From 4f3e59224321c0267033b3ccae11cb49fa2dbdc6 Mon Sep 17 00:00:00 2001 From: Jordi Enric Date: Thu, 18 Jun 2026 16:55:41 +0200 Subject: [PATCH] feat(studio): migrate SharedAPIReport charts to OTEL endpoint behind a flag The /observability reports (except database) query the BigQuery logs.all analytics endpoint. This starts migrating them to the OTEL ClickHouse endpoint (logs.all.otel) behind a new otelReports feature flag (default off). This PR covers the SharedAPIReport charts that power the auth, postgrest, and realtime reports (total requests, top routes, error counts, top error routes, response speed, top slow routes, network traffic), plus the shared plumbing: - Add generateOtelWhereSafe in Reports.constants.ts: the ClickHouse counterpart of generateRegexpWhereSafe, mapping filter keys to log_attributes[...] lookups (match() for `matches`, int cast for ordering, string equality otherwise). - Add safeSqlOtel ClickHouse variants for all 7 SharedAPIReport queries (FROM logs WHERE source = '...', toStartOfHour, count()/avg, toInt64OrZero/ toFloat64OrZero), reusing shared route projection/grouping fragments. - Gate useSharedAPIReport on useFlag('otelReports'): pick builder + endpoint via logsAllEndpointUrl, key queries by otel state. BigQuery stays the default. - Add the useOtel param to fetchLogs (report.utils) so the edge-functions follow-up only needs ClickHouse SQL. - Tests for generateOtelWhereSafe. The generated OTEL SQL for all 7 charts was validated against the staging OTEL endpoint with real project data. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Reports/Reports.constants.test.ts | 68 ++++++++- .../interfaces/Reports/Reports.constants.ts | 62 ++++++++ .../SharedAPIReport.constants.ts | 142 ++++++++++++++---- apps/studio/data/reports/report.utils.ts | 8 +- 4 files changed, 248 insertions(+), 32 deletions(-) diff --git a/apps/studio/components/interfaces/Reports/Reports.constants.test.ts b/apps/studio/components/interfaces/Reports/Reports.constants.test.ts index e9ef58cf375..b11b5b50832 100644 --- a/apps/studio/components/interfaces/Reports/Reports.constants.test.ts +++ b/apps/studio/components/interfaces/Reports/Reports.constants.test.ts @@ -1,8 +1,11 @@ import { describe, expect, it } from 'vitest' -import { generateRegexpWhereSafe } from './Reports.constants' +import { generateOtelWhereSafe, generateRegexpWhereSafe } from './Reports.constants' import type { ReportFilterItem } from './Reports.types' +// Collapse whitespace so OTEL assertions are resilient to formatting. +const sqlText = (fragment: string) => fragment.replace(/\s+/g, ' ').trim() + describe('generateRegexpWhereSafe', () => { it('should return empty fragment when no filters provided', () => { const result = generateRegexpWhereSafe([]) @@ -119,3 +122,66 @@ describe('generateRegexpWhereSafe', () => { expect(result).toBe("WHERE `request`.`method` = 'get'") }) }) + +describe('generateOtelWhereSafe', () => { + it('returns an empty fragment when no filters provided', () => { + expect(sqlText(generateOtelWhereSafe([]))).toBe('') + }) + + it('maps `matches` to ClickHouse match() over the log_attributes lookup', () => { + const filters: ReportFilterItem[] = [ + { key: 'request.path', value: '/auth', compare: 'matches' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toBe( + "WHERE match(log_attributes['request.path'], '/auth')" + ) + }) + + it('maps `is` to a lowercased string equality against the attribute', () => { + const filters: ReportFilterItem[] = [{ key: 'request.method', value: 'GET', compare: 'is' }] + expect(sqlText(generateOtelWhereSafe(filters))).toBe( + "WHERE log_attributes['request.method'] = 'get'" + ) + }) + + it('casts the attribute to an int for ordering comparisons', () => { + const filters: ReportFilterItem[] = [ + { key: 'response.status_code', value: '400', compare: '>=' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toBe( + "WHERE toInt64OrZero(log_attributes['response.status_code']) >= 400" + ) + }) + + it('drops an ordering comparison whose value is not numeric', () => { + const filters: ReportFilterItem[] = [ + { key: 'response.status_code', value: 'abc', compare: '>' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toBe('') + }) + + it('normalizes a deep key to its last two segments', () => { + const filters: ReportFilterItem[] = [ + { key: 'metadata.request.path', value: '/rest', compare: 'matches' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toContain("log_attributes['request.path']") + }) + + it('escapes the key inside the log_attributes subscript so it cannot break out', () => { + const filters: ReportFilterItem[] = [{ key: "x'] = '1", value: 'info', compare: 'is' }] + // Single quotes in the key are doubled by the literal escaping, keeping the + // injection attempt confined to the string subscript. + expect(sqlText(generateOtelWhereSafe(filters))).toContain("log_attributes['x''] = ''1']") + }) + + it('joins multiple conditions with AND and supports the non-prepend form', () => { + const filters: ReportFilterItem[] = [ + { key: 'request.path', value: '/auth', compare: 'matches' }, + { key: 'request.method', value: 'POST', compare: 'is' }, + ] + const result = sqlText(generateOtelWhereSafe(filters, false)) + expect(result.startsWith('AND ')).toBe(true) + expect(result).toContain("match(log_attributes['request.path'], '/auth')") + expect(result).toContain("log_attributes['request.method'] = 'post'") + }) +}) diff --git a/apps/studio/components/interfaces/Reports/Reports.constants.ts b/apps/studio/components/interfaces/Reports/Reports.constants.ts index 3f917f04db0..35a98b5ef33 100644 --- a/apps/studio/components/interfaces/Reports/Reports.constants.ts +++ b/apps/studio/components/interfaces/Reports/Reports.constants.ts @@ -143,6 +143,68 @@ export function generateRegexpWhereSafe( return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}` } +/** + * OTEL/ClickHouse counterpart of `generateRegexpWhereSafe`. The OTEL `logs` table + * stores per-request fields in a `log_attributes` Map keyed by the same dotted path + * (e.g. `request.path`), so each filter key maps to a `log_attributes['']` + * lookup. `matches` uses ClickHouse `match()` (re2), ordering comparisons cast the + * (string) value to an int, and equality compares string-to-string. Mirrors the + * BigQuery generator's key normalization and value lowercasing for parity. + */ +export function generateOtelWhereSafe( + filters: ReportFilterItem[], + prepend = true +): SafeLogSqlFragment { + if (filters.length === 0) return safeLogSql`` + + const conditions = filters + .map((filter) => { + const splitKey = filter.key.split('.') + const normalizedKey = filter.key.includes('.') + ? [splitKey[splitKey.length - 2], splitKey[splitKey.length - 1]].join('.') + : filter.key + + let col: SafeLogSqlFragment + try { + col = safeLogSql`log_attributes[${analyticsLiteral(normalizedKey)}]` + } catch { + return null + } + + const valueIsNumber = !isNaN(Number(filter.value)) + const stringLit = analyticsLiteral(String(filter.value).toLowerCase()) + + switch (filter.compare) { + case 'matches': + return safeLogSql`match(${col}, ${stringLit})` + case 'is': + return safeLogSql`${col} = ${stringLit}` + case '!=': + return safeLogSql`${col} != ${stringLit}` + case '>=': + case '<=': + case '>': + case '<': { + if (!valueIsNumber) return null + const num = analyticsLiteral(Number(filter.value)) + const lhs = safeLogSql`toInt64OrZero(${col})` + if (filter.compare === '>=') return safeLogSql`${lhs} >= ${num}` + if (filter.compare === '<=') return safeLogSql`${lhs} <= ${num}` + if (filter.compare === '>') return safeLogSql`${lhs} > ${num}` + return safeLogSql`${lhs} < ${num}` + } + default: + return safeLogSql`${col} = ${stringLit}` + } + }) + .filter((c) => c !== null) + + if (conditions.length === 0) return safeLogSql`` + + const joined = joinSqlFragments(conditions, ' AND ') + return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}` +} + export const PRESET_CONFIG: Record = { [Presets.API]: { title: 'API', diff --git a/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts b/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts index 06d3ca0e3da..c8cf3702758 100644 --- a/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts +++ b/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts @@ -1,13 +1,14 @@ import * as Sentry from '@sentry/nextjs' import { useQueries, useQueryClient } from '@tanstack/react-query' -import { useParams } from 'common' +import { useFlag, useParams } from 'common' import { isEqual } from 'lodash' import { useState } from 'react' -import { generateRegexpWhereSafe } from '../Reports.constants' +import { generateOtelWhereSafe, generateRegexpWhereSafe } from '../Reports.constants' import { ReportFilterItem } from '../Reports.types' import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql' -import { safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql' +import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint' +import { analyticsLiteral, safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql' const SOURCE_TABLE: Record = { edge_logs: safeSql`edge_logs`, @@ -19,6 +20,37 @@ function sourceTable(src: string): SafeLogSqlFragment { return SOURCE_TABLE[src] ?? SOURCE_TABLE.edge_logs } +// --- OTEL / ClickHouse variants ------------------------------------------------- +// The OTEL `logs` table is a single table keyed by `source`, with request/response +// fields stored in the `log_attributes` Map. These helpers mirror the BigQuery +// builders above so the chart-facing result columns (timestamp/count/avg/path/...) +// stay identical; only the SQL dialect changes. + +const OTEL_SOURCE = new Set(['edge_logs', 'function_edge_logs']) +const otelSourceName = (src: string): string => (OTEL_SOURCE.has(src) ? src : 'edge_logs') + +/** `WHERE source = '' [AND ] [AND ]` for the OTEL logs table. */ +function otelWhere( + src: string, + filters: ReportFilterItem[], + extra?: SafeLogSqlFragment +): SafeLogSqlFragment { + const base = extra + ? safeSql`where source = ${analyticsLiteral(otelSourceName(src))} and ${extra}` + : safeSql`where source = ${analyticsLiteral(otelSourceName(src))}` + return safeSql`${base} ${generateOtelWhereSafe(filters, false)}` +} + +// Route grouping columns, shared by the top-routes style queries. +const OTEL_ROUTE_SELECT: SafeLogSqlFragment = safeSql` + log_attributes['request.path'] as path, + log_attributes['request.method'] as method, + log_attributes['request.search'] as search, + log_attributes['response.status_code'] as status_code` +const OTEL_ROUTE_GROUP_BY: SafeLogSqlFragment = safeSql`log_attributes['request.path'], log_attributes['request.method'], log_attributes['request.search'], log_attributes['response.status_code']` +const OTEL_STATUS_IS_ERROR: SafeLogSqlFragment = safeSql`toInt64OrZero(log_attributes['response.status_code']) >= 400` +const OTEL_ORIGIN_TIME: SafeLogSqlFragment = safeSql`toFloat64OrZero(log_attributes['response.origin_time'])` + export const SHARED_API_REPORT_SQL = { totalRequests: { queryType: 'logs', @@ -38,6 +70,14 @@ export const SHARED_API_REPORT_SQL = { timestamp ORDER BY timestamp ASC`, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-total-requests (otel) + select toStartOfHour(timestamp) as timestamp, count() as count + from logs + ${otelWhere(src, filters)} + group by timestamp + order by timestamp asc`, }, topRoutes: { queryType: 'logs', @@ -62,6 +102,15 @@ export const SHARED_API_REPORT_SQL = { count desc limit 10 `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-top-routes (otel) + select ${OTEL_ROUTE_SELECT}, count() as count + from logs + ${otelWhere(src, filters)} + group by ${OTEL_ROUTE_GROUP_BY} + order by count desc + limit 10`, }, errorCounts: { queryType: 'logs', @@ -84,6 +133,14 @@ export const SHARED_API_REPORT_SQL = { ORDER BY timestamp ASC `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-error-counts (otel) + select toStartOfHour(timestamp) as timestamp, count() as count + from logs + ${otelWhere(src, filters, OTEL_STATUS_IS_ERROR)} + group by timestamp + order by timestamp asc`, }, topErrorRoutes: { queryType: 'logs', @@ -110,6 +167,15 @@ export const SHARED_API_REPORT_SQL = { count desc limit 10 `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-top-error-routes (otel) + select ${OTEL_ROUTE_SELECT}, count() as count + from logs + ${otelWhere(src, filters, OTEL_STATUS_IS_ERROR)} + group by ${OTEL_ROUTE_GROUP_BY} + order by count desc + limit 10`, }, responseSpeed: { queryType: 'logs', @@ -131,6 +197,14 @@ export const SHARED_API_REPORT_SQL = { ORDER BY timestamp ASC `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-response-speed (otel) + select toStartOfHour(timestamp) as timestamp, avg(${OTEL_ORIGIN_TIME}) as avg + from logs + ${otelWhere(src, filters)} + group by timestamp + order by timestamp asc`, }, topSlowRoutes: { queryType: 'logs', @@ -156,6 +230,15 @@ export const SHARED_API_REPORT_SQL = { avg desc limit 10 `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-top-slow-routes (otel) + select ${OTEL_ROUTE_SELECT}, count() as count, avg(${OTEL_ORIGIN_TIME}) as avg + from logs + ${otelWhere(src, filters)} + group by ${OTEL_ROUTE_GROUP_BY} + order by avg desc + limit 10`, }, networkTraffic: { queryType: 'logs', @@ -195,6 +278,17 @@ export const SHARED_API_REPORT_SQL = { ORDER BY timestamp ASC `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-network-traffic (otel) + select + toStartOfHour(timestamp) as timestamp, + sum(toInt64OrZero(log_attributes['request.headers.content_length'])) / 1000000 as ingress_mb, + sum(toInt64OrZero(log_attributes['response.headers.content_length'])) / 1000000 as egress_mb + from logs + ${otelWhere(src, filters)} + group by timestamp + order by timestamp asc`, }, } @@ -225,6 +319,12 @@ export const useSharedAPIReport = ({ const { ref } = useParams() as { ref: string } const [filters, setFilters] = useState([]) const queryClient = useQueryClient() + + // When enabled, route report queries through the OTEL ClickHouse endpoint + // (logs.all.otel) with the ClickHouse SQL variants instead of BigQuery. + const useOtel = useFlag('otelReports') + const buildSql = (entry: (typeof SHARED_API_REPORT_SQL)[SharedAPIReportKey]) => + useOtel ? entry.safeSqlOtel : entry.safeSql const filterByMapSource = { functions: 'function_edge_logs', realtime: 'edge_logs', @@ -262,14 +362,15 @@ export const useSharedAPIReport = ({ start, end, ref, + { otel: useOtel }, ], enabled: enabled && !!ref && !!filterBy, queryFn: async () => { try { const data = await executeAnalyticsSql({ projectRef: ref, - endpoint: '/platform/projects/{ref}/analytics/endpoints/logs.all', - sql: value.safeSql(allFilters, filterByMapSource[filterBy]), + endpoint: logsAllEndpointUrl(useOtel), + sql: buildSql(value)(allFilters, filterByMapSource[filterBy]), iso_timestamp_start: start, iso_timestamp_end: end, method: 'get', @@ -333,30 +434,13 @@ export const useSharedAPIReport = ({ const isLoadingData = Object.values(isLoading).some(Boolean) - const SQLMap: Record = { - totalRequests: SHARED_API_REPORT_SQL.totalRequests.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - topRoutes: SHARED_API_REPORT_SQL.topRoutes.safeSql(allFilters, filterByMapSource[filterBy]), - errorCounts: SHARED_API_REPORT_SQL.errorCounts.safeSql(allFilters, filterByMapSource[filterBy]), - topErrorRoutes: SHARED_API_REPORT_SQL.topErrorRoutes.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - responseSpeed: SHARED_API_REPORT_SQL.responseSpeed.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - topSlowRoutes: SHARED_API_REPORT_SQL.topSlowRoutes.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - networkTraffic: SHARED_API_REPORT_SQL.networkTraffic.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - } + const SQLMap = keys.reduce( + (acc, key) => { + acc[key] = buildSql(SHARED_API_REPORT_SQL[key])(allFilters, filterByMapSource[filterBy]) + return acc + }, + {} as Record + ) return { data, diff --git a/apps/studio/data/reports/report.utils.ts b/apps/studio/data/reports/report.utils.ts index 9c52a8759d4..0a859b6f271 100644 --- a/apps/studio/data/reports/report.utils.ts +++ b/apps/studio/data/reports/report.utils.ts @@ -2,6 +2,7 @@ import { type ComparisonOperator } from '@/components/interfaces/Reports/v2/Repo import { AnalyticsInterval } from '@/data/analytics/constants' import { useEdgeFunctionsQuery } from '@/data/edge-functions/edge-functions-query' import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql' +import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint' import { safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql' export type Granularity = 'minute' | 'hour' | 'day' @@ -85,11 +86,14 @@ export async function fetchLogs( projectRef: string, sql: SafeLogSqlFragment, startDate: string, - endDate: string + endDate: string, + // Pass true (with ClickHouse SQL) to route through the OTEL endpoint. Defaults + // to the BigQuery endpoint until the edge-functions report SQL is migrated. + useOtel = false ) { return await executeAnalyticsSql({ projectRef, - endpoint: '/platform/projects/{ref}/analytics/endpoints/logs.all', + endpoint: logsAllEndpointUrl(useOtel), sql, iso_timestamp_start: startDate, iso_timestamp_end: endDate,