diff --git a/apps/studio/components/interfaces/Reports/Reports.constants.test.ts b/apps/studio/components/interfaces/Reports/Reports.constants.test.ts index e9ef58cf375..b11b5b50832 100644 --- a/apps/studio/components/interfaces/Reports/Reports.constants.test.ts +++ b/apps/studio/components/interfaces/Reports/Reports.constants.test.ts @@ -1,8 +1,11 @@ import { describe, expect, it } from 'vitest' -import { generateRegexpWhereSafe } from './Reports.constants' +import { generateOtelWhereSafe, generateRegexpWhereSafe } from './Reports.constants' import type { ReportFilterItem } from './Reports.types' +// Collapse whitespace so OTEL assertions are resilient to formatting. +const sqlText = (fragment: string) => fragment.replace(/\s+/g, ' ').trim() + describe('generateRegexpWhereSafe', () => { it('should return empty fragment when no filters provided', () => { const result = generateRegexpWhereSafe([]) @@ -119,3 +122,66 @@ describe('generateRegexpWhereSafe', () => { expect(result).toBe("WHERE `request`.`method` = 'get'") }) }) + +describe('generateOtelWhereSafe', () => { + it('returns an empty fragment when no filters provided', () => { + expect(sqlText(generateOtelWhereSafe([]))).toBe('') + }) + + it('maps `matches` to ClickHouse match() over the log_attributes lookup', () => { + const filters: ReportFilterItem[] = [ + { key: 'request.path', value: '/auth', compare: 'matches' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toBe( + "WHERE match(log_attributes['request.path'], '/auth')" + ) + }) + + it('maps `is` to a lowercased string equality against the attribute', () => { + const filters: ReportFilterItem[] = [{ key: 'request.method', value: 'GET', compare: 'is' }] + expect(sqlText(generateOtelWhereSafe(filters))).toBe( + "WHERE log_attributes['request.method'] = 'get'" + ) + }) + + it('casts the attribute to an int for ordering comparisons', () => { + const filters: ReportFilterItem[] = [ + { key: 'response.status_code', value: '400', compare: '>=' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toBe( + "WHERE toInt64OrZero(log_attributes['response.status_code']) >= 400" + ) + }) + + it('drops an ordering comparison whose value is not numeric', () => { + const filters: ReportFilterItem[] = [ + { key: 'response.status_code', value: 'abc', compare: '>' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toBe('') + }) + + it('normalizes a deep key to its last two segments', () => { + const filters: ReportFilterItem[] = [ + { key: 'metadata.request.path', value: '/rest', compare: 'matches' }, + ] + expect(sqlText(generateOtelWhereSafe(filters))).toContain("log_attributes['request.path']") + }) + + it('escapes the key inside the log_attributes subscript so it cannot break out', () => { + const filters: ReportFilterItem[] = [{ key: "x'] = '1", value: 'info', compare: 'is' }] + // Single quotes in the key are doubled by the literal escaping, keeping the + // injection attempt confined to the string subscript. + expect(sqlText(generateOtelWhereSafe(filters))).toContain("log_attributes['x''] = ''1']") + }) + + it('joins multiple conditions with AND and supports the non-prepend form', () => { + const filters: ReportFilterItem[] = [ + { key: 'request.path', value: '/auth', compare: 'matches' }, + { key: 'request.method', value: 'POST', compare: 'is' }, + ] + const result = sqlText(generateOtelWhereSafe(filters, false)) + expect(result.startsWith('AND ')).toBe(true) + expect(result).toContain("match(log_attributes['request.path'], '/auth')") + expect(result).toContain("log_attributes['request.method'] = 'post'") + }) +}) diff --git a/apps/studio/components/interfaces/Reports/Reports.constants.ts b/apps/studio/components/interfaces/Reports/Reports.constants.ts index 3f917f04db0..35a98b5ef33 100644 --- a/apps/studio/components/interfaces/Reports/Reports.constants.ts +++ b/apps/studio/components/interfaces/Reports/Reports.constants.ts @@ -143,6 +143,68 @@ export function generateRegexpWhereSafe( return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}` } +/** + * OTEL/ClickHouse counterpart of `generateRegexpWhereSafe`. The OTEL `logs` table + * stores per-request fields in a `log_attributes` Map keyed by the same dotted path + * (e.g. `request.path`), so each filter key maps to a `log_attributes['']` + * lookup. `matches` uses ClickHouse `match()` (re2), ordering comparisons cast the + * (string) value to an int, and equality compares string-to-string. Mirrors the + * BigQuery generator's key normalization and value lowercasing for parity. + */ +export function generateOtelWhereSafe( + filters: ReportFilterItem[], + prepend = true +): SafeLogSqlFragment { + if (filters.length === 0) return safeLogSql`` + + const conditions = filters + .map((filter) => { + const splitKey = filter.key.split('.') + const normalizedKey = filter.key.includes('.') + ? [splitKey[splitKey.length - 2], splitKey[splitKey.length - 1]].join('.') + : filter.key + + let col: SafeLogSqlFragment + try { + col = safeLogSql`log_attributes[${analyticsLiteral(normalizedKey)}]` + } catch { + return null + } + + const valueIsNumber = !isNaN(Number(filter.value)) + const stringLit = analyticsLiteral(String(filter.value).toLowerCase()) + + switch (filter.compare) { + case 'matches': + return safeLogSql`match(${col}, ${stringLit})` + case 'is': + return safeLogSql`${col} = ${stringLit}` + case '!=': + return safeLogSql`${col} != ${stringLit}` + case '>=': + case '<=': + case '>': + case '<': { + if (!valueIsNumber) return null + const num = analyticsLiteral(Number(filter.value)) + const lhs = safeLogSql`toInt64OrZero(${col})` + if (filter.compare === '>=') return safeLogSql`${lhs} >= ${num}` + if (filter.compare === '<=') return safeLogSql`${lhs} <= ${num}` + if (filter.compare === '>') return safeLogSql`${lhs} > ${num}` + return safeLogSql`${lhs} < ${num}` + } + default: + return safeLogSql`${col} = ${stringLit}` + } + }) + .filter((c) => c !== null) + + if (conditions.length === 0) return safeLogSql`` + + const joined = joinSqlFragments(conditions, ' AND ') + return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}` +} + export const PRESET_CONFIG: Record = { [Presets.API]: { title: 'API', diff --git a/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts b/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts index 06d3ca0e3da..c8cf3702758 100644 --- a/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts +++ b/apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts @@ -1,13 +1,14 @@ import * as Sentry from '@sentry/nextjs' import { useQueries, useQueryClient } from '@tanstack/react-query' -import { useParams } from 'common' +import { useFlag, useParams } from 'common' import { isEqual } from 'lodash' import { useState } from 'react' -import { generateRegexpWhereSafe } from '../Reports.constants' +import { generateOtelWhereSafe, generateRegexpWhereSafe } from '../Reports.constants' import { ReportFilterItem } from '../Reports.types' import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql' -import { safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql' +import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint' +import { analyticsLiteral, safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql' const SOURCE_TABLE: Record = { edge_logs: safeSql`edge_logs`, @@ -19,6 +20,37 @@ function sourceTable(src: string): SafeLogSqlFragment { return SOURCE_TABLE[src] ?? SOURCE_TABLE.edge_logs } +// --- OTEL / ClickHouse variants ------------------------------------------------- +// The OTEL `logs` table is a single table keyed by `source`, with request/response +// fields stored in the `log_attributes` Map. These helpers mirror the BigQuery +// builders above so the chart-facing result columns (timestamp/count/avg/path/...) +// stay identical; only the SQL dialect changes. + +const OTEL_SOURCE = new Set(['edge_logs', 'function_edge_logs']) +const otelSourceName = (src: string): string => (OTEL_SOURCE.has(src) ? src : 'edge_logs') + +/** `WHERE source = '' [AND ] [AND ]` for the OTEL logs table. */ +function otelWhere( + src: string, + filters: ReportFilterItem[], + extra?: SafeLogSqlFragment +): SafeLogSqlFragment { + const base = extra + ? safeSql`where source = ${analyticsLiteral(otelSourceName(src))} and ${extra}` + : safeSql`where source = ${analyticsLiteral(otelSourceName(src))}` + return safeSql`${base} ${generateOtelWhereSafe(filters, false)}` +} + +// Route grouping columns, shared by the top-routes style queries. +const OTEL_ROUTE_SELECT: SafeLogSqlFragment = safeSql` + log_attributes['request.path'] as path, + log_attributes['request.method'] as method, + log_attributes['request.search'] as search, + log_attributes['response.status_code'] as status_code` +const OTEL_ROUTE_GROUP_BY: SafeLogSqlFragment = safeSql`log_attributes['request.path'], log_attributes['request.method'], log_attributes['request.search'], log_attributes['response.status_code']` +const OTEL_STATUS_IS_ERROR: SafeLogSqlFragment = safeSql`toInt64OrZero(log_attributes['response.status_code']) >= 400` +const OTEL_ORIGIN_TIME: SafeLogSqlFragment = safeSql`toFloat64OrZero(log_attributes['response.origin_time'])` + export const SHARED_API_REPORT_SQL = { totalRequests: { queryType: 'logs', @@ -38,6 +70,14 @@ export const SHARED_API_REPORT_SQL = { timestamp ORDER BY timestamp ASC`, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-total-requests (otel) + select toStartOfHour(timestamp) as timestamp, count() as count + from logs + ${otelWhere(src, filters)} + group by timestamp + order by timestamp asc`, }, topRoutes: { queryType: 'logs', @@ -62,6 +102,15 @@ export const SHARED_API_REPORT_SQL = { count desc limit 10 `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-top-routes (otel) + select ${OTEL_ROUTE_SELECT}, count() as count + from logs + ${otelWhere(src, filters)} + group by ${OTEL_ROUTE_GROUP_BY} + order by count desc + limit 10`, }, errorCounts: { queryType: 'logs', @@ -84,6 +133,14 @@ export const SHARED_API_REPORT_SQL = { ORDER BY timestamp ASC `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-error-counts (otel) + select toStartOfHour(timestamp) as timestamp, count() as count + from logs + ${otelWhere(src, filters, OTEL_STATUS_IS_ERROR)} + group by timestamp + order by timestamp asc`, }, topErrorRoutes: { queryType: 'logs', @@ -110,6 +167,15 @@ export const SHARED_API_REPORT_SQL = { count desc limit 10 `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-top-error-routes (otel) + select ${OTEL_ROUTE_SELECT}, count() as count + from logs + ${otelWhere(src, filters, OTEL_STATUS_IS_ERROR)} + group by ${OTEL_ROUTE_GROUP_BY} + order by count desc + limit 10`, }, responseSpeed: { queryType: 'logs', @@ -131,6 +197,14 @@ export const SHARED_API_REPORT_SQL = { ORDER BY timestamp ASC `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-response-speed (otel) + select toStartOfHour(timestamp) as timestamp, avg(${OTEL_ORIGIN_TIME}) as avg + from logs + ${otelWhere(src, filters)} + group by timestamp + order by timestamp asc`, }, topSlowRoutes: { queryType: 'logs', @@ -156,6 +230,15 @@ export const SHARED_API_REPORT_SQL = { avg desc limit 10 `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-top-slow-routes (otel) + select ${OTEL_ROUTE_SELECT}, count() as count, avg(${OTEL_ORIGIN_TIME}) as avg + from logs + ${otelWhere(src, filters)} + group by ${OTEL_ROUTE_GROUP_BY} + order by avg desc + limit 10`, }, networkTraffic: { queryType: 'logs', @@ -195,6 +278,17 @@ export const SHARED_API_REPORT_SQL = { ORDER BY timestamp ASC `, + safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment => + safeSql` + -- reports-api-network-traffic (otel) + select + toStartOfHour(timestamp) as timestamp, + sum(toInt64OrZero(log_attributes['request.headers.content_length'])) / 1000000 as ingress_mb, + sum(toInt64OrZero(log_attributes['response.headers.content_length'])) / 1000000 as egress_mb + from logs + ${otelWhere(src, filters)} + group by timestamp + order by timestamp asc`, }, } @@ -225,6 +319,12 @@ export const useSharedAPIReport = ({ const { ref } = useParams() as { ref: string } const [filters, setFilters] = useState([]) const queryClient = useQueryClient() + + // When enabled, route report queries through the OTEL ClickHouse endpoint + // (logs.all.otel) with the ClickHouse SQL variants instead of BigQuery. + const useOtel = useFlag('otelReports') + const buildSql = (entry: (typeof SHARED_API_REPORT_SQL)[SharedAPIReportKey]) => + useOtel ? entry.safeSqlOtel : entry.safeSql const filterByMapSource = { functions: 'function_edge_logs', realtime: 'edge_logs', @@ -262,14 +362,15 @@ export const useSharedAPIReport = ({ start, end, ref, + { otel: useOtel }, ], enabled: enabled && !!ref && !!filterBy, queryFn: async () => { try { const data = await executeAnalyticsSql({ projectRef: ref, - endpoint: '/platform/projects/{ref}/analytics/endpoints/logs.all', - sql: value.safeSql(allFilters, filterByMapSource[filterBy]), + endpoint: logsAllEndpointUrl(useOtel), + sql: buildSql(value)(allFilters, filterByMapSource[filterBy]), iso_timestamp_start: start, iso_timestamp_end: end, method: 'get', @@ -333,30 +434,13 @@ export const useSharedAPIReport = ({ const isLoadingData = Object.values(isLoading).some(Boolean) - const SQLMap: Record = { - totalRequests: SHARED_API_REPORT_SQL.totalRequests.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - topRoutes: SHARED_API_REPORT_SQL.topRoutes.safeSql(allFilters, filterByMapSource[filterBy]), - errorCounts: SHARED_API_REPORT_SQL.errorCounts.safeSql(allFilters, filterByMapSource[filterBy]), - topErrorRoutes: SHARED_API_REPORT_SQL.topErrorRoutes.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - responseSpeed: SHARED_API_REPORT_SQL.responseSpeed.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - topSlowRoutes: SHARED_API_REPORT_SQL.topSlowRoutes.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - networkTraffic: SHARED_API_REPORT_SQL.networkTraffic.safeSql( - allFilters, - filterByMapSource[filterBy] - ), - } + const SQLMap = keys.reduce( + (acc, key) => { + acc[key] = buildSql(SHARED_API_REPORT_SQL[key])(allFilters, filterByMapSource[filterBy]) + return acc + }, + {} as Record + ) return { data, diff --git a/apps/studio/data/reports/report.utils.ts b/apps/studio/data/reports/report.utils.ts index 9c52a8759d4..0a859b6f271 100644 --- a/apps/studio/data/reports/report.utils.ts +++ b/apps/studio/data/reports/report.utils.ts @@ -2,6 +2,7 @@ import { type ComparisonOperator } from '@/components/interfaces/Reports/v2/Repo import { AnalyticsInterval } from '@/data/analytics/constants' import { useEdgeFunctionsQuery } from '@/data/edge-functions/edge-functions-query' import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql' +import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint' import { safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql' export type Granularity = 'minute' | 'hour' | 'day' @@ -85,11 +86,14 @@ export async function fetchLogs( projectRef: string, sql: SafeLogSqlFragment, startDate: string, - endDate: string + endDate: string, + // Pass true (with ClickHouse SQL) to route through the OTEL endpoint. Defaults + // to the BigQuery endpoint until the edge-functions report SQL is migrated. + useOtel = false ) { return await executeAnalyticsSql({ projectRef, - endpoint: '/platform/projects/{ref}/analytics/endpoints/logs.all', + endpoint: logsAllEndpointUrl(useOtel), sql, iso_timestamp_start: startDate, iso_timestamp_end: endDate,