feat(studio): migrate SharedAPIReport charts to OTEL endpoint behind a flag

The /observability reports (except database) query the BigQuery logs.all
analytics endpoint. This starts migrating them to the OTEL ClickHouse
endpoint (logs.all.otel) behind a new otelReports feature flag (default off).

This PR covers the SharedAPIReport charts that power the auth, postgrest, and
realtime reports (total requests, top routes, error counts, top error routes,
response speed, top slow routes, network traffic), plus the shared plumbing:

- Add generateOtelWhereSafe in Reports.constants.ts: the ClickHouse counterpart
  of generateRegexpWhereSafe, mapping filter keys to log_attributes[...] lookups
  (match() for `matches`, int cast for ordering, string equality otherwise).
- Add safeSqlOtel ClickHouse variants for all 7 SharedAPIReport queries (FROM
  logs WHERE source = '...', toStartOfHour, count()/avg, toInt64OrZero/
  toFloat64OrZero), reusing shared route projection/grouping fragments.
- Gate useSharedAPIReport on useFlag('otelReports'): pick builder + endpoint via
  logsAllEndpointUrl, key queries by otel state. BigQuery stays the default.
- Add the useOtel param to fetchLogs (report.utils) so the edge-functions
  follow-up only needs ClickHouse SQL.
- Tests for generateOtelWhereSafe. The generated OTEL SQL for all 7 charts was
  validated against the staging OTEL endpoint with real project data.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

apps/studio/components/interfaces/Reports/Reports.constants.test.ts

@@ -1,8 +1,11 @@
 import { describe, expect, it } from 'vitest'
 
-import { generateRegexpWhereSafe } from './Reports.constants'
+import { generateOtelWhereSafe, generateRegexpWhereSafe } from './Reports.constants'
 import type { ReportFilterItem } from './Reports.types'
 
+// Collapse whitespace so OTEL assertions are resilient to formatting.
+const sqlText = (fragment: string) => fragment.replace(/\s+/g, ' ').trim()
+
 describe('generateRegexpWhereSafe', () => {
   it('should return empty fragment when no filters provided', () => {
     const result = generateRegexpWhereSafe([])
@@ -119,3 +122,66 @@ describe('generateRegexpWhereSafe', () => {
     expect(result).toBe("WHERE `request`.`method` = 'get'")
   })
 })
+
+describe('generateOtelWhereSafe', () => {
+  it('returns an empty fragment when no filters provided', () => {
+    expect(sqlText(generateOtelWhereSafe([]))).toBe('')
+  })
+
+  it('maps `matches` to ClickHouse match() over the log_attributes lookup', () => {
+    const filters: ReportFilterItem[] = [
+      { key: 'request.path', value: '/auth', compare: 'matches' },
+    ]
+    expect(sqlText(generateOtelWhereSafe(filters))).toBe(
+      "WHERE match(log_attributes['request.path'], '/auth')"
+    )
+  })
+
+  it('maps `is` to a lowercased string equality against the attribute', () => {
+    const filters: ReportFilterItem[] = [{ key: 'request.method', value: 'GET', compare: 'is' }]
+    expect(sqlText(generateOtelWhereSafe(filters))).toBe(
+      "WHERE log_attributes['request.method'] = 'get'"
+    )
+  })
+
+  it('casts the attribute to an int for ordering comparisons', () => {
+    const filters: ReportFilterItem[] = [
+      { key: 'response.status_code', value: '400', compare: '>=' },
+    ]
+    expect(sqlText(generateOtelWhereSafe(filters))).toBe(
+      "WHERE toInt64OrZero(log_attributes['response.status_code']) >= 400"
+    )
+  })
+
+  it('drops an ordering comparison whose value is not numeric', () => {
+    const filters: ReportFilterItem[] = [
+      { key: 'response.status_code', value: 'abc', compare: '>' },
+    ]
+    expect(sqlText(generateOtelWhereSafe(filters))).toBe('')
+  })
+
+  it('normalizes a deep key to its last two segments', () => {
+    const filters: ReportFilterItem[] = [
+      { key: 'metadata.request.path', value: '/rest', compare: 'matches' },
+    ]
+    expect(sqlText(generateOtelWhereSafe(filters))).toContain("log_attributes['request.path']")
+  })
+
+  it('escapes the key inside the log_attributes subscript so it cannot break out', () => {
+    const filters: ReportFilterItem[] = [{ key: "x'] = '1", value: 'info', compare: 'is' }]
+    // Single quotes in the key are doubled by the literal escaping, keeping the
+    // injection attempt confined to the string subscript.
+    expect(sqlText(generateOtelWhereSafe(filters))).toContain("log_attributes['x''] = ''1']")
+  })
+
+  it('joins multiple conditions with AND and supports the non-prepend form', () => {
+    const filters: ReportFilterItem[] = [
+      { key: 'request.path', value: '/auth', compare: 'matches' },
+      { key: 'request.method', value: 'POST', compare: 'is' },
+    ]
+    const result = sqlText(generateOtelWhereSafe(filters, false))
+    expect(result.startsWith('AND ')).toBe(true)
+    expect(result).toContain("match(log_attributes['request.path'], '/auth')")
+    expect(result).toContain("log_attributes['request.method'] = 'post'")
+  })
+})

apps/studio/components/interfaces/Reports/Reports.constants.ts

@@ -143,6 +143,68 @@ export function generateRegexpWhereSafe(
   return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}`
 }
 
+/**
+ * OTEL/ClickHouse counterpart of `generateRegexpWhereSafe`. The OTEL `logs` table
+ * stores per-request fields in a `log_attributes` Map keyed by the same dotted path
+ * (e.g. `request.path`), so each filter key maps to a `log_attributes['<key>']`
+ * lookup. `matches` uses ClickHouse `match()` (re2), ordering comparisons cast the
+ * (string) value to an int, and equality compares string-to-string. Mirrors the
+ * BigQuery generator's key normalization and value lowercasing for parity.
+ */
+export function generateOtelWhereSafe(
+  filters: ReportFilterItem[],
+  prepend = true
+): SafeLogSqlFragment {
+  if (filters.length === 0) return safeLogSql``
+
+  const conditions = filters
+    .map((filter) => {
+      const splitKey = filter.key.split('.')
+      const normalizedKey = filter.key.includes('.')
+        ? [splitKey[splitKey.length - 2], splitKey[splitKey.length - 1]].join('.')
+        : filter.key
+
+      let col: SafeLogSqlFragment
+      try {
+        col = safeLogSql`log_attributes[${analyticsLiteral(normalizedKey)}]`
+      } catch {
+        return null
+      }
+
+      const valueIsNumber = !isNaN(Number(filter.value))
+      const stringLit = analyticsLiteral(String(filter.value).toLowerCase())
+
+      switch (filter.compare) {
+        case 'matches':
+          return safeLogSql`match(${col}, ${stringLit})`
+        case 'is':
+          return safeLogSql`${col} = ${stringLit}`
+        case '!=':
+          return safeLogSql`${col} != ${stringLit}`
+        case '>=':
+        case '<=':
+        case '>':
+        case '<': {
+          if (!valueIsNumber) return null
+          const num = analyticsLiteral(Number(filter.value))
+          const lhs = safeLogSql`toInt64OrZero(${col})`
+          if (filter.compare === '>=') return safeLogSql`${lhs} >= ${num}`
+          if (filter.compare === '<=') return safeLogSql`${lhs} <= ${num}`
+          if (filter.compare === '>') return safeLogSql`${lhs} > ${num}`
+          return safeLogSql`${lhs} < ${num}`
+        }
+        default:
+          return safeLogSql`${col} = ${stringLit}`
+      }
+    })
+    .filter((c) => c !== null)
+
+  if (conditions.length === 0) return safeLogSql``
+
+  const joined = joinSqlFragments(conditions, ' AND ')
+  return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}`
+}
+
 export const PRESET_CONFIG: Record<Presets, PresetConfig> = {
   [Presets.API]: {
     title: 'API',

apps/studio/components/interfaces/Reports/SharedAPIReport/SharedAPIReport.constants.ts

@@ -1,13 +1,14 @@
 import * as Sentry from '@sentry/nextjs'
 import { useQueries, useQueryClient } from '@tanstack/react-query'
-import { useParams } from 'common'
+import { useFlag, useParams } from 'common'
 import { isEqual } from 'lodash'
 import { useState } from 'react'
 
-import { generateRegexpWhereSafe } from '../Reports.constants'
+import { generateOtelWhereSafe, generateRegexpWhereSafe } from '../Reports.constants'
 import { ReportFilterItem } from '../Reports.types'
 import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql'
-import { safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql'
+import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint'
+import { analyticsLiteral, safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql'
 
 const SOURCE_TABLE: Record<string, SafeLogSqlFragment> = {
   edge_logs: safeSql`edge_logs`,
@@ -19,6 +20,37 @@ function sourceTable(src: string): SafeLogSqlFragment {
   return SOURCE_TABLE[src] ?? SOURCE_TABLE.edge_logs
 }
 
+// --- OTEL / ClickHouse variants -------------------------------------------------
+// The OTEL `logs` table is a single table keyed by `source`, with request/response
+// fields stored in the `log_attributes` Map. These helpers mirror the BigQuery
+// builders above so the chart-facing result columns (timestamp/count/avg/path/...)
+// stay identical; only the SQL dialect changes.
+
+const OTEL_SOURCE = new Set(['edge_logs', 'function_edge_logs'])
+const otelSourceName = (src: string): string => (OTEL_SOURCE.has(src) ? src : 'edge_logs')
+
+/** `WHERE source = '<src>' [AND <extra>] [AND <user filters>]` for the OTEL logs table. */
+function otelWhere(
+  src: string,
+  filters: ReportFilterItem[],
+  extra?: SafeLogSqlFragment
+): SafeLogSqlFragment {
+  const base = extra
+    ? safeSql`where source = ${analyticsLiteral(otelSourceName(src))} and ${extra}`
+    : safeSql`where source = ${analyticsLiteral(otelSourceName(src))}`
+  return safeSql`${base} ${generateOtelWhereSafe(filters, false)}`
+}
+
+// Route grouping columns, shared by the top-routes style queries.
+const OTEL_ROUTE_SELECT: SafeLogSqlFragment = safeSql`
+  log_attributes['request.path'] as path,
+  log_attributes['request.method'] as method,
+  log_attributes['request.search'] as search,
+  log_attributes['response.status_code'] as status_code`
+const OTEL_ROUTE_GROUP_BY: SafeLogSqlFragment = safeSql`log_attributes['request.path'], log_attributes['request.method'], log_attributes['request.search'], log_attributes['response.status_code']`
+const OTEL_STATUS_IS_ERROR: SafeLogSqlFragment = safeSql`toInt64OrZero(log_attributes['response.status_code']) >= 400`
+const OTEL_ORIGIN_TIME: SafeLogSqlFragment = safeSql`toFloat64OrZero(log_attributes['response.origin_time'])`
+
 export const SHARED_API_REPORT_SQL = {
   totalRequests: {
     queryType: 'logs',
@@ -38,6 +70,14 @@ export const SHARED_API_REPORT_SQL = {
           timestamp
         ORDER BY
           timestamp ASC`,
+    safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment =>
+      safeSql`
+        -- reports-api-total-requests (otel)
+        select toStartOfHour(timestamp) as timestamp, count() as count
+        from logs
+        ${otelWhere(src, filters)}
+        group by timestamp
+        order by timestamp asc`,
   },
   topRoutes: {
     queryType: 'logs',
@@ -62,6 +102,15 @@ export const SHARED_API_REPORT_SQL = {
           count desc
         limit 10
         `,
+    safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment =>
+      safeSql`
+        -- reports-api-top-routes (otel)
+        select ${OTEL_ROUTE_SELECT}, count() as count
+        from logs
+        ${otelWhere(src, filters)}
+        group by ${OTEL_ROUTE_GROUP_BY}
+        order by count desc
+        limit 10`,
   },
   errorCounts: {
     queryType: 'logs',
@@ -84,6 +133,14 @@ export const SHARED_API_REPORT_SQL = {
         ORDER BY
           timestamp ASC
         `,
+    safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment =>
+      safeSql`
+        -- reports-api-error-counts (otel)
+        select toStartOfHour(timestamp) as timestamp, count() as count
+        from logs
+        ${otelWhere(src, filters, OTEL_STATUS_IS_ERROR)}
+        group by timestamp
+        order by timestamp asc`,
   },
   topErrorRoutes: {
     queryType: 'logs',
@@ -110,6 +167,15 @@ export const SHARED_API_REPORT_SQL = {
           count desc
         limit 10
         `,
+    safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment =>
+      safeSql`
+        -- reports-api-top-error-routes (otel)
+        select ${OTEL_ROUTE_SELECT}, count() as count
+        from logs
+        ${otelWhere(src, filters, OTEL_STATUS_IS_ERROR)}
+        group by ${OTEL_ROUTE_GROUP_BY}
+        order by count desc
+        limit 10`,
   },
   responseSpeed: {
     queryType: 'logs',
@@ -131,6 +197,14 @@ export const SHARED_API_REPORT_SQL = {
         ORDER BY
           timestamp ASC
       `,
+    safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment =>
+      safeSql`
+        -- reports-api-response-speed (otel)
+        select toStartOfHour(timestamp) as timestamp, avg(${OTEL_ORIGIN_TIME}) as avg
+        from logs
+        ${otelWhere(src, filters)}
+        group by timestamp
+        order by timestamp asc`,
   },
   topSlowRoutes: {
     queryType: 'logs',
@@ -156,6 +230,15 @@ export const SHARED_API_REPORT_SQL = {
           avg desc
         limit 10
         `,
+    safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment =>
+      safeSql`
+        -- reports-api-top-slow-routes (otel)
+        select ${OTEL_ROUTE_SELECT}, count() as count, avg(${OTEL_ORIGIN_TIME}) as avg
+        from logs
+        ${otelWhere(src, filters)}
+        group by ${OTEL_ROUTE_GROUP_BY}
+        order by avg desc
+        limit 10`,
   },
   networkTraffic: {
     queryType: 'logs',
@@ -195,6 +278,17 @@ export const SHARED_API_REPORT_SQL = {
         ORDER BY
           timestamp ASC
         `,
+    safeSqlOtel: (filters: ReportFilterItem[], src = 'edge_logs'): SafeLogSqlFragment =>
+      safeSql`
+        -- reports-api-network-traffic (otel)
+        select
+          toStartOfHour(timestamp) as timestamp,
+          sum(toInt64OrZero(log_attributes['request.headers.content_length'])) / 1000000 as ingress_mb,
+          sum(toInt64OrZero(log_attributes['response.headers.content_length'])) / 1000000 as egress_mb
+        from logs
+        ${otelWhere(src, filters)}
+        group by timestamp
+        order by timestamp asc`,
   },
 }
 
@@ -225,6 +319,12 @@ export const useSharedAPIReport = ({
   const { ref } = useParams() as { ref: string }
   const [filters, setFilters] = useState<ReportFilterItem[]>([])
   const queryClient = useQueryClient()
+
+  // When enabled, route report queries through the OTEL ClickHouse endpoint
+  // (logs.all.otel) with the ClickHouse SQL variants instead of BigQuery.
+  const useOtel = useFlag('otelReports')
+  const buildSql = (entry: (typeof SHARED_API_REPORT_SQL)[SharedAPIReportKey]) =>
+    useOtel ? entry.safeSqlOtel : entry.safeSql
   const filterByMapSource = {
     functions: 'function_edge_logs',
     realtime: 'edge_logs',
@@ -262,14 +362,15 @@ export const useSharedAPIReport = ({
         start,
         end,
         ref,
+        { otel: useOtel },
       ],
       enabled: enabled && !!ref && !!filterBy,
       queryFn: async () => {
         try {
           const data = await executeAnalyticsSql({
             projectRef: ref,
-            endpoint: '/platform/projects/{ref}/analytics/endpoints/logs.all',
-            sql: value.safeSql(allFilters, filterByMapSource[filterBy]),
+            endpoint: logsAllEndpointUrl(useOtel),
+            sql: buildSql(value)(allFilters, filterByMapSource[filterBy]),
             iso_timestamp_start: start,
             iso_timestamp_end: end,
             method: 'get',
@@ -333,30 +434,13 @@ export const useSharedAPIReport = ({
 
   const isLoadingData = Object.values(isLoading).some(Boolean)
 
-  const SQLMap: Record<SharedAPIReportKey, SafeLogSqlFragment> = {
-    totalRequests: SHARED_API_REPORT_SQL.totalRequests.safeSql(
-      allFilters,
-      filterByMapSource[filterBy]
-    ),
-    topRoutes: SHARED_API_REPORT_SQL.topRoutes.safeSql(allFilters, filterByMapSource[filterBy]),
-    errorCounts: SHARED_API_REPORT_SQL.errorCounts.safeSql(allFilters, filterByMapSource[filterBy]),
-    topErrorRoutes: SHARED_API_REPORT_SQL.topErrorRoutes.safeSql(
-      allFilters,
-      filterByMapSource[filterBy]
-    ),
-    responseSpeed: SHARED_API_REPORT_SQL.responseSpeed.safeSql(
-      allFilters,
-      filterByMapSource[filterBy]
-    ),
-    topSlowRoutes: SHARED_API_REPORT_SQL.topSlowRoutes.safeSql(
-      allFilters,
-      filterByMapSource[filterBy]
-    ),
-    networkTraffic: SHARED_API_REPORT_SQL.networkTraffic.safeSql(
-      allFilters,
-      filterByMapSource[filterBy]
-    ),
-  }
+  const SQLMap = keys.reduce(
+    (acc, key) => {
+      acc[key] = buildSql(SHARED_API_REPORT_SQL[key])(allFilters, filterByMapSource[filterBy])
+      return acc
+    },
+    {} as Record<SharedAPIReportKey, SafeLogSqlFragment>
+  )
 
   return {
     data,

apps/studio/data/reports/report.utils.ts

@@ -2,6 +2,7 @@ import { type ComparisonOperator } from '@/components/interfaces/Reports/v2/Repo
 import { AnalyticsInterval } from '@/data/analytics/constants'
 import { useEdgeFunctionsQuery } from '@/data/edge-functions/edge-functions-query'
 import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql'
+import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint'
 import { safeSql, type SafeLogSqlFragment } from '@/data/logs/safe-analytics-sql'
 
 export type Granularity = 'minute' | 'hour' | 'day'
@@ -85,11 +86,14 @@ export async function fetchLogs(
   projectRef: string,
   sql: SafeLogSqlFragment,
   startDate: string,
-  endDate: string
+  endDate: string,
+  // Pass true (with ClickHouse SQL) to route through the OTEL endpoint. Defaults
+  // to the BigQuery endpoint until the edge-functions report SQL is migrated.
+  useOtel = false
 ) {
   return await executeAnalyticsSql({
     projectRef,
-    endpoint: '/platform/projects/{ref}/analytics/endpoints/logs.all',
+    endpoint: logsAllEndpointUrl(useOtel),
     sql,
     iso_timestamp_start: startDate,
     iso_timestamp_end: endDate,