feat(studio): deterministic BigQuery->ClickHouse rewrite for Logs Explorer

Add rewriteBqLogsSqlToClickhouse: parses the user's BigQuery logs query with
the polyglot SDK, rewrites it onto the OTEL logs schema on the AST (FROM logs +
source filter, drop unnest joins, nested fields -> log_attributes[...] with
numeric casts), and regenerates ClickHouse SQL. The banner's button now runs
this directly on the editor and only falls back to the AI Assistant when it
can't. Lazy-loaded so the WASM stays out of the main bundle.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

apps/studio/components/interfaces/Settings/Logs/LogsExplorerOtelBanner.tsx

@@ -1,7 +1,10 @@
 import { useLocalStorage } from '@uidotdev/usehooks'
+import { useState } from 'react'
+import { toast } from 'sonner'
 import { Button } from 'ui'
 import { Admonition } from 'ui-patterns'
 
+import { rewriteBqLogsSqlToClickhouse } from './logs-sql-rewrite'
 import { buildClickhouseRewritePrompt } from './Logs.utils'
 import { SIDEBAR_KEYS } from '@/components/layouts/ProjectLayout/LayoutSidebar/LayoutSidebarProvider'
 import { useAiAssistantStateSnapshot } from '@/state/ai-assistant-state'
@@ -10,16 +13,23 @@ import { useSidebarManagerSnapshot } from '@/state/sidebar-manager-state'
 interface LogsExplorerOtelBannerProps {
   projectRef: string
   sql: string
+  onApplyRewrite: (sql: string) => void
 }
 
 /**
  * Shown in the Logs Explorer when it runs against the ClickHouse-backed OTEL
- * endpoint, warning that the SQL dialect changed from BigQuery and offering the
- * AI Assistant to rewrite existing queries. Dismissal is persisted per project.
+ * endpoint, warning that the SQL dialect changed from BigQuery. The primary
+ * action rewrites the current query to ClickHouse deterministically and falls
+ * back to the AI Assistant when it can't. Dismissal is persisted per project.
  */
-export const LogsExplorerOtelBanner = ({ projectRef, sql }: LogsExplorerOtelBannerProps) => {
+export const LogsExplorerOtelBanner = ({
+  projectRef,
+  sql,
+  onApplyRewrite,
+}: LogsExplorerOtelBannerProps) => {
   const { openSidebar } = useSidebarManagerSnapshot()
   const aiSnap = useAiAssistantStateSnapshot()
+  const [isRewriting, setIsRewriting] = useState(false)
   const [dismissed, setDismissed] = useLocalStorage<boolean>(
     `logs-explorer-clickhouse-banner-dismissed-${projectRef}`,
     false
@@ -35,6 +45,29 @@ export const LogsExplorerOtelBanner = ({ projectRef, sql }: LogsExplorerOtelBann
     })
   }
 
+  const handleRewrite = async () => {
+    if (!sql.trim()) {
+      openRewriteAssistant()
+      return
+    }
+    setIsRewriting(true)
+    try {
+      const { sql: rewritten, changed } = await rewriteBqLogsSqlToClickhouse(sql)
+      if (changed) {
+        onApplyRewrite(rewritten)
+        toast.success('Rewrote the query for ClickHouse')
+      } else {
+        // Nothing to rewrite deterministically, let the assistant help.
+        openRewriteAssistant()
+      }
+    } catch {
+      toast.info("Couldn't rewrite automatically, opening the AI Assistant")
+      openRewriteAssistant()
+    } finally {
+      setIsRewriting(false)
+    }
+  }
+
   return (
     <Admonition
       type="warning"
@@ -43,8 +76,8 @@ export const LogsExplorerOtelBanner = ({ projectRef, sql }: LogsExplorerOtelBann
       description="This project's logs run on a new ClickHouse-backed engine, which uses a different SQL dialect than BigQuery. Existing saved queries may need to be rewritten."
       actions={
         <div className="flex items-center gap-2">
-          <Button variant="default" size="tiny" onClick={openRewriteAssistant}>
-            Rewrite with AI Assistant
+          <Button variant="default" size="tiny" loading={isRewriting} onClick={handleRewrite}>
+            Rewrite to ClickHouse
           </Button>
           <Button variant="text" size="tiny" onClick={() => setDismissed(true)}>
             Dismiss

apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from 'vitest'
+
+import { rewriteBqLogsSqlToClickhouse } from './logs-sql-rewrite'
+
+const collapse = (sql: string) => sql.replace(/\s+/g, ' ').trim()
+
+describe('rewriteBqLogsSqlToClickhouse', () => {
+  it('rewrites an edge_logs query onto the OTEL logs schema', async () => {
+    const { sql, changed } =
+      await rewriteBqLogsSqlToClickhouse(`select id, request.method, response.status_code
+from edge_logs
+cross join unnest(metadata) as m
+cross join unnest(m.request) as request
+cross join unnest(m.response) as response
+where response.status_code >= 500 and request.method = 'POST'
+order by edge_logs.timestamp desc
+limit 100`)
+    const out = collapse(sql)
+    expect(changed).toBe(true)
+    expect(out).toContain('FROM logs')
+    expect(out).toContain("source = 'edge_logs'")
+    // unnest joins folded into map access
+    expect(out.toLowerCase()).not.toContain('unnest')
+    expect(out.toLowerCase()).not.toContain('arrayjoin')
+    // column refs rewritten, alias preserved
+    expect(out).toContain("log_attributes['request.method'] AS method")
+    expect(out).toContain("log_attributes['response.status_code']")
+    // numeric field cast for the comparison
+    expect(out).toContain("toInt32OrZero(log_attributes['response.status_code']) >= 500")
+    // string field not cast
+    expect(out).toContain("log_attributes['request.method'] = 'POST'")
+  })
+
+  it('rewrites postgres severity columns', async () => {
+    const { sql, changed } = await rewriteBqLogsSqlToClickhouse(`select id, parsed.error_severity
+from postgres_logs
+cross join unnest(metadata) as m
+cross join unnest(m.parsed) as parsed`)
+    expect(changed).toBe(true)
+    expect(collapse(sql)).toContain("source = 'postgres_logs'")
+    expect(collapse(sql)).toContain("log_attributes['parsed.error_severity']")
+  })
+
+  it('leaves a query with no legacy log table unchanged', async () => {
+    const { sql, changed } = await rewriteBqLogsSqlToClickhouse('select 1')
+    expect(changed).toBe(false)
+    expect(sql).toBe('select 1')
+  })
+})

apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.ts

@@ -0,0 +1,177 @@
+/**
+ * Deterministically rewrites a legacy BigQuery logs query into the ClickHouse
+ * dialect + schema used by the OTEL `logs.all.otel` endpoint.
+ *
+ * The old logs live in per-service tables (`edge_logs`, `postgres_logs`, …) with
+ * nested fields reached through `cross join unnest(metadata)`. The OTEL endpoint
+ * stores everything in one `logs` table keyed by `source`, with those fields in a
+ * `log_attributes` map. So the rewrite, done on the parsed query tree:
+ *   - points the FROM at `logs` and adds `source = '<table>'`
+ *   - drops the `unnest` joins
+ *   - turns `request.method` style refs into `log_attributes['request.method']`
+ *   - wraps numeric fields in `toInt32OrZero` so comparisons still work
+ *
+ * Parsing/generation runs through the WASM-backed polyglot SDK, loaded lazily so
+ * it never enters the main bundle.
+ */
+
+// Legacy BigQuery logs tables. A FROM referencing one of these is remapped onto
+// the OTEL `logs` table with a matching `source` filter (the names are identical).
+const LOG_TABLES = new Set<string>([
+  'edge_logs',
+  'postgres_logs',
+  'function_logs',
+  'function_edge_logs',
+  'auth_logs',
+  'auth_audit_logs',
+  'realtime_logs',
+  'storage_logs',
+  'postgrest_logs',
+  'supavisor_logs',
+  'pgbouncer_logs',
+  'pg_upgrade_logs',
+  'pg_cron_logs',
+  'etl_replication_logs',
+  'multigres_logs',
+])
+
+// BigQuery unnest alias -> OTEL `log_attributes` key prefix. The legacy queries
+// unnest `metadata` (aliased `m`/`metadata`, the root, so no prefix) and its
+// `request` / `response` / `parsed` structs (which keep their name as the prefix).
+const ALIAS_PREFIX: Record<string, string> = {
+  request: 'request.',
+  response: 'response.',
+  parsed: 'parsed.',
+  m: '',
+  metadata: '',
+}
+
+// `log_attributes` values are strings; these keys are compared/aggregated as
+// numbers, so they need `toInt32OrZero(...)`.
+const NUMERIC_KEYS = new Set([
+  'response.status_code',
+  'request.status_code',
+  'status',
+  'execution_time_ms',
+])
+
+export interface RewriteResult {
+  sql: string
+  /** False when the query referenced no legacy log table, so nothing was rewritten. */
+  changed: boolean
+}
+
+type Sdk = typeof import('@polyglot-sql/sdk')
+
+let sdkPromise: Promise<Sdk> | null = null
+const loadSdk = (): Promise<Sdk> => {
+  if (sdkPromise === null) {
+    sdkPromise = import('@polyglot-sql/sdk').then(async (sdk) => {
+      await sdk.init?.()
+      return sdk
+    })
+  }
+  return sdkPromise
+}
+
+const isUnnestColumn = (node: any): boolean =>
+  !!node?.column?.table && ALIAS_PREFIX[node.column.table.name] !== undefined
+
+const otelKey = (node: any): string =>
+  `${ALIAS_PREFIX[node.column.table.name]}${node.column.name.name}`
+
+const isSourceColumn = (node: any, source: string): boolean =>
+  !!node?.column?.table && (node.column.table.name === source || node.column.table.name === 'logs')
+
+export async function rewriteBqLogsSqlToClickhouse(input: string): Promise<RewriteResult> {
+  const sdk = await loadSdk()
+
+  const parsed = sdk.parse(input, 'bigquery')
+  if (!parsed.success || !parsed.ast) {
+    throw new Error(parsed.error || 'Could not parse the query.')
+  }
+
+  // Build a `log_attributes['key']` expression node (numeric-cast when needed) by
+  // lifting it out of a tiny parsed query.
+  const attrExpr = (key: string): any => {
+    const expr = NUMERIC_KEYS.has(key)
+      ? `toInt32OrZero(log_attributes['${key}'])`
+      : `log_attributes['${key}']`
+    return sdk.parse(`select ${expr} as x from t`, 'clickhouse').ast[0].select.expressions[0].alias
+      .this
+  }
+
+  const rewriteColumns = (node: any, source: string): void => {
+    if (!node || typeof node !== 'object') return
+    for (const k of Object.keys(node)) {
+      const child = node[k]
+      if (isUnnestColumn(child)) node[k] = attrExpr(otelKey(child))
+      else if (isSourceColumn(child, source)) child.column.table = null
+      else rewriteColumns(child, source)
+    }
+  }
+
+  const injectSourceFilter = (select: any, source: string): void => {
+    const srcWhere = sdk.parse(`select 1 from t where source = '${source}'`, 'clickhouse').ast[0]
+      .select.where_clause
+    if (!select.where_clause) {
+      select.where_clause = srcWhere
+      return
+    }
+    const andWhere = sdk.parse(`select 1 from t where x and y`, 'clickhouse').ast[0].select
+      .where_clause
+    const conj = Object.keys(andWhere.this)[0]
+    andWhere.this[conj].left = srcWhere.this
+    andWhere.this[conj].right = select.where_clause.this
+    select.where_clause = andWhere
+  }
+
+  const remapSelect = (select: any): boolean => {
+    const fromTable = select?.from?.expressions?.[0]?.table
+    const source = fromTable?.name?.name
+    if (!source || !LOG_TABLES.has(source)) return false
+
+    fromTable.name.name = 'logs'
+    // Legacy log queries only join via `unnest`; those joins are folded into the
+    // `log_attributes` map access, so drop them.
+    select.joins = []
+
+    // SELECT list: keep the original leaf name as the alias so result columns
+    // (and the renderers reading them) are unchanged.
+    select.expressions = select.expressions.map((expr: any) => {
+      if (isUnnestColumn(expr)) {
+        const leaf = expr.column.name.name
+        const aliased = sdk.parse(`select x as ${leaf} from t`, 'clickhouse').ast[0].select
+          .expressions[0]
+        aliased.alias.this = attrExpr(otelKey(expr))
+        return aliased
+      }
+      if (isSourceColumn(expr, source)) {
+        expr.column.table = null
+        return expr
+      }
+      rewriteColumns(expr, source)
+      return expr
+    })
+
+    rewriteColumns(select.where_clause, source)
+    rewriteColumns(select.group_by, source)
+    rewriteColumns(select.having, source)
+    rewriteColumns(select.order_by, source)
+
+    injectSourceFilter(select, source)
+    return true
+  }
+
+  let changed = false
+  for (const statement of parsed.ast) {
+    if (statement?.select) changed = remapSelect(statement.select) || changed
+  }
+  if (!changed) return { sql: input, changed: false }
+
+  const generated = sdk.generate(parsed.ast, 'clickhouse')
+  if (!generated.success || !generated.sql?.[0]) {
+    throw new Error(generated.error || 'Could not generate ClickHouse SQL.')
+  }
+  return { sql: generated.sql[0], changed: true }
+}

apps/studio/package.json

@@ -54,6 +54,7 @@
     "@monaco-editor/react": "^4.6.0",
     "@next/bundle-analyzer": "16.2.3",
     "@number-flow/react": "^0.3.2",
+    "@polyglot-sql/sdk": "^0.5.7",
     "@sentry/nextjs": "catalog:",
     "@std/path": "npm:@jsr/std__path@^1.0.8",
     "@stripe/react-stripe-js": "6.1.0",

apps/studio/pages/project/[ref]/logs/explorer/index.tsx

@@ -200,6 +200,22 @@ export const LogsExplorerPage: NextPageWithLayout = () => {
     addRecentLogSqlSnippet({ sql: template.searchString })
   }
 
+  const applyRewrite = (newSql: string) => {
+    setEditorValue(newSql)
+    if (editorRef.current && monaco) {
+      const editorModel = editorRef.current.getModel()
+      editorRef.current.pushUndoStop()
+      editorRef.current.executeEdits('rewrite-clickhouse', [
+        {
+          text: newSql,
+          range: editorModel?.getFullModelRange() ?? new monaco.Range(1, 1, 1, 1),
+        },
+      ])
+      editorRef.current.pushUndoStop()
+      editorRef.current.focus()
+    }
+  }
+
   const handleRun = (value?: string | React.MouseEvent) => {
     track('log_explorer_query_run_button_clicked', { is_saved_query: !!queryId })
 
@@ -387,7 +403,13 @@ export const LogsExplorerPage: NextPageWithLayout = () => {
             onSelectTemplate={onSelectTemplate}
             warnings={warnings}
           />
-          {useOtelEndpoint && <LogsExplorerOtelBanner projectRef={projectRef} sql={editorValue} />}
+          {useOtelEndpoint && (
+            <LogsExplorerOtelBanner
+              projectRef={projectRef}
+              sql={editorValue}
+              onApplyRewrite={applyRewrite}
+            />
+          )}
           <ShimmerLine active={isLoading} />
           <CodeEditor
             // Ensure we reset the editor to the query content whenever the selected query changes

pnpm-lock.yaml

@@ -925,6 +925,9 @@ importers:
       '@number-flow/react':
         specifier: ^0.3.2
         version: 0.3.2(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
+      '@polyglot-sql/sdk':
+        specifier: ^0.5.7
+        version: 0.5.7
       '@sentry/nextjs':
         specifier: 'catalog:'
         version: 10.27.0(@opentelemetry/context-async-hooks@2.2.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.2.6(@babel/core@7.29.7(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6)(supports-color@8.1.1)(webpack@5.105.4(esbuild@0.28.1))
@@ -1336,7 +1339,7 @@ importers:
         version: 2.11.3(@types/node@22.13.14)(typescript@6.0.2)
       next-router-mock:
         specifier: ^0.9.13
-        version: 0.9.13(next@16.2.6(@babel/core@7.29.7(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6)
+        version: 0.9.13(next@16.2.6(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6)
       node-mocks-http:
         specifier: ^1.17.2
         version: 1.17.2(@types/node@22.13.14)
@@ -2217,7 +2220,7 @@ importers:
         version: link:../config
       next-router-mock:
         specifier: ^0.9.13
-        version: 0.9.13(next@16.2.6(@babel/core@7.29.7(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6)
+        version: 0.9.13(next@16.2.6(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6)
       tailwindcss:
         specifier: 'catalog:'
         version: 4.2.4
@@ -2705,7 +2708,7 @@ importers:
         version: link:../config
       next-router-mock:
         specifier: ^0.9.13
-        version: 0.9.13(next@16.2.6(@babel/core@7.29.7(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6)
+        version: 0.9.13(next@16.2.6(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6)
       tailwindcss:
         specifier: ^4.2.4
         version: 4.2.4
@@ -6037,6 +6040,10 @@ packages:
   '@polka/url@1.0.0-next.25':
     resolution: {integrity: sha512-j7P6Rgr3mmtdkeDGTe0E/aYyWEWVtc5yFXtHCRHs28/jptDEWfaVOc5T7cblqy1XKPPfCxJc/8DwQ5YgLOZOVQ==}
 
+  '@polyglot-sql/sdk@0.5.7':
+    resolution: {integrity: sha512-qcB+RpI9RNmeLvAdtpZIcuiBlevqwt80mbHV8Ielyz5KROAsD/cmWg765lTXHGDDxvWT7SQECpxGruDB0uY7OQ==}
+    engines: {node: '>=22', pnpm: '>=10'}
+
   '@popperjs/core@2.11.8':
     resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==}
 
@@ -15429,6 +15436,7 @@ packages:
   recharts@2.15.4:
     resolution: {integrity: sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==}
     engines: {node: '>=14'}
+    deprecated: 1.x and 2.x branches are no longer active. Bump to Recharts v3 to receive latest features and bugfixes. See https://github.com/recharts/recharts/wiki/3.0-migration-guide
     peerDependencies:
       react: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
       react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
@@ -16717,6 +16725,7 @@ packages:
   tsconfck@3.0.3:
     resolution: {integrity: sha512-4t0noZX9t6GcPTfBAbIbbIU4pfpCwh0ueq3S4O/5qXI1VwK1outmxhe9dOiEWqMz3MW2LKgDTpqWV+37IWuVbA==}
     engines: {node: ^18 || >=20}
+    deprecated: unmaintained
     hasBin: true
     peerDependencies:
       typescript: ^5.0.0
@@ -21998,6 +22007,8 @@ snapshots:
 
   '@polka/url@1.0.0-next.25': {}
 
+  '@polyglot-sql/sdk@0.5.7': {}
+
   '@popperjs/core@2.11.8': {}
 
   '@poppinss/colors@4.1.6':
@@ -31440,7 +31451,7 @@ snapshots:
     dependencies:
       js-yaml-loader: 1.2.2
 
-  next-router-mock@0.9.13(next@16.2.6(@babel/core@7.29.7(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6):
+  next-router-mock@0.9.13(next@16.2.6(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4))(react@19.2.6):
     dependencies:
       next: 16.2.6(@babel/core@7.29.7(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.59.1)(babel-plugin-macros@3.1.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(sass@1.77.4)
       react: 19.2.6