From 2aa1b5223430aabdf0fa20543e705f1f7fe741f5 Mon Sep 17 00:00:00 2001 From: Jordi Enric <37541088+jordienr@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:31:18 +0200 Subject: [PATCH] feat(studio): add feature to rewrite queries DEBUG-145 (#47266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Moving the Logs Explorer to ClickHouse means users' saved BigQuery queries no longer run. CleanShot 2026-06-29 at 11 36 04@2x ## Fix Rewrite the query with AI instead of a SQL transpiler. AI handles the long tail of nested fields and dialect differences far better than a rule-based rewriter, and it needs no extra runtime dependency. - `rewriteLogsSqlWithAI` posts the current query to `/api/ai/code/complete` with `dialect: 'clickhouse'`. The endpoint skips the Postgres schema and best-practices for that dialect and uses logs-specific instructions and model so the output is ClickHouse logs SQL (FROM `logs` + `source` filter, no `unnest` joins, nested fields read from `log_attributes['...']`). - The query's `source` is detected and its real `log_attributes` keys are fetched and passed to the model, so it maps to exact paths instead of guessing. - The rewrite runs in the background and is proposed as a side-by-side accept/discard diff in the editor. The AI Assistant panel is not opened. - Entry points: a banner shown only for legacy-looking queries (dismissal persisted), and a "Fix Query" button next to Field Reference. - The Field Reference drawers discover `log_attributes` keys from real data so the listed fields match what the source actually emits. ## Dependencies Built on top of #47265 (Logs Explorer -> OTEL endpoint) — that is the base branch of this PR. Merge #47265 first. Behind `otelLegacyLogs` (off by default). Part of DEBUG-145 (split from #47087). ## How to test - Open the Logs Explorer with a BigQuery logs query (the templates have some), click "Fix Query", and confirm the diff shows valid ClickHouse SQL. Accept it and confirm the applied query runs. ## Summary by CodeRabbit * **New Features** * Added an OTEL legacy logs workflow (behind a feature flag) with an interactive banner and a “Fix Query” ClickHouse rewrite action, including an accept/discard diff review overlay. * Introduced OTEL-aware field reference rendering with dynamic discovery of `log_attributes` keys and updated OTEL source insertion behavior. * Enabled dialect-aware SQL completion for ClickHouse logs, using logs-specific instructions and output constraints. * **Bug Fixes** * Improved rewrite flow validation and handling, including log source detection and cleanup of AI-generated SQL formatting. * **Tests** * Added Vitest coverage for rewrite prompt generation, detection/classification utilities, SQL fence stripping, OTEL field mapping, and OTEL log attribute key discovery. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Joshen Lim --- .../Settings/Logs/Logs.fieldReference.test.ts | 102 ++++++ .../Settings/Logs/Logs.fieldReference.ts | 42 +++ .../DefaultErrorRenderer.tsx | 2 +- .../Settings/Logs/LogsExplorerOtelBanner.tsx | 34 ++ .../Settings/Logs/LogsQueryPanel.tsx | 297 +++++++++++------- .../Settings/Logs/logs-sql-rewrite.test.ts | 149 +++++++++ .../Settings/Logs/logs-sql-rewrite.ts | 142 +++++++++ .../components/ui/Logs/LogsExplorerHeader.tsx | 103 +++--- apps/studio/data/logs/otel-log-keys-query.ts | 45 +++ .../studio/hooks/analytics/useLogsUrlState.ts | 58 ++-- apps/studio/lib/ai/model.utils.ts | 5 + apps/studio/lib/ai/prompts.ts | 9 + apps/studio/pages/api/ai/code/complete.ts | 52 +-- .../project/[ref]/logs/explorer/index.tsx | 173 ++++++++-- .../features/logs/LogsQueryPanel.test.tsx | 2 +- 15 files changed, 986 insertions(+), 229 deletions(-) create mode 100644 apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.test.ts create mode 100644 apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.ts create mode 100644 apps/studio/components/interfaces/Settings/Logs/LogsExplorerOtelBanner.tsx create mode 100644 apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.test.ts create mode 100644 apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.ts create mode 100644 apps/studio/data/logs/otel-log-keys-query.ts diff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.test.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.test.ts new file mode 100644 index 00000000000..3d234244ab5 --- /dev/null +++ b/apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' + +import { otelFieldsFromKeys, toOtelFieldSchemas, type LogFieldSchema } from './Logs.fieldReference' + +const edgeSchema: LogFieldSchema = { + name: 'API Gateway', + reference: 'edge_logs', + fields: [ + { path: 'id', type: 'string' }, + { path: 'timestamp', type: 'datetime' }, + { path: 'event_message', type: 'string' }, + { path: 'identifier', type: 'string' }, + { path: 'metadata.request.method', type: 'string' }, + { path: 'metadata.request.cf.asn', type: 'number' }, + ], +} + +describe('toOtelFieldSchemas', () => { + it('keeps source name and reference', () => { + const [otel] = toOtelFieldSchemas([edgeSchema]) + expect(otel.name).toBe('API Gateway') + expect(otel.reference).toBe('edge_logs') + }) + + it('keeps id/timestamp/event_message as real columns and adds source + severity_text', () => { + const [otel] = toOtelFieldSchemas([edgeSchema]) + const paths = otel.fields.map((f) => f.path) + expect(paths.slice(0, 5)).toEqual([ + 'id', + 'timestamp', + 'event_message', + 'severity_text', + 'source', + ]) + }) + + it('moves non-column fields into log_attributes, dropping the metadata root', () => { + const [otel] = toOtelFieldSchemas([edgeSchema]) + const paths = otel.fields.map((f) => f.path) + expect(paths).toContain("log_attributes['identifier']") + expect(paths).toContain("log_attributes['request.method']") + expect(paths).toContain("log_attributes['request.cf.asn']") + expect(paths).not.toContain('metadata.request.method') + expect(paths).not.toContain('identifier') + }) + + it('preserves the original field type for casting hints', () => { + const [otel] = toOtelFieldSchemas([edgeSchema]) + const asn = otel.fields.find((f) => f.path === "log_attributes['request.cf.asn']") + expect(asn?.type).toBe('number') + }) +}) + +describe('otelFieldsFromKeys', () => { + it('starts with the five base OTEL columns', () => { + const fields = otelFieldsFromKeys(['request.cf.asn']) + const paths = fields.map((f) => f.path) + expect(paths.slice(0, 5)).toEqual([ + 'id', + 'timestamp', + 'event_message', + 'severity_text', + 'source', + ]) + }) + + it('maps discovered keys to log_attributes lookups', () => { + const fields = otelFieldsFromKeys(['request.cf.asn', 'response.headers.x-foo']) + const paths = fields.map((f) => f.path) + expect(paths).toContain("log_attributes['request.cf.asn']") + expect(paths).toContain("log_attributes['response.headers.x-foo']") + }) + + it('excludes keys that are real OTEL columns', () => { + const fields = otelFieldsFromKeys([ + 'id', + 'timestamp', + 'event_message', + 'severity_text', + 'source', + 'trace_id', + ]) + const paths = fields.map((f) => f.path) + for (const column of ['id', 'timestamp', 'event_message', 'severity_text', 'source']) { + expect(paths.filter((p) => p === column)).toHaveLength(1) + expect(paths).not.toContain(`log_attributes['${column}']`) + } + expect(paths).toContain("log_attributes['trace_id']") + }) + + it('escapes quotes and backslashes in discovered keys', () => { + const fields = otelFieldsFromKeys(["weird'key", 'back\\slash']) + const paths = fields.map((f) => f.path) + expect(paths).toContain("log_attributes['weird''key']") + expect(paths).toContain("log_attributes['back\\\\slash']") + }) + + it('returns only base fields for an empty key list', () => { + const fields = otelFieldsFromKeys([]) + expect(fields).toHaveLength(5) + }) +}) diff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.ts new file mode 100644 index 00000000000..3f89c77b372 --- /dev/null +++ b/apps/studio/components/interfaces/Settings/Logs/Logs.fieldReference.ts @@ -0,0 +1,42 @@ +import { logConstants } from 'shared-data' + +export type LogFieldSchema = (typeof logConstants.schemas)[number] +type LogField = LogFieldSchema['fields'][number] + +const OTEL_BASE_FIELDS: LogField[] = [ + { path: 'id', type: 'string' }, + { path: 'timestamp', type: 'datetime' }, + { path: 'event_message', type: 'string' }, + { path: 'severity_text', type: 'string' }, + { path: 'source', type: 'string' }, +] + +const OTEL_COLUMN_PATHS = new Set(OTEL_BASE_FIELDS.map(({ path }) => path)) + +const toOtelAttributeKey = (path: string) => path.replace(/^metadata\./, '') + +const toLogAttributePath = (key: string) => + `log_attributes['${key.replace(/\\/g, '\\\\').replace(/'/g, "''")}']` + +const toOtelFieldSchema = (schema: LogFieldSchema): LogFieldSchema => ({ + ...schema, + fields: [ + ...OTEL_BASE_FIELDS, + ...schema.fields + .filter((field) => !OTEL_COLUMN_PATHS.has(field.path)) + .map((field) => ({ + path: toLogAttributePath(toOtelAttributeKey(field.path)), + type: field.type, + })), + ], +}) + +export const toOtelFieldSchemas = (schemas: LogFieldSchema[]): LogFieldSchema[] => + schemas.map(toOtelFieldSchema) + +export function otelFieldsFromKeys(keys: string[]): LogFieldSchema['fields'] { + const attributeFields = keys + .filter((key) => !OTEL_COLUMN_PATHS.has(key)) + .map((key) => ({ path: toLogAttributePath(key), type: 'string' as const })) + return [...OTEL_BASE_FIELDS, ...attributeFields] +} diff --git a/apps/studio/components/interfaces/Settings/Logs/LogsErrorRenderers/DefaultErrorRenderer.tsx b/apps/studio/components/interfaces/Settings/Logs/LogsErrorRenderers/DefaultErrorRenderer.tsx index 444cddf7969..ed9d6df9cf5 100644 --- a/apps/studio/components/interfaces/Settings/Logs/LogsErrorRenderers/DefaultErrorRenderer.tsx +++ b/apps/studio/components/interfaces/Settings/Logs/LogsErrorRenderers/DefaultErrorRenderer.tsx @@ -14,7 +14,7 @@ export const DefaultErrorRenderer: React.FC = ({ error }) => language="json" hideLineNumbers value={typeof error === 'string' ? error : JSON.stringify(error, null, 2)} - className="w-full font-mono" + className="w-full font-mono px-4" /> ) diff --git a/apps/studio/components/interfaces/Settings/Logs/LogsExplorerOtelBanner.tsx b/apps/studio/components/interfaces/Settings/Logs/LogsExplorerOtelBanner.tsx new file mode 100644 index 00000000000..050b5e2d793 --- /dev/null +++ b/apps/studio/components/interfaces/Settings/Logs/LogsExplorerOtelBanner.tsx @@ -0,0 +1,34 @@ +import { Button } from 'ui' +import { Admonition } from 'ui-patterns' + +interface LogsExplorerOtelBannerProps { + isRewriting: boolean + onRewrite: () => void + onDismiss: () => void +} + +export const LogsExplorerOtelBanner = ({ + isRewriting, + onRewrite, + onDismiss, +}: LogsExplorerOtelBannerProps) => { + return ( + + + + + } + /> + ) +} diff --git a/apps/studio/components/interfaces/Settings/Logs/LogsQueryPanel.tsx b/apps/studio/components/interfaces/Settings/Logs/LogsQueryPanel.tsx index a6b583f1587..c71c9fdae2c 100644 --- a/apps/studio/components/interfaces/Settings/Logs/LogsQueryPanel.tsx +++ b/apps/studio/components/interfaces/Settings/Logs/LogsQueryPanel.tsx @@ -1,10 +1,12 @@ -import { BookOpen, Check, ChevronDown, ChevronsUpDown, Copy, ExternalLink, X } from 'lucide-react' +import { useFlag, useParams } from 'common' +import { BookOpen, Check, ChevronDown, ChevronsUpDown, Copy, ExternalLink } from 'lucide-react' import Link from 'next/link' import { ReactNode, useEffect, useState } from 'react' import { logConstants } from 'shared-data' import { Badge, Button, + Card, cn, Command, CommandEmpty, @@ -20,11 +22,24 @@ import { Popover, PopoverContent, PopoverTrigger, - SidePanel, + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetSection, + SheetTitle, + SheetTrigger, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, Tooltip, TooltipContent, TooltipTrigger, } from 'ui' +import { ShimmeringLoader } from 'ui-patterns' import { EXPLORER_DATEPICKER_HELPERS, @@ -32,19 +47,23 @@ import { LogsTableName, } from './Logs.constants' import { DatePickerValue, LogsDatePicker } from './Logs.DatePickers' +import { otelFieldsFromKeys, toOtelFieldSchemas } from './Logs.fieldReference' import { LogsWarning, LogTemplate } from './Logs.types' -import Table from '@/components/to-be-cleaned/Table' +import { useOtelLogKeysQuery } from '@/data/logs/otel-log-keys-query' import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled' import { useShowMultigresLogs } from '@/hooks/misc/useShowMultigresLogs' import { DOCS_URL } from '@/lib/constants' -export interface LogsQueryPanelProps { +interface LogsQueryPanelProps { templates?: LogTemplate[] value: DatePickerValue warnings: LogsWarning[] onSelectTemplate: (template: LogTemplate) => void onSelectSource: (source: string) => void onDateChange: (value: DatePickerValue) => void + showRewriteAction?: boolean + isRewriting?: boolean + onRewrite?: () => void } function DropdownMenuItemContent({ name, desc }: { name: ReactNode; desc?: string }) { @@ -56,13 +75,16 @@ function DropdownMenuItemContent({ name, desc }: { name: ReactNode; desc?: strin ) } -const LogsQueryPanel = ({ +export const LogsQueryPanel = ({ templates = [], value, warnings, onSelectTemplate, onSelectSource, onDateChange, + showRewriteAction = false, + isRewriting = false, + onRewrite, }: LogsQueryPanelProps) => { const [showReference, setShowReference] = useState(false) const { logsTemplates } = useIsFeatureEnabled(['logs:templates']) @@ -92,11 +114,20 @@ const LogsQueryPanel = ({ const [open, setOpen] = useState(false) const showMultigresLogs = useShowMultigresLogs() - const schemas = logConstants.schemas.filter( + const useOtel = useFlag('otelLegacyLogs') + const baseSchemas = logConstants.schemas.filter( (schema) => schema.reference !== 'multigres_logs' || showMultigresLogs ) + const schemas = useOtel ? toOtelFieldSchemas(baseSchemas) : baseSchemas - const [selectedSchema, setSelectedSchema] = useState(schemas[0]) + const [selectedRef, setSelectedRef] = useState(schemas[0]?.reference) + const selectedSchema = schemas.find((s) => s.reference === selectedRef) ?? schemas[0] + + const { ref: projectRef } = useParams() + const { data: discoveredKeys, isPending: isLoadingKeys } = useOtelLogKeysQuery( + { projectRef, source: selectedRef }, + { enabled: useOtel && showReference } + ) return (
@@ -187,116 +218,142 @@ const LogsQueryPanel = ({
- -

Field Reference

- - } - > - -
-

- The following table shows all the available paths that can be queried from each - respective source. Do note that to access nested keys, you would need to perform - the necessary{' '} - - unnesting joins - - -

-
- - - -
- - +
+ {showRewriteAction && onRewrite && ( + + - - - - - - No source found. - - {schemas.map((schema) => ( - { - setSelectedSchema(schema) - setOpen(false) - }} - > - - {schema.name} - - ))} - - - - - - - Path - , - - Type - , - ]} - body={selectedSchema.fields.map((field) => ( - - ))} - /> - - + + + Logs now run on a ClickHouse-backed engine. Click to rewrite this query with the + Assistant. + + + )} + + + + + + + + Field Reference + {useOtel ? ( + + The following table shows the fields available on each source. Nested fields + live in the{' '} + log_attributes{' '} + map and are read with{' '} + + log_attributes['key'] + {' '} + — no unnesting joins needed. + + ) : ( + + The following table shows all the available paths that can be queried from + each respective source. Do note that to access nested keys, you would need to + perform the necessary{' '} + + unnesting joins + + + + )} + + + + + + + + + + + + No source found. + + {schemas.map((schema) => ( + { + setSelectedRef(schema.reference) + setOpen(false) + }} + > + + {schema.name} + + ))} + + + + + + + +
+ + + Path + Type + + + + {useOtel && isLoadingKeys + ? Array.from({ length: 3 }).map((_, i) => ( + + + + + + )) + : (useOtel + ? otelFieldsFromKeys(discoveredKeys ?? []) + : selectedSchema.fields + ).map((field) => )} + +
+ + + + +
@@ -314,8 +371,8 @@ const Field = ({ const [isCopied, setIsCopied] = useState(false) return ( - - + copyToClipboard(field.path, () => { @@ -340,10 +397,8 @@ const Field = ({ Copy value )} - - {field.type} - + + {field.type} + ) } - -export default LogsQueryPanel diff --git a/apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.test.ts b/apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.test.ts new file mode 100644 index 00000000000..1b4f0646410 --- /dev/null +++ b/apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.test.ts @@ -0,0 +1,149 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { + buildClickhouseRewritePrompt, + detectLogSource, + looksLikeLegacyLogsQuery, + rewriteLogsSqlWithAI, + stripSqlCodeFences, +} from './logs-sql-rewrite' + +describe('buildClickhouseRewritePrompt', () => { + it('includes the query, the schema, and a reply-with-only-SQL instruction', () => { + const prompt = buildClickhouseRewritePrompt('select count(*) from edge_logs') + expect(prompt).toContain('select count(*) from edge_logs') + expect(prompt).toContain('log_attributes') + expect(prompt).toContain("source = 'edge_logs'") + expect(prompt.toLowerCase()).toContain('reply with only') + }) + + it('spells out the FROM-to-logs conversion and shows a worked example', () => { + const prompt = buildClickhouseRewritePrompt('select 1 from postgres_logs') + expect(prompt).toContain("from logs where source = 'postgres_logs'") + expect(prompt.toLowerCase()).toContain('remove every') + expect(prompt).toContain('cross join unnest') + expect(prompt).toContain('BigQuery:') + expect(prompt).toContain('ClickHouse:') + expect(prompt).toContain("log_attributes['parsed.error_severity']") + }) + + it('lists the real log_attributes keys when provided and demands exact paths', () => { + const prompt = buildClickhouseRewritePrompt('select 1 from edge_logs', [ + 'request.headers.x_real_ip', + 'request.cf.country', + ]) + expect(prompt).toContain("log_attributes['request.headers.x_real_ip']") + expect(prompt).toContain("log_attributes['request.cf.country']") + expect(prompt.toLowerCase()).toContain('exact') + }) + + it('omits the keys section when none are provided', () => { + const prompt = buildClickhouseRewritePrompt('select 1 from edge_logs') + expect(prompt).not.toContain('actual log_attributes keys present') + }) +}) + +describe('detectLogSource', () => { + it('reads an explicit source filter', () => { + expect(detectLogSource("select 1 from logs where source = 'edge_logs'")).toBe('edge_logs') + }) + + it('falls back to the legacy FROM table name', () => { + expect(detectLogSource('select 1 from edge_logs as t')).toBe('edge_logs') + }) + + it('maps pg_cron_logs to postgres_logs from either the FROM table or the source filter', () => { + expect(detectLogSource('select 1 from pg_cron_logs')).toBe('postgres_logs') + expect(detectLogSource("select 1 from logs where source = 'pg_cron_logs'")).toBe( + 'postgres_logs' + ) + }) + + it('returns undefined for the bare logs table with no source', () => { + expect(detectLogSource('select 1 from logs limit 5')).toBeUndefined() + }) + + it('returns undefined when nothing matches', () => { + expect(detectLogSource('select 1')).toBeUndefined() + }) +}) + +describe('looksLikeLegacyLogsQuery', () => { + it('flags per-service FROM tables', () => { + expect(looksLikeLegacyLogsQuery('select 1 from edge_logs')).toBe(true) + }) + + it('flags unnest joins and the cast-timestamp idiom', () => { + expect(looksLikeLegacyLogsQuery('select 1 from logs cross join unnest(metadata) as m')).toBe( + true + ) + expect(looksLikeLegacyLogsQuery('select cast(timestamp as datetime) from logs')).toBe(true) + }) + + it('does not flag a ClickHouse query against the logs table', () => { + expect( + looksLikeLegacyLogsQuery("select timestamp from logs where source = 'edge_logs' limit 5") + ).toBe(false) + }) +}) + +describe('stripSqlCodeFences', () => { + it('removes a ```sql fenced block', () => { + expect(stripSqlCodeFences('```sql\nselect 1 from logs\n```')).toBe('select 1 from logs') + }) + + it('removes a plain ``` fenced block', () => { + expect(stripSqlCodeFences('```\nselect 1\n```')).toBe('select 1') + }) + + it('leaves unfenced SQL untouched (trimmed)', () => { + expect(stripSqlCodeFences(' select 1 from logs ')).toBe('select 1 from logs') + }) + + it('extracts the fenced block when wrapped in prose', () => { + expect(stripSqlCodeFences('Here is the rewrite:\n```sql\nselect 1 from logs\n```')).toBe( + 'select 1 from logs' + ) + }) +}) + +describe('rewriteLogsSqlWithAI', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('posts to the completion endpoint and returns the cleaned query', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => '```sql\nselect 1 from logs\n```', + }) + vi.stubGlobal('fetch', fetchMock) + + const result = await rewriteLogsSqlWithAI({ + sql: 'select 1 from edge_logs', + projectRef: 'abc', + }) + + expect(result).toBe('select 1 from logs') + const [url, init] = fetchMock.mock.calls[0] + expect(url).toContain('/api/ai/code/complete') + const body = JSON.parse(init.body) + expect(body.dialect).toBe('clickhouse') + expect(body.completionMetadata.selection).toBe('select 1 from edge_logs') + expect(body.completionMetadata.prompt.toLowerCase()).toContain('reply with only') + }) + + it('throws when the request fails', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, text: async () => 'boom' })) + await expect(rewriteLogsSqlWithAI({ sql: 'select 1', projectRef: 'abc' })).rejects.toThrow( + 'boom' + ) + }) + + it('throws when the model returns an empty query', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => ' ' })) + await expect(rewriteLogsSqlWithAI({ sql: 'select 1', projectRef: 'abc' })).rejects.toThrow( + 'empty' + ) + }) +}) diff --git a/apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.ts b/apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.ts new file mode 100644 index 00000000000..ca532f204a4 --- /dev/null +++ b/apps/studio/components/interfaces/Settings/Logs/logs-sql-rewrite.ts @@ -0,0 +1,142 @@ +import { BASE_PATH } from '@/lib/constants' + +export const LOGS_SCHEMA_REFERENCE = `The logs table (ClickHouse) has these columns: +- id (String) +- timestamp (DateTime64, UTC) formatted like 2026-06-22T09:34:06.215000 (ISO 8601, microsecond precision, no trailing Z) +- event_message (String): the raw log line +- severity_text (String): log level when present +- source (String): the service the log belongs to. Always filter by it, e.g. where source = 'edge_logs'. +- log_attributes (Map(String, String)): structured per-source fields, read as log_attributes['key']. Values are strings, so wrap numeric ones in toInt32OrZero(...) for comparisons. + +Sources and their common log_attributes keys: +- edge_logs: request.method, request.path, request.search, response.status_code, identifier +- postgres_logs: parsed.error_severity, parsed.detail, parsed.hint, parsed.query, identifier +- pg_cron logs live under source = 'postgres_logs' (parsed.error_severity, parsed.query) +- auth_logs: level, status, path, msg, error +- function_edge_logs: response.status_code, request.method, request.pathname, function_id, execution_id, execution_time_ms +- function_logs: event_type, function_id, execution_id, level +- storage_logs, realtime_logs, postgrest_logs, supavisor_logs, pgbouncer_logs: mostly id, timestamp, event_message, with extra fields in log_attributes + +Rules: always filter by source; the editor applies the selected time range so a timestamp filter is usually unnecessary; the old BigQuery unnest joins become log_attributes['key'] lookups (drop the metadata root).` + +function renderAvailableKeys(availableKeys?: string[]): string { + if (!availableKeys || availableKeys.length === 0) return '' + const list = availableKeys.map((key) => `- log_attributes['${key}']`).join('\n') + return `\nThe actual log_attributes keys present for this source are listed below. Use these EXACT keys — do not invent, shorten, or drop any dotted prefix. If a BigQuery field maps to one of these (e.g. request.headers.x_real_ip, request.cf.country), use the full key shown here: +${list}\n` +} + +export function buildClickhouseRewritePrompt(sql: string, availableKeys?: string[]): string { + return `${LOGS_SCHEMA_REFERENCE} +${renderAvailableKeys(availableKeys)} +Convert the BigQuery logs query below to ClickHouse SQL for the logs table. There are no per-service tables and no unnest joins in ClickHouse. Follow these rules exactly: + +1. Replace the FROM table with the single logs table and filter by source. The old table name is the source value: "from postgres_logs as t" becomes "from logs where source = 'postgres_logs'". This is required, never select from a table like postgres_logs or edge_logs. +2. Remove every join that unnests metadata or its structs. This includes "cross join unnest(...)" and "left join unnest(...) on true". +3. Replace any column that came from an unnest alias with a log_attributes lookup. A field off unnest(metadata) becomes log_attributes['field']; a field off a nested struct like unnest(m.parsed) becomes log_attributes['parsed.field'] (keep the struct name as a dotted prefix, drop the metadata root and every alias). When the actual keys are listed above, match against them and use the full dotted key exactly. +4. Wrap numeric fields in toInt32OrZero(...) before comparing or aggregating them. +5. Replace BigQuery functions with ClickHouse equivalents: regexp_contains(x, 'p') becomes match(x, 'p'), or x ILIKE '%p%' for a plain substring. Replace cast(timestamp as datetime) with timestamp. Use count() instead of count(*). +6. Preserve the original select list, filters, group by, order by, and limit intent. + +Example. +BigQuery: +select count(t.timestamp) as count, p.error_severity +from postgres_logs as t +cross join unnest(metadata) as m +cross join unnest(m.parsed) as p +where p.error_severity in ('ERROR', 'FATAL', 'PANIC') +group by p.error_severity +order by count desc +limit 100 + +ClickHouse: +select count() as count, log_attributes['parsed.error_severity'] as error_severity +from logs +where source = 'postgres_logs' + and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC') +group by log_attributes['parsed.error_severity'] +order by count desc +limit 100 + +Reply with ONLY the rewritten SQL query: no explanation, no comments, and no markdown code fences. + +${sql}` +} + +export function stripSqlCodeFences(text: string): string { + const trimmed = text.trim() + const fenced = trimmed.match(/```(?:sql)?\s*\n?([\s\S]*?)\n?```/i) + return (fenced ? fenced[1] : trimmed).trim() +} + +const SOURCE_ALIASES: Record = { + pg_cron_logs: 'postgres_logs', +} + +export function detectLogSource(sql: string): string | undefined { + const bySource = sql.match(/source\s*=\s*'([^']+)'/i) + if (bySource) { + const source = bySource[1].toLowerCase() + return SOURCE_ALIASES[source] ?? source + } + const byFrom = sql.match(/\bfrom\s+([a-z_][a-z0-9_]*)/i) + if (byFrom) { + const table = byFrom[1].toLowerCase() + if (table === 'logs') return undefined + return SOURCE_ALIASES[table] ?? table + } + return undefined +} + +export function looksLikeLegacyLogsQuery(sql: string): boolean { + const lower = sql.toLowerCase() + if (/\bunnest\s*\(/.test(lower)) return true + if (/cast\s*\(\s*timestamp\s+as\s+datetime\s*\)/.test(lower)) return true + const byFrom = lower.match(/\bfrom\s+([a-z_][a-z0-9_]*)/) + return byFrom ? byFrom[1] !== 'logs' : false +} + +export interface RewriteLogsSqlArgs { + sql: string + projectRef: string + connectionString?: string | null + orgSlug?: string + authorizationHeader?: string | null + availableKeys?: string[] +} + +export async function rewriteLogsSqlWithAI(args: RewriteLogsSqlArgs) { + const { sql, projectRef, connectionString, orgSlug, authorizationHeader, availableKeys } = args + + const response = await fetch(`${BASE_PATH}/api/ai/code/complete`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(authorizationHeader ? { Authorization: authorizationHeader } : {}), + }, + body: JSON.stringify({ + projectRef, + connectionString, + language: 'sql', + dialect: 'clickhouse', + orgSlug, + completionMetadata: { + textBeforeCursor: '', + textAfterCursor: '', + language: 'pgsql', + prompt: buildClickhouseRewritePrompt(sql, availableKeys), + selection: sql, + }, + }), + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(errorText || 'Failed to rewrite the query') + } + + const raw = await response.json() + const rewritten = stripSqlCodeFences(typeof raw === 'string' ? raw : String(raw)) + if (!rewritten) throw new Error('The assistant returned an empty query') + return rewritten +} diff --git a/apps/studio/components/ui/Logs/LogsExplorerHeader.tsx b/apps/studio/components/ui/Logs/LogsExplorerHeader.tsx index 1a2f195b8b0..26e7b376dc1 100644 --- a/apps/studio/components/ui/Logs/LogsExplorerHeader.tsx +++ b/apps/studio/components/ui/Logs/LogsExplorerHeader.tsx @@ -1,3 +1,4 @@ +import { useFlag, useParams } from 'common' import { BookOpen, Check, ChevronsUpDown, Copy, ExternalLink, List, X } from 'lucide-react' import Link from 'next/link' import { useState } from 'react' @@ -23,7 +24,12 @@ import { import { DocsButton } from '../DocsButton' import { LOGS_EXPLORER_DOCS_URL } from '@/components/interfaces/Settings/Logs/Logs.constants' +import { + otelFieldsFromKeys, + toOtelFieldSchemas, +} from '@/components/interfaces/Settings/Logs/Logs.fieldReference' import Table from '@/components/to-be-cleaned/Table' +import { useOtelLogKeysQuery } from '@/data/logs/otel-log-keys-query' import { DOCS_URL } from '@/lib/constants' export interface LogsExplorerHeaderProps { @@ -33,7 +39,16 @@ export interface LogsExplorerHeaderProps { const LogsExplorerHeader = ({ subtitle }: LogsExplorerHeaderProps) => { const [showReference, setShowReference] = useState(false) const [open, setOpen] = useState(false) - const [selectedSchema, setSelectedSchema] = useState(logConstants.schemas[0]) + const useOtel = useFlag('otelLegacyLogs') + const schemas = useOtel ? toOtelFieldSchemas(logConstants.schemas) : logConstants.schemas + const [selectedRef, setSelectedRef] = useState(schemas[0]?.reference) + const selectedSchema = schemas.find((s) => s.reference === selectedRef) ?? schemas[0] + + const { ref: projectRef } = useParams() + const { data: discoveredKeys, isPending: isLoadingKeys } = useOtelLogKeysQuery( + { projectRef, source: selectedRef }, + { enabled: useOtel && showReference } + ) return (
@@ -79,24 +94,33 @@ const LogsExplorerHeader = ({ subtitle }: LogsExplorerHeaderProps) => { >
-

- The following table shows all the available paths that can be queried from each - respective source. Do note that to access nested keys, you would need to perform the - necessary{' '} - - unnesting joins - - -

+ {useOtel ? ( +

+ The following table shows the fields available on each source. Nested fields live + in the log_attributes map and are read with{' '} + log_attributes['key'] — no unnesting joins + needed. +

+ ) : ( +

+ The following table shows all the available paths that can be queried from each + respective source. Do note that to access nested keys, you would need to perform + the necessary{' '} + + unnesting joins + + +

+ )}
@@ -120,19 +144,21 @@ const LogsExplorerHeader = ({ subtitle }: LogsExplorerHeaderProps) => { No source found. - {logConstants.schemas.map((schema) => ( + {schemas.map((schema) => ( { - setSelectedSchema(schema) + setSelectedRef(schema.reference) setOpen(false) }} > {schema.name} @@ -143,19 +169,26 @@ const LogsExplorerHeader = ({ subtitle }: LogsExplorerHeaderProps) => { - - Path - , - - Type - , - ]} - body={selectedSchema.fields.map((field) => ( - - ))} - /> + {useOtel && isLoadingKeys ? ( +

Loading fields…

+ ) : ( +
+ Path + , + + Type + , + ]} + body={(() => { + const fields = useOtel + ? otelFieldsFromKeys(discoveredKeys ?? []) + : selectedSchema.fields + return fields.map((field) => ) + })()} + /> + )} diff --git a/apps/studio/data/logs/otel-log-keys-query.ts b/apps/studio/data/logs/otel-log-keys-query.ts new file mode 100644 index 00000000000..c4918af0af8 --- /dev/null +++ b/apps/studio/data/logs/otel-log-keys-query.ts @@ -0,0 +1,45 @@ +import { useQuery } from '@tanstack/react-query' + +import { executeAnalyticsSql } from './execute-analytics-sql' +import { logsAllEndpointUrl } from './logs-endpoint' +import { analyticsLiteral, safeSql } from './safe-analytics-sql' + +const LOOKBACK_HOURS = 24 * 7 + +export async function fetchOtelLogKeys({ + projectRef, + source, + signal, +}: { + projectRef: string + source: string + signal?: AbortSignal +}): Promise { + const end = new Date() + const start = new Date(end.getTime() - LOOKBACK_HOURS * 60 * 60 * 1000) + const sql = safeSql`SELECT arrayJoin(mapKeys(log_attributes)) AS key, count() AS n FROM logs WHERE source = ${analyticsLiteral(source)} GROUP BY key ORDER BY n DESC LIMIT 500` + const data = await executeAnalyticsSql({ + projectRef, + endpoint: logsAllEndpointUrl(true), + sql, + iso_timestamp_start: start.toISOString(), + iso_timestamp_end: end.toISOString(), + method: 'post', + signal, + }) + const rows = (data?.result ?? []) as { key: string }[] + return rows.map((r) => r.key).filter(Boolean) +} + +export function useOtelLogKeysQuery( + { projectRef, source }: { projectRef?: string; source?: string }, + { enabled = true }: { enabled?: boolean } = {} +) { + return useQuery({ + queryKey: ['projects', projectRef, 'otel-log-keys', source], + queryFn: ({ signal }) => + fetchOtelLogKeys({ projectRef: projectRef ?? '', source: source ?? '', signal }), + enabled: enabled && Boolean(projectRef) && Boolean(source), + staleTime: 5 * 60 * 1000, + }) +} diff --git a/apps/studio/hooks/analytics/useLogsUrlState.ts b/apps/studio/hooks/analytics/useLogsUrlState.ts index ea3b670faad..939efd3891b 100644 --- a/apps/studio/hooks/analytics/useLogsUrlState.ts +++ b/apps/studio/hooks/analytics/useLogsUrlState.ts @@ -1,4 +1,5 @@ import { createParser, useQueryState } from 'nuqs' +import { useCallback } from 'react' import { Filters } from '@/components/interfaces/Settings/Logs/Logs.types' @@ -67,34 +68,55 @@ export function useLogsUrlState() { const selectedLogId = selectedLogIdValue || null const filters = filtersValue || defaultState.filters - const setSearch = (value: string) => { - setSearchValue(value || defaultState.search) - } + const setSearch = useCallback( + (value: string) => { + setSearchValue(value || defaultState.search) + }, + [setSearchValue] + ) - const setTimeRange = (start: string, end: string) => { - setTimestampStartValue(start || defaultState.timestampStart) - setTimestampEndValue(end || defaultState.timestampEnd) - } + const setTimeRange = useCallback( + (start: string, end: string) => { + setTimestampStartValue(start || defaultState.timestampStart) + setTimestampEndValue(end || defaultState.timestampEnd) + }, + [setTimestampStartValue, setTimestampEndValue] + ) - const setSelectedLogId = (value: string | null) => { - setSelectedLogIdValue(value || '') - } + const setSelectedLogId = useCallback( + (value: string | null) => { + setSelectedLogIdValue(value || '') + }, + [setSelectedLogIdValue] + ) - const setFilters = (value: Filters) => { - setFiltersValue(value || defaultState.filters) - } + const setFilters = useCallback( + (value: Filters) => { + setFiltersValue(value || defaultState.filters) + }, + [setFiltersValue] + ) - const updateFilter = (key: string, value: any) => { - setFilters({ ...filters, [key]: value }) - } + const updateFilter = useCallback( + (key: string, value: any) => { + setFilters({ ...filters, [key]: value }) + }, + [filters, setFilters] + ) - const reset = () => { + const reset = useCallback(() => { setSearch(defaultState.search) setTimestampStartValue(defaultState.timestampStart) setTimestampEndValue(defaultState.timestampEnd) setSelectedLogIdValue('') setFiltersValue(defaultState.filters) - } + }, [ + setSearch, + setTimestampStartValue, + setTimestampEndValue, + setSelectedLogIdValue, + setFiltersValue, + ]) return { // State diff --git a/apps/studio/lib/ai/model.utils.ts b/apps/studio/lib/ai/model.utils.ts index 905c650dff3..0174c2904ec 100644 --- a/apps/studio/lib/ai/model.utils.ts +++ b/apps/studio/lib/ai/model.utils.ts @@ -51,6 +51,11 @@ export const DEFAULT_COMPLETION_MODEL = openaiModelEntry({ reasoningEffort: 'none', }) +export const LOGS_REWRITE_MODEL = openaiModelEntry({ + id: 'gpt-5.4-nano', + reasoningEffort: 'low', +}) + // Single source of truth for all Assistant chat model variants and their reasoning levels. // Models with requiresAdvanceModelEntitlement false are available to all users; true requires the assistant.advance_model entitlement. export const ASSISTANT_MODELS = [ diff --git a/apps/studio/lib/ai/prompts.ts b/apps/studio/lib/ai/prompts.ts index 81c6f0435f6..3a74b8b6dcc 100644 --- a/apps/studio/lib/ai/prompts.ts +++ b/apps/studio/lib/ai/prompts.ts @@ -786,6 +786,15 @@ export const SQL_COMPLETION_INSTRUCTIONS = ` Do not quote identifiers unless they actually require it (uppercase letters, reserved words, or special characters). Plain lowercase identifiers should not be quoted. ` +export const CLICKHOUSE_LOGS_COMPLETION_INSTRUCTIONS = ` +# Supabase logs SQL (ClickHouse) +You are writing SQL for Supabase logs, which run on a ClickHouse-backed engine. This is NOT Postgres and NOT BigQuery. Output valid ClickHouse SQL only. +- All logs are in a single table named \`logs\`, keyed by a \`source\` column. There are no per-service tables (no \`edge_logs\`, \`postgres_logs\`, and so on) and no \`unnest\` joins. +- Per-source fields live in the \`log_attributes\` Map(String, String), read as \`log_attributes['key']\`. Map values are strings, so wrap numeric ones in \`toInt32OrZero(...)\`. +- Use ClickHouse functions, not Postgres or BigQuery ones. Use \`match(col, 'regex')\` or \`col ILIKE '%text%'\` instead of \`regexp_contains\`, \`count()\` instead of \`count(*)\`, and select the \`timestamp\` column directly instead of \`cast(timestamp as datetime)\`. +- Do not quote identifiers with double quotes and do not append a trailing semicolon. +` + export const LIMITATIONS_PROMPT = ` # Limitations - You are to only answer Supabase, database, or edge function related questions. All other questions should be declined with a polite message. diff --git a/apps/studio/pages/api/ai/code/complete.ts b/apps/studio/pages/api/ai/code/complete.ts index 46ab01332db..1c2d8218662 100644 --- a/apps/studio/pages/api/ai/code/complete.ts +++ b/apps/studio/pages/api/ai/code/complete.ts @@ -9,8 +9,9 @@ import { executeSql } from '@/data/sql/execute-sql-mutation' import { AiOptInLevel } from '@/hooks/misc/useOrgOptedIntoAi' import { getOrgAIDetails } from '@/lib/ai/ai-details' import { getModel } from '@/lib/ai/model' -import { DEFAULT_COMPLETION_MODEL } from '@/lib/ai/model.utils' +import { DEFAULT_COMPLETION_MODEL, LOGS_REWRITE_MODEL } from '@/lib/ai/model.utils' import { + CLICKHOUSE_LOGS_COMPLETION_INSTRUCTIONS, COMPLETION_PROMPT, EDGE_FUNCTION_PROMPT, PG_BEST_PRACTICES, @@ -128,6 +129,7 @@ const requestBodySchema = z.object({ connectionString: z.string().nullish(), orgSlug: z.string().optional(), language: z.string().optional(), + dialect: z.enum(['postgres', 'clickhouse']).optional(), }) async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -148,8 +150,9 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { return res.status(400).json({ error: 'Invalid request body', issues: parseError.issues }) } - const { completionMetadata, projectRef, connectionString, orgSlug, language } = data + const { completionMetadata, projectRef, connectionString, orgSlug, language, dialect } = data const { textBeforeCursor, textAfterCursor, prompt, selection } = completionMetadata + const isClickhouse = dialect === 'clickhouse' const authorization = req.headers.authorization let aiOptInLevel: AiOptInLevel = IS_PLATFORM ? 'disabled' : 'schema' @@ -168,7 +171,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { systemProviderOptions, } = await getModel({ provider: 'openai', - modelEntry: DEFAULT_COMPLETION_MODEL, + modelEntry: isClickhouse ? LOGS_REWRITE_MODEL : DEFAULT_COMPLETION_MODEL, }) if (modelError) { @@ -180,7 +183,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { ...(authorization && { Authorization: authorization }), } - const includeSchema = aiOptInLevel !== 'disabled' + const includeSchema = !isClickhouse && aiOptInLevel !== 'disabled' // Fetch schema list first so we can determine which schemas to load DDL for. // These are best-effort — if they fail, we proceed without DDL context. @@ -227,29 +230,36 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { otherSchemas: schemas.filter((s) => !fetchedSchemaSet.has(s.name)).map((s) => s.name), } - // Important: do not use dynamic content in the system prompt or Bedrock will not cache it - const system = source` - ${COMPLETION_PROMPT} - ${language === 'sql' ? SQL_COMPLETION_INSTRUCTIONS : ''} - ${language === 'sql' ? PG_BEST_PRACTICES : EDGE_FUNCTION_PROMPT} - ${SECURITY_PROMPT} - ` + const system = isClickhouse + ? source` + You rewrite SQL queries to ClickHouse SQL for the Supabase logs table. + Output only the rewritten SQL query: no explanation, no markdown, and no code fences. + ${CLICKHOUSE_LOGS_COMPLETION_INSTRUCTIONS} + ${SECURITY_PROMPT} + ` + : source` + ${COMPLETION_PROMPT} + ${language === 'sql' ? `${SQL_COMPLETION_INSTRUCTIONS}\n${PG_BEST_PRACTICES}` : EDGE_FUNCTION_PROMPT} + ${SECURITY_PROMPT} + ` - const userMessage = source` - ## Database Schema + const userMessage = isClickhouse + ? prompt + : source` + ## Database Schema - ${buildDatabaseSchemaSection({ includeSchema, schemaListResult, schemaDDLResult })} + ${buildDatabaseSchemaSection({ includeSchema, schemaListResult, schemaDDLResult })} - ## Code + ## Code - \`\`\`${language ?? ''} - ${textBeforeCursor}${selection}${textAfterCursor} - \`\`\` + \`\`\`${language ?? ''} + ${textBeforeCursor}${selection}${textAfterCursor} + \`\`\` - ## Instruction + ## Instruction - ${prompt} - ` + ${prompt} + ` // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions` // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88 diff --git a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx b/apps/studio/pages/project/[ref]/logs/explorer/index.tsx index ddaca7aa3ae..fbdabf6e159 100644 --- a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx +++ b/apps/studio/pages/project/[ref]/logs/explorer/index.tsx @@ -6,8 +6,13 @@ import type { editor } from 'monaco-editor' import { useRouter } from 'next/router' import { useEffect, useMemo, useRef, useState } from 'react' import { toast } from 'sonner' -import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from 'ui' +import { Button, ResizableHandle, ResizablePanel, ResizablePanelGroup } from 'ui' +import { + detectLogSource, + looksLikeLegacyLogsQuery, + rewriteLogsSqlWithAI, +} from '@/components/interfaces/Settings/Logs/logs-sql-rewrite' import { EXPLORER_DATEPICKER_HELPERS, getDefaultHelper, @@ -25,12 +30,14 @@ import { buildLogQueryParams, resolveLogDateRange, } from '@/components/interfaces/Settings/Logs/logsDateRange' -import LogsQueryPanel from '@/components/interfaces/Settings/Logs/LogsQueryPanel' +import { LogsExplorerOtelBanner } from '@/components/interfaces/Settings/Logs/LogsExplorerOtelBanner' +import { LogsQueryPanel } from '@/components/interfaces/Settings/Logs/LogsQueryPanel' import { LogTable } from '@/components/interfaces/Settings/Logs/LogTable' import UpgradePrompt from '@/components/interfaces/Settings/Logs/UpgradePrompt' import DefaultLayout from '@/components/layouts/DefaultLayout' import LogsLayout from '@/components/layouts/LogsLayout/LogsLayout' import { CodeEditor } from '@/components/ui/CodeEditor/CodeEditor' +import { DiffEditor } from '@/components/ui/DiffEditor' import LoadingOpacity from '@/components/ui/LoadingOpacity' import ShimmerLine from '@/components/ui/ShimmerLine' import { useContentQuery } from '@/data/content/content-query' @@ -38,17 +45,23 @@ import { UpsertContentPayload, useContentUpsertMutation, } from '@/data/content/content-upsert-mutation' +import { constructHeaders } from '@/data/fetchers' +import { fetchOtelLogKeys } from '@/data/logs/otel-log-keys-query' import useLogsQuery from '@/hooks/analytics/useLogsQuery' import { useLogsUrlState } from '@/hooks/analytics/useLogsUrlState' import { useCustomContent } from '@/hooks/custom-content/useCustomContent' import { useCheckEntitlements } from '@/hooks/misc/useCheckEntitlements' import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled' +import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganization' +import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject' import { useUpgradePrompt } from '@/hooks/misc/useUpgradePrompt' import { uuidv4 } from '@/lib/helpers' import { useProfile } from '@/lib/profile' import { useTrack } from '@/lib/telemetry/track' import type { LogSqlSnippets, NextPageWithLayout } from '@/types' +type SaveQueryFormValues = { name: string; description?: string } + const LOCAL_PLACEHOLDER_QUERY = 'select\n timestamp, event_message, metadata\n from edge_logs limit 5' @@ -58,11 +71,16 @@ const PLATFORM_PLACEHOLDER_QUERY = const OTEL_PLACEHOLDER_QUERY = "select\n timestamp,\n event_message,\n log_attributes\nfrom logs\nwhere source = 'edge_logs'\norder by timestamp desc\nlimit 5" +const otelSourceQuery = (source: string) => + `select\n timestamp,\n event_message,\n log_attributes\nfrom logs\nwhere source = '${source}'\norder by timestamp desc\nlimit 100` + export const LogsExplorerPage: NextPageWithLayout = () => { useEditorHints() const monaco = useMonaco() const router = useRouter() const { profile } = useProfile() + const { data: project } = useSelectedProjectQuery() + const { data: organization } = useSelectedOrganizationQuery() const { ref, q, queryId } = useParams() const track = useTrack() const projectRef = ref as string @@ -106,6 +124,15 @@ export const LogsExplorerPage: NextPageWithLayout = () => { const [saveModalOpen, setSaveModalOpen] = useState(false) const [warnings, setWarnings] = useState([]) const [selectedLog, setSelectedLog] = useState(null) + const [rewriteProposal, setRewriteProposal] = useState<{ + original: string + modified: string + } | null>(null) + const [isRewriting, setIsRewriting] = useState(false) + const [rewriteBannerDismissed, setRewriteBannerDismissed] = useLocalStorage( + `project-${projectRef}-logs-rewrite-banner-dismissed`, + false + ) const [recentLogs, setRecentLogs] = useLocalStorage( `project-content-${projectRef}-recent-log-sql`, @@ -150,6 +177,8 @@ export const LogsExplorerPage: NextPageWithLayout = () => { const results = logData const isLoading = logsLoading + const showRewriteCTA = useOtelEndpoint && looksLikeLegacyLogsQuery(editorValue) + const { mutateAsync: upsertContent, isPending: isUpsertingContent } = useContentUpsertMutation({ onError: (e) => { const error = e as { message: string } @@ -202,6 +231,52 @@ export const LogsExplorerPage: NextPageWithLayout = () => { addRecentLogSqlSnippet({ sql: template.searchString }) } + const handleRewrite = async () => { + const currentSql = editorRef.current?.getValue() ?? editorValue + if (!currentSql.trim()) { + toast.info('Write a query to rewrite first') + return + } + setIsRewriting(true) + try { + const headerData = await constructHeaders() + const source = detectLogSource(currentSql) + const availableKeys = source + ? await fetchOtelLogKeys({ projectRef, source }).catch(() => undefined) + : undefined + const rewritten = await rewriteLogsSqlWithAI({ + sql: currentSql, + projectRef, + connectionString: project?.connectionString, + orgSlug: organization?.slug, + authorizationHeader: headerData.get('Authorization'), + availableKeys, + }) + // The editor may have changed while awaiting key discovery and the AI call; + // don't offer a proposal that would clobber intervening edits. + const latestSql = editorRef.current?.getValue() ?? editorValue + if (latestSql !== currentSql) { + toast.info('The query changed while rewriting. Please try again.') + return + } + setRewriteProposal({ original: currentSql, modified: rewritten }) + } catch (error) { + toast.error(`Couldn't rewrite the query: ${(error as Error).message}`) + } finally { + setIsRewriting(false) + } + } + + const acceptRewrite = () => { + if (!rewriteProposal) return + editorRef.current?.setValue(rewriteProposal.modified) + setEditorValue(rewriteProposal.modified) + setRewriteProposal(null) + toast.success('Applied the ClickHouse rewrite') + } + + const discardRewrite = () => setRewriteProposal(null) + const handleRun = (value?: string | React.MouseEvent) => { track('log_explorer_query_run_button_clicked', { is_saved_query: !!queryId }) @@ -232,12 +307,20 @@ export const LogsExplorerPage: NextPageWithLayout = () => { if (editorRef.current && monaco) { const editorModel = editorRef.current?.getModel() const currentValue = editorRef.current.getValue() - const index = currentValue.indexOf('from') - const updatedValue = - index < 0 - ? `${currentValue}${source}` - : `${currentValue.substring(0, index + 4)} ${source} ${currentValue.substring(index + 5)}` + let updatedValue: string + if (useOtelEndpoint) { + const sourceFilter = /source\s*=\s*'[^']*'/i + updatedValue = sourceFilter.test(currentValue) + ? currentValue.replace(sourceFilter, `source = '${source}'`) + : otelSourceQuery(source) + } else { + const index = currentValue.indexOf('from') + updatedValue = + index < 0 + ? `${currentValue}${source}` + : `${currentValue.substring(0, index + 4)} ${source} ${currentValue.substring(index + 5)}` + } editorRef.current.pushUndoStop() editorRef.current.executeEdits(`insert-identifier`, [ @@ -251,8 +334,6 @@ export const LogsExplorerPage: NextPageWithLayout = () => { } } - type SaveQueryFormValues = { name: string; description?: string } - const handleCreateQuery = async (values: SaveQueryFormValues) => { if (!projectRef) return console.error('Project ref is required') if (!profile) return console.error('Profile is required') @@ -333,20 +414,11 @@ export const LogsExplorerPage: NextPageWithLayout = () => { } else if (q) { setEditorValue(q) setSearch(q) + } else if (!queryId) { + setEditorValue(PLACEHOLDER_QUERY) + editorRef.current?.setValue(PLACEHOLDER_QUERY) } - }, [q, search, setSearch]) - - useEffect(() => { - if (!useOtelEndpoint || q || search || queryId) return - if (editorValue === OTEL_PLACEHOLDER_QUERY) return - const isUntouchedDefault = - editorValue === LOCAL_PLACEHOLDER_QUERY || - editorValue === PLATFORM_PLACEHOLDER_QUERY || - editorValue === logsDefaultQuery - if (!isUntouchedDefault) return - setEditorValue(OTEL_PLACEHOLDER_QUERY) - editorRef.current?.setValue(OTEL_PLACEHOLDER_QUERY) - }, [useOtelEndpoint, q, search, queryId, editorValue, logsDefaultQuery]) + }, [q, search, queryId, setSearch, PLACEHOLDER_QUERY]) useEffect(() => { // prevents overwriting when the user selects a helper. @@ -402,18 +474,55 @@ export const LogsExplorerPage: NextPageWithLayout = () => { templates={allTemplates.filter((template) => template.mode === 'custom')} onSelectTemplate={onSelectTemplate} warnings={warnings} + showRewriteAction={showRewriteCTA && rewriteBannerDismissed} + isRewriting={isRewriting} + onRewrite={handleRewrite} /> + {showRewriteCTA && !rewriteBannerDismissed && ( + setRewriteBannerDismissed(true)} + /> + )} - setEditorValue(v || '')} - actions={{ runQuery: { enabled: true, callback: handleRun } }} - /> +
+ setEditorValue(v || '')} + actions={{ runQuery: { enabled: true, callback: handleRun } }} + /> + {rewriteProposal && ( +
+
+ + Review the ClickHouse SQL rewrite before accepting it + +
+ + +
+
+
+ +
+
+ )} +
diff --git a/apps/studio/tests/features/logs/LogsQueryPanel.test.tsx b/apps/studio/tests/features/logs/LogsQueryPanel.test.tsx index 124257c52c6..e6d2a6556dd 100644 --- a/apps/studio/tests/features/logs/LogsQueryPanel.test.tsx +++ b/apps/studio/tests/features/logs/LogsQueryPanel.test.tsx @@ -1,7 +1,7 @@ import { screen } from '@testing-library/react' import { expect, test } from 'vitest' -import LogsQueryPanel from '@/components/interfaces/Settings/Logs/LogsQueryPanel' +import { LogsQueryPanel } from '@/components/interfaces/Settings/Logs/LogsQueryPanel' import { render } from '@/tests/helpers' test('run and clear', async () => {