From b9f76d95f796440150668dee1ec532f4be5687d1 Mon Sep 17 00:00:00 2001 From: Jordi Enric <37541088+jordienr@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:57:36 +0200 Subject: [PATCH] feat(studio): load ClickHouse query templates when OTEL logs are on (#47390) --- .../Settings/Logs/Logs.constants.ts | 151 ++++++++++++++++++ .../Settings/Logs/Logs.templates.test.ts | 59 +++++++ .../project/[ref]/logs/explorer/index.tsx | 12 +- .../project/[ref]/logs/explorer/templates.tsx | 10 +- 4 files changed, 222 insertions(+), 10 deletions(-) create mode 100644 apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts diff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts index ec5fc44dbaf..ab6671faf79 100644 --- a/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts +++ b/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts @@ -286,6 +286,157 @@ limit 100; }, ] +// ClickHouse rewrites of the custom BigQuery templates above, used when the OTEL +// logs engine is on. Keyed by template label. Simple-mode templates are plain +// event_message searches that work unchanged, so they have no entry here. +const OTEL_TEMPLATE_SEARCH_STRINGS: Record = { + 'Commits By User': `select + log_attributes['parsed.user_name'] as user_name, + count() as count +from logs +where source = 'postgres_logs' + and event_message like '%COMMIT%' +group by user_name +order by count desc +limit 100`, + 'Metadata IP': `select + timestamp, + log_attributes['request.headers.x_real_ip'] as x_real_ip +from logs +where source = 'edge_logs' + and log_attributes['request.headers.x_real_ip'] != '' +order by timestamp desc +limit 100`, + 'Requests by Geography': `select + log_attributes['request.cf.country'] as country, + count() as count +from logs +where source = 'edge_logs' +group by country +order by count desc +limit 100`, + 'Slow Response Time': `select + timestamp, + event_message, + toInt32OrZero(log_attributes['response.origin_time']) as origin_time +from logs +where source = 'edge_logs' + and toInt32OrZero(log_attributes['response.origin_time']) > 1000 +order by timestamp desc +limit 100`, + '500 Request Codes': `select + timestamp, + event_message, + toInt32OrZero(log_attributes['response.status_code']) as status_code +from logs +where source = 'edge_logs' + and toInt32OrZero(log_attributes['response.status_code']) >= 500 +order by timestamp desc +limit 100`, + 'Top Paths': `select + log_attributes['request.path'] as path, + log_attributes['request.search'] as params, + count() as c +from logs +where source = 'edge_logs' +group by path, params +order by c desc +limit 100`, + 'REST Requests': `select + timestamp, + event_message +from logs +where source = 'edge_logs' + and log_attributes['request.path'] like '%rest/v1%' +order by timestamp desc +limit 100`, + Errors: `select + timestamp, + log_attributes['parsed.error_severity'] as error_severity, + event_message +from logs +where source = 'postgres_logs' + and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC') +order by timestamp desc +limit 100`, + 'Error Count by User': `select + count() as count, + log_attributes['parsed.user_name'] as user_name, + log_attributes['parsed.error_severity'] as error_severity +from logs +where source = 'postgres_logs' + and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC') +group by user_name, error_severity +order by count desc +limit 100`, + 'Auth Endpoint Events': `select + timestamp, + event_message +from logs +where source = 'auth_logs' + and match(event_message, 'level.{3}(info|warning|error|fatal)') +order by timestamp desc +limit 100`, + 'Auth Audit Logs': `select + timestamp, + event_message, + log_attributes +from logs +where source = 'auth_audit_logs' +order by timestamp desc +limit 10`, + 'Storage Object Requests': `select + log_attributes['request.method'] as http_verb, + log_attributes['request.path'] as filepath, + count() as num_requests +from logs +where source = 'edge_logs' + and log_attributes['request.path'] like '%storage/v1/object/%' +group by http_verb, filepath +order by num_requests desc +limit 100`, + 'Storage Egress Requests': `select + log_attributes['request.method'] as http_verb, + log_attributes['request.path'] as filepath, + (log_attributes['response.headers.cf_cache_status'] = 'HIT') as cached, + count() as num_requests +from logs +where source = 'edge_logs' + and ( + log_attributes['request.path'] like '%storage/v1/object/%' + or log_attributes['request.path'] like '%storage/v1/render/%' + ) + and log_attributes['request.method'] = 'GET' +group by http_verb, filepath, cached +order by num_requests desc +limit 100`, + 'Storage Top Cache Misses': `select + log_attributes['request.path'] as path, + log_attributes['request.search'] as search, + count() as count +from logs +where source = 'edge_logs' + and startsWith(log_attributes['request.path'], '/storage/v1/object') + and log_attributes['request.method'] = 'GET' + and log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC') +group by path, search +order by count desc +limit 100`, +} + +/** + * Returns the log templates for the active engine. On the OTEL/ClickHouse engine + * the custom templates, written for BigQuery, are swapped for their ClickHouse + * rewrites; everything else is returned unchanged. + */ +export function getLogsTemplates(useOtel: boolean): LogTemplate[] { + if (!useOtel) return TEMPLATES + return TEMPLATES.map((template) => { + const otelSearchString = template.label && OTEL_TEMPLATE_SEARCH_STRINGS[template.label] + return otelSearchString ? { ...template, searchString: otelSearchString } : template + }) +} + type SqlFilterFn = (value: any) => SafeLogSqlFragment export type SqlFilterEntry = SafeLogSqlFragment | SqlFilterFn diff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts new file mode 100644 index 00000000000..f0ee03d2768 --- /dev/null +++ b/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from 'vitest' + +import { getLogsTemplates, TEMPLATES } from './Logs.constants' + +const customTemplates = TEMPLATES.filter((t) => t.mode === 'custom') +const bigQueryConstructs = + /unnest|cross join|left join|regexp_contains|cast\(\s*timestamp|count\(\*\)/i + +describe('getLogsTemplates', () => { + it('returns the original templates when the OTEL engine is off', () => { + expect(getLogsTemplates(false)).toBe(TEMPLATES) + }) + + it('gives every custom template a ClickHouse rewrite when OTEL is on', () => { + const otel = getLogsTemplates(true) + for (const template of customTemplates) { + const rewritten = otel.find((t) => t.label === template.label) + expect(rewritten, `missing OTEL rewrite for "${template.label}"`).toBeDefined() + expect( + rewritten!.searchString, + `"${template.label}" was not rewritten for ClickHouse` + ).not.toBe(template.searchString) + } + }) + + it('produces ClickHouse SQL with no BigQuery constructs', () => { + const otel = getLogsTemplates(true) + for (const template of otel.filter((t) => t.mode === 'custom')) { + expect(template.searchString, `"${template.label}" still targets logs table`).toMatch( + /from logs/i + ) + expect(template.searchString, `"${template.label}" filters by source`).toMatch(/source\s*=/i) + expect( + template.searchString, + `"${template.label}" still contains a BigQuery construct` + ).not.toMatch(bigQueryConstructs) + } + }) + + it('leaves simple-mode templates unchanged on the OTEL engine', () => { + const otel = getLogsTemplates(true) + for (const template of TEMPLATES.filter((t) => t.mode === 'simple')) { + const same = otel.find((t) => t.label === template.label) + expect(same!.searchString).toBe(template.searchString) + } + }) + + it('maps nested BigQuery fields to the right log_attributes keys', () => { + const otel = getLogsTemplates(true) + const byLabel = (label: string) => otel.find((t) => t.label === label)!.searchString + + expect(byLabel('Requests by Geography')).toContain("log_attributes['request.cf.country']") + expect(byLabel('Metadata IP')).toContain("log_attributes['request.headers.x_real_ip']") + expect(byLabel('Errors')).toContain("log_attributes['parsed.error_severity']") + expect(byLabel('Slow Response Time')).toContain( + "toInt32OrZero(log_attributes['response.origin_time'])" + ) + }) +}) diff --git a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx b/apps/studio/pages/project/[ref]/logs/explorer/index.tsx index 1fb54164151..c246e2da385 100644 --- a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx +++ b/apps/studio/pages/project/[ref]/logs/explorer/index.tsx @@ -16,8 +16,8 @@ import { import { EXPLORER_DATEPICKER_HELPERS, getDefaultHelper, + getLogsTemplates, LOGS_LARGE_DATE_RANGE_DAYS_THRESHOLD, - TEMPLATES, } from '@/components/interfaces/Settings/Logs/Logs.constants' import { DatePickerValue } from '@/components/interfaces/Settings/Logs/Logs.DatePickers' import { LogData, LogsWarning, LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types' @@ -85,11 +85,13 @@ export const LogsExplorerPage: NextPageWithLayout = () => { const track = useTrack() const projectRef = ref as string const { logsShowMetadataIpTemplate } = useIsFeatureEnabled(['logs:show_metadata_ip_template']) + const useOtelEndpoint = useFlag('otelLegacyLogs') const allTemplates = useMemo(() => { - if (logsShowMetadataIpTemplate) return TEMPLATES - else return TEMPLATES.filter((x) => x.label !== 'Metadata IP') - }, [logsShowMetadataIpTemplate]) + const templates = getLogsTemplates(useOtelEndpoint) + if (logsShowMetadataIpTemplate) return templates + else return templates.filter((x) => x.label !== 'Metadata IP') + }, [logsShowMetadataIpTemplate, useOtelEndpoint]) const editorRef = useRef(null) const [editorId] = useState(uuidv4()) @@ -111,8 +113,6 @@ export const LogsExplorerPage: NextPageWithLayout = () => { }, [timestampStart, timestampEnd, defaultHelper]) const [datePickerValue, setDatePickerValue] = useState(initialDatePickerValue) - const useOtelEndpoint = useFlag('otelLegacyLogs') - const { logsDefaultQuery } = useCustomContent(['logs:default_query']) const PLACEHOLDER_QUERY = useOtelEndpoint ? OTEL_PLACEHOLDER_QUERY diff --git a/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx b/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx index d0f507ed041..d5af19ee7cd 100644 --- a/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx +++ b/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx @@ -1,9 +1,9 @@ -import { useParams } from 'common' +import { useFlag, useParams } from 'common' import { CodeIcon } from 'lucide-react' import { useState } from 'react' import { Button, cn, Popover, PopoverContent, PopoverTrigger } from 'ui' -import { TEMPLATES } from '@/components/interfaces/Settings/Logs/Logs.constants' +import { getLogsTemplates } from '@/components/interfaces/Settings/Logs/Logs.constants' import type { LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types' import { DefaultLayout } from '@/components/layouts/DefaultLayout' import LogsLayout from '@/components/layouts/LogsLayout/LogsLayout' @@ -17,14 +17,16 @@ export const LogsTemplatesPage: NextPageWithLayout = () => { const { ref: projectRef } = useParams() const { logsTemplates: isTemplatesEnabled, logsShowMetadataIpTemplate: showMetadataIpTemplate } = useIsFeatureEnabled(['logs:templates', 'logs:show_metadata_ip_template']) + const useOtelEndpoint = useFlag('otelLegacyLogs') if (!isTemplatesEnabled) { return } + const templates = getLogsTemplates(useOtelEndpoint) const allTemplates = showMetadataIpTemplate - ? TEMPLATES - : TEMPLATES.filter((template) => template.label !== 'Metadata IP') + ? templates + : templates.filter((template) => template.label !== 'Metadata IP') return (