feat(studio): load ClickHouse query templates when OTEL logs are on (#47390)

apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts

@@ -286,6 +286,157 @@ limit 100;
   },
 ]
 
+// ClickHouse rewrites of the custom BigQuery templates above, used when the OTEL
+// logs engine is on. Keyed by template label. Simple-mode templates are plain
+// event_message searches that work unchanged, so they have no entry here.
+const OTEL_TEMPLATE_SEARCH_STRINGS: Record<string, string> = {
+  'Commits By User': `select
+  log_attributes['parsed.user_name'] as user_name,
+  count() as count
+from logs
+where source = 'postgres_logs'
+  and event_message like '%COMMIT%'
+group by user_name
+order by count desc
+limit 100`,
+  'Metadata IP': `select
+  timestamp,
+  log_attributes['request.headers.x_real_ip'] as x_real_ip
+from logs
+where source = 'edge_logs'
+  and log_attributes['request.headers.x_real_ip'] != ''
+order by timestamp desc
+limit 100`,
+  'Requests by Geography': `select
+  log_attributes['request.cf.country'] as country,
+  count() as count
+from logs
+where source = 'edge_logs'
+group by country
+order by count desc
+limit 100`,
+  'Slow Response Time': `select
+  timestamp,
+  event_message,
+  toInt32OrZero(log_attributes['response.origin_time']) as origin_time
+from logs
+where source = 'edge_logs'
+  and toInt32OrZero(log_attributes['response.origin_time']) > 1000
+order by timestamp desc
+limit 100`,
+  '500 Request Codes': `select
+  timestamp,
+  event_message,
+  toInt32OrZero(log_attributes['response.status_code']) as status_code
+from logs
+where source = 'edge_logs'
+  and toInt32OrZero(log_attributes['response.status_code']) >= 500
+order by timestamp desc
+limit 100`,
+  'Top Paths': `select
+  log_attributes['request.path'] as path,
+  log_attributes['request.search'] as params,
+  count() as c
+from logs
+where source = 'edge_logs'
+group by path, params
+order by c desc
+limit 100`,
+  'REST Requests': `select
+  timestamp,
+  event_message
+from logs
+where source = 'edge_logs'
+  and log_attributes['request.path'] like '%rest/v1%'
+order by timestamp desc
+limit 100`,
+  Errors: `select
+  timestamp,
+  log_attributes['parsed.error_severity'] as error_severity,
+  event_message
+from logs
+where source = 'postgres_logs'
+  and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC')
+order by timestamp desc
+limit 100`,
+  'Error Count by User': `select
+  count() as count,
+  log_attributes['parsed.user_name'] as user_name,
+  log_attributes['parsed.error_severity'] as error_severity
+from logs
+where source = 'postgres_logs'
+  and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC')
+group by user_name, error_severity
+order by count desc
+limit 100`,
+  'Auth Endpoint Events': `select
+  timestamp,
+  event_message
+from logs
+where source = 'auth_logs'
+  and match(event_message, 'level.{3}(info|warning|error|fatal)')
+order by timestamp desc
+limit 100`,
+  'Auth Audit Logs': `select
+  timestamp,
+  event_message,
+  log_attributes
+from logs
+where source = 'auth_audit_logs'
+order by timestamp desc
+limit 10`,
+  'Storage Object Requests': `select
+  log_attributes['request.method'] as http_verb,
+  log_attributes['request.path'] as filepath,
+  count() as num_requests
+from logs
+where source = 'edge_logs'
+  and log_attributes['request.path'] like '%storage/v1/object/%'
+group by http_verb, filepath
+order by num_requests desc
+limit 100`,
+  'Storage Egress Requests': `select
+  log_attributes['request.method'] as http_verb,
+  log_attributes['request.path'] as filepath,
+  (log_attributes['response.headers.cf_cache_status'] = 'HIT') as cached,
+  count() as num_requests
+from logs
+where source = 'edge_logs'
+  and (
+    log_attributes['request.path'] like '%storage/v1/object/%'
+    or log_attributes['request.path'] like '%storage/v1/render/%'
+  )
+  and log_attributes['request.method'] = 'GET'
+group by http_verb, filepath, cached
+order by num_requests desc
+limit 100`,
+  'Storage Top Cache Misses': `select
+  log_attributes['request.path'] as path,
+  log_attributes['request.search'] as search,
+  count() as count
+from logs
+where source = 'edge_logs'
+  and startsWith(log_attributes['request.path'], '/storage/v1/object')
+  and log_attributes['request.method'] = 'GET'
+  and log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC')
+group by path, search
+order by count desc
+limit 100`,
+}
+
+/**
+ * Returns the log templates for the active engine. On the OTEL/ClickHouse engine
+ * the custom templates, written for BigQuery, are swapped for their ClickHouse
+ * rewrites; everything else is returned unchanged.
+ */
+export function getLogsTemplates(useOtel: boolean): LogTemplate[] {
+  if (!useOtel) return TEMPLATES
+  return TEMPLATES.map((template) => {
+    const otelSearchString = template.label && OTEL_TEMPLATE_SEARCH_STRINGS[template.label]
+    return otelSearchString ? { ...template, searchString: otelSearchString } : template
+  })
+}
+
 type SqlFilterFn = (value: any) => SafeLogSqlFragment
 export type SqlFilterEntry = SafeLogSqlFragment | SqlFilterFn
 

apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from 'vitest'
+
+import { getLogsTemplates, TEMPLATES } from './Logs.constants'
+
+const customTemplates = TEMPLATES.filter((t) => t.mode === 'custom')
+const bigQueryConstructs =
+  /unnest|cross join|left join|regexp_contains|cast\(\s*timestamp|count\(\*\)/i
+
+describe('getLogsTemplates', () => {
+  it('returns the original templates when the OTEL engine is off', () => {
+    expect(getLogsTemplates(false)).toBe(TEMPLATES)
+  })
+
+  it('gives every custom template a ClickHouse rewrite when OTEL is on', () => {
+    const otel = getLogsTemplates(true)
+    for (const template of customTemplates) {
+      const rewritten = otel.find((t) => t.label === template.label)
+      expect(rewritten, `missing OTEL rewrite for "${template.label}"`).toBeDefined()
+      expect(
+        rewritten!.searchString,
+        `"${template.label}" was not rewritten for ClickHouse`
+      ).not.toBe(template.searchString)
+    }
+  })
+
+  it('produces ClickHouse SQL with no BigQuery constructs', () => {
+    const otel = getLogsTemplates(true)
+    for (const template of otel.filter((t) => t.mode === 'custom')) {
+      expect(template.searchString, `"${template.label}" still targets logs table`).toMatch(
+        /from logs/i
+      )
+      expect(template.searchString, `"${template.label}" filters by source`).toMatch(/source\s*=/i)
+      expect(
+        template.searchString,
+        `"${template.label}" still contains a BigQuery construct`
+      ).not.toMatch(bigQueryConstructs)
+    }
+  })
+
+  it('leaves simple-mode templates unchanged on the OTEL engine', () => {
+    const otel = getLogsTemplates(true)
+    for (const template of TEMPLATES.filter((t) => t.mode === 'simple')) {
+      const same = otel.find((t) => t.label === template.label)
+      expect(same!.searchString).toBe(template.searchString)
+    }
+  })
+
+  it('maps nested BigQuery fields to the right log_attributes keys', () => {
+    const otel = getLogsTemplates(true)
+    const byLabel = (label: string) => otel.find((t) => t.label === label)!.searchString
+
+    expect(byLabel('Requests by Geography')).toContain("log_attributes['request.cf.country']")
+    expect(byLabel('Metadata IP')).toContain("log_attributes['request.headers.x_real_ip']")
+    expect(byLabel('Errors')).toContain("log_attributes['parsed.error_severity']")
+    expect(byLabel('Slow Response Time')).toContain(
+      "toInt32OrZero(log_attributes['response.origin_time'])"
+    )
+  })
+})

apps/studio/pages/project/[ref]/logs/explorer/index.tsx

@@ -16,8 +16,8 @@ import {
 import {
   EXPLORER_DATEPICKER_HELPERS,
   getDefaultHelper,
+  getLogsTemplates,
   LOGS_LARGE_DATE_RANGE_DAYS_THRESHOLD,
-  TEMPLATES,
 } from '@/components/interfaces/Settings/Logs/Logs.constants'
 import { DatePickerValue } from '@/components/interfaces/Settings/Logs/Logs.DatePickers'
 import { LogData, LogsWarning, LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types'
@@ -85,11 +85,13 @@ export const LogsExplorerPage: NextPageWithLayout = () => {
   const track = useTrack()
   const projectRef = ref as string
   const { logsShowMetadataIpTemplate } = useIsFeatureEnabled(['logs:show_metadata_ip_template'])
+  const useOtelEndpoint = useFlag('otelLegacyLogs')
 
   const allTemplates = useMemo(() => {
-    if (logsShowMetadataIpTemplate) return TEMPLATES
-    else return TEMPLATES.filter((x) => x.label !== 'Metadata IP')
-  }, [logsShowMetadataIpTemplate])
+    const templates = getLogsTemplates(useOtelEndpoint)
+    if (logsShowMetadataIpTemplate) return templates
+    else return templates.filter((x) => x.label !== 'Metadata IP')
+  }, [logsShowMetadataIpTemplate, useOtelEndpoint])
 
   const editorRef = useRef<editor.IStandaloneCodeEditor>(null)
   const [editorId] = useState<string>(uuidv4())
@@ -111,8 +113,6 @@ export const LogsExplorerPage: NextPageWithLayout = () => {
   }, [timestampStart, timestampEnd, defaultHelper])
   const [datePickerValue, setDatePickerValue] = useState<DatePickerValue>(initialDatePickerValue)
 
-  const useOtelEndpoint = useFlag('otelLegacyLogs')
-
   const { logsDefaultQuery } = useCustomContent(['logs:default_query'])
   const PLACEHOLDER_QUERY = useOtelEndpoint
     ? OTEL_PLACEHOLDER_QUERY

apps/studio/pages/project/[ref]/logs/explorer/templates.tsx

@@ -1,9 +1,9 @@
-import { useParams } from 'common'
+import { useFlag, useParams } from 'common'
 import { CodeIcon } from 'lucide-react'
 import { useState } from 'react'
 import { Button, cn, Popover, PopoverContent, PopoverTrigger } from 'ui'
 
-import { TEMPLATES } from '@/components/interfaces/Settings/Logs/Logs.constants'
+import { getLogsTemplates } from '@/components/interfaces/Settings/Logs/Logs.constants'
 import type { LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types'
 import { DefaultLayout } from '@/components/layouts/DefaultLayout'
 import LogsLayout from '@/components/layouts/LogsLayout/LogsLayout'
@@ -17,14 +17,16 @@ export const LogsTemplatesPage: NextPageWithLayout = () => {
   const { ref: projectRef } = useParams()
   const { logsTemplates: isTemplatesEnabled, logsShowMetadataIpTemplate: showMetadataIpTemplate } =
     useIsFeatureEnabled(['logs:templates', 'logs:show_metadata_ip_template'])
+  const useOtelEndpoint = useFlag('otelLegacyLogs')
 
   if (!isTemplatesEnabled) {
     return <UnknownInterface urlBack={`/project/${projectRef}/logs/explorer`} />
   }
 
+  const templates = getLogsTemplates(useOtelEndpoint)
   const allTemplates = showMetadataIpTemplate
-    ? TEMPLATES
-    : TEMPLATES.filter((template) => template.label !== 'Metadata IP')
+    ? templates
+    : templates.filter((template) => template.label !== 'Metadata IP')
 
   return (
     <div className="mx-auto h-full w-full px-5 py-6">