mirror of
https://github.com/supabase/supabase.git
synced 2026-10-06 09:55:06 +03:00
studio(logs): use safe sql escaping for new logs queries (#45887)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Introduced a safe SQL fragment system and helpers to build composable, validated log queries and aggregations. * **Refactor** * Rewrote unified log query builders and inspection flows to use the new safe fragments and identifier/literal validators. * **Bug Fixes** * Improved validation and error handling for filter keys and literal escaping to prevent malformed or injectable queries. * **Tests** * Added tests covering identifier quoting, value escaping, and rejection of invalid filter inputs. <!-- review_stack_entry_start --> [](https://app.coderabbit.ai/change-stack/supabase/supabase/pull/45887) <!-- review_stack_entry_end --> <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
1 parent
c104beabb6
commit
ec21e68eee
6 files changed
+454
-255
No files matched your search
@@ -4,11 +4,17 @@
|
||||
// during the migration. This file should be deleted once the flag is
|
||||
// removed.
|
||||
|
||||
import { literal } from '@supabase/pg-meta/src/pg-format'
|
||||
import dayjs from 'dayjs'
|
||||
|
||||
import { DEFAULT_LOG_TYPES } from './UnifiedLogs.constants'
|
||||
import { QuerySearchParamsType, SearchParamsType } from './UnifiedLogs.types'
|
||||
import {
|
||||
bqIdent,
|
||||
joinSqlFragments,
|
||||
analyticsLiteral as lit,
|
||||
safeSql,
|
||||
type SafeLogSqlFragment,
|
||||
} from '@/data/logs/safe-analytics-sql'
|
||||
|
||||
// Pagination and control parameters
|
||||
const PAGINATION_PARAMS = ['sort', 'start', 'size', 'uuid', 'cursor', 'direction', 'live'] as const
|
||||
@@ -19,55 +25,77 @@ const SPECIAL_FILTER_PARAMS = ['date'] as const
|
||||
// Combined list of all parameters to exclude from standard filtering
|
||||
const EXCLUDED_QUERY_PARAMS = [...PAGINATION_PARAMS, ...SPECIAL_FILTER_PARAMS] as const
|
||||
|
||||
// Strips `literal()`'s surrounding quotes so the value can be concatenated
|
||||
// inside a `LIKE '%...%'` clause without re-escaping; quotes and back-
|
||||
// slashes are already escaped by literal().
|
||||
const likeValue = (value: unknown): string =>
|
||||
literal(String(value)).replace(/^E?'/, '').replace(/'$/, '')
|
||||
|
||||
/**
|
||||
* Builds query conditions from search parameters and returns WHERE clause
|
||||
* @param search SearchParamsType object containing query parameters
|
||||
* @returns Object with whereConditions array and formatted WHERE clause
|
||||
* Builds WHERE-clause fragments from a search-param map. Identifier-position
|
||||
* keys are validated via `bqIdent()` (regex allowlist) and value-position
|
||||
* inputs via `analyticsLiteral` — both throw on disallowed input, in which
|
||||
* case we drop the predicate rather than emit unsafe SQL.
|
||||
*
|
||||
* @param search Search params (URL-derived filter values)
|
||||
* @param excludeKey Optional key to skip — used by facet-count branches that
|
||||
* need every filter applied *except* the one being faceted
|
||||
* @returns Array of SafeLogSqlFragment predicates ready to be AND-joined
|
||||
*/
|
||||
const buildQueryConditions = (search: QuerySearchParamsType) => {
|
||||
const whereConditions: string[] = []
|
||||
const buildConditions = (
|
||||
search: QuerySearchParamsType,
|
||||
excludeKey?: string
|
||||
): SafeLogSqlFragment[] => {
|
||||
const conditions: SafeLogSqlFragment[] = []
|
||||
|
||||
// Process all search parameters for filtering
|
||||
Object.entries(search).forEach(([key, value]) => {
|
||||
// Skip pagination/control parameters
|
||||
if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) {
|
||||
return
|
||||
}
|
||||
if (key === excludeKey) return
|
||||
if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) return
|
||||
|
||||
// Handle array filters (IN clause)
|
||||
if (Array.isArray(value) && value.length > 0) {
|
||||
whereConditions.push(`${key} IN (${value.map((v) => literal(String(v))).join(',')})`)
|
||||
return
|
||||
}
|
||||
try {
|
||||
// `key` is interpolated as a column identifier. `bqIdent()` rejects
|
||||
// anything outside `[A-Za-z_][A-Za-z0-9_]*` (notably no spaces, so a
|
||||
// crafted URL key like `level OR id IS NOT NULL` is dropped rather
|
||||
// than emitted into the WHERE clause).
|
||||
const col = bqIdent(key)
|
||||
|
||||
// Handle scalar values
|
||||
if (value !== null && value !== undefined) {
|
||||
if (['host', 'pathname'].includes(key)) {
|
||||
whereConditions.push(`${key} LIKE '%${likeValue(value)}%'`)
|
||||
} else {
|
||||
whereConditions.push(`${key} = ${literal(String(value))}`)
|
||||
if (Array.isArray(value) && value.length > 0) {
|
||||
const inList = joinSqlFragments(
|
||||
value.map((v) => lit(String(v))),
|
||||
','
|
||||
)
|
||||
conditions.push(safeSql`${col} IN (${inList})`)
|
||||
return
|
||||
}
|
||||
|
||||
if (value !== null && value !== undefined) {
|
||||
if (key === 'host' || key === 'pathname') {
|
||||
conditions.push(safeSql`${col} LIKE ${lit('%' + String(value) + '%')}`)
|
||||
} else {
|
||||
conditions.push(safeSql`${col} = ${lit(String(value))}`)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// bqIdent() or analyticsLiteral() rejected the input — drop the predicate.
|
||||
}
|
||||
})
|
||||
|
||||
// Create final WHERE clause
|
||||
const finalWhere = whereConditions.length > 0 ? `WHERE ${whereConditions.join(' AND ')}` : ''
|
||||
|
||||
return { whereConditions, finalWhere }
|
||||
return conditions
|
||||
}
|
||||
|
||||
const whereClause = (conditions: SafeLogSqlFragment[]): SafeLogSqlFragment =>
|
||||
conditions.length > 0 ? safeSql`WHERE ${joinSqlFragments(conditions, ' AND ')}` : safeSql``
|
||||
|
||||
/**
|
||||
* Calculates how much the chart start datetime should be offset given the current datetime filter params
|
||||
* and determines the appropriate bucketing level (minute, hour, day)
|
||||
* Ported from the older implementation (apps/studio/components/interfaces/Settings/Logs/Logs.utils.ts)
|
||||
*/
|
||||
const calculateChartBucketing = (search: SearchParamsType | Record<string, unknown>): string => {
|
||||
type TruncationLevel = 'MINUTE' | 'HOUR' | 'DAY'
|
||||
|
||||
const TRUNCATION_LEVEL_SQL: Record<TruncationLevel, SafeLogSqlFragment> = {
|
||||
MINUTE: safeSql`MINUTE`,
|
||||
HOUR: safeSql`HOUR`,
|
||||
DAY: safeSql`DAY`,
|
||||
}
|
||||
|
||||
const calculateChartBucketing = (
|
||||
search: SearchParamsType | Record<string, unknown>
|
||||
): TruncationLevel => {
|
||||
// Extract start and end times from the date array if available
|
||||
const dateRange = (search.date as Array<Date | string | number | null | undefined>) || []
|
||||
|
||||
@@ -101,21 +129,12 @@ const calculateChartBucketing = (search: SearchParamsType | Record<string, unkno
|
||||
const startTime = dayjs(startMillis)
|
||||
const endTime = dayjs(endMillis)
|
||||
|
||||
let truncationLevel = 'MINUTE'
|
||||
|
||||
const hourDiff = endTime.diff(startTime, 'hour')
|
||||
const dayDiff = endTime.diff(startTime, 'day')
|
||||
|
||||
// Adjust bucketing based on time range
|
||||
if (dayDiff >= 2) {
|
||||
truncationLevel = 'DAY'
|
||||
} else if (hourDiff >= 12) {
|
||||
truncationLevel = 'HOUR'
|
||||
} else {
|
||||
truncationLevel = 'MINUTE'
|
||||
}
|
||||
|
||||
return truncationLevel
|
||||
if (dayDiff >= 2) return 'DAY'
|
||||
if (hourDiff >= 12) return 'HOUR'
|
||||
return 'MINUTE'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -123,8 +142,7 @@ const calculateChartBucketing = (search: SearchParamsType | Record<string, unkno
|
||||
*
|
||||
* excludes `/rest/` in the path
|
||||
*/
|
||||
const getEdgeLogsQuery = () => {
|
||||
return `
|
||||
const getEdgeLogsQuery = (): SafeLogSqlFragment => safeSql`
|
||||
select
|
||||
id,
|
||||
null as source_id,
|
||||
@@ -150,15 +168,10 @@ const getEdgeLogsQuery = () => {
|
||||
-- ONLY include logs where the path does not include /rest/
|
||||
WHERE edge_logs_request.path NOT LIKE '%/rest/%'
|
||||
AND edge_logs_request.path NOT LIKE '%/storage/%'
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
// Postgrest logs
|
||||
|
||||
// WHERE pathname includes `/rest/`
|
||||
const getPostgrestLogsQuery = () => {
|
||||
return `
|
||||
// Postgrest logs — WHERE pathname includes `/rest/`
|
||||
const getPostgrestLogsQuery = (): SafeLogSqlFragment => safeSql`
|
||||
select
|
||||
id,
|
||||
null as source_id,
|
||||
@@ -184,13 +197,11 @@ const getPostgrestLogsQuery = () => {
|
||||
-- ONLY include logs where the path includes /rest/
|
||||
WHERE edge_logs_request.path LIKE '%/rest/%'
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Postgres logs query fragment
|
||||
*/
|
||||
const getPostgresLogsQuery = () => {
|
||||
return `
|
||||
const getPostgresLogsQuery = (): SafeLogSqlFragment => safeSql`
|
||||
select
|
||||
id,
|
||||
null as source_id,
|
||||
@@ -213,13 +224,11 @@ const getPostgresLogsQuery = () => {
|
||||
cross join unnest(pgl.metadata) as pgl_metadata
|
||||
cross join unnest(pgl_metadata.parsed) as pgl_parsed
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Edge function logs query fragment
|
||||
*/
|
||||
const getEdgeFunctionLogsQuery = () => {
|
||||
return `
|
||||
const getEdgeFunctionLogsQuery = (): SafeLogSqlFragment => safeSql`
|
||||
select
|
||||
id,
|
||||
null as source_id,
|
||||
@@ -252,13 +261,11 @@ const getEdgeFunctionLogsQuery = () => {
|
||||
GROUP BY fl_metadata.request_id
|
||||
) as function_logs_agg on fel_metadata.request_id = function_logs_agg.request_id
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Auth logs query fragment
|
||||
*/
|
||||
const getAuthLogsQuery = () => {
|
||||
return `
|
||||
const getAuthLogsQuery = (): SafeLogSqlFragment => safeSql`
|
||||
select
|
||||
el_in_al.id as id,
|
||||
al.id as source_id,
|
||||
@@ -277,24 +284,22 @@ const getAuthLogsQuery = () => {
|
||||
null as log_count,
|
||||
null as logs
|
||||
from auth_logs as al
|
||||
cross join unnest(metadata) as al_metadata
|
||||
cross join unnest(metadata) as al_metadata
|
||||
left join (
|
||||
edge_logs as el_in_al
|
||||
cross join unnest (metadata) as el_in_al_metadata
|
||||
cross join unnest (el_in_al_metadata.response) as el_in_al_response
|
||||
cross join unnest (el_in_al_response.headers) as el_in_al_response_headers
|
||||
cross join unnest (metadata) as el_in_al_metadata
|
||||
cross join unnest (el_in_al_metadata.response) as el_in_al_response
|
||||
cross join unnest (el_in_al_response.headers) as el_in_al_response_headers
|
||||
cross join unnest (el_in_al_metadata.request) as el_in_al_request
|
||||
)
|
||||
on al_metadata.request_id = el_in_al_response_headers.cf_ray
|
||||
WHERE al_metadata.request_id is not null
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Supabase storage logs query fragment
|
||||
*/
|
||||
const getSupabaseStorageLogsQuery = () => {
|
||||
return `
|
||||
const getSupabaseStorageLogsQuery = (): SafeLogSqlFragment => safeSql`
|
||||
select
|
||||
id,
|
||||
null as source_id,
|
||||
@@ -319,9 +324,8 @@ const getSupabaseStorageLogsQuery = () => {
|
||||
-- ONLY include logs where the path includes /storage/
|
||||
WHERE edge_logs_request.path LIKE '%/storage/%'
|
||||
`
|
||||
}
|
||||
|
||||
const LOG_TYPE_QUERIES: Record<string, () => string> = {
|
||||
const LOG_TYPE_QUERIES: Record<string, () => SafeLogSqlFragment> = {
|
||||
edge: getEdgeLogsQuery,
|
||||
postgrest: getPostgrestLogsQuery,
|
||||
postgres: getPostgresLogsQuery,
|
||||
@@ -334,17 +338,19 @@ const LOG_TYPE_QUERIES: Record<string, () => string> = {
|
||||
* Combine the requested log sources to create the unified logs CTE.
|
||||
* Defaults to postgres + postgrest on first load to reduce query cost.
|
||||
*/
|
||||
export const getUnifiedLogsCTE = (logTypes: string[] = [...DEFAULT_LOG_TYPES]) => {
|
||||
export const getUnifiedLogsCTE = (
|
||||
logTypes: string[] = [...DEFAULT_LOG_TYPES]
|
||||
): SafeLogSqlFragment => {
|
||||
const queries = logTypes
|
||||
.filter((type) => type in LOG_TYPE_QUERIES)
|
||||
.map((type) => LOG_TYPE_QUERIES[type]())
|
||||
|
||||
const effectiveQueries =
|
||||
queries.length > 0 ? queries : DEFAULT_LOG_TYPES.map((type) => LOG_TYPE_QUERIES[type]())
|
||||
const effective =
|
||||
queries.length > 0 ? queries : DEFAULT_LOG_TYPES.map((t) => LOG_TYPE_QUERIES[t]())
|
||||
|
||||
return `
|
||||
return safeSql`
|
||||
WITH unified_logs AS (
|
||||
${effectiveQueries.join('\n union all\n ')}
|
||||
${joinSqlFragments(effective, ' union all ')}
|
||||
)
|
||||
`
|
||||
}
|
||||
@@ -352,11 +358,11 @@ WITH unified_logs AS (
|
||||
/**
|
||||
* Unified logs SQL query
|
||||
*/
|
||||
export const getUnifiedLogsQuery = (search: QuerySearchParamsType): string => {
|
||||
const { finalWhere } = buildQueryConditions(search)
|
||||
export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
|
||||
const conditions = buildConditions(search)
|
||||
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
|
||||
|
||||
const sql = `
|
||||
return safeSql`
|
||||
${getUnifiedLogsCTE(effectiveLogTypes)}
|
||||
SELECT
|
||||
id,
|
||||
@@ -371,42 +377,8 @@ SELECT
|
||||
log_count,
|
||||
logs
|
||||
FROM unified_logs
|
||||
${finalWhere}
|
||||
${whereClause(conditions)}
|
||||
`
|
||||
|
||||
return sql
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a count query for the total logs within the timeframe
|
||||
* Uses proper faceted search behavior where facets show "what would I get if I selected ONLY this option"
|
||||
*/
|
||||
|
||||
// Helper function to build WHERE clause excluding a specific field
|
||||
const buildFacetWhere = (search: QuerySearchParamsType, excludeField: string): string => {
|
||||
const conditions: string[] = []
|
||||
|
||||
Object.entries(search).forEach(([key, value]) => {
|
||||
if (key === excludeField) return // Skip the field we're getting facets for
|
||||
if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) return // Skip pagination and special params
|
||||
|
||||
// Handle array filters (IN clause)
|
||||
if (Array.isArray(value) && value.length > 0) {
|
||||
conditions.push(`${key} IN (${value.map((v) => literal(String(v))).join(',')})`)
|
||||
return
|
||||
}
|
||||
|
||||
// Handle scalar values
|
||||
if (value !== null && value !== undefined) {
|
||||
if (['host', 'pathname'].includes(key)) {
|
||||
conditions.push(`${key} LIKE '%${likeValue(value)}%'`)
|
||||
} else {
|
||||
conditions.push(`${key} = ${literal(String(value))}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''
|
||||
}
|
||||
|
||||
export const getFacetCountCTE = ({
|
||||
@@ -417,24 +389,36 @@ export const getFacetCountCTE = ({
|
||||
search: QuerySearchParamsType
|
||||
facet: string
|
||||
facetSearch?: string
|
||||
}) => {
|
||||
}): SafeLogSqlFragment => {
|
||||
const MAX_FACETS_QUANTITY = 20
|
||||
|
||||
return `
|
||||
${facet}_count AS (
|
||||
SELECT '${facet}' as dimension, ${facet} as value, COUNT(*) as count
|
||||
// `facet` is used both as a column reference and to derive a CTE name;
|
||||
// quote each appropriately with bqIdent() to reject non-identifier inputs.
|
||||
const facetCol = bqIdent(facet)
|
||||
const facetCte = bqIdent(facet + '_count')
|
||||
const baseConditions = buildConditions(search, facet)
|
||||
const facetSearchClause = facetSearch
|
||||
? safeSql`AND ${facetCol} LIKE ${lit('%' + facetSearch + '%')}`
|
||||
: safeSql``
|
||||
|
||||
const where =
|
||||
baseConditions.length > 0
|
||||
? safeSql`WHERE ${joinSqlFragments(baseConditions, ' AND ')} AND ${facetCol} IS NOT NULL`
|
||||
: safeSql`WHERE ${facetCol} IS NOT NULL`
|
||||
|
||||
return safeSql`
|
||||
${facetCte} AS (
|
||||
SELECT ${lit(facet)} as dimension, ${facetCol} as value, COUNT(*) as count
|
||||
FROM unified_logs
|
||||
${buildFacetWhere(search, `${facet}`) || `WHERE ${facet} IS NOT NULL`}
|
||||
${buildFacetWhere(search, `${facet}`) ? ` AND ${facet} IS NOT NULL` : ''}
|
||||
${!!facetSearch ? `AND ${facet} LIKE '%${likeValue(facetSearch)}%'` : ''}
|
||||
GROUP BY ${facet}
|
||||
LIMIT ${MAX_FACETS_QUANTITY}
|
||||
${where}
|
||||
${facetSearchClause}
|
||||
GROUP BY ${facetCol}
|
||||
LIMIT ${lit(MAX_FACETS_QUANTITY)}
|
||||
)
|
||||
`.trim()
|
||||
`
|
||||
}
|
||||
|
||||
export const getUnifiedLogsCountCTE = () => {
|
||||
return `
|
||||
export const getUnifiedLogsCountCTE = (): SafeLogSqlFragment => safeSql`
|
||||
WITH unified_logs AS (
|
||||
-- Single scan of edge_logs covering edge gateway, postgrest, and storage
|
||||
select
|
||||
@@ -526,14 +510,21 @@ WITH unified_logs AS (
|
||||
WHERE al_metadata.request_id is not null
|
||||
)
|
||||
`
|
||||
}
|
||||
|
||||
export const getLogsCountQuery = (search: QuerySearchParamsType): string => {
|
||||
export const getLogsCountQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
|
||||
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
|
||||
const logTypeWhere = buildFacetWhere(search, 'log_type') || 'WHERE log_type IS NOT NULL'
|
||||
const levelWhere = buildFacetWhere(search, 'level') || 'WHERE level IS NOT NULL'
|
||||
const logTypeConditions = buildConditions(search, 'log_type')
|
||||
const levelConditions = buildConditions(search, 'level')
|
||||
const logTypeWhere: SafeLogSqlFragment =
|
||||
logTypeConditions.length > 0
|
||||
? safeSql`WHERE ${joinSqlFragments(logTypeConditions, ' AND ')}`
|
||||
: safeSql`WHERE log_type IS NOT NULL`
|
||||
const levelWhere: SafeLogSqlFragment =
|
||||
levelConditions.length > 0
|
||||
? safeSql`WHERE ${joinSqlFragments(levelConditions, ' AND ')}`
|
||||
: safeSql`WHERE level IS NOT NULL`
|
||||
|
||||
const sql = `
|
||||
return safeSql`
|
||||
${getUnifiedLogsCTE(effectiveLogTypes)},
|
||||
|
||||
-- Single COUNTIF pass for all log_type buckets + total (no GROUP BY / sort needed)
|
||||
@@ -579,29 +570,27 @@ UNION ALL SELECT dimension, value, count FROM method_count
|
||||
UNION ALL SELECT dimension, value, count FROM status_count
|
||||
UNION ALL SELECT dimension, value, count FROM pathname_count
|
||||
`
|
||||
|
||||
return sql
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced logs chart query with dynamic bucketing based on time range
|
||||
* Incorporates dynamic bucketing from the older implementation
|
||||
*/
|
||||
export const getLogsChartQuery = (search: QuerySearchParamsType): string => {
|
||||
const { finalWhere } = buildQueryConditions(search)
|
||||
export const getLogsChartQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
|
||||
const conditions = buildConditions(search)
|
||||
const truncationLevel = calculateChartBucketing(search)
|
||||
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
|
||||
|
||||
return `
|
||||
return safeSql`
|
||||
${getUnifiedLogsCTE(effectiveLogTypes)}
|
||||
SELECT
|
||||
TIMESTAMP_TRUNC(timestamp, ${truncationLevel}) as time_bucket,
|
||||
TIMESTAMP_TRUNC(timestamp, ${TRUNCATION_LEVEL_SQL[truncationLevel]}) as time_bucket,
|
||||
COUNTIF(level = 'success') as success,
|
||||
COUNTIF(level = 'warning') as warning,
|
||||
COUNTIF(level = 'error') as error,
|
||||
COUNT(*) as total_per_bucket
|
||||
FROM unified_logs
|
||||
${finalWhere}
|
||||
${whereClause(conditions)}
|
||||
GROUP BY time_bucket
|
||||
ORDER BY time_bucket ASC
|
||||
`
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
getLogsCountQuery,
|
||||
getUnifiedLogsQuery,
|
||||
} from './UnifiedLogs.queries'
|
||||
import { getUnifiedLogsQuery as getUnifiedLogsQueryBQ } from './UnifiedLogs.queries.bq'
|
||||
|
||||
const baseSearch = {
|
||||
date: [new Date('2026-05-08T09:00:00Z'), new Date('2026-05-08T10:00:00Z')],
|
||||
@@ -134,4 +135,52 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
|
||||
expect(sql).toContain('LIMIT 20')
|
||||
})
|
||||
})
|
||||
|
||||
describe('analyticsLiteral escaping', () => {
|
||||
it('emits ClickHouse / BigQuery escape syntax (doubled `\\\\`, no `E` prefix)', () => {
|
||||
// pg-meta's literal() would emit `E'a\\b'` for `a\b` — the `E` prefix is
|
||||
// Postgres-only and rejected by both analytics engines. analyticsLiteral
|
||||
// doubles the backslash inside plain `'…'` delimiters instead.
|
||||
const sql = getUnifiedLogsQuery({ ...baseSearch, method: 'a\\b' } as any)
|
||||
expect(sql).toContain(`log_attributes['request.method'] = 'a\\\\b'`)
|
||||
expect(sql).not.toContain(`E'a`)
|
||||
})
|
||||
|
||||
it("escapes single quotes by doubling them ('' rather than \\')", () => {
|
||||
const sql = getUnifiedLogsQuery({ ...baseSearch, method: "GET' OR '1'='1" } as any)
|
||||
expect(sql).toContain(`log_attributes['request.method'] = 'GET'' OR ''1''=''1'`)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('UnifiedLogs.queries.bq', () => {
|
||||
it('backtick-quotes column identifiers (BigQuery syntax)', () => {
|
||||
const sql = getUnifiedLogsQueryBQ({ ...baseSearch, method: 'GET' } as any)
|
||||
expect(sql).toContain("`method` = 'GET'")
|
||||
})
|
||||
|
||||
it('rejects keys with non-identifier characters', () => {
|
||||
// A key like "foo; DROP TABLE" fails the bqIdent regex (the `;` is not
|
||||
// in `[A-Za-z_][A-Za-z0-9_]*`), so the predicate is dropped entirely.
|
||||
const sql = getUnifiedLogsQueryBQ({ ...baseSearch, 'foo; DROP TABLE x': 'y' } as any)
|
||||
expect(sql).not.toContain('DROP TABLE')
|
||||
expect(sql).not.toContain('foo;')
|
||||
})
|
||||
|
||||
it('rejects keys containing spaces', () => {
|
||||
const sql = getUnifiedLogsQueryBQ({
|
||||
...baseSearch,
|
||||
'level OR id IS NOT NULL': 'anything',
|
||||
} as any)
|
||||
expect(sql).not.toContain('IS NOT NULL')
|
||||
expect(sql).not.toContain('level OR')
|
||||
})
|
||||
|
||||
it('escapes injection attempts in filter values via analyticsLiteral', () => {
|
||||
const sql = getUnifiedLogsQueryBQ({ ...baseSearch, method: "GET' OR '1'='1" } as any)
|
||||
// The value is single-quote-escaped, so the synthetic OR can't break out
|
||||
// of the string literal.
|
||||
expect(sql).toContain("`method` = 'GET'' OR ''1''=''1'")
|
||||
expect(sql).not.toMatch(/`method` = 'GET' OR '1'='1'/)
|
||||
})
|
||||
})
|
||||
@@ -1,19 +1,13 @@
|
||||
import { literal } from '@supabase/pg-meta/src/pg-format'
|
||||
import dayjs from 'dayjs'
|
||||
|
||||
import { DEFAULT_LOG_TYPES } from './UnifiedLogs.constants'
|
||||
import { QuerySearchParamsType, SearchParamsType } from './UnifiedLogs.types'
|
||||
|
||||
// Escapes a substring of a `LIKE '%...%'` clause so a value containing a
|
||||
// single quote can't terminate the literal early. `literal()` would wrap
|
||||
// the whole value in quotes, which we don't want when concatenating wild-
|
||||
// card characters; this strips its quoting back off.
|
||||
const likeValue = (value: unknown): string => {
|
||||
const escaped = literal(String(value))
|
||||
// literal() wraps strings as 'value' (and prefixes with E for backslash
|
||||
// values); strip the surrounding quotes since we glue this between %.
|
||||
return escaped.replace(/^E?'/, '').replace(/'$/, '')
|
||||
}
|
||||
import {
|
||||
joinSqlFragments,
|
||||
analyticsLiteral as lit,
|
||||
safeSql,
|
||||
type SafeLogSqlFragment,
|
||||
} from '@/data/logs/safe-analytics-sql'
|
||||
|
||||
// Pagination and control parameters
|
||||
const PAGINATION_PARAMS = ['sort', 'start', 'size', 'uuid', 'cursor', 'direction', 'live'] as const
|
||||
@@ -23,14 +17,18 @@ const SPECIAL_FILTER_PARAMS = ['date'] as const
|
||||
|
||||
// Combined list of all parameters to exclude from standard filtering
|
||||
const EXCLUDED_QUERY_PARAMS = [...PAGINATION_PARAMS, ...SPECIAL_FILTER_PARAMS] as const
|
||||
|
||||
// Facets the count query is allowed to be invoked for. Reject anything else
|
||||
// at the entry point rather than letting an unsupported value reach
|
||||
// `log_attributes[…]` lookups.
|
||||
const FACET_FIELDS = ['log_type', 'level', 'method', 'status', 'pathname'] as const
|
||||
|
||||
// OTEL log_attributes keys for HTTP-style fields. Centralized so they can be
|
||||
// adjusted in one place if the backend conventions change.
|
||||
const ATTR = {
|
||||
method: `log_attributes['request.method']`,
|
||||
status: `log_attributes['response.status_code']`,
|
||||
path: `log_attributes['request.path']`,
|
||||
method: safeSql`log_attributes['request.method']`,
|
||||
status: safeSql`log_attributes['response.status_code']`,
|
||||
path: safeSql`log_attributes['request.path']`,
|
||||
} as const
|
||||
|
||||
/**
|
||||
@@ -43,17 +41,17 @@ const ATTR = {
|
||||
* logs from postgREST / storage-api and are intentionally not part of unified
|
||||
* logs; the UI surfaces gateway HTTP traffic for those buckets.
|
||||
*/
|
||||
const LOG_TYPE_PREDICATE: Record<string, string> = {
|
||||
edge: `source = 'edge_logs' AND ${ATTR.path} NOT LIKE '%/rest/%' AND ${ATTR.path} NOT LIKE '%/storage/%'`,
|
||||
postgrest: `source = 'edge_logs' AND ${ATTR.path} LIKE '%/rest/%'`,
|
||||
storage: `source = 'edge_logs' AND ${ATTR.path} LIKE '%/storage/%'`,
|
||||
postgres: `source = 'postgres_logs'`,
|
||||
'edge function': `source = 'function_edge_logs'`,
|
||||
auth: `source = 'auth_logs'`,
|
||||
const LOG_TYPE_PREDICATE: Record<string, SafeLogSqlFragment> = {
|
||||
edge: safeSql`source = 'edge_logs' AND ${ATTR.path} NOT LIKE '%/rest/%' AND ${ATTR.path} NOT LIKE '%/storage/%'`,
|
||||
postgrest: safeSql`source = 'edge_logs' AND ${ATTR.path} LIKE '%/rest/%'`,
|
||||
storage: safeSql`source = 'edge_logs' AND ${ATTR.path} LIKE '%/storage/%'`,
|
||||
postgres: safeSql`source = 'postgres_logs'`,
|
||||
'edge function': safeSql`source = 'function_edge_logs'`,
|
||||
auth: safeSql`source = 'auth_logs'`,
|
||||
}
|
||||
|
||||
// Derived `log_type` column for SELECT / GROUP BY / countIf use.
|
||||
const LOG_TYPE_EXPR = `CASE
|
||||
const LOG_TYPE_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
WHEN source = 'edge_logs' AND ${ATTR.path} LIKE '%/rest/%' THEN 'postgrest'
|
||||
WHEN source = 'edge_logs' AND ${ATTR.path} LIKE '%/storage/%' THEN 'storage'
|
||||
WHEN source = 'edge_logs' THEN 'edge'
|
||||
@@ -65,7 +63,7 @@ const LOG_TYPE_EXPR = `CASE
|
||||
|
||||
// Status code is sourced from the HTTP response for gateway-style rows and
|
||||
// from the Postgres `parsed.sql_state_code` (e.g. `42P01`) for postgres rows.
|
||||
const STATUS_EXPR = `CASE
|
||||
const STATUS_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
WHEN source = 'postgres_logs' THEN toString(log_attributes['parsed.sql_state_code'])
|
||||
ELSE toString(${ATTR.status})
|
||||
END`
|
||||
@@ -78,7 +76,7 @@ const STATUS_EXPR = `CASE
|
||||
// `severity_text` of `INFO` regardless of response code) bucket as
|
||||
// success/warning/error by status. Postgres-style severity is the
|
||||
// fallback for rows without a status code.
|
||||
const LEVEL_EXPR = `CASE
|
||||
const LEVEL_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
WHEN ${ATTR.status} != '' AND toInt32OrZero(${ATTR.status}) >= 500 THEN 'error'
|
||||
WHEN ${ATTR.status} != '' AND toInt32OrZero(${ATTR.status}) BETWEEN 400 AND 499 THEN 'warning'
|
||||
WHEN ${ATTR.status} != '' AND toInt32OrZero(${ATTR.status}) BETWEEN 200 AND 299 THEN 'success'
|
||||
@@ -88,10 +86,11 @@ const LEVEL_EXPR = `CASE
|
||||
ELSE 'success'
|
||||
END`
|
||||
|
||||
const logTypeWherePredicate = (logTypes: string[]) => {
|
||||
const logTypeWherePredicate = (logTypes: string[]): SafeLogSqlFragment => {
|
||||
const effective = logTypes.filter((t) => t in LOG_TYPE_PREDICATE)
|
||||
const types = effective.length ? effective : [...DEFAULT_LOG_TYPES]
|
||||
return `(${types.map((t) => `(${LOG_TYPE_PREDICATE[t]})`).join(' OR ')})`
|
||||
const branches = types.map((t) => safeSql`(${LOG_TYPE_PREDICATE[t]})`)
|
||||
return safeSql`(${joinSqlFragments(branches, ' OR ')})`
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -100,71 +99,90 @@ const logTypeWherePredicate = (logTypes: string[]) => {
|
||||
* `log_type` or `level` in WHERE for some shapes, so we always emit raw-column
|
||||
* predicates (source/severity_text/log_attributes[…]).
|
||||
*/
|
||||
const translateFilter = (key: string, value: unknown): string | null => {
|
||||
const translateFilter = (key: string, value: unknown): SafeLogSqlFragment | null => {
|
||||
if (value === null || value === undefined) return null
|
||||
|
||||
const arr = Array.isArray(value) ? (value.length > 0 ? value : null) : null
|
||||
if (Array.isArray(value) && !arr) return null
|
||||
|
||||
const inList = (values: readonly unknown[]) =>
|
||||
`(${values.map((v) => literal(String(v))).join(',')})`
|
||||
const inList = (values: readonly unknown[]): SafeLogSqlFragment =>
|
||||
safeSql`(${joinSqlFragments(
|
||||
values.map((v) => lit(String(v))),
|
||||
','
|
||||
)})`
|
||||
|
||||
switch (key) {
|
||||
case 'log_type': {
|
||||
const types = (arr ?? [value]).map((v) => String(v))
|
||||
return `(${types
|
||||
.map((t) => `(${LOG_TYPE_PREDICATE[t] ?? `source = ${literal(t)}`})`)
|
||||
.join(' OR ')})`
|
||||
const branches = types.map(
|
||||
(t) => safeSql`(${LOG_TYPE_PREDICATE[t] ?? safeSql`source = ${lit(t)}`})`
|
||||
)
|
||||
return safeSql`(${joinSqlFragments(branches, ' OR ')})`
|
||||
}
|
||||
case 'level': {
|
||||
// No simple raw column for level; reference the inline CASE expression.
|
||||
const levels = arr ?? [value]
|
||||
return `(${LEVEL_EXPR}) IN ${inList(levels.map((v) => String(v)))}`
|
||||
return safeSql`(${LEVEL_EXPR}) IN ${inList(levels.map((v) => String(v)))}`
|
||||
}
|
||||
case 'method':
|
||||
return arr ? `${ATTR.method} IN ${inList(arr)}` : `${ATTR.method} = ${literal(String(value))}`
|
||||
return arr
|
||||
? safeSql`${ATTR.method} IN ${inList(arr)}`
|
||||
: safeSql`${ATTR.method} = ${lit(String(value))}`
|
||||
case 'status':
|
||||
return arr ? `${ATTR.status} IN ${inList(arr)}` : `${ATTR.status} = ${literal(String(value))}`
|
||||
return arr
|
||||
? safeSql`${ATTR.status} IN ${inList(arr)}`
|
||||
: safeSql`${ATTR.status} = ${lit(String(value))}`
|
||||
case 'pathname':
|
||||
return arr
|
||||
? `(${arr.map((v) => `${ATTR.path} LIKE '%${likeValue(v)}%'`).join(' OR ')})`
|
||||
: `${ATTR.path} LIKE '%${likeValue(value)}%'`
|
||||
? safeSql`(${joinSqlFragments(
|
||||
arr.map((v) => safeSql`${ATTR.path} LIKE ${lit('%' + String(v) + '%')}`),
|
||||
' OR '
|
||||
)})`
|
||||
: safeSql`${ATTR.path} LIKE ${lit('%' + String(value) + '%')}`
|
||||
case 'host':
|
||||
// Best-effort: use full request URL since `host` isn't a top-level field.
|
||||
return arr
|
||||
? `(${arr.map((v) => `log_attributes['request.url'] LIKE '%${likeValue(v)}%'`).join(' OR ')})`
|
||||
: `log_attributes['request.url'] LIKE '%${likeValue(value)}%'`
|
||||
? safeSql`(${joinSqlFragments(
|
||||
arr.map(
|
||||
(v) => safeSql`log_attributes['request.url'] LIKE ${lit('%' + String(v) + '%')}`
|
||||
),
|
||||
' OR '
|
||||
)})`
|
||||
: safeSql`log_attributes['request.url'] LIKE ${lit('%' + String(value) + '%')}`
|
||||
default:
|
||||
// Unknown filter key — fall back to a generic equality on log_attributes.
|
||||
// We don't pass `key` through literal() because Map key access in
|
||||
// ClickHouse uses bracket syntax with a string literal; the existing
|
||||
// EXCLUDED_QUERY_PARAMS list and the typed search params surface this
|
||||
// from a static allow-list, not user input.
|
||||
return arr
|
||||
? `log_attributes['${key}'] IN ${inList(arr)}`
|
||||
: `log_attributes['${key}'] = ${literal(String(value))}`
|
||||
? safeSql`log_attributes[${lit(key)}] IN ${inList(arr)}`
|
||||
: safeSql`log_attributes[${lit(key)}] = ${lit(String(value))}`
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an array of WHERE predicate strings from search params, optionally
|
||||
* Builds an array of WHERE predicate fragments from search params, optionally
|
||||
* skipping a specific facet field (used when computing faceted counts).
|
||||
* `log_type` is always handled separately (see `logTypeWherePredicate`).
|
||||
*/
|
||||
const buildPredicates = (search: QuerySearchParamsType, excludeField?: string) => {
|
||||
const predicates: string[] = []
|
||||
const buildPredicates = (
|
||||
search: QuerySearchParamsType,
|
||||
excludeField?: string
|
||||
): SafeLogSqlFragment[] => {
|
||||
const predicates: SafeLogSqlFragment[] = []
|
||||
Object.entries(search).forEach(([key, value]) => {
|
||||
if (key === excludeField) return
|
||||
if (key === 'log_type') return
|
||||
if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) return
|
||||
const predicate = translateFilter(key, value)
|
||||
if (predicate) predicates.push(predicate)
|
||||
try {
|
||||
const predicate = translateFilter(key, value)
|
||||
if (predicate) predicates.push(predicate)
|
||||
} catch {
|
||||
// analyticsLiteral rejected an unsupported input — drop the predicate.
|
||||
}
|
||||
})
|
||||
return predicates
|
||||
}
|
||||
|
||||
const whereClause = (predicates: string[]) =>
|
||||
predicates.length > 0 ? `WHERE ${predicates.join(' AND ')}` : ''
|
||||
const whereClause = (predicates: SafeLogSqlFragment[]): SafeLogSqlFragment =>
|
||||
predicates.length > 0 ? safeSql`WHERE ${joinSqlFragments(predicates, ' AND ')}` : safeSql``
|
||||
|
||||
/**
|
||||
* Calculates the chart bucketing level (minute/hour/day) given the date range.
|
||||
@@ -203,15 +221,15 @@ const calculateChartBucketing = (
|
||||
return 'MINUTE'
|
||||
}
|
||||
|
||||
const truncationFunction = (level: 'MINUTE' | 'HOUR' | 'DAY') => {
|
||||
const truncationFunction = (level: 'MINUTE' | 'HOUR' | 'DAY'): SafeLogSqlFragment => {
|
||||
switch (level) {
|
||||
case 'DAY':
|
||||
return 'toStartOfDay'
|
||||
return safeSql`toStartOfDay`
|
||||
case 'HOUR':
|
||||
return 'toStartOfHour'
|
||||
return safeSql`toStartOfHour`
|
||||
case 'MINUTE':
|
||||
default:
|
||||
return 'toStartOfMinute'
|
||||
return safeSql`toStartOfMinute`
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,7 +238,7 @@ const truncationFunction = (level: 'MINUTE' | 'HOUR' | 'DAY') => {
|
||||
* inlined so the result can be referenced (or filtered) at the same query
|
||||
* level — the OTEL endpoint rejects subqueries.
|
||||
*/
|
||||
const rowProjection = () => `
|
||||
const rowProjection = (): SafeLogSqlFragment => safeSql`
|
||||
id,
|
||||
null AS source_id,
|
||||
timestamp,
|
||||
@@ -234,9 +252,12 @@ const rowProjection = () => `
|
||||
null AS logs
|
||||
`
|
||||
|
||||
const buildBaseWhere = (search: QuerySearchParamsType, excludeField?: string) => {
|
||||
const buildBaseWhere = (
|
||||
search: QuerySearchParamsType,
|
||||
excludeField?: string
|
||||
): SafeLogSqlFragment[] => {
|
||||
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
|
||||
const parts: string[] = []
|
||||
const parts: SafeLogSqlFragment[] = []
|
||||
if (excludeField !== 'log_type') {
|
||||
parts.push(logTypeWherePredicate(effectiveLogTypes))
|
||||
}
|
||||
@@ -247,9 +268,9 @@ const buildBaseWhere = (search: QuerySearchParamsType, excludeField?: string) =>
|
||||
/**
|
||||
* Unified logs row query — flat SELECT, no subquery wrapper.
|
||||
*/
|
||||
export const getUnifiedLogsQuery = (search: QuerySearchParamsType): string => {
|
||||
export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
|
||||
const predicates = buildBaseWhere(search)
|
||||
return `
|
||||
return safeSql`
|
||||
SELECT ${rowProjection()}
|
||||
FROM logs
|
||||
${whereClause(predicates)}
|
||||
@@ -267,14 +288,14 @@ export const getFacetCountQuery = ({
|
||||
search: QuerySearchParamsType
|
||||
facet: string
|
||||
facetSearch?: string
|
||||
}) => {
|
||||
}): SafeLogSqlFragment => {
|
||||
if (!(FACET_FIELDS as readonly string[]).includes(facet)) {
|
||||
throw new Error('Invalid unified logs facet')
|
||||
}
|
||||
|
||||
const MAX_FACETS_QUANTITY = 20
|
||||
|
||||
const facetExpr =
|
||||
const facetExpr: SafeLogSqlFragment =
|
||||
facet === 'log_type'
|
||||
? LOG_TYPE_EXPR
|
||||
: facet === 'level'
|
||||
@@ -285,78 +306,89 @@ export const getFacetCountQuery = ({
|
||||
? STATUS_EXPR
|
||||
: facet === 'pathname'
|
||||
? ATTR.path
|
||||
: `log_attributes['${facet}']`
|
||||
: safeSql`log_attributes[${lit(facet)}]`
|
||||
|
||||
const predicates = [
|
||||
const predicates: SafeLogSqlFragment[] = [
|
||||
...buildBaseWhere(search, facet),
|
||||
`(${facetExpr}) IS NOT NULL AND (${facetExpr}) != ''`,
|
||||
safeSql`(${facetExpr}) IS NOT NULL AND (${facetExpr}) != ''`,
|
||||
]
|
||||
if (facetSearch) {
|
||||
predicates.push(`(${facetExpr}) LIKE '%${likeValue(facetSearch)}%'`)
|
||||
predicates.push(safeSql`(${facetExpr}) LIKE ${lit('%' + facetSearch + '%')}`)
|
||||
}
|
||||
|
||||
return `
|
||||
SELECT '${facet}' AS dimension, (${facetExpr}) AS value, count() AS count
|
||||
return safeSql`
|
||||
SELECT ${lit(facet)} AS dimension, (${facetExpr}) AS value, count() AS count
|
||||
FROM logs
|
||||
${whereClause(predicates)}
|
||||
GROUP BY value
|
||||
LIMIT ${MAX_FACETS_QUANTITY}
|
||||
`.trim()
|
||||
LIMIT ${lit(MAX_FACETS_QUANTITY)}
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Bundled count query — UNION ALL of (dimension, value, count) rows so the
|
||||
* frontend can render facet counts and total in one round trip.
|
||||
*/
|
||||
export const getLogsCountQuery = (search: QuerySearchParamsType): string => {
|
||||
const baseFiltersFor = (excludeField?: string) =>
|
||||
buildBaseWhere(search, excludeField).join(' AND ')
|
||||
export const getLogsCountQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
|
||||
// When no predicates remain, fall back to `1` so we emit a valid
|
||||
// tautology rather than a bare `WHERE`.
|
||||
const baseFiltersFor = (excludeField?: string): SafeLogSqlFragment => {
|
||||
const predicates = buildBaseWhere(search, excludeField)
|
||||
return predicates.length > 0 ? joinSqlFragments(predicates, ' AND ') : safeSql`1`
|
||||
}
|
||||
|
||||
// The "total" badge should reflect the user's *current* filter set,
|
||||
// including any active log_type filter. Pass no excludeField so the
|
||||
// log_type predicate is included.
|
||||
const totalSql = `
|
||||
const totalSql = safeSql`
|
||||
SELECT 'total' AS dimension, 'all' AS value, count() AS count
|
||||
FROM logs
|
||||
WHERE ${baseFiltersFor()}
|
||||
`.trim()
|
||||
`
|
||||
|
||||
const logTypeBranches = Object.entries(LOG_TYPE_PREDICATE)
|
||||
.map(([logType, predicate]) =>
|
||||
`
|
||||
SELECT 'log_type' AS dimension, '${logType}' AS value, countIf(${predicate}) AS count
|
||||
const logTypeBranches = joinSqlFragments(
|
||||
Object.entries(LOG_TYPE_PREDICATE).map(
|
||||
([logType, predicate]) =>
|
||||
safeSql`
|
||||
SELECT 'log_type' AS dimension, ${lit(logType)} AS value, countIf(${predicate}) AS count
|
||||
FROM logs
|
||||
WHERE ${baseFiltersFor('log_type')}
|
||||
`.trim()
|
||||
)
|
||||
.join('\nUNION ALL\n')
|
||||
`
|
||||
),
|
||||
' UNION ALL '
|
||||
)
|
||||
|
||||
const levelBranches = ['success', 'warning', 'error']
|
||||
.map((lvl) =>
|
||||
`
|
||||
SELECT 'level' AS dimension, '${lvl}' AS value, countIf((${LEVEL_EXPR}) = '${lvl}') AS count
|
||||
const levelBranches = joinSqlFragments(
|
||||
(['success', 'warning', 'error'] as const).map(
|
||||
(lvl) =>
|
||||
safeSql`
|
||||
SELECT 'level' AS dimension, ${lit(lvl)} AS value, countIf((${LEVEL_EXPR}) = ${lit(lvl)}) AS count
|
||||
FROM logs
|
||||
WHERE ${baseFiltersFor('level')}
|
||||
`.trim()
|
||||
)
|
||||
.join('\nUNION ALL\n')
|
||||
`
|
||||
),
|
||||
' UNION ALL '
|
||||
)
|
||||
|
||||
const facetBranches = ['method', 'status', 'pathname']
|
||||
.map((facet) => getFacetCountQuery({ search, facet }))
|
||||
.join('\nUNION ALL\n')
|
||||
const facetBranches = joinSqlFragments(
|
||||
(['method', 'status', 'pathname'] as const).map((facet) =>
|
||||
getFacetCountQuery({ search, facet })
|
||||
),
|
||||
' UNION ALL '
|
||||
)
|
||||
|
||||
return [totalSql, logTypeBranches, levelBranches, facetBranches].join('\nUNION ALL\n')
|
||||
return joinSqlFragments([totalSql, logTypeBranches, levelBranches, facetBranches], ' UNION ALL ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs chart query with dynamic bucketing based on time range.
|
||||
*/
|
||||
export const getLogsChartQuery = (search: QuerySearchParamsType): string => {
|
||||
export const getLogsChartQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
|
||||
const truncationLevel = calculateChartBucketing(search)
|
||||
const truncFn = truncationFunction(truncationLevel)
|
||||
const predicates = buildBaseWhere(search)
|
||||
|
||||
return `
|
||||
return safeSql`
|
||||
SELECT
|
||||
${truncFn}(timestamp) AS time_bucket,
|
||||
countIf((${LEVEL_EXPR}) = 'success') AS success,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// pg-meta's `literal()` and `ident()` are Postgres-specific: `literal()` emits
|
||||
// `E'…'` for backslash-bearing strings and `::jsonb` casts for objects;
|
||||
// `ident()` quotes identifiers with double-quotes, which BigQuery rejects
|
||||
// (double-quoted tokens are string literals there, not identifiers). We add
|
||||
// analytics-engine-specific helpers here rather than extend pg-meta, which
|
||||
// would cross-cut unrelated Postgres callers.
|
||||
//
|
||||
// The brand `SafeLogSqlFragment` is intentionally distinct from pg-meta's
|
||||
// `SafeSqlFragment`: escaping that is safe for Postgres (`E'…'` strings,
|
||||
// `::jsonb` casts, double-quoted identifiers) is not safe for BigQuery or
|
||||
// ClickHouse, and vice versa. Keeping the brands disjoint prevents a
|
||||
// Postgres-escaped fragment from being composed into an analytics query
|
||||
// (or vice versa) and silently emitting unsafe SQL.
|
||||
//
|
||||
// String literals: ClickHouse and BigQuery share the same convention —
|
||||
// double the single quote (`''`) and double the backslash (`\\`), inside
|
||||
// plain `'…'` delimiters.
|
||||
//
|
||||
// Identifiers: BigQuery requires backticks. ClickHouse accepts both
|
||||
// backticks and double-quotes; we use double-quotes (SQL-standard form).
|
||||
// In both engines a backslash inside a quoted identifier is an escape
|
||||
// character, so we reject any non-`[A-Za-z_][A-Za-z0-9_]*` input rather than
|
||||
// try to escape it — column names never need special characters in practice.
|
||||
|
||||
/**
|
||||
* A branded string type representing a SQL fragment that is safe to compose
|
||||
* into BigQuery or ClickHouse queries. Intentionally distinct from pg-meta's
|
||||
* `SafeSqlFragment` (Postgres-only).
|
||||
*
|
||||
* Values of this type are either:
|
||||
* - Static strings in source code (no interpolation) via `rawSql`
|
||||
* - Outputs of `analyticsLiteral`, `bqIdent`, or `clickhouseIdent`
|
||||
* - Compositions via the `safeSql` template tag (which only accepts
|
||||
* `SafeLogSqlFragment` interpolations)
|
||||
* - Compositions via `joinSqlFragments`
|
||||
*
|
||||
* Never cast arbitrary strings to this type.
|
||||
*/
|
||||
export type SafeLogSqlFragment = string & { readonly __safeLogSqlFragmentBrand: never }
|
||||
|
||||
export type LogSqlFragmentSeparator =
|
||||
| ','
|
||||
| ', '
|
||||
| ';\n'
|
||||
| ' and '
|
||||
| ' AND '
|
||||
| ' or '
|
||||
| ' OR '
|
||||
| ' union all '
|
||||
| ' union '
|
||||
| ' UNION ALL '
|
||||
| ' UNION '
|
||||
| '\n'
|
||||
| '\n\n'
|
||||
| ' '
|
||||
|
||||
/**
|
||||
* Tagged template literal for composing log-SQL fragments safely.
|
||||
* Only accepts `SafeLogSqlFragment` interpolations — plain strings and
|
||||
* Postgres-branded `SafeSqlFragment` values are rejected at compile time.
|
||||
*/
|
||||
export function safeSql(
|
||||
strings: TemplateStringsArray,
|
||||
...interpolated: Array<SafeLogSqlFragment>
|
||||
): SafeLogSqlFragment {
|
||||
return strings.reduce(
|
||||
(result, string, i) => result + string + (interpolated[i] ?? ''),
|
||||
''
|
||||
) as SafeLogSqlFragment
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a hand-written log-SQL string as a `SafeLogSqlFragment`. Use only
|
||||
* for static SQL authored in source code; never call with arbitrary input.
|
||||
*/
|
||||
export function rawSql(sql: string): SafeLogSqlFragment {
|
||||
return sql as SafeLogSqlFragment
|
||||
}
|
||||
|
||||
/** Joins already-safe log-SQL fragments with a fixed structural separator. */
|
||||
export function joinSqlFragments(
|
||||
fragments: Array<SafeLogSqlFragment>,
|
||||
separator: LogSqlFragmentSeparator
|
||||
): SafeLogSqlFragment {
|
||||
return fragments.join(separator) as SafeLogSqlFragment
|
||||
}
|
||||
|
||||
export function analyticsLiteral(value: string | number | boolean): SafeLogSqlFragment {
|
||||
if (typeof value === 'number') {
|
||||
if (!Number.isFinite(value)) {
|
||||
throw new Error('analyticsLiteral: non-finite numbers are not supported')
|
||||
}
|
||||
return rawSql(String(value))
|
||||
}
|
||||
if (typeof value === 'boolean') {
|
||||
return rawSql(value ? 'true' : 'false')
|
||||
}
|
||||
if (typeof value !== 'string') {
|
||||
throw new Error('analyticsLiteral: only string, number, or boolean inputs are supported')
|
||||
}
|
||||
let escaped = ''
|
||||
for (const c of value) {
|
||||
if (c === "'") escaped += "''"
|
||||
else if (c === '\\') escaped += '\\\\'
|
||||
else escaped += c
|
||||
}
|
||||
return rawSql(`'${escaped}'`)
|
||||
}
|
||||
|
||||
const SAFE_IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/
|
||||
|
||||
/** Quote an identifier for BigQuery using backticks. */
|
||||
export function bqIdent(value: string): SafeLogSqlFragment {
|
||||
if (typeof value !== 'string' || !SAFE_IDENT_RE.test(value)) {
|
||||
throw new Error(`bqIdent: invalid BigQuery identifier "${value}"`)
|
||||
}
|
||||
return rawSql('`' + value + '`')
|
||||
}
|
||||
|
||||
/** Quote an identifier for ClickHouse using double-quotes. */
|
||||
export function clickhouseIdent(value: string): SafeLogSqlFragment {
|
||||
if (typeof value !== 'string' || !SAFE_IDENT_RE.test(value)) {
|
||||
throw new Error(`clickhouseIdent: invalid ClickHouse identifier "${value}"`)
|
||||
}
|
||||
return rawSql('"' + value + '"')
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
import { literal, safeSql } from '@supabase/pg-meta/src/pg-format'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { useFlag } from 'common'
|
||||
|
||||
@@ -8,6 +7,7 @@ import {
|
||||
flattenOtelInspectionRow,
|
||||
type OtelLogRow,
|
||||
} from './otel-inspection.utils'
|
||||
import { analyticsLiteral as lit, safeSql } from './safe-analytics-sql'
|
||||
import {
|
||||
getUnifiedLogsISOStartEnd,
|
||||
UNIFIED_LOGS_QUERY_OPTIONS,
|
||||
@@ -23,7 +23,7 @@ import { QuerySearchParamsType } from '@/components/interfaces/UnifiedLogs/Unifi
|
||||
import { handleError, post } from '@/data/fetchers'
|
||||
import type { ResponseError, UseCustomQueryOptions } from '@/types'
|
||||
|
||||
// Service flow types — subset of LOG_TYPES that support service flows.
|
||||
// Service flow types - subset of LOG_TYPES that support service flows
|
||||
export const SERVICE_FLOW_TYPES = [
|
||||
'postgrest',
|
||||
'auth',
|
||||
@@ -184,17 +184,19 @@ export async function getUnifiedLogInspection(
|
||||
// so existing panel components that read `enrichedData['request.path']`
|
||||
// etc. keep working without per service flow SQL.
|
||||
//
|
||||
// `logId` ultimately originates from a URL query parameter, so we route
|
||||
// every interpolation through pg-meta's `literal()` / `safeSql` helpers.
|
||||
// The analytics endpoint uses ClickHouse SQL string literal escaping
|
||||
// rules (single quotes doubled), which matches Postgres and what
|
||||
// `literal()` produces.
|
||||
// logId comes from the row data we fetched (a uuid-shaped value), but it
|
||||
// ultimately originates from a URL query parameter. Reject anything that
|
||||
// isn't a plain uuid before interpolating it into SQL so a crafted id
|
||||
// can't break out of the string literal.
|
||||
if (!/^[0-9a-fA-F-]{1,64}$/.test(logId)) {
|
||||
throw new Error('Invalid logId')
|
||||
}
|
||||
const sql = safeSql`
|
||||
SELECT id, timestamp, source, event_message, severity_text, log_attributes
|
||||
FROM logs
|
||||
WHERE id = ${literal(logId)}
|
||||
WHERE id = ${lit(logId)}
|
||||
LIMIT 1
|
||||
`.trim()
|
||||
`
|
||||
|
||||
const { data, error } = await post('/platform/projects/{ref}/analytics/endpoints/logs.all.otel', {
|
||||
params: { path: { ref: projectRef } },
|
||||
@@ -216,21 +218,21 @@ LIMIT 1
|
||||
return { result: [] }
|
||||
}
|
||||
|
||||
const entry = flattenOtelInspectionRow(row)
|
||||
const entry = flattenOtelInspectionRow(row) as UnifiedLogInspectionEntry & Record<string, unknown>
|
||||
|
||||
// For edge function rows, fetch and aggregate the related `function_logs`
|
||||
// (the per-execution console.log output) — this is the only legitimate
|
||||
// cross-source join in the legacy BigQuery service-flow queries.
|
||||
if (row.source === 'function_edge_logs') {
|
||||
const executionId = row.log_attributes?.['execution_id'] ?? row.log_attributes?.['request_id']
|
||||
if (typeof executionId === 'string') {
|
||||
if (typeof executionId === 'string' && /^[0-9a-fA-F-]{1,64}$/.test(executionId)) {
|
||||
const fnSql = safeSql`
|
||||
SELECT id, timestamp, source, event_message, severity_text, log_attributes
|
||||
FROM logs
|
||||
WHERE source = 'function_logs' AND log_attributes['execution_id'] = ${literal(executionId)}
|
||||
WHERE source = 'function_logs' AND log_attributes['execution_id'] = ${lit(executionId)}
|
||||
ORDER BY timestamp ASC
|
||||
LIMIT 100
|
||||
`.trim()
|
||||
`
|
||||
|
||||
const { data: fnData, error: fnError } = await post(
|
||||
'/platform/projects/{ref}/analytics/endpoints/logs.all.otel',
|
||||
@@ -253,7 +255,7 @@ LIMIT 100
|
||||
}
|
||||
}
|
||||
|
||||
return { result: [entry] }
|
||||
return { result: [entry as UnifiedLogInspectionEntry] }
|
||||
}
|
||||
|
||||
export type UnifiedLogInspectionData = Awaited<ReturnType<typeof getUnifiedLogInspection>>
|
||||
@@ -266,7 +268,7 @@ export const useUnifiedLogInspectionQuery = <TData = UnifiedLogInspectionData>(
|
||||
...options
|
||||
}: UseCustomQueryOptions<UnifiedLogInspectionData, UnifiedLogInspectionError, TData> = {}
|
||||
) => {
|
||||
const useOtel = useFlag('otelUnifiedLogs')
|
||||
const useOtel = !!useFlag('otelUnifiedLogs')
|
||||
return useQuery<UnifiedLogInspectionData, UnifiedLogInspectionError, TData>({
|
||||
queryKey: [...logsKeys.serviceFlow(projectRef, search, logId), { otel: useOtel }],
|
||||
queryFn: ({ signal }) =>
|
||||
|
||||
@@ -3,6 +3,7 @@ import { useFlag } from 'common'
|
||||
|
||||
import { logsKeys } from './keys'
|
||||
import { logsAllEndpointUrl } from './logs-endpoint'
|
||||
import { bqIdent, safeSql } from './safe-analytics-sql'
|
||||
import {
|
||||
getUnifiedLogsISOStartEnd,
|
||||
UNIFIED_LOGS_QUERY_OPTIONS,
|
||||
@@ -35,11 +36,11 @@ export async function getUnifiedLogsFacetCount(
|
||||
const { isoTimestampStart, isoTimestampEnd } = getUnifiedLogsISOStartEnd(search)
|
||||
const sql = useOtel
|
||||
? getFacetCountQuery({ search, facet, facetSearch })
|
||||
: `
|
||||
: safeSql`
|
||||
${getUnifiedLogsCTE()},
|
||||
${getFacetCountCTE({ search, facet, facetSearch })}
|
||||
SELECT dimension, value, count from ${facet}_count;
|
||||
`.trim()
|
||||
SELECT dimension, value, count from ${bqIdent(facet + '_count')};
|
||||
`
|
||||
|
||||
const endpoint = logsAllEndpointUrl(useOtel)
|
||||
const { data, error } = await post(endpoint, {
|
||||
@@ -62,7 +63,7 @@ export const useUnifiedLogsFacetCountQuery = <TData = UnifiedLogsFacetCountData>
|
||||
...options
|
||||
}: UseCustomQueryOptions<UnifiedLogsFacetCountData, UnifiedLogsFacetCountError, TData> = {}
|
||||
) => {
|
||||
const useOtel = useFlag('otelUnifiedLogs')
|
||||
const useOtel = !!useFlag('otelUnifiedLogs')
|
||||
return useQuery<UnifiedLogsFacetCountData, UnifiedLogsFacetCountError, TData>({
|
||||
queryKey: [
|
||||
...logsKeys.unifiedLogsFacetCount(projectRef, facet, facetSearch, search),
|
||||
|
||||
Reference in new issue
Block a user