diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.constants.tsx b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.constants.tsx index ab206f8ddc0..47e7b5c2ab7 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.constants.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.constants.tsx @@ -9,8 +9,6 @@ import { } from 'nuqs' import { - ARRAY_DELIMITER, - LEVELS, RANGE_DELIMITER, SLIDER_DELIMITER, SORT_DELIMITER, @@ -33,20 +31,18 @@ const parseAsSort = createParser({ }) export const SEARCH_PARAMS_PARSER = { - // CUSTOM FILTERS - level: parseAsArrayOf(parseAsStringLiteral(LEVELS), ARRAY_DELIMITER), - log_type: parseAsArrayOf(parseAsString, ARRAY_DELIMITER), + // Equality filters. Repeatable `?filter=column:opAbbrev:value` (e.g. `level:eq:info`, + // `host:neq:foo`). Parsed with the helpers in `UnifiedLogs.filters.ts`. + filter: parseAsArrayOf(parseAsString), + + // Range/special filters keep their dedicated keys — their semantics are inherently + // multi-value (sliders, time ranges) and don't fit the eq/neq model. latency: parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER), 'timing.dns': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER), 'timing.connection': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER), 'timing.tls': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER), 'timing.ttfb': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER), 'timing.transfer': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER), - status: parseAsArrayOf(parseAsString, ARRAY_DELIMITER), - regions: parseAsArrayOf(parseAsStringLiteral(REGIONS), ARRAY_DELIMITER), - method: parseAsArrayOf(parseAsStringLiteral(METHODS), ARRAY_DELIMITER), - host: parseAsString, - pathname: parseAsString, date: parseAsArrayOf(parseAsTimestamp, RANGE_DELIMITER), // REQUIRED FOR SORTING & PAGINATION diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.filters.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.filters.ts new file mode 100644 index 00000000000..e327798e678 --- /dev/null +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.filters.ts @@ -0,0 +1,95 @@ +// Operators mirror PostgREST / Table Editor conventions: +// `=` / `<>` — exact equality, available on most columns +// `~~*` / `!~~*` — ILIKE / NOT ILIKE (case-insensitive substring +// contains / does-not-contain), wired up only for the +// `event_message` column +export type LogsFilterOperator = '=' | '<>' | '~~*' | '!~~*' + +export interface LogsFilter { + column: string + operator: LogsFilterOperator + value: string +} + +export interface LogsColumnFilterValue { + operator: LogsFilterOperator + values: string[] +} + +export const LOGS_FILTER_OPERATORS = [ + '=', + '<>', + '~~*', + '!~~*', +] as const satisfies readonly LogsFilterOperator[] + +const OPERATOR_TO_ABBREV: Record = { + '=': 'eq', + '<>': 'neq', + '~~*': 'ilike', + '!~~*': 'notilike', +} + +const ABBREV_TO_OPERATOR: Record = { + eq: '=', + neq: '<>', + ilike: '~~*', + notilike: '!~~*', +} + +export const isLogsFilterColumnValue = (value: unknown): value is LogsColumnFilterValue => { + return ( + typeof value === 'object' && + value !== null && + 'operator' in value && + 'values' in value && + Array.isArray((value as LogsColumnFilterValue).values) + ) +} + +export const parseLogsFilterUrlParams = (filter?: string[] | null): LogsFilter[] => { + if (!Array.isArray(filter)) return [] + const parsed: LogsFilter[] = [] + for (const raw of filter) { + const [column, abbrev, ...rest] = raw.split(':') + const operator = ABBREV_TO_OPERATOR[abbrev] + if (!column || !operator) continue + parsed.push({ column, operator, value: rest.join(':') }) + } + return parsed +} + +export const logsFiltersToUrlParams = (filters: LogsFilter[]): string[] => { + return filters.map((f) => `${f.column}:${OPERATOR_TO_ABBREV[f.operator]}:${f.value}`) +} + +export const groupLogsFiltersByColumn = ( + filters: LogsFilter[] +): Record => { + const grouped: Record = {} + for (const { column, operator, value } of filters) { + const existing = grouped[column] + if (!existing) { + grouped[column] = { operator, values: [value] } + } else { + existing.values.push(value) + // Mixed operators on the same column aren't expressible in the column-filter + // shape (one operator per column). Last write wins. + if (existing.operator !== operator) existing.operator = operator + } + } + return grouped +} + +export const columnFiltersToLogsFilters = ( + columnFilters: { id: string; value: unknown }[] +): LogsFilter[] => { + const filters: LogsFilter[] = [] + for (const { id, value } of columnFilters) { + if (!isLogsFilterColumnValue(value)) continue + for (const v of value.values) { + filters.push({ column: id, operator: value.operator, value: String(v) }) + } + } + return filters +} diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts index 1351862eab9..120d5861f11 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts @@ -7,6 +7,7 @@ import dayjs from 'dayjs' import { DEFAULT_LOG_TYPES } from './UnifiedLogs.constants' +import { groupLogsFiltersByColumn, parseLogsFilterUrlParams } from './UnifiedLogs.filters' import { QuerySearchParamsType, SearchParamsType } from './UnifiedLogs.types' import { joinSqlFragments, @@ -16,23 +17,37 @@ import { type SafeLogSqlFragment, } from '@/data/logs/safe-analytics-sql' -// Pagination and control parameters -const PAGINATION_PARAMS = ['sort', 'start', 'size', 'uuid', 'cursor', 'direction', 'live'] as const +// Operator fragments for SQL emission. `safeSql` rejects plain strings, so we +// pre-brand the keywords we want to switch between. +const IN_OP = safeSql`IN` +const NOT_IN_OP = safeSql`NOT IN` +const LIKE_OP = safeSql`LIKE` +const NOT_LIKE_OP = safeSql`NOT LIKE` -// Special filter parameters that need custom handling -const SPECIAL_FILTER_PARAMS = ['date'] as const - -// Combined list of all parameters to exclude from standard filtering -const EXCLUDED_QUERY_PARAMS = [...PAGINATION_PARAMS, ...SPECIAL_FILTER_PARAMS] as const +const ALL_LOG_TYPES = ['edge', 'postgrest', 'storage', 'postgres', 'edge function', 'auth'] /** - * Builds WHERE-clause fragments from a search-param map. Identifier-position - * keys are validated via `quotedIdent()` (regex allowlist) and value-position - * inputs via `analyticsLiteral` — both throw on disallowed input, in which - * case we drop the predicate rather than emit unsafe SQL. + * Computes the log_type set that the CTE should union. With no filter, defaults + * to the cheap two-source set. With `=` filters, narrows to those values. With + * `<>` filters, excludes them from the full set. + */ +const getEffectiveLogTypes = (search: QuerySearchParamsType): string[] => { + const filters = parseLogsFilterUrlParams(search.filter).filter((f) => f.column === 'log_type') + if (filters.length === 0) return [...DEFAULT_LOG_TYPES] + const included = filters.filter((f) => f.operator === '=').map((f) => f.value) + const excluded = new Set(filters.filter((f) => f.operator === '<>').map((f) => f.value)) + const base = included.length > 0 ? included : ALL_LOG_TYPES + return base.filter((t) => !excluded.has(t)) +} + +/** + * Builds WHERE-clause fragments from the parsed `filter` URL array. Identifier- + * position keys are validated via `quotedIdent()` (regex allowlist) and value- + * position inputs via `analyticsLiteral` — both throw on disallowed input, in + * which case we drop the predicate rather than emit unsafe SQL. * * @param search Search params (URL-derived filter values) - * @param excludeKey Optional key to skip — used by facet-count branches that + * @param excludeKey Optional column to skip — used by facet-count branches that * need every filter applied *except* the one being faceted * @returns Array of SafeLogSqlFragment predicates ready to be AND-joined */ @@ -41,10 +56,11 @@ const buildConditions = ( excludeKey?: string ): SafeLogSqlFragment[] => { const conditions: SafeLogSqlFragment[] = [] + const grouped = groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter)) - Object.entries(search).forEach(([key, value]) => { - if (key === excludeKey) return - if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) return + for (const [key, { operator, values }] of Object.entries(grouped)) { + if (key === excludeKey) continue + if (values.length === 0) continue try { // `key` is interpolated as a column identifier. `quotedIdent()` rejects @@ -52,27 +68,37 @@ const buildConditions = ( // crafted URL key like `level OR id IS NOT NULL` is dropped rather // than emitted into the WHERE clause). const col = quotedIdent(key) + const isNeq = operator === '<>' + const inOp = isNeq ? NOT_IN_OP : IN_OP + const likeOp = isNeq ? NOT_LIKE_OP : LIKE_OP + const joinAndOr = isNeq ? ' AND ' : ' OR ' - if (Array.isArray(value) && value.length > 0) { + if (key === 'event_message' && (operator === '~~*' || operator === '!~~*')) { + // BigQuery has no ILIKE; emulate via LOWER(col) (NOT) LIKE LOWER('%v%'). + // Auto-wrap with `%…%` unless the user already included one. Multiple + // ILIKE values join with OR; NOT ILIKE joins with AND (row must contain + // none of the given substrings). + const pattern = (v: string) => (v.includes('%') ? v : '%' + v + '%') + const likeKeyword = operator === '!~~*' ? safeSql`NOT LIKE` : safeSql`LIKE` + const join = operator === '!~~*' ? ' AND ' : ' OR ' + const branches = values.map( + (v) => safeSql`LOWER(${col}) ${likeKeyword} LOWER(${lit(pattern(v))})` + ) + conditions.push(safeSql`(${joinSqlFragments(branches, join)})`) + } else if (key === 'host' || key === 'pathname') { + const branches = values.map((v) => safeSql`${col} ${likeOp} ${lit('%' + v + '%')}`) + conditions.push(safeSql`(${joinSqlFragments(branches, joinAndOr)})`) + } else { const inList = joinSqlFragments( - value.map((v) => lit(String(v))), + values.map((v) => lit(v)), ',' ) - conditions.push(safeSql`${col} IN (${inList})`) - return - } - - if (value !== null && value !== undefined) { - if (key === 'host' || key === 'pathname') { - conditions.push(safeSql`${col} LIKE ${lit('%' + String(value) + '%')}`) - } else { - conditions.push(safeSql`${col} = ${lit(String(value))}`) - } + conditions.push(safeSql`${col} ${inOp} (${inList})`) } } catch { // quotedIdent() or analyticsLiteral() rejected the input — drop the predicate. } - }) + } return conditions } @@ -360,7 +386,7 @@ WITH unified_logs AS ( */ export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => { const conditions = buildConditions(search) - const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES] + const effectiveLogTypes = getEffectiveLogTypes(search) return safeSql` ${getUnifiedLogsCTE(effectiveLogTypes)} @@ -513,7 +539,7 @@ WITH unified_logs AS ( ` export const getLogsCountQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => { - const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES] + const effectiveLogTypes = getEffectiveLogTypes(search) const logTypeConditions = buildConditions(search, 'log_type') const levelConditions = buildConditions(search, 'level') const logTypeWhere: SafeLogSqlFragment = @@ -580,7 +606,7 @@ UNION ALL SELECT dimension, value, count FROM pathname_count export const getLogsChartQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => { const conditions = buildConditions(search) const truncationLevel = calculateChartBucketing(search) - const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES] + const effectiveLogTypes = getEffectiveLogTypes(search) return safeSql` ${getUnifiedLogsCTE(effectiveLogTypes)} diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts index 03088995abc..a8bea182e51 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts @@ -12,6 +12,9 @@ const baseSearch = { date: [new Date('2026-05-08T09:00:00Z'), new Date('2026-05-08T10:00:00Z')], } as any +// Helper: build a search with extra `filter` URL entries on top of the base. +const withFilters = (...entries: string[]) => ({ ...baseSearch, filter: entries }) as any + describe('UnifiedLogs.queries (OTEL flat)', () => { describe('getUnifiedLogsQuery', () => { it('defaults to postgres + postgrest log types when none specified', () => { @@ -24,7 +27,7 @@ describe('UnifiedLogs.queries (OTEL flat)', () => { }) it('routes the `edge` log type to edge_logs without /rest/ or /storage/ paths', () => { - const sql = getUnifiedLogsQuery({ ...baseSearch, log_type: ['edge'] } as any) + const sql = getUnifiedLogsQuery(withFilters('log_type:eq:edge')) expect(sql).toContain(`NOT LIKE '%/rest/%'`) expect(sql).toContain(`NOT LIKE '%/storage/%'`) const where = sql.split(/\bWHERE\b/)[1] ?? '' @@ -32,18 +35,16 @@ describe('UnifiedLogs.queries (OTEL flat)', () => { }) it('routes the `storage` log type to edge_logs filtered by /storage/', () => { - const sql = getUnifiedLogsQuery({ ...baseSearch, log_type: ['storage'] } as any) + const sql = getUnifiedLogsQuery(withFilters('log_type:eq:storage')) expect(sql).toContain( `source = 'edge_logs' AND log_attributes['request.path'] LIKE '%/storage/%'` ) }) it('escapes single quotes in filter values to prevent SQL injection', () => { - const sql = getUnifiedLogsQuery({ - ...baseSearch, - method: [`G'ET`], - pathname: `/customers'; DROP TABLE logs --`, - } as any) + const sql = getUnifiedLogsQuery( + withFilters(`method:eq:G'ET`, `pathname:eq:/customers'; DROP TABLE logs --`) + ) // Single quotes are doubled (SQL-standard escaping) by pg-meta's // literal(); a raw single quote from user input never closes its // string literal early. @@ -52,12 +53,9 @@ describe('UnifiedLogs.queries (OTEL flat)', () => { }) it('translates method/status/pathname filters to log_attributes predicates', () => { - const sql = getUnifiedLogsQuery({ - ...baseSearch, - method: ['GET'], - status: ['401'], - pathname: '/customers', - } as any) + const sql = getUnifiedLogsQuery( + withFilters('method:eq:GET', 'status:eq:401', 'pathname:eq:/customers') + ) expect(sql).toContain(`log_attributes['request.method'] IN ('GET')`) // Status filter wraps the CASE that picks HTTP code or Postgres SQLSTATE // so e.g. '00000' matches postgres success rows. @@ -66,6 +64,43 @@ describe('UnifiedLogs.queries (OTEL flat)', () => { expect(sql).toContain(`log_attributes['request.path'] LIKE '%/customers%'`) }) + it('flips IN to NOT IN when the operator is `<>`', () => { + const sql = getUnifiedLogsQuery(withFilters('method:neq:GET', 'status:neq:401')) + expect(sql).toContain(`log_attributes['request.method'] NOT IN ('GET')`) + expect(sql).toMatch(/END\) NOT IN \('401'\)/) + }) + + it('flips LIKE to NOT LIKE when the pathname/host operator is `<>`', () => { + const sql = getUnifiedLogsQuery(withFilters('pathname:neq:/health', 'host:neq:cdn.foo')) + expect(sql).toContain(`log_attributes['request.path'] NOT LIKE '%/health%'`) + expect(sql).toContain(`log_attributes['request.url'] NOT LIKE '%cdn.foo%'`) + }) + + it('emits ILIKE with auto-wrapped `%…%` for event_message `~~*`', () => { + const sql = getUnifiedLogsQuery(withFilters('event_message:ilike:Permission Denied')) + expect(sql).toContain(`event_message ILIKE '%Permission Denied%'`) + }) + + it('emits NOT ILIKE for event_message `!~~*` so rows containing the term are excluded', () => { + const sql = getUnifiedLogsQuery(withFilters('event_message:notilike:cron')) + expect(sql).toContain(`event_message NOT ILIKE '%cron%'`) + }) + + it('joins multiple NOT ILIKE values with AND (row must contain none)', () => { + const sql = getUnifiedLogsQuery( + withFilters('event_message:notilike:cron', 'event_message:notilike:heartbeat') + ) + expect(sql).toMatch( + /event_message NOT ILIKE '%cron%' AND event_message NOT ILIKE '%heartbeat%'/ + ) + }) + + it('passes through user-supplied `%` wildcards on event_message ILIKE without double-wrapping', () => { + const sql = getUnifiedLogsQuery(withFilters('event_message:ilike:error%')) + expect(sql).toContain(`event_message ILIKE 'error%'`) + expect(sql).not.toContain(`'%error%%'`) + }) + it('excludes connection log messages by default (hide_connection_logs=true)', () => { const sql = getUnifiedLogsQuery({ ...baseSearch, hide_connection_logs: true } as any) expect(sql).toContain("source != 'postgres_logs'") @@ -104,7 +139,7 @@ describe('UnifiedLogs.queries (OTEL flat)', () => { }) it('honours an active log_type filter in the total count branch', () => { - const sql = getLogsCountQuery({ ...baseSearch, log_type: ['edge'] } as any) + const sql = getLogsCountQuery(withFilters('log_type:eq:edge')) // The first branch is the total — its WHERE must include the edge // log_type predicate, otherwise the total badge would over-count // when a log_type filter is active. @@ -141,7 +176,7 @@ describe('UnifiedLogs.queries (OTEL flat)', () => { describe('getFacetCountQuery', () => { it('groups by the requested facet and excludes that facet from the WHERE filters', () => { const sql = getFacetCountQuery({ - search: { ...baseSearch, method: ['GET'], status: ['200'] } as any, + search: withFilters('method:eq:GET', 'status:eq:200'), facet: 'method', }) // Filtered facet is excluded from WHERE; other filters still applied. @@ -160,46 +195,56 @@ describe('UnifiedLogs.queries (OTEL flat)', () => { // pg-meta's literal() would emit `E'a\\b'` for `a\b` — the `E` prefix is // Postgres-only and rejected by both analytics engines. analyticsLiteral // doubles the backslash inside plain `'…'` delimiters instead. - const sql = getUnifiedLogsQuery({ ...baseSearch, method: 'a\\b' } as any) - expect(sql).toContain(`log_attributes['request.method'] = 'a\\\\b'`) + const sql = getUnifiedLogsQuery(withFilters('method:eq:a\\b')) + expect(sql).toContain(`log_attributes['request.method'] IN ('a\\\\b')`) expect(sql).not.toContain(`E'a`) }) it("escapes single quotes by doubling them ('' rather than \\')", () => { - const sql = getUnifiedLogsQuery({ ...baseSearch, method: "GET' OR '1'='1" } as any) - expect(sql).toContain(`log_attributes['request.method'] = 'GET'' OR ''1''=''1'`) + const sql = getUnifiedLogsQuery(withFilters("method:eq:GET' OR '1'='1")) + expect(sql).toContain(`log_attributes['request.method'] IN ('GET'' OR ''1''=''1')`) }) }) }) describe('UnifiedLogs.queries.bq', () => { it('backtick-quotes column identifiers (BigQuery syntax)', () => { - const sql = getUnifiedLogsQueryBQ({ ...baseSearch, method: 'GET' } as any) - expect(sql).toContain("`method` = 'GET'") + const sql = getUnifiedLogsQueryBQ(withFilters('method:eq:GET')) + expect(sql).toContain("`method` IN ('GET')") }) it('rejects keys with non-identifier characters', () => { - // A key like "foo; DROP TABLE" fails the quotedIdent regex (the `;` is not + // A column like "foo; DROP TABLE x" fails the quotedIdent regex (the `;` is not // in `[A-Za-z_][A-Za-z0-9_]*`), so the predicate is dropped entirely. - const sql = getUnifiedLogsQueryBQ({ ...baseSearch, 'foo; DROP TABLE x': 'y' } as any) + const sql = getUnifiedLogsQueryBQ(withFilters('foo; DROP TABLE x:eq:y')) expect(sql).not.toContain('DROP TABLE') expect(sql).not.toContain('foo;') }) it('rejects keys containing spaces', () => { - const sql = getUnifiedLogsQueryBQ({ - ...baseSearch, - 'level OR id IS NOT NULL': 'anything', - } as any) + const sql = getUnifiedLogsQueryBQ(withFilters('level OR id IS NOT NULL:eq:anything')) expect(sql).not.toContain('IS NOT NULL') expect(sql).not.toContain('level OR') }) it('escapes injection attempts in filter values via analyticsLiteral', () => { - const sql = getUnifiedLogsQueryBQ({ ...baseSearch, method: "GET' OR '1'='1" } as any) + const sql = getUnifiedLogsQueryBQ(withFilters("method:eq:GET' OR '1'='1")) // The value is single-quote-escaped, so the synthetic OR can't break out // of the string literal. - expect(sql).toContain("`method` = 'GET'' OR ''1''=''1'") - expect(sql).not.toMatch(/`method` = 'GET' OR '1'='1'/) + expect(sql).toContain("`method` IN ('GET'' OR ''1''=''1')") + expect(sql).not.toMatch(/`method` IN \('GET' OR '1'='1'\)/) + }) + + it('emulates ILIKE with LOWER()/LOWER() since BigQuery has no ILIKE keyword', () => { + const sql = getUnifiedLogsQueryBQ(withFilters('event_message:ilike:Permission Denied')) + expect(sql).toContain("LOWER(`event_message`) LIKE LOWER('%Permission Denied%')") + // Sanity check: never emit a raw ILIKE keyword for BQ. + expect(sql).not.toMatch(/\bILIKE\b/) + }) + + it('emulates NOT ILIKE with LOWER()/NOT LIKE/LOWER() for event_message `!~~*`', () => { + const sql = getUnifiedLogsQueryBQ(withFilters('event_message:notilike:cron')) + expect(sql).toContain("LOWER(`event_message`) NOT LIKE LOWER('%cron%')") + expect(sql).not.toMatch(/\bILIKE\b/) }) }) diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts index 4252369ce39..5d111ac7ec2 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts @@ -1,6 +1,11 @@ import dayjs from 'dayjs' import { DEFAULT_LOG_TYPES } from './UnifiedLogs.constants' +import { + groupLogsFiltersByColumn, + parseLogsFilterUrlParams, + type LogsFilterOperator, +} from './UnifiedLogs.filters' import { QuerySearchParamsType, SearchParamsType } from './UnifiedLogs.types' import { joinSqlFragments, @@ -9,14 +14,14 @@ import { type SafeLogSqlFragment, } from '@/data/logs/safe-analytics-sql' -// Pagination and control parameters -const PAGINATION_PARAMS = ['sort', 'start', 'size', 'uuid', 'cursor', 'direction', 'live'] as const - -// Special filter parameters that need custom handling -const SPECIAL_FILTER_PARAMS = ['date', 'hide_connection_logs'] as const - -// Combined list of all parameters to exclude from standard filtering -const EXCLUDED_QUERY_PARAMS = [...PAGINATION_PARAMS, ...SPECIAL_FILTER_PARAMS] as const +// Operator fragments for SQL emission. `safeSql` rejects plain strings, so we +// pre-brand the keywords we want to switch between. +const IN_OP = safeSql`IN` +const NOT_IN_OP = safeSql`NOT IN` +const LIKE_OP = safeSql`LIKE` +const NOT_LIKE_OP = safeSql`NOT LIKE` +const ILIKE_OP = safeSql`ILIKE` +const NOT_ILIKE_OP = safeSql`NOT ILIKE` // Facets the count query is allowed to be invoked for. Reject anything else // at the entry point rather than letting an unsupported value reach @@ -94,94 +99,83 @@ const logTypeWherePredicate = (logTypes: string[]): SafeLogSqlFragment => { } /** - * Translates a frontend filter key/value pair into an underlying SQL predicate. - * The OTEL endpoint won't accept queries that reference derived aliases like - * `log_type` or `level` in WHERE for some shapes, so we always emit raw-column - * predicates (source/severity_text/log_attributes[…]). + * Translates one (column, values, operator) group from the parsed `filter` URL + * param into an underlying SQL predicate. The OTEL endpoint rejects queries + * that reference derived aliases like `log_type` or `level` in WHERE for some + * shapes, so we always emit raw-column predicates (source/severity_text/ + * log_attributes[…]). For `<>`, IN becomes NOT IN, LIKE becomes NOT LIKE, and + * multi-value lists are joined with AND so the row must not match *any* value. */ -const translateFilter = (key: string, value: unknown): SafeLogSqlFragment | null => { - if (value === null || value === undefined) return null +const translateFilter = ( + key: string, + values: readonly string[], + operator: LogsFilterOperator +): SafeLogSqlFragment | null => { + if (values.length === 0) return null - const arr = Array.isArray(value) ? (value.length > 0 ? value : null) : null - if (Array.isArray(value) && !arr) return null + const isNeq = operator === '<>' + const inOp = isNeq ? NOT_IN_OP : IN_OP + const likeOp = isNeq ? NOT_LIKE_OP : LIKE_OP + const joinAndOr = isNeq ? ' AND ' : ' OR ' - const inList = (values: readonly unknown[]): SafeLogSqlFragment => + const inList = (vals: readonly string[]): SafeLogSqlFragment => safeSql`(${joinSqlFragments( - values.map((v) => lit(String(v))), + vals.map((v) => lit(v)), ',' )})` switch (key) { case 'log_type': { - const types = (arr ?? [value]).map((v) => String(v)) - const branches = types.map( - (t) => safeSql`(${LOG_TYPE_PREDICATE[t] ?? safeSql`source = ${lit(t)}`})` - ) - return safeSql`(${joinSqlFragments(branches, ' OR ')})` + const branches = values.map((t) => { + const pred = LOG_TYPE_PREDICATE[t] ?? safeSql`source = ${lit(t)}` + return isNeq ? safeSql`NOT (${pred})` : safeSql`(${pred})` + }) + return safeSql`(${joinSqlFragments(branches, joinAndOr)})` } - case 'level': { + case 'level': // No simple raw column for level; reference the inline CASE expression. - const levels = arr ?? [value] - return safeSql`(${LEVEL_EXPR}) IN ${inList(levels.map((v) => String(v)))}` - } + return safeSql`(${LEVEL_EXPR}) ${inOp} ${inList(values)}` case 'method': - return arr - ? safeSql`${ATTR.method} IN ${inList(arr)}` - : safeSql`${ATTR.method} = ${lit(String(value))}` - case 'status': { + return safeSql`${ATTR.method} ${inOp} ${inList(values)}` + case 'status': // Match the displayed status: HTTP response code for gateway rows, // Postgres SQLSTATE for postgres rows. Inline STATUS_EXPR so e.g. // filtering on '00000' picks up postgres success rows. - const statuses = arr ?? [value] - return safeSql`(${STATUS_EXPR}) IN ${inList(statuses.map((v) => String(v)))}` - } + return safeSql`(${STATUS_EXPR}) ${inOp} ${inList(values)}` case 'pathname': - return arr - ? safeSql`(${joinSqlFragments( - arr.map((v) => safeSql`${ATTR.path} LIKE ${lit('%' + String(v) + '%')}`), - ' OR ' - )})` - : safeSql`${ATTR.path} LIKE ${lit('%' + String(value) + '%')}` + return safeSql`(${joinSqlFragments( + values.map((v) => safeSql`${ATTR.path} ${likeOp} ${lit('%' + v + '%')}`), + joinAndOr + )})` case 'host': // Best-effort: use full request URL since `host` isn't a top-level field. - return arr - ? safeSql`(${joinSqlFragments( - arr.map( - (v) => safeSql`log_attributes['request.url'] LIKE ${lit('%' + String(v) + '%')}` - ), - ' OR ' - )})` - : safeSql`log_attributes['request.url'] LIKE ${lit('%' + String(value) + '%')}` - default: - // Unknown filter key — fall back to a generic equality on log_attributes. - return arr - ? safeSql`log_attributes[${lit(key)}] IN ${inList(arr)}` - : safeSql`log_attributes[${lit(key)}] = ${lit(String(value))}` - } -} - -/** - * Builds an array of WHERE predicate fragments from search params, optionally - * skipping a specific facet field (used when computing faceted counts). - * `log_type` is always handled separately (see `logTypeWherePredicate`). - */ -const buildPredicates = ( - search: QuerySearchParamsType, - excludeField?: string -): SafeLogSqlFragment[] => { - const predicates: SafeLogSqlFragment[] = [] - Object.entries(search).forEach(([key, value]) => { - if (key === excludeField) return - if (key === 'log_type') return - if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) return - try { - const predicate = translateFilter(key, value) - if (predicate) predicates.push(predicate) - } catch { - // analyticsLiteral rejected an unsupported input — drop the predicate. + return safeSql`(${joinSqlFragments( + values.map((v) => safeSql`log_attributes['request.url'] ${likeOp} ${lit('%' + v + '%')}`), + joinAndOr + )})` + case 'event_message': { + // event_message is a top-level column, not a log_attributes key. ILIKE/NOT ILIKE + // auto-wrap with `%…%` so the user can type "permission denied" as a substring + // search; explicit `%`s in the input are passed through unchanged. Multiple + // ILIKE values join with OR (match any); multiple NOT ILIKE values join with AND + // (the row must contain none of them). + if (operator === '~~*' || operator === '!~~*') { + const op = operator === '~~*' ? ILIKE_OP : NOT_ILIKE_OP + const join = operator === '!~~*' ? ' AND ' : ' OR ' + const pattern = (v: string) => (v.includes('%') ? v : '%' + v + '%') + return safeSql`(${joinSqlFragments( + values.map((v) => safeSql`event_message ${op} ${lit(pattern(v))}`), + join + )})` + } + // = / <> still emit exact match against the column — defensive; the + // FilterBar doesn't expose them for event_message, but a hand-crafted URL + // might. + return safeSql`event_message ${inOp} ${inList(values)}` } - }) - return predicates + default: + return safeSql`log_attributes[${lit(key)}] ${inOp} ${inList(values)}` + } } const whereClause = (predicates: SafeLogSqlFragment[]): SafeLogSqlFragment => @@ -259,12 +253,30 @@ const buildBaseWhere = ( search: QuerySearchParamsType, excludeField?: string ): SafeLogSqlFragment[] => { - const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES] + const grouped = groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter)) const parts: SafeLogSqlFragment[] = [] + if (excludeField !== 'log_type') { - parts.push(logTypeWherePredicate(effectiveLogTypes)) + const logTypeFilter = grouped.log_type + if (logTypeFilter) { + const pred = translateFilter('log_type', logTypeFilter.values, logTypeFilter.operator) + if (pred) parts.push(pred) + } else { + parts.push(logTypeWherePredicate([...DEFAULT_LOG_TYPES])) + } } - parts.push(...buildPredicates(search, excludeField)) + + for (const [key, { operator, values }] of Object.entries(grouped)) { + if (key === excludeField) continue + if (key === 'log_type') continue // handled above + try { + const predicate = translateFilter(key, values, operator) + if (predicate) parts.push(predicate) + } catch { + // analyticsLiteral rejected an unsupported input — drop the predicate. + } + } + return parts } diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx index 0c994a427d1..18096f8d8f6 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx @@ -38,6 +38,12 @@ import { RowSelectionHeader } from './RowSelectionHeader' import { ServiceFlowPanel } from './ServiceFlowPanel' import { SEARCH_PARAMS_PARSER } from './UnifiedLogs.constants' import { filterFields as defaultFilterFields } from './UnifiedLogs.fields' +import { + columnFiltersToLogsFilters, + groupLogsFiltersByColumn, + logsFiltersToUrlParams, + parseLogsFilterUrlParams, +} from './UnifiedLogs.filters' import { useLiveMode, useResetFocus } from './UnifiedLogs.hooks' import { ColumnSchema } from './UnifiedLogs.schema' import { QuerySearchParamsType } from './UnifiedLogs.types' @@ -84,22 +90,14 @@ export const UnifiedLogs = () => { const track = useTrack() const [search, setSearch] = useQueryStates(SEARCH_PARAMS_PARSER) - const { - sort, - start, - size, - id, - cursor, - direction, - live, - hide_connection_logs: _hideConnectionLogs, - ...filter - } = search - const defaultColumnSorting = sort ? [sort] : [] + const defaultColumnSorting = search.sort ? [search.sort] : [] const defaultColumnVisibility = { uuid: false } - const defaultColumnFilters = Object.entries(filter) - .map(([key, value]) => ({ id: key, value })) - .filter(({ value }) => value ?? undefined) + // Column filters are seeded from the repeatable `filter` URL param. Each entry + // (`col:opAbbrev:value`) is grouped per column into a wrapped { operator, values } + // shape that the query builder reads back. + const defaultColumnFilters = Object.entries( + groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter)) + ).map(([id, value]) => ({ id, value })) const [topBarHeight, setTopBarHeight] = useState(0) const topBarRef = useRef(null) @@ -211,15 +209,17 @@ export const UnifiedLogs = () => { const facets = counts?.facets const totalFetched = flatData?.length - // Create a filtered version of the chart config based on selected levels + // Create a filtered version of the chart config based on level filters in the URL. const filteredChartConfig = useMemo(() => { - const levelFilter = search.level || LEVELS + const levelFilters = parseLogsFilterUrlParams(search.filter).filter((f) => f.column === 'level') + const included = levelFilters.filter((f) => f.operator === '=').map((f) => f.value) + const excluded = new Set(levelFilters.filter((f) => f.operator === '<>').map((f) => f.value)) + const baseLevels: readonly string[] = included.length > 0 ? included : LEVELS + const activeLevels = baseLevels.filter((l) => !excluded.has(l)) return Object.fromEntries( - Object.entries(CHART_CONFIG).filter(([key]) => - levelFilter.includes(key as (typeof LEVELS)[number]) - ) + Object.entries(CHART_CONFIG).filter(([key]) => activeLevels.includes(key)) ) as ChartConfig - }, [search.level]) + }, [search.filter]) const getRowClassName = < TData extends { date: Date; level: (typeof LEVELS)[number]; timestamp: number }, @@ -270,10 +270,6 @@ export const UnifiedLogs = () => { return table.getCoreRowModel().flatRows.find((row) => row.id === openRowId) }, [isLoading, isFetching, flatData.length, table, openRowId]) - // REMINDER: this is currently needed for the cmdk search - // [Joshen] This is where facets are getting dynamically loaded - // TODO: auto search via API when the user changes the filter instead of hardcoded - // Will need to refactor this bit // - Each facet just handles its own state, rather than getting passed down like this const filterFields = useMemo(() => { @@ -298,24 +294,21 @@ export const UnifiedLogs = () => { }) }, [facets]) - // Debounced filter application to avoid too many API calls when user clicks multiple filters quickly + // Debounced filter application to avoid too many API calls when user clicks multiple filters quickly. + // All equality/pattern column filters serialize into the repeatable `filter` URL param. Slider/timerange + // column filters keep their dedicated per-column URL keys so things like the timeline brush still + // round-trip — they have their own range semantics and aren't covered by eq/neq/like. const applyFilterSearch = () => { - const columnFiltersWithNullable = filterFields.map((field) => { - const filterValue = columnFilters.find((filter) => filter.id === field.value) - if (!filterValue) return { id: field.value, value: null } - return { id: field.value, value: filterValue.value } - }) - - const search = columnFiltersWithNullable.reduce( - (prev, curr) => { - // Add to search parameters - prev[curr.id as string] = curr.value - return prev - }, - {} as Record - ) - - setSearch(search) + const filterEntries = logsFiltersToUrlParams(columnFiltersToLogsFilters(columnFilters)) + const update: Record = { + filter: filterEntries.length > 0 ? filterEntries : null, + } + for (const field of filterFields) { + if (field.type !== 'timerange') continue + const current = columnFilters.find((c) => c.id === field.value)?.value + update[field.value] = current ?? null + } + setSearch(update) } const debouncedApplyFilterSearch = useDebounce(applyFilterSearch, 250) diff --git a/apps/studio/components/interfaces/UnifiedLogs/components/Columns.tsx b/apps/studio/components/interfaces/UnifiedLogs/components/Columns.tsx index 9bb50e54274..24226623f0f 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/components/Columns.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/components/Columns.tsx @@ -184,7 +184,10 @@ export function generateDynamicColumns({ data }: { data: ColumnSchema[] }): { { accessorKey: 'method', header: 'Method', - filterFn: 'arrIncludesSome', + // Filtering is server-side via the `filter` URL param, like every other + // column in this table. The built-in `arrIncludesSome` would receive the + // wrapped { operator, values } shape and reject it. + filterFn: (_row, _columnId, _filterValue) => true, cell: ({ row }) => { const value = row.getValue('method') return {value} @@ -221,6 +224,11 @@ export function generateDynamicColumns({ data }: { data: ColumnSchema[] }): { { accessorKey: 'event_message', header: 'Event message', + // No client-side filterFn — event_message uses server-side LIKE/ILIKE via + // the `filter` URL param (see translateFilter in UnifiedLogs.queries.ts). + // We still need a no-op so TanStack's default `includesString` doesn't run + // against the wrapped { operator, values } shape we write. + filterFn: () => true, cell: ({ row }) => { const value = row.getValue('event_message') const logType = row.original.log_type diff --git a/apps/studio/components/interfaces/UnifiedLogs/components/LogsFilterBar.tsx b/apps/studio/components/interfaces/UnifiedLogs/components/LogsFilterBar.tsx index 7eb6e432967..bfcf2ed626e 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/components/LogsFilterBar.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/components/LogsFilterBar.tsx @@ -1,8 +1,12 @@ -import { groupBy } from 'lodash' import { LoaderCircle, Search } from 'lucide-react' import { useEffect, useState } from 'react' import { FilterBar, FilterCondition, type FilterGroup, type FilterProperty } from 'ui-patterns' +import { + isLogsFilterColumnValue, + type LogsColumnFilterValue, + type LogsFilterOperator, +} from '../UnifiedLogs.filters' import { useDataTable } from '@/components/ui/DataTable/providers/DataTableProvider' import { useStaticEffectEvent } from '@/hooks/useStaticEffectEvent' @@ -13,12 +17,16 @@ const buildFilterGroup = ( const conditions: FilterCondition[] = [] for (const { id, value } of columnFilters) { if (!filterableNames.has(id) || value === null || value === undefined) continue - const values = Array.isArray(value) ? value : [value] + // Equality filters carry their operator inside a wrapped value; range/slider + // filters arrive as plain arrays and default to `=`. + const { operator, values } = isLogsFilterColumnValue(value) + ? value + : { operator: '=' as LogsFilterOperator, values: Array.isArray(value) ? value : [value] } for (const v of values) { conditions.push({ propertyName: id, value: v as FilterCondition['value'], - operator: '=', + operator, }) } } @@ -37,7 +45,16 @@ export const LogsFilterBar = () => { name: filter.value, type: 'string', options: filter.options ?? [], - operators: ['='], + operators: + filter.value === 'event_message' + ? [ + { label: 'iLike', value: '~~*', group: 'pattern' }, + { label: 'Not iLike', value: '!~~*', group: 'pattern' }, + ] + : [ + { label: 'Equals', value: '=', group: 'comparison' }, + { label: 'Not equal', value: '<>', group: 'comparison' }, + ], })) // Local state because the FilterBar carries transient states @@ -60,16 +77,28 @@ export const LogsFilterBar = () => { ) if (!isValid) return - const filterConditions = next.conditions as FilterCondition[] - const groupedFilterConditions = groupBy(filterConditions, 'propertyName') - Object.entries(groupedFilterConditions).forEach(([name, conditions]) => { - table.getColumn(name)?.setFilterValue(conditions.map((x) => x.value)) - }) + // Coalesce conditions into one wrapped value per column. Mixed operators on + // the same column aren't expressible in the column-filter shape — last wins. + const wrappedByColumn = new Map() + for (const cond of next.conditions as FilterCondition[]) { + const operator = cond.operator as LogsFilterOperator + const existing = wrappedByColumn.get(cond.propertyName) + if (!existing) { + wrappedByColumn.set(cond.propertyName, { operator, values: [String(cond.value)] }) + } else { + existing.values.push(String(cond.value)) + if (existing.operator !== operator) existing.operator = operator + } + } + + for (const [name, wrapped] of wrappedByColumn) { + table.getColumn(name)?.setFilterValue(wrapped) + } // Only clear filters owned by this bar — leaves externally-set filters // (e.g. the timeline date range) untouched. const managedNames = new Set(filterProperties.map((p) => p.name)) - const nextNames = new Set(Object.keys(groupedFilterConditions)) + const nextNames = new Set(wrappedByColumn.keys()) const filtersToRemove = table .getState() .columnFilters.filter((x) => managedNames.has(x.id) && !nextNames.has(x.id)) diff --git a/apps/studio/components/ui/DataTable/DataTableSheetRowAction.tsx b/apps/studio/components/ui/DataTable/DataTableSheetRowAction.tsx index 7c3f9655285..2408d4d9714 100644 --- a/apps/studio/components/ui/DataTable/DataTableSheetRowAction.tsx +++ b/apps/studio/components/ui/DataTable/DataTableSheetRowAction.tsx @@ -21,6 +21,10 @@ import { DropdownMenuTrigger, } from 'ui' +import { + isLogsFilterColumnValue, + type LogsColumnFilterValue, +} from '@/components/interfaces/UnifiedLogs/UnifiedLogs.filters' import { DataTableFilterField } from '@/components/ui/DataTable/DataTable.types' import { useCopyToClipboard } from '@/hooks/ui/useCopyToClipboard' @@ -61,12 +65,16 @@ export function DataTableSheetRowAction { - const filterValue = column?.getFilterValue() as undefined | Array - const newValue = filterValue?.includes(value) - ? filterValue - : [...(filterValue || []), value] - - column?.setFilterValue(newValue) + // Equality filters use the wrapped { operator, values } shape so the + // row action stays compatible with the FilterBar (which writes `=` and `<>`). + const current = column?.getFilterValue() + const existing: LogsColumnFilterValue = isLogsFilterColumnValue(current) + ? current + : { operator: '=', values: [] } + const next: LogsColumnFilterValue = existing.values.includes(String(value)) + ? existing + : { operator: existing.operator, values: [...existing.values, String(value)] } + column?.setFilterValue(next) }} className="flex items-center gap-2" > @@ -77,7 +85,12 @@ export function DataTableSheetRowAction column?.setFilterValue(value)} + onClick={() => + column?.setFilterValue({ + operator: field.value === 'event_message' ? '~~*' : '=', + values: [String(value)], + } satisfies LogsColumnFilterValue) + } className="flex items-center gap-2" > diff --git a/apps/studio/data/logs/unified-logs-chart-query.ts b/apps/studio/data/logs/unified-logs-chart-query.ts index ea078151dd3..82c5ae7db92 100644 --- a/apps/studio/data/logs/unified-logs-chart-query.ts +++ b/apps/studio/data/logs/unified-logs-chart-query.ts @@ -5,6 +5,7 @@ import { executeAnalyticsSql } from './execute-analytics-sql' import { logsKeys } from './keys' import { logsAllEndpointUrl, pickLogsQueryBuilder } from './logs-endpoint' import { UNIFIED_LOGS_QUERY_OPTIONS, UnifiedLogsVariables } from './unified-logs-infinite-query' +import { parseLogsFilterUrlParams } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.filters' import { getLogsChartQuery } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries' import { getLogsChartQuery as getLogsChartQueryBq } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq' import { ExecuteSqlError } from '@/data/sql/execute-sql-query' @@ -88,13 +89,21 @@ export async function getUnifiedLogsChart( error: Number(row.error) || 0, } - // Filter levels if needed - const levelFilter = search.level - if (levelFilter && levelFilter.length > 0) { - // Reset levels not in the filter - if (!levelFilter.includes('success')) dataPoint.success = 0 - if (!levelFilter.includes('warning')) dataPoint.warning = 0 - if (!levelFilter.includes('error')) dataPoint.error = 0 + // Zero out levels excluded by the active filter set. + // `=` filters narrow to an allow-list; `<>` filters carve out a deny-list. + const levelFilters = parseLogsFilterUrlParams(search.filter).filter( + (f) => f.column === 'level' + ) + if (levelFilters.length > 0) { + const included = levelFilters.filter((f) => f.operator === '=').map((f) => f.value) + const excluded = new Set( + levelFilters.filter((f) => f.operator === '<>').map((f) => f.value) + ) + const isActive = (lvl: 'success' | 'warning' | 'error') => + (included.length === 0 || included.includes(lvl)) && !excluded.has(lvl) + if (!isActive('success')) dataPoint.success = 0 + if (!isActive('warning')) dataPoint.warning = 0 + if (!isActive('error')) dataPoint.error = 0 } dataByTimestamp.set(milliseconds, dataPoint)