Joshen/fe 3475 add operator to event message filter (#46457)

## Context

Original task was to support searching `!=` on `event_message`, but this
PR addresses some things regarding searching on `event_message` in
unified logs that I found while working on this.

### `=` and `!=` are technically inaccurate
We're doing pattern matching when searching on event_message rather than
a strict equality check, so a more accurate operator would be `ilike
(~~*)` and `not ilike(!~~*)` - both of which would be case insensitive
for easier checking.

Am thus swapping to use these 2 operators when filtering on
`event_message`:
<img width="430" height="134" alt="image"
src="https://github.com/user-attachments/assets/c8a320b6-e016-44ae-aed0-1e7b6cefbda9"
/>

### Filtering on `event_message` was never server side
It seems like we have been only doing client side searching on
`event_message` which is inaccurate as we're only filtering against rows
that are on the current page. The `event_message` filtering was never
appended to the URL state as well so the changes in this PR ensures that
all search including `event_message` is server side.

### Rework on unified logs filtering via URL params
Because we're now supporting more than just `=` in unified logs, the
current filter system is insufficient (e.g can't just be
`status=x&method=y`). Am opting to use the same system as per how we do
filtering in the table editor where search params follow the syntax:
`{column}:{operator}:{value}`
<img width="521" height="46" alt="image"
src="https://github.com/user-attachments/assets/54e72eb2-1581-4c1a-910e-58d993da1766"
/>

## To test
- [ ] Verify that searching for logs in unified logs still works
- [ ] Verify that searching against event_message in unified logs works
as expected (both ilike and not ilike)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Repeatable URL-based column filters with operator support (e.g.,
equals, not-equals, pattern matching).
* Expanded pattern-style operators for message searches
(case-insensitive/contains, negation).

* **Improvements**
* Unified filter handling across logs list, charts, and counts for
consistent results.
* Range/slider filters and pagination remain supported and round-trip
via URL parameters.

<!-- review_stack_entry_start -->

[![Review Change
Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/supabase/supabase/pull/46457?utm_source=github_walkthrough&utm_medium=github&utm_campaign=change_stack)

<!-- review_stack_entry_end -->
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Joshen Lim authored and GitHub committed 2026-05-28 22:19:08 +08:00
1 parent ebd13ffaeb
commit b281d3fcf5
10 files changed
+444 -218

No files matched your search

@@ -9,8 +9,6 @@ import {
} from 'nuqs'
import {
ARRAY_DELIMITER,
LEVELS,
RANGE_DELIMITER,
SLIDER_DELIMITER,
SORT_DELIMITER,
@@ -33,20 +31,18 @@ const parseAsSort = createParser({
})
export const SEARCH_PARAMS_PARSER = {
// CUSTOM FILTERS
level: parseAsArrayOf(parseAsStringLiteral(LEVELS), ARRAY_DELIMITER),
log_type: parseAsArrayOf(parseAsString, ARRAY_DELIMITER),
// Equality filters. Repeatable `?filter=column:opAbbrev:value` (e.g. `level:eq:info`,
// `host:neq:foo`). Parsed with the helpers in `UnifiedLogs.filters.ts`.
filter: parseAsArrayOf(parseAsString),
// Range/special filters keep their dedicated keys — their semantics are inherently
// multi-value (sliders, time ranges) and don't fit the eq/neq model.
latency: parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER),
'timing.dns': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER),
'timing.connection': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER),
'timing.tls': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER),
'timing.ttfb': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER),
'timing.transfer': parseAsArrayOf(parseAsInteger, SLIDER_DELIMITER),
status: parseAsArrayOf(parseAsString, ARRAY_DELIMITER),
regions: parseAsArrayOf(parseAsStringLiteral(REGIONS), ARRAY_DELIMITER),
method: parseAsArrayOf(parseAsStringLiteral(METHODS), ARRAY_DELIMITER),
host: parseAsString,
pathname: parseAsString,
date: parseAsArrayOf(parseAsTimestamp, RANGE_DELIMITER),
// REQUIRED FOR SORTING & PAGINATION
@@ -0,0 +1,95 @@
// Operators mirror PostgREST / Table Editor conventions:
// `=` / `<>` — exact equality, available on most columns
// `~~*` / `!~~*` — ILIKE / NOT ILIKE (case-insensitive substring
// contains / does-not-contain), wired up only for the
// `event_message` column
export type LogsFilterOperator = '=' | '<>' | '~~*' | '!~~*'
export interface LogsFilter {
column: string
operator: LogsFilterOperator
value: string
}
export interface LogsColumnFilterValue {
operator: LogsFilterOperator
values: string[]
}
export const LOGS_FILTER_OPERATORS = [
'=',
'<>',
'~~*',
'!~~*',
] as const satisfies readonly LogsFilterOperator[]
const OPERATOR_TO_ABBREV: Record<LogsFilterOperator, string> = {
'=': 'eq',
'<>': 'neq',
'~~*': 'ilike',
'!~~*': 'notilike',
}
const ABBREV_TO_OPERATOR: Record<string, LogsFilterOperator> = {
eq: '=',
neq: '<>',
ilike: '~~*',
notilike: '!~~*',
}
export const isLogsFilterColumnValue = (value: unknown): value is LogsColumnFilterValue => {
return (
typeof value === 'object' &&
value !== null &&
'operator' in value &&
'values' in value &&
Array.isArray((value as LogsColumnFilterValue).values)
)
}
export const parseLogsFilterUrlParams = (filter?: string[] | null): LogsFilter[] => {
if (!Array.isArray(filter)) return []
const parsed: LogsFilter[] = []
for (const raw of filter) {
const [column, abbrev, ...rest] = raw.split(':')
const operator = ABBREV_TO_OPERATOR[abbrev]
if (!column || !operator) continue
parsed.push({ column, operator, value: rest.join(':') })
}
return parsed
}
export const logsFiltersToUrlParams = (filters: LogsFilter[]): string[] => {
return filters.map((f) => `${f.column}:${OPERATOR_TO_ABBREV[f.operator]}:${f.value}`)
}
export const groupLogsFiltersByColumn = (
filters: LogsFilter[]
): Record<string, LogsColumnFilterValue> => {
const grouped: Record<string, LogsColumnFilterValue> = {}
for (const { column, operator, value } of filters) {
const existing = grouped[column]
if (!existing) {
grouped[column] = { operator, values: [value] }
} else {
existing.values.push(value)
// Mixed operators on the same column aren't expressible in the column-filter
// shape (one operator per column). Last write wins.
if (existing.operator !== operator) existing.operator = operator
}
}
return grouped
}
export const columnFiltersToLogsFilters = (
columnFilters: { id: string; value: unknown }[]
): LogsFilter[] => {
const filters: LogsFilter[] = []
for (const { id, value } of columnFilters) {
if (!isLogsFilterColumnValue(value)) continue
for (const v of value.values) {
filters.push({ column: id, operator: value.operator, value: String(v) })
}
}
return filters
}
@@ -7,6 +7,7 @@
import dayjs from 'dayjs'
import { DEFAULT_LOG_TYPES } from './UnifiedLogs.constants'
import { groupLogsFiltersByColumn, parseLogsFilterUrlParams } from './UnifiedLogs.filters'
import { QuerySearchParamsType, SearchParamsType } from './UnifiedLogs.types'
import {
joinSqlFragments,
@@ -16,23 +17,37 @@ import {
type SafeLogSqlFragment,
} from '@/data/logs/safe-analytics-sql'
// Pagination and control parameters
const PAGINATION_PARAMS = ['sort', 'start', 'size', 'uuid', 'cursor', 'direction', 'live'] as const
// Operator fragments for SQL emission. `safeSql` rejects plain strings, so we
// pre-brand the keywords we want to switch between.
const IN_OP = safeSql`IN`
const NOT_IN_OP = safeSql`NOT IN`
const LIKE_OP = safeSql`LIKE`
const NOT_LIKE_OP = safeSql`NOT LIKE`
// Special filter parameters that need custom handling
const SPECIAL_FILTER_PARAMS = ['date'] as const
// Combined list of all parameters to exclude from standard filtering
const EXCLUDED_QUERY_PARAMS = [...PAGINATION_PARAMS, ...SPECIAL_FILTER_PARAMS] as const
const ALL_LOG_TYPES = ['edge', 'postgrest', 'storage', 'postgres', 'edge function', 'auth']
/**
* Builds WHERE-clause fragments from a search-param map. Identifier-position
* keys are validated via `quotedIdent()` (regex allowlist) and value-position
* inputs via `analyticsLiteral` — both throw on disallowed input, in which
* case we drop the predicate rather than emit unsafe SQL.
* Computes the log_type set that the CTE should union. With no filter, defaults
* to the cheap two-source set. With `=` filters, narrows to those values. With
* `<>` filters, excludes them from the full set.
*/
const getEffectiveLogTypes = (search: QuerySearchParamsType): string[] => {
const filters = parseLogsFilterUrlParams(search.filter).filter((f) => f.column === 'log_type')
if (filters.length === 0) return [...DEFAULT_LOG_TYPES]
const included = filters.filter((f) => f.operator === '=').map((f) => f.value)
const excluded = new Set(filters.filter((f) => f.operator === '<>').map((f) => f.value))
const base = included.length > 0 ? included : ALL_LOG_TYPES
return base.filter((t) => !excluded.has(t))
}
/**
* Builds WHERE-clause fragments from the parsed `filter` URL array. Identifier-
* position keys are validated via `quotedIdent()` (regex allowlist) and value-
* position inputs via `analyticsLiteral` — both throw on disallowed input, in
* which case we drop the predicate rather than emit unsafe SQL.
*
* @param search Search params (URL-derived filter values)
* @param excludeKey Optional key to skip — used by facet-count branches that
* @param excludeKey Optional column to skip — used by facet-count branches that
* need every filter applied *except* the one being faceted
* @returns Array of SafeLogSqlFragment predicates ready to be AND-joined
*/
@@ -41,10 +56,11 @@ const buildConditions = (
excludeKey?: string
): SafeLogSqlFragment[] => {
const conditions: SafeLogSqlFragment[] = []
const grouped = groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter))
Object.entries(search).forEach(([key, value]) => {
if (key === excludeKey) return
if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) return
for (const [key, { operator, values }] of Object.entries(grouped)) {
if (key === excludeKey) continue
if (values.length === 0) continue
try {
// `key` is interpolated as a column identifier. `quotedIdent()` rejects
@@ -52,27 +68,37 @@ const buildConditions = (
// crafted URL key like `level OR id IS NOT NULL` is dropped rather
// than emitted into the WHERE clause).
const col = quotedIdent(key)
const isNeq = operator === '<>'
const inOp = isNeq ? NOT_IN_OP : IN_OP
const likeOp = isNeq ? NOT_LIKE_OP : LIKE_OP
const joinAndOr = isNeq ? ' AND ' : ' OR '
if (Array.isArray(value) && value.length > 0) {
if (key === 'event_message' && (operator === '~~*' || operator === '!~~*')) {
// BigQuery has no ILIKE; emulate via LOWER(col) (NOT) LIKE LOWER('%v%').
// Auto-wrap with `%…%` unless the user already included one. Multiple
// ILIKE values join with OR; NOT ILIKE joins with AND (row must contain
// none of the given substrings).
const pattern = (v: string) => (v.includes('%') ? v : '%' + v + '%')
const likeKeyword = operator === '!~~*' ? safeSql`NOT LIKE` : safeSql`LIKE`
const join = operator === '!~~*' ? ' AND ' : ' OR '
const branches = values.map(
(v) => safeSql`LOWER(${col}) ${likeKeyword} LOWER(${lit(pattern(v))})`
)
conditions.push(safeSql`(${joinSqlFragments(branches, join)})`)
} else if (key === 'host' || key === 'pathname') {
const branches = values.map((v) => safeSql`${col} ${likeOp} ${lit('%' + v + '%')}`)
conditions.push(safeSql`(${joinSqlFragments(branches, joinAndOr)})`)
} else {
const inList = joinSqlFragments(
value.map((v) => lit(String(v))),
values.map((v) => lit(v)),
','
)
conditions.push(safeSql`${col} IN (${inList})`)
return
}
if (value !== null && value !== undefined) {
if (key === 'host' || key === 'pathname') {
conditions.push(safeSql`${col} LIKE ${lit('%' + String(value) + '%')}`)
} else {
conditions.push(safeSql`${col} = ${lit(String(value))}`)
}
conditions.push(safeSql`${col} ${inOp} (${inList})`)
}
} catch {
// quotedIdent() or analyticsLiteral() rejected the input — drop the predicate.
}
})
}
return conditions
}
@@ -360,7 +386,7 @@ WITH unified_logs AS (
*/
export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
const conditions = buildConditions(search)
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
const effectiveLogTypes = getEffectiveLogTypes(search)
return safeSql`
${getUnifiedLogsCTE(effectiveLogTypes)}
@@ -513,7 +539,7 @@ WITH unified_logs AS (
`
export const getLogsCountQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
const effectiveLogTypes = getEffectiveLogTypes(search)
const logTypeConditions = buildConditions(search, 'log_type')
const levelConditions = buildConditions(search, 'level')
const logTypeWhere: SafeLogSqlFragment =
@@ -580,7 +606,7 @@ UNION ALL SELECT dimension, value, count FROM pathname_count
export const getLogsChartQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
const conditions = buildConditions(search)
const truncationLevel = calculateChartBucketing(search)
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
const effectiveLogTypes = getEffectiveLogTypes(search)
return safeSql`
${getUnifiedLogsCTE(effectiveLogTypes)}
@@ -12,6 +12,9 @@ const baseSearch = {
date: [new Date('2026-05-08T09:00:00Z'), new Date('2026-05-08T10:00:00Z')],
} as any
// Helper: build a search with extra `filter` URL entries on top of the base.
const withFilters = (...entries: string[]) => ({ ...baseSearch, filter: entries }) as any
describe('UnifiedLogs.queries (OTEL flat)', () => {
describe('getUnifiedLogsQuery', () => {
it('defaults to postgres + postgrest log types when none specified', () => {
@@ -24,7 +27,7 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
})
it('routes the `edge` log type to edge_logs without /rest/ or /storage/ paths', () => {
const sql = getUnifiedLogsQuery({ ...baseSearch, log_type: ['edge'] } as any)
const sql = getUnifiedLogsQuery(withFilters('log_type:eq:edge'))
expect(sql).toContain(`NOT LIKE '%/rest/%'`)
expect(sql).toContain(`NOT LIKE '%/storage/%'`)
const where = sql.split(/\bWHERE\b/)[1] ?? ''
@@ -32,18 +35,16 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
})
it('routes the `storage` log type to edge_logs filtered by /storage/', () => {
const sql = getUnifiedLogsQuery({ ...baseSearch, log_type: ['storage'] } as any)
const sql = getUnifiedLogsQuery(withFilters('log_type:eq:storage'))
expect(sql).toContain(
`source = 'edge_logs' AND log_attributes['request.path'] LIKE '%/storage/%'`
)
})
it('escapes single quotes in filter values to prevent SQL injection', () => {
const sql = getUnifiedLogsQuery({
...baseSearch,
method: [`G'ET`],
pathname: `/customers'; DROP TABLE logs --`,
} as any)
const sql = getUnifiedLogsQuery(
withFilters(`method:eq:G'ET`, `pathname:eq:/customers'; DROP TABLE logs --`)
)
// Single quotes are doubled (SQL-standard escaping) by pg-meta's
// literal(); a raw single quote from user input never closes its
// string literal early.
@@ -52,12 +53,9 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
})
it('translates method/status/pathname filters to log_attributes predicates', () => {
const sql = getUnifiedLogsQuery({
...baseSearch,
method: ['GET'],
status: ['401'],
pathname: '/customers',
} as any)
const sql = getUnifiedLogsQuery(
withFilters('method:eq:GET', 'status:eq:401', 'pathname:eq:/customers')
)
expect(sql).toContain(`log_attributes['request.method'] IN ('GET')`)
// Status filter wraps the CASE that picks HTTP code or Postgres SQLSTATE
// so e.g. '00000' matches postgres success rows.
@@ -66,6 +64,43 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
expect(sql).toContain(`log_attributes['request.path'] LIKE '%/customers%'`)
})
it('flips IN to NOT IN when the operator is `<>`', () => {
const sql = getUnifiedLogsQuery(withFilters('method:neq:GET', 'status:neq:401'))
expect(sql).toContain(`log_attributes['request.method'] NOT IN ('GET')`)
expect(sql).toMatch(/END\) NOT IN \('401'\)/)
})
it('flips LIKE to NOT LIKE when the pathname/host operator is `<>`', () => {
const sql = getUnifiedLogsQuery(withFilters('pathname:neq:/health', 'host:neq:cdn.foo'))
expect(sql).toContain(`log_attributes['request.path'] NOT LIKE '%/health%'`)
expect(sql).toContain(`log_attributes['request.url'] NOT LIKE '%cdn.foo%'`)
})
it('emits ILIKE with auto-wrapped `%…%` for event_message `~~*`', () => {
const sql = getUnifiedLogsQuery(withFilters('event_message:ilike:Permission Denied'))
expect(sql).toContain(`event_message ILIKE '%Permission Denied%'`)
})
it('emits NOT ILIKE for event_message `!~~*` so rows containing the term are excluded', () => {
const sql = getUnifiedLogsQuery(withFilters('event_message:notilike:cron'))
expect(sql).toContain(`event_message NOT ILIKE '%cron%'`)
})
it('joins multiple NOT ILIKE values with AND (row must contain none)', () => {
const sql = getUnifiedLogsQuery(
withFilters('event_message:notilike:cron', 'event_message:notilike:heartbeat')
)
expect(sql).toMatch(
/event_message NOT ILIKE '%cron%' AND event_message NOT ILIKE '%heartbeat%'/
)
})
it('passes through user-supplied `%` wildcards on event_message ILIKE without double-wrapping', () => {
const sql = getUnifiedLogsQuery(withFilters('event_message:ilike:error%'))
expect(sql).toContain(`event_message ILIKE 'error%'`)
expect(sql).not.toContain(`'%error%%'`)
})
it('excludes connection log messages by default (hide_connection_logs=true)', () => {
const sql = getUnifiedLogsQuery({ ...baseSearch, hide_connection_logs: true } as any)
expect(sql).toContain("source != 'postgres_logs'")
@@ -104,7 +139,7 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
})
it('honours an active log_type filter in the total count branch', () => {
const sql = getLogsCountQuery({ ...baseSearch, log_type: ['edge'] } as any)
const sql = getLogsCountQuery(withFilters('log_type:eq:edge'))
// The first branch is the total — its WHERE must include the edge
// log_type predicate, otherwise the total badge would over-count
// when a log_type filter is active.
@@ -141,7 +176,7 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
describe('getFacetCountQuery', () => {
it('groups by the requested facet and excludes that facet from the WHERE filters', () => {
const sql = getFacetCountQuery({
search: { ...baseSearch, method: ['GET'], status: ['200'] } as any,
search: withFilters('method:eq:GET', 'status:eq:200'),
facet: 'method',
})
// Filtered facet is excluded from WHERE; other filters still applied.
@@ -160,46 +195,56 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
// pg-meta's literal() would emit `E'a\\b'` for `a\b` — the `E` prefix is
// Postgres-only and rejected by both analytics engines. analyticsLiteral
// doubles the backslash inside plain `'…'` delimiters instead.
const sql = getUnifiedLogsQuery({ ...baseSearch, method: 'a\\b' } as any)
expect(sql).toContain(`log_attributes['request.method'] = 'a\\\\b'`)
const sql = getUnifiedLogsQuery(withFilters('method:eq:a\\b'))
expect(sql).toContain(`log_attributes['request.method'] IN ('a\\\\b')`)
expect(sql).not.toContain(`E'a`)
})
it("escapes single quotes by doubling them ('' rather than \\')", () => {
const sql = getUnifiedLogsQuery({ ...baseSearch, method: "GET' OR '1'='1" } as any)
expect(sql).toContain(`log_attributes['request.method'] = 'GET'' OR ''1''=''1'`)
const sql = getUnifiedLogsQuery(withFilters("method:eq:GET' OR '1'='1"))
expect(sql).toContain(`log_attributes['request.method'] IN ('GET'' OR ''1''=''1')`)
})
})
})
describe('UnifiedLogs.queries.bq', () => {
it('backtick-quotes column identifiers (BigQuery syntax)', () => {
const sql = getUnifiedLogsQueryBQ({ ...baseSearch, method: 'GET' } as any)
expect(sql).toContain("`method` = 'GET'")
const sql = getUnifiedLogsQueryBQ(withFilters('method:eq:GET'))
expect(sql).toContain("`method` IN ('GET')")
})
it('rejects keys with non-identifier characters', () => {
// A key like "foo; DROP TABLE" fails the quotedIdent regex (the `;` is not
// A column like "foo; DROP TABLE x" fails the quotedIdent regex (the `;` is not
// in `[A-Za-z_][A-Za-z0-9_]*`), so the predicate is dropped entirely.
const sql = getUnifiedLogsQueryBQ({ ...baseSearch, 'foo; DROP TABLE x': 'y' } as any)
const sql = getUnifiedLogsQueryBQ(withFilters('foo; DROP TABLE x:eq:y'))
expect(sql).not.toContain('DROP TABLE')
expect(sql).not.toContain('foo;')
})
it('rejects keys containing spaces', () => {
const sql = getUnifiedLogsQueryBQ({
...baseSearch,
'level OR id IS NOT NULL': 'anything',
} as any)
const sql = getUnifiedLogsQueryBQ(withFilters('level OR id IS NOT NULL:eq:anything'))
expect(sql).not.toContain('IS NOT NULL')
expect(sql).not.toContain('level OR')
})
it('escapes injection attempts in filter values via analyticsLiteral', () => {
const sql = getUnifiedLogsQueryBQ({ ...baseSearch, method: "GET' OR '1'='1" } as any)
const sql = getUnifiedLogsQueryBQ(withFilters("method:eq:GET' OR '1'='1"))
// The value is single-quote-escaped, so the synthetic OR can't break out
// of the string literal.
expect(sql).toContain("`method` = 'GET'' OR ''1''=''1'")
expect(sql).not.toMatch(/`method` = 'GET' OR '1'='1'/)
expect(sql).toContain("`method` IN ('GET'' OR ''1''=''1')")
expect(sql).not.toMatch(/`method` IN \('GET' OR '1'='1'\)/)
})
it('emulates ILIKE with LOWER()/LOWER() since BigQuery has no ILIKE keyword', () => {
const sql = getUnifiedLogsQueryBQ(withFilters('event_message:ilike:Permission Denied'))
expect(sql).toContain("LOWER(`event_message`) LIKE LOWER('%Permission Denied%')")
// Sanity check: never emit a raw ILIKE keyword for BQ.
expect(sql).not.toMatch(/\bILIKE\b/)
})
it('emulates NOT ILIKE with LOWER()/NOT LIKE/LOWER() for event_message `!~~*`', () => {
const sql = getUnifiedLogsQueryBQ(withFilters('event_message:notilike:cron'))
expect(sql).toContain("LOWER(`event_message`) NOT LIKE LOWER('%cron%')")
expect(sql).not.toMatch(/\bILIKE\b/)
})
})
@@ -1,6 +1,11 @@
import dayjs from 'dayjs'
import { DEFAULT_LOG_TYPES } from './UnifiedLogs.constants'
import {
groupLogsFiltersByColumn,
parseLogsFilterUrlParams,
type LogsFilterOperator,
} from './UnifiedLogs.filters'
import { QuerySearchParamsType, SearchParamsType } from './UnifiedLogs.types'
import {
joinSqlFragments,
@@ -9,14 +14,14 @@ import {
type SafeLogSqlFragment,
} from '@/data/logs/safe-analytics-sql'
// Pagination and control parameters
const PAGINATION_PARAMS = ['sort', 'start', 'size', 'uuid', 'cursor', 'direction', 'live'] as const
// Special filter parameters that need custom handling
const SPECIAL_FILTER_PARAMS = ['date', 'hide_connection_logs'] as const
// Combined list of all parameters to exclude from standard filtering
const EXCLUDED_QUERY_PARAMS = [...PAGINATION_PARAMS, ...SPECIAL_FILTER_PARAMS] as const
// Operator fragments for SQL emission. `safeSql` rejects plain strings, so we
// pre-brand the keywords we want to switch between.
const IN_OP = safeSql`IN`
const NOT_IN_OP = safeSql`NOT IN`
const LIKE_OP = safeSql`LIKE`
const NOT_LIKE_OP = safeSql`NOT LIKE`
const ILIKE_OP = safeSql`ILIKE`
const NOT_ILIKE_OP = safeSql`NOT ILIKE`
// Facets the count query is allowed to be invoked for. Reject anything else
// at the entry point rather than letting an unsupported value reach
@@ -94,94 +99,83 @@ const logTypeWherePredicate = (logTypes: string[]): SafeLogSqlFragment => {
}
/**
* Translates a frontend filter key/value pair into an underlying SQL predicate.
* The OTEL endpoint won't accept queries that reference derived aliases like
* `log_type` or `level` in WHERE for some shapes, so we always emit raw-column
* predicates (source/severity_text/log_attributes[…]).
* Translates one (column, values, operator) group from the parsed `filter` URL
* param into an underlying SQL predicate. The OTEL endpoint rejects queries
* that reference derived aliases like `log_type` or `level` in WHERE for some
* shapes, so we always emit raw-column predicates (source/severity_text/
* log_attributes[…]). For `<>`, IN becomes NOT IN, LIKE becomes NOT LIKE, and
* multi-value lists are joined with AND so the row must not match *any* value.
*/
const translateFilter = (key: string, value: unknown): SafeLogSqlFragment | null => {
if (value === null || value === undefined) return null
const translateFilter = (
key: string,
values: readonly string[],
operator: LogsFilterOperator
): SafeLogSqlFragment | null => {
if (values.length === 0) return null
const arr = Array.isArray(value) ? (value.length > 0 ? value : null) : null
if (Array.isArray(value) && !arr) return null
const isNeq = operator === '<>'
const inOp = isNeq ? NOT_IN_OP : IN_OP
const likeOp = isNeq ? NOT_LIKE_OP : LIKE_OP
const joinAndOr = isNeq ? ' AND ' : ' OR '
const inList = (values: readonly unknown[]): SafeLogSqlFragment =>
const inList = (vals: readonly string[]): SafeLogSqlFragment =>
safeSql`(${joinSqlFragments(
values.map((v) => lit(String(v))),
vals.map((v) => lit(v)),
','
)})`
switch (key) {
case 'log_type': {
const types = (arr ?? [value]).map((v) => String(v))
const branches = types.map(
(t) => safeSql`(${LOG_TYPE_PREDICATE[t] ?? safeSql`source = ${lit(t)}`})`
)
return safeSql`(${joinSqlFragments(branches, ' OR ')})`
const branches = values.map((t) => {
const pred = LOG_TYPE_PREDICATE[t] ?? safeSql`source = ${lit(t)}`
return isNeq ? safeSql`NOT (${pred})` : safeSql`(${pred})`
})
return safeSql`(${joinSqlFragments(branches, joinAndOr)})`
}
case 'level': {
case 'level':
// No simple raw column for level; reference the inline CASE expression.
const levels = arr ?? [value]
return safeSql`(${LEVEL_EXPR}) IN ${inList(levels.map((v) => String(v)))}`
}
return safeSql`(${LEVEL_EXPR}) ${inOp} ${inList(values)}`
case 'method':
return arr
? safeSql`${ATTR.method} IN ${inList(arr)}`
: safeSql`${ATTR.method} = ${lit(String(value))}`
case 'status': {
return safeSql`${ATTR.method} ${inOp} ${inList(values)}`
case 'status':
// Match the displayed status: HTTP response code for gateway rows,
// Postgres SQLSTATE for postgres rows. Inline STATUS_EXPR so e.g.
// filtering on '00000' picks up postgres success rows.
const statuses = arr ?? [value]
return safeSql`(${STATUS_EXPR}) IN ${inList(statuses.map((v) => String(v)))}`
}
return safeSql`(${STATUS_EXPR}) ${inOp} ${inList(values)}`
case 'pathname':
return arr
? safeSql`(${joinSqlFragments(
arr.map((v) => safeSql`${ATTR.path} LIKE ${lit('%' + String(v) + '%')}`),
' OR '
)})`
: safeSql`${ATTR.path} LIKE ${lit('%' + String(value) + '%')}`
return safeSql`(${joinSqlFragments(
values.map((v) => safeSql`${ATTR.path} ${likeOp} ${lit('%' + v + '%')}`),
joinAndOr
)})`
case 'host':
// Best-effort: use full request URL since `host` isn't a top-level field.
return arr
? safeSql`(${joinSqlFragments(
arr.map(
(v) => safeSql`log_attributes['request.url'] LIKE ${lit('%' + String(v) + '%')}`
),
' OR '
)})`
: safeSql`log_attributes['request.url'] LIKE ${lit('%' + String(value) + '%')}`
default:
// Unknown filter key — fall back to a generic equality on log_attributes.
return arr
? safeSql`log_attributes[${lit(key)}] IN ${inList(arr)}`
: safeSql`log_attributes[${lit(key)}] = ${lit(String(value))}`
}
}
/**
* Builds an array of WHERE predicate fragments from search params, optionally
* skipping a specific facet field (used when computing faceted counts).
* `log_type` is always handled separately (see `logTypeWherePredicate`).
*/
const buildPredicates = (
search: QuerySearchParamsType,
excludeField?: string
): SafeLogSqlFragment[] => {
const predicates: SafeLogSqlFragment[] = []
Object.entries(search).forEach(([key, value]) => {
if (key === excludeField) return
if (key === 'log_type') return
if ((EXCLUDED_QUERY_PARAMS as readonly string[]).includes(key)) return
try {
const predicate = translateFilter(key, value)
if (predicate) predicates.push(predicate)
} catch {
// analyticsLiteral rejected an unsupported input — drop the predicate.
return safeSql`(${joinSqlFragments(
values.map((v) => safeSql`log_attributes['request.url'] ${likeOp} ${lit('%' + v + '%')}`),
joinAndOr
)})`
case 'event_message': {
// event_message is a top-level column, not a log_attributes key. ILIKE/NOT ILIKE
// auto-wrap with `%…%` so the user can type "permission denied" as a substring
// search; explicit `%`s in the input are passed through unchanged. Multiple
// ILIKE values join with OR (match any); multiple NOT ILIKE values join with AND
// (the row must contain none of them).
if (operator === '~~*' || operator === '!~~*') {
const op = operator === '~~*' ? ILIKE_OP : NOT_ILIKE_OP
const join = operator === '!~~*' ? ' AND ' : ' OR '
const pattern = (v: string) => (v.includes('%') ? v : '%' + v + '%')
return safeSql`(${joinSqlFragments(
values.map((v) => safeSql`event_message ${op} ${lit(pattern(v))}`),
join
)})`
}
// = / <> still emit exact match against the column — defensive; the
// FilterBar doesn't expose them for event_message, but a hand-crafted URL
// might.
return safeSql`event_message ${inOp} ${inList(values)}`
}
})
return predicates
default:
return safeSql`log_attributes[${lit(key)}] ${inOp} ${inList(values)}`
}
}
const whereClause = (predicates: SafeLogSqlFragment[]): SafeLogSqlFragment =>
@@ -259,12 +253,30 @@ const buildBaseWhere = (
search: QuerySearchParamsType,
excludeField?: string
): SafeLogSqlFragment[] => {
const effectiveLogTypes = search.log_type?.length ? search.log_type : [...DEFAULT_LOG_TYPES]
const grouped = groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter))
const parts: SafeLogSqlFragment[] = []
if (excludeField !== 'log_type') {
parts.push(logTypeWherePredicate(effectiveLogTypes))
const logTypeFilter = grouped.log_type
if (logTypeFilter) {
const pred = translateFilter('log_type', logTypeFilter.values, logTypeFilter.operator)
if (pred) parts.push(pred)
} else {
parts.push(logTypeWherePredicate([...DEFAULT_LOG_TYPES]))
}
}
parts.push(...buildPredicates(search, excludeField))
for (const [key, { operator, values }] of Object.entries(grouped)) {
if (key === excludeField) continue
if (key === 'log_type') continue // handled above
try {
const predicate = translateFilter(key, values, operator)
if (predicate) parts.push(predicate)
} catch {
// analyticsLiteral rejected an unsupported input — drop the predicate.
}
}
return parts
}
@@ -38,6 +38,12 @@ import { RowSelectionHeader } from './RowSelectionHeader'
import { ServiceFlowPanel } from './ServiceFlowPanel'
import { SEARCH_PARAMS_PARSER } from './UnifiedLogs.constants'
import { filterFields as defaultFilterFields } from './UnifiedLogs.fields'
import {
columnFiltersToLogsFilters,
groupLogsFiltersByColumn,
logsFiltersToUrlParams,
parseLogsFilterUrlParams,
} from './UnifiedLogs.filters'
import { useLiveMode, useResetFocus } from './UnifiedLogs.hooks'
import { ColumnSchema } from './UnifiedLogs.schema'
import { QuerySearchParamsType } from './UnifiedLogs.types'
@@ -84,22 +90,14 @@ export const UnifiedLogs = () => {
const track = useTrack()
const [search, setSearch] = useQueryStates(SEARCH_PARAMS_PARSER)
const {
sort,
start,
size,
id,
cursor,
direction,
live,
hide_connection_logs: _hideConnectionLogs,
...filter
} = search
const defaultColumnSorting = sort ? [sort] : []
const defaultColumnSorting = search.sort ? [search.sort] : []
const defaultColumnVisibility = { uuid: false }
const defaultColumnFilters = Object.entries(filter)
.map(([key, value]) => ({ id: key, value }))
.filter(({ value }) => value ?? undefined)
// Column filters are seeded from the repeatable `filter` URL param. Each entry
// (`col:opAbbrev:value`) is grouped per column into a wrapped { operator, values }
// shape that the query builder reads back.
const defaultColumnFilters = Object.entries(
groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter))
).map(([id, value]) => ({ id, value }))
const [topBarHeight, setTopBarHeight] = useState(0)
const topBarRef = useRef<HTMLDivElement>(null)
@@ -211,15 +209,17 @@ export const UnifiedLogs = () => {
const facets = counts?.facets
const totalFetched = flatData?.length
// Create a filtered version of the chart config based on selected levels
// Create a filtered version of the chart config based on level filters in the URL.
const filteredChartConfig = useMemo(() => {
const levelFilter = search.level || LEVELS
const levelFilters = parseLogsFilterUrlParams(search.filter).filter((f) => f.column === 'level')
const included = levelFilters.filter((f) => f.operator === '=').map((f) => f.value)
const excluded = new Set(levelFilters.filter((f) => f.operator === '<>').map((f) => f.value))
const baseLevels: readonly string[] = included.length > 0 ? included : LEVELS
const activeLevels = baseLevels.filter((l) => !excluded.has(l))
return Object.fromEntries(
Object.entries(CHART_CONFIG).filter(([key]) =>
levelFilter.includes(key as (typeof LEVELS)[number])
)
Object.entries(CHART_CONFIG).filter(([key]) => activeLevels.includes(key))
) as ChartConfig
}, [search.level])
}, [search.filter])
const getRowClassName = <
TData extends { date: Date; level: (typeof LEVELS)[number]; timestamp: number },
@@ -270,10 +270,6 @@ export const UnifiedLogs = () => {
return table.getCoreRowModel().flatRows.find((row) => row.id === openRowId)
}, [isLoading, isFetching, flatData.length, table, openRowId])
// REMINDER: this is currently needed for the cmdk search
// [Joshen] This is where facets are getting dynamically loaded
// TODO: auto search via API when the user changes the filter instead of hardcoded
// Will need to refactor this bit
// - Each facet just handles its own state, rather than getting passed down like this
const filterFields = useMemo(() => {
@@ -298,24 +294,21 @@ export const UnifiedLogs = () => {
})
}, [facets])
// Debounced filter application to avoid too many API calls when user clicks multiple filters quickly
// Debounced filter application to avoid too many API calls when user clicks multiple filters quickly.
// All equality/pattern column filters serialize into the repeatable `filter` URL param. Slider/timerange
// column filters keep their dedicated per-column URL keys so things like the timeline brush still
// round-trip — they have their own range semantics and aren't covered by eq/neq/like.
const applyFilterSearch = () => {
const columnFiltersWithNullable = filterFields.map((field) => {
const filterValue = columnFilters.find((filter) => filter.id === field.value)
if (!filterValue) return { id: field.value, value: null }
return { id: field.value, value: filterValue.value }
})
const search = columnFiltersWithNullable.reduce(
(prev, curr) => {
// Add to search parameters
prev[curr.id as string] = curr.value
return prev
},
{} as Record<string, unknown>
)
setSearch(search)
const filterEntries = logsFiltersToUrlParams(columnFiltersToLogsFilters(columnFilters))
const update: Record<string, unknown> = {
filter: filterEntries.length > 0 ? filterEntries : null,
}
for (const field of filterFields) {
if (field.type !== 'timerange') continue
const current = columnFilters.find((c) => c.id === field.value)?.value
update[field.value] = current ?? null
}
setSearch(update)
}
const debouncedApplyFilterSearch = useDebounce(applyFilterSearch, 250)
@@ -184,7 +184,10 @@ export function generateDynamicColumns({ data }: { data: ColumnSchema[] }): {
{
accessorKey: 'method',
header: 'Method',
filterFn: 'arrIncludesSome',
// Filtering is server-side via the `filter` URL param, like every other
// column in this table. The built-in `arrIncludesSome` would receive the
// wrapped { operator, values } shape and reject it.
filterFn: (_row, _columnId, _filterValue) => true,
cell: ({ row }) => {
const value = row.getValue<ColumnSchema['method']>('method')
return <span className="text-foreground-lighter">{value}</span>
@@ -221,6 +224,11 @@ export function generateDynamicColumns({ data }: { data: ColumnSchema[] }): {
{
accessorKey: 'event_message',
header: 'Event message',
// No client-side filterFn — event_message uses server-side LIKE/ILIKE via
// the `filter` URL param (see translateFilter in UnifiedLogs.queries.ts).
// We still need a no-op so TanStack's default `includesString` doesn't run
// against the wrapped { operator, values } shape we write.
filterFn: () => true,
cell: ({ row }) => {
const value = row.getValue<ColumnSchema['event_message']>('event_message')
const logType = row.original.log_type
@@ -1,8 +1,12 @@
import { groupBy } from 'lodash'
import { LoaderCircle, Search } from 'lucide-react'
import { useEffect, useState } from 'react'
import { FilterBar, FilterCondition, type FilterGroup, type FilterProperty } from 'ui-patterns'
import {
isLogsFilterColumnValue,
type LogsColumnFilterValue,
type LogsFilterOperator,
} from '../UnifiedLogs.filters'
import { useDataTable } from '@/components/ui/DataTable/providers/DataTableProvider'
import { useStaticEffectEvent } from '@/hooks/useStaticEffectEvent'
@@ -13,12 +17,16 @@ const buildFilterGroup = (
const conditions: FilterCondition[] = []
for (const { id, value } of columnFilters) {
if (!filterableNames.has(id) || value === null || value === undefined) continue
const values = Array.isArray(value) ? value : [value]
// Equality filters carry their operator inside a wrapped value; range/slider
// filters arrive as plain arrays and default to `=`.
const { operator, values } = isLogsFilterColumnValue(value)
? value
: { operator: '=' as LogsFilterOperator, values: Array.isArray(value) ? value : [value] }
for (const v of values) {
conditions.push({
propertyName: id,
value: v as FilterCondition['value'],
operator: '=',
operator,
})
}
}
@@ -37,7 +45,16 @@ export const LogsFilterBar = () => {
name: filter.value,
type: 'string',
options: filter.options ?? [],
operators: ['='],
operators:
filter.value === 'event_message'
? [
{ label: 'iLike', value: '~~*', group: 'pattern' },
{ label: 'Not iLike', value: '!~~*', group: 'pattern' },
]
: [
{ label: 'Equals', value: '=', group: 'comparison' },
{ label: 'Not equal', value: '<>', group: 'comparison' },
],
}))
// Local state because the FilterBar carries transient states
@@ -60,16 +77,28 @@ export const LogsFilterBar = () => {
)
if (!isValid) return
const filterConditions = next.conditions as FilterCondition[]
const groupedFilterConditions = groupBy(filterConditions, 'propertyName')
Object.entries(groupedFilterConditions).forEach(([name, conditions]) => {
table.getColumn(name)?.setFilterValue(conditions.map((x) => x.value))
})
// Coalesce conditions into one wrapped value per column. Mixed operators on
// the same column aren't expressible in the column-filter shape — last wins.
const wrappedByColumn = new Map<string, LogsColumnFilterValue>()
for (const cond of next.conditions as FilterCondition[]) {
const operator = cond.operator as LogsFilterOperator
const existing = wrappedByColumn.get(cond.propertyName)
if (!existing) {
wrappedByColumn.set(cond.propertyName, { operator, values: [String(cond.value)] })
} else {
existing.values.push(String(cond.value))
if (existing.operator !== operator) existing.operator = operator
}
}
for (const [name, wrapped] of wrappedByColumn) {
table.getColumn(name)?.setFilterValue(wrapped)
}
// Only clear filters owned by this bar — leaves externally-set filters
// (e.g. the timeline date range) untouched.
const managedNames = new Set(filterProperties.map((p) => p.name))
const nextNames = new Set(Object.keys(groupedFilterConditions))
const nextNames = new Set(wrappedByColumn.keys())
const filtersToRemove = table
.getState()
.columnFilters.filter((x) => managedNames.has(x.id) && !nextNames.has(x.id))
@@ -21,6 +21,10 @@ import {
DropdownMenuTrigger,
} from 'ui'
import {
isLogsFilterColumnValue,
type LogsColumnFilterValue,
} from '@/components/interfaces/UnifiedLogs/UnifiedLogs.filters'
import { DataTableFilterField } from '@/components/ui/DataTable/DataTable.types'
import { useCopyToClipboard } from '@/hooks/ui/useCopyToClipboard'
@@ -61,12 +65,16 @@ export function DataTableSheetRowAction<TData, TFields extends DataTableFilterFi
return (
<DropdownMenuItem
onClick={() => {
const filterValue = column?.getFilterValue() as undefined | Array<unknown>
const newValue = filterValue?.includes(value)
? filterValue
: [...(filterValue || []), value]
column?.setFilterValue(newValue)
// Equality filters use the wrapped { operator, values } shape so the
// row action stays compatible with the FilterBar (which writes `=` and `<>`).
const current = column?.getFilterValue()
const existing: LogsColumnFilterValue = isLogsFilterColumnValue(current)
? current
: { operator: '=', values: [] }
const next: LogsColumnFilterValue = existing.values.includes(String(value))
? existing
: { operator: existing.operator, values: [...existing.values, String(value)] }
column?.setFilterValue(next)
}}
className="flex items-center gap-2"
>
@@ -77,7 +85,12 @@ export function DataTableSheetRowAction<TData, TFields extends DataTableFilterFi
case 'input':
return (
<DropdownMenuItem
onClick={() => column?.setFilterValue(value)}
onClick={() =>
column?.setFilterValue({
operator: field.value === 'event_message' ? '~~*' : '=',
values: [String(value)],
} satisfies LogsColumnFilterValue)
}
className="flex items-center gap-2"
>
<Filter size={12} />
@@ -5,6 +5,7 @@ import { executeAnalyticsSql } from './execute-analytics-sql'
import { logsKeys } from './keys'
import { logsAllEndpointUrl, pickLogsQueryBuilder } from './logs-endpoint'
import { UNIFIED_LOGS_QUERY_OPTIONS, UnifiedLogsVariables } from './unified-logs-infinite-query'
import { parseLogsFilterUrlParams } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.filters'
import { getLogsChartQuery } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries'
import { getLogsChartQuery as getLogsChartQueryBq } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq'
import { ExecuteSqlError } from '@/data/sql/execute-sql-query'
@@ -88,13 +89,21 @@ export async function getUnifiedLogsChart(
error: Number(row.error) || 0,
}
// Filter levels if needed
const levelFilter = search.level
if (levelFilter && levelFilter.length > 0) {
// Reset levels not in the filter
if (!levelFilter.includes('success')) dataPoint.success = 0
if (!levelFilter.includes('warning')) dataPoint.warning = 0
if (!levelFilter.includes('error')) dataPoint.error = 0
// Zero out levels excluded by the active filter set.
// `=` filters narrow to an allow-list; `<>` filters carve out a deny-list.
const levelFilters = parseLogsFilterUrlParams(search.filter).filter(
(f) => f.column === 'level'
)
if (levelFilters.length > 0) {
const included = levelFilters.filter((f) => f.operator === '=').map((f) => f.value)
const excluded = new Set(
levelFilters.filter((f) => f.operator === '<>').map((f) => f.value)
)
const isActive = (lvl: 'success' | 'warning' | 'error') =>
(included.length === 0 || included.includes(lvl)) && !excluded.has(lvl)
if (!isActive('success')) dataPoint.success = 0
if (!isActive('warning')) dataPoint.warning = 0
if (!isActive('error')) dataPoint.error = 0
}
dataByTimestamp.set(milliseconds, dataPoint)