mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 09:25:06 +03:00
feat(studio): add workers to unified logs FE-4281 (#49682)
## Problem Unified Logs does not expose Workers logs, so users cannot search Workers ingress, runtime, or build events alongside other services. ## Fix Add a Workers log type that classifies all three Workers OTEL streams. Gate the option and any persisted Workers filters with the existing Workers feature flag. ## How to test - Enable the Workers feature flag and open Unified Logs. - Select Workers from the Log Type filter. - Expected result: Unified Logs shows ingress, runtime, and build events with the Workers icon. - Disable the Workers feature flag and load a URL containing `log_type:eq:workers`. - Expected result: the Workers option and filter are removed, and Workers logs are not queried. - Run `./node_modules/.bin/vitest --run components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts components/interfaces/UnifiedLogs/UnifiedLogs.utils.test.ts data/workers/worker-logs-query.test.ts` from `apps/studio`. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added Workers as a selectable log type in Unified Logs. * Unified Logs now combines worker ingress, guest, and API streams under the Workers category. * Added a dedicated Workers icon and worker log filtering. * **Improvements** * Worker filters and URL parameters respect feature availability. * Worker details show relevant metadata while omitting unavailable HTTP fields. * Improved handling of worker log levels, statuses, and raw data. * Added stronger validation for unified log data. * **Tests** * Added coverage for worker routing, filtering, feature visibility, parsing, and metadata redaction. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
1 parent
ec1029dff0
commit
be030229bf
21 files changed
+695
-173
No files matched your search
+1
-1
@@ -52,7 +52,7 @@ export const DetailRow = ({
|
||||
const valueEl = isLoading ? (
|
||||
<Skeleton className="h-4 w-24" />
|
||||
) : (
|
||||
<FieldValue config={config} value={value} wrap={config.wrap} level={level} />
|
||||
<FieldValue config={config} value={value} wrap={config.wrap} level={level ?? undefined} />
|
||||
)
|
||||
|
||||
const rowClass = cn(
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { getLogDataForMetadataVisibility } from './ServiceFlowPanel'
|
||||
|
||||
describe('getLogDataForMetadataVisibility', () => {
|
||||
const logData = {
|
||||
id: 'worker-log',
|
||||
metadata: { source: 'worker_guest_logs', worker: 'api' },
|
||||
raw_log_data: {
|
||||
event_message: 'Worker failed',
|
||||
metadata: { request_id: 'request-id' },
|
||||
},
|
||||
}
|
||||
|
||||
it('redacts top-level and nested metadata when metadata is hidden', () => {
|
||||
const visibleData = JSON.parse(JSON.stringify(getLogDataForMetadataVisibility(logData, false)))
|
||||
|
||||
expect(visibleData).toEqual({
|
||||
id: 'worker-log',
|
||||
raw_log_data: { event_message: 'Worker failed' },
|
||||
})
|
||||
})
|
||||
|
||||
it('preserves metadata when metadata is visible', () => {
|
||||
expect(getLogDataForMetadataVisibility(logData, true)).toBe(logData)
|
||||
})
|
||||
})
|
||||
@@ -27,7 +27,7 @@ import { ServiceFlowPanelControls } from './ServiceFlow/components/ServiceFlowPa
|
||||
import { DetailSectionHeader } from './ServiceFlow/components/shared/DetailSection'
|
||||
import { ColumnSchema } from './UnifiedLogs.schema'
|
||||
import { QuerySearchParamsType } from './UnifiedLogs.types'
|
||||
import { getRowTimestampMs } from './UnifiedLogs.utils'
|
||||
import { getRawLogData, getRowTimestampMs } from './UnifiedLogs.utils'
|
||||
import { useDataTable } from '@/components/ui/DataTable/providers/DataTableProvider'
|
||||
import {
|
||||
SERVICE_FLOW_TYPES,
|
||||
@@ -44,6 +44,20 @@ interface ServiceFlowPanelProps {
|
||||
searchParameters: QuerySearchParamsType
|
||||
}
|
||||
|
||||
export function getLogDataForMetadataVisibility(data: unknown, metadataVisible: boolean) {
|
||||
if (metadataVisible || typeof data !== 'object' || data === null) return data
|
||||
|
||||
const redactedData = { ...data, metadata: undefined }
|
||||
const rawLogData = 'raw_log_data' in data ? data.raw_log_data : undefined
|
||||
|
||||
if (typeof rawLogData !== 'object' || rawLogData === null) return redactedData
|
||||
|
||||
return {
|
||||
...redactedData,
|
||||
raw_log_data: { ...rawLogData, metadata: undefined },
|
||||
}
|
||||
}
|
||||
|
||||
export function ServiceFlowPanel({
|
||||
dock,
|
||||
setDock,
|
||||
@@ -105,14 +119,8 @@ export function ServiceFlowPanel({
|
||||
// Prepare JSON data for Raw JSON tab
|
||||
const jsonData =
|
||||
shouldShowServiceFlow && serviceFlowData?.result?.[0] ? serviceFlowData.result[0] : selectedRow
|
||||
|
||||
const formattedJsonData =
|
||||
!logsMetadata && 'raw_log_data' in jsonData && 'metadata' in jsonData.raw_log_data
|
||||
? {
|
||||
...jsonData,
|
||||
raw_log_data: { ...jsonData.raw_log_data, metadata: undefined },
|
||||
}
|
||||
: jsonData
|
||||
const rawLogData = getRawLogData(jsonData)
|
||||
const formattedJsonData = getLogDataForMetadataVisibility(rawLogData, logsMetadata)
|
||||
|
||||
return (
|
||||
<>
|
||||
|
||||
@@ -27,17 +27,14 @@ export const LOG_TYPES_LABELS = {
|
||||
supavisor: 'Supavisor',
|
||||
pgbouncer: 'PgBouncer',
|
||||
multigres: 'Multigres',
|
||||
workers: 'Workers',
|
||||
}
|
||||
|
||||
type LogType = keyof typeof LOG_TYPES_LABELS
|
||||
export const LOG_TYPES = Object.keys(LOG_TYPES_LABELS) as [LogType, ...LogType[]]
|
||||
export const DEFAULT_LOG_TYPES = ['postgres', 'edge'] as const
|
||||
|
||||
// ClickHouse `source` value for each unified log type. Single source of truth
|
||||
// consumed by both the unified logs list query (LOG_TYPE_CONDITION in
|
||||
// UnifiedLogs.queries.ts) and the single-log inspection point lookup
|
||||
// (unified-log-inspection-query.ts).
|
||||
export const LOG_TYPE_TO_SOURCE: Record<LogType, string> = {
|
||||
export const LOG_TYPE_TO_SOURCE: Record<Exclude<LogType, 'workers'>, string> = {
|
||||
edge: 'edge_logs',
|
||||
postgrest: 'postgrest_logs',
|
||||
storage: 'storage_logs',
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { act, renderHook } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { useFilterSearchSync } from './UnifiedLogs.hooks'
|
||||
|
||||
describe('useFilterSearchSync', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('defers platform filter synchronization until feature flags load', () => {
|
||||
vi.useFakeTimers()
|
||||
const applyFilterSearch = vi.fn()
|
||||
const workersFilter = [{ id: 'log_type', value: ['workers'] }]
|
||||
const { rerender } = renderHook(
|
||||
({ enabled }) =>
|
||||
useFilterSearchSync({ applyFilterSearch, columnFilters: workersFilter, enabled }),
|
||||
{ initialProps: { enabled: false } }
|
||||
)
|
||||
|
||||
act(() => vi.advanceTimersByTime(1_000))
|
||||
expect(applyFilterSearch).not.toHaveBeenCalled()
|
||||
|
||||
rerender({ enabled: true })
|
||||
act(() => vi.advanceTimersByTime(250))
|
||||
|
||||
expect(applyFilterSearch).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('synchronizes self-hosted filters without waiting for feature flags', () => {
|
||||
vi.useFakeTimers()
|
||||
const applyFilterSearch = vi.fn()
|
||||
|
||||
renderHook(() => useFilterSearchSync({ applyFilterSearch, columnFilters: [], enabled: true }))
|
||||
|
||||
act(() => vi.advanceTimersByTime(250))
|
||||
|
||||
expect(applyFilterSearch).toHaveBeenCalledOnce()
|
||||
})
|
||||
})
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { ColumnFiltersState } from '@tanstack/react-table'
|
||||
import { useDebounce } from 'common'
|
||||
import { useQueryState } from 'nuqs'
|
||||
import { useEffect, useMemo, useRef } from 'react'
|
||||
|
||||
@@ -5,6 +7,25 @@ import { SEARCH_PARAMS_PARSER } from './UnifiedLogs.constants'
|
||||
import { SHORTCUT_IDS } from '@/state/shortcuts/registry'
|
||||
import { useShortcut } from '@/state/shortcuts/useShortcut'
|
||||
|
||||
export const useFilterSearchSync = ({
|
||||
applyFilterSearch,
|
||||
columnFilters,
|
||||
enabled,
|
||||
}: {
|
||||
applyFilterSearch: () => void
|
||||
columnFilters: ColumnFiltersState
|
||||
enabled: boolean
|
||||
}) => {
|
||||
const debouncedApplyFilterSearch = useDebounce(applyFilterSearch, 250)
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) return
|
||||
|
||||
debouncedApplyFilterSearch()
|
||||
return () => debouncedApplyFilterSearch.cancel()
|
||||
}, [columnFilters, debouncedApplyFilterSearch, enabled])
|
||||
}
|
||||
|
||||
export const useResetFocus = () => {
|
||||
useShortcut(SHORTCUT_IDS.UNIFIED_LOGS_RESET_FOCUS, () => {
|
||||
// FIXME: some dedicated div[tabindex="0"] do not auto-unblur (e.g. the DataTableFilterResetButton)
|
||||
|
||||
@@ -46,6 +46,41 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
|
||||
expect(where).not.toContain(`log_attributes['request.path'] LIKE '%/storage/%'`)
|
||||
})
|
||||
|
||||
it('routes the `workers` log type to every worker OTEL stream', () => {
|
||||
const sql = getUnifiedLogsQuery(withFilters('log_type:eq:workers'))
|
||||
const where = sql.split(/\bWHERE\b/)[1] ?? ''
|
||||
expect(where).toContain(
|
||||
`log_attributes['source'] IN ('worker_ingress_logs','worker_guest_logs','worker_api_logs')`
|
||||
)
|
||||
expect(where).not.toContain(`source = 'workers'`)
|
||||
})
|
||||
|
||||
it('classifies every worker OTEL stream as workers in the projected log type', () => {
|
||||
const sql = getUnifiedLogsQuery(baseSearch)
|
||||
expect(sql).toContain(
|
||||
`WHEN log_attributes['source'] IN ('worker_ingress_logs','worker_guest_logs','worker_api_logs') THEN 'workers'`
|
||||
)
|
||||
})
|
||||
|
||||
it('excludes every worker OTEL stream when the workers log type is negated', () => {
|
||||
const sql = getUnifiedLogsQuery(withFilters('log_type:neq:workers'))
|
||||
const where = sql.split(/\bWHERE\b/)[1] ?? ''
|
||||
expect(where).toContain(
|
||||
`NOT (log_attributes['source'] IN ('worker_ingress_logs','worker_guest_logs','worker_api_logs'))`
|
||||
)
|
||||
})
|
||||
|
||||
it('projects only Workers fields that exist on worker logs', () => {
|
||||
const sql = getUnifiedLogsQuery(withFilters('log_type:eq:workers'))
|
||||
const workerCondition =
|
||||
"log_attributes['source'] IN ('worker_ingress_logs','worker_guest_logs','worker_api_logs')"
|
||||
|
||||
expect(sql).toContain(`WHEN ${workerCondition} THEN null`)
|
||||
expect(sql).toContain(`if(${workerCondition}, null, log_attributes['request.method'])`)
|
||||
expect(sql).toContain(`if(${workerCondition}, null, log_attributes['request.path'])`)
|
||||
expect(sql).toContain(`if(${workerCondition}, log_attributes, map()) AS metadata`)
|
||||
})
|
||||
|
||||
it('escapes single quotes in filter values to prevent SQL injection', () => {
|
||||
const sql = getUnifiedLogsQuery(
|
||||
withFilters(`method:eq:G'ET`, `pathname:eq:/customers'; DROP TABLE logs --`)
|
||||
@@ -262,6 +297,15 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
|
||||
`if(source = 'auth_logs', log_attributes['status'], log_attributes['response.status_code'])`
|
||||
)
|
||||
})
|
||||
|
||||
it('does not classify Workers rows into a severity bucket', () => {
|
||||
const sql = getLogsChartQuery(withFilters('log_type:eq:workers'))
|
||||
const workerCondition =
|
||||
"log_attributes['source'] IN ('worker_ingress_logs','worker_guest_logs','worker_api_logs')"
|
||||
|
||||
expect(sql).toContain(`WHEN ${workerCondition} THEN null`)
|
||||
expect(sql).not.toContain(`WHEN ${workerCondition} THEN 'success'`)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getFacetCountQuery', () => {
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
safeSql,
|
||||
type SafeLogSqlFragment,
|
||||
} from '@/data/logs/safe-analytics-sql'
|
||||
import { WORKER_LOG_SOURCES } from '@/lib/constants/workers'
|
||||
|
||||
// Operator fragments for SQL emission. `safeSql` rejects plain strings, so we
|
||||
// pre-brand the keywords we want to switch between.
|
||||
@@ -54,12 +55,21 @@ const HTTP_STATUS_EXPR: SafeLogSqlFragment = safeSql`if(source = 'auth_logs', lo
|
||||
* logs from postgREST / storage-api and are intentionally not part of unified
|
||||
* logs; the UI surfaces gateway HTTP traffic for those buckets.
|
||||
*/
|
||||
const LOG_TYPE_CONDITION: Record<string, SafeLogSqlFragment> = Object.fromEntries(
|
||||
Object.entries(LOG_TYPE_TO_SOURCE).map(([type, source]) => [
|
||||
type,
|
||||
safeSql`source = ${lit(source)}`,
|
||||
])
|
||||
)
|
||||
const WORKER_LOG_SOURCE_VALUES = Object.values(WORKER_LOG_SOURCES)
|
||||
const WORKER_LOG_SOURCE_CONDITION = safeSql`log_attributes['source'] IN (${joinSqlFragments(
|
||||
WORKER_LOG_SOURCE_VALUES.map((source) => lit(source)),
|
||||
','
|
||||
)})`
|
||||
|
||||
const LOG_TYPE_CONDITION: Record<string, SafeLogSqlFragment> = {
|
||||
...Object.fromEntries(
|
||||
Object.entries(LOG_TYPE_TO_SOURCE).map(([type, source]) => [
|
||||
type,
|
||||
safeSql`source = ${lit(source)}`,
|
||||
])
|
||||
),
|
||||
workers: WORKER_LOG_SOURCE_CONDITION,
|
||||
}
|
||||
|
||||
// Derived `log_type` column for SELECT / GROUP BY / countIf use.
|
||||
// WHEN source = 'edge_logs' AND ${ATTR.path} LIKE '%/rest/%' THEN 'postgrest'
|
||||
@@ -75,6 +85,7 @@ const LOG_TYPE_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
WHEN source = 'supavisor_logs' THEN 'supavisor'
|
||||
WHEN source = 'pgbouncer_logs' THEN 'pgbouncer'
|
||||
WHEN source = 'multigres_logs' THEN 'multigres'
|
||||
WHEN ${WORKER_LOG_SOURCE_CONDITION} THEN 'workers'
|
||||
ELSE source
|
||||
END`
|
||||
|
||||
@@ -82,10 +93,15 @@ const LOG_TYPE_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
// auth-service `status` attribute for auth rows, and the Postgres
|
||||
// `parsed.sql_state_code` (e.g. `42P01`) for postgres rows.
|
||||
const STATUS_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
WHEN ${WORKER_LOG_SOURCE_CONDITION} THEN null
|
||||
WHEN source = 'postgres_logs' THEN toString(log_attributes['parsed.sql_state_code'])
|
||||
ELSE toString((${HTTP_STATUS_EXPR}))
|
||||
END`
|
||||
|
||||
const METHOD_EXPR: SafeLogSqlFragment = safeSql`if(${WORKER_LOG_SOURCE_CONDITION}, null, ${ATTR.method})`
|
||||
const PATHNAME_EXPR: SafeLogSqlFragment = safeSql`if(${WORKER_LOG_SOURCE_CONDITION}, null, ${ATTR.path})`
|
||||
const METADATA_EXPR: SafeLogSqlFragment = safeSql`if(${WORKER_LOG_SOURCE_CONDITION}, log_attributes, map())`
|
||||
|
||||
// SQL expression for derived `level`. Used inline (not as alias reference)
|
||||
// because the OTEL endpoint can't resolve aliases inside countIf when the
|
||||
// alias is not in GROUP BY.
|
||||
@@ -95,6 +111,7 @@ const STATUS_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
// success/warning/error by status. Postgres-style severity is the
|
||||
// fallback for rows without a status code.
|
||||
const LEVEL_EXPR: SafeLogSqlFragment = safeSql`CASE
|
||||
WHEN ${WORKER_LOG_SOURCE_CONDITION} THEN null
|
||||
WHEN (${HTTP_STATUS_EXPR}) != '' AND toInt32OrZero((${HTTP_STATUS_EXPR})) >= 500 THEN 'error'
|
||||
WHEN (${HTTP_STATUS_EXPR}) != '' AND toInt32OrZero((${HTTP_STATUS_EXPR})) BETWEEN 400 AND 499 THEN 'warning'
|
||||
WHEN (${HTTP_STATUS_EXPR}) != '' AND toInt32OrZero((${HTTP_STATUS_EXPR})) BETWEEN 200 AND 299 THEN 'success'
|
||||
@@ -262,10 +279,11 @@ const ROW_PROJECTION: SafeLogSqlFragment = safeSql`
|
||||
${LOG_TYPE_EXPR} AS log_type,
|
||||
${STATUS_EXPR} AS status,
|
||||
${LEVEL_EXPR} AS level,
|
||||
${ATTR.path} AS pathname,
|
||||
${PATHNAME_EXPR} AS pathname,
|
||||
event_message,
|
||||
${ATTR.method} AS method,
|
||||
${METHOD_EXPR} AS method,
|
||||
${AUTH_USER_EXPR} AS auth_user,
|
||||
${METADATA_EXPR} AS metadata,
|
||||
null AS log_count,
|
||||
null AS logs
|
||||
`
|
||||
@@ -446,11 +464,11 @@ export const getFacetCountQuery = ({
|
||||
: facet === 'level'
|
||||
? LEVEL_EXPR
|
||||
: facet === 'method'
|
||||
? ATTR.method
|
||||
? METHOD_EXPR
|
||||
: facet === 'status'
|
||||
? STATUS_EXPR
|
||||
: facet === 'pathname'
|
||||
? ATTR.path
|
||||
? PATHNAME_EXPR
|
||||
: safeSql`log_attributes[${lit(facet)}]`
|
||||
|
||||
const conditions: SafeLogSqlFragment[] = [
|
||||
@@ -488,7 +506,7 @@ export const getLogsCountQuery = (search: QuerySearchParamsType): SafeLogSqlFrag
|
||||
total: safeSql`'all'`,
|
||||
log_type: LOG_TYPE_EXPR,
|
||||
level: LEVEL_EXPR,
|
||||
method: ATTR.method,
|
||||
method: METHOD_EXPR,
|
||||
status: STATUS_EXPR,
|
||||
}
|
||||
|
||||
|
||||
@@ -10,16 +10,17 @@ import {
|
||||
export const columnSchema = z.object({
|
||||
id: z.string(),
|
||||
log_type: z.enum(LOG_TYPES),
|
||||
method: z.enum(METHODS),
|
||||
pathname: z.string(),
|
||||
level: z.enum(LEVELS),
|
||||
status: z.number(),
|
||||
method: z.enum(METHODS).nullable(),
|
||||
pathname: z.string().nullable(),
|
||||
level: z.enum(LEVELS).nullable(),
|
||||
status: z.number().nullable(),
|
||||
date: z.date(),
|
||||
timestamp: z.number(),
|
||||
event_message: z.string().optional(),
|
||||
log_count: z.number().optional(), // used to count function logs for a given execution_id
|
||||
logs: z.array(z.any()).optional(), // array of function logs
|
||||
auth_user: z.string().optional(),
|
||||
auth_user: z.string().nullable().optional(),
|
||||
metadata: z.record(z.string(), z.unknown()).nullable().optional(),
|
||||
})
|
||||
|
||||
export type ColumnSchema = z.infer<typeof columnSchema>
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
useReactTable,
|
||||
VisibilityState,
|
||||
} from '@tanstack/react-table'
|
||||
import { LOCAL_STORAGE_KEYS, useDebounce, useParams } from 'common'
|
||||
import { IS_PLATFORM, LOCAL_STORAGE_KEYS, useFeatureFlags, useFlag, useParams } from 'common'
|
||||
import { Loader2, PanelLeftClose, PanelLeftOpen } from 'lucide-react'
|
||||
import { useQueryStates } from 'nuqs'
|
||||
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||
@@ -42,14 +42,16 @@ import {
|
||||
buildFilterSearchUpdate,
|
||||
parseLogsFilterUrlParams,
|
||||
} from './UnifiedLogs.filters'
|
||||
import { useLiveMode, useResetFocus } from './UnifiedLogs.hooks'
|
||||
import { useFilterSearchSync, useLiveMode, useResetFocus } from './UnifiedLogs.hooks'
|
||||
import { isUserFilterUnreachable } from './UnifiedLogs.queries'
|
||||
import { ColumnSchema } from './UnifiedLogs.schema'
|
||||
import { QuerySearchParamsType } from './UnifiedLogs.types'
|
||||
import {
|
||||
gateMultigresLogType,
|
||||
gateLogTypeFilters,
|
||||
gateLogTypeOptions,
|
||||
getFacetedUniqueValues,
|
||||
getLevelRowClassName,
|
||||
getWorkersLogsAvailability,
|
||||
} from './UnifiedLogs.utils'
|
||||
import { LEVELS } from '@/components/ui/DataTable/DataTable.constants'
|
||||
import { Option } from '@/components/ui/DataTable/DataTable.types'
|
||||
@@ -93,10 +95,25 @@ export const UnifiedLogs = () => {
|
||||
const { ref: projectRef } = useParams()
|
||||
const track = useTrack()
|
||||
const [search, setSearch] = useQueryStates(SEARCH_PARAMS_PARSER)
|
||||
const showMultigresLogs = useShowMultigresLogs()
|
||||
const { hasLoaded: flagsLoaded } = useFeatureFlags()
|
||||
const workersEnabled = !!useFlag('workers')
|
||||
const workersAvailability = getWorkersLogsAvailability({
|
||||
isPlatform: IS_PLATFORM,
|
||||
flagsLoaded,
|
||||
workersEnabled,
|
||||
})
|
||||
const visibleSearchFilters = gateLogTypeFilters(search.filter, {
|
||||
multigres: showMultigresLogs,
|
||||
workers: workersAvailability.preserveWorkersFilter,
|
||||
})
|
||||
|
||||
const defaultColumnSorting = search.sort ? [search.sort] : []
|
||||
const defaultColumnVisibility = { uuid: false }
|
||||
const defaultColumnFilters = buildDefaultColumnFilters(search)
|
||||
const defaultColumnFilters = buildDefaultColumnFilters({
|
||||
...search,
|
||||
filter: visibleSearchFilters,
|
||||
})
|
||||
|
||||
const [topBarHeight, setTopBarHeight] = useState(0)
|
||||
const topBarRef = useRef<HTMLDivElement>(null)
|
||||
@@ -112,8 +129,6 @@ export const UnifiedLogs = () => {
|
||||
return () => observer.unobserve(topBar)
|
||||
}, [])
|
||||
|
||||
const showMultigresLogs = useShowMultigresLogs()
|
||||
|
||||
const [sorting, setSorting] = useState<SortingState>(defaultColumnSorting)
|
||||
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>(defaultColumnFilters)
|
||||
const [rowSelection, setRowSelection] = useState<RowSelectionState>({})
|
||||
@@ -135,19 +150,26 @@ export const UnifiedLogs = () => {
|
||||
|
||||
// Create a stable query key object by removing nulls/undefined, id, and live
|
||||
// Mainly to prevent the react queries from unnecessarily re-fetching
|
||||
const searchParameters = useMemo(
|
||||
() =>
|
||||
Object.entries(search).reduce(
|
||||
(acc, [key, value]) => {
|
||||
if (!['id', 'live'].includes(key) && value !== null && value !== undefined) {
|
||||
acc[key] = value
|
||||
}
|
||||
return acc
|
||||
},
|
||||
{} as Record<string, unknown>
|
||||
) as QuerySearchParamsType,
|
||||
[search]
|
||||
)
|
||||
const searchParameters = useMemo(() => {
|
||||
const parameters = Object.entries(search).reduce(
|
||||
(acc, [key, value]) => {
|
||||
if (!['id', 'live'].includes(key) && value !== null && value !== undefined) {
|
||||
acc[key] = value
|
||||
}
|
||||
return acc
|
||||
},
|
||||
{} as Record<string, unknown>
|
||||
) as QuerySearchParamsType
|
||||
|
||||
if (parameters.filter) {
|
||||
parameters.filter =
|
||||
gateLogTypeFilters(parameters.filter, {
|
||||
multigres: showMultigresLogs,
|
||||
workers: workersAvailability.canQueryWorkers,
|
||||
}) ?? null
|
||||
}
|
||||
return parameters
|
||||
}, [search, showMultigresLogs, workersAvailability.canQueryWorkers])
|
||||
|
||||
const {
|
||||
data: unifiedLogsData,
|
||||
@@ -223,7 +245,7 @@ export const UnifiedLogs = () => {
|
||||
}, [search.filter])
|
||||
|
||||
const getRowClassName = <
|
||||
TData extends { date: Date; level: (typeof LEVELS)[number]; timestamp: number },
|
||||
TData extends { date: Date; level: (typeof LEVELS)[number] | null; timestamp: number },
|
||||
>(
|
||||
row: Row<TData>
|
||||
) => {
|
||||
@@ -274,7 +296,10 @@ export const UnifiedLogs = () => {
|
||||
// Will need to refactor this bit
|
||||
// - Each facet just handles its own state, rather than getting passed down like this
|
||||
const filterFields = useMemo(() => {
|
||||
const gatedFields = gateMultigresLogType(defaultFilterFields, showMultigresLogs)
|
||||
const gatedFields = gateLogTypeOptions(defaultFilterFields, {
|
||||
multigres: showMultigresLogs,
|
||||
workers: workersAvailability.canQueryWorkers,
|
||||
})
|
||||
|
||||
return gatedFields.map((field) => {
|
||||
const facetsField = facets?.[field.value]
|
||||
@@ -302,17 +327,24 @@ export const UnifiedLogs = () => {
|
||||
|
||||
return { ...field, options }
|
||||
})
|
||||
}, [facets, showMultigresLogs])
|
||||
}, [facets, showMultigresLogs, workersAvailability.canQueryWorkers])
|
||||
|
||||
const applyFilterSearch = () => {
|
||||
setSearch(buildFilterSearchUpdate(columnFilters, filterFields))
|
||||
const update = buildFilterSearchUpdate(columnFilters, filterFields)
|
||||
if (Array.isArray(update.filter)) {
|
||||
update.filter = gateLogTypeFilters(update.filter.map(String), {
|
||||
multigres: showMultigresLogs,
|
||||
workers: workersAvailability.canQueryWorkers,
|
||||
})
|
||||
}
|
||||
setSearch(update)
|
||||
}
|
||||
|
||||
const debouncedApplyFilterSearch = useDebounce(applyFilterSearch, 250)
|
||||
|
||||
useEffect(() => {
|
||||
debouncedApplyFilterSearch()
|
||||
}, [columnFilters, debouncedApplyFilterSearch])
|
||||
useFilterSearchSync({
|
||||
applyFilterSearch,
|
||||
columnFilters,
|
||||
enabled: workersAvailability.readyToSyncFilters,
|
||||
})
|
||||
|
||||
useEffect(() => {
|
||||
setSearch({ sort: sorting?.[0] || null })
|
||||
@@ -485,7 +517,7 @@ export const UnifiedLogs = () => {
|
||||
setColumnVisibility={setColumnVisibility}
|
||||
searchParamsParser={SEARCH_PARAMS_PARSER}
|
||||
emptyStateMessage={
|
||||
isUserFilterUnreachable(search) ? (
|
||||
isUserFilterUnreachable(searchParameters) ? (
|
||||
<div className="text-sm flex flex-col gap-y-1">
|
||||
<p className="text-foreground-light">No results found</p>
|
||||
<p className="text-foreground-lighter">
|
||||
|
||||
@@ -2,8 +2,11 @@ import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
buildUnifiedLogsUrl,
|
||||
gateMultigresLogType,
|
||||
gateLogTypeFilters,
|
||||
gateLogTypeOptions,
|
||||
getEventMessageDisplay,
|
||||
getRawLogData,
|
||||
getWorkersLogsAvailability,
|
||||
parseMultigresEventMessage,
|
||||
} from './UnifiedLogs.utils'
|
||||
|
||||
@@ -94,7 +97,55 @@ describe('getEventMessageDisplay', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('gateMultigresLogType', () => {
|
||||
describe('getRawLogData', () => {
|
||||
it('returns only the real Workers payload fields', () => {
|
||||
const row = {
|
||||
event_message: 'Error: Dynamic require of "path" is not supported',
|
||||
id: '51a29911-9293-4616-8984-743cc548b629',
|
||||
metadata: {
|
||||
cw_event_id: '39883203917805946105278943454814281535421893832620638214',
|
||||
launch_id: '1788424715503435269',
|
||||
log_group: '/aws/lambda-microvms/workers/cxkpapyhaaywrtudnqpl/api',
|
||||
log_stream: 'launch-1788424715503435269',
|
||||
source: 'worker_guest_logs',
|
||||
worker: 'api',
|
||||
},
|
||||
project: 'cxkpapyhaaywrtudnqpl',
|
||||
timestamp: 1788424716876000,
|
||||
log_type: 'workers' as const,
|
||||
status: null,
|
||||
level: null,
|
||||
method: null,
|
||||
pathname: null,
|
||||
auth_user: null,
|
||||
date: new Date(1788424716876),
|
||||
}
|
||||
|
||||
expect(getRawLogData(row)).toEqual({
|
||||
id: '51a29911-9293-4616-8984-743cc548b629',
|
||||
timestamp: 1788424716876000,
|
||||
event_message: 'Error: Dynamic require of "path" is not supported',
|
||||
metadata: row.metadata,
|
||||
})
|
||||
})
|
||||
|
||||
it('returns non-Workers rows unchanged', () => {
|
||||
const row = {
|
||||
id: 'edge-log',
|
||||
timestamp: 1788424716876000,
|
||||
log_type: 'edge' as const,
|
||||
status: 200,
|
||||
method: 'GET' as const,
|
||||
pathname: '/rest/v1',
|
||||
level: 'success' as const,
|
||||
date: new Date(1788424716876),
|
||||
}
|
||||
|
||||
expect(getRawLogData(row)).toBe(row)
|
||||
})
|
||||
})
|
||||
|
||||
describe('gateLogTypeOptions', () => {
|
||||
const fields = [
|
||||
{ value: 'date' },
|
||||
{
|
||||
@@ -102,23 +153,110 @@ describe('gateMultigresLogType', () => {
|
||||
options: [
|
||||
{ label: 'Postgres', value: 'postgres' },
|
||||
{ label: 'Multigres', value: 'multigres' },
|
||||
{ label: 'Workers', value: 'workers' },
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
it('drops the multigres log_type option when the flag is disabled', () => {
|
||||
const gated = gateMultigresLogType(fields, false)
|
||||
it('drops log_type options whose flags are disabled', () => {
|
||||
const gated = gateLogTypeOptions(fields, { multigres: false, workers: false })
|
||||
const logType = gated.find((field) => field.value === 'log_type')
|
||||
expect(logType?.options?.map((option) => option.value)).toEqual(['postgres'])
|
||||
})
|
||||
|
||||
it('keeps the multigres option when the flag is enabled', () => {
|
||||
const gated = gateMultigresLogType(fields, true)
|
||||
it('keeps independently enabled log types', () => {
|
||||
const gated = gateLogTypeOptions(fields, { multigres: false, workers: true })
|
||||
const logType = gated.find((field) => field.value === 'log_type')
|
||||
expect(logType?.options?.map((option) => option.value)).toEqual(['postgres', 'workers'])
|
||||
})
|
||||
|
||||
it('returns the original fields when every gated log type is enabled', () => {
|
||||
const gated = gateLogTypeOptions(fields, { multigres: true, workers: true })
|
||||
expect(gated).toBe(fields)
|
||||
})
|
||||
|
||||
it('leaves non log_type fields untouched', () => {
|
||||
const gated = gateMultigresLogType(fields, false)
|
||||
const gated = gateLogTypeOptions(fields, { workers: false })
|
||||
expect(gated.find((field) => field.value === 'date')).toEqual({ value: 'date' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('gateLogTypeFilters', () => {
|
||||
it('removes disabled log types from equality and inequality filters', () => {
|
||||
expect(
|
||||
gateLogTypeFilters(
|
||||
['log_type:eq:workers', 'log_type:neq:multigres', 'log_type:eq:postgres', 'method:eq:GET'],
|
||||
{ workers: false, multigres: false }
|
||||
)
|
||||
).toEqual(['log_type:eq:postgres', 'method:eq:GET'])
|
||||
})
|
||||
|
||||
it('keeps enabled log types and unrelated filters unchanged', () => {
|
||||
const filters = ['log_type:eq:workers', 'method:eq:GET']
|
||||
expect(gateLogTypeFilters(filters, { workers: true })).toBe(filters)
|
||||
})
|
||||
|
||||
it('preserves absent filter values', () => {
|
||||
expect(gateLogTypeFilters(undefined, { workers: false })).toBeUndefined()
|
||||
expect(gateLogTypeFilters(null, { workers: false })).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getWorkersLogsAvailability', () => {
|
||||
const workersFilter = ['log_type:eq:workers']
|
||||
|
||||
it('preserves an unresolved platform filter without allowing it into queries or sync', () => {
|
||||
const availability = getWorkersLogsAvailability({
|
||||
isPlatform: true,
|
||||
flagsLoaded: false,
|
||||
workersEnabled: false,
|
||||
})
|
||||
|
||||
expect(gateLogTypeFilters(workersFilter, { workers: availability.preserveWorkersFilter })).toBe(
|
||||
workersFilter
|
||||
)
|
||||
expect(gateLogTypeFilters(workersFilter, { workers: availability.canQueryWorkers })).toEqual([])
|
||||
expect(availability.readyToSyncFilters).toBe(false)
|
||||
})
|
||||
|
||||
it('allows Workers filters and queries when the platform flag is enabled', () => {
|
||||
const availability = getWorkersLogsAvailability({
|
||||
isPlatform: true,
|
||||
flagsLoaded: true,
|
||||
workersEnabled: true,
|
||||
})
|
||||
|
||||
expect(availability).toEqual({
|
||||
canQueryWorkers: true,
|
||||
preserveWorkersFilter: true,
|
||||
readyToSyncFilters: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('removes Workers filters and queries when the platform flag is disabled', () => {
|
||||
const availability = getWorkersLogsAvailability({
|
||||
isPlatform: true,
|
||||
flagsLoaded: true,
|
||||
workersEnabled: false,
|
||||
})
|
||||
|
||||
expect(availability).toEqual({
|
||||
canQueryWorkers: false,
|
||||
preserveWorkersFilter: false,
|
||||
readyToSyncFilters: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('syncs generic filters immediately while keeping Workers unavailable on self-hosted', () => {
|
||||
const availability = getWorkersLogsAvailability({
|
||||
isPlatform: false,
|
||||
workersEnabled: false,
|
||||
})
|
||||
|
||||
expect(availability).toEqual({
|
||||
canQueryWorkers: false,
|
||||
preserveWorkersFilter: false,
|
||||
readyToSyncFilters: true,
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -2,12 +2,32 @@ import { type Table as TTable } from '@tanstack/react-table'
|
||||
import { cn } from 'ui'
|
||||
|
||||
import { LOG_TYPES_LABELS } from './UnifiedLogs.constants'
|
||||
import { FacetMetadataSchema } from './UnifiedLogs.schema'
|
||||
import { parseLogsFilterUrlParams } from './UnifiedLogs.filters'
|
||||
import { ColumnSchema, FacetMetadataSchema } from './UnifiedLogs.schema'
|
||||
import { LEVELS } from '@/components/ui/DataTable/DataTable.constants'
|
||||
import { Option } from '@/components/ui/DataTable/DataTable.types'
|
||||
import type { UnifiedLogInspectionEntry } from '@/data/logs/unified-log-inspection-query'
|
||||
|
||||
export type UnifiedLogType = keyof typeof LOG_TYPES_LABELS
|
||||
|
||||
export function getWorkersLogsAvailability({
|
||||
isPlatform,
|
||||
flagsLoaded,
|
||||
workersEnabled,
|
||||
}: {
|
||||
isPlatform: boolean
|
||||
flagsLoaded?: boolean
|
||||
workersEnabled: boolean
|
||||
}) {
|
||||
const flagsReady = flagsLoaded === true
|
||||
|
||||
return {
|
||||
canQueryWorkers: isPlatform && flagsReady && workersEnabled,
|
||||
preserveWorkersFilter: isPlatform && (!flagsReady || workersEnabled),
|
||||
readyToSyncFilters: !isPlatform || flagsReady,
|
||||
}
|
||||
}
|
||||
|
||||
export const buildUnifiedLogsUrl = ({
|
||||
projectRef,
|
||||
logType,
|
||||
@@ -63,6 +83,21 @@ export function getRowTimestampMs(
|
||||
return null
|
||||
}
|
||||
|
||||
type WorkersRawLogData = Pick<ColumnSchema, 'id' | 'timestamp' | 'event_message' | 'metadata'>
|
||||
|
||||
export function getRawLogData(
|
||||
row: ColumnSchema | UnifiedLogInspectionEntry
|
||||
): ColumnSchema | UnifiedLogInspectionEntry | WorkersRawLogData {
|
||||
if (!('log_type' in row) || row.log_type !== 'workers') return row
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
timestamp: row.timestamp,
|
||||
event_message: row.event_message,
|
||||
metadata: row.metadata,
|
||||
}
|
||||
}
|
||||
|
||||
export const getLevelLabel = (value: (typeof LEVELS)[number]): string => {
|
||||
switch (value) {
|
||||
case 'success':
|
||||
@@ -86,7 +121,7 @@ export const getStatusLevel = (status?: number | string): string => {
|
||||
return 'success'
|
||||
}
|
||||
|
||||
export function getLevelRowClassName(value: (typeof LEVELS)[number]): string {
|
||||
export function getLevelRowClassName(value: (typeof LEVELS)[number] | null | undefined): string {
|
||||
switch (value) {
|
||||
case 'success':
|
||||
return ''
|
||||
@@ -205,19 +240,43 @@ export function getEventMessageDisplay(
|
||||
return { message: value, capitalize: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* Multigres logs are gated behind the `showMultigresLogs` flag, so the multigres
|
||||
* log_type option is removed from the filter fields when the flag is disabled.
|
||||
*/
|
||||
export function gateMultigresLogType<T extends { value: string; options?: Option[] }>(
|
||||
export function gateLogTypeOptions<T extends { value: string; options?: Option[] }>(
|
||||
fields: T[],
|
||||
showMultigresLogs: boolean
|
||||
visibility: Partial<Record<UnifiedLogType, boolean>>
|
||||
): T[] {
|
||||
if (showMultigresLogs) return fields
|
||||
|
||||
return fields.map((field) =>
|
||||
field.value === 'log_type' && field.options
|
||||
? ({ ...field, options: field.options.filter((option) => option.value !== 'multigres') } as T)
|
||||
: field
|
||||
const hiddenLogTypes = new Set(
|
||||
Object.entries(visibility)
|
||||
.filter(([, visible]) => !visible)
|
||||
.map(([logType]) => logType)
|
||||
)
|
||||
|
||||
if (hiddenLogTypes.size === 0) return fields
|
||||
|
||||
return fields.map((field) => {
|
||||
if (field.value !== 'log_type' || !field.options) return field
|
||||
return {
|
||||
...field,
|
||||
options: field.options.filter((option) => !hiddenLogTypes.has(option.value)),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function gateLogTypeFilters(
|
||||
filters: string[] | null | undefined,
|
||||
visibility: Partial<Record<UnifiedLogType, boolean>>
|
||||
): string[] | null | undefined {
|
||||
if (!filters) return filters
|
||||
|
||||
const hiddenLogTypes = new Set(
|
||||
Object.entries(visibility)
|
||||
.filter(([, visible]) => !visible)
|
||||
.map(([logType]) => logType)
|
||||
)
|
||||
|
||||
if (hiddenLogTypes.size === 0) return filters
|
||||
|
||||
return filters.filter((filter) => {
|
||||
const parsed = parseLogsFilterUrlParams([filter])[0]
|
||||
return parsed?.column !== 'log_type' || !hiddenLogTypes.has(parsed.value)
|
||||
})
|
||||
}
|
||||
@@ -71,7 +71,7 @@ export function generateDynamicColumns({ data }: { data: ColumnSchema[] }): {
|
||||
header: '',
|
||||
cell: ({ row }) => {
|
||||
const level = row.getValue<ColumnSchema['level']>('level')
|
||||
return <DataTableColumnLevelIndicator value={level} />
|
||||
return level ? <DataTableColumnLevelIndicator value={level} /> : null
|
||||
},
|
||||
enableHiding: false,
|
||||
enableResizing: false,
|
||||
@@ -153,8 +153,8 @@ export function generateDynamicColumns({ data }: { data: ColumnSchema[] }): {
|
||||
<TooltipTrigger asChild>
|
||||
<span>
|
||||
<DataTableColumnStatusCode
|
||||
value={value}
|
||||
level={row.getValue<ColumnSchema['level']>('level')}
|
||||
value={value ?? undefined}
|
||||
level={row.getValue<ColumnSchema['level']>('level') ?? undefined}
|
||||
/>
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
@@ -162,8 +162,8 @@ export function generateDynamicColumns({ data }: { data: ColumnSchema[] }): {
|
||||
</Tooltip>
|
||||
) : (
|
||||
<DataTableColumnStatusCode
|
||||
value={value}
|
||||
level={row.getValue<ColumnSchema['level']>('level')}
|
||||
value={value ?? undefined}
|
||||
level={row.getValue<ColumnSchema['level']>('level') ?? undefined}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Auth, EdgeFunctions, Realtime, Storage } from 'icons'
|
||||
import { Auth, EdgeFunctions, Realtime, Storage, Workers } from 'icons'
|
||||
import { Box, Cable, Code2, Database, Network } from 'lucide-react'
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from 'ui'
|
||||
|
||||
@@ -11,11 +11,7 @@ interface LogTypeIconProps {
|
||||
className?: string
|
||||
}
|
||||
|
||||
type IconComponent = React.ComponentType<{
|
||||
size?: number
|
||||
strokeWidth?: number
|
||||
className?: string
|
||||
}>
|
||||
type IconComponent = typeof Box | typeof Auth
|
||||
|
||||
// [Alaister]: commented out types coming in the future
|
||||
// edge: Globe,
|
||||
@@ -29,6 +25,7 @@ const ICON_MAP: Partial<Record<(typeof LOG_TYPES)[number], IconComponent>> = {
|
||||
supavisor: Cable,
|
||||
pgbouncer: Cable,
|
||||
multigres: Network,
|
||||
workers: Workers,
|
||||
}
|
||||
|
||||
export const LogTypeIcon = ({
|
||||
|
||||
@@ -4,6 +4,7 @@ import { toast } from 'sonner'
|
||||
|
||||
import { logsAllEndpointUrl, pickLogsQueryBuilder } from './logs-endpoint'
|
||||
import { getUnifiedLogsISOStartEnd } from './unified-logs-infinite-query'
|
||||
import { mapUnifiedLogRow, parseUnifiedLogsQueryRows } from './unified-logs.utils'
|
||||
import { getUnifiedLogsQuery } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries'
|
||||
import { getUnifiedLogsQuery as getUnifiedLogsQueryBq } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq'
|
||||
import { QuerySearchParamsType } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.types'
|
||||
@@ -41,34 +42,8 @@ export async function retrieveUnifiedLogs({
|
||||
|
||||
if (error) handleError(error)
|
||||
|
||||
const resultData = data?.result ?? []
|
||||
|
||||
const result = resultData.map((row: any) => {
|
||||
const ts = String(row.timestamp ?? '')
|
||||
const looksLikeIso = /[T-]/.test(ts)
|
||||
const date = looksLikeIso
|
||||
? new Date(/Z$|[+-]\d{2}:?\d{2}$/.test(ts) ? ts : `${ts}Z`)
|
||||
: new Date(Number(ts) / 1000)
|
||||
return {
|
||||
id: row.id,
|
||||
date,
|
||||
timestamp: row.timestamp,
|
||||
level: row.level,
|
||||
status: row.status || 200,
|
||||
method: row.method,
|
||||
host: row.host,
|
||||
pathname: (row.url || '').replace(/^https?:\/\/[^\/]+/, '') || row.pathname || '',
|
||||
event_message: row.event_message || row.body || '',
|
||||
headers:
|
||||
typeof row.headers === 'string' ? JSON.parse(row.headers || '{}') : row.headers || {},
|
||||
regions: row.region ? [row.region] : [],
|
||||
log_type: row.log_type || '',
|
||||
latency: row.latency || 0,
|
||||
log_count: row.log_count || null,
|
||||
logs: row.logs || [],
|
||||
auth_user: row.auth_user || null,
|
||||
}
|
||||
})
|
||||
const resultData = parseUnifiedLogsQueryRows(data?.result)
|
||||
const result = resultData.map(mapUnifiedLogRow)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -4,9 +4,8 @@ import { useFlag } from 'common'
|
||||
import { executeAnalyticsSql } from './execute-analytics-sql'
|
||||
import { logsKeys } from './keys'
|
||||
import { logsAllEndpointUrl, pickLogsQueryBuilder } from './logs-endpoint'
|
||||
import { parseOtelTimestamp } from './otel-inspection.utils'
|
||||
import { analyticsLiteral, safeSql } from './safe-analytics-sql'
|
||||
import { extractLogMetadata } from './unified-logs.utils'
|
||||
import { mapUnifiedLogRow, parseUnifiedLogsQueryRows } from './unified-logs.utils'
|
||||
import { getUnifiedLogsQuery } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries'
|
||||
import { getUnifiedLogsQuery as getUnifiedLogsQueryBq } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq'
|
||||
import {
|
||||
@@ -17,7 +16,6 @@ import { handleError } from '@/data/fetchers'
|
||||
import type { ResponseError, UseCustomInfiniteQueryOptions } from '@/types'
|
||||
|
||||
const LOGS_PAGE_LIMIT = 50
|
||||
type LogLevel = 'success' | 'warning' | 'error'
|
||||
|
||||
export const UNIFIED_LOGS_QUERY_OPTIONS = {
|
||||
refetchOnWindowFocus: false,
|
||||
@@ -122,33 +120,8 @@ export async function getUnifiedLogs(
|
||||
|
||||
if (data.error) handleError(new Error(data.error as string))
|
||||
|
||||
const resultData = data?.result ?? []
|
||||
|
||||
const result = resultData.map((row: any) => {
|
||||
const date = parseOtelTimestamp(row.timestamp)
|
||||
|
||||
const { status, method, pathname } = extractLogMetadata(row)
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
date,
|
||||
method,
|
||||
pathname,
|
||||
status,
|
||||
timestamp: row.timestamp,
|
||||
level: row.level as LogLevel,
|
||||
host: row.host,
|
||||
event_message: row.event_message || row.body || '',
|
||||
headers:
|
||||
typeof row.headers === 'string' ? JSON.parse(row.headers || '{}') : row.headers || {},
|
||||
regions: row.region ? [row.region] : [],
|
||||
log_type: row.log_type || '',
|
||||
latency: row.latency || 0,
|
||||
log_count: row.log_count || null,
|
||||
logs: row.logs || [],
|
||||
auth_user: row.auth_user || null,
|
||||
}
|
||||
})
|
||||
const resultData = parseUnifiedLogsQueryRows(data?.result)
|
||||
const result = resultData.map(mapUnifiedLogRow)
|
||||
|
||||
const firstRow = result.length > 0 ? result[0] : null
|
||||
const lastRow = result.length > 0 ? result[result.length - 1] : null
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { extractLogMetadata } from './unified-logs.utils'
|
||||
import {
|
||||
extractLogMetadata,
|
||||
mapUnifiedLogRow,
|
||||
parseUnifiedLogsQueryRows,
|
||||
} from './unified-logs.utils'
|
||||
|
||||
describe('extractLogMetadata', () => {
|
||||
describe('non-auth logs', () => {
|
||||
it('returns the row status, method, and url-derived pathname', () => {
|
||||
it('returns projected status, method, and pathname', () => {
|
||||
const row = {
|
||||
log_type: 'api',
|
||||
status: 404,
|
||||
method: 'GET',
|
||||
url: 'https://example.supabase.co/rest/v1/users?select=id',
|
||||
pathname: '/ignored',
|
||||
pathname: '/rest/v1/users?select=id',
|
||||
event_message: 'irrelevant',
|
||||
}
|
||||
|
||||
@@ -21,12 +24,11 @@ describe('extractLogMetadata', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to row.pathname when url is missing', () => {
|
||||
it('returns the projected pathname', () => {
|
||||
const row = {
|
||||
log_type: 'api',
|
||||
status: 500,
|
||||
method: 'POST',
|
||||
url: '',
|
||||
pathname: '/fallback',
|
||||
event_message: '',
|
||||
}
|
||||
@@ -34,7 +36,7 @@ describe('extractLogMetadata', () => {
|
||||
expect(extractLogMetadata(row).pathname).toBe('/fallback')
|
||||
})
|
||||
|
||||
it('returns empty string for pathname when both url and pathname are missing', () => {
|
||||
it('returns an empty string when pathname is missing', () => {
|
||||
const row = {
|
||||
log_type: 'api',
|
||||
status: 200,
|
||||
@@ -49,7 +51,6 @@ describe('extractLogMetadata', () => {
|
||||
const row = {
|
||||
log_type: 'api',
|
||||
method: 'GET',
|
||||
url: 'https://example.supabase.co/health',
|
||||
event_message: '',
|
||||
}
|
||||
|
||||
@@ -63,7 +64,6 @@ describe('extractLogMetadata', () => {
|
||||
log_type: 'auth',
|
||||
status: 999,
|
||||
method: 'IGNORED',
|
||||
url: 'https://ignored',
|
||||
event_message: JSON.stringify({
|
||||
status: 400,
|
||||
method: 'POST',
|
||||
@@ -146,7 +146,6 @@ describe('extractLogMetadata', () => {
|
||||
log_type: 'auth',
|
||||
status: 200,
|
||||
method: 'GET',
|
||||
url: 'https://example.supabase.co/token',
|
||||
event_message: 'not json',
|
||||
}
|
||||
|
||||
@@ -170,4 +169,141 @@ describe('extractLogMetadata', () => {
|
||||
expect(extractLogMetadata(row).status).toBe(200)
|
||||
})
|
||||
})
|
||||
|
||||
describe('workers logs', () => {
|
||||
it('does not synthesize unsupported request metadata', () => {
|
||||
const row = {
|
||||
event_message: 'Error: Dynamic require of "path" is not supported',
|
||||
id: '51a29911-9293-4616-8984-743cc548b629',
|
||||
log_type: 'workers',
|
||||
metadata: {
|
||||
cw_event_id: '39883203917805946105278943454814281535421893832620638214',
|
||||
launch_id: '1788424715503435269',
|
||||
log_group: '/aws/lambda-microvms/workers/cxkpapyhaaywrtudnqpl/api',
|
||||
log_stream: 'launch-1788424715503435269',
|
||||
source: 'worker_guest_logs',
|
||||
worker: 'api',
|
||||
},
|
||||
project: 'cxkpapyhaaywrtudnqpl',
|
||||
timestamp: 1788424716876000,
|
||||
}
|
||||
|
||||
expect(extractLogMetadata(row)).toEqual({ status: null, method: null, pathname: null })
|
||||
})
|
||||
|
||||
it('preserves only Workers metadata while keeping unsupported fields null', () => {
|
||||
const metadata = {
|
||||
cw_event_id: '39883203917805946105278943454814281535421893832620638214',
|
||||
launch_id: '1788424715503435269',
|
||||
log_group: '/aws/lambda-microvms/workers/cxkpapyhaaywrtudnqpl/api',
|
||||
log_stream: 'launch-1788424715503435269',
|
||||
source: 'worker_guest_logs',
|
||||
worker: 'api',
|
||||
}
|
||||
const mapped = mapUnifiedLogRow({
|
||||
event_message: 'Error: Dynamic require of "path" is not supported',
|
||||
id: '51a29911-9293-4616-8984-743cc548b629',
|
||||
log_type: 'workers',
|
||||
metadata,
|
||||
timestamp: 1788424716876000,
|
||||
status: 200,
|
||||
level: 'success',
|
||||
method: 'GET',
|
||||
pathname: '/invented',
|
||||
auth_user: 'invented-user',
|
||||
log_count: null,
|
||||
logs: null,
|
||||
})
|
||||
|
||||
expect(mapped).toMatchObject({
|
||||
id: '51a29911-9293-4616-8984-743cc548b629',
|
||||
timestamp: 1788424716876000,
|
||||
event_message: 'Error: Dynamic require of "path" is not supported',
|
||||
metadata,
|
||||
status: null,
|
||||
level: null,
|
||||
method: null,
|
||||
pathname: null,
|
||||
auth_user: null,
|
||||
})
|
||||
expect(mapped).not.toHaveProperty('project')
|
||||
})
|
||||
|
||||
it('does not add metadata to non-Workers rows', () => {
|
||||
const mapped = mapUnifiedLogRow({
|
||||
id: 'edge-log',
|
||||
timestamp: 1788424716876000,
|
||||
log_type: 'edge',
|
||||
metadata: { request: 'existing metadata' },
|
||||
status: 200,
|
||||
level: 'success',
|
||||
method: 'GET',
|
||||
pathname: '/rest/v1',
|
||||
event_message: null,
|
||||
log_count: 0,
|
||||
logs: null,
|
||||
})
|
||||
|
||||
expect(mapped).not.toHaveProperty('metadata')
|
||||
expect(mapped.log_count).toBe(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseUnifiedLogsQueryRows', () => {
|
||||
const workersRow = {
|
||||
event_message: 'Error: Dynamic require of "path" is not supported',
|
||||
id: '51a29911-9293-4616-8984-743cc548b629',
|
||||
metadata: {
|
||||
cw_event_id: '39883203917805946105278943454814281535421893832620638214',
|
||||
launch_id: '1788424715503435269',
|
||||
log_group: '/aws/lambda-microvms/workers/cxkpapyhaaywrtudnqpl/api',
|
||||
log_stream: 'launch-1788424715503435269',
|
||||
source: 'worker_guest_logs',
|
||||
worker: 'api',
|
||||
},
|
||||
project: 'cxkpapyhaaywrtudnqpl',
|
||||
timestamp: 1788424716876000,
|
||||
log_type: 'workers',
|
||||
status: null,
|
||||
level: null,
|
||||
pathname: null,
|
||||
method: null,
|
||||
log_count: null,
|
||||
logs: null,
|
||||
auth_user: null,
|
||||
}
|
||||
|
||||
it('parses the Workers projection and strips project', () => {
|
||||
const [parsed] = parseUnifiedLogsQueryRows([workersRow])
|
||||
|
||||
expect(parsed).toMatchObject({
|
||||
id: workersRow.id,
|
||||
timestamp: workersRow.timestamp,
|
||||
event_message: workersRow.event_message,
|
||||
metadata: workersRow.metadata,
|
||||
log_type: 'workers',
|
||||
status: null,
|
||||
level: null,
|
||||
pathname: null,
|
||||
method: null,
|
||||
})
|
||||
expect(parsed).not.toHaveProperty('project')
|
||||
})
|
||||
|
||||
it('returns an empty array for undefined results', () => {
|
||||
expect(parseUnifiedLogsQueryRows(undefined)).toEqual([])
|
||||
})
|
||||
|
||||
it('rejects invalid metadata', () => {
|
||||
expect(() => parseUnifiedLogsQueryRows([{ ...workersRow, metadata: 'invalid' }])).toThrow()
|
||||
})
|
||||
|
||||
it.each([{ id: 42 }, { timestamp: true }])('rejects invalid identity fields', (invalidFields) => {
|
||||
expect(() => parseUnifiedLogsQueryRows([{ ...workersRow, ...invalidFields }])).toThrow()
|
||||
})
|
||||
|
||||
it('rejects invalid projected field types', () => {
|
||||
expect(() => parseUnifiedLogsQueryRows([{ ...workersRow, level: 'info' }])).toThrow()
|
||||
})
|
||||
})
|
||||
@@ -1,13 +1,47 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
import { parseOtelTimestamp } from './otel-inspection.utils'
|
||||
import { LEVELS } from '@/components/ui/DataTable/DataTable.constants'
|
||||
import { tryParseJson } from '@/lib/helpers'
|
||||
|
||||
type UnifiedLogMetadataRow = {
|
||||
log_type?: string | null
|
||||
status?: string | number | null
|
||||
method?: string | null
|
||||
pathname?: string | null
|
||||
event_message?: string | null
|
||||
}
|
||||
|
||||
const unifiedLogsQueryRowSchema = z.object({
|
||||
id: z.string(),
|
||||
timestamp: z.union([z.string(), z.number()]),
|
||||
log_type: z.string(),
|
||||
status: z.union([z.string(), z.number()]).nullable(),
|
||||
level: z.enum(LEVELS).nullable(),
|
||||
pathname: z.string().nullable(),
|
||||
event_message: z.string().nullable(),
|
||||
method: z.string().nullable(),
|
||||
log_count: z.number().nullable(),
|
||||
logs: z.array(z.unknown()).nullable(),
|
||||
auth_user: z.string().nullish(),
|
||||
metadata: z.record(z.string(), z.unknown()).nullish(),
|
||||
})
|
||||
|
||||
export type UnifiedLogsQueryRow = z.infer<typeof unifiedLogsQueryRowSchema>
|
||||
|
||||
export const parseUnifiedLogsQueryRows = (value: unknown): UnifiedLogsQueryRow[] =>
|
||||
z.array(unifiedLogsQueryRowSchema).parse(value ?? [])
|
||||
|
||||
const extractLeadingStatus = (s?: string) => {
|
||||
const m = typeof s === 'string' ? s.match(/^(\d{3})\b/) : null
|
||||
return m ? Number(m[1]) : undefined
|
||||
}
|
||||
|
||||
// [Joshen] Row has an unknown type in this case so `any` is accurate
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export const extractLogMetadata = (row: any) => {
|
||||
export const extractLogMetadata = (row: UnifiedLogMetadataRow) => {
|
||||
if (row.log_type === 'workers') {
|
||||
return { status: null, method: null, pathname: null }
|
||||
}
|
||||
|
||||
// [Joshen] For auth logs, these metadata are nested within event_message,
|
||||
// so opting to bring them out at the query level
|
||||
const eventMessage = tryParseJson(row.event_message)
|
||||
@@ -18,10 +52,30 @@ export const extractLogMetadata = (row: any) => {
|
||||
extractLeadingStatus(eventMessage?.error))
|
||||
: (row.status ?? 200)
|
||||
const method = row.log_type === 'auth' ? eventMessage?.method : row.method
|
||||
const pathname =
|
||||
row.log_type === 'auth'
|
||||
? eventMessage?.path
|
||||
: (row.url || '').replace(/^https?:\/\/[^\/]+/, '') || row.pathname || ''
|
||||
const pathname = row.log_type === 'auth' ? eventMessage?.path : row.pathname || ''
|
||||
|
||||
return { status, method, pathname }
|
||||
}
|
||||
|
||||
export const mapUnifiedLogRow = (row: UnifiedLogsQueryRow) => {
|
||||
const isWorkersLog = row.log_type === 'workers'
|
||||
const { status, method, pathname } = extractLogMetadata(row)
|
||||
|
||||
const mappedRow = {
|
||||
id: row.id,
|
||||
date: parseOtelTimestamp(row.timestamp),
|
||||
method,
|
||||
pathname,
|
||||
status,
|
||||
timestamp: row.timestamp,
|
||||
level: isWorkersLog ? null : row.level,
|
||||
event_message: row.event_message ?? '',
|
||||
log_type: row.log_type,
|
||||
log_count: row.log_count ?? null,
|
||||
logs: row.logs ?? [],
|
||||
auth_user: isWorkersLog ? null : row.auth_user || null,
|
||||
}
|
||||
|
||||
if (isWorkersLog) return { ...mappedRow, metadata: row.metadata ?? null }
|
||||
return mappedRow
|
||||
}
|
||||
@@ -8,12 +8,7 @@ import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql'
|
||||
import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint'
|
||||
import { analyticsLiteral, safeSql } from '@/data/logs/safe-analytics-sql'
|
||||
import { IS_PLATFORM } from '@/lib/constants'
|
||||
|
||||
export const WORKER_LOG_SOURCES = {
|
||||
requests: 'worker_ingress_logs',
|
||||
output: 'worker_guest_logs',
|
||||
builds: 'worker_api_logs',
|
||||
} as const
|
||||
import { WORKER_LOG_SOURCES } from '@/lib/constants/workers'
|
||||
|
||||
export type WorkerLogStream = keyof typeof WORKER_LOG_SOURCES
|
||||
|
||||
|
||||
@@ -1,2 +1,8 @@
|
||||
export const PRODUCT_NAME = 'Workers'
|
||||
export const CLI_NAME = 'workers'
|
||||
|
||||
export const WORKER_LOG_SOURCES = {
|
||||
requests: 'worker_ingress_logs',
|
||||
output: 'worker_guest_logs',
|
||||
builds: 'worker_api_logs',
|
||||
} as const
|
||||
@@ -3708,6 +3708,7 @@ export interface UnifiedLogsRowClickedEvent {
|
||||
| 'supavisor'
|
||||
| 'pgbouncer'
|
||||
| 'multigres'
|
||||
| 'workers'
|
||||
}
|
||||
groups: TelemetryGroups
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user