From 4e1c6b5907f89ec3dddaab6cdc124ce58a0554c9 Mon Sep 17 00:00:00 2001 From: Jordi Enric <37541088+jordienr@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:22:13 +0200 Subject: [PATCH] fix(logs): preserve pagination cursor precision DEBUG-229 (#51597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Rhe dashboard used only the last log's time to decide where to continue, and shortened that time to milliseconds. This could skip logs with the same recorded time, or times less than a millisecond apart, between one page and the next. This could happen even with Live turned off. ## Fix Use both the last log's full recorded time and its unique ID to decide where the next page starts. This lets the dashboard continue from the right log, even when several logs have the same time. The fix works with both supported log services. This PR fixes missing logs when loading older results. It does not change how Live fetches new logs; that improvement is handled separately in [#51599](https://github.com/supabase/supabase/pull/51599). Part 1/9 of the replacement stack for #51090, tracked by [DEBUG-229](https://linear.app/supabase/issue/DEBUG-229/make-live-mode-incremental). Targets master; review and merge the stack in order. This diff contains 389 changed lines, including tests. Stack: [#51597](https://github.com/supabase/supabase/pull/51597) → [#51598](https://github.com/supabase/supabase/pull/51598) → [#51599](https://github.com/supabase/supabase/pull/51599) → [#51600](https://github.com/supabase/supabase/pull/51600) → [#51601](https://github.com/supabase/supabase/pull/51601) → [#51602](https://github.com/supabase/supabase/pull/51602) → [#51603](https://github.com/supabase/supabase/pull/51603) → [#51604](https://github.com/supabase/supabase/pull/51604) → [#51605](https://github.com/supabase/supabase/pull/51605). ## How to test - `pnpm --filter studio exec vitest run data/logs/unified-logs-pagination.test.ts data/logs/unified-logs.utils.test.ts` - Check both log services: logs with the same recorded time, or times less than a millisecond apart, should appear once when loading older results. - Validation: focused tests, full Studio typecheck, changed-file lint, and formatting pass. --- .../UnifiedLogs/LogSelectionActions.utils.ts | 3 +- .../UnifiedLogs/UnifiedLogs.queries.bq.ts | 6 +- .../UnifiedLogs/UnifiedLogs.queries.ts | 6 +- .../UnifiedLogs/UnifiedLogs.schema.ts | 12 +- .../interfaces/UnifiedLogs/UnifiedLogs.tsx | 19 ++- .../UnifiedLogs/components/LogTypeIcon.tsx | 30 +++-- .../data/logs/unified-logs-infinite-query.ts | 111 ++++++++++-------- .../data/logs/unified-logs-pagination.test.ts | 62 ++++++++++ .../data/logs/unified-logs.test-utils.ts | 101 ++++++++++++++++ .../data/logs/unified-logs.utils.test.ts | 28 +++++ apps/studio/data/logs/unified-logs.utils.ts | 9 ++ 11 files changed, 302 insertions(+), 85 deletions(-) create mode 100644 apps/studio/data/logs/unified-logs-pagination.test.ts create mode 100644 apps/studio/data/logs/unified-logs.test-utils.ts diff --git a/apps/studio/components/interfaces/UnifiedLogs/LogSelectionActions.utils.ts b/apps/studio/components/interfaces/UnifiedLogs/LogSelectionActions.utils.ts index ee26f4cac3f..51fb32339db 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/LogSelectionActions.utils.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/LogSelectionActions.utils.ts @@ -1,6 +1,7 @@ import { z } from 'zod' import { getLogDataForMetadataVisibility } from './ServiceFlowPanel.utils' +import { LOG_TYPES } from './UnifiedLogs.constants' import { columnSchema } from './UnifiedLogs.schema' import { getRawLogData } from './UnifiedLogs.utils' import { parseOtelTimestamp } from '@/data/logs/otel-inspection.utils' @@ -28,7 +29,7 @@ const selectedLogsSchema = z .array( columnSchema .pick({ id: true, log_type: true, event_message: true, metadata: true }) - .extend({ timestamp: z.union([z.string(), z.number()]) }) + .extend({ log_type: z.enum(LOG_TYPES), timestamp: z.union([z.string(), z.number()]) }) .passthrough() ) .transform((rows) => diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts index bb180254823..bbd6e837396 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq.ts @@ -387,8 +387,12 @@ WITH unified_logs AS ( /** * Unified logs SQL query */ -export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => { +export const getUnifiedLogsQuery = ( + search: QuerySearchParamsType, + paginationFilter?: SafeLogSqlFragment +): SafeLogSqlFragment => { const conditions = buildConditions(search) + if (paginationFilter) conditions.push(paginationFilter) const effectiveLogTypes = getEffectiveLogTypes(search) return safeSql` diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts index fc923e5e23d..33dddd8285c 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts @@ -448,8 +448,12 @@ const applySearchParamsFilter = (search: QuerySearchParamsType): SafeLogSqlFragm /** * Unified logs row query — flat SELECT, no subquery wrapper. */ -export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => { +export const getUnifiedLogsQuery = ( + search: QuerySearchParamsType, + paginationFilter?: SafeLogSqlFragment +): SafeLogSqlFragment => { const conditions = buildBaseWhere(search) + if (paginationFilter) conditions.push(paginationFilter) return safeSql`-- unified logs: row list SELECT ${ROW_PROJECTION} FROM logs diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.schema.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.schema.ts index b5726f6e7e5..983f96d62ed 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.schema.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.schema.ts @@ -1,6 +1,6 @@ import { z } from 'zod' -import { LOG_TYPES, METHODS, REGIONS } from './UnifiedLogs.constants' +import { METHODS, REGIONS } from './UnifiedLogs.constants' import { ARRAY_DELIMITER, LEVELS, @@ -9,15 +9,15 @@ import { export const columnSchema = z.object({ id: z.string(), - log_type: z.enum(LOG_TYPES), - method: z.enum(METHODS).nullable(), + log_type: z.string(), + method: z.string().nullable(), pathname: z.string().nullable(), level: z.enum(LEVELS).nullable(), - status: z.number().nullable(), + status: z.union([z.string(), z.number()]).nullable(), date: z.date(), - timestamp: z.number(), + timestamp: z.union([z.string(), z.number()]), event_message: z.string().optional(), - log_count: z.number().optional(), // used to count function logs for a given execution_id + log_count: z.number().nullish(), logs: z.array(z.any()).optional(), // array of function logs auth_user: z.string().nullable().optional(), metadata: z.record(z.string(), z.unknown()).nullable().optional(), diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx index 8a84a655533..1967824e696 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx @@ -33,7 +33,7 @@ import { LogsFilterBar } from './components/LogsFilterBar' import { LogsListPanel } from './components/LogsListPanel' import { TooltipLabel } from './components/TooltipLabel' import { ServiceFlowPanel } from './ServiceFlowPanel' -import { SEARCH_PARAMS_PARSER } from './UnifiedLogs.constants' +import { LOG_TYPES, SEARCH_PARAMS_PARSER } from './UnifiedLogs.constants' import { filterFields as defaultFilterFields } from './UnifiedLogs.fields' import { buildDefaultColumnFilters, @@ -68,6 +68,7 @@ import { ShortcutTooltip } from '@/components/ui/ShortcutTooltip' import { useUnifiedLogsChartQuery } from '@/data/logs/unified-logs-chart-query' import { useUnifiedLogsCountQuery } from '@/data/logs/unified-logs-count-query' import { useUnifiedLogsInfiniteQuery } from '@/data/logs/unified-logs-infinite-query' +import { deduplicateUnifiedLogs } from '@/data/logs/unified-logs.utils' import { useLocalStorageQuery } from '@/hooks/misc/useLocalStorage' import { useShowMultigresLogs } from '@/hooks/misc/useShowMultigresLogs' import { useTrack } from '@/lib/telemetry/track' @@ -226,12 +227,7 @@ export const UnifiedLogs = () => { const rawFlatData = useMemo(() => { return unifiedLogsData?.pages?.flatMap((page) => page.data ?? []) ?? [] }, [unifiedLogsData?.pages]) - // [Joshen] Refer to unified-logs-infinite-query on why the need to deupe - const flatData = useMemo(() => { - return rawFlatData.filter((value, idx) => { - return idx === rawFlatData.findIndex((x) => x.id === value.id) - }) - }, [rawFlatData]) + const flatData = useMemo(() => deduplicateUnifiedLogs(rawFlatData), [rawFlatData]) const liveMode = useLiveMode(flatData) const totalDBRowCount = counts?.totalRowCount @@ -253,11 +249,11 @@ export const UnifiedLogs = () => { }, [search.filter]) const getRowClassName = < - TData extends { date: Date; level: (typeof LEVELS)[number] | null; timestamp: number }, + TData extends { date: Date; level: (typeof LEVELS)[number] | null; timestamp: number | string }, >( row: Row ) => { - const rowTimestamp = row.original.timestamp + const rowTimestamp = Number(row.original.timestamp) const isPast = rowTimestamp <= (liveMode.timestamp || -1) const levelClassName = getLevelRowClassName(row.original.level) return cn(levelClassName, isPast ? 'opacity-50' : 'opacity-100', 'h-[30px]') @@ -268,7 +264,7 @@ export const UnifiedLogs = () => { return generateDynamicColumns({ data: flatData }) }, [flatData]) - const table: Table = useReactTable({ + const table: Table = useReactTable({ data: flatData, columns: dynamicColumns, state: { @@ -381,7 +377,8 @@ export const UnifiedLogs = () => { setOpenRowId(undefined) } else if (openRowId && selectedRow) { setSearch({ id: openRowId }) - track('unified_logs_row_clicked', { logType: selectedRow.original.log_type }) + const logType = LOG_TYPES.find((type) => type === selectedRow.original.log_type) + if (logType) track('unified_logs_row_clicked', { logType }) } else if (!openRowId && search.id) { // Clear the URL parameter when no row is open setSearch({ id: null }) diff --git a/apps/studio/components/interfaces/UnifiedLogs/components/LogTypeIcon.tsx b/apps/studio/components/interfaces/UnifiedLogs/components/LogTypeIcon.tsx index 0c1512570c5..8b4a59a6527 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/components/LogTypeIcon.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/components/LogTypeIcon.tsx @@ -2,10 +2,8 @@ import { Auth, Compute, EdgeFunctions, Realtime, Storage } from 'icons' import { BoxIcon, CableIcon, Code2Icon, DatabaseIcon, NetworkIcon } from 'lucide-react' import { Tooltip, TooltipContent, TooltipTrigger } from 'ui' -import { type LOG_TYPES } from '../UnifiedLogs.constants' - interface LogTypeIconProps { - type: (typeof LOG_TYPES)[number] + type: string size?: number strokeWidth?: number className?: string @@ -15,18 +13,18 @@ type IconComponent = typeof BoxIcon | typeof Auth // [Alaister]: commented out types coming in the future // edge: Globe, -const ICON_MAP: Partial> = { - postgrest: Code2Icon, - auth: Auth, - 'edge function': EdgeFunctions, - postgres: DatabaseIcon, - storage: Storage, - realtime: Realtime, - supavisor: CableIcon, - pgbouncer: CableIcon, - multigres: NetworkIcon, - compute: Compute, -} +const ICON_MAP = new Map([ + ['postgrest', Code2Icon], + ['auth', Auth], + ['edge function', EdgeFunctions], + ['postgres', DatabaseIcon], + ['storage', Storage], + ['realtime', Realtime], + ['supavisor', CableIcon], + ['pgbouncer', CableIcon], + ['multigres', NetworkIcon], + ['compute', Compute], +]) export const LogTypeIcon = ({ type, @@ -34,7 +32,7 @@ export const LogTypeIcon = ({ strokeWidth = 1.5, className, }: LogTypeIconProps) => { - const Icon = ICON_MAP[type] ?? BoxIcon + const Icon = ICON_MAP.get(type) ?? BoxIcon return ( diff --git a/apps/studio/data/logs/unified-logs-infinite-query.ts b/apps/studio/data/logs/unified-logs-infinite-query.ts index 86f3c68b8cd..c9d7be5bc36 100644 --- a/apps/studio/data/logs/unified-logs-infinite-query.ts +++ b/apps/studio/data/logs/unified-logs-infinite-query.ts @@ -5,7 +5,11 @@ import { executeAnalyticsSql } from './execute-analytics-sql' import { logsKeys } from './keys' import { logsAllEndpointUrl, pickLogsQueryBuilder } from './logs-endpoint' import { analyticsLiteral, safeSql } from './safe-analytics-sql' -import { mapUnifiedLogRow, parseUnifiedLogsQueryRows } from './unified-logs.utils' +import { + mapUnifiedLogRow, + parseUnifiedLogsQueryRows, + type UnifiedLogsQueryRow, +} from './unified-logs.utils' import { getUnifiedLogsQuery } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries' import { getUnifiedLogsQuery as getUnifiedLogsQueryBq } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.queries.bq' import { @@ -17,6 +21,9 @@ import type { ResponseError, UseCustomInfiniteQueryOptions } from '@/types' const LOGS_PAGE_LIMIT = 50 +type RowCursor = Pick +type UnifiedLogsPageParam = (NonNullable & { row?: RowCursor }) | null | undefined + export const UNIFIED_LOGS_QUERY_OPTIONS = { refetchOnWindowFocus: false, refetchOnMount: false, @@ -25,7 +32,7 @@ export const UNIFIED_LOGS_QUERY_OPTIONS = { staleTime: 1000 * 60 * 5, // 5 minutes, } -export type UnifiedLogsData = any +export type UnifiedLogsData = Awaited> export type UnifiedLogsError = ResponseError export type UnifiedLogsVariables = { projectRef?: string; search: QuerySearchParamsType } @@ -62,85 +69,87 @@ export async function getUnifiedLogs( search, pageParam, useOtel = false, - }: UnifiedLogsVariables & { pageParam: PageParam | null; useOtel?: boolean }, + }: UnifiedLogsVariables & { + pageParam: UnifiedLogsPageParam + useOtel?: boolean + }, signal?: AbortSignal, headersInit?: HeadersInit ) { if (typeof projectRef === 'undefined') throw new Error('projectRef is required for getUnifiedLogs') - /** - * [Joshen] RE infinite loading pagination logic for unified logs, these all really should live in the API - * but for now we're doing these on the FE to move quickly while figuring out what data we need before we - * migrate this logic to the BE. Just thought to leave a small explanation on the logic here: - * - * We're leveraging on the log's timestamp to essentially fetch the next page - * Given that the logs are ordered descending (latest logs come first, and we're fetching older logs as we scroll down) - * Hence why the cursor is basically the last row's timestamp from the latest page - * - * iso_timestamp_start will always be the current timestamp - * iso_timestamp_end will default to the last hour for the first page, followed by the last row's timestamp from - * the previous page. - * - * However, just note that this isn't a perfect solution as there's always the edge case where by there's multiple rows - * with identical timestamps, hence why FE will need a de-duping logic (in UnifiedLogs.tsx) unless we can figure a cleaner - * solution when we move all this logic to the BE (e.g using composite columns for the cursor like timestamp + id) - * - */ - const { isoTimestampStart, isoTimestampEnd } = getUnifiedLogsISOStartEnd(search) const buildQuery = pickLogsQueryBuilder(useOtel, getUnifiedLogsQuery, getUnifiedLogsQueryBq) - const sql = safeSql`${buildQuery(search)} ORDER BY timestamp DESC, id DESC LIMIT ${analyticsLiteral(LOGS_PAGE_LIMIT)}` const cursorValue = pageParam?.cursor const cursorDirection = pageParam?.direction - let timestampEnd: string + let timestampEnd = isoTimestampEnd if (cursorDirection === 'prev') { - // Live mode: fetch logs newer than the cursor timestampEnd = new Date().toISOString() - } else if (cursorDirection === 'next') { - // Regular pagination: fetch logs older than the cursor. - // The cursor is stored as milliseconds (set below from `date.getTime()`), - // so we can convert it directly without worrying about the wire format. + } else if (cursorDirection === 'next' && !pageParam?.row) { timestampEnd = cursorValue !== null && cursorValue !== undefined ? new Date(Number(cursorValue)).toISOString() : isoTimestampEnd - } else { - timestampEnd = isoTimestampEnd } const endpoint = logsAllEndpointUrl(useOtel) - const data = await executeAnalyticsSql({ - projectRef, - endpoint, - sql, - iso_timestamp_start: isoTimestampStart, - iso_timestamp_end: timestampEnd, - signal, - headers: headersInit, - }) + const idColumn = useOtel ? safeSql`toString(id)` : safeSql`id` + const cursorTimestamp = (cursor: RowCursor) => { + const rawTimestamp = String(cursor.timestamp) + const isIsoTimestamp = /[T-]/.test(rawTimestamp) + let timestamp + if (isIsoTimestamp) { + timestamp = useOtel + ? safeSql`parseDateTime64BestEffort(${analyticsLiteral(rawTimestamp)}, 9, 'UTC')` + : safeSql`CAST(${analyticsLiteral(rawTimestamp)} AS TIMESTAMP)` + } else { + timestamp = useOtel + ? safeSql`fromUnixTimestamp64Micro(${analyticsLiteral(Number(rawTimestamp))})` + : safeSql`TIMESTAMP_MICROS(${analyticsLiteral(Number(rawTimestamp))})` + } + return timestamp + } + const fetchPage = async (cursor?: RowCursor) => { + let paginationFilter + if (cursor) { + const timestamp = cursorTimestamp(cursor) + paginationFilter = safeSql`(timestamp < ${timestamp} OR (timestamp = ${timestamp} AND ${idColumn} < ${analyticsLiteral(cursor.id)}))` + } + const sql = safeSql`${buildQuery(search, paginationFilter)} ORDER BY timestamp DESC, ${idColumn} DESC LIMIT ${analyticsLiteral(LOGS_PAGE_LIMIT)}` + const data = await executeAnalyticsSql({ + projectRef, + endpoint, + sql, + iso_timestamp_start: isoTimestampStart, + iso_timestamp_end: timestampEnd, + signal, + headers: headersInit, + }) - if (data.error) handleError(new Error(data.error as string)) + if (data.error) handleError(new Error(data.error as string)) + return parseUnifiedLogsQueryRows(data?.result) + } - const resultData = parseUnifiedLogsQueryRows(data?.result) - const result = resultData.map(mapUnifiedLogRow) + const rows = await fetchPage(pageParam?.row) + const result = rows.map(mapUnifiedLogRow) const firstRow = result.length > 0 ? result[0] : null const lastRow = result.length > 0 ? result[result.length - 1] : null + const lastQueryRow = rows.length > 0 ? rows[rows.length - 1] : pageParam?.row const hasMore = result.length >= LOGS_PAGE_LIMIT - 1 - // Cursors are stored as milliseconds (Date.getTime()) so the OTEL endpoint's - // wire format (ISO string vs numeric microseconds) doesn't bleed into pagination. - const nextCursor = lastRow ? lastRow.date.getTime() : null + const nextCursor = lastRow ? lastRow.date.getTime() : (pageParam?.cursor ?? null) const prevCursor = firstRow ? firstRow.date.getTime() : new Date().getTime() return { data: result, nextCursor: hasMore ? nextCursor : null, prevCursor, + nextRow: lastQueryRow ? { id: lastQueryRow.id, timestamp: lastQueryRow.timestamp } : null, } } @@ -154,7 +163,7 @@ export const useUnifiedLogsInfiniteQuery = ( UnifiedLogsError, InfiniteData, readonly unknown[], - PageParam | null + UnifiedLogsPageParam > = {} ) => { const useOtel = useUnifiedLogsBackend() @@ -170,8 +179,12 @@ export const useUnifiedLogsInfiniteQuery = ( }, initialPageParam: null, getNextPageParam(lastPage) { - if (!lastPage.nextCursor || lastPage.data.length === 0) return null - return { cursor: lastPage.nextCursor, direction: 'next' } as const + if (!lastPage.nextCursor || !lastPage.nextRow) return null + return { + cursor: lastPage.nextCursor, + direction: 'next', + row: lastPage.nextRow ?? undefined, + } as const }, ...UNIFIED_LOGS_QUERY_OPTIONS, ...options, diff --git a/apps/studio/data/logs/unified-logs-pagination.test.ts b/apps/studio/data/logs/unified-logs-pagination.test.ts new file mode 100644 index 00000000000..11f649b01a4 --- /dev/null +++ b/apps/studio/data/logs/unified-logs-pagination.test.ts @@ -0,0 +1,62 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { getUnifiedLogs } from './unified-logs-infinite-query' +import { compareRows, createRow, mockLogs, now, search } from './unified-logs.test-utils' + +afterEach(() => { + vi.useRealTimers() +}) + +describe('getUnifiedLogs forward pagination', () => { + it.each([false, true])('paginates equal microsecond timestamps with otel=%s', async (useOtel) => { + vi.useFakeTimers() + vi.setSystemTime(now) + const timestamp = (now.getTime() - 1000) * 1000 + 123 + const rows = Array.from({ length: 120 }, (_, index) => + createRow(`log-${String(index).padStart(3, '0')}`, timestamp) + ) + const requests = mockLogs(rows, useOtel) + const variables = { projectRef: 'default', search, useOtel } + const first = await getUnifiedLogs({ ...variables, pageParam: null }) + const second = await getUnifiedLogs({ + ...variables, + pageParam: { cursor: first.nextCursor!, row: first.nextRow!, direction: 'next' }, + }) + const third = await getUnifiedLogs({ + ...variables, + pageParam: { cursor: second.nextCursor!, row: second.nextRow!, direction: 'next' }, + }) + + expect([first.data.length, second.data.length, third.data.length]).toEqual([50, 50, 20]) + expect([...first.data, ...second.data, ...third.data].map((row) => row.id)).toEqual( + [...rows].sort(compareRows).map((row) => row.id) + ) + expect(third.nextCursor).toBeNull() + expect(requests[1].sql).toContain(String(timestamp)) + expect(requests[1].sql).not.toContain('OFFSET') + }) + + it('preserves nanosecond ISO cursors on OTEL', async () => { + vi.useFakeTimers() + vi.setSystemTime(now) + const rows = Array.from({ length: 51 }, (_, index) => + createRow( + `log-${String(index).padStart(3, '0')}`, + `2026-09-30T11:59:59.000000${String(index).padStart(3, '0')}` + ) + ) + const requests = mockLogs(rows, true) + const variables = { projectRef: 'default', search, useOtel: true } + const first = await getUnifiedLogs({ ...variables, pageParam: null }) + const second = await getUnifiedLogs({ + ...variables, + pageParam: { cursor: first.nextCursor!, row: first.nextRow!, direction: 'next' }, + }) + expect([...first.data, ...second.data].map((row) => row.id)).toEqual( + [...rows].sort(compareRows).map((row) => row.id) + ) + expect(requests[1].sql).toContain( + "parseDateTime64BestEffort('2026-09-30T11:59:59.000000001', 9, 'UTC')" + ) + }) +}) diff --git a/apps/studio/data/logs/unified-logs.test-utils.ts b/apps/studio/data/logs/unified-logs.test-utils.ts new file mode 100644 index 00000000000..0efdfb7fd6e --- /dev/null +++ b/apps/studio/data/logs/unified-logs.test-utils.ts @@ -0,0 +1,101 @@ +import { platformComponents } from 'api-types' +import { HttpResponse } from 'msw' +import { createLoader } from 'nuqs/server' +import { z } from 'zod' + +import { SEARCH_PARAMS_PARSER } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.constants' +import { addAPIMock } from '@/tests/lib/msw' + +export const now = new Date('2026-09-30T12:00:00.000Z') +export const search = createLoader(SEARCH_PARAMS_PARSER)(new URLSearchParams()) + +export const createRow = (id: string, timestamp: number | string = now.getTime() * 1000) => ({ + id, + timestamp, + log_type: 'edge', + status: 200, + level: 'success', + pathname: '/', + event_message: id, + method: 'GET', + log_count: null, + logs: null, +}) + +const requestBodySchema = z.object({ + iso_timestamp_start: z.string(), + iso_timestamp_end: z.string(), + sql: z.string(), +}) +type RequestBody = z.infer + +const timestampToNanos = (timestamp: number | string) => { + if (!/[T-]/.test(String(timestamp))) return BigInt(timestamp) * 1000n + const fractionalNanos = + String(timestamp) + .match(/\.(\d+)/)?.[1] + .padEnd(9, '0') + .slice(3, 9) ?? '0' + const isoTimestamp = String(timestamp).endsWith('Z') ? String(timestamp) : `${timestamp}Z` + return BigInt(Date.parse(isoTimestamp)) * 1_000_000n + BigInt(fractionalNanos) +} + +export const compareRows = (a: ReturnType, b: ReturnType) => { + const timestampA = timestampToNanos(a.timestamp) + const timestampB = timestampToNanos(b.timestamp) + if (timestampA !== timestampB) return timestampA > timestampB ? -1 : 1 + if (a.id === b.id) return 0 + return a.id > b.id ? -1 : 1 +} + +export const mockLogs = ( + rows: ReturnType[], + useOtel = false, + onRequest?: (requestNumber: number) => void +) => { + const requests: RequestBody[] = [] + addAPIMock({ + method: 'post', + path: useOtel + ? '/platform/projects/:ref/analytics/endpoints/logs.all.otel' + : '/platform/projects/:ref/analytics/endpoints/logs.all', + response: async ({ request }) => { + const body = requestBodySchema.parse(await request.json()) + requests.push(body) + onRequest?.(requests.length) + const numericCursor = body.sql.match( + /timestamp < (?:TIMESTAMP_MICROS|fromUnixTimestamp64Micro)\((\d+)\)/ + )?.[1] + const isoCursor = body.sql.match( + /timestamp < (?:CAST|parseDateTime64BestEffort)\('([^']+)'/ + )?.[1] + const cursorTimestamp = numericCursor ?? isoCursor + const cursorId = body.sql.match(/(?:id|toString\(id\)) < '([^']+)'/)?.[1] + const boundaryTimestamp = + body.sql.match(/timestamp > (?:TIMESTAMP_MICROS|fromUnixTimestamp64Micro)\((\d+)\)/)?.[1] ?? + body.sql.match(/timestamp > (?:CAST|parseDateTime64BestEffort)\('([^']+)'/)?.[1] + const boundaryId = body.sql.match(/(?:id|toString\(id\)) >= '([^']+)'/)?.[1] + const start = timestampToNanos(body.iso_timestamp_start) + const end = timestampToNanos(body.iso_timestamp_end) + return HttpResponse.json({ + result: rows + .filter((row) => { + const timestamp = timestampToNanos(row.timestamp) + if (timestamp <= start || timestamp > end) return false + if (boundaryTimestamp && boundaryId) { + const boundary = timestampToNanos(boundaryTimestamp) + if (timestamp < boundary || (timestamp === boundary && row.id < boundaryId)) { + return false + } + } + if (!cursorTimestamp || !cursorId) return true + const cursor = timestampToNanos(cursorTimestamp) + return timestamp < cursor || (timestamp === cursor && row.id < cursorId) + }) + .sort(compareRows) + .slice(0, 50), + }) + }, + }) + return requests +} diff --git a/apps/studio/data/logs/unified-logs.utils.test.ts b/apps/studio/data/logs/unified-logs.utils.test.ts index 625f9353a6f..10d679b250a 100644 --- a/apps/studio/data/logs/unified-logs.utils.test.ts +++ b/apps/studio/data/logs/unified-logs.utils.test.ts @@ -5,6 +5,7 @@ import { mapUnifiedLogRow, parseUnifiedLogsQueryRows, } from './unified-logs.utils' +import { columnSchema } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.schema' describe('extractLogMetadata', () => { describe('non-auth logs', () => { @@ -295,6 +296,33 @@ describe('parseUnifiedLogsQueryRows', () => { expect(parseUnifiedLogsQueryRows(undefined)).toEqual([]) }) + it.each([1788424716876123, '2026-09-03T08:38:36.876123456'])( + 'preserves backend fields and raw timestamp %s in displayed rows', + (timestamp) => { + const [parsed] = parseUnifiedLogsQueryRows([ + { + ...computeRow, + timestamp, + log_type: 'custom_logs', + status: '42P01', + method: 'PATCH', + log_count: null, + }, + ]) + const mapped = mapUnifiedLogRow(parsed) + + expect(columnSchema.parse(mapped)).toMatchObject({ + timestamp, + log_type: 'custom_logs', + status: '42P01', + method: 'PATCH', + log_count: null, + date: new Date('2026-09-03T08:38:36.876Z'), + }) + expect(parsed.timestamp).toBe(timestamp) + } + ) + it('rejects invalid metadata', () => { expect(() => parseUnifiedLogsQueryRows([{ ...computeRow, metadata: 'invalid' }])).toThrow() }) diff --git a/apps/studio/data/logs/unified-logs.utils.ts b/apps/studio/data/logs/unified-logs.utils.ts index 9a50c5639c7..3d2c4974522 100644 --- a/apps/studio/data/logs/unified-logs.utils.ts +++ b/apps/studio/data/logs/unified-logs.utils.ts @@ -32,6 +32,15 @@ export type UnifiedLogsQueryRow = z.infer export const parseUnifiedLogsQueryRows = (value: unknown): UnifiedLogsQueryRow[] => z.array(unifiedLogsQueryRowSchema).parse(value ?? []) +export const deduplicateUnifiedLogs = (logs: T[]) => { + const ids = new Set() + return logs.filter((log) => { + if (ids.has(log.id)) return false + ids.add(log.id) + return true + }) +} + const extractLeadingStatus = (s?: string) => { const m = typeof s === 'string' ? s.match(/^(\d{3})\b/) : null return m ? Number(m[1]) : undefined