From 232ce7e68cf50969da75254b220c1f439420cea6 Mon Sep 17 00:00:00 2001 From: Saxon Fletcher Date: Wed, 16 Sep 2026 16:52:03 +1000 Subject: [PATCH] docs: focus Logs on the unified view and export queryable fields (#50073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. Yes. ## What kind of change does this PR introduce? Documentation update and a small Studio copy correction. ## What is the current behavior? The Logs guide mixes unified filtering with retired SQL Explorer instructions, and the Markdown field reference loses query semantics. ## What is the new behavior? The Logs guide mixed filtering and event inspection with the retired SQL Logs Explorer workflow. It now documents the unified Logs view: default sources, filter semantics, event details, Live, sharing, bounded exports, and missing results. The log field reference uses one mapping for HTML and Markdown, preserving source IDs, query expressions, and source/query types. The Studio User-filter empty state and comments now match its Auth and API Gateway scope. ## Additional context Validation: shared-field mapping tests, guides Markdown generation, docs typecheck, targeted docs/Studio ESLint, formatting, and browser inspection of the field table. Exported Markdown includes the source IDs and usable ClickHouse expressions. Repository-wide MDX lint has existing failures; changed pages have no reported violations. Self-review: hosted Studio filtering and log ingestion were checked against the implementation, not exercised against a hosted project. The documentation assumes the unified Logs experience is the default. Stage 1 of 3. Review and merge from the bottom of the stack. Stack: #50073 → #50074 → #50075. Production docs build also passes at the stack tip after standard reference generation. Initial CI note: the spelling action failed while building its container because Debian package downloads returned 404, before checking content. The stack has no merge conflicts. ## Summary by CodeRabbit * **Documentation** * Expanded the log field reference with ClickHouse fields, nested-field query examples, types, schema references, and capture limits. * Reworked the Logs guide with clearer instructions for filtering, event inspection, live mode, sharing, exports, retention, and missing results. * Clarified service and Postgres log behavior and updated navigation and metadata. * **Bug Fixes** * Corrected user-filtering guidance and empty-state messaging to identify Auth and API Gateway logs as supported sources. --------- Co-authored-by: Steven Eubank Co-authored-by: Steven Eubank <47563310+smeubank@users.noreply.github.com> --- apps/docs/components/SharedData.tsx | 4 +- apps/docs/components/SharedData.utils.test.ts | 39 ++++++++ apps/docs/components/SharedData.utils.ts | 40 ++++++++ .../observability/log-field-reference.mdx | 80 +++++++++------ .../content/guides/observability/logs.mdx | 97 ++++++++----------- .../internals/markdown-schema/SharedData.ts | 30 +++--- .../UnifiedLogs/UnifiedLogs.queries.ts | 14 +-- .../interfaces/UnifiedLogs/UnifiedLogs.tsx | 2 +- 8 files changed, 197 insertions(+), 109 deletions(-) create mode 100644 apps/docs/components/SharedData.utils.test.ts diff --git a/apps/docs/components/SharedData.tsx b/apps/docs/components/SharedData.tsx index 744be85b829..9e91e59cedb 100644 --- a/apps/docs/components/SharedData.tsx +++ b/apps/docs/components/SharedData.tsx @@ -1,11 +1,11 @@ import { ReactNode } from 'react' import { config, logConstants } from 'shared-data' -import { resolveSharedDataPath } from './SharedData.utils' +import { getLogFieldReference, resolveSharedDataPath } from './SharedData.utils' const sharedData = { config, - logConstants, + logConstants: { schemas: getLogFieldReference(logConstants.schemas) }, } /** diff --git a/apps/docs/components/SharedData.utils.test.ts b/apps/docs/components/SharedData.utils.test.ts new file mode 100644 index 00000000000..5bc8bcc86bc --- /dev/null +++ b/apps/docs/components/SharedData.utils.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' + +import { getLogFieldReference } from './SharedData.utils' + +describe('getLogFieldReference', () => { + it('distinguishes ClickHouse columns from both prefixed and unprefixed attributes', () => { + const source = { + name: 'API Gateway', + reference: 'edge_logs', + fields: [ + { path: 'id', type: 'string' }, + { path: 'identifier', type: 'string' }, + { path: 'metadata.response.status_code', type: 'number' }, + ], + } + const [result] = getLogFieldReference([source]) + expect(result.reference).toBe('edge_logs') + expect(result.fields).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: 'id', queryField: 'id', queryType: 'String' }), + expect.objectContaining({ path: 'identifier', queryField: "log_attributes['identifier']" }), + expect.objectContaining({ + path: 'metadata.response.status_code', + type: 'number', + queryField: "log_attributes['response.status_code']", + queryType: 'String', + }), + expect.objectContaining({ + path: 'timestamp', + queryField: 'timestamp', + queryType: 'DateTime64', + }), + expect.objectContaining({ path: 'source', queryField: 'source' }), + ]) + ) + expect(result.fields.filter((field) => field.path === 'id')).toHaveLength(1) + expect(source.fields).toHaveLength(3) + }) +}) diff --git a/apps/docs/components/SharedData.utils.ts b/apps/docs/components/SharedData.utils.ts index 512eab59cfe..c0fbbff1fc1 100644 --- a/apps/docs/components/SharedData.utils.ts +++ b/apps/docs/components/SharedData.utils.ts @@ -17,3 +17,43 @@ export function resolveSharedDataPath(dataset: unknown, path: string): string | } return selected } + +type LogSourceSchema = { + name: string + reference: string + fields: { path: string; type: string }[] +} + +const LOG_COLUMNS = new Map([ + ['id', 'String'], + ['timestamp', 'DateTime64'], + ['event_message', 'String'], + ['severity_text', 'String'], + ['source', 'String'], +]) + +/** One field mapping for the HTML reference and its Markdown export. */ +export function getLogFieldReference(schemas: LogSourceSchema[]) { + return schemas.map((schema) => { + const fields = [...schema.fields] + for (const [path, type] of LOG_COLUMNS) { + if (!fields.some((field) => field.path === path)) fields.push({ path, type }) + } + return { + ...schema, + fields: fields + .sort((a, b) => a.path.localeCompare(b.path)) + .map((field) => { + const key = field.path + .replace(/^metadata\./, '') + .replace(/\\/g, '\\\\') + .replace(/'/g, "''") + return { + ...field, + queryField: LOG_COLUMNS.has(field.path) ? field.path : `log_attributes['${key}']`, + queryType: LOG_COLUMNS.get(field.path) ?? 'String', + } + }), + } + }) +} diff --git a/apps/docs/content/guides/observability/log-field-reference.mdx b/apps/docs/content/guides/observability/log-field-reference.mdx index 23bec25e0eb..99e444585ef 100644 --- a/apps/docs/content/guides/observability/log-field-reference.mdx +++ b/apps/docs/content/guides/observability/log-field-reference.mdx @@ -1,42 +1,68 @@ --- id: 'logs-field-reference' title: 'Logs field reference' -description: 'Supabase Logs field reference' +description: 'Log sources, ClickHouse fields, and event capture limits' --- -Use this reference to find the fields available for each log source. Query `id`, `timestamp`, `event_message`, and `source` as top-level columns. Other structured fields are keys in the `log_attributes` map: drop the `metadata.` prefix shown in the source schema and keep the rest of the dotted path. +Each event is a row in the ClickHouse `logs` table. Filter the `source` column to select a service. The tables below list its fields; a field is not necessarily populated on every event. -For example, the schema path `metadata.request.cf.country` is queried as `log_attributes['request.cf.country']`. See [Query and filter logs](/docs/guides/observability/advanced-log-filtering) for complete ClickHouse examples. +`id`, `timestamp`, `event_message`, `severity_text`, and `source` are top-level columns. Service fields are string values in `log_attributes`, even when the original event contains a number or boolean. Use the **ClickHouse query field** column directly. See [Query logs with SQL](/docs/guides/observability/advanced-log-filtering) for casting and field discovery. + +## Capture limits + +- Hosted Postgres events longer than 100,000 characters and Edge Function log messages longer than 10,000 characters are truncated. +- Internal Supabase service connection events are not recorded in hosted Postgres logs. +- An Edge Function invocation uses `function_edge_logs`; its console output uses `function_logs`. +- For [API Load Balancer](/docs/guides/platform/read-replicas#api-load-balancer) traffic, `log_attributes['load_balancer_redirect_identifier']` identifies the upstream database. + +## Fields by source {(logConstants) => ( {logConstants.schemas.map((schema) => ( - - - - - - - - - - {schema.fields - .sort((a, b) => a.path.localeCompare(b.path)) - .map((field) => ( - - - - - - ))} - -
Schema pathClickHouse query fieldType
{field.path} - {field.path.startsWith('metadata.') - ? `log_attributes['${field.path.slice('metadata.'.length)}']` - : field.path} - {field.type}
+

+ Source: {schema.reference} +

+
+ {schema.fields + .sort((a, b) => a.path.localeCompare(b.path)) + .map((field) => { + const shortName = field.path.replace(/^metadata\./, '') + const isTopLevel = field.queryField === field.path + return ( +
+
+ {shortName} + + {field.type} + +
+ {!isTopLevel && ( +
+
+ + schema + + + {field.path} + +
+
+ + clickhouse + + + {field.queryField} + +
+
+ )} +
+ ) + })} +
))} diff --git a/apps/docs/content/guides/observability/logs.mdx b/apps/docs/content/guides/observability/logs.mdx index 6d8b9f62278..28e03353b82 100644 --- a/apps/docs/content/guides/observability/logs.mdx +++ b/apps/docs/content/guides/observability/logs.mdx @@ -1,81 +1,68 @@ --- id: 'logs' -title: 'Logs' -description: 'Inspect project log events in the unified Logs view in Studio' +title: 'Logs in Studio' +description: 'Filter, inspect, and export project events in the Logs view' --- -This guide explains how to inspect project logs in Studio. Log retention is based on your [project's pricing plan](/pricing). For details on how Logs usage is billed, see [Manage Logs usage](/docs/guides/platform/manage-your-usage/logs). +Use [Logs](/dashboard/project/_/logs) to inspect events across your hosted project's services. For SQL queries through [Explorer](/dashboard/project/_/explorer), MCP, or the API, see [Query logs with SQL](/docs/guides/observability/advanced-log-filtering). -Use this page to filter and inspect events in [Logs](#product-logs). To query the same data with SQL from Studio, MCP, the API, or a script, or to record extra Postgres, API, and Realtime events, see [Query and filter logs](/docs/guides/observability/advanced-log-filtering). - - - -If you already have a specific error, start at [Diagnosing](/docs/guides/troubleshooting). To pick up a signal from these events, see [Detecting](/docs/guides/observability/detecting). - - - -## Filter and inspect events [#product-logs] - -Open [Logs](/dashboard/project/_/logs). The page shows a timeline of success, warning, and error events, a filterable table, and a detail panel when you select a row. - -If you don't select a log type, Logs queries **Postgres** and **API Gateway** events. Selecting log types replaces that default set. - - - -For regular expression filtering, structured-field queries, and field discovery, see [Query and filter logs](/docs/guides/observability/advanced-log-filtering). - - - -### Filter logs +## Find events [#product-logs] 1. Open [Logs](/dashboard/project/_/logs). -2. Set the **Time Range** in the sidebar. -3. Select one or more **Log Type** values. Nested toggles under API Gateway include or exclude Auth, Storage, and PostgREST request paths. The nested toggle under Postgres shows or hides connection logs. -4. Optionally filter by **Level**, **Status**, **Method**, **Pathname**, or **Event message**. Type in the filter bar to search event messages. -5. Optionally filter by **User**. This filter only matches Auth and Postgres events. +2. Set the **Time Range** in the sidebar, or select a range on the timeline. +3. Select one or more **Log Type** values. +4. Add filters in the filter bar, or type text to search event messages. +5. Select a row to inspect the event. -Refresh the table, hide columns, download matching rows as CSV or JSON, or turn on live mode to stream new events. +Without a log type selection, Logs queries **Postgres** and **API Gateway**. Selecting types replaces this default set. The timeline groups events by success, warning, and error. -### Log types +## Filter events -Selecting a log type in Studio queries the matching ClickHouse `source`. For the `source` names to use in SQL, see [Sources](/docs/guides/observability/advanced-log-filtering#logs-explorer). +{/* supa-mdx-lint-disable Rule003Spelling */} +| Filter | Behavior | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Log Type | Select API Gateway, Postgres, Auth, Storage, PostgREST, Edge Function, Realtime, or pooler events. | +| Level | Match success, warning, or error. | +| Status | Match an HTTP status or Postgres SQLSTATE. | +| Method | Match an HTTP method. | +| Pathname | Match a request path. | +| Event message | Use **iLike** or **Not iLike** for case-insensitive text matching or exclusion. Plain text matches anywhere in the message; `%` specifies a wildcard pattern. | +| User | Match the user's ID in Auth actor IDs or API Gateway JWT subjects. Other log types cannot match this filter. | -| Log type | Events | -| ------------- | ----------------------------------------------------------------- | -| API Gateway | HTTP requests through the API gateway, including REST and GraphQL | -| Postgres | Database queries and activity | -| PostgREST | PostgREST server logs | -| Auth | Auth server logs | -| Storage | Storage API server logs | -| Edge Function | Edge Function HTTP invocations and `console` output | -| Realtime | Realtime server logs | -| Supavisor | Connection pooler logs | -| PgBouncer | PgBouncer logs | +{/* supa-mdx-lint-enable Rule003Spelling */} -Selecting **API Gateway** is not the same as selecting **Auth**, **Storage**, or **PostgREST**. The nested API Gateway toggles filter HTTP paths on the gateway. The Auth, Storage, and PostgREST log types query those services' own logs. +Filters other than **Event message** and **User** support **Equals** and **Not equal**. **User** supports **Equals**. Included values within a field match any selected value; exclusions remove every selected value. Filters on different fields must all match. + +### Gateway and service logs + +The nested service toggles under **API Gateway** include or exclude gateway request paths. Selecting the separate **Auth**, **Storage**, or **PostgREST** log type retrieves that service's own logs. These are different events. + +For SQL source names, see the [Log field reference](/docs/guides/observability/log-field-reference). ### Postgres [#postgres] -Postgres logs show queries and activity for your database. Connection lifecycle events appear here when [connection logging](/docs/guides/observability/advanced-log-filtering#logging-postgres-connections) is enabled. They are included by default; clear **Connection logs** under the Postgres log type to hide them. +Postgres logs contain database activity and errors. Connection events appear when [connection logging](/docs/guides/platform/postgres-connection-logging) is enabled. Clear **Connection logs** under **Postgres** to hide them. To record additional statement classes, see [Logging Postgres queries](/docs/guides/observability/advanced-log-filtering#logging-postgres-queries). -### Inspect a log +## Inspect an event [#expanding-results] -1. Select a row in the table. -2. Open **Overview** to follow the request through the services that handled it. Open **Raw JSON** for the full event. -3. Dock the panel at the bottom or on the right. +Select a row to open its detail panel. **Overview**, when available for the log type, shows service details. **Raw JSON** shows the event data. Dock the panel at the bottom or on the right. -Edge Function rows include console output from that invocation. In SQL, the HTTP request is `function_edge_logs` and console output is `function_logs`. Function log messages longer than 10,000 characters are truncated. +Edge Function invocations can include associated console output. In SQL, invocation events use `function_edge_logs` and console events use `function_logs`. -### Expanding results [#expanding-results] +## Watch, share, and export -In the [Logs Explorer](/dashboard/project/_/logs/explorer), query results can be hard to read in the table. Double-click a row to expand it as JSON: +- Select **Live** to fetch new events automatically. Select it again to pause. Starting live mode clears the fixed time range and sort; selecting a time range or sort stops live mode. +- Copy the page URL to share the current filters. Recipients need access to the project. +- Open **Download logs**, choose CSV or JSON, and select a result limit of 100, 500, or 1,000 rows. The export applies the current filters. Without a fixed time range, choose the duration to retrieve. -![Expanding log results](/docs/img/guides/platform/expanded-log-results.png) +For continuous export, use [Log drains](/docs/guides/observability/log-drains). -### Single-service collections [#single-service-collections] +## Missing results [#single-service-collections] -The Logs sidebar still lists collections for one service at a time, such as [API Gateway](/dashboard/project/_/logs/edge-logs) or [Postgres](/dashboard/project/_/logs/postgres-logs). Use a collection when you want a dedicated view. +Check the time range, selected log types, and exclusions first. **User** combined with only Postgres or another unsupported type returns no matches. An empty result does not establish that the user had no activity. -If [Read Replicas](/docs/guides/platform/read-replicas) are enabled, collections can filter by database with the **Source** control. For API logs from the [API Load Balancer](/docs/guides/platform/read-replicas#api-load-balancer), the upstream database is the Redirect Identifier field (`log_attributes['load_balancer_redirect_identifier']` in SQL). +Events must be recorded before they can appear in Logs. See [Logging configuration](/docs/guides/observability/advanced-log-filtering#logging-postgres-connections) and the [source limitations](/docs/guides/observability/log-field-reference#capture-limits). + +Retention depends on your [pricing plan](/pricing). See [Manage Logs usage](/docs/guides/platform/manage-your-usage/logs) for billing details. diff --git a/apps/docs/internals/markdown-schema/SharedData.ts b/apps/docs/internals/markdown-schema/SharedData.ts index c915f95d7c2..430b6740246 100644 --- a/apps/docs/internals/markdown-schema/SharedData.ts +++ b/apps/docs/internals/markdown-schema/SharedData.ts @@ -1,25 +1,29 @@ import { createRequire } from 'node:module' -import { resolveSharedDataPath } from '../../components/SharedData.utils' +import { getLogFieldReference, resolveSharedDataPath } from '../../components/SharedData.utils' // tsx's ESM loader can't pick up named exports from the `shared-data` package // (CJS, no `"type": "module"`). Load via `createRequire` to use CJS interop — // this file only runs in the build script, never in the Next.js bundle. const { config, logConstants } = createRequire(import.meta.url)('shared-data') -type Field = { path: string; type: string } -type Schema = { name: string; fields: Field[] } - const sharedData: Record = { config, logConstants } -const renderLogConstants = (data: { schemas: Schema[] }): string => - data.schemas - .map( - (s) => - `#### ${s.name}\n${[...s.fields] - .sort((a, b) => a.path.localeCompare(b.path)) - .map((f) => ` - \`${f.path}\`, \`${f.type}\``) - .join('\n')}` +const renderLogConstants = (data: typeof logConstants): string => + getLogFieldReference(data.schemas) + .map((schema) => + [ + `### ${schema.name}`, + '', + `Source: \`${schema.reference}\``, + '', + '| Schema path | ClickHouse query field | Source type | Query value type |', + '| --- | --- | --- | --- |', + ...schema.fields.map( + (field) => + `| \`${field.path}\` | \`${field.queryField}\` | \`${field.type}\` | \`${field.queryType}\` |` + ), + ].join('\n') ) .join('\n\n') @@ -41,6 +45,6 @@ export const SharedData = ({ // The schema walker strips the MDX expression children before this handler // runs, and we can't evaluate the function statically anyway — hardcode the // markdown for the only dataset that uses this form today. - if (props.data === 'logConstants') return renderLogConstants(dataset as { schemas: Schema[] }) + if (props.data === 'logConstants') return renderLogConstants(dataset as typeof logConstants) return '' } diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts index 973e24e75b3..777a6e49ba3 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts @@ -357,14 +357,8 @@ const userFilterValue = (search: QuerySearchParamsType): string => typeof search.user === 'string' ? search.user.trim() : '' /** - * Cross-cutting "attributable to one user" condition. Only the two sources that can - * be positively tied to a user are eligible, each with its own match: - * - auth_logs: structured identity (`auth_event.actor_id`) - * - postgres_logs: the identifier appears verbatim in the error text (e.g. a 23502 - * failing row echoing the id column). - * edge_logs / storage_logs / realtime_logs carry no per-user field and can't satisfy - * either branch, so they're auto-excluded while the filter is active — never guessed - * at via IP or timestamp proximity. + * Matches structured user identifiers in Auth events and API Gateway JWT subjects. + * Sources without these fields cannot match the user filter. */ const userAttributionCondition = (search: QuerySearchParamsType): SafeLogSqlFragment | null => { const value = userFilterValue(search) @@ -381,12 +375,10 @@ const USER_ATTRIBUTABLE_SOURCES = new Set([LOG_TYPE_TO_SOURCE.auth, LOG_TYPE_TO_ /** * True when the user filter is active but an explicit log_type filter restricts the - * view to source(s) that can never satisfy `userAttributionCondition` (e.g. `edge`) — + * view to source(s) that can never satisfy `userAttributionCondition` (e.g. `postgres`) — * the combination is guaranteed to match zero rows. Consumed by the UI to show a * specific empty state instead of the generic "No results found". * - * [Joshen] Basically filtering by user only works on Auth and Postgres logs atm - * Refer to userAttributeCondition above */ export const isUserFilterUnreachable = (search: QuerySearchParamsType): boolean => { if (!userFilterValue(search)) return false diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx index 06724b70d65..f67b3f1040a 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx @@ -521,7 +521,7 @@ export const UnifiedLogs = () => {

No results found

- Filtering by user is only supported for Auth and Postgres log types + Filtering by user is only supported for Auth and API Gateway log types

) : undefined