diff --git a/apps/docs/components/SharedData.tsx b/apps/docs/components/SharedData.tsx index 744be85b829..9e91e59cedb 100644 --- a/apps/docs/components/SharedData.tsx +++ b/apps/docs/components/SharedData.tsx @@ -1,11 +1,11 @@ import { ReactNode } from 'react' import { config, logConstants } from 'shared-data' -import { resolveSharedDataPath } from './SharedData.utils' +import { getLogFieldReference, resolveSharedDataPath } from './SharedData.utils' const sharedData = { config, - logConstants, + logConstants: { schemas: getLogFieldReference(logConstants.schemas) }, } /** diff --git a/apps/docs/components/SharedData.utils.test.ts b/apps/docs/components/SharedData.utils.test.ts new file mode 100644 index 00000000000..5bc8bcc86bc --- /dev/null +++ b/apps/docs/components/SharedData.utils.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' + +import { getLogFieldReference } from './SharedData.utils' + +describe('getLogFieldReference', () => { + it('distinguishes ClickHouse columns from both prefixed and unprefixed attributes', () => { + const source = { + name: 'API Gateway', + reference: 'edge_logs', + fields: [ + { path: 'id', type: 'string' }, + { path: 'identifier', type: 'string' }, + { path: 'metadata.response.status_code', type: 'number' }, + ], + } + const [result] = getLogFieldReference([source]) + expect(result.reference).toBe('edge_logs') + expect(result.fields).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: 'id', queryField: 'id', queryType: 'String' }), + expect.objectContaining({ path: 'identifier', queryField: "log_attributes['identifier']" }), + expect.objectContaining({ + path: 'metadata.response.status_code', + type: 'number', + queryField: "log_attributes['response.status_code']", + queryType: 'String', + }), + expect.objectContaining({ + path: 'timestamp', + queryField: 'timestamp', + queryType: 'DateTime64', + }), + expect.objectContaining({ path: 'source', queryField: 'source' }), + ]) + ) + expect(result.fields.filter((field) => field.path === 'id')).toHaveLength(1) + expect(source.fields).toHaveLength(3) + }) +}) diff --git a/apps/docs/components/SharedData.utils.ts b/apps/docs/components/SharedData.utils.ts index 512eab59cfe..c0fbbff1fc1 100644 --- a/apps/docs/components/SharedData.utils.ts +++ b/apps/docs/components/SharedData.utils.ts @@ -17,3 +17,43 @@ export function resolveSharedDataPath(dataset: unknown, path: string): string | } return selected } + +type LogSourceSchema = { + name: string + reference: string + fields: { path: string; type: string }[] +} + +const LOG_COLUMNS = new Map([ + ['id', 'String'], + ['timestamp', 'DateTime64'], + ['event_message', 'String'], + ['severity_text', 'String'], + ['source', 'String'], +]) + +/** One field mapping for the HTML reference and its Markdown export. */ +export function getLogFieldReference(schemas: LogSourceSchema[]) { + return schemas.map((schema) => { + const fields = [...schema.fields] + for (const [path, type] of LOG_COLUMNS) { + if (!fields.some((field) => field.path === path)) fields.push({ path, type }) + } + return { + ...schema, + fields: fields + .sort((a, b) => a.path.localeCompare(b.path)) + .map((field) => { + const key = field.path + .replace(/^metadata\./, '') + .replace(/\\/g, '\\\\') + .replace(/'/g, "''") + return { + ...field, + queryField: LOG_COLUMNS.has(field.path) ? field.path : `log_attributes['${key}']`, + queryType: LOG_COLUMNS.get(field.path) ?? 'String', + } + }), + } + }) +} diff --git a/apps/docs/content/guides/observability/log-field-reference.mdx b/apps/docs/content/guides/observability/log-field-reference.mdx index 23bec25e0eb..99e444585ef 100644 --- a/apps/docs/content/guides/observability/log-field-reference.mdx +++ b/apps/docs/content/guides/observability/log-field-reference.mdx @@ -1,42 +1,68 @@ --- id: 'logs-field-reference' title: 'Logs field reference' -description: 'Supabase Logs field reference' +description: 'Log sources, ClickHouse fields, and event capture limits' --- -Use this reference to find the fields available for each log source. Query `id`, `timestamp`, `event_message`, and `source` as top-level columns. Other structured fields are keys in the `log_attributes` map: drop the `metadata.` prefix shown in the source schema and keep the rest of the dotted path. +Each event is a row in the ClickHouse `logs` table. Filter the `source` column to select a service. The tables below list its fields; a field is not necessarily populated on every event. -For example, the schema path `metadata.request.cf.country` is queried as `log_attributes['request.cf.country']`. See [Query and filter logs](/docs/guides/observability/advanced-log-filtering) for complete ClickHouse examples. +`id`, `timestamp`, `event_message`, `severity_text`, and `source` are top-level columns. Service fields are string values in `log_attributes`, even when the original event contains a number or boolean. Use the **ClickHouse query field** column directly. See [Query logs with SQL](/docs/guides/observability/advanced-log-filtering) for casting and field discovery. + +## Capture limits + +- Hosted Postgres events longer than 100,000 characters and Edge Function log messages longer than 10,000 characters are truncated. +- Internal Supabase service connection events are not recorded in hosted Postgres logs. +- An Edge Function invocation uses `function_edge_logs`; its console output uses `function_logs`. +- For [API Load Balancer](/docs/guides/platform/read-replicas#api-load-balancer) traffic, `log_attributes['load_balancer_redirect_identifier']` identifies the upstream database. + +## Fields by source {(logConstants) => ( {logConstants.schemas.map((schema) => ( - - - - - - - - - - {schema.fields - .sort((a, b) => a.path.localeCompare(b.path)) - .map((field) => ( - - - - - - ))} - -
Schema pathClickHouse query fieldType
{field.path} - {field.path.startsWith('metadata.') - ? `log_attributes['${field.path.slice('metadata.'.length)}']` - : field.path} - {field.type}
+

+ Source: {schema.reference} +

+
+ {schema.fields + .sort((a, b) => a.path.localeCompare(b.path)) + .map((field) => { + const shortName = field.path.replace(/^metadata\./, '') + const isTopLevel = field.queryField === field.path + return ( +
+
+ {shortName} + + {field.type} + +
+ {!isTopLevel && ( +
+
+ + schema + + + {field.path} + +
+
+ + clickhouse + + + {field.queryField} + +
+
+ )} +
+ ) + })} +
))} diff --git a/apps/docs/content/guides/observability/logs.mdx b/apps/docs/content/guides/observability/logs.mdx index 6d8b9f62278..28e03353b82 100644 --- a/apps/docs/content/guides/observability/logs.mdx +++ b/apps/docs/content/guides/observability/logs.mdx @@ -1,81 +1,68 @@ --- id: 'logs' -title: 'Logs' -description: 'Inspect project log events in the unified Logs view in Studio' +title: 'Logs in Studio' +description: 'Filter, inspect, and export project events in the Logs view' --- -This guide explains how to inspect project logs in Studio. Log retention is based on your [project's pricing plan](/pricing). For details on how Logs usage is billed, see [Manage Logs usage](/docs/guides/platform/manage-your-usage/logs). +Use [Logs](/dashboard/project/_/logs) to inspect events across your hosted project's services. For SQL queries through [Explorer](/dashboard/project/_/explorer), MCP, or the API, see [Query logs with SQL](/docs/guides/observability/advanced-log-filtering). -Use this page to filter and inspect events in [Logs](#product-logs). To query the same data with SQL from Studio, MCP, the API, or a script, or to record extra Postgres, API, and Realtime events, see [Query and filter logs](/docs/guides/observability/advanced-log-filtering). - - - -If you already have a specific error, start at [Diagnosing](/docs/guides/troubleshooting). To pick up a signal from these events, see [Detecting](/docs/guides/observability/detecting). - - - -## Filter and inspect events [#product-logs] - -Open [Logs](/dashboard/project/_/logs). The page shows a timeline of success, warning, and error events, a filterable table, and a detail panel when you select a row. - -If you don't select a log type, Logs queries **Postgres** and **API Gateway** events. Selecting log types replaces that default set. - - - -For regular expression filtering, structured-field queries, and field discovery, see [Query and filter logs](/docs/guides/observability/advanced-log-filtering). - - - -### Filter logs +## Find events [#product-logs] 1. Open [Logs](/dashboard/project/_/logs). -2. Set the **Time Range** in the sidebar. -3. Select one or more **Log Type** values. Nested toggles under API Gateway include or exclude Auth, Storage, and PostgREST request paths. The nested toggle under Postgres shows or hides connection logs. -4. Optionally filter by **Level**, **Status**, **Method**, **Pathname**, or **Event message**. Type in the filter bar to search event messages. -5. Optionally filter by **User**. This filter only matches Auth and Postgres events. +2. Set the **Time Range** in the sidebar, or select a range on the timeline. +3. Select one or more **Log Type** values. +4. Add filters in the filter bar, or type text to search event messages. +5. Select a row to inspect the event. -Refresh the table, hide columns, download matching rows as CSV or JSON, or turn on live mode to stream new events. +Without a log type selection, Logs queries **Postgres** and **API Gateway**. Selecting types replaces this default set. The timeline groups events by success, warning, and error. -### Log types +## Filter events -Selecting a log type in Studio queries the matching ClickHouse `source`. For the `source` names to use in SQL, see [Sources](/docs/guides/observability/advanced-log-filtering#logs-explorer). +{/* supa-mdx-lint-disable Rule003Spelling */} +| Filter | Behavior | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Log Type | Select API Gateway, Postgres, Auth, Storage, PostgREST, Edge Function, Realtime, or pooler events. | +| Level | Match success, warning, or error. | +| Status | Match an HTTP status or Postgres SQLSTATE. | +| Method | Match an HTTP method. | +| Pathname | Match a request path. | +| Event message | Use **iLike** or **Not iLike** for case-insensitive text matching or exclusion. Plain text matches anywhere in the message; `%` specifies a wildcard pattern. | +| User | Match the user's ID in Auth actor IDs or API Gateway JWT subjects. Other log types cannot match this filter. | -| Log type | Events | -| ------------- | ----------------------------------------------------------------- | -| API Gateway | HTTP requests through the API gateway, including REST and GraphQL | -| Postgres | Database queries and activity | -| PostgREST | PostgREST server logs | -| Auth | Auth server logs | -| Storage | Storage API server logs | -| Edge Function | Edge Function HTTP invocations and `console` output | -| Realtime | Realtime server logs | -| Supavisor | Connection pooler logs | -| PgBouncer | PgBouncer logs | +{/* supa-mdx-lint-enable Rule003Spelling */} -Selecting **API Gateway** is not the same as selecting **Auth**, **Storage**, or **PostgREST**. The nested API Gateway toggles filter HTTP paths on the gateway. The Auth, Storage, and PostgREST log types query those services' own logs. +Filters other than **Event message** and **User** support **Equals** and **Not equal**. **User** supports **Equals**. Included values within a field match any selected value; exclusions remove every selected value. Filters on different fields must all match. + +### Gateway and service logs + +The nested service toggles under **API Gateway** include or exclude gateway request paths. Selecting the separate **Auth**, **Storage**, or **PostgREST** log type retrieves that service's own logs. These are different events. + +For SQL source names, see the [Log field reference](/docs/guides/observability/log-field-reference). ### Postgres [#postgres] -Postgres logs show queries and activity for your database. Connection lifecycle events appear here when [connection logging](/docs/guides/observability/advanced-log-filtering#logging-postgres-connections) is enabled. They are included by default; clear **Connection logs** under the Postgres log type to hide them. +Postgres logs contain database activity and errors. Connection events appear when [connection logging](/docs/guides/platform/postgres-connection-logging) is enabled. Clear **Connection logs** under **Postgres** to hide them. To record additional statement classes, see [Logging Postgres queries](/docs/guides/observability/advanced-log-filtering#logging-postgres-queries). -### Inspect a log +## Inspect an event [#expanding-results] -1. Select a row in the table. -2. Open **Overview** to follow the request through the services that handled it. Open **Raw JSON** for the full event. -3. Dock the panel at the bottom or on the right. +Select a row to open its detail panel. **Overview**, when available for the log type, shows service details. **Raw JSON** shows the event data. Dock the panel at the bottom or on the right. -Edge Function rows include console output from that invocation. In SQL, the HTTP request is `function_edge_logs` and console output is `function_logs`. Function log messages longer than 10,000 characters are truncated. +Edge Function invocations can include associated console output. In SQL, invocation events use `function_edge_logs` and console events use `function_logs`. -### Expanding results [#expanding-results] +## Watch, share, and export -In the [Logs Explorer](/dashboard/project/_/logs/explorer), query results can be hard to read in the table. Double-click a row to expand it as JSON: +- Select **Live** to fetch new events automatically. Select it again to pause. Starting live mode clears the fixed time range and sort; selecting a time range or sort stops live mode. +- Copy the page URL to share the current filters. Recipients need access to the project. +- Open **Download logs**, choose CSV or JSON, and select a result limit of 100, 500, or 1,000 rows. The export applies the current filters. Without a fixed time range, choose the duration to retrieve. -![Expanding log results](/docs/img/guides/platform/expanded-log-results.png) +For continuous export, use [Log drains](/docs/guides/observability/log-drains). -### Single-service collections [#single-service-collections] +## Missing results [#single-service-collections] -The Logs sidebar still lists collections for one service at a time, such as [API Gateway](/dashboard/project/_/logs/edge-logs) or [Postgres](/dashboard/project/_/logs/postgres-logs). Use a collection when you want a dedicated view. +Check the time range, selected log types, and exclusions first. **User** combined with only Postgres or another unsupported type returns no matches. An empty result does not establish that the user had no activity. -If [Read Replicas](/docs/guides/platform/read-replicas) are enabled, collections can filter by database with the **Source** control. For API logs from the [API Load Balancer](/docs/guides/platform/read-replicas#api-load-balancer), the upstream database is the Redirect Identifier field (`log_attributes['load_balancer_redirect_identifier']` in SQL). +Events must be recorded before they can appear in Logs. See [Logging configuration](/docs/guides/observability/advanced-log-filtering#logging-postgres-connections) and the [source limitations](/docs/guides/observability/log-field-reference#capture-limits). + +Retention depends on your [pricing plan](/pricing). See [Manage Logs usage](/docs/guides/platform/manage-your-usage/logs) for billing details. diff --git a/apps/docs/internals/markdown-schema/SharedData.ts b/apps/docs/internals/markdown-schema/SharedData.ts index c915f95d7c2..430b6740246 100644 --- a/apps/docs/internals/markdown-schema/SharedData.ts +++ b/apps/docs/internals/markdown-schema/SharedData.ts @@ -1,25 +1,29 @@ import { createRequire } from 'node:module' -import { resolveSharedDataPath } from '../../components/SharedData.utils' +import { getLogFieldReference, resolveSharedDataPath } from '../../components/SharedData.utils' // tsx's ESM loader can't pick up named exports from the `shared-data` package // (CJS, no `"type": "module"`). Load via `createRequire` to use CJS interop — // this file only runs in the build script, never in the Next.js bundle. const { config, logConstants } = createRequire(import.meta.url)('shared-data') -type Field = { path: string; type: string } -type Schema = { name: string; fields: Field[] } - const sharedData: Record = { config, logConstants } -const renderLogConstants = (data: { schemas: Schema[] }): string => - data.schemas - .map( - (s) => - `#### ${s.name}\n${[...s.fields] - .sort((a, b) => a.path.localeCompare(b.path)) - .map((f) => ` - \`${f.path}\`, \`${f.type}\``) - .join('\n')}` +const renderLogConstants = (data: typeof logConstants): string => + getLogFieldReference(data.schemas) + .map((schema) => + [ + `### ${schema.name}`, + '', + `Source: \`${schema.reference}\``, + '', + '| Schema path | ClickHouse query field | Source type | Query value type |', + '| --- | --- | --- | --- |', + ...schema.fields.map( + (field) => + `| \`${field.path}\` | \`${field.queryField}\` | \`${field.type}\` | \`${field.queryType}\` |` + ), + ].join('\n') ) .join('\n\n') @@ -41,6 +45,6 @@ export const SharedData = ({ // The schema walker strips the MDX expression children before this handler // runs, and we can't evaluate the function statically anyway — hardcode the // markdown for the only dataset that uses this form today. - if (props.data === 'logConstants') return renderLogConstants(dataset as { schemas: Schema[] }) + if (props.data === 'logConstants') return renderLogConstants(dataset as typeof logConstants) return '' } diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts index 973e24e75b3..777a6e49ba3 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts @@ -357,14 +357,8 @@ const userFilterValue = (search: QuerySearchParamsType): string => typeof search.user === 'string' ? search.user.trim() : '' /** - * Cross-cutting "attributable to one user" condition. Only the two sources that can - * be positively tied to a user are eligible, each with its own match: - * - auth_logs: structured identity (`auth_event.actor_id`) - * - postgres_logs: the identifier appears verbatim in the error text (e.g. a 23502 - * failing row echoing the id column). - * edge_logs / storage_logs / realtime_logs carry no per-user field and can't satisfy - * either branch, so they're auto-excluded while the filter is active — never guessed - * at via IP or timestamp proximity. + * Matches structured user identifiers in Auth events and API Gateway JWT subjects. + * Sources without these fields cannot match the user filter. */ const userAttributionCondition = (search: QuerySearchParamsType): SafeLogSqlFragment | null => { const value = userFilterValue(search) @@ -381,12 +375,10 @@ const USER_ATTRIBUTABLE_SOURCES = new Set([LOG_TYPE_TO_SOURCE.auth, LOG_TYPE_TO_ /** * True when the user filter is active but an explicit log_type filter restricts the - * view to source(s) that can never satisfy `userAttributionCondition` (e.g. `edge`) — + * view to source(s) that can never satisfy `userAttributionCondition` (e.g. `postgres`) — * the combination is guaranteed to match zero rows. Consumed by the UI to show a * specific empty state instead of the generic "No results found". * - * [Joshen] Basically filtering by user only works on Auth and Postgres logs atm - * Refer to userAttributeCondition above */ export const isUserFilterUnreachable = (search: QuerySearchParamsType): boolean => { if (!userFilterValue(search)) return false diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx index 06724b70d65..f67b3f1040a 100644 --- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx @@ -521,7 +521,7 @@ export const UnifiedLogs = () => {

No results found

- Filtering by user is only supported for Auth and Postgres log types + Filtering by user is only supported for Auth and API Gateway log types

) : undefined