From 32341830b3bf9f38f25ae42e0e48bf7fd5fb23d9 Mon Sep 17 00:00:00 2001 From: Saxon Fletcher Date: Wed, 16 Sep 2026 17:03:43 +1000 Subject: [PATCH] docs: organize observability by task and move SQL logs to Explorer (#50074) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. Yes. ## What kind of change does this PR introduce? Documentation update. ## What is the current behavior? The observability overview and access page overlap; configuration interrupts querying; related guides send log queries to the old editor. ## What is the new behavior? The observability overview and navigation follow the same four sections: Read project data, Detect and diagnose, Hire an agent, and Configure and export. The overview absorbs the redundant access page, with permanent redirects for both HTML and Markdown URLs. “Query logs with SQL” owns ClickHouse querying through MCP, the Management API, and Explorer with query source Logs. Logging configuration moves to its own guide; sources, captured headers, and limits live in the field reference. Inspection links to canonical diagnostic SQL. Related Storage and database guides use the replacement Explorer workflow and retain existing anchors where headings move. ## Additional context Validation: Markdown generation, docs typecheck, targeted ESLint, formatting, and content-listing tests. Browser overview/navigation checked; old HTML and Markdown URLs return 308, and the new configuration page returns 200 in both formats. Three ClickHouse examples and the Postgres configuration query ran in a disposable container sandbox. Changed pages have no MDX lint violations; repository-wide existing failures remain. Self-review: the Management API request was verified against its published schema but not sent to a hosted project. Realtime ingestion and hosted logging configuration still need a hosted smoke check. No compatibility path for the deprecated logs engine is documented. Stage 2 of 3; depends on stage 1. Stack: #50073 → #50074 → #50075. Production docs build also passes at the stack tip after standard reference generation. ## Summary by CodeRabbit - **Documentation** - Reorganized observability guidance around reading data, detecting issues, diagnosing problems, agent setup, and exporting data. - Added a guide for configuring Postgres and Realtime logging. - Updated log investigation instructions to use Explorer, SQL queries, and clearer filters. - Added log source, field, and captured-header references. - Improved advisor guidance and database performance troubleshooting. - Added redirects for moved observability content. - **Accessibility** - Improved screen-reader labels for copy and feature-selection controls. --------- Co-authored-by: Claude Opus 5 --- .../NavigationMenu.constants.ts | 139 +---- apps/docs/content/guides/ai-tools/mcp.mdx | 2 +- .../guides/api/rest/postgrest-error-codes.mdx | 2 +- .../guides/database/extensions/pgaudit.mdx | 2 +- .../guides/database/postgres/timeouts.mdx | 4 +- apps/docs/content/guides/database/prisma.mdx | 2 +- apps/docs/content/guides/observability.mdx | 30 +- .../guides/observability/access-data.mdx | 31 - .../observability/advanced-log-filtering.mdx | 549 +++--------------- .../content/guides/observability/advisors.mdx | 6 +- .../observability/configure-logging.mdx | 47 ++ .../guides/observability/detecting.mdx | 6 +- .../content/guides/observability/inspect.mdx | 152 +---- .../observability/log-field-reference.mdx | 56 +- .../content/guides/observability/logs.mdx | 4 +- .../platform/manage-your-usage/egress.mdx | 2 +- .../content/guides/storage/cdn/metrics.mdx | 6 +- .../content/guides/storage/debugging/logs.mdx | 6 +- .../guides/storage/serving/bandwidth.mdx | 4 +- .../data/content-listings/telemetry.data.ts | 28 +- apps/docs/next.config.mjs | 10 + apps/www/lib/redirects.js | 10 + .../ui-patterns/src/CodeBlock/CodeBlock.tsx | 1 + .../components/McpConfigurationOptions.tsx | 1 + 24 files changed, 280 insertions(+), 820 deletions(-) delete mode 100644 apps/docs/content/guides/observability/access-data.mdx create mode 100644 apps/docs/content/guides/observability/configure-logging.mdx diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index d1617528709..a578dd53fa5 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -3071,143 +3071,58 @@ export const telemetry: NavMenuConstant = { items: [ { name: 'Overview', url: '/guides/observability' }, { - name: 'Observe the data', - url: '/guides/observability/access-data' as `/${string}`, + name: 'Read project data', items: [ - { - name: 'Logs', - url: '/guides/observability/advanced-log-filtering' as `/${string}`, - items: [ - { - name: 'Query and filter logs', - url: '/guides/observability/advanced-log-filtering' as `/${string}`, - }, - { - name: 'Sources', - url: '/guides/observability/advanced-log-filtering#logs-explorer' as `/${string}`, - }, - { - name: 'Logs field reference', - url: '/guides/observability/log-field-reference' as `/${string}`, - }, - { - name: 'Logs in Studio', - url: '/guides/observability/logs' as `/${string}`, - }, - ], - }, + { name: 'Query logs with SQL', url: '/guides/observability/advanced-log-filtering' }, + { name: 'Logs in Studio', url: '/guides/observability/logs' }, + { name: 'Log sources and fields', url: '/guides/observability/log-field-reference' }, + { name: 'Inspect the database', url: '/guides/observability/inspect' }, + { name: 'Advisors', url: '/guides/observability/advisors' }, + { name: 'Reports', url: '/guides/observability/reports' }, { name: 'Metrics API', - url: '/guides/observability/metrics' as `/${string}`, + url: '/guides/observability/metrics', items: [ - { - name: 'Grafana Cloud', - url: '/guides/observability/metrics/grafana-cloud' as `/${string}`, - }, + { name: 'Grafana Cloud', url: '/guides/observability/metrics/grafana-cloud' }, { name: 'Grafana self-hosted', - url: '/guides/observability/metrics/grafana-self-hosted' as `/${string}`, - }, - { - name: 'Datadog', - url: 'https://docs.datadoghq.com/integrations/supabase/', - }, - { - name: 'Elastic', - url: 'https://www.elastic.co/docs/reference/integrations/supabase', - }, - { - name: 'Vendor-agnostic setup', - url: '/guides/observability/metrics/vendor-agnostic' as `/${string}`, + url: '/guides/observability/metrics/grafana-self-hosted', }, + { name: 'Datadog', url: 'https://docs.datadoghq.com/integrations/supabase/' }, + { name: 'Elastic', url: 'https://www.elastic.co/docs/reference/integrations/supabase' }, + { name: 'Vendor-agnostic setup', url: '/guides/observability/metrics/vendor-agnostic' }, ], }, - { - name: 'Database', - url: '/guides/observability/inspect' as `/${string}`, - items: [ - { - name: 'CLI commands', - url: '/guides/observability/inspect#using-the-cli' as `/${string}`, - }, - { - name: 'SQL', - url: '/guides/observability/inspect#using-sql' as `/${string}`, - }, - ], - }, - { - name: 'Advisors', - url: '/guides/observability/advisors' as `/${string}`, - }, - { - name: 'Reports', - url: '/guides/observability/reports' as `/${string}`, - }, ], }, { - name: 'Detect issues', - url: '/guides/observability/detecting' as `/${string}`, + name: 'Detect and diagnose', items: [ - { - name: 'Detection checks', - url: '/guides/observability/detecting' as `/${string}`, - }, - ], - }, - { - name: 'Diagnose and resolve', - url: '/guides/troubleshooting' as `/${string}`, - items: [ - { - name: 'Troubleshooting', - url: '/guides/troubleshooting' as `/${string}`, - }, + { name: 'Detection checks', url: '/guides/observability/detecting' }, + { name: 'Troubleshooting', url: '/guides/troubleshooting' }, ], }, { name: 'Hire an agent', - url: '/guides/observability/automate-with-agents' as `/${string}`, items: [ - { - name: 'Generalist', - url: '/guides/observability/automate-with-agents/all' as `/${string}`, - }, - { - name: 'Health monitor', - url: '/guides/observability/automate-with-agents/health' as `/${string}`, - }, - { - name: 'Security monitor', - url: '/guides/observability/automate-with-agents/security' as `/${string}`, - }, + { name: 'Set up an agent', url: '/guides/observability/automate-with-agents' }, + { name: 'Generalist', url: '/guides/observability/automate-with-agents/all' }, + { name: 'Health monitor', url: '/guides/observability/automate-with-agents/health' }, + { name: 'Security monitor', url: '/guides/observability/automate-with-agents/security' }, { name: 'Performance monitor', - url: '/guides/observability/automate-with-agents/performance' as `/${string}`, - }, - { - name: 'Capacity monitor', - url: '/guides/observability/automate-with-agents/usage' as `/${string}`, + url: '/guides/observability/automate-with-agents/performance', }, + { name: 'Capacity monitor', url: '/guides/observability/automate-with-agents/usage' }, ], }, { - name: 'Export', - url: undefined, + name: 'Configure and export', items: [ - { - name: 'Log drains', - url: '/guides/observability/log-drains' as `/${string}`, - }, - { - name: 'Client-side tracing', - url: '/guides/observability/client-side-tracing' as `/${string}`, - }, - { - name: 'Sentry integration', - url: '/guides/observability/sentry-monitoring' as `/${string}`, - }, + { name: 'Configure logging', url: '/guides/observability/configure-logging' }, + { name: 'Log drains', url: '/guides/observability/log-drains' }, + { name: 'Client-side tracing', url: '/guides/observability/client-side-tracing' }, + { name: 'Sentry integration', url: '/guides/observability/sentry-monitoring' }, ], }, ], diff --git a/apps/docs/content/guides/ai-tools/mcp.mdx b/apps/docs/content/guides/ai-tools/mcp.mdx index d0701342afc..abfc36d5f60 100644 --- a/apps/docs/content/guides/ai-tools/mcp.mdx +++ b/apps/docs/content/guides/ai-tools/mcp.mdx @@ -56,7 +56,7 @@ The Supabase MCP server provides tools organized into feature groups. All groups ### Debugging -- `query_logs` - Run a read-only SQL query against project logs to filter, aggregate, or join across log fields. See [Query and filter logs](/docs/guides/observability/advanced-log-filtering). +- `query_logs` - Run a read-only SQL query against project logs to filter, aggregate, or join across log fields. See [Query logs with SQL](/docs/guides/observability/advanced-log-filtering). - `get_advisors` - Get security and performance advisors ### Development diff --git a/apps/docs/content/guides/api/rest/postgrest-error-codes.mdx b/apps/docs/content/guides/api/rest/postgrest-error-codes.mdx index 36decb42cb9..19288d73d08 100644 --- a/apps/docs/content/guides/api/rest/postgrest-error-codes.mdx +++ b/apps/docs/content/guides/api/rest/postgrest-error-codes.mdx @@ -145,7 +145,7 @@ Data API error unspecified ## Viewing errors in the logs -One can filter for API errors in the [SQL Editor](/dashboard/project/_/sql/new?skip=true&source=logs) with the query source set to **Logs**. Below are useful queries for filtering and analyzing API errors: +One can filter for API errors in the [Explorer](/dashboard/project/_/explorer) after selecting **Run SQL**, query source **Logs**, and a time range. Below are useful queries for filtering and analyzing API errors: ### Find all API errors that occurred at the database level diff --git a/apps/docs/content/guides/database/extensions/pgaudit.mdx b/apps/docs/content/guides/database/extensions/pgaudit.mdx index 78b20cff9fd..7514e339dea 100644 --- a/apps/docs/content/guides/database/extensions/pgaudit.mdx +++ b/apps/docs/content/guides/database/extensions/pgaudit.mdx @@ -254,7 +254,7 @@ Generates the following log in the [Dashboard's Postgres Logs](/dashboard/projec ## Finding and filtering audit logs -Logs generated by PGAudit can be found in [Postgres Logs](/dashboard/project/_/logs/postgres-logs?s=AUDIT). To find a specific log, you can use the [SQL Editor](/dashboard/project/_/sql/new?skip=true&source=logs) with the query source set to **Logs**. Below is a basic example to extract logs referencing `CREATE TABLE` events +Find pgAudit events in [Logs](/dashboard/project/_/logs): select **Postgres** as the log type and filter **Event message** for `AUDIT`. To find a specific log, you can use the [Explorer](/dashboard/project/_/explorer) after selecting **Run SQL**, query source **Logs**, and a time range. Below is a basic example to extract logs referencing `CREATE TABLE` events ```sql select diff --git a/apps/docs/content/guides/database/postgres/timeouts.mdx b/apps/docs/content/guides/database/postgres/timeouts.mdx index c831515f0fe..13e803959e2 100644 --- a/apps/docs/content/guides/database/postgres/timeouts.mdx +++ b/apps/docs/content/guides/database/postgres/timeouts.mdx @@ -126,9 +126,9 @@ language sql; The Supabase Dashboard contains tools to help you identify timed-out and long-running queries. -### Using the SQL Editor +### Query timeout logs [#using-the-sql-editor] -Go to the [SQL Editor](/dashboard/project/_/sql/new?skip=true&source=logs), set the query source to **Logs**, and run the following query to identify timed-out events (`statement timeout`) and queries that successfully run for longer than 10 seconds (`duration`). +Go to the [Explorer](/dashboard/project/_/explorer), select **Run SQL**, choose query source **Logs**, set a time range, and run the following query to identify timed-out events (`statement timeout`) and queries that successfully run for longer than 10 seconds (`duration`). ```sql select diff --git a/apps/docs/content/guides/database/prisma.mdx b/apps/docs/content/guides/database/prisma.mdx index 17f83390eb5..f7b9b40d783 100644 --- a/apps/docs/content/guides/database/prisma.mdx +++ b/apps/docs/content/guides/database/prisma.mdx @@ -18,7 +18,7 @@ If you plan to solely use Prisma instead of the Supabase Data API (PostgREST), t - In the [SQL Editor](/dashboard/project/_/sql/new), create a Prisma DB user with full privileges on the public schema. - - This gives you better control over Prisma's access and makes it easier to monitor using Supabase tools like the [Query Performance Dashboard](/dashboard/project/_/advisors/query-performance) and [Log Explorer](/dashboard/project/_/logs/explorer). + - This gives you better control over Prisma's access and makes it easier to monitor using Supabase tools like the [Query Performance Dashboard](/dashboard/project/_/advisors/query-performance) and [Logs](/dashboard/project/_/logs). For security, consider using a [password generator](https://bitwarden.com/password-generator/) for the Prisma role. diff --git a/apps/docs/content/guides/observability.mdx b/apps/docs/content/guides/observability.mdx index 7b3f6b054a9..31c7817a010 100644 --- a/apps/docs/content/guides/observability.mdx +++ b/apps/docs/content/guides/observability.mdx @@ -1,38 +1,28 @@ --- title: Observability -description: 'Access project data, detect issues, diagnose findings, and automate repeatable checks with an agent.' +description: 'Read project data, diagnose issues, and hire an agent to monitor your project' --- - +Use project data to understand what is happening, investigate issues, and give an agent repeatable checks to run. -Monitor your Supabase project with the tools you already use, as a person or an agent. +## Read project data [#metrics-api] -## 1. Observe the data - -The sources you can query, and where to read them. +Query logs for events, inspect database statistics, or review advisor findings. Use Reports to visualize signals and the Metrics API to export them. -## 2. Detect issues +## Detect and diagnose -Use queries and checks against those sources to pick up health, security, performance, and usage signals. +Run [detection checks](/docs/guides/observability/detecting) to identify health, security, performance, or capacity issues. Take the resulting error code, time window, or affected object to the [troubleshooting guides](/docs/guides/troubleshooting), then rerun the check after a fix. - +## Hire an agent -## 3. Diagnose and resolve - -Use a concrete finding, symptom, or error code to identify the cause and apply a known solution. - - - -## 4. Hire an agent - -Turn the checks you trust into a read-only routine in your agent harness and run it on a schedule. +Give an agent recurring checks to run and findings to report. [Set up an agent](/docs/guides/observability/automate-with-agents) with read-only access to your project. -## Export your data +## Configure and export -Send logs and traces to the tools you already run. +Record additional events or send telemetry to your monitoring tools. diff --git a/apps/docs/content/guides/observability/access-data.mdx b/apps/docs/content/guides/observability/access-data.mdx deleted file mode 100644 index 4e5cdbe7ab8..00000000000 --- a/apps/docs/content/guides/observability/access-data.mdx +++ /dev/null @@ -1,31 +0,0 @@ ---- -id: 'access-data' -title: 'Observe the data' -description: 'Query logs, metrics, database diagnostics, and advisors. Each source page lists Studio, MCP, the API, and the CLI.' ---- - -This guide lists the project data you can query. Each source page lists where to read that source. To pick up a signal from this data, see [Detecting](/docs/guides/observability/detecting). - -## Logs - -Request, database, Auth, Storage, Realtime, and function events in ClickHouse. - -Query them with SQL in [Query and filter logs](/docs/guides/observability/advanced-log-filtering) from the [Logs Explorer](/dashboard/project/_/logs/explorer), MCP `query_logs`, or the [Management API](/docs/reference/api/v1-get-project-logs). Filter events without SQL in [Logs](/docs/guides/observability/logs). See the [Logs field reference](/docs/guides/observability/log-field-reference) for sources and fields. - -The CLI does not query ClickHouse logs. Call the Management API from a script, or [inspect the database](/docs/guides/observability/inspect) for Postgres diagnostics. - -## Metrics [#metrics-api] - -Prometheus-compatible CPU, IO, WAL, connections, and query stats. Scrape the [Metrics API](/docs/guides/observability/metrics) for custom dashboards, alerting, or retention beyond Studio. Chart a subset of the same window in [Reports](/docs/guides/observability/reports). - -## Database - -Live Postgres statistics such as bloat, cache hit rate, blocking sessions, index usage, and slow queries. Run the same checks from the [SQL Editor](/dashboard/project/_/sql), MCP `execute_sql`, or `supabase inspect db`. See [Inspect the database](/docs/guides/observability/inspect). - -## Advisors - -Deterministic security and performance findings. Pull them from Studio, MCP `get_advisors`, [`supabase db advisors`](/docs/reference/cli/usage#supabase-db-advisors), or the Management API. See [Advisors](/docs/guides/observability/advisors). - -## Reports - -Studio dashboards for API, Auth, Storage, Realtime, and database signals. Use them to pick a time window or resource, then follow [Detecting](/docs/guides/observability/detecting). See [Reports](/docs/guides/observability/reports). diff --git a/apps/docs/content/guides/observability/advanced-log-filtering.mdx b/apps/docs/content/guides/observability/advanced-log-filtering.mdx index dc4e02d7f75..fc46a5c8b22 100644 --- a/apps/docs/content/guides/observability/advanced-log-filtering.mdx +++ b/apps/docs/content/guides/observability/advanced-log-filtering.mdx @@ -1,539 +1,120 @@ --- -title: 'Query and filter logs' -description: 'Query project logs from Studio, MCP, the API, or a script. Record extra Postgres, API, and Realtime events.' +title: 'Query logs with SQL' +description: 'Query ClickHouse logs through MCP, the Management API, or Explorer' --- -This guide explains how to query project logs and how to record extra events. The same ClickHouse SQL runs in the [Logs Explorer](/dashboard/project/_/logs/explorer), the MCP [`query_logs`](/docs/guides/ai-tools/mcp) tool, and the [Management API](/docs/reference/api/v1-get-project-logs). Filter events without SQL in [Logs](/docs/guides/observability/logs) in Studio. From a terminal, call the Management API; the CLI inspects the database rather than ClickHouse logs. +This guide explains how to query project logs with ClickHouse SQL. Use [MCP](#mcp) or the [Management API](#api) for programmatic access, or [Explorer](#studio) in Studio. To filter events without SQL, use [Logs in Studio](/docs/guides/observability/logs). -Use this page to: +## Query events [#querying-with-the-logs-explorer] -- Query logs from [Studio](#studio), [MCP](#mcp), the [API](#api), or a [script](#cli) -- Pick a [`source`](#logs-explorer) for the layer that reported the error -- Record extra [API](#working-with-api-logs), [Postgres](#logging-postgres-queries), and [Realtime](#logging-realtime-connections) events -- Write [ClickHouse SQL](#querying-with-the-logs-explorer) +Every event is a row in `logs`. Select a service with `source`, use a bounded time range, and limit the returned rows. For example, this query returns the latest API server errors within the supplied time range: -Every log line is one row in a single `logs` table, tagged by a `source` column. Structured fields live in a `log_attributes` map, and the raw line is in `event_message`. Filter by `source` to scope a query to one service. +```sql +-- recent API server errors +select timestamp, id, + toInt32OrZero(log_attributes['response.status_code']) as status, + log_attributes['request.path'] as path +from logs +where source = 'edge_logs' + and toInt32OrZero(log_attributes['response.status_code']) between 500 and 599 +order by timestamp desc +limit 100; +``` - - -ClickHouse has been the default engine since June 2026. Projects created before this date use BigQuery, whose `cross join unnest(metadata)` syntax is deprecated. We recommend rewriting those queries in the ClickHouse syntax shown in this guide. - - - -On hosted projects, prefer `query_logs` over `get_logs`. `get_logs` returns a service's recent logs without SQL; it remains the option for local and self-hosted projects. - -## Query from Studio, MCP, the API, or the CLI - -### Studio [#studio] - -Open [Logs](/dashboard/project/_/logs) to filter and inspect events. Open the [Logs Explorer](/dashboard/project/_/logs/explorer) to run ClickHouse SQL. See [Logs](/docs/guides/observability/logs) for the unified Logs interface. +Use the returned timestamp, ID, status, and path to investigate an event. No rows means no matching recorded events in that window; check the source, filters, and retention before concluding that there were no errors. ### MCP [#mcp] -On hosted projects, call [`query_logs`](/docs/guides/ai-tools/mcp) with the same SQL as this guide. Keep the connection project-scoped and read-only. +Connect [Supabase MCP](/docs/guides/ai-tools/mcp) with `project_ref` and `read_only=true`. Call `query_logs` with the SQL and an explicit time range, using the tool's input schema. Use `execute_sql` for Postgres database diagnostics, not ClickHouse logs. -### API [#api] +### Management API [#api] -Pass ClickHouse SQL in the `sql` parameter of the [Management API logs endpoint](/docs/reference/api/v1-get-project-logs). Unless you pass `sql`, that endpoint queries `edge_logs` only. Supply `iso_timestamp_start` and `iso_timestamp_end`; the range must be 24 hours or less. +Set `SUPABASE_ACCESS_TOKEN` to a Management API access token authorized to read project logs, and `PROJECT_REF` to the project reference. Set `START` and `END` to UTC timestamps such as `2026-09-07T09:00:00Z`, with a range of 24 hours or less. Save the query above as `logs.sql`, then run: -### CLI [#cli] - -The Supabase CLI does not query ClickHouse logs. Call the [Management API](/docs/reference/api/v1-get-project-logs) from a script, or use [`supabase inspect db`](/docs/guides/observability/inspect) for database diagnostics. - -## Sources [#logs-explorer] - -Filter by `source` to query one service. The Logs Explorer **Sources** drop-down lists these values. - -Pick the source for the layer that reported the error. A request hits the API gateway first, then one service, then the pooler and Postgres. The layer that _reports_ an error is often not the layer that _caused_ it. When two sources could fit, start closer to the database. - -```mermaid -flowchart TD - Client --> Gateway["API gateway — edge_logs"] - Gateway --> PostgREST - Gateway --> Auth - Gateway --> Storage - Gateway --> Realtime - PostgREST --> Pooler["Pooler — supavisor_logs, pgbouncer_logs"] - Auth --> Pooler - Storage --> Pooler - Pooler --> Postgres["Postgres — postgres_logs"] +```bash +curl --get "https://api.supabase.com/v1/projects/$PROJECT_REF/analytics/endpoints/logs" \ + --header "Authorization: Bearer $SUPABASE_ACCESS_TOKEN" \ + --data-urlencode "sql@logs.sql" \ + --data-urlencode "iso_timestamp_start=$START" \ + --data-urlencode "iso_timestamp_end=$END" ``` -Edge Functions sit outside that path: `function_edge_logs` is the HTTP request to the function, and `function_logs` is `console` output from inside it. +Inspect both the HTTP status and the response for query errors before interpreting the results. Without `sql`, this endpoint queries API Gateway events only. See the [logs endpoint reference](/docs/reference/api/v1-get-project-logs) for request and response fields. -A permission error or an empty result at the API is often row-level security in `postgres_logs`. +### Explorer [#studio] -| `source` | Events | -| -------------------- | ------------------------------------------------------------------------------------------------------ | -| `edge_logs` | HTTP requests through the API gateway, including REST and GraphQL | -| `postgres_logs` | Database queries, SQLSTATE, RLS, and functions | -| `postgrest_logs` | PostgREST process logs. Low-signal; `PGRST*` evidence usually lives in `edge_logs` and `postgres_logs` | -| `auth_logs` | Auth server: login, JWT, OAuth, email | -| `auth_audit_logs` | Auth audit events | -| `storage_logs` | Storage API: uploads and object access | -| `realtime_logs` | Realtime server: channels, presence, broadcast | -| `function_edge_logs` | HTTP request and response for an Edge Function invocation | -| `function_logs` | `console` output from inside an Edge Function | -| `supavisor_logs` | Shared pooler: pooling and timeouts | -| `pgbouncer_logs` | Dedicated pooler | -| `pg_upgrade_logs` | Database version upgrade | +1. Open [Explorer](/dashboard/project/_/explorer) and select **Run SQL**. +2. Open the query source menu and select **Logs**. +3. Choose the time range in that menu. +4. Enter the query and select **Run**. -For `postgres_logs`, statement text and error detail live in `event_message`. `parsed.query` and `parsed.detail` are usually empty. +The selected range is applied to the query. The **Logs** query source chooses ClickHouse; `source = 'edge_logs'` chooses API Gateway events within it. Select **Database** instead when running Postgres SQL. -For API Load Balancer traffic, the upstream database is `log_attributes['load_balancer_redirect_identifier']`. +### Terminal access [#cli] -See the [Logs field reference](/docs/guides/observability/log-field-reference) for the ClickHouse field names on each source. +The Supabase CLI does not query ClickHouse logs. Use the Management API command above. For live database statistics, use [`supabase inspect db`](/docs/guides/observability/inspect). -## Working with API logs [#working-with-api-logs] +## Sources and fields [#logs-explorer] -API Gateway logs run through Cloudflare and include Cloudflare metadata on the request. +Use the [Log sources and fields reference](/docs/guides/observability/log-field-reference) to choose the service and query expressions. API Gateway events and a service's own logs describe different layers of a request. -### Allowed headers +### Read structured fields [#understanding-field-references] -A strict list of request and response headers are permitted in the API logs. Request and response headers will still be received by the server(s) and client(s), but will not be attached to the API logs generated. +Read a map key with bracket access, retaining its full dotted path. Values in `log_attributes` are strings. Cast numeric values before comparing them. `toInt32OrZero` treats missing or non-numeric values as zero; do not interpret that zero as a measured status or duration. -Request headers: - -- `accept` -- `cf-connecting-ip` -- `cf-ipcountry` -- `host` -- `user-agent` -- `x-forwarded-proto` -- `referer` -- `content-length` -- `x-real-ip` -- `x-client-info` -- `x-forwarded-user-agent` -- `range` -- `prefer` - -Response headers: - -- `cf-cache-status` -- `cf-ray` -- `content-location` -- `content-range` -- `content-type` -- `content-length` -- `date` -- `transfer-encoding` -- `x-kong-proxy-latency` -- `x-kong-upstream-latency` -- `sb-gateway-mode` -- `sb-gateway-version` - -### Additional request metadata - -To attach additional metadata to a request, it is recommended to use the `User-Agent` header for purposes such as device or version identification. - -For example: - -``` -node MyApp/1.2.3 (device-id:abc123) -Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 MyApp/1.2.3 (Foo v1.3.2; Bar v2.2.2) -``` - - - -Do not log Personal Identifiable Information (PII) within the `User-Agent` header, to avoid infringing data protection privacy laws. Overly fine-grained and detailed user agents may allow fingerprinting and identification of the end user through PII. - - - -## Logging Postgres connections [#logging-postgres-connections] - -Postgres can log connection lifecycle events to your project's Postgres logs, for example when a client connects or authenticates. By default, Supabase sets `log_connections` to off for new projects and you must enable it first. - -To enable connection logging for audit or compliance, see [Postgres connection logging](/docs/guides/platform/postgres-connection-logging). - -In Logs, connection lifecycle messages are included when the Postgres log type is selected. Clear **Connection logs** under Postgres to hide them. - -## Logging Postgres queries [#logging-postgres-queries] - -To enable query logs for other categories of statements: - -1. [Enable the pgAudit extension](/dashboard/project/_/database/extensions). -2. Configure `pgaudit.log` (see below). Perform a fast reboot if needed. -3. View your query logs in [Logs](/dashboard/project/_/logs). Filter **Log Type** to Postgres. - -### Configuring `pgaudit.log` [#configuring-pgauditlog] - -The stored value under `pgaudit.log` determines the classes of statements that are logged by [pgAudit extension](https://www.pgaudit.org/). Refer to the pgAudit documentation for the [full list of values](https://github.com/pgaudit/pgaudit/blob/master/README.md#pgauditlog). - -To enable logging for function calls/do blocks, writes, and DDL statements for a single session, execute the following within the session: +When a field is missing or unfamiliar, discover the keys present on recorded events: ```sql --- temporary single-session config update -set pgaudit.log = 'function, write, ddl'; -``` - -To _permanently_ set a logging configuration (beyond a single session), execute the following, then perform a fast reboot: - -```sql --- equivalent permanent config update. -alter role postgres set pgaudit.log to 'function, write, ddl'; -``` - -To help with debugging, we recommend adjusting the log scope to only relevant statements as having too wide of a scope would result in a lot of noise in your Postgres logs. - -Note that in the above example, the role is set to `postgres`. To log user traffic flowing through the [HTTP APIs](/docs/guides/api#rest-api-overview), which use PostgREST, set your configuration values for the `authenticator`. - -```sql --- for API-related logs -alter role authenticator set pgaudit.log to 'write'; -``` - -By default, the log level will be set to `log`. To view other levels, run the following: - -```sql --- adjust log level -alter role postgres set pgaudit.log_level to 'info'; -alter role postgres set pgaudit.log_level to 'debug5'; -``` - -Note that as per the pgAudit [log_level documentation](https://github.com/pgaudit/pgaudit/blob/master/README.md#pgauditlog_level), `error`, `fatal`, and `panic` are not allowed. - -To reset system-wide settings, execute the following, then perform a fast reboot: - -```sql --- resets stored config. -alter role postgres reset pgaudit.log -``` - - - -If any permission errors are encountered when executing `alter role postgres ...`, it is likely that your project has yet to receive the patch to the latest version of [supautils](https://github.com/supabase/supautils), which is currently being rolled out. - - - -### `RAISE`d log messages in Postgres - -Messages that are manually logged via `RAISE INFO`, `RAISE NOTICE`, `RAISE WARNING`, and `RAISE LOG` are shown in Postgres Logs. Note that only messages at or above your logging level are shown. Syncing of messages to Postgres Logs may take a few minutes. - -If your logs aren't showing, check your logging level by running: - -```sql -show log_min_messages; -``` - -Note that `LOG` is a higher level than `WARNING` and `ERROR`, so if your level is set to `LOG`, you will not see `WARNING` and `ERROR` messages. - -### Limits and caveats - -- Postgres log events on the Supabase Platform are limited to 100,000 characters. If a log event exceeds this limit, it will be truncated. This does not apply to self-hosting. -- Internal connection logs to Postgres within the Supabase Platform by internal services are not logged. This does not apply to self-hosting. - -## Logging realtime connections [#logging-realtime-connections] - -Realtime doesn't log new WebSocket connections or Channel joins by default. Enable connection logging per client by including an `info` `log_level` parameter when instantiating the Supabase client. - -```javascript -import { createClient } from '@supabase/supabase-js' - -const options = { - realtime: { - params: { - log_level: 'info', - }, - }, -} -const supabase = createClient('https://xyzcompany.supabase.co', 'sb_publishable_...', options) -``` - -## Querying logs [#querying-with-the-logs-explorer] - -Read fields with bracket access, keeping the full dotted key, for example `log_attributes['request.path']` rather than `path`. Wrap numeric values in `toInt32OrZero(...)`, which returns `0` for a missing or non-numeric value. Use `count()` rather than `count(*)`. - -For example, to find failing API requests: - -```sql -select timestamp, - toInt32OrZero(log_attributes['response.status_code']) as status, - log_attributes['request.path'] as path -from logs -where source = 'edge_logs' - and toInt32OrZero(log_attributes['response.status_code']) >= 400 -order by timestamp desc -limit 100; -``` - -For example, to find a specific Postgres SQLSTATE (`42501` permission denied, `42P01` relation missing, `23505` duplicate key): - -```sql -select timestamp, log_attributes['parsed.user_name'] as role, event_message -from logs -where source = 'postgres_logs' - and log_attributes['parsed.sql_state_code'] = '42501' -order by timestamp desc -limit 100; -``` - -The Management API accepts this SQL in the `sql` parameter. Unless you pass `sql`, that endpoint queries `edge_logs` only. Supply `iso_timestamp_start` and `iso_timestamp_end`; the range must be 24 hours or less. - -## Timestamp display and behavior - -The `timestamp` column is a `DateTime64` value in UTC, formatted as an ISO-8601 string like `2026-06-22T09:34:06.215000`. You can order and compare it directly, so no conversion function is needed. In the Logs Explorer the selected time range is applied for you, so you rarely need to filter on `timestamp` by hand. MCP and the Management API require an explicit time range. - -```sql -select timestamp, event_message -from logs -where source = 'edge_logs' -order by timestamp desc -limit 100; -``` - -## Reading fields from log_attributes - -Structured fields live in the `log_attributes` map. Read a field with bracket access, keeping the full dotted key. There are no unnesting joins. - -```sql -select - log_attributes['request.method'] as method, - log_attributes['request.path'] as path, - log_attributes['response.status_code'] as status -from logs -where source = 'edge_logs' -limit 100; -``` - -The key keeps the full dotted path, with the `metadata` root dropped. What BigQuery expressed as `metadata.request.cf.country` is `log_attributes['request.cf.country']`. Keep the full prefix rather than shortening it. - -Map values are always strings. To compare or aggregate a numeric field, wrap it in `toInt32OrZero`, which returns `0` for a missing or non-numeric value: - -```sql -select count() as server_errors -from logs -where source = 'edge_logs' - and toInt32OrZero(log_attributes['response.status_code']) between 500 and 599; -``` - -Do not guess keys. Discover the keys a source sets from recent rows: - -```sql -select arrayJoin(mapKeys(log_attributes)) as key, count() as n +-- discover Postgres attributes +select arrayJoin(mapKeys(log_attributes)) as key, count() as events from logs where source = 'postgres_logs' group by key -order by n desc +order by events desc limit 100; ``` -## LIMIT and result row limitations +### Time ranges [#timestamp-display-and-behavior] -The Logs Explorer has a maximum of 1000 rows per run. Use `LIMIT` to reduce the number of rows returned further. +`timestamp` is a UTC `DateTime64` value. Compare and order it directly. Explorer supplies the chosen time range; MCP and API callers must supply their own bounded range. To compare more than 24 hours through the API, fetch separate windows within retention and combine their aggregates. -## Best practices +## Search messages [#filtering-with-regular-expressions] -1. **Use a narrow time range.** - -The Logs Explorer applies the time range you select, so keep it tight. Querying a very large range risks timeouts, especially for Enterprise customers with long retention, because of the extra data scanned. - -2. **Select only the fields you need.** - -Selecting the whole `log_attributes` map, or every column, reads far more data than you need and slows the query down. Select the specific keys instead. +Use `ilike` for a case-insensitive substring, or ClickHouse's [`match`](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match) for a regular expression: ```sql --- ❌ Avoid this: selecting the whole attributes map -select timestamp, log_attributes +-- find connection failures +select timestamp, id, event_message from logs -where source = 'edge_logs'; - --- ✅ Do this: select only the keys you need -select timestamp, log_attributes['request.method'] as method -from logs -where source = 'edge_logs'; -``` - -3. **Query one source at a time.** - -Identify which service owns the problem from the error or status code first, then query only that source. Scanning every source at once buries the signal you need and scans far more data than the investigation requires. - -4. **Follow a request across sources with an anchor.** Once a query gives you an anchor such as a timestamp, request id, or SQL state, filter the adjacent source by that anchor to correlate the request across layers (for example `edge_logs` to `postgres_logs`), instead of re-scanning each source from scratch. - -5. **Reference only fields you have confirmed.** - -A misspelled or non-existent field name either errors or silently returns nothing, which leaves a working query look empty. Confirm field names in the [Logs field reference](/docs/guides/observability/log-field-reference), or select `event_message` and inspect a sample row first. - -## Examples and templates - -The Logs Explorer includes **Templates** (available in the Templates tab or the dropdown in the Query tab) to help you get started. - -For example, you can enter the following query in the SQL Editor to retrieve each user's IP address: - -```sql -select timestamp, log_attributes['request.headers.x_real_ip'] as x_real_ip -from logs -where source = 'edge_logs' - and log_attributes['request.headers.x_real_ip'] != '' - and log_attributes['request.method'] = 'GET' +where source = 'postgres_logs' + and event_message ilike '%connection%' + and match(event_message, '(?i)failed|refused|timeout') order by timestamp desc limit 100; ``` -## Understanding field references +Combine predicates with `and`, `or`, and `not`. Select only the fields needed for the investigation. To correlate sources, use an identifier present in both; a shared timestamp alone does not establish that events belong to the same request. -Every log source shares the same `logs` table. Each row has these columns: +## Query limits [#limit-and-result-row-limitations] -| column | description | -| ---------------- | -------------------------------------------------- | -| `id` | unique log identifier | -| `timestamp` | time the event was recorded | -| `event_message` | the log's message | -| `severity_text` | log level, when the source sets one | -| `source` | the service the log came from | -| `log_attributes` | structured per-source fields, keyed by dotted path | +Use an explicit `limit` and narrow time range. The logs query surface rejects `select *` and `count(*)`; list columns and use `count()`. A result limit bounds returned rows, not the time range scanned. -Service-specific details live in `log_attributes`. For example, in `postgres_logs` the `log_attributes['parsed.error_severity']` field holds the error level of an event. Read those fields with bracket access: +## Record additional events [#working-with-api-logs] -```sql -select - event_message, - log_attributes['parsed.error_severity'] as error_severity, - log_attributes['parsed.user_name'] as user_name -from logs -where source = 'postgres_logs' -limit 100; -``` +For HTTP header capture, see [Captured HTTP headers](/docs/guides/observability/log-field-reference#captured-http-headers). Configure event recording separately from querying: -## Filtering with [regular expressions](https://en.wikipedia.org/wiki/Regular_expression) +### Postgres connections [#logging-postgres-connections] -Use the ClickHouse [`match` function](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match) for regular expressions. In its most basic form, it checks whether a pattern is present in a column. +See [Configure connection logging](/docs/guides/observability/configure-logging#postgres-connections). -```sql -select timestamp, event_message -from logs -where source = 'postgres_logs' - and match(event_message, 'is present') -limit 100; -``` +### Postgres statements [#logging-postgres-queries] -There are multiple operators to consider using. +See [Configure statement logging](/docs/guides/observability/configure-logging#postgres-statements). -### Find messages that start with a phrase +### Statement classes [#configuring-pgauditlog] -`^` only looks for values at the start of a string +See [pgAudit configuration](/docs/guides/database/extensions/pgaudit#configure-the-extension) for session and role scope. -```sql --- find only messages that start with connection -match(event_message, '^connection') -``` +### Realtime connections [#logging-realtime-connections] -### Find messages that end with a phrase - -`$` only looks for values at the end of the string - -```sql --- find only messages that end with port=12345 -match(event_message, 'port=12345$') -``` - -### Ignore case sensitivity - -`(?i)` ignores capitalization for all proceeding characters - -```sql --- find all event_messages with the word "connection" -match(event_message, '(?i)COnnecTion') -``` - -For a plain case-insensitive substring match, `ilike` is simpler: - -```sql --- find all event_messages containing "connection", in any case -event_message ilike '%connection%' -``` - -### Wildcards - -`.` matches any single character, and `.*` matches any sequence of characters - -```sql --- find event_messages like "helloworld" -match(event_message, 'hello.*world') -``` - -### Alphanumeric ranges - -`[0-9a-zA-Z]` matches a single alphanumeric character. Anchor it with `^[0-9a-zA-Z]+$` to match a value that is entirely alphanumeric. - -```sql --- find event_messages that contain a digit between 1 and 5 (inclusive) -match(event_message, '[1-5]') -``` - -### Repeated values - -`x*` zero or more x -`x+` one or more x -`x?` zero or one x -`x{4,}` four or more x -`x{3}` exactly 3 x - -```sql --- find event_messages that contain any sequence of 3 digits -match(event_message, '[0-9]{3}') -``` - -### Escaping reserved characters - -`\.` is interpreted as a period `.` instead of as a wildcard - -```sql --- escapes . -match(event_message, 'hello world\.') -``` - -### `or` statements - -`x|y` any string with `x` or `y` present - -```sql --- find event_messages that have the word 'started' followed by either "host" or "authenticated" -match(event_message, 'started (host|authenticated)') -``` - -### `and`/`or`/`not` statements in SQL - -`and`, `or`, and `not` are native terms in SQL and can be used with regular expressions to filter results - -```sql -select timestamp, event_message -from logs -where source = 'postgres_logs' - and ( - (match(event_message, 'connection') and match(event_message, 'host')) - or not match(event_message, 'received') - ) -limit 100; -``` - -### Filtering example - -Filter for Postgres errors: - -```sql -select - timestamp, - log_attributes['parsed.error_severity'] as error_severity, - log_attributes['parsed.user_name'] as user_name, - event_message -from logs -where source = 'postgres_logs' - and match(log_attributes['parsed.error_severity'], 'ERROR|FATAL|PANIC') -order by timestamp desc -limit 100; -``` - -## Limitations - -### The wildcard operator `*` is not supported - -The logs query surface rejects `select *` and `count(*)`. List the columns you need, and use `count()` for row counts: - -```sql -select timestamp, event_message, log_attributes['parsed.error_severity'] as error_severity -from logs -where source = 'postgres_logs' -order by timestamp desc -limit 100; -``` +See [Configure Realtime logging](/docs/guides/observability/configure-logging#realtime-connections). diff --git a/apps/docs/content/guides/observability/advisors.mdx b/apps/docs/content/guides/observability/advisors.mdx index 7f64deff229..16098634142 100644 --- a/apps/docs/content/guides/observability/advisors.mdx +++ b/apps/docs/content/guides/observability/advisors.mdx @@ -6,16 +6,16 @@ description: 'Deterministic security and performance findings you or an agent ca Advisors are programmatic checks that ship with the platform. They inspect the live schema and return deterministic findings, such as missing indexes or incorrectly configured RLS policies. -Use them as part of ongoing observability, together with [logs](/docs/guides/observability/advanced-log-filtering). A finding is not a fix. Confirm it against recent log evidence, then search [Diagnosing](/docs/guides/troubleshooting) for the check name or the object it names. +Confirm each finding against the intended schema and access model. Search [Troubleshooting](/docs/guides/troubleshooting) for its check name or affected object; logs can provide additional context but are not required to establish a schema finding. You or an agent can pull the same checks from: - Studio: [Security Advisor](/dashboard/project/_/advisors/security) and [Performance Advisor](/dashboard/project/_/advisors/performance) -- MCP: `get_advisors` +- MCP: `get_advisors` with `type` set to `security` or `performance` - CLI: [`supabase db advisors`](/docs/reference/cli/supabase-db-advisors) - Management API: [security advisors](/docs/reference/api/v1-get-security-advisors) and [performance advisors](/docs/reference/api/v1-get-performance-advisors) -The advisors run automatically in Studio. Rerun them after you resolve an issue. +Prioritize warning and error findings. Each finding names a check, severity, affected object, and remediation guidance. Informational findings provide context and do not always require a change. The advisors run automatically in Studio. After an authorized fix, rerun the relevant advisor and confirm that the finding no longer appears. ## Available checks diff --git a/apps/docs/content/guides/observability/configure-logging.mdx b/apps/docs/content/guides/observability/configure-logging.mdx new file mode 100644 index 00000000000..1b268fe1ce0 --- /dev/null +++ b/apps/docs/content/guides/observability/configure-logging.mdx @@ -0,0 +1,47 @@ +--- +title: 'Configure logging' +description: 'Record additional Postgres and Realtime events for an investigation' +--- + +This guide explains how to record events that are not logged by default. Logging changes affect future events; they cannot recover past activity. Keep the scope limited to the investigation, because recorded statements and messages can contain sensitive values. + +## Postgres connections + +To record connection and authentication events, follow [Postgres connection logging](/docs/guides/platform/postgres-connection-logging). Note the current setting before changing it. + +After enabling logging, open a new database connection and find its event in [Logs](/dashboard/project/_/logs) with **Log Type** set to **Postgres** and **Connection logs** enabled. Restore the previous setting when the investigation is complete, unless continued logging is required. + +## Postgres statements + +1. Enable [pgAudit](/docs/guides/database/extensions/pgaudit#enable-the-extension). +2. Select the statement classes and session or role scope in [pgAudit configuration](/docs/guides/database/extensions/pgaudit#configure-the-extension). Record the previous setting first. API traffic through PostgREST uses the `authenticator` role. +3. Run an authorized operation in the configured scope, then find its audit event in [Logs](/dashboard/project/_/logs) with **Log Type** set to **Postgres**. +4. Restore the previous logging configuration when finished. + +Session settings apply only to that database connection. Studio queries do not maintain a persistent session. For persistent logging, follow the role-scoped instructions in the pgAudit guide. + +### Messages from database functions + +Whether a `RAISE` message reaches Postgres logs depends on `log_min_messages`. Read the current value from a database connection: + +```sql +show log_min_messages; +``` + +Allow a few minutes for messages to appear. See [Postgres message levels](https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-LOG-MIN-MESSAGES) before changing the threshold; their ordering differs from client message levels. + +## Realtime connections + +Realtime does not log new WebSocket connections or channel joins by default. Enable connection logging for the client under investigation: + +```javascript +import { createClient } from '@supabase/supabase-js' + +const supabase = createClient('https://your-project.supabase.co', 'sb_publishable_...', { + realtime: { params: { log_level: 'info' } }, +}) +``` + +Reconnect that client and join a channel, then inspect **Realtime** events in [Logs](/dashboard/project/_/logs). Remove `log_level: 'info'` and recreate the client to restore the default behavior. + +For truncation and capture constraints, see [Log sources and fields](/docs/guides/observability/log-field-reference#capture-limits). diff --git a/apps/docs/content/guides/observability/detecting.mdx b/apps/docs/content/guides/observability/detecting.mdx index 7196a9779cd..c8ec348a221 100644 --- a/apps/docs/content/guides/observability/detecting.mdx +++ b/apps/docs/content/guides/observability/detecting.mdx @@ -4,9 +4,9 @@ title: 'Detecting issues' description: 'Run Health, Security, Performance, and Usage checks against logs and database statistics to pick up actionable signals.' --- -Detection is the step between accessing project data and troubleshooting a specific problem. Use the sources in [Observe the data](/docs/guides/observability/access-data) to produce a count, rate, trend, or named finding. Do not try to prove the root cause yet. +Detection is the step between accessing project data and troubleshooting a specific problem. Use the sources in [Observability](/docs/guides/observability) to produce a count, rate, trend, or named finding. Do not try to prove the root cause yet. -This guide provides starting checks for [Health](#health), [Security](#security), [Performance](#performance), and [Usage](#usage). The log examples use ClickHouse SQL in the [Logs Explorer](/dashboard/project/_/logs/explorer) or MCP `query_logs`. The database examples use Postgres SQL in the [SQL Editor](/dashboard/project/_/sql) or MCP `execute_sql`. +This guide provides starting checks for [Health](#health), [Security](#security), [Performance](#performance), and [Usage](#usage). The log examples use ClickHouse SQL in the [Explorer](/dashboard/project/_/explorer) with query source **Logs** or MCP `query_logs`. The database examples use Postgres SQL in the [Explorer](/dashboard/project/_/explorer) with query source **Database** or MCP `execute_sql`. Use a time range that represents normal traffic, then compare it with the same period after a deployment or configuration change. When a check returns a spike, error code, SQLSTATE, object name, or advisor finding, take that evidence to [Diagnosing](/docs/guides/troubleshooting). @@ -280,4 +280,4 @@ order by connections desc; A detection result should name an affected time window and at least one concrete anchor: a path, status, SQLSTATE, request ID, query, relation, PID, policy, or advisor lint. Take that evidence to [Diagnosing](/docs/guides/troubleshooting), identify the cause, apply the smallest relevant solution, and rerun the same detection check to verify the result. -After a check is useful and repeatable, [hire an agent](/docs/guides/observability/automate-with-agents) to run it on a schedule. +After a check is useful and repeatable, [automate monitoring](/docs/guides/observability/automate-with-agents) to run it on a schedule. diff --git a/apps/docs/content/guides/observability/inspect.mdx b/apps/docs/content/guides/observability/inspect.mdx index 41dde492d6a..07f3116a2ce 100644 --- a/apps/docs/content/guides/observability/inspect.mdx +++ b/apps/docs/content/guides/observability/inspect.mdx @@ -1,23 +1,23 @@ --- id: 'inspect' title: 'Inspect the database' -description: 'Read live Postgres statistics such as bloat, cache hit rate, locks, and slow queries from the CLI, SQL Editor, or MCP.' +description: 'Read live Postgres statistics such as bloat, cache hit rate, locks, and slow queries from the CLI, Explorer, or MCP.' --- -Database performance is a large topic and many factors can contribute. Common causes of poor performance include inefficient schemas or queries, missing or unused indexes, insufficient memory, lock contention, and table bloat. +This guide explains how to read live database statistics using the CLI, MCP, or Explorer. -Use the live Postgres statistics in this guide to check for those conditions. You or an agent can run the same checks from: +Read database statistics from: -- Studio: [SQL Editor](/dashboard/project/_/sql) +- Studio: [Explorer](/dashboard/project/_/explorer) with query source **Database** - MCP: `execute_sql` - CLI: [`supabase inspect db`](/docs/reference/cli/supabase-inspect-db) Use this page to: - Run [CLI inspection commands](#using-the-cli) -- Copy the matching [SQL](#using-sql) +- Run [SQL checks](#using-sql) -To pick up a signal from these checks, see [Detecting](/docs/guides/observability/detecting). For the other sources, see [Observe the data](/docs/guides/observability/access-data). +To pick up a signal from these checks, see [Detecting](/docs/guides/observability/detecting). For the other sources, see [Observability](/docs/guides/observability). ## Using the CLI @@ -104,144 +104,12 @@ The commands below are useful if your Postgres database consumes a lot of resour - [role-connections](/docs/reference/cli/supabase-inspect-db-role-connections) - shows number of active connections for all database roles (Supabase-specific command) - [replication-slots](/docs/reference/cli/supabase-inspect-db-replication-slots) - shows information about replication slots on the database -### Notes on `pg_stat_statements` - -Following commands require `pg_stat_statements` to be enabled: calls, locks, cache-hit, blocking, unused-indexes, index-usage, bloat, outliers, table-record-counts, replication-slots, seq-scans, vacuum-stats, long-running-queries. - -When using `pg_stat_statements` also take note that it only stores the latest 5,000 statements. Moreover, consider resetting the analysis after optimizing any queries by running `select pg_stat_statements_reset();` - -Learn more about [`pg_stat_statements`](/docs/guides/database/extensions/pg_stat_statements). - ## Using SQL - +Open [Explorer](/dashboard/project/_/explorer), select **Run SQL**, and choose **Database** as the query source. You can also run read-only diagnostics through MCP `execute_sql`. -If you're seeing an `insufficient privilege` error when viewing the Query Performance page from the dashboard, run this command: +Use [Performance checks](/docs/guides/observability/detecting#performance) for active sessions, blockers, expensive statements, and cache hit rates. Use [Capacity checks](/docs/guides/observability/detecting#usage) for relation sizes and connection counts. -```shell -$ grant pg_read_all_stats to postgres; -``` +`pg_stat_activity` is a live snapshot. `pg_stat_statements` and cache counters are cumulative since their last reset; they do not describe an arbitrary historical window. Compare saved snapshots with the same reset interval when measuring changes. Check the [pg_stat_statements guide](/docs/guides/database/extensions/pg_stat_statements) for extension requirements. - - -### Postgres cumulative statistics system - -Postgres collects data about its own operations using the [cumulative statistics system](https://www.postgresql.org/docs/current/monitoring-stats.html). In addition to this, every Supabase project has the [pg_stat_statements extension](/docs/guides/database/extensions/pg_stat_statements) enabled by default. This extension records query execution performance details. - -Here are some example queries to get you started. - -### Most frequently called queries - -```sql -select - auth.rolname, - statements.query, - statements.calls, - -- -- Postgres 13, 14, 15 - statements.total_exec_time + statements.total_plan_time as total_time, - statements.min_exec_time + statements.min_plan_time as min_time, - statements.max_exec_time + statements.max_plan_time as max_time, - statements.mean_exec_time + statements.mean_plan_time as mean_time, - -- -- Postgres <= 12 - -- total_time, - -- min_time, - -- max_time, - -- mean_time, - statements.rows / statements.calls as avg_rows -from - pg_stat_statements as statements - inner join pg_authid as auth on statements.userid = auth.oid -order by statements.calls desc -limit 100; -``` - -This query shows: - -- query statistics, ordered by the number of times each query has been executed -- the role that ran the query -- the number of times it has been called -- the average number of rows returned -- the cumulative total time the query has spent running -- the min, max and mean query times. - -This provides useful information about the queries you run most frequently. Queries that have high `max_time` or `mean_time` times and are being called often can be good candidates for optimization. - -### Slowest queries by execution time - -```sql -select - auth.rolname, - statements.query, - statements.calls, - -- -- Postgres 13, 14, 15 - statements.total_exec_time + statements.total_plan_time as total_time, - statements.min_exec_time + statements.min_plan_time as min_time, - statements.max_exec_time + statements.max_plan_time as max_time, - statements.mean_exec_time + statements.mean_plan_time as mean_time, - -- -- Postgres <= 12 - -- total_time, - -- min_time, - -- max_time, - -- mean_time, - statements.rows / statements.calls as avg_rows -from - pg_stat_statements as statements - inner join pg_authid as auth on statements.userid = auth.oid -order by max_time desc -limit 100; -``` - -This query will show you statistics about queries ordered by the maximum execution time. It is similar to the query above ordered by calls, but this one highlights outliers that may have high executions times. Queries which have high or mean execution times are good candidates for optimization. - -### Most time consuming queries - -```sql -select - auth.rolname, - statements.query, - statements.calls, - statements.total_exec_time + statements.total_plan_time as total_time, - to_char( - ( - (statements.total_exec_time + statements.total_plan_time) / sum( - statements.total_exec_time + statements.total_plan_time - ) over () - ) * 100, - 'FM90D0' - ) || '%' as prop_total_time -from - pg_stat_statements as statements - inner join pg_authid as auth on statements.userid = auth.oid -order by total_time desc -limit 100; -``` - -This query will show you statistics about queries ordered by the cumulative total execution time. It shows the total time the query has spent running as well as the proportion of total execution time the query has taken up. - -Queries which are the most time consuming are not necessarily bad, you may have a very efficient and frequently ran queries that end up taking a large total % time, but it can be useful to help spot queries that are taking up more time than they should. - -### Hit rate - -Generally for most applications a small percentage of data is accessed more regularly than the rest. To make sure that your regularly accessed data is available, Postgres tracks your data access patterns and keeps this in its [shared_buffers](https://www.postgresql.org/docs/15/runtime-config-resource.html#RUNTIME-CONFIG-RESOURCE-MEMORY) cache. - -Applications with lower cache hit rates generally perform more poorly since they have to hit the disk to get results rather than serving them from memory. Very poor hit rates can also cause you to burst past your [Disk IO limits](/docs/guides/platform/compute-and-disk#disk) causing significant performance issues. - -You can view your cache and index hit rate by executing the following query: - -```sql -select - 'index hit rate' as name, - (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read), 0) * 100 as ratio -from pg_statio_user_indexes -union all -select - 'table hit rate' as name, - sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read), 0) * 100 as ratio -from pg_statio_user_tables; -``` - -This shows the ratio of data blocks fetched from the Postgres [shared_buffers](https://www.postgresql.org/docs/15/runtime-config-resource.html#RUNTIME-CONFIG-RESOURCE-MEMORY) cache against the data blocks that were read from disk or the OS cache. - -A ratio below 99% means more than 1% of observed block accesses missed `shared_buffers`. Postgres cannot distinguish whether those reads were served by the operating system cache or physical disk. Treat that as a [Performance](/docs/guides/observability/detecting#performance) signal, then search [Diagnosing](/docs/guides/troubleshooting). - -When a check names a slow statement, get a query plan with [`explain`](/docs/guides/database/query-optimization#analyze-the-query-plan) in SQL, or [`explain()`](/docs/guides/database/debugging-performance) on the Data API. Pair `pg_stat_statements` with the [Metrics API](/docs/guides/observability/metrics) to read the same window from Postgres stats and host metrics. +When a check identifies a statement, inspect its [query plan](/docs/guides/database/query-optimization#analyze-the-query-plan). A long-running session or high cumulative query time is evidence to investigate, not a reason by itself to cancel a query or reset statistics. diff --git a/apps/docs/content/guides/observability/log-field-reference.mdx b/apps/docs/content/guides/observability/log-field-reference.mdx index 99e444585ef..d3e7ef6db3c 100644 --- a/apps/docs/content/guides/observability/log-field-reference.mdx +++ b/apps/docs/content/guides/observability/log-field-reference.mdx @@ -1,6 +1,6 @@ --- id: 'logs-field-reference' -title: 'Logs field reference' +title: 'Log sources and fields' description: 'Log sources, ClickHouse fields, and event capture limits' --- @@ -8,6 +8,60 @@ Each event is a row in the ClickHouse `logs` table. Filter the `source` column t `id`, `timestamp`, `event_message`, `severity_text`, and `source` are top-level columns. Service fields are string values in `log_attributes`, even when the original event contains a number or boolean. Use the **ClickHouse query field** column directly. See [Query logs with SQL](/docs/guides/observability/advanced-log-filtering) for casting and field discovery. +## Sources + +| `source` | Events | +| -------------------- | ----------------------------------------------------------------- | +| `edge_logs` | HTTP requests through the API gateway, including REST and GraphQL | +| `postgres_logs` | Database activity, statements, and errors | +| `postgrest_logs` | PostgREST server logs | +| `auth_logs` | Auth server: login, JWT, OAuth, email | +| `auth_audit_logs` | Auth audit events | +| `storage_logs` | Storage API: uploads and object access | +| `realtime_logs` | Realtime server: channels, presence, broadcast | +| `function_edge_logs` | HTTP request and response for an Edge Function invocation | +| `function_logs` | `console` output from inside an Edge Function | +| `supavisor_logs` | Shared pooler: pooling and timeouts | +| `pgbouncer_logs` | Dedicated pooler | +| `pg_upgrade_logs` | Database version upgrade | + +For `postgres_logs`, statement text and error details can appear in `event_message`. A missing structured field does not mean the event has no detail. + +## Captured HTTP headers + +API Gateway logs capture only the headers below. Other headers still reach the application and client but are omitted from these logs. + +Request headers: + +- `accept` +- `cf-connecting-ip` +- `cf-ipcountry` +- `host` +- `user-agent` +- `x-forwarded-proto` +- `referer` +- `content-length` +- `x-real-ip` +- `x-client-info` +- `x-forwarded-user-agent` +- `range` +- `prefer` + +Response headers: + +- `cf-cache-status` +- `cf-ray` +- `content-location` +- `content-range` +- `content-type` +- `content-length` +- `date` +- `transfer-encoding` +- `x-kong-proxy-latency` +- `x-kong-upstream-latency` +- `sb-gateway-mode` +- `sb-gateway-version` + ## Capture limits - Hosted Postgres events longer than 100,000 characters and Edge Function log messages longer than 10,000 characters are truncated. diff --git a/apps/docs/content/guides/observability/logs.mdx b/apps/docs/content/guides/observability/logs.mdx index 28e03353b82..2d82edf23a2 100644 --- a/apps/docs/content/guides/observability/logs.mdx +++ b/apps/docs/content/guides/observability/logs.mdx @@ -43,7 +43,7 @@ For SQL source names, see the [Log field reference](/docs/guides/observability/l Postgres logs contain database activity and errors. Connection events appear when [connection logging](/docs/guides/platform/postgres-connection-logging) is enabled. Clear **Connection logs** under **Postgres** to hide them. -To record additional statement classes, see [Logging Postgres queries](/docs/guides/observability/advanced-log-filtering#logging-postgres-queries). +To record additional statement classes, see [Configure statement logging](/docs/guides/observability/configure-logging#postgres-statements). ## Inspect an event [#expanding-results] @@ -63,6 +63,6 @@ For continuous export, use [Log drains](/docs/guides/observability/log-drains). Check the time range, selected log types, and exclusions first. **User** combined with only Postgres or another unsupported type returns no matches. An empty result does not establish that the user had no activity. -Events must be recorded before they can appear in Logs. See [Logging configuration](/docs/guides/observability/advanced-log-filtering#logging-postgres-connections) and the [source limitations](/docs/guides/observability/log-field-reference#capture-limits). +Events must be recorded before they can appear in Logs. See [Configure logging](/docs/guides/observability/configure-logging) and the [source limitations](/docs/guides/observability/log-field-reference#capture-limits). Retention depends on your [pricing plan](/pricing). See [Manage Logs usage](/docs/guides/platform/manage-your-usage/logs) for billing details. diff --git a/apps/docs/content/guides/platform/manage-your-usage/egress.mdx b/apps/docs/content/guides/platform/manage-your-usage/egress.mdx index 09e49705282..6b11627f331 100644 --- a/apps/docs/content/guides/platform/manage-your-usage/egress.mdx +++ b/apps/docs/content/guides/platform/manage-your-usage/egress.mdx @@ -195,7 +195,7 @@ height={510} ### Most requested API endpoints -In the [Logs Explorer](/dashboard/project/_/logs/explorer) you can access Edge Logs, and review the top paths to identify heavily queried endpoints. These logs currently do not include response byte data. That data will be available in the future too. +In [Explorer](/dashboard/project/_/explorer), select query source **Logs** and [query API Gateway events](/docs/guides/observability/advanced-log-filtering) to identify heavily queried paths. These logs currently do not include response byte data. That data will be available in the future too. Top paths -For more details on filtering the log tables, see [Query and filter logs](/docs/guides/observability/advanced-log-filtering) +For more details on filtering the log tables, see [Query logs with SQL](/docs/guides/observability/advanced-log-filtering) diff --git a/apps/docs/content/guides/storage/serving/bandwidth.mdx b/apps/docs/content/guides/storage/serving/bandwidth.mdx index 815133e8103..9334f352774 100644 --- a/apps/docs/content/guides/storage/serving/bandwidth.mdx +++ b/apps/docs/content/guides/storage/serving/bandwidth.mdx @@ -10,9 +10,9 @@ sidebar_label: 'Bandwidth & Storage Egress' Free Plan Organizations in Supabase have a limit of 10 GB of bandwidth (5 GB cached + 5 GB uncached). This limit is calculated by the sum of all the data transferred from the Supabase servers to the client. This includes all the data transferred from the database, storage, and functions. -### Checking Storage egress requests in the SQL Editor +### Query storage egress requests [#checking-storage-egress-requests-in-the-sql-editor] -You can use the following query to get the number of requests for each object. Run it in the [SQL Editor](/dashboard/project/_/sql/new?skip=true&source=logs) with the query source set to **Logs**. +You can use the following query to get the number of requests for each object. Run it in the [Explorer](/dashboard/project/_/explorer) after selecting **Run SQL**, query source **Logs**, and a time range. ```sql select diff --git a/apps/docs/data/content-listings/telemetry.data.ts b/apps/docs/data/content-listings/telemetry.data.ts index b901c3ba481..65d8062d898 100644 --- a/apps/docs/data/content-listings/telemetry.data.ts +++ b/apps/docs/data/content-listings/telemetry.data.ts @@ -8,10 +8,19 @@ export const telemetryAccessWhat: ContentListingGroup = { columns: 2, items: [ { - title: 'Logs', + title: 'Query logs with SQL', href: '/guides/observability/advanced-log-filtering', - description: - 'Query ClickHouse logs from Studio, MCP, or the API. Filter events in the Logs UI.', + description: 'Query ClickHouse events through MCP, the API, or Explorer.', + }, + { + title: 'Logs in Studio', + href: '/guides/observability/logs', + description: 'Filter, inspect, and export events in the unified Logs view.', + }, + { + title: 'Log sources and fields', + href: '/guides/observability/log-field-reference', + description: 'Look up sources, ClickHouse query fields, and capture limits.', }, { title: 'Metrics API', @@ -19,9 +28,9 @@ export const telemetryAccessWhat: ContentListingGroup = { description: 'Scrape Prometheus-compatible database metrics, or chart a subset in Reports.', }, { - title: 'Database', + title: 'Inspect the database', href: '/guides/observability/inspect', - description: 'Inspect live Postgres stats from the CLI, the SQL Editor, or MCP.', + description: 'Inspect live Postgres stats from the CLI, Explorer, or MCP.', }, { title: 'Advisors', @@ -44,7 +53,7 @@ export const telemetryDetect: ContentListingGroup = { title: 'Detect issues', href: '/guides/observability/detecting', description: - 'Run health, security, performance, and usage checks against logs and database statistics to pick up a signal.', + 'Run health, security, performance, and capacity checks against logs and database statistics to pick up a signal.', }, ], } @@ -72,7 +81,7 @@ export const telemetryHireAgent: ContentListingGroup = { href: '/guides/observability/automate-with-agents/all', subtitle: getScheduleLabel(monitoringAgents.all), description: - 'Run all four checks — health, security, performance, and usage — in one daily pass.', + 'Run all four checks — health, security, performance, and capacity — in one daily pass.', }, { title: monitoringAgents.health.name, @@ -106,6 +115,11 @@ export const telemetryExport: ContentListingGroup = { type: 'grid', columns: 3, items: [ + { + title: 'Configure logging', + href: '/guides/observability/configure-logging', + description: 'Record additional Postgres and Realtime events.', + }, { title: 'Log drains', href: '/guides/observability/log-drains', diff --git a/apps/docs/next.config.mjs b/apps/docs/next.config.mjs index 1f522ef0604..d416bea5b85 100644 --- a/apps/docs/next.config.mjs +++ b/apps/docs/next.config.mjs @@ -132,6 +132,16 @@ const nextConfig = { */ async redirects() { return [ + { + source: '/guides/observability/access-data', + destination: '/guides/observability', + permanent: true, + }, + { + source: '/guides/observability/access-data.md', + destination: '/guides/observability.md', + permanent: true, + }, // Redirect root to docs base path in dev/preview envs { source: '/', diff --git a/apps/www/lib/redirects.js b/apps/www/lib/redirects.js index ce4e019f7d1..7cea0b4e1a9 100644 --- a/apps/www/lib/redirects.js +++ b/apps/www/lib/redirects.js @@ -185,6 +185,16 @@ module.exports = [ source: '/storage/Storage', destination: '/storage', }, + { + permanent: true, + source: '/docs/guides/observability/access-data', + destination: '/docs/guides/observability', + }, + { + permanent: true, + source: '/docs/guides/observability/access-data.md', + destination: '/docs/guides/observability.md', + }, { permanent: true, source: '/docs/guides/reports/:match*', diff --git a/packages/ui-patterns/src/CodeBlock/CodeBlock.tsx b/packages/ui-patterns/src/CodeBlock/CodeBlock.tsx index 0d40dd320b7..ed2b57644e5 100644 --- a/packages/ui-patterns/src/CodeBlock/CodeBlock.tsx +++ b/packages/ui-patterns/src/CodeBlock/CodeBlock.tsx @@ -277,6 +277,7 @@ export const CodeBlock = ({ className="px-1.5 dark:bg-200! dark:hover:bg-button! hover:bg-alternative!" icon={copied ? : } onClick={() => onSelectCopy(value || children)} + aria-label={copied ? 'Copied' : 'Copy'} > {copied ? 'Copied' : ''} diff --git a/packages/ui-patterns/src/McpUrlBuilder/components/McpConfigurationOptions.tsx b/packages/ui-patterns/src/McpUrlBuilder/components/McpConfigurationOptions.tsx index 3729a0c7dc0..4cfb476ebca 100644 --- a/packages/ui-patterns/src/McpUrlBuilder/components/McpConfigurationOptions.tsx +++ b/packages/ui-patterns/src/McpUrlBuilder/components/McpConfigurationOptions.tsx @@ -58,6 +58,7 @@ export function McpConfigurationOptions({