diff --git a/apps/docs/components/CustomHTMLElements/Heading.tsx b/apps/docs/components/CustomHTMLElements/Heading.tsx index 9d427e79b99..336dba4d0f7 100644 --- a/apps/docs/components/CustomHTMLElements/Heading.tsx +++ b/apps/docs/components/CustomHTMLElements/Heading.tsx @@ -26,7 +26,7 @@ interface Props { * * In tsx files, we can generate this tocList directly. For these files, we don't * need to parse the and generate anchors. Custom anchors are used in tsx files. - * (see: /pages/reference/cli/config.tsx) + * (see: /pages/guides/cli/config.tsx) */ const Heading: React.FC = ({ tag, customAnchor, children }) => { const HeadingTag = `${tag}` as any diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index f12d0ecd7f8..37803d4823b 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -568,6 +568,7 @@ export const database: NavMenuConstant = { { name: 'Implementing Cascade Deletes', url: '/guides/database/postgres/cascade-deletes' }, { name: 'Implementing column encryption', url: '/guides/database/column-encryption' }, { name: 'Partitioning your tables', url: '/guides/database/partitions' }, + { name: 'Query Optimization', url: '/guides/database/query-optimization' }, { name: 'Testing your database', url: '/guides/database/testing' }, { name: 'Managing Timeouts', url: '/guides/database/timeouts' }, { name: 'Managing Passwords', url: '/guides/database/managing-passwords' }, @@ -588,6 +589,10 @@ export const database: NavMenuConstant = { url: '/guides/database/extensions/plv8', }, { name: 'http: RESTful Client', url: '/guides/database/extensions/http' }, + { + name: 'index_advisor: Query optimization', + url: '/guides/database/extensions/index_advisor', + }, { name: 'PGAudit: Postgres Auditing', url: '/guides/database/extensions/pgaudit', @@ -1037,6 +1042,10 @@ export const supabase_cli: NavMenuConstant = { name: 'Testing and linting', url: '/guides/cli/testing-and-linting', }, + { + name: 'Customizing email templates', + url: '/guides/cli/customizing-email-templates', + }, ], }, { diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx index 452eb693116..bb5b6035dbd 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx @@ -95,7 +95,7 @@ const menus: Menu[] = [ }, { id: 'supabase_cli', - // TODO: Add path '/reference/cli/config' + // TODO: Add path '/guides/cli/config' path: '/guides/cli', type: 'guide', }, diff --git a/apps/docs/data/extensions.json b/apps/docs/data/extensions.json index 53c1c69b19e..96fb984b821 100644 --- a/apps/docs/data/extensions.json +++ b/apps/docs/data/extensions.json @@ -113,6 +113,12 @@ "tags": ["Admin", "Utility"], "link": "https://www.postgresql.org/docs/current/oldsnapshot.html" }, + { + "name": "index_advisor", + "comment": "optimize query performance with automatic index recommendation", + "tags": ["Utility"], + "link": "/guides/database/extensions/index_advisor" + }, { "name": "intarray", "comment": "functions, operators, and index support for 1-D arrays of integers", diff --git a/apps/docs/data/nav/supabase-cli.ts b/apps/docs/data/nav/supabase-cli.ts index 65a5bce9419..9eb1a91e66c 100644 --- a/apps/docs/data/nav/supabase-cli.ts +++ b/apps/docs/data/nav/supabase-cli.ts @@ -4,7 +4,7 @@ const Nav = [ items: [ { name: 'Supabase CLI', url: '/reference/cli', items: [] }, { name: 'Usage', url: '/reference/cli/usage', items: [] }, - { name: 'Configuration', url: '/reference/cli/config', items: [] }, + { name: 'Configuration', url: '/guides/cli/config', items: [] }, { name: 'Release Notes', url: '/reference/cli/release-notes', items: [] }, ], }, diff --git a/apps/docs/docs/ref/self-hosting-analytics/introduction.mdx b/apps/docs/docs/ref/self-hosting-analytics/introduction.mdx index 1158463ba48..b020dd6110b 100644 --- a/apps/docs/docs/ref/self-hosting-analytics/introduction.mdx +++ b/apps/docs/docs/ref/self-hosting-analytics/introduction.mdx @@ -20,92 +20,100 @@ When self-hosting the Analytics server, the full logging experience matching tha However, it's important to note that certain [differences](#differences) may arise due to the platform's infrastructure. - All logflare technical docs are available at - [https://docs.logflare.app](https://docs.logflare.app) + +All Logflare technical documentation is available at https://docs.logflare.app. + +## Backends Supported + +The Analytics server supports either **Postgres** or **BigQuery** as the backend. The `supabase-cli` experience uses the Postgres backend out-of-the-box. However, the Supabase Platform uses the BigQuery backend for storing all platform logs. + +When using the BigQuery backend, a BigQuery dataset is created in the provided Google Cloud project, and tables are created for each source. Log events are streamed into each table, and all queries generated by Studio or by the Logs Explorer are executed against the BigQuery API. This backend requires internet access to work, and cannot be run fully locally. + +When using the Postgres backend, tables are created for each source within the provided schema (for supabase-cli, this would be `_analytics`). Log events received by Logflare are inserted directly into the respective tables. All BigQuery-dialect SQL queries from Studio will be handled by a translation layer within the Analytics server. This translation layer translates the query to PostgreSQL dialect, and then executes it against the Postgres database. + +The Postgres backend is not yet optimized for a high volume of inserts, or for heavy query usage. Today the translation layer only handles a limited subset of the BigQuery dialect. As such, the [Log Explorer](https://supabase.com/docs/guides/platform/logs#logs-explorer) may produce errors for more advanced queries when using the Postgres Backend. + ## Getting Started -### Pre-requisites +The Postgres backend is recommended when familiarizing and experimenting with self-hosting Supabase. For production, we recommend using the BigQuery backend. See [production recommendations](#production-recommendations) for more information. -Currently the Analytics service supports two backends for ingestion: +To set up logging in self-hosted Supabase, see the [docker-compose example](https://github.com/supabase/supabase/tree/master/docker). +Two compose services are required: Logflare, and Vector. Logflare is the HTTP Analytics server, while Vector is the logging pipeline to route all compose services' syslog to the Logflare sever. -- Postgres ingestion -- Big Query ingestion +Regardless of the backend chosen, the following environment variables **must** be set for the `supabase/logflare` docker image: -### Postgres Ingestion Requirements +- `LOGFLARE_SINGLE_TENANT_MODE=true`: The feature flag for enabling single tenant mode for Logflare. Must be set to `true` +- `LOGFLARE_SUPABASE_MODE=true`: The feature flag for seeding Supabase-related data. Must be set to `true` -- Requires you to use supabase/logflare:1.3.26 or above -- Requires the following environment variables set: - - Required - POSTGRES_BACKEND_URL: `postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}` - - Optional - POSTGRES_BACKEND_SCHEMA: `{SCHEMA_NAME}` +For all other configuration environment variables, please refer to the [Logflare self-hosting documentation](https://docs.logflare.app/self-hosting/#configuration). -### Big Query Ingestion Requirements +## Postgres Backend Setup -- Google Cloud Account with billing enabled -- Big Query enabled - -The remainder of the instructions are specified in the [Setup Big Query ingestion](#setup-big-query-ingestion) section. - -#### Setup Big Query ingestion - -Logflare currently requires BigQuery usage. You will need to create a Google Cloud project with billing enabled. - -The requirements are as follows after creating the project: - -- Project ID -- Project number -- A service account key - -##### Setting up BigQuery Service Account - -To ensure that you have sufficient permissions to insert into your Google Cloud BigQuery, ensure that you have created a service account with either: - -- BigQuery Admin role; or -- The following permissions: - - bigquery.datasets.create - - bigquery.datasets.get - - bigquery.datasets.getIamPolicy - - bigquery.datasets.update - - bigquery.jobs.create - - bigquery.routines.create - - bigquery.routines.update - - bigquery.tables.create - - bigquery.tables.delete - - bigquery.tables.get - - bigquery.tables.getData - - bigquery.tables.update - - bigquery.tables.updateData - -You can create the service account via the web console or `gcloud`, as per the [Google Cloud documentation](https://cloud.google.com/iam/docs/keys-create-delete). In the web console, you can create the key by navigating to IAM > Service Accounts > Actions (dropdown) > Manage Keys - -We recommend setting the BigQuery Admin role, as it simplifies permissions setup. - -##### Downloading the Service Account key - -After the service account is created, you will need to create a key for the service account. This key will sign the JWTs for API requests that the Analytics server makes with BigQuery. - -### Docker Compose - -For analytics to work we're adding two services: Logflare and Vector. - -Logflare is the analytics server that will ingest your logs into your preferred backend while Vector is the log router that will route all the docker logs using syslog unto Logflare. - -#### Using the Postgres Backend - -You can simply startup using the following command +The [example docker-compose](https://github.com/supabase/supabase/tree/master/docker) uses the Postgres backend out of the box. ```bash -# assuming you clone the supabase/supabase repo. +# clone the supabase/supabase repo, and run the following cd docker docker compose -f docker-compose.yml up ``` -> ⚠️ WARNING: Please be aware of the heavy consumption of analytics. For heavy usage you might want to consider the usage of a different database for analytics only by updating the `POSTGRES_BACKEND_URL` in the `analytics` service on the `docker-compose.yml` file. +### Configuration and Requirements -Also be aware that you won't be able to use [Log Explorer](https://supabase.com/docs/guides/platform/logs#logs-explorer) when using the Postgres Backend as some of the querying features are still not supported by our translation layer. +- `supabase/logflare:1.4.0` or above +- Relevant environment variables: + - `POSTGRES_BACKEND_URL` : Required. The connection string to the Postgres database. + - `POSTGRES_BACKEND_SCHEMA` : Optional. Allows customization of the schema used to scope all backend operations within the database. -#### Using the Big Query Backend +## BigQuery Backend Setup + +The BigQuery backend is a more robust and scalable backend option that is battle-tested and production ready. Use this backend if you intend to have heavy logging usage and require advanced querying features such as the Logs Explorer. + +### Configuration and Requirements + +The requirements are as follows after creating the project: + +- Google Cloud project with billing enabled +- Project ID +- Project number +- A service account key. + + + +You must enable billing on your Google Cloud project, as a valid billing account is required for streaming inserts. + + + +#### Setting up BigQuery Service Account + +The service account used must have sufficient permissions to insert into your Google Cloud BigQuery. Ensure that the service account has either: + +- BigQuery Admin role; or +- The following permissions: + - `bigquery.datasets.create` + - `bigquery.datasets.get` + - `bigquery.datasets.getIamPolicy` + - `bigquery.datasets.update` + - `bigquery.jobs.create` + - `bigquery.routines.create` + - `bigquery.routines.update` + - `bigquery.tables.create` + - `bigquery.tables.delete` + - `bigquery.tables.get` + - `bigquery.tables.getData` + - `bigquery.tables.update` + - `bigquery.tables.updateData` + +You can create the service account via the web console or `gcloud` CLI, as per the [Google Cloud documentation](https://cloud.google.com/iam/docs/keys-create-delete). In the web console, you can create the key by navigating to IAM > Service Accounts > Actions (dropdown) > Manage Keys + +We recommend setting the BigQuery Admin role, as it simplifies permissions setup. + +#### Download the Service Account Keys + +After the service account is created, you will need to create a key for the service account. This key will sign the JWTs for API requests that the Analytics server makes with BigQuery. This can be done through the IAM section in Google Cloud console. + +#### Docker Image Configuration Using the example [self-hosting stack based on docker-compose](https://github.com/supabase/supabase/tree/master/docker), you include the logging related services using the following command @@ -118,7 +126,7 @@ Using the example [self-hosting stack based on docker-compose](https://github.co 3. On `docker-compose.yml`, uncomment the block section below the commentary `# Uncomment to use Big Query backend for analytics` 4. On `docker-compose.yml`, comment the block section below the commentary `# Comment variables to use Big Query backend for analytics` -Thereafter, you can simply startup using the following command +Thereafter, you can start the example stack using the following command: ```bash # assuming you clone the supabase/supabase repo. @@ -126,55 +134,25 @@ cd docker docker compose -f docker-compose.yml ``` - - Read more about [self-hosting Logflare](https://docs.logflare.app/self-hosting) as your analytics - server. - +#### BigQuery Datset Storage Location -#### Vector Usage +Currently, all BigQuery datasets stored and managed by Analytics, whether via CLI or self-hosted, will default to the US region. -In the Docker Compose example, we utilize vector to coordinate the logging pipeline between the services. However, if you peer into the [vector configuration file](https://github.com/supabase/supabase/blob/master/docker/volumes/logs/vector.yml), you will be able to see that Vector sends logs to the Analytics ingestion endpoint. +## Vector Usage -If you need to customize the logging pipeline for your own needs, you must ensure that the payloads matches the expected event schema structure. Without the correct structure, it would cause the Studio Logs UI features to break. +In the Docker Compose example, Vector is used for the logging pipieline, where log events are forwarded to the Analytics API for ingestion. -### Standalone Docker Container +Please refer to the [Vector configuration file](https://github.com/supabase/supabase/blob/master/docker/volumes/logs/vector.yml) when customizing your own setup. -Supplementary technical documentation on self-hosting and using the `supabase/logflare` image for a full Logflare experience is available at the [official Logflare documentation](https://docs.logflare.app/self-hosting/). +You **must** ensure that the payloads matches the expected event schema structure. Without the correct structure, it would cause the Studio Logs UI features to break. -## Differences +## Differences from Platform API logs rely on Kong instead of the Supabase Cloud API Gateway. Logs from Kong are not enriched with platform-only data. Within the self-hosted setup, all logs are routed to Logflare via Vector. As Kong routes API requests to PostgREST, self-hosted or local deployments will result in Kong request logs instead. This would result in differences in the log event metadata between self-hosted API requests and Supabase Platform requests. -## Postgres - -Log event data is stored and queried from a Postgres database. - -Make sure to set the `POSTGRES_BACKEND_URL` environment variable. - -### Storage - -All data will be stored on the Postgres Database configured by setting `POSTGRES_BACKEND_URL` - -## BigQuery - -All log event data is stored in and queried from BigQuery. To use the Analytics server with Supabase you'll need a Google Cloud Platform account for access to BigQuery. - -Make sure to set the `GOOGLE_DATASET_ID_APPEND`, `GOOGLE_PROJECT_ID` and `GOOGLE_PROJECT_NUMBER` environment variables. - -Download your Google Cloud API JWT and store it under gcloud.json in your working directory. - - - You must also enable billing on your Google Cloud project, as the streaming inserts feature is - required. - - -### Storage Location - -Currently, all BigQuery datasets stored and managed by Analytics, whether via CLI or self-hosted, will default to the US region. - ## Production Recommendations To self-host in a production setting, we recommend performing the following for a better experience. @@ -194,6 +172,8 @@ The self-hosted example is only used as a minimal example on running the entire The current Postgres Ingestion backend isn't optimized for production usage. We recommend using Big Query for more heavy use cases. +**We recommend using the BigQuery backend for production environments as it offers better scaling and querying/debugging experiences.** + diff --git a/apps/docs/pages/guides/auth/social-login/auth-facebook.mdx b/apps/docs/pages/guides/auth/social-login/auth-facebook.mdx index f46284137b6..36db6d0bbee 100644 --- a/apps/docs/pages/guides/auth/social-login/auth-facebook.mdx +++ b/apps/docs/pages/guides/auth/social-login/auth-facebook.mdx @@ -43,12 +43,13 @@ From the `Add Products to your App` screen: - Enter this in the `Valid OAuth Redirect URIs` box - Click `Save Changes` at the bottom right -Be aware that you have to set the right access levels on your Facebook App to enable 3rd party applications to read the email address. -From the `App Review -> Permissions and Features` screen: +Be aware that you have to set the right use case permissions to enable Third party applications to read the email address. To do so: -- Click the button `Request Advanced Access` on the right side of `public_profile` and `email` +Under `Build Your App`, click on `Use Cases` screen. From there, do the following steps: -You can read more about access levels [here](https://developers.facebook.com/docs/graph-api/overview/access-levels/) +- Click the Edit button in `Authentication and Account Creation` on the right side. This action will lead to the other page. +- `public_profile` is set by default, so make sure it and `email` have status of **Ready for testing** in the redirected page. +- If not, click the **Add** button in email on right side. ## Copy your Facebook App ID and Secret @@ -81,6 +82,9 @@ async function signout() { } ``` +Now, you should be able to login with Facebook and alert you to `Submit for Login Review` when users try to sign into your app. Follow the instructions there to make your app go live for full features and products. +You can read more about App Review [here](https://developers.facebook.com/docs/app-review/). + ## Resources - [Supabase Account - Free Plan OK](https://supabase.com) diff --git a/apps/docs/pages/guides/cli/customizing-email-templates.mdx b/apps/docs/pages/guides/cli/customizing-email-templates.mdx new file mode 100644 index 00000000000..4702b09ad72 --- /dev/null +++ b/apps/docs/pages/guides/cli/customizing-email-templates.mdx @@ -0,0 +1,124 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'customizing-email-templates', + title: 'Customizing email templates', + description: 'Customizing local email templates using config.toml.', + subtitle: 'Customizing local email templates using config.toml.', +} + +You can customize the email templates for local development [using the `config.toml` settings](/docs/guides/cli/config#auth-config). + +## Configuring Templates + +You should provide a relative URL to the `content_path` parameter, pointing to an HTML file which contains the template. For example + + + +```toml supabase/config.toml +[auth.email.template.invite] +subject = "You are invited to Acme Inc" +content_path = "./supabase/templates/invite.html" +``` + +```html supabase/templates/invite.html + + +

Confirm your signup

+

Confirm your email

+ + +``` + + + +## Available email templates + +There are several Auth email templates which can be configured: + +- `auth.email.template.invite` +- `auth.email.template.confirmation` +- `auth.email.template.recovery` +- `auth.email.template.magic_link` +- `auth.email.template.email_change` + +## Template variables + +The templating system provides the following variables for use: + +### ConfirmationURL + +Contains the confirmation URL. For example, a signup confirmation URL would look like: + +``` +https://project-ref.supabase.co/auth/v1/verify?token={{ .TokenHash }}&type=signup&redirect_to=https://example.com/path +``` + +**Usage** + +```html +

Click here to confirm: {{ .ConfirmationURL }}

+``` + +### Token + +Contains a 6-digit One-Time-Password (OTP) that can be used instead of the `ConfirmationURL`. + +**Usage** + +```html +

Here is your one time password: {{ .Token }}

+``` + +### TokenHash + +Contains a hashed version of the `Token`. This is useful for constructing your own email link in the email template. + +**Usage** + +```html +

Follow this link to confirm your user:

+

+ Confirm your email +

+``` + +### SiteURL + +Contains your application's Site URL. This can be configured in your project's [authentication settings](/dashboard/project/_/auth/url-configuration). + +**Usage** + +```html +

Visit here to log in.

+``` + +### Email + +Contains the user's email address. + +**Usage** + +```html +

A recovery request was sent to {{ .Email }}.

+``` + +### NewEmail + +Contains the new user's email address. This is only available in the `email_change` email template. + +**Usage** + +```html +

You are requesting to update your email address to {{ .NewEmail }}.

+``` + +## Deploying email templates + +These settings are only for local development. To update your hosted project, please copy the templates into the [Email Templates](/dashboard/project/_/auth/templates) section of the Dashboard. + +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/pages/guides/cli/local-development.mdx b/apps/docs/pages/guides/cli/local-development.mdx index 57b4e5abf52..8c719d931be 100644 --- a/apps/docs/pages/guides/cli/local-development.mdx +++ b/apps/docs/pages/guides/cli/local-development.mdx @@ -364,12 +364,12 @@ Associate your project with your remote project using [`supabase link`](/docs/re supabase link --project-ref # You can get from your project's dashboard URL: https://supabase.com/dashboard/project/ -supabase db remote commit +supabase db pull # Capture any changes that you have made to your remote database before you went through the steps above # If you have not made any changes to the remote database, skip this step ``` -`supabase/migrations` is now populated with a migration in `..._remote_commit.sql`. +`supabase/migrations` is now populated with a migration in `_remote_schema.sql`. This migration captures any changes required for your local database to match the schema of your remote Supabase project. @@ -420,7 +420,7 @@ For these changes to take effect, you need to run `supabase stop` and `supabase If you have additional triggers or RLS policies defined on your `auth` schema, you can pull them as a migration file locally. ```bash -supabase db remote commit --schema auth +supabase db pull --schema auth ``` ### Local Logging diff --git a/apps/docs/pages/guides/cli/managing-config.mdx b/apps/docs/pages/guides/cli/managing-config.mdx index 1f93c9e52cf..d63eeafd81c 100644 --- a/apps/docs/pages/guides/cli/managing-config.mdx +++ b/apps/docs/pages/guides/cli/managing-config.mdx @@ -12,7 +12,7 @@ The Supabase CLI uses a `config.toml` file to manage local configuration. This f The `config.toml` file is automatically created when you run `supabase start`. -There are a wide variety of options available, which can be found in the [CLI Config Reference](/docs/reference/cli/config). +There are a wide variety of options available, which can be found in the [CLI Config Reference](/docs/guides/cli/config). For example, to enable the "Apple" OAuth provider for local development, you can append the following information to `config.toml`: diff --git a/apps/docs/pages/guides/cli/managing-environments.mdx b/apps/docs/pages/guides/cli/managing-environments.mdx index b20a4c5c5f1..59bd4ed51fe 100644 --- a/apps/docs/pages/guides/cli/managing-environments.mdx +++ b/apps/docs/pages/guides/cli/managing-environments.mdx @@ -38,10 +38,10 @@ If you're using an existing Supabase project, you might have made schema changes Run the following command to pull these changes before making local schema changes from the CLI: ```sql -supabase db remote commit +supabase db pull ``` -This command creates a new migration in `supabase/migrations/_remote_commit.sql` which reflects the schema changes you have made previously. +This command creates a new migration in `supabase/migrations/_remote_schema.sql` which reflects the schema changes you have made previously. Now commit your local changes to Git and run the local development setup: @@ -348,9 +348,9 @@ Once pushed, check that the migration version is up to date for both local and r supabase migration list ``` -### Permission denied on db remote commit +### Permission denied on db pull -If you have been using Supabase hosted projects for a long time, you might encounter the following permission error when executing `db remote commit`. +If you have been using Supabase hosted projects for a long time, you might encounter the following permission error when executing `db pull`. ```bash Error: Error running pg_dump on remote database: pg_dump: error: query failed: ERROR: permission denied for table _type diff --git a/apps/docs/pages/guides/database/extensions/index_advisor.mdx b/apps/docs/pages/guides/database/extensions/index_advisor.mdx new file mode 100644 index 00000000000..d155c36a49b --- /dev/null +++ b/apps/docs/pages/guides/database/extensions/index_advisor.mdx @@ -0,0 +1,161 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'index-advisor', + title: 'index_advisor: query optimization', + description: 'Automatically optimize SQL queries', +} + +[Index advisor](https://github.com/supabase/index_advisor) is a Postgres extension for recommending indexes to improve query performance. + +For example: + +```sql +select + * +from + index_advisor('select book.id from book where title = $1'); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +Features: + +- Supports generic parameters e.g. `$1`, `$2` +- Supports materialized views +- Identifies tables/columns obfuscaed by views +- Skips duplicate indexes + +## Installation + +index_advisor is a trusted language extension, which means it is directly installable by users from the [database.dev](database.dev) SQL package repository. + +To get started, enable the dbdev client by executing the [setup SQL script](https://database.dev/installer). + +Then, install index_advisor by running + +```sql +select dbdev.install('olirice-index_advisor'); +create extension if not exists hypopg; +create extension "olirice-index_advisor"; +``` + +## API + +Index advisor exposes a single function `index_advisor(query text)` that accepts a query and searches for a set of SQL DDL `create index` statements that improve the query's execution time. + +The function's signature is: + +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) +``` + +## Usage + +As a minimal example, the `index_advisor` function can be given a single table query with a filter on an unindexed column. + +```sql +create extension if not exists index_advisor cascade; + +create table book( + id int primary key, + title text not null +); + +select + * +from + index_advisor('select book.id from book where title = $1'); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +and will return a row recommending an index on the unindexed column. + +More complex queries may generate additional suggested indexes: + +```sql +create extension if not exists index_advisor cascade; + +create table author( + id serial primary key, + name text not null +); + +create table publisher( + id serial primary key, + name text not null, + corporate_address text +); + +create table book( + id serial primary key, + author_id int not null references author(id), + publisher_id int not null references publisher(id), + title text +); + +create table review( + id serial primary key, + book_id int references book(id), + body text not null +); + +select + * +from + index_advisor(' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + '); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------------+-------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", | {} + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(3 rows) +``` + +## Limitations + +- index_advisor will only recommend single column, B-tree indexes. More complex indexes will be supported in future releases. +- when a generic argument's type is not discernible from context, an error is returned in the `errors` field. To resolve those errors, add explicit type casting to the argument. e.g. `$1::int`. + +## Resources + +- Official [`index_advisor` GitHub Repository](https://github.com/supabase/index_advisor) + +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/pages/guides/database/extensions/postgis.mdx b/apps/docs/pages/guides/database/extensions/postgis.mdx index da0562db7af..97c8f8438ff 100644 --- a/apps/docs/pages/guides/database/extensions/postgis.mdx +++ b/apps/docs/pages/guides/database/extensions/postgis.mdx @@ -157,7 +157,7 @@ Sorting datasets from closest to farthest, sometimes called nearest-neighbor sor ```sql create or replace function nearby_restaurants(lat float, long float) -returns setof record +returns table (id public.restaurants.id%TYPE, name public.restaurants.name%TYPE, location text, dist_meters float) language sql as $$ select id, name, st_astext(location) as location, st_distance(location, st_point(long, lat)::geography) as dist_meters @@ -230,7 +230,7 @@ When you are working on a map-based application where the user scrolls through y ```sql create or replace function restaurants_in_view(min_lat float, min_long float, max_lat float, max_long float) -returns setof record +returns table (id public.restaurants.id%TYPE, name public.restaurants.name%TYPE, location text) language sql as $$ select id, name, st_astext(location) as location diff --git a/apps/docs/pages/guides/database/query-optimization.mdx b/apps/docs/pages/guides/database/query-optimization.mdx new file mode 100644 index 00000000000..3176c49adaf --- /dev/null +++ b/apps/docs/pages/guides/database/query-optimization.mdx @@ -0,0 +1,134 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + title: 'Query Optimization', + description: 'Choosing indexes', + footerHelpType: 'postgres', +} + +When working with PostgreSQL, or any relational database, indexing is key to improving query performance. +Aligning indexes with common query patterns can speed up data retrieval by an order of magnitude. + +This guide is intended to: + +- help identify parts of a query that have the potential to be improved by indexes +- introduce tooling to help identify useful indexes + +This is not a comprehensive resource, but rather a helpful starting point for your optimization journey. + +If you're new to query optimization, you may be interested in [index_advisor](/docs/guides/database/extensions/index_advisor), our tool for automatically detecting indexes that improve performance on a given query. + +## Example Query + +Consider the following example query that retrieves customer names and purchase dates from two tables: + +```sql +select + a.name, + b.date_of_purchase +from + customers a + join orders b + on a.id = b.customer_id +where + a.sign_up_date > '2023-01-01' + and b.status = 'shipped' +order by + b.date_of_purchase; +limit 10 +``` + +In this query, there are several parts that indexes could likely help in optimizing the performance: + +### `where` Clause: + +The `where` clause filters rows based on certain conditions, and indexing the columns involved can improve this process: + +- `a.sign_up_date`: If filtering by `sign_up_date` is common, indexing this column can speed up the query. +- `b.status`: Indexing the status may be beneficial if the column has diverse values. + +```sql +create index idx_customers_sign_up_date on customers (sign_up_date); + +create index idx_orders_status on orders (status); +``` + +### `join` Columns + +Indexes on the columns used for joining tables can help Postgres avoid scanning tables in their entirety when connecting tables. + +- Indexing and `b.customer_id` would likely improve the performance of the join in this query. +- Note that if `a.id` is the primary key of the `customers` table it is already indexed + +```sql +create index idx_orders_customer_id on orders (customer_id); +``` + +### `order by` Clause + +Sorting can also be optimized by indexing: + +- An index on `b.date_of_purchase` wcan improve the sorting process, and is particularly beneficial when a subset of rows is being returned with a `limit` clause. + +```sql +create index idx_orders_date_of_purchase on orders (date_of_purchase); +``` + +## Key Concepts + +Here are some concepts and tools to keep in mind to help you identify the best index for the job, and measure the impact that your index had: + +### Analyze the Query Plan + +Use the `explain` command to understand the query's execution. Look for slow parts, such as Sequential Scans or high cost numbers. If creating an index does not reduce the cost of the query plan, remove it. + +For example: + +```sql +explain select * from customers where sign_up_date > 25; +``` + +### Use Appropriate Index Types + +PostgreSQL offers various index types like [B-tree, Hash, GIN, etc](https://www.postgresql.org/docs/current/indexes-types.html). Select the type that best suits your data and query pattern. Using the right index type can make a significant difference. For example, using a BRIN index on a field that always increases and lives within a table that updates infrequently - like `created_at` on an `orders` table - routinely results in indexes that are +10x smaller than the equivalent default B-tree index. That translates into better scalability. + +```sql +create index idx_orders_created_at ON customers using brin(created_at); +``` + +### Partial Indexes + +For queries that frequently target a subset of data, a partial index could be faster and smaller than indexing the entire column. A partial index contains a `where` clause to filter the values included in the index. Note that a query's `where` clause must match the index for it to be used. + +```sql +create index idx_orders_status on orders (status) +where status = 'shipped'; +``` + +### Composite Indexes + +If filtering or joining on multiple columns, a composite index prevents Postgres from referring to multiple indexes when identifying the relevant rows. + +```sql +create index idx_customers_sign_up_date_priority on customers (sign_up_date, priority); +``` + +### Over-Indexing + +Avoid the urge to index columns you operate on infrequently. While indexes can speed up reads, they also slow down writes, so it's important to balance those factors when making indexing decisions. + +### Statistics + +Postgres maintains a set of statistics about the contents of your tables. Those statistics are used by the query planner to decide when it's is more efficient to use an index vs scanning the entire table. If the collected statistics drift too far from reality, the query planner may make poor decisions. To avoid this risk, you can periodically `analyze` tables. + +```sql +analyze customers; +``` + +--- + +By following this guide, you'll be able to discern where indexes can optimize queries and enhance your PostgreSQL performance. Remember that each database is unique, so always consider the specific context and use case of your queries. + +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/pages/guides/functions/cicd-workflow.mdx b/apps/docs/pages/guides/functions/cicd-workflow.mdx index 4827bde8473..b729044c17e 100644 --- a/apps/docs/pages/guides/functions/cicd-workflow.mdx +++ b/apps/docs/pages/guides/functions/cicd-workflow.mdx @@ -38,7 +38,7 @@ jobs: Since Supabase CLI [v1.62.0](https://github.com/supabase/cli/releases/tag/v1.62.0) you can deploy all functions with a single command. -Individual function configuration like [JWT verification](/docs/reference/cli/config#functions.function_name.verify_jwt) and [import map location](/docs/reference/cli/config#functions.function_name.import_map) can be set via the `config.toml` file. +Individual function configuration like [JWT verification](/docs/guides/cli/config#functions.function_name.verify_jwt) and [import map location](/docs/guides/cli/config#functions.function_name.import_map) can be set via the `config.toml` file. ```toml [functions.hello-world] diff --git a/apps/docs/pages/guides/functions/examples/github-actions.mdx b/apps/docs/pages/guides/functions/examples/github-actions.mdx index 7baf600b40d..d29fd6f1981 100644 --- a/apps/docs/pages/guides/functions/examples/github-actions.mdx +++ b/apps/docs/pages/guides/functions/examples/github-actions.mdx @@ -47,7 +47,7 @@ jobs: Since Supabase CLI [v1.62.0](https://github.com/supabase/cli/releases/tag/v1.62.0) you can deploy all functions with a single command. -Individual function configuration like [JWT verification](/docs/reference/cli/config#functions.function_name.verify_jwt) and [import map location](/docs/reference/cli/config#functions.function_name.import_map) can be set via the `config.toml` file. +Individual function configuration like [JWT verification](/docs/guides/cli/config#functions.function_name.verify_jwt) and [import map location](/docs/guides/cli/config#functions.function_name.import_map) can be set via the `config.toml` file. ```toml [functions.hello-world] diff --git a/apps/docs/pages/guides/functions/quickstart.mdx b/apps/docs/pages/guides/functions/quickstart.mdx index 14de4f79620..7c32c36db06 100644 --- a/apps/docs/pages/guides/functions/quickstart.mdx +++ b/apps/docs/pages/guides/functions/quickstart.mdx @@ -66,7 +66,7 @@ supabase functions deploy Since Supabase CLI [v1.62.0](https://github.com/supabase/cli/releases/tag/v1.62.0) you can deploy all functions with a single command. This is useful for example when [deploying with GitHub Actions](/docs/guides/functions/cicd-workflow). -Individual function configuration like [JWT verification](/docs/reference/cli/config#functions.function_name.verify_jwt) and [import map location](/docs/reference/cli/config#functions.function_name.import_map) can be set via the `config.toml` file. +Individual function configuration like [JWT verification](/docs/guides/cli/config#functions.function_name.verify_jwt) and [import map location](/docs/guides/cli/config#functions.function_name.import_map) can be set via the `config.toml` file. ```toml [functions.hello-world] diff --git a/apps/docs/pages/guides/getting-started/quickstarts/nextjs.mdx b/apps/docs/pages/guides/getting-started/quickstarts/nextjs.mdx index b586d9ff914..24fde38da52 100644 --- a/apps/docs/pages/guides/getting-started/quickstarts/nextjs.mdx +++ b/apps/docs/pages/guides/getting-started/quickstarts/nextjs.mdx @@ -100,7 +100,7 @@ export const meta = { const { data: countries } = await supabase.from("countries").select(); return ( -