From 9df7008228fa2c67ab161e446589db85991ccd19 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Tue, 4 Jun 2024 17:53:18 +0300 Subject: [PATCH] created new section for replication --- .../NavigationMenu.constants.ts | 14 +++++ apps/docs/components/index.tsx | 2 + .../replication-analytics/etl-airbyte.mdx | 0 .../replication-analytics/etl-estuary.mdx | 0 .../replication-analytics/etl-fivetran.mdx | 0 .../replication-analytics/etl-materialize.mdx | 0 .../replication-analytics/etl-stitch.mdx | 0 .../replication-analytics/overview.mdx | 53 ------------------- .../database/replication/considerations.mdx | 23 ++++++++ .../replication/logical-replication-intro.mdx | 42 +++++++++++++++ .../guides/database/replication/overview.mdx | 37 +++++++++++++ .../replication/services/etl-airbyte.mdx | 6 +++ .../replication/services/etl-estuary.mdx | 6 +++ .../replication/services/etl-fivetran.mdx | 6 +++ .../replication/services/etl-materialize.mdx | 6 +++ .../replication/services/etl-stitch.mdx | 6 +++ .../content/guides/platform/read-replicas.mdx | 2 +- 17 files changed, 149 insertions(+), 54 deletions(-) delete mode 100644 apps/docs/content/guides/database/replication-analytics/etl-airbyte.mdx delete mode 100644 apps/docs/content/guides/database/replication-analytics/etl-estuary.mdx delete mode 100644 apps/docs/content/guides/database/replication-analytics/etl-fivetran.mdx delete mode 100644 apps/docs/content/guides/database/replication-analytics/etl-materialize.mdx delete mode 100644 apps/docs/content/guides/database/replication-analytics/etl-stitch.mdx delete mode 100644 apps/docs/content/guides/database/replication-analytics/overview.mdx create mode 100644 apps/docs/content/guides/database/replication/considerations.mdx create mode 100644 apps/docs/content/guides/database/replication/logical-replication-intro.mdx create mode 100644 apps/docs/content/guides/database/replication/overview.mdx create mode 100644 apps/docs/content/guides/database/replication/services/etl-airbyte.mdx create mode 100644 apps/docs/content/guides/database/replication/services/etl-estuary.mdx create mode 100644 apps/docs/content/guides/database/replication/services/etl-fivetran.mdx create mode 100644 apps/docs/content/guides/database/replication/services/etl-materialize.mdx create mode 100644 apps/docs/content/guides/database/replication/services/etl-stitch.mdx diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index 63f6ba37fe8..db286bd1fe2 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -811,6 +811,20 @@ export const database: NavMenuConstant = { { name: 'Testing your database', url: '/guides/database/testing' }, ], }, + { + name: 'Replication', + url: '/guides/database/replication-analytics/overview', + items: [ + { name: 'Overview', url: '/guides/database/replication/overview' }, + { name: 'Considerations', url: '/guides/database/replication/considerations' }, + { name: 'Logical Replication', url: '/guides/database/replication/logical-replication-intro' }, + { name: 'Replication with Airbyte', url: '/guides/database/replication/services/etl-airbyte' }, + { name: 'Replication with Estuary', url: '/guides/database/replication/services/etl-estuary' }, + { name: 'Replication with Fivetran', url: '/guides/database/replication/services/etl-fivetran' }, + { name: 'Replication with Materialize', url: '/guides/database/replication/services/etl-materialize' }, + { name: 'Replication with Stitch', url: '/guides/database/replication/services/etl-stitch' }, + ], + }, { name: 'Extensions', url: undefined, diff --git a/apps/docs/components/index.tsx b/apps/docs/components/index.tsx index 76ed777d98d..caecf9cc47e 100644 --- a/apps/docs/components/index.tsx +++ b/apps/docs/components/index.tsx @@ -79,6 +79,7 @@ import { Extensions } from './Extensions' import { JwtGenerator } from './JwtGenerator' import { Mermaid } from './Mermaid' import { RealtimeLimitsEstimator } from './RealtimeLimitsEstimator' +import { DatabaseReplicationConfig } from './DatabaseReplicationConfig' const components = { ...markdownComponents, @@ -93,6 +94,7 @@ const components = { AuthProviders, AuthRateLimits, AuthSmsProviderConfig, + DatabaseReplicationConfig, Button, ButtonCard, CH, diff --git a/apps/docs/content/guides/database/replication-analytics/etl-airbyte.mdx b/apps/docs/content/guides/database/replication-analytics/etl-airbyte.mdx deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/docs/content/guides/database/replication-analytics/etl-estuary.mdx b/apps/docs/content/guides/database/replication-analytics/etl-estuary.mdx deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/docs/content/guides/database/replication-analytics/etl-fivetran.mdx b/apps/docs/content/guides/database/replication-analytics/etl-fivetran.mdx deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/docs/content/guides/database/replication-analytics/etl-materialize.mdx b/apps/docs/content/guides/database/replication-analytics/etl-materialize.mdx deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/docs/content/guides/database/replication-analytics/etl-stitch.mdx b/apps/docs/content/guides/database/replication-analytics/etl-stitch.mdx deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/docs/content/guides/database/replication-analytics/overview.mdx b/apps/docs/content/guides/database/replication-analytics/overview.mdx deleted file mode 100644 index 414d350705e..00000000000 --- a/apps/docs/content/guides/database/replication-analytics/overview.mdx +++ /dev/null @@ -1,53 +0,0 @@ ---- -id: 'etl' -title: 'ETL with Supabase' -description: 'Performing Extract Transform Load (ETL) with Supabase' -sidebar_label: 'Overview' ---- - -# Replication, Analytics and ETL - -You have launched your app. You want to understand what works and what doesn't so you start some queries and soon notice that your queries are taking minutes or that you need to combine these data with another system you are using. Not to worry, you are in the right place. - - -## Why not use my database? - -If you want to run analytics on your app, you may wonder why you could not just plug in your database connection string to an analytics tool and go. In some cases, you could, but we do not recommend it. Here's why: - -### Security - -Adding a new system to your database can expose data and settings that you may want to keep private (or may be required to keep private due to compliance or legislation). Using a read-only user is possible but you must ensure that access is controlled and regularly audited. - -### Performance - -Your users want to create or view new content and your analytics queries want to perform complex calculations on the past 3 months. Both of these wants will be sharing the same resources and one might win over the other; causing your users to have a bad experience or your analytics report to time out. - -### Recovery - -If you accidentally drop a table, you might not want to restore your whole project from a backup so you can check these values in your external system before your boss finds out. - - - -## How? - -OK, so now we know why you should not use your primary database for this, what should you use? This is not an easy question and it comes with the "it depends" answer so let's go through some options: - -### Read Replicas - -Read-Replicas are available on Supabase and allow you to create a second database that is a direct, read-only copy of your primary database that stays in sync. This can be great for many things, like load-balancing, and, in this case, they can be great for analytics workloads. - - -### External Systems - -Your use case might go beyond what a Read-Replica can offer and you may need to move your data out of Supabase and into a Data Warehouse, like Clickhouse or Google BigQuery. This could be because of long running jobs, the amount of data or because you need to combine your Supabase data with data from other systems. - -When you want to do this, this is where Extract Transform Load (ETL) comes in. - -1. Extract - Getting the data out of your source, typically using a method called [Logical Replication](https://www.postgresql.org/docs/current/logical-replication.html) -2. Transform - Modifying the data for ingestion -3. Load - Loading the transformed data into the destination - -To perform each of these steps, some destinations will pull data directly from Postgresql but, in most cases, you will want to use a tool, such as Estuary or Airbyte or Fivetran. These perform the `Extract` and `Transform` in much the same way, as well as support a large number of destinations to `Load` your data into, ranging from S3 to Salesforce. - -To get started, choose the tool you want to use below: - diff --git a/apps/docs/content/guides/database/replication/considerations.mdx b/apps/docs/content/guides/database/replication/considerations.mdx new file mode 100644 index 00000000000..4340fbff233 --- /dev/null +++ b/apps/docs/content/guides/database/replication/considerations.mdx @@ -0,0 +1,23 @@ +--- +id: 'replication-considerations' +title: 'Replication and Analytics with Supabase' +description: 'Considerations and FAQs when setting up replication' +sidebar_label: 'Considerations' +--- + +## Why not use my Primary database? + +If you want to run analytics on your app, you may wonder why you could not just plug in your database connection string to an analytics tool and go. In some cases, you could, but we do not recommend it. Here's why: + +### Security + +Adding a new system to your database can expose data and settings that you may want to keep private (or may be required to keep private due to compliance or legislation). Using a read-only user is possible but you must ensure that access is controlled and regularly audited. + +### Performance + +Your users want to create or view new content and your analytics queries want to perform complex calculations on the past 3 months. Both of these wants will be sharing the same resources and one might win over the other; causing your users to have a bad experience or your analytics report to time out. + +### Recovery + +If you accidentally drop a table, you might not want to restore your whole project from a backup so you can check these values in your external system before your boss finds out. + diff --git a/apps/docs/content/guides/database/replication/logical-replication-intro.mdx b/apps/docs/content/guides/database/replication/logical-replication-intro.mdx new file mode 100644 index 00000000000..bf9fe01f54b --- /dev/null +++ b/apps/docs/content/guides/database/replication/logical-replication-intro.mdx @@ -0,0 +1,42 @@ +--- +id: 'logical-replication' +title: 'Logical Replication: An Introduction' +description: 'Covering the concepts, terms and config of Logical Replication' +sidebar_label: 'Logical Replication' +--- + +# Concepts and Terms + +## Write Ahead Log (WAL) + +You may have heard this term, seen it in error messages or configuration files. Postgresql uses a system called the Write Ahead Log to managae changes to the database. As you make changes, they are appended to the WAL (which is a series of files (also called "segments"), where the file size can be specified). Once one segment is full, Postgresql will start appending to a new segment. After a period of time, a checkpoint occurs and Postgresql synchronizes the WAL with your database. Once the checkpoint is complete, then the WAL files can be removed from disk and free up space. + +## Logical Replication and WAL + +Logical replication is a method of replication where Postgresql can reuse the WAL files and transmit those changes to another Postgres database, or a system that supports reading WAL files. + +## Logical Replication Architecture + +When using logical replication, you have a few active components: + +- `publication` - A set of tables that you want to replicate +- `replication slot` - A slot used for replicating the data from a single publication. The slot, when created, will specify the output format of the changes +- `subscription` - A subscription is made from an external system (i.e. another Postgresql database) and must specify the name of the `publication`. If you do not specify a replication slot, one is automatically created for you + +#### Logical Replication Output Format + +Logical replication is typically output in 2 forms, `pgoutput` and `wal2json`. The output method is how Postgresql sends changes to any active replication slot. + +# Logical Replication Configuration + +When you use logical replication, you are usually telling Postgresql to keep WAL files around for longer than it needs them. If the files are removed too quickly, then your `replication slot` will become inactive and, if the database receives a large number of changes in a short time, then the `replication slot` can become lost as it was not able to keep up. + +In order to mitigate this, Postgresql has many options and settings that can be tweaked to manage the WAL usage effectively. Not all of these settings are user configurable as they can impact the stability of your database. For those that are, these should be considered as advanced configuration and not changed without understanding that they can cause additional disk space and resources to be used, as well as incur additional costs. + +| Setting | Description | User-facing | Default | +| -------------------------------------------------------------------------------------- | ------------------------------------------------- | ----------- | ------- | +| [`max_replication_slots`](https://postgresqlco.nf/doc/en/param/max_replication_slots/) | Max count of replication slots allowed | | | +| [`wal_keep_size`](https://postgresqlco.nf/doc/en/param/wal_keep_size/) | Minimum size of WAL files to keep for replication | | | +| [`max_slot_wal_keep_size`](https://postgresqlco.nf/doc/en/param/max_slot_wal_keep_size/) | Max WAL size that can be reserved by replication slots | | | +| [`checkpoint_timeout`](https://postgresqlco.nf/doc/en/param/checkpoint_timeout/) | Max time between WAL checkpoints | | | + diff --git a/apps/docs/content/guides/database/replication/overview.mdx b/apps/docs/content/guides/database/replication/overview.mdx new file mode 100644 index 00000000000..65dec528978 --- /dev/null +++ b/apps/docs/content/guides/database/replication/overview.mdx @@ -0,0 +1,37 @@ +--- +id: 'replication' +title: 'Replication and Analytics with Supabase' +description: 'Performing Extract Transform Load (ETL) with Supabase' +sidebar_label: 'Overview' +--- + +# Replication, Analytics and ETL + +You have launched your app. You want to understand what works and what doesn't so you start some queries and soon notice that your queries are taking minutes or that you need to combine these data with another system you are using. Not to worry, you are in the right place. + +## How? + +OK, so now we know why you should not use your primary database for this, what should you use? This is not an easy question and it comes with the "it depends" answer so let's go through some options: + +### Read Replicas + +[Read-Replicas](/guides/platform/read-replicas) are available on Supabase and allow you to create a second database that is a direct, read-only copy of your primary database that stays in sync. This can be great for many things, like load-balancing, and, in this case, they can be great for analytics workloads. + +### Logical Replication + +Your use case might go beyond what a Read-Replica can offer and you may need to replicate your data from Supabase into a Data Warehouse, like Clickhouse or Google BigQuery. This could be because of long running queries, the amount of data or because you need to combine your Supabase data with data from other systems. + +When you want to do this, this is where Extract Transform Load (ETL) comes in. + +1. Extract - Getting the data out of your source, typically using a method called [Logical Replication](https://www.postgresql.org/docs/current/logical-replication.html) +2. Transform - Modifying the data for ingestion +3. Load - Loading the transformed data into the destination + +To perform each of these steps, some destinations will pull data directly from Postgresql but, in most cases, you will want to use a tool, such as Estuary or Airbyte or Fivetran. These perform the `Extract` and `Transform` in much the same way, as well as support a large number of destinations to `Load` your data into, ranging from S3 to Salesforce. + +Regardless of the tool you choose, it is very likely you will be using logical replication, so let's take a moment to coer that. + + +To get started, choose the tool you want to use below: + + \ No newline at end of file diff --git a/apps/docs/content/guides/database/replication/services/etl-airbyte.mdx b/apps/docs/content/guides/database/replication/services/etl-airbyte.mdx new file mode 100644 index 00000000000..c954aac612a --- /dev/null +++ b/apps/docs/content/guides/database/replication/services/etl-airbyte.mdx @@ -0,0 +1,6 @@ +--- +id: 'replication-airbyte' +title: 'Replicating from Supabase to Airbyte' +description: 'How to set up Logical Replication with Airbyte' +sidebar_label: 'Replicating to Airbyte' +--- \ No newline at end of file diff --git a/apps/docs/content/guides/database/replication/services/etl-estuary.mdx b/apps/docs/content/guides/database/replication/services/etl-estuary.mdx new file mode 100644 index 00000000000..c318296a691 --- /dev/null +++ b/apps/docs/content/guides/database/replication/services/etl-estuary.mdx @@ -0,0 +1,6 @@ +--- +id: 'replication-estuary' +title: 'Replicating from Supabase to Estuary' +description: 'How to set up Logical Replication with Estuary' +sidebar_label: 'Replicating to Estuary' +--- \ No newline at end of file diff --git a/apps/docs/content/guides/database/replication/services/etl-fivetran.mdx b/apps/docs/content/guides/database/replication/services/etl-fivetran.mdx new file mode 100644 index 00000000000..be73839b2c9 --- /dev/null +++ b/apps/docs/content/guides/database/replication/services/etl-fivetran.mdx @@ -0,0 +1,6 @@ +--- +id: 'replication-fivetrain' +title: 'Replicating from Supabase to Fivetran' +description: 'How to set up Logical Replication with Fivetran' +sidebar_label: 'Replicating to Fivetran' +--- \ No newline at end of file diff --git a/apps/docs/content/guides/database/replication/services/etl-materialize.mdx b/apps/docs/content/guides/database/replication/services/etl-materialize.mdx new file mode 100644 index 00000000000..b23d3c50810 --- /dev/null +++ b/apps/docs/content/guides/database/replication/services/etl-materialize.mdx @@ -0,0 +1,6 @@ +--- +id: 'replication-materialize' +title: 'Replicating from Supabase to Materialize' +description: 'How to set up Logical Replication with Materialize' +sidebar_label: 'Replicating to Materialize' +--- \ No newline at end of file diff --git a/apps/docs/content/guides/database/replication/services/etl-stitch.mdx b/apps/docs/content/guides/database/replication/services/etl-stitch.mdx new file mode 100644 index 00000000000..f82f6438c03 --- /dev/null +++ b/apps/docs/content/guides/database/replication/services/etl-stitch.mdx @@ -0,0 +1,6 @@ +--- +id: 'replication-stitch' +title: 'Replicating from Supabase to Stitch' +description: 'How to set up Logical Replication with Stitch' +sidebar_label: 'Replicating to Stitch' +--- \ No newline at end of file diff --git a/apps/docs/content/guides/platform/read-replicas.mdx b/apps/docs/content/guides/platform/read-replicas.mdx index 882892700d6..91ef1ba2e16 100644 --- a/apps/docs/content/guides/platform/read-replicas.mdx +++ b/apps/docs/content/guides/platform/read-replicas.mdx @@ -28,7 +28,7 @@ Read Replicas are additional databases that are kept in sync with your Primary d ## About Read Replicas -The database you start with when launching a Supabase project is your Primary database. Read Replicas are kept in sync with the Primary through a process called "replication." Replication is asynchronous to ensure that transactions on the Primary aren't blocked. There is a delay between an update on the Primary and the time that a Read Replica receives the change. This delay is called "replication lag." +The database you start with when launching a Supabase project is your Primary database. Read Replicas are kept in sync with the Primary through a process called "[physical replication](https://www.postgresql.org/docs/16/protocol-replication.html)." Replication is asynchronous to ensure that transactions on the Primary aren't blocked. There is a delay between an update on the Primary and the time that a Read Replica receives the change. This delay is called "replication lag." You can only read data from a Read Replica. This is in contrast to a Primary database, where you can both read and write: