diff --git a/apps/docs/components/Navigation/Navigation.constants.ts b/apps/docs/components/Navigation/Navigation.constants.ts index 94f40fce1ee..beb5bdc962d 100644 --- a/apps/docs/components/Navigation/Navigation.constants.ts +++ b/apps/docs/components/Navigation/Navigation.constants.ts @@ -241,12 +241,57 @@ export const menuItems: NavMenu = { { label: 'Realtime', items: [ - { name: 'Overview', url: '/guides/realtime', items: [] }, - { name: 'Quickstart', url: '/guides/realtime/quickstart', items: [] }, - { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] }, - { name: 'Presence', url: '/guides/realtime/presence', items: [] }, - { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] }, - { name: 'Rate Limits', url: '/guides/realtime/rate-limits', items: [] }, + { + name: 'Overview', + url: '/guides/realtime', + items: [], + }, + { + name: 'Quickstart', + url: '/guides/realtime/quickstart', + items: [], + }, + { + name: 'Features', + url: undefined, + items: [ + { name: 'Channels', url: '/guides/realtime/channels', items: [] }, + { + name: 'Extensions', + url: '/guides/realtime/extensions', + items: [ + { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] }, + { name: 'Presence', url: '/guides/realtime/presence', items: [] }, + { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] }, + ], + }, + ], + }, + { + name: 'Guides', + url: undefined, + items: [ + { + name: 'Subscribing to Database Changes', + url: '/guides/realtime/subscribing-to-database-changes', + items: [], + }, + { + name: 'Using Realtime with Next.js', + url: '/guides/realtime/realtime-with-nextjs', + items: [], + }, + ], + }, + { + name: 'Deep dive', + url: undefined, + items: [ + { name: 'Rate Limits', url: '/guides/realtime/rate-limits', items: [] }, + { name: 'Architecture', url: '/guides/realtime/architecture', items: [] }, + { name: 'Protocol', url: '/guides/realtime/protocol', items: [] }, + ], + }, ], }, { diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index f30bfec7b43..f677f5ebc76 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -604,16 +604,53 @@ export const realtime = { label: 'Realtime', url: '/guides/realtime', items: [ - { name: 'Overview', url: '/guides/realtime', items: [] }, - { name: 'Quickstart', url: '/guides/realtime/quickstart', items: [] }, { - name: 'Channels', + name: 'Overview', + url: '/guides/realtime', + }, + { + name: 'Quickstart', + url: '/guides/realtime/quickstart', + }, + { + name: 'Features', + url: undefined, + items: [ + { name: 'Channels', url: '/guides/realtime/channels', items: [] }, + { + name: 'Extensions', + url: '/guides/realtime/extensions', + items: [ + { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] }, + { name: 'Presence', url: '/guides/realtime/presence', items: [] }, + { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] }, + ], + }, + ], + }, + { + name: 'Guides', + url: undefined, + items: [ + { + name: 'Subscribing to Database Changes', + url: '/guides/realtime/subscribing-to-database-changes', + items: [], + }, + { + name: 'Using Realtime with Next.js', + url: '/guides/realtime/realtime-with-nextjs', + items: [], + }, + ], + }, + { + name: 'Deep dive', url: undefined, items: [ - { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] }, - { name: 'Presence', url: '/guides/realtime/presence', items: [] }, - { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] }, { name: 'Rate Limits', url: '/guides/realtime/rate-limits', items: [] }, + { name: 'Architecture', url: '/guides/realtime/architecture', items: [] }, + { name: 'Protocol', url: '/guides/realtime/protocol', items: [] }, ], }, ], diff --git a/apps/docs/pages/guides/platform/logs.mdx b/apps/docs/pages/guides/platform/logs.mdx index a748b241fbc..897f22cebda 100644 --- a/apps/docs/pages/guides/platform/logs.mdx +++ b/apps/docs/pages/guides/platform/logs.mdx @@ -60,6 +60,12 @@ Supabase provides a logging interface specific to each product. You can use simp [Realtime logs](https://app.supabase.com/project/_/logs/realtime-logs) show all server logs for your [Realtime API usage](../../guides/realtime). + + +Realtime connections are not logged by default. Turn on [Realtime connection logs per client](#logging-realtime-connections) with the `log_level` parameter. + + + ![Realtime Logs](/docs/img/guides/platform/logs/logs-realtime.png) @@ -126,6 +132,21 @@ If any permission errors are encountered when executing `alter role postgres ... +## Logging Realtime Connections + +Realtime doesn't log new WebSocket connections or Channel joins by default. Enable connection logging per client by including an `info` `log_level` parameter when instantiating the Supabase client. + +```javascript +import { createClient } from '@supabase/supabase-js' + +const options = { + realtime: { + log_level: 'info', + }, +} +const supabase = createClient('https://xyzcompany.supabase.co', 'public-anon-key', options) +``` + ## Logs Explorer The [Logs Explorer](https://app.supabase.com/project/_/logs-explorer) exposes logs from each part of the Supabase stack as a separate table that can be queried and joined using SQL. diff --git a/apps/docs/pages/guides/realtime/architecture.mdx b/apps/docs/pages/guides/realtime/architecture.mdx new file mode 100644 index 00000000000..5a4173bdb88 --- /dev/null +++ b/apps/docs/pages/guides/realtime/architecture.mdx @@ -0,0 +1,58 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'architecture', + title: 'Realtime Architecture', + description: 'Architecture of the Supabase Realtime service', + sidebar_label: 'Architecture', +} + +Realtime is a globally distributed Elixir cluster. Clients can connect to any node in the cluster via WebSockets and send messages to any other client connected to the cluster. + +Realtime is written in [Elixir](https://elixir-lang.org/), which compiles to [Erlang](https://www.erlang.org/), and utilizes many tools the [Phoenix Framework](https://www.phoenixframework.org/) provides out of the box. + +![Global Architecture](/docs/img/guides/realtime/realtime-arch.png) + +## Elixir & Phoenix + +Phoenix is fast and able to handle millions of concurrent connections. + +Phoenix can handle many concurrent connections because Elixir provides lightweight processes (not OS processes) to work with. + +Client-facing WebSocket servers need to handle many concurrent connections. Elixir & Phoenix let the Supabase Realtime cluster do this easily. + +## Global Cluster + +Presence is an in-memory key-value store backed by a CRDT. When a user is connected to the cluster the state of that user is sent to all connected Realtime nodes. + +Broadcast lets you send a message from any connected client to a Channel. Any other client connected to that same Channel will receive that message. + +This works globally. A client connected to a Realtime node in the United States can send a message to another client connected to a node in Singapore. Simply connect two clients to the same Realtime Channel and they'll all receive the same messages. + +Broadcast is useful for getting messages to users in the same location very quickly. If a group of clients are connected to a node in Singapore, the message only needs to go to that Realtime node in Singapore and back down. If users are close to a Realtime node they'll get Broadcast messages in the time it takes to ping the cluster. + +Thanks to the Realtime cluster, you (an amazing Supabase user) don't have to think about which regions your clients are connected to. + +If you're using Broadcast, Presence, or streaming database changes, messages will always get to your users via the shortest path possible. + +## Connecting to a Database + +Realtime allows you to listen to changes from your Postgres database. When a new client connects to Realtime and initializes the `postgres_changes` Realtime Extension the cluster will connect to your Postgres database and start streaming changes from a replication slot. + +Realtime knows the region your database is in, and connects to it from the closest region possible. + +Every Realtime region has at least two nodes so if one node goes offline the other node should reconnect and start streaming changes again. + +## Streaming the Write-Ahead Log + +A Postgres logical replication slot is acquired when connecting to your database. + +Realtime delivers changes by polling the replication slot and appending channel subscription IDs to each wal record. + +Subscription IDs are Erlang processes representing underlying sockets on the cluster. These IDs are globally unique and messages to processes are routed automatically by the Erlang virtual machine. + +After receiving results from the polling query, with subscription IDs appended, Realtime delivers records to those clients. + +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/pages/guides/realtime/channels.mdx b/apps/docs/pages/guides/realtime/channels.mdx new file mode 100644 index 00000000000..b016f4bf6f6 --- /dev/null +++ b/apps/docs/pages/guides/realtime/channels.mdx @@ -0,0 +1,22 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'channels', + title: 'Realtime Channels', + description: 'WebSockets are Channels on Supabase Realtime', + sidebar_label: 'Channels', +} + +You can uniquely reference a channel by its `topic` you define when you initialize your Supabase Realtime client. Everyone connected to the same Channel topic will receive the same messages. + +Clients can send and receive messages bi-directionally over a Channel. The Realtime backend can also push messages to all clients connected to the same Channel. + +A single client can receive change records from Postgres, Broadcast messages from other clients and Presence updates all over the same Channel. + +Channels are implemented over [Phoenix Channels](https://hexdocs.pm/phoenix/channels.html) which uses [Phoenix.PubSub](https://hexdocs.pm/phoenix_pubsub/Phoenix.PubSub.html) with the default `Phoenix.PubSub.PG2` adapter. + +The PG2 adapter utilizes Erlang [process groups](https://www.erlang.org/docs/18/man/pg2.html) to implement the PubSub model where a publisher can send messages to many subscribers. + +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/pages/guides/realtime/extensions.mdx b/apps/docs/pages/guides/realtime/extensions.mdx new file mode 100644 index 00000000000..bad8b1daf66 --- /dev/null +++ b/apps/docs/pages/guides/realtime/extensions.mdx @@ -0,0 +1,27 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'extensions', + title: 'Realtime Extensions', + description: + 'Extensions enable powerful Realtime features like Presence, Broadcast and database change streaming', + sidebar_label: 'Extensions', +} + +Supabase Realtime provides multiplayer features so that you can build real-time applications. The server is has several extensions which are built to solve a specific task. + +- [Broadcast](/docs/guides/realtime/broadcast): for sending rapid, ephemeral messages to other connected clients. A typical use-case is tracking mouse movements. +- [Presence](/docs/guides/realtime/presence): for sending user state between connected clients. A typical use-case is simply an "online" status, which will disappear when a user is disconnected. +- [Postgres Changes](/docs/guides/realtime/postgres-changes): for receiving database changes in real-time. + +## Choosing between Broadcast and Presence + +Defaulting to using Broadcast is a good idea, and then use Presence sparingly. The Presence extension merges all changes into a shared state for every client connected to the same "channel". If you _do_ use Presence, it's best to throttle your changes so that you are sending updates less frequently. + +## Future possibilities + +Realtime extensions are still under development. We have many more extensions planned. You can follow our progress by watching the [Realtime](https://github.com/supabase/realtime) repo on GitHub. + +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/pages/guides/realtime/postgres-changes.mdx b/apps/docs/pages/guides/realtime/postgres-changes.mdx index 0f3b15e6bac..07b819990ac 100644 --- a/apps/docs/pages/guides/realtime/postgres-changes.mdx +++ b/apps/docs/pages/guides/realtime/postgres-changes.mdx @@ -15,7 +15,9 @@ Clients can choose to receive `INSERT`, `UPDATE`, `DELETE`, or `*` (all) changes Postgres Changes works out of the box for tables in the `public` schema. You can listen to tables in your private schemas by granting table `SELECT` permissions to the database role found in your access token. You can run a query similar to the following: ```sql -GRANT SELECT ON "private_schema"."table" TO authenticated; +grant +select + on "private_schema"."table" to authenticated; ``` @@ -29,15 +31,19 @@ You can do this in the [Replication](https://app.supabase.com/project/_/database ```sql begin; - -- remove the supabase_realtime publication - drop publication if exists supabase_realtime; - -- re-create the supabase_realtime publication with no tables - create publication supabase_realtime; +-- remove the supabase_realtime publication +drop + publication if exists supabase_realtime; + +-- re-create the supabase_realtime publication with no tables +create publication supabase_realtime; + commit; -- add a table to the publication -alter publication supabase_realtime add table messages; +alter + publication supabase_realtime add table messages; ``` ### Full `old` Record @@ -46,9 +52,16 @@ By default, only `new` record changes are sent but if you want to receive the `o you can set the `replica identity` of your table to `full`: ```sql -alter table messages replica identity full; +alter table + messages replica identity full; ``` + + +RLS policies are not applied to `DELETE` statements. When RLS is enabled and `replica identity` is set to `full` on a table, the `old` record contains only the primary key(s). + + + ## Schema Changes To listen to all changes in the `public` schema: diff --git a/apps/docs/pages/guides/realtime/protocol.mdx b/apps/docs/pages/guides/realtime/protocol.mdx new file mode 100644 index 00000000000..488d2975612 --- /dev/null +++ b/apps/docs/pages/guides/realtime/protocol.mdx @@ -0,0 +1,194 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'protocol', + title: 'Realtime Protocol', + description: 'Understanding Realtime Protocol', +} + +The Realtime Protocol is a set of message formats used for communication over a WebSocket connection between a Realtime client and server. These messages are used to initiate a connection, update access tokens, receive system status updates, and receive real-time updates from the Postgres database. + +## Connection + +In the initial message, the client sends a message specifying the features they want to use (Broadcast, Presence, Postgres Changes). + +```ts +{ + "event": "phx_join", + "topic": string, + "payload": { + "config": { + "broadcast": { + "self": boolean + }, + "presence": { + "key": string + }, + "postgres_changes": [ + { + "event": "*" | "INSERT" | "UPDATE" | "DELETE", + "schema": string, + "table": string, + "filter": string + '=' + "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "in" + '.' + string + } + ] + } + }, + "ref": string +} +``` + + +The `in` filter has the format `COLUMN_NAME=in.(value1,value2,value3)`. However, other filters use the format `COLUMN_NAME=FILTER_NAME.value`. + + +In response, the server sends the Postgres configuration with a unique ID. With this ID, the client should route incoming changes to the appropriate callback. + +```ts +{ + "event": "phx_reply", + "topic": string, + "payload": { + "response": { + "postgres_changes": [ + { + "id": number, + "event": "*" | "INSERT" | "UPDATE" | "DELETE", + "schema": string, + "table": string, + "filter": string + '=' + "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "in" + '.' + string + } + ] + }, + "status": "ok" | "error" + }, + "ref": string +} +``` + +## System messages + +System message are used to inform a client about the status of the Postgres subscription. The `payload.status` indicates if the subscription successful or not. +The body of the `payload.message` can be "Subscribed to PostgreSQL" or "Subscribing to PostgreSQL failed" with subscription params. + +```ts +{ + "event": "system", + "topic": string, + "payload":{ + "channel": string, + "extension": "postgres_changes", + "message": "Subscribed to PostgreSQL" | "Subscribing to PostgreSQL failed", + "status": "ok" | "error" + }, + "ref": null, +} +``` + +## Heartbeat + +The heartbeat message should be sent every 30 seconds to avoid a connection timeout. + +```ts +{ + "event": "heartbeat", + "topic": "phoenix", + "payload": {}, + "ref": string +} +``` + +## Access Token + +To update the access token, you need to send to the server a message specifying a new token in the `payload.access_token` value. + +```ts +{ + "event": "access_token", + "topic": string, + "payload":{ + "access_token": string + }, + "ref": string +} +``` + +## Postgres CDC message + +Realtime sends a message with the following structure + +```ts +{ + "event": "postgres_changes", + "topic": string, + "payload": { + "data": { + "columns": Array<{name: string, type: string}>, + "commit_timestamp": string, + "errors": null | string, + "old_record": {"id": number | string}, + "record": {[key: string]: boolean | number | string | null}, + "type": "*" | "INSERT" | "UPDATE" | "DELETE", + "schema": string, + "table": string + }, + "ids": Array + }, + "ref": null +} +``` + +## Broadcast message + +Structure of the broadcast event + +```ts +{ + "event": "broadcast", + "topic": string, + "payload": { + "event": string, + "payload": {[key: string]: boolean | number | string | null | undefined}, + "type": "broadcast" + }, + "ref": null +} +``` + +## Presence message + +The Presence events allow clients to monitor the online status of other clients in real-time. + +### State Update + +After joining, the server sends a `presence_state` message to a client with presence information. The payload field contains keys in UUID format, where each key represents a client and its value is a JSON object containing information about that client. + +```ts +{ + "event": "presence_state", + "topic": string, + "payload": { + [key: string]: {metas: Array<{phx_ref: string, name: string, t: float}>} + }, + "ref": null +} +``` + +### Diff Update + +After a change to the presence state, such as a client joining or leaving, the server sends a presence_diff message to update the client's view of the presence state. The payload field contains two keys, `joins` and `leaves`, which represent clients that have joined and left, respectively. The values associated with each key are UUIDs of the clients. + +```ts +{ + "event": "presence_diff", + "topic": string, + "payload": { + "joins": {metas: Array<{phx_ref: string, name: string, t: float}>}, + "leaves": {metas: Array<{phx_ref: string, name: string, t: float}>} + }, + "ref": null +} +``` + +export const Page = ({ children }) => +export default Page diff --git a/apps/docs/pages/guides/realtime/realtime-with-nextjs.mdx b/apps/docs/pages/guides/realtime/realtime-with-nextjs.mdx new file mode 100644 index 00000000000..e60afb4f74a --- /dev/null +++ b/apps/docs/pages/guides/realtime/realtime-with-nextjs.mdx @@ -0,0 +1,24 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'realtime-with-nextjs', + title: 'Using Realtime with Next.js', + description: 'Client & Server Components in Next.js with Realtime Updates', + sidebar_label: 'Videos', +} + +In this guide we explore the best ways to receive realtime Postgres changes with your Next.js application. +We'll show both client and serverside updates, and explore the which option is best. + +
+ +
+ +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/pages/guides/realtime/subscribing-to-database-changes.mdx b/apps/docs/pages/guides/realtime/subscribing-to-database-changes.mdx new file mode 100644 index 00000000000..f22764ac7c8 --- /dev/null +++ b/apps/docs/pages/guides/realtime/subscribing-to-database-changes.mdx @@ -0,0 +1,80 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'subscribing-to-database-changes', + title: 'Subscribing to Database Changes', + description: 'Listen to database changes in real-time from your website or application.', + sidebar_label: 'Videos', +} + +Supabase allows you to subscribe to real-time changes on your database from your client application. + +You can listen to database changes using the [Postgres Changes](/docs/guides/realtime/postgres-changes) extension. + +## Demo + +
+ +
+ +## Streaming inserts + +You can use the `INSERT` event to stream all new rows. + +```js +const { createClient } = require('@supabase/supabase-js') + +const supabase = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_KEY) + +const channel = supabase + .channel('schema-db-changes') + .on( + 'postgres_changes', + { + event: 'INSERT', + schema: 'public', + }, + (payload) => console.log(payload) + ) + .subscribe() +``` + +## Streaming updates + +You can use the `UPDATE` event to stream all updated rows. + +```js +const { createClient } = require('@supabase/supabase-js') + +const supabase = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_KEY) + +const channel = supabase + .channel('schema-db-changes') + .on( + 'postgres_changes', + { + event: 'UPDATE', + schema: 'public', + }, + (payload) => console.log(payload) + ) + .subscribe() +``` + +## More resources + +- Learn more about the [Postgres Changes](/docs/guides/realtime/postgres-changes) extension. +- Client Libraries: + - [JavaScript](/docs/reference/javascript/subscribe) + - [Flutter](/docs/reference/dart/stream) + - [Python](/docs/reference/python/subscribe) + - [C#](/docs/reference/csharp/subscribe) + +export const Page = ({ children }) => + +export default Page diff --git a/apps/docs/public/img/guides/realtime/realtime-arch.png b/apps/docs/public/img/guides/realtime/realtime-arch.png new file mode 100644 index 00000000000..fca216142b1 Binary files /dev/null and b/apps/docs/public/img/guides/realtime/realtime-arch.png differ