Merge pull request #12424 from supabase/docs/realtime-architecture

feat: more Realtime docs

apps/docs/components/Navigation/Navigation.constants.ts

@@ -241,12 +241,57 @@ export const menuItems: NavMenu = {
     {
       label: 'Realtime',
       items: [
-        { name: 'Overview', url: '/guides/realtime', items: [] },
-        { name: 'Quickstart', url: '/guides/realtime/quickstart', items: [] },
-        { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] },
-        { name: 'Presence', url: '/guides/realtime/presence', items: [] },
-        { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] },
-        { name: 'Rate Limits', url: '/guides/realtime/rate-limits', items: [] },
+        {
+          name: 'Overview',
+          url: '/guides/realtime',
+          items: [],
+        },
+        {
+          name: 'Quickstart',
+          url: '/guides/realtime/quickstart',
+          items: [],
+        },
+        {
+          name: 'Features',
+          url: undefined,
+          items: [
+            { name: 'Channels', url: '/guides/realtime/channels', items: [] },
+            {
+              name: 'Extensions',
+              url: '/guides/realtime/extensions',
+              items: [
+                { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] },
+                { name: 'Presence', url: '/guides/realtime/presence', items: [] },
+                { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] },
+              ],
+            },
+          ],
+        },
+        {
+          name: 'Guides',
+          url: undefined,
+          items: [
+            {
+              name: 'Subscribing to Database Changes',
+              url: '/guides/realtime/subscribing-to-database-changes',
+              items: [],
+            },
+            {
+              name: 'Using Realtime with Next.js',
+              url: '/guides/realtime/realtime-with-nextjs',
+              items: [],
+            },
+          ],
+        },
+        {
+          name: 'Deep dive',
+          url: undefined,
+          items: [
+            { name: 'Rate Limits', url: '/guides/realtime/rate-limits', items: [] },
+            { name: 'Architecture', url: '/guides/realtime/architecture', items: [] },
+            { name: 'Protocol', url: '/guides/realtime/protocol', items: [] },
+          ],
+        },
       ],
     },
     {

apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts

@@ -604,16 +604,53 @@ export const realtime = {
   label: 'Realtime',
   url: '/guides/realtime',
   items: [
-    { name: 'Overview', url: '/guides/realtime', items: [] },
-    { name: 'Quickstart', url: '/guides/realtime/quickstart', items: [] },
     {
-      name: 'Channels',
+      name: 'Overview',
+      url: '/guides/realtime',
+    },
+    {
+      name: 'Quickstart',
+      url: '/guides/realtime/quickstart',
+    },
+    {
+      name: 'Features',
+      url: undefined,
+      items: [
+        { name: 'Channels', url: '/guides/realtime/channels', items: [] },
+        {
+          name: 'Extensions',
+          url: '/guides/realtime/extensions',
+          items: [
+            { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] },
+            { name: 'Presence', url: '/guides/realtime/presence', items: [] },
+            { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] },
+          ],
+        },
+      ],
+    },
+    {
+      name: 'Guides',
+      url: undefined,
+      items: [
+        {
+          name: 'Subscribing to Database Changes',
+          url: '/guides/realtime/subscribing-to-database-changes',
+          items: [],
+        },
+        {
+          name: 'Using Realtime with Next.js',
+          url: '/guides/realtime/realtime-with-nextjs',
+          items: [],
+        },
+      ],
+    },
+    {
+      name: 'Deep dive',
       url: undefined,
       items: [
-        { name: 'Broadcast', url: '/guides/realtime/broadcast', items: [] },
-        { name: 'Presence', url: '/guides/realtime/presence', items: [] },
-        { name: 'Postgres Changes', url: '/guides/realtime/postgres-changes', items: [] },
         { name: 'Rate Limits', url: '/guides/realtime/rate-limits', items: [] },
+        { name: 'Architecture', url: '/guides/realtime/architecture', items: [] },
+        { name: 'Protocol', url: '/guides/realtime/protocol', items: [] },
       ],
     },
   ],

apps/docs/pages/guides/platform/logs.mdx

@@ -60,6 +60,12 @@ Supabase provides a logging interface specific to each product. You can use simp
 
 [Realtime logs](https://app.supabase.com/project/_/logs/realtime-logs) show all server logs for your [Realtime API usage](../../guides/realtime).
 
+<Admonition type="note">
+
+Realtime connections are not logged by default. Turn on [Realtime connection logs per client](#logging-realtime-connections) with the `log_level` parameter.
+
+</Admonition>
+
 ![Realtime Logs](/docs/img/guides/platform/logs/logs-realtime.png)
 
 </TabPanel>
@@ -126,6 +132,21 @@ If any permission errors are encountered when executing `alter role postgres ...
 
 </Admonition>
 
+## Logging Realtime Connections
+
+Realtime doesn't log new WebSocket connections or Channel joins by default. Enable connection logging per client by including an `info` `log_level` parameter when instantiating the Supabase client.
+
+```javascript
+import { createClient } from '@supabase/supabase-js'
+
+const options = {
+  realtime: {
+    log_level: 'info',
+  },
+}
+const supabase = createClient('https://xyzcompany.supabase.co', 'public-anon-key', options)
+```
+
 ## Logs Explorer
 
 The [Logs Explorer](https://app.supabase.com/project/_/logs-explorer) exposes logs from each part of the Supabase stack as a separate table that can be queried and joined using SQL.

apps/docs/pages/guides/realtime/architecture.mdx

@@ -0,0 +1,58 @@
+import Layout from '~/layouts/DefaultGuideLayout'
+
+export const meta = {
+  id: 'architecture',
+  title: 'Realtime Architecture',
+  description: 'Architecture of the Supabase Realtime service',
+  sidebar_label: 'Architecture',
+}
+
+Realtime is a globally distributed Elixir cluster. Clients can connect to any node in the cluster via WebSockets and send messages to any other client connected to the cluster.
+
+Realtime is written in [Elixir](https://elixir-lang.org/), which compiles to [Erlang](https://www.erlang.org/), and utilizes many tools the [Phoenix Framework](https://www.phoenixframework.org/) provides out of the box.
+
+![Global Architecture](/docs/img/guides/realtime/realtime-arch.png)
+
+## Elixir & Phoenix
+
+Phoenix is fast and able to handle millions of concurrent connections.
+
+Phoenix can handle many concurrent connections because Elixir provides lightweight processes (not OS processes) to work with.
+
+Client-facing WebSocket servers need to handle many concurrent connections. Elixir & Phoenix let the Supabase Realtime cluster do this easily.
+
+## Global Cluster
+
+Presence is an in-memory key-value store backed by a CRDT. When a user is connected to the cluster the state of that user is sent to all connected Realtime nodes.
+
+Broadcast lets you send a message from any connected client to a Channel. Any other client connected to that same Channel will receive that message.
+
+This works globally. A client connected to a Realtime node in the United States can send a message to another client connected to a node in Singapore. Simply connect two clients to the same Realtime Channel and they'll all receive the same messages.
+
+Broadcast is useful for getting messages to users in the same location very quickly. If a group of clients are connected to a node in Singapore, the message only needs to go to that Realtime node in Singapore and back down. If users are close to a Realtime node they'll get Broadcast messages in the time it takes to ping the cluster.
+
+Thanks to the Realtime cluster, you (an amazing Supabase user) don't have to think about which regions your clients are connected to.
+
+If you're using Broadcast, Presence, or streaming database changes, messages will always get to your users via the shortest path possible.
+
+## Connecting to a Database
+
+Realtime allows you to listen to changes from your Postgres database. When a new client connects to Realtime and initializes the `postgres_changes` Realtime Extension the cluster will connect to your Postgres database and start streaming changes from a replication slot.
+
+Realtime knows the region your database is in, and connects to it from the closest region possible.
+
+Every Realtime region has at least two nodes so if one node goes offline the other node should reconnect and start streaming changes again.
+
+## Streaming the Write-Ahead Log
+
+A Postgres logical replication slot is acquired when connecting to your database.
+
+Realtime delivers changes by polling the replication slot and appending channel subscription IDs to each wal record.
+
+Subscription IDs are Erlang processes representing underlying sockets on the cluster. These IDs are globally unique and messages to processes are routed automatically by the Erlang virtual machine.
+
+After receiving results from the polling query, with subscription IDs appended, Realtime delivers records to those clients.
+
+export const Page = ({ children }) => <Layout meta={meta} children={children} />
+
+export default Page

apps/docs/pages/guides/realtime/channels.mdx

@@ -0,0 +1,22 @@
+import Layout from '~/layouts/DefaultGuideLayout'
+
+export const meta = {
+  id: 'channels',
+  title: 'Realtime Channels',
+  description: 'WebSockets are Channels on Supabase Realtime',
+  sidebar_label: 'Channels',
+}
+
+You can uniquely reference a channel by its `topic` you define when you initialize your Supabase Realtime client. Everyone connected to the same Channel topic will receive the same messages.
+
+Clients can send and receive messages bi-directionally over a Channel. The Realtime backend can also push messages to all clients connected to the same Channel.
+
+A single client can receive change records from Postgres, Broadcast messages from other clients and Presence updates all over the same Channel.
+
+Channels are implemented over [Phoenix Channels](https://hexdocs.pm/phoenix/channels.html) which uses [Phoenix.PubSub](https://hexdocs.pm/phoenix_pubsub/Phoenix.PubSub.html) with the default `Phoenix.PubSub.PG2` adapter.
+
+The PG2 adapter utilizes Erlang [process groups](https://www.erlang.org/docs/18/man/pg2.html) to implement the PubSub model where a publisher can send messages to many subscribers.
+
+export const Page = ({ children }) => <Layout meta={meta} children={children} />
+
+export default Page

apps/docs/pages/guides/realtime/extensions.mdx

@@ -0,0 +1,27 @@
+import Layout from '~/layouts/DefaultGuideLayout'
+
+export const meta = {
+  id: 'extensions',
+  title: 'Realtime Extensions',
+  description:
+    'Extensions enable powerful Realtime features like Presence, Broadcast and database change streaming',
+  sidebar_label: 'Extensions',
+}
+
+Supabase Realtime provides multiplayer features so that you can build real-time applications. The server is has several extensions which are built to solve a specific task.
+
+- [Broadcast](/docs/guides/realtime/broadcast): for sending rapid, ephemeral messages to other connected clients. A typical use-case is tracking mouse movements.
+- [Presence](/docs/guides/realtime/presence): for sending user state between connected clients. A typical use-case is simply an "online" status, which will disappear when a user is disconnected.
+- [Postgres Changes](/docs/guides/realtime/postgres-changes): for receiving database changes in real-time.
+
+## Choosing between Broadcast and Presence
+
+Defaulting to using Broadcast is a good idea, and then use Presence sparingly. The Presence extension merges all changes into a shared state for every client connected to the same "channel". If you _do_ use Presence, it's best to throttle your changes so that you are sending updates less frequently.
+
+## Future possibilities
+
+Realtime extensions are still under development. We have many more extensions planned. You can follow our progress by watching the [Realtime](https://github.com/supabase/realtime) repo on GitHub.
+
+export const Page = ({ children }) => <Layout meta={meta} children={children} />
+
+export default Page

apps/docs/pages/guides/realtime/postgres-changes.mdx

@@ -15,7 +15,9 @@ Clients can choose to receive `INSERT`, `UPDATE`, `DELETE`, or `*` (all) changes
 Postgres Changes works out of the box for tables in the `public` schema. You can listen to tables in your private schemas by granting table `SELECT` permissions to the database role found in your access token. You can run a query similar to the following:
 
 ```sql
-GRANT SELECT ON "private_schema"."table" TO authenticated;
+grant
+select
+  on "private_schema"."table" to authenticated;
 ```
 
 <Admonition type="caution">
@@ -29,15 +31,19 @@ You can do this in the [Replication](https://app.supabase.com/project/_/database
 
 ```sql
 begin;
-  -- remove the supabase_realtime publication
-  drop publication if exists supabase_realtime;
 
-  -- re-create the supabase_realtime publication with no tables
-  create publication supabase_realtime;
+-- remove the supabase_realtime publication
+drop
+  publication if exists supabase_realtime;
+
+-- re-create the supabase_realtime publication with no tables
+create publication supabase_realtime;
+
 commit;
 
 -- add a table to the publication
-alter publication supabase_realtime add table messages;
+alter
+  publication supabase_realtime add table messages;
 ```
 
 ### Full `old` Record
@@ -46,9 +52,16 @@ By default, only `new` record changes are sent but if you want to receive the `o
 you can set the `replica identity` of your table to `full`:
 
 ```sql
-alter table messages replica identity full;
+alter table
+  messages replica identity full;
 ```
 
+<Admonition type="caution">
+
+RLS policies are not applied to `DELETE` statements. When RLS is enabled and `replica identity` is set to `full` on a table, the `old` record contains only the primary key(s).
+
+</Admonition>
+
 ## Schema Changes
 
 To listen to all changes in the `public` schema:

apps/docs/pages/guides/realtime/protocol.mdx

@@ -0,0 +1,194 @@
+import Layout from '~/layouts/DefaultGuideLayout'
+
+export const meta = {
+  id: 'protocol',
+  title: 'Realtime Protocol',
+  description: 'Understanding Realtime Protocol',
+}
+
+The Realtime Protocol is a set of message formats used for communication over a WebSocket connection between a Realtime client and server. These messages are used to initiate a connection, update access tokens, receive system status updates, and receive real-time updates from the Postgres database.
+
+## Connection
+
+In the initial message, the client sends a message specifying the features they want to use (Broadcast, Presence, Postgres Changes).
+
+```ts
+{
+   "event": "phx_join",
+   "topic": string,
+   "payload": {
+      "config": {
+         "broadcast": {
+            "self": boolean
+         },
+         "presence": {
+            "key": string
+         },
+         "postgres_changes": [
+            {
+               "event": "*" | "INSERT" | "UPDATE" | "DELETE",
+               "schema": string,
+               "table": string,
+               "filter": string + '=' + "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "in" +  '.' + string
+            }
+         ]
+      }
+   },
+   "ref": string
+}
+```
+
+<Admonition type="note">
+The `in` filter has the format `COLUMN_NAME=in.(value1,value2,value3)`. However, other filters use the format `COLUMN_NAME=FILTER_NAME.value`.
+</Admonition>
+
+In response, the server sends the Postgres configuration with a unique ID. With this ID, the client should route incoming changes to the appropriate callback.
+
+```ts
+{
+   "event": "phx_reply",
+   "topic": string,
+   "payload": {
+      "response": {
+         "postgres_changes": [
+            {
+               "id": number,
+               "event": "*" | "INSERT" | "UPDATE" | "DELETE",
+               "schema": string,
+               "table": string,
+               "filter": string + '=' + "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "in" +  '.' + string
+            }
+         ]
+      },
+      "status": "ok" | "error"
+   },
+   "ref": string
+}
+```
+
+## System messages
+
+System message are used to inform a client about the status of the Postgres subscription. The `payload.status` indicates if the subscription successful or not.
+The body of the `payload.message` can be "Subscribed to PostgreSQL" or "Subscribing to PostgreSQL failed" with subscription params.
+
+```ts
+{
+   "event": "system",
+   "topic": string,
+   "payload":{
+      "channel": string,
+      "extension": "postgres_changes",
+      "message": "Subscribed to PostgreSQL" | "Subscribing to PostgreSQL failed",
+      "status": "ok" | "error"
+   },
+   "ref": null,
+}
+```
+
+## Heartbeat
+
+The heartbeat message should be sent every 30 seconds to avoid a connection timeout.
+
+```ts
+{
+   "event": "heartbeat",
+   "topic": "phoenix",
+   "payload": {},
+   "ref": string
+}
+```
+
+## Access Token
+
+To update the access token, you need to send to the server a message specifying a new token in the `payload.access_token` value.
+
+```ts
+{
+   "event": "access_token",
+   "topic": string,
+   "payload":{
+      "access_token": string
+   },
+   "ref": string
+}
+```
+
+## Postgres CDC message
+
+Realtime sends a message with the following structure
+
+```ts
+{
+   "event": "postgres_changes",
+   "topic": string,
+   "payload": {
+      "data": {
+         "columns": Array<{name: string, type: string}>,
+         "commit_timestamp": string,
+         "errors": null | string,
+         "old_record": {"id": number | string},
+         "record": {[key: string]: boolean | number | string | null},
+         "type": "*" | "INSERT" | "UPDATE" | "DELETE",
+         "schema": string,
+         "table": string
+      },
+      "ids": Array<number>
+   },
+   "ref": null
+}
+```
+
+## Broadcast message
+
+Structure of the broadcast event
+
+```ts
+{
+   "event": "broadcast",
+   "topic": string,
+   "payload": {
+      "event": string,
+      "payload": {[key: string]: boolean | number | string | null | undefined},
+      "type": "broadcast"
+   },
+   "ref": null
+}
+```
+
+## Presence message
+
+The Presence events allow clients to monitor the online status of other clients in real-time.
+
+### State Update
+
+After joining, the server sends a `presence_state` message to a client with presence information. The payload field contains keys in UUID format, where each key represents a client and its value is a JSON object containing information about that client.
+
+```ts
+{
+   "event": "presence_state",
+   "topic": string,
+   "payload": {
+      [key: string]: {metas: Array<{phx_ref: string, name: string, t: float}>}
+   },
+   "ref": null
+}
+```
+
+### Diff Update
+
+After a change to the presence state, such as a client joining or leaving, the server sends a presence_diff message to update the client's view of the presence state. The payload field contains two keys, `joins` and `leaves`, which represent clients that have joined and left, respectively. The values associated with each key are UUIDs of the clients.
+
+```ts
+{
+   "event": "presence_diff",
+   "topic": string,
+   "payload": {
+      "joins": {metas: Array<{phx_ref: string, name: string, t: float}>},
+      "leaves": {metas: Array<{phx_ref: string, name: string, t: float}>}
+   },
+   "ref": null
+}
+```
+
+export const Page = ({ children }) => <Layout meta={meta} children={children} />
+export default Page

apps/docs/pages/guides/realtime/realtime-with-nextjs.mdx

@@ -0,0 +1,24 @@
+import Layout from '~/layouts/DefaultGuideLayout'
+
+export const meta = {
+  id: 'realtime-with-nextjs',
+  title: 'Using Realtime with Next.js',
+  description: 'Client & Server Components in Next.js with Realtime Updates',
+  sidebar_label: 'Videos',
+}
+
+In this guide we explore the best ways to receive realtime Postgres changes with your Next.js application.
+We'll show both client and serverside updates, and explore the which option is best.
+
+<div className="video-container">
+  <iframe
+    src="https://www.youtube-nocookie.com/embed/YR-xP6PPXXA"
+    frameBorder="1"
+    allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+    allowFullScreen
+  ></iframe>
+</div>
+
+export const Page = ({ children }) => <Layout meta={meta} children={children} />
+
+export default Page

apps/docs/pages/guides/realtime/subscribing-to-database-changes.mdx

@@ -0,0 +1,80 @@
+import Layout from '~/layouts/DefaultGuideLayout'
+
+export const meta = {
+  id: 'subscribing-to-database-changes',
+  title: 'Subscribing to Database Changes',
+  description: 'Listen to database changes in real-time from your website or application.',
+  sidebar_label: 'Videos',
+}
+
+Supabase allows you to subscribe to real-time changes on your database from your client application.
+
+You can listen to database changes using the [Postgres Changes](/docs/guides/realtime/postgres-changes) extension.
+
+## Demo
+
+<div className="video-container">
+  <iframe
+    src="https://www.youtube-nocookie.com/embed/2rUjcmgZDwQ"
+    frameBorder="1"
+    allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+    allowFullScreen
+  ></iframe>
+</div>
+
+## Streaming inserts
+
+You can use the `INSERT` event to stream all new rows.
+
+```js
+const { createClient } = require('@supabase/supabase-js')
+
+const supabase = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_KEY)
+
+const channel = supabase
+  .channel('schema-db-changes')
+  .on(
+    'postgres_changes',
+    {
+      event: 'INSERT',
+      schema: 'public',
+    },
+    (payload) => console.log(payload)
+  )
+  .subscribe()
+```
+
+## Streaming updates
+
+You can use the `UPDATE` event to stream all updated rows.
+
+```js
+const { createClient } = require('@supabase/supabase-js')
+
+const supabase = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_KEY)
+
+const channel = supabase
+  .channel('schema-db-changes')
+  .on(
+    'postgres_changes',
+    {
+      event: 'UPDATE',
+      schema: 'public',
+    },
+    (payload) => console.log(payload)
+  )
+  .subscribe()
+```
+
+## More resources
+
+- Learn more about the [Postgres Changes](/docs/guides/realtime/postgres-changes) extension.
+- Client Libraries:
+  - [JavaScript](/docs/reference/javascript/subscribe)
+  - [Flutter](/docs/reference/dart/stream)
+  - [Python](/docs/reference/python/subscribe)
+  - [C#](/docs/reference/csharp/subscribe)
+
+export const Page = ({ children }) => <Layout meta={meta} children={children} />
+
+export default Page