feat(replication): Add dialog to clarify replication requirements and limitations (#39032)

apps/studio/components/interfaces/Database/Replication/DestinationPanel.tsx

@@ -5,12 +5,15 @@ import { toast } from 'sonner'
 import * as z from 'zod'
 
 import { useParams } from 'common'
+import { InlineLink } from 'components/ui/InlineLink'
+import { useCheckPrimaryKeysExists } from 'data/database/primary-keys-exists-query'
 import { useCreateDestinationPipelineMutation } from 'data/replication/create-destination-pipeline-mutation'
 import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query'
 import { useReplicationPipelineByIdQuery } from 'data/replication/pipeline-by-id-query'
 import { useReplicationPublicationsQuery } from 'data/replication/publications-query'
 import { useStartPipelineMutation } from 'data/replication/start-pipeline-mutation'
 import { useUpdateDestinationPipelineMutation } from 'data/replication/update-destination-pipeline-mutation'
+import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
 import {
   PipelineStatusRequestStatus,
   usePipelineRequestStatus,
@@ -44,6 +47,7 @@ import { Admonition } from 'ui-patterns'
 import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
 import NewPublicationPanel from './NewPublicationPanel'
 import PublicationsComboBox from './PublicationsComboBox'
+import { ReplicationDisclaimerDialog } from './ReplicationDisclaimerDialog'
 
 const formId = 'destination-editor'
 const types = ['BigQuery'] as const
@@ -80,10 +84,15 @@ export const DestinationPanel = ({
   existingDestination,
 }: DestinationPanelProps) => {
   const { ref: projectRef } = useParams()
+  const { data: project } = useSelectedProjectQuery()
   const { setRequestStatus } = usePipelineRequestStatus()
 
   const editMode = !!existingDestination
   const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)
+  const [showDisclaimerDialog, setShowDisclaimerDialog] = useState(false)
+  const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(
+    null
+  )
 
   const { mutateAsync: createDestinationPipeline, isLoading: creatingDestinationPipeline } =
     useCreateDestinationPipelineMutation({
@@ -142,9 +151,21 @@ export const DestinationPanel = ({
   const isSelectedPublicationMissing =
     isSuccessPublications && !!publicationName && !publicationNames.includes(publicationName)
 
-  const isSubmitDisabled = isSaving || isSelectedPublicationMissing
+  const selectedPublication = publications.find((pub) => pub.name === publicationName)
+  const { data: checkPrimaryKeysExistsData, isLoading: isLoadingCheck } = useCheckPrimaryKeysExists(
+    {
+      projectRef: project?.ref,
+      connectionString: project?.connectionString,
+      tables: selectedPublication?.tables ?? [],
+    },
+    { enabled: visible && !!selectedPublication }
+  )
+  const hasTablesWithNoPrimaryKeys = (checkPrimaryKeysExistsData?.offendingTables ?? []).length > 0
 
-  const onSubmit = async (data: z.infer<typeof FormSchema>) => {
+  const isSubmitDisabled =
+    isSaving || isSelectedPublicationMissing || isLoadingCheck || hasTablesWithNoPrimaryKeys
+
+  const submitPipeline = async (data: z.infer<typeof FormSchema>) => {
     if (!projectRef) return console.error('Project ref is required')
     if (!sourceId) return console.error('Source id is required')
     if (isSelectedPublicationMissing) {
@@ -236,6 +257,32 @@ export const DestinationPanel = ({
     }
   }
 
+  const onSubmit = async (data: z.infer<typeof FormSchema>) => {
+    if (!editMode) {
+      setPendingFormValues(data)
+      setShowDisclaimerDialog(true)
+      return
+    }
+
+    await submitPipeline(data)
+  }
+
+  const handleDisclaimerDialogChange = (open: boolean) => {
+    setShowDisclaimerDialog(open)
+    if (!open) {
+      setPendingFormValues(null)
+    }
+  }
+
+  const handleDisclaimerConfirm = async () => {
+    if (!pendingFormValues) return
+
+    const values = pendingFormValues
+    setPendingFormValues(null)
+    setShowDisclaimerDialog(false)
+    await submitPipeline(values)
+  }
+
   useEffect(() => {
     if (editMode && destinationData && pipelineData) {
       form.reset(defaultValues)
@@ -303,12 +350,13 @@ export const DestinationPanel = ({
                           <FormControl_Shadcn_>
                             <PublicationsComboBox
                               publications={publicationNames}
-                              loading={isLoadingPublications}
+                              isLoadingPublications={isLoadingPublications}
+                              isLoadingCheck={!!selectedPublication && isLoadingCheck}
                               field={field}
                               onNewPublicationClick={() => setPublicationPanelVisible(true)}
                             />
                           </FormControl_Shadcn_>
-                          {isSelectedPublicationMissing && (
+                          {isSelectedPublicationMissing ? (
                             <Admonition type="warning" className="mt-2 mb-0">
                               <p className="!leading-normal">
                                 The publication{' '}
@@ -317,7 +365,29 @@ export const DestinationPanel = ({
                                 another one.
                               </p>
                             </Admonition>
-                          )}
+                          ) : hasTablesWithNoPrimaryKeys ? (
+                            <Admonition type="warning" className="mt-2 mb-0">
+                              <p className="!leading-normal">
+                                Replication requires every table in the publication to have a
+                                primary key to work, which these tables are missing:
+                              </p>
+                              <ul className="list-disc pl-6 mb-2">
+                                {(checkPrimaryKeysExistsData?.offendingTables ?? []).map((x) => {
+                                  const value = `${x.schema}.${x.name}`
+                                  return (
+                                    <li key={value} className="!leading-normal">
+                                      <InlineLink href={`/project/${projectRef}/editor/${x.id}`}>
+                                        {value}
+                                      </InlineLink>
+                                    </li>
+                                  )
+                                })}
+                              </ul>
+                              <p className="!leading-normal">
+                                Ensure that these tables have primary keys first.
+                              </p>
+                            </Admonition>
+                          ) : null}
                         </FormItemLayout>
                       )}
                     />
@@ -487,6 +557,13 @@ export const DestinationPanel = ({
         </SheetContent>
       </Sheet>
 
+      <ReplicationDisclaimerDialog
+        open={showDisclaimerDialog}
+        onOpenChange={handleDisclaimerDialogChange}
+        isLoading={isSaving}
+        onConfirm={handleDisclaimerConfirm}
+      />
+
       <NewPublicationPanel
         visible={publicationPanelVisible}
         sourceId={sourceId}

apps/studio/components/interfaces/Database/Replication/PublicationsComboBox.tsx

@@ -1,7 +1,9 @@
 import { Check, ChevronsUpDown, Loader2, Plus } from 'lucide-react'
 import { useEffect, useState } from 'react'
+import { ControllerRenderProps } from 'react-hook-form'
 import {
   Button,
+  cn,
   Command_Shadcn_,
   CommandEmpty_Shadcn_,
   CommandGroup_Shadcn_,
@@ -14,18 +16,19 @@ import {
   PopoverTrigger_Shadcn_,
   ScrollArea,
 } from 'ui'
-import { ControllerRenderProps } from 'react-hook-form'
 
 interface PublicationsComboBoxProps {
   publications: string[]
-  loading: boolean
+  isLoadingPublications: boolean
+  isLoadingCheck: boolean
   onNewPublicationClick: () => void
   field: ControllerRenderProps<any, 'publicationName'>
 }
 
 const PublicationsComboBox = ({
   publications,
-  loading,
+  isLoadingPublications,
+  isLoadingCheck,
   onNewPublicationClick,
   field,
 }: PublicationsComboBoxProps) => {
@@ -62,13 +65,16 @@ const PublicationsComboBox = ({
         <Button
           type="default"
           size="medium"
-          className={`w-full [&>span]:w-full text-left`}
+          className={cn(
+            'w-full [&>span]:w-full text-left',
+            !selectedPublication && 'text-foreground-muted'
+          )}
           iconRight={
-            <ChevronsUpDown
-              className="text-foreground-muted"
-              strokeWidth={2}
-              size={14}
-            ></ChevronsUpDown>
+            isLoadingCheck ? (
+              <Loader2 className="animate-spin" size={14} />
+            ) : (
+              <ChevronsUpDown className="text-foreground-muted" strokeWidth={2} size={14} />
+            )
           }
           name={field.name}
           onBlur={field.onBlur}
@@ -82,10 +88,10 @@ const PublicationsComboBox = ({
             placeholder="Find publication..."
             value={searchTerm}
             onValueChange={handleSearchChange}
-          ></CommandInput_Shadcn_>
+          />
           <CommandList_Shadcn_>
             <CommandEmpty_Shadcn_>
-              {loading ? (
+              {isLoadingPublications ? (
                 <div className="flex items-center gap-2 text-center justify-center">
                   <Loader2 size={12} className="animate-spin" />
                   Loading...

apps/studio/components/interfaces/Database/Replication/ReplicationDisclaimerDialog.tsx

@@ -0,0 +1,77 @@
+import {
+  Button,
+  Dialog,
+  DialogContent,
+  DialogFooter,
+  DialogHeader,
+  DialogSection,
+  DialogSectionSeparator,
+  DialogTitle,
+} from 'ui'
+
+interface ReplicationDisclaimerDialogProps {
+  open: boolean
+  isLoading: boolean
+  onOpenChange: (value: boolean) => void
+  onConfirm: () => void
+}
+
+export const ReplicationDisclaimerDialog = ({
+  open,
+  isLoading,
+  onOpenChange,
+  onConfirm,
+}: ReplicationDisclaimerDialogProps) => {
+  return (
+    <Dialog open={open} onOpenChange={onOpenChange}>
+      <DialogContent className="sm:max-w-lg">
+        <DialogHeader>
+          <DialogTitle>Replication limitations</DialogTitle>
+        </DialogHeader>
+        <DialogSectionSeparator />
+        <DialogSection className="space-y-4 text-sm">
+          <p className="text-foreground">
+            Creating this replication pipeline will immediately start syncing data from your
+            publication into the destination. Make sure you understand the limitations of the system
+            before proceeding.
+          </p>
+
+          <div className="text-foreground-light">
+            <ul className="list-disc flex flex-col gap-y-1.5 pl-5 text-sm leading-snug">
+              <li>
+                <strong className="text-foreground">Custom data types replicate as strings.</strong>{' '}
+                Check that the destination can interpret those string values correctly.
+              </li>
+              <li>
+                <strong className="text-foreground">Generated columns are skipped.</strong> Replace
+                them with triggers or materialized views if you need the derived values downstream.
+              </li>
+              <li>
+                <strong className="text-foreground">
+                  FULL replica identity is strongly recommended.
+                </strong>{' '}
+                With FULL replica identity deletes and updates include the payload that is needed to
+                correctly apply those changes.
+              </li>
+              <li>
+                <strong className="text-foreground">Schema changes aren’t supported yet.</strong>{' '}
+                Plan for manual adjustments if you need to alter replicated tables.
+              </li>
+            </ul>
+          </div>
+        </DialogSection>
+
+        <DialogSectionSeparator />
+
+        <DialogFooter>
+          <Button type="default" disabled={isLoading} onClick={() => onOpenChange(false)}>
+            Cancel
+          </Button>
+          <Button loading={isLoading} onClick={onConfirm}>
+            Understood, start replication
+          </Button>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  )
+}

apps/studio/data/database/keys.ts

@@ -37,4 +37,8 @@ export const databaseKeys = {
     ['projects', projectRef, 'pgbouncer', 'status'] as const,
   pgbouncerConfig: (projectRef: string | undefined) =>
     ['projects', projectRef, 'pgbouncer', 'config'] as const,
+  checkPrimaryKeysExists: (
+    projectRef: string | undefined,
+    tables: { name: string; schema: string }[]
+  ) => ['projects', projectRef, 'check-primary-keys', tables] as const,
 }

apps/studio/data/database/primary-keys-exists-query.ts

@@ -0,0 +1,57 @@
+import { getCheckPrimaryKeysExistsSQL } from '@supabase/pg-meta/src/sql/studio/check-primary-keys-exists'
+import { useQuery, UseQueryOptions } from '@tanstack/react-query'
+
+import { executeSql } from 'data/sql/execute-sql-query'
+import type { ResponseError } from 'types'
+import { databaseKeys } from './keys'
+
+type CheckPrimaryKeysExistsVariables = {
+  projectRef?: string
+  connectionString?: string | null
+  tables: { name: string; schema: string }[]
+}
+
+type CheckPrimaryKeysExistResponse = {
+  id: string
+  name: string
+  schema: string
+  has_primary_key: boolean
+}[]
+
+export async function checkPrimaryKeysExists({
+  projectRef,
+  connectionString,
+  tables,
+}: CheckPrimaryKeysExistsVariables) {
+  if (!projectRef) throw new Error('Project ref is required')
+
+  const { result } = await executeSql({
+    projectRef,
+    connectionString,
+    sql: getCheckPrimaryKeysExistsSQL(tables),
+  })
+
+  return {
+    offendingTables: (result as CheckPrimaryKeysExistResponse).filter((x) => !x.has_primary_key),
+  }
+}
+
+export type CheckPrimaryKeysExistsData = Awaited<ReturnType<typeof checkPrimaryKeysExists>>
+export type CheckPrimaryKeysExistsError = ResponseError
+
+export const useCheckPrimaryKeysExists = <TData = CheckPrimaryKeysExistsData>(
+  { projectRef, connectionString, tables }: CheckPrimaryKeysExistsVariables,
+  {
+    enabled = true,
+    ...options
+  }: UseQueryOptions<CheckPrimaryKeysExistsData, CheckPrimaryKeysExistsError, TData> = {}
+) =>
+  useQuery<CheckPrimaryKeysExistsData, CheckPrimaryKeysExistsError, TData>(
+    databaseKeys.checkPrimaryKeysExists(projectRef, tables),
+    () => checkPrimaryKeysExists({ projectRef, connectionString, tables }),
+    {
+      retry: false,
+      enabled: enabled && typeof projectRef !== 'undefined' && tables.length > 0,
+      ...options,
+    }
+  )

packages/pg-meta/src/sql/studio/check-primary-keys-exists.ts

@@ -0,0 +1,20 @@
+export const getCheckPrimaryKeysExistsSQL = (tables: { name: string; schema: string }[]) => {
+  const formattedTables = tables.map((table) => `'${table.schema}.${table.name}'`).join(',')
+
+  return /* SQL */ `
+WITH targets(rel) AS (
+  SELECT unnest(ARRAY[${formattedTables}]::regclass[])
+)
+SELECT
+  c.oid AS id,
+  n.nspname AS schema,
+  c.relname AS name,
+  (con.conrelid IS NOT NULL) AS has_primary_key
+FROM targets t
+JOIN pg_class c ON c.oid = t.rel
+JOIN pg_namespace n ON n.oid = c.relnamespace
+LEFT JOIN pg_constraint con
+  ON con.conrelid = c.oid AND con.contype = 'p'
+ORDER BY n.nspname, c.relname;
+`
+}