From 6143441493991f511f18f104d1f89bc7533efdd9 Mon Sep 17 00:00:00 2001
From: Danny White <3104761+dnywh@users.noreply.github.com>
Date: Fri, 2 Oct 2026 10:34:49 +1000
Subject: [PATCH] fix(pipelines): clarify DuckLake destination setup (#51013)
## Problem
The DuckLake setup form makes it hard to choose between Supabase
projects and external connection details. Bucket creation, catalog
settings, and the guide do not clearly follow the setup flow.
## Solution
- Show **Configuration method** as two clear choices: **Select Supabase
projects** and **Enter connection details**.
- Group catalog and storage fields, move **Pool size** to **Advanced
settings**, and add **New bucket** to the bucket selector.
- Clarify the custom Postgres and S3 fields, including the metadata
schema, connection URL, and storage options.
- Update the [DuckLake destination
guide](https://docs-git-dnywh-ducklake-pipelines-setup-supabase.vercel.app/docs/guides/database/replication/pipelines/ducklake)
to follow the form and explain resource preparation and validation.
| Before | After |
| --- | --- |
| | |
## Review instructions
1. Open **Database > Pipelines > Add pipeline** and select **DuckLake**.
2. Select **Select Supabase projects**. Check the catalog and storage
fields, create a bucket from the bucket selector, and find **Pool size**
under **Advanced settings**.
3. Select **Enter connection details**. Check the Catalog URL, S3 URL
style, and Use SSL guidance.
4. Compare both routes with the [DuckLake destination
guide](https://docs-git-dnywh-ducklake-pipelines-setup-supabase.vercel.app/docs/guides/database/replication/pipelines/ducklake).
## Checklist
- [x] I have read
[CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md)
- [x] I used the `/edit-the-docs` skill and the docs [style
guide](https://github.com/supabase/supabase/tree/master/apps/docs/style-guide)
## Summary by CodeRabbit
* **New Features**
* DuckLake destinations support Supabase-managed projects or an existing
Postgres catalog with S3-compatible storage.
* Select a storage bucket using search, configure a metadata schema, and
access clearer guidance for catalog and storage settings.
* Advanced settings provide a connection pool size from 1 to 6, with a
default of 4. Credential fields include show and hide controls.
* **Documentation**
* Updated setup steps, configuration guidance, query credential details,
and troubleshooting instructions for both configuration modes.
---
.../replication/pipelines/ducklake.mdx | 62 ++-
.../DestinationForm/AdvancedSettings.tsx | 37 +-
.../DestinationForm.utils.test.ts | 10 +
.../DestinationForm/DestinationForm.utils.ts | 4 +-
.../DestinationFormFieldCopy.ts | 12 +-
.../DuckLake/DuckLake.schema.ts | 12 +-
.../DestinationForm/DuckLake/Fields.tsx | 436 ++++++++++--------
7 files changed, 325 insertions(+), 248 deletions(-)
diff --git a/apps/docs/content/guides/database/replication/pipelines/ducklake.mdx b/apps/docs/content/guides/database/replication/pipelines/ducklake.mdx
index f2c31fcdc92..917f68bcaef 100644
--- a/apps/docs/content/guides/database/replication/pipelines/ducklake.mdx
+++ b/apps/docs/content/guides/database/replication/pipelines/ducklake.mdx
@@ -18,7 +18,7 @@ Insert-only tables don't require a primary key or replica identity. Updates and
## Prepare DuckLake resources [#understand-the-ducklake-components]
-Prepare a Postgres catalog, object storage, and a compatible query engine:
+DuckLake stores metadata in a Postgres catalog and data in object storage. With **Select Supabase projects**, choose a project for the catalog, then a project and bucket for storage. The catalog and storage can use the same project. Pipelines creates the connection credentials. With **Enter connection details**, you provide the catalog URL and storage credentials. You also need a compatible query engine to read the replicated data.
| Component | Purpose |
| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
@@ -30,56 +30,54 @@ You can query replicated tables, but treat them and their underlying catalog and
## Configure DuckLake as a destination [#choose-a-configuration-mode]
-Choose a mode below for its resource requirements and configuration steps.
+Follow [Set up Pipelines](/docs/guides/database/replication/pipelines#setup-overview). In **Add pipeline**, select **DuckLake**, enter a **Pipeline name**, choose a **Publication**, and set **Initial sync**. Then choose how to configure the catalog and storage.
-Follow [Set up Pipelines](/docs/guides/database/replication/pipelines#setup-overview), select **DuckLake**, then choose **Use Supabase** or **Custom parameters** for the catalog and storage.
+### Select Supabase projects [#use-supabase]
-### Use Supabase
-
-Use this mode to back the DuckLake with Supabase projects. Pipelines provisions its own catalog and object-storage credentials when you create the destination.
+Choose Supabase projects for the Postgres catalog and Storage bucket. Pipelines creates the connection credentials when you create the destination.
Before you begin:
- Choose active, healthy, non-branch projects from the same organization for the catalog and storage. You can use the same project for both.
- Make sure your organization role can administer SQL in the catalog project and Storage in the storage project.
-- Create a private standard Storage bucket, or create one from the destination form.
-- Choose a metadata schema unique to this DuckLake. Use only letters, numbers, and underscores.
+- Choose a Files bucket for DuckLake data. You can create a private bucket from the destination form.
+- Choose an unused metadata schema name for this DuckLake. Pipelines creates the schema and its tables. Use only letters, numbers, and underscores. If the catalog project already uses the default `ducklake` schema for Warehouse, enter a different name.
- Keep catalog and storage in the same region when possible, near the [managed pipeline region](/docs/guides/database/replication/pipelines#region).
To configure the destination:
-1. Select **Use Supabase**.
-2. Choose the **Catalog project**, **Pool size**, and **Metadata schema**. Pool size allows `1` to `6` concurrent DuckDB connections; the default is `4`.
-3. Choose the **Storage project** and private **Bucket**.
-4. Click **Create and start pipeline** and complete the validation and cost confirmations.
+1. Select **Select Supabase projects** under **Configuration method**.
+2. Choose the **Catalog project** and **Metadata schema**.
+3. Choose the **Storage project** and **Bucket**. Select **New bucket** at the bottom of the bucket list if you haven't created one yet.
+4. Optionally adjust **Pool size** under **Advanced settings**. It allows 1 to 6 concurrent DuckDB connections; the default is 4.
+5. Click **Start pipeline**. Review any validation warnings, then confirm the estimated cost with **Create and start pipeline**.
Credential-provisioning warnings are expected before creation: catalog and Storage credentials are provisioned when you save the destination. Review the selected resources before proceeding.
-### Custom parameters
+### Enter connection details [#custom-parameters]
-Use this mode with a Postgres catalog and S3-compatible object storage that you control.
+Use this mode to connect an existing Postgres database and S3-compatible object storage. You provide their connection details and credentials.
Prepare the following resources:
-1. A Postgres database reachable from managed Pipelines. Create a dedicated user that can create and modify the DuckLake metadata schema and its tables.
+1. A Postgres database reachable from managed Pipelines. Create a dedicated user with permission to create the DuckLake metadata schema and its tables.
2. An S3-compatible bucket and a dedicated prefix for this DuckLake.
3. Object-storage credentials that can list, read, write, and delete objects under that prefix. Delete access is required for managed file cleanup.
-Use a new catalog metadata schema and data prefix for each destination. Reusing an existing schema or prefix can mix the metadata or files of different DuckLakes.
+Choose an unused metadata schema name and a new data prefix for each destination. Pipelines creates the schema and its tables. Initialising a DuckLake catalog in that schema beforehand triggers a validation warning; reusing its schema or prefix can mix metadata or files from different DuckLakes.
-Configure these fields in the destination form:
+Select **Enter connection details** under **Configuration method**, then configure these fields:
-- **Catalog URL**: A `postgres://` or `postgresql://` connection URL, including credentials and an `sslmode` appropriate for your provider
+- **Catalog URL**: A `postgres://` or `postgresql://` URL for your existing database, including credentials. If your provider requires TLS, append `?sslmode=require` after the database name, or `&sslmode=require` if the URL already has query parameters. Keep the database name from your provider's URL; it doesn't need to be `ducklake_catalog`.
- **Data path**: An `s3:///` URL
-- **Pool size**: From `1` to `6`; the default is `4`
- **S3 access key ID** and **S3 secret access key**: A credential pair for the data path
+- **S3 endpoint**: A publicly reachable provider endpoint without `http://` or `https://`
- **S3 region**: The storage provider's region
-- **S3 endpoint**: The provider endpoint without `http://` or `https://`
-- **S3 URL style**: `path` for Supabase Storage and many S3-compatible providers, or `vhost` for virtual-host-style addressing
-- **Use SSL**: Keep enabled for production endpoints
+- **S3 URL style**: Path style if the bucket is in the URL path, or virtual-host style if the bucket is in the hostname
+- **Use SSL**: Choose **On** for HTTPS, or **Off** if your provider requires HTTP
- **Metadata schema**: A unique Postgres schema for DuckLake metadata, using only letters, numbers, and underscores
-Click **Create and start pipeline** and complete the validation and cost confirmations.
+Optionally adjust **Pool size** under **Advanced settings**. It allows 1 to 6 concurrent DuckDB connections; the default is 4. Click **Start pipeline**, review the validation results, then confirm the estimated cost with **Create and start pipeline**.
The catalog URL and storage credentials are stored as secrets and aren't returned after creation. When editing the destination, leave a secret field empty to keep its stored value, or enter a new value to replace it.
@@ -95,7 +93,7 @@ A source `TRUNCATE` truncates the DuckLake table. A [table restart](/docs/guides
Connect DuckDB with its `ducklake` extension, or another compatible engine, to the same catalog and storage path. Query through the catalog; reading raw Parquet files can miss inlined changes, delete files, and the current snapshot.
-For **Use Supabase** mode, create separate read credentials for the selected catalog and Storage projects. The writer credentials generated for Pipelines aren't exposed. For **Custom parameters**, use separate read-only credentials when your catalog and storage provider support them.
+If you selected Supabase projects, create separate read credentials for the catalog and Storage projects. The writer credentials generated for Pipelines aren't exposed. If you entered connection details, use separate read-only credentials when your catalog and storage provider support them.
After attaching the catalog under an alias such as `my_ducklake`, source schemas and tables are available as qualified DuckLake tables:
@@ -163,14 +161,14 @@ For type changes, unsupported changes, and interrupted schema changes, see the s
## Troubleshooting
-| Issue | Resolution |
-| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Credential-provisioning warnings | Expected in [Use Supabase](#use-supabase) mode before saving. Review the selected resources. |
-| Catalog or storage validation fails | Check [custom parameters](#custom-parameters), credentials, connectivity, and permissions for the configured prefix. Local `file://` paths are unsupported. |
-| Metadata schema exists | Choose a new schema, unless intentionally reusing the same DuckLake and its corresponding data path. |
-| Inserts work but updates or deletes fail | Check [replica identity and published columns](#source-table-requirements). |
-| Queries omit changes or deleted rows | [Query through the catalog](#query-the-destination) with credentials for both catalog and storage. |
-| A schema change fails | Review [supported changes](#schema-change-support). Don't modify catalog tables or files manually. |
+| Issue | Resolution |
+| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Credential-provisioning warnings | Expected when you [select Supabase projects](#use-supabase) before saving. Review the selected resources. |
+| Catalog or storage validation fails | Check the [connection details](#custom-parameters), credentials, connectivity, and permissions for the configured prefix. Local `file://` paths are unsupported. |
+| Metadata schema exists | Choose a new schema, unless intentionally reusing the same DuckLake and its corresponding data path. |
+| Inserts work but updates or deletes fail | Check [replica identity and published columns](#source-table-requirements). |
+| Queries omit changes or deleted rows | [Query through the catalog](#query-the-destination) with credentials for both catalog and storage. |
+| A schema change fails | Review [supported changes](#schema-change-support). Don't modify catalog tables or files manually. |
Use [pipeline monitoring](/docs/guides/database/replication/pipelines-monitoring) to inspect errors. For unresolved failures, [contact support](/dashboard/support/new) with the pipeline ID and error details.
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
index 4c706a8ca98..996cb4366ee 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
@@ -23,6 +23,7 @@ import { DestinationType } from '../DestinationPanel.types'
import { TableOptions } from './BigQuery/TableOptions'
import {
DEFAULT_CONNECTION_POOL_SIZE,
+ DEFAULT_DUCKLAKE_POOL_SIZE,
DEFAULT_MAX_COPY_CONNECTIONS_PER_TABLE,
DEFAULT_MAX_FILL_MS,
DEFAULT_MAX_TABLE_SYNC_WORKERS,
@@ -55,11 +56,39 @@ export const AdvancedSettings = ({
Advanced settings
- Customize how the pipeline syncs and replicates data.
+ {type === 'DuckLake'
+ ? 'Adjust catalog connections and replication settings.'
+ : 'Customize how the pipeline syncs and replicates data.'}
+ {type === 'DuckLake' && (
+ (
+
+
+
+
+
+ )}
+ />
+ )}
+
workers
@@ -136,7 +165,7 @@ export const AdvancedSettings = ({
step={1}
value={field.value ?? ''}
onChange={handleNumberChange(field)}
- placeholder={`Default: ${DEFAULT_MAX_COPY_CONNECTIONS_PER_TABLE}`}
+ placeholder={String(DEFAULT_MAX_COPY_CONNECTIONS_PER_TABLE)}
/>
connections
@@ -201,7 +230,7 @@ export const AdvancedSettings = ({
step={1}
value={field.value ?? ''}
onChange={handleNumberChange(field)}
- placeholder={`Default: ${DEFAULT_CONNECTION_POOL_SIZE}`}
+ placeholder={String(DEFAULT_CONNECTION_POOL_SIZE)}
/>
connections
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
index 85d29ea363a..d21f4478114 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
@@ -311,6 +311,16 @@ const baseClickHouseFormData = {
}
describe('DestinationForm.utils DuckLake', () => {
+ it('uses the default pool size when the advanced field is cleared', () => {
+ const config = buildDestinationConfigForValidation({
+ projectRef: 'project-ref',
+ selectedType: 'DuckLake',
+ data: { ...baseDucklakeFormData, ducklakePoolSize: '' },
+ })
+
+ expect(config).toMatchObject({ ducklake: { poolSize: undefined } })
+ })
+
it('builds DuckLake validation config with required fields trimmed and blank optionals removed', () => {
const config = buildDestinationConfigForValidation({
projectRef: 'project-ref',
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
index 4f3fb97e1e2..007201431d8 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
@@ -332,7 +332,7 @@ const buildDucklakeConfig = (
catalogProjectRef: normalizeRequiredString(data.ducklakeCatalogProjectRef),
storageProjectRef: normalizeRequiredString(data.ducklakeStorageProjectRef),
bucket: normalizeRequiredString(data.ducklakeStorageBucket),
- poolSize: data.ducklakePoolSize,
+ poolSize: data.ducklakePoolSize === '' ? undefined : data.ducklakePoolSize,
metadataSchema: normalizeOptionalString(data.ducklakeMetadataSchema),
}
return supabaseConfig
@@ -341,7 +341,7 @@ const buildDucklakeConfig = (
const manualConfig: DucklakeManualDestinationConfig = {
catalogUrl: data.ducklakeCatalogUrl ?? '',
dataPath: data.ducklakeDataPath ?? '',
- poolSize: data.ducklakePoolSize,
+ poolSize: data.ducklakePoolSize === '' ? undefined : data.ducklakePoolSize,
s3AccessKeyId: normalizeRequiredString(data.ducklakeS3AccessKeyId),
s3SecretAccessKey: normalizeRequiredString(data.ducklakeS3SecretAccessKey),
s3Region: normalizeRequiredString(data.ducklakeS3Region),
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationFormFieldCopy.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationFormFieldCopy.ts
index 10fde3b4e7f..d264964e23e 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationFormFieldCopy.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationFormFieldCopy.ts
@@ -25,29 +25,29 @@ export const ANALYTICS_BUCKET_NAMESPACE_FIELD_COPY = {
export const DUCKLAKE_CATALOG_PROJECT_FIELD_COPY = {
label: 'Catalog project',
- description:
- "Pipelines connects to this project's Postgres instance to store the DuckLake catalog.",
+ description: 'Postgres project that stores this DuckLake’s metadata.',
} as const
export const DUCKLAKE_STORAGE_PROJECT_FIELD_COPY = {
label: 'Storage project',
- description: 'The project whose object storage holds the DuckLake data files.',
+ description: 'Supabase project that stores the DuckLake data files.',
} as const
export const DUCKLAKE_BUCKET_FIELD_COPY = {
label: 'Bucket',
- description: 'The bucket in which DuckLake data files will be stored.',
+ description: 'Files bucket for DuckLake data.',
} as const
export const DUCKLAKE_CATALOG_URL_FIELD_COPY = {
label: 'Catalog URL',
- createDescription: 'A Postgres connection string for the DuckLake catalog.',
+ createDescription:
+ 'Postgres URL for an existing database. Add TLS settings to the URL if required.',
editDescription: 'Stored catalog URL is hidden. Enter a new URL to replace it.',
} as const
export const DUCKLAKE_DATA_PATH_FIELD_COPY = {
label: 'Data path',
- description: 'An S3 path where DuckLake data files will be written.',
+ description: 'S3 path for DuckLake data files.',
} as const
export const SNOWFLAKE_ACCOUNT_ID_FIELD_COPY = {
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts
index 50a9ace4a91..e78683d1e3e 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts
@@ -12,10 +12,14 @@ export const DuckLakeFormSchema = z.object({
ducklakeCatalogUrl: z.string().optional(),
ducklakeDataPath: z.string().optional(),
ducklakePoolSize: z
- .number()
- .int()
- .min(1, 'Pool size must be greater than 0.')
- .max(6, 'Pool size must be 6 or less.')
+ .union([
+ z.literal(''),
+ z
+ .number()
+ .int()
+ .min(1, 'Pool size must be greater than 0.')
+ .max(6, 'Pool size must be 6 or less.'),
+ ])
.optional(),
ducklakeS3AccessKeyId: z.string().optional(),
ducklakeS3SecretAccessKey: z.string().optional(),
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/Fields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/Fields.tsx
index ea5231e0743..7fedb790004 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/Fields.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/Fields.tsx
@@ -1,12 +1,21 @@
-import { Check, Database, Eye, EyeOff, Plus, SlidersHorizontal } from 'lucide-react'
+import { Check, Eye, EyeOff, Plus } from 'lucide-react'
import { useMemo, useState } from 'react'
import { useWatch, type UseFormReturn } from 'react-hook-form'
import { toast } from 'sonner'
import {
Button,
cn,
+ ComboboxTrigger,
+ Command,
+ CommandEmpty,
+ CommandGroup,
+ CommandInput,
+ CommandItem,
+ CommandList,
+ CommandSeparator,
Dialog,
DialogContent,
+ DialogDescription,
DialogFooter,
DialogHeader,
DialogSection,
@@ -15,6 +24,12 @@ import {
FormControl,
FormField,
Input,
+ Popover,
+ PopoverContent,
+ PopoverTrigger,
+ RadioGroupStacked,
+ RadioGroupStackedItem,
+ ScrollArea,
Select,
SelectContent,
SelectGroup,
@@ -26,7 +41,7 @@ import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import { SelectionListState } from 'ui-patterns/SelectionListState'
-import { DEFAULT_DUCKLAKE_POOL_SIZE, STORED_SECRET_PLACEHOLDER } from '../DestinationForm.constants'
+import { STORED_SECRET_PLACEHOLDER } from '../DestinationForm.constants'
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
import {
DUCKLAKE_BUCKET_FIELD_COPY,
@@ -55,16 +70,15 @@ import { PROJECT_STATUS } from '@/lib/constants'
const DUCKLAKE_MODE_OPTIONS = [
{
value: DUCKLAKE_MODE_SUPABASE,
- icon: Database,
- label: 'Use Supabase',
+ label: 'Select Supabase projects',
description:
- 'Create or use a DuckLake backed by your Supabase projects. Catalog and storage are managed for you.',
+ 'Choose projects for the Postgres catalog and Storage bucket. They can be the same project; Pipelines creates credentials.',
},
{
value: DUCKLAKE_MODE_CUSTOM,
- icon: SlidersHorizontal,
- label: 'Custom parameters',
- description: 'Bring your own Postgres catalog and S3-compatible object storage credentials.',
+ label: 'Enter connection details',
+ description:
+ 'Provide a Postgres catalog URL and S3-compatible storage details and credentials.',
},
] as const
@@ -76,44 +90,20 @@ const DuckLakeModeSelector = ({
onChange: (value: DucklakeMode) => void
}) => {
return (
-