From 2ce01473bbe1df2f726936416119856b5dc6435f Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Fri, 19 Jun 2026 16:07:20 +0200 Subject: [PATCH] feat(replication): add better form to create ducklake replication (#47069) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What kind of change does this PR introduce? Add support to use supabase projects as a pg catalog and storage when adding a ducklake replication. ## What is the current behavior? Only simple form with raw input text for custom parameters is available. ## What is the new behavior? Being able to select supabase project to directly use projects in supabase for the ducklake. I also fixed a warning we had in the console for this form (cf screenshot) ## Additional context [API Changes ](https://github.com/supabase/platform/pull/34282) https://github.com/user-attachments/assets/4ff9ee65-6ba4-4f17-9ea1-9aebad34171c Capture d’écran 2026-06-18 à 09 58
50 ### Test scenario Follow the screencast, go to your supabase project (better if it's in ap-southeast-1) Create a test table with 1 row for example -> Database -> Replication -> New destination -> Select ducklake and use supabase option -> Keep the same current supabase project selected for both catalog and storage -> Create destination -> You'll get a warning about the storage and credentials -> Confirm creation -> Wait until it's in status Running, if it's runing then it works ## Summary by CodeRabbit ## Release Notes * **New Features** * Added DuckLake replication destination with **Use Supabase** and **Custom parameters** modes. * Added DuckLake bucket selection with a **“New bucket”** creation dialog. * Added/expanded BigQuery, Analytics Bucket, and Snowflake destination configuration. * **Improvements** * Updated DuckLake create vs edit behavior: mode selection is hidden in edit mode and configuration is mapped correctly for the selected variant. * Enhanced field-level validation (including whitespace-only handling) and added clearer validation issue messages. * Added a cross-region warning for DuckLake when catalog and storage regions differ. --------- Signed-off-by: Benjamin <5719034+bnjjj@users.noreply.github.com> Co-authored-by: Joshen Lim --- .../AnalyticsBucket/AnalyticsBucket.utils.ts | 48 + .../AnalyticsBucket/Fields.tsx | 418 +++++ .../BigQuery/BigQuery.utils.ts | 19 + .../DestinationForm/BigQuery/Fields.tsx | 68 + .../DestinationForm/DestinationForm.schema.ts | 8 + .../DestinationForm.utils.test.ts | 241 ++- .../DestinationForm/DestinationForm.utils.ts | 345 ++-- .../DestinationPanelFields.tsx | 896 --------- .../DuckLake/DuckLake.constants.ts | 5 + .../DuckLake/DuckLake.utils.ts | 137 ++ .../DestinationForm/DuckLake/Fields.tsx | 829 +++++++++ .../DestinationForm/NewPublicationPanel.tsx | 10 +- .../DestinationForm/Snowflake/Fields.tsx | 170 ++ .../Snowflake/Snowflake.utils.ts | 36 + .../DestinationForm/index.tsx | 161 +- .../DestinationTypeSelection.tsx | 2 +- ...eate-destination-pipeline-mutation.test.ts | 81 + .../create-destination-pipeline-mutation.ts | 111 +- .../update-destination-pipeline-mutation.ts | 35 +- .../validate-destination-mutation.ts | 32 +- packages/api-types/types/platform.d.ts | 1603 +++++++++++++---- 21 files changed, 3550 insertions(+), 1705 deletions(-) create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.utils.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/Fields.tsx delete mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationPanelFields.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.constants.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.utils.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/Fields.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/Snowflake/Fields.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/Snowflake/Snowflake.utils.ts create mode 100644 apps/studio/data/replication/create-destination-pipeline-mutation.test.ts diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.utils.ts new file mode 100644 index 00000000000..3fe1f083c9a --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.utils.ts @@ -0,0 +1,48 @@ +import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from '../DestinationForm.constants' +import { type DestinationPanelSchemaType } from '../DestinationForm.schema' + +type AnalyticsBucketFieldPath = + | 'warehouseName' + | 'namespace' + | 'newNamespaceName' + | 's3Region' + | 's3AccessKeyId' + | 's3SecretAccessKey' + +export type AnalyticsBucketValidationIssue = { + path: AnalyticsBucketFieldPath + message: string +} + +// Fields that are always required regardless of the namespace / access key selections. +const ANALYTICS_BUCKET_REQUIRED_FIELDS: AnalyticsBucketValidationIssue[] = [ + { path: 'warehouseName', message: 'Bucket is required' }, + { path: 's3Region', message: 'S3 Region is required' }, + { path: 's3AccessKeyId', message: 'S3 Access Key ID is required' }, +] + +export const getAnalyticsBucketValidationIssues = ( + data: Pick +): AnalyticsBucketValidationIssue[] => { + const issues = ANALYTICS_BUCKET_REQUIRED_FIELDS.filter(({ path }) => !data[path]?.trim().length) + + const isCreatingNewNamespace = data.namespace === CREATE_NEW_NAMESPACE + const hasValidNamespace = + (data.namespace?.trim().length && !isCreatingNewNamespace) || + (isCreatingNewNamespace && data.newNamespaceName?.trim().length) + + if (!hasValidNamespace) { + issues.push( + isCreatingNewNamespace + ? { path: 'newNamespaceName', message: 'Namespace name is required' } + : { path: 'namespace', message: 'Namespace is required' } + ) + } + + // Creating a new key generates the secret later, so only require it for existing keys. + if (data.s3AccessKeyId !== CREATE_NEW_KEY && !data.s3SecretAccessKey?.trim().length) { + issues.push({ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' }) + } + + return issues +} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx new file mode 100644 index 00000000000..ffb4ce2ecd7 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx @@ -0,0 +1,418 @@ +import { PermissionAction } from '@supabase/shared-types/out/constants' +import { useParams } from 'common' +import { Eye, EyeOff, Loader2 } from 'lucide-react' +import { useState } from 'react' +import type { UseFormReturn } from 'react-hook-form' +import { + Button, + FormControl, + FormField, + Input, + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectSeparator, + SelectTrigger, + WarningIcon, +} from 'ui' +import { Admonition } from 'ui-patterns' +import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input' +import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' + +import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from '../DestinationForm.constants' +import type { DestinationPanelSchemaType } from '../DestinationForm.schema' +import { InlineLink } from '@/components/ui/InlineLink' +import { useAPIKeys } from '@/data/api-keys/api-keys-query' +import { useAnalyticsBucketsQuery } from '@/data/storage/analytics-buckets-query' +import { useIcebergNamespacesQuery } from '@/data/storage/iceberg-namespaces-query' +import { useStorageCredentialsQuery } from '@/data/storage/s3-access-key-query' +import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions' + +/** + * [Joshen] JFYI I'd foresee a possible UX friction point here regarding S3 access key IDs and secret access keys + * - We'd allow users to select access key IDs via a dropdown here, but require a text input for secret access keys + * - Chances are most users wouldn't have the corresponding secret access key for the selected key ID at the top of their heads + * - So highly likely may have to default to "Create a new key" -> which from here they won't know the secret access key thereafter + * - And it'll end up just creating more keys for each destination + * Ideal scenario: Just select an access key ID, we then apply the secret access key in the PATCH request, so FE has no + * context of the secret access key at any point + */ +export const AnalyticsBucketFields = ({ + form, + setIsFormInteracting, + onSelectNewBucket, +}: { + form: UseFormReturn + setIsFormInteracting: (value: boolean) => void + onSelectNewBucket: () => void +}) => { + const { warehouseName, s3AccessKeyId, namespace } = form.watch() + const [showCatalogToken, setShowCatalogToken] = useState(false) + const [showSecretAccessKey, setShowSecretAccessKey] = useState(false) + + const { ref: projectRef } = useParams() + + const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*') + const { data: apiKeysData } = useAPIKeys( + { projectRef, reveal: true }, + { enabled: canReadAPIKeys } + ) + const { serviceKey } = apiKeysData ?? {} + const serviceApiKey = serviceKey?.api_key ?? '' + + const { + data: keysData, + isSuccess: isSuccessKeys, + isPending: isLoadingKeys, + isError: isErrorKeys, + } = useStorageCredentialsQuery({ projectRef }) + const s3Keys = keysData?.data ?? [] + const keyNoLongerExists = + (s3AccessKeyId ?? '').length > 0 && + s3AccessKeyId !== CREATE_NEW_KEY && + !s3Keys.find((k) => k.access_key === s3AccessKeyId) + + const { + data: analyticsBuckets = [], + isPending: isLoadingBuckets, + isError: isErrorBuckets, + } = useAnalyticsBucketsQuery({ projectRef }) + + const canSelectNamespace = !!warehouseName && !!serviceApiKey + + const { + data: namespaces = [], + isPending: isLoadingNamespaces, + isError: isErrorNamespaces, + } = useIcebergNamespacesQuery( + { projectRef, warehouse: warehouseName }, + { enabled: !!serviceApiKey } + ) + + return ( +
+

Analytics Bucket settings

+ +
+ ( + + {isLoadingBuckets ? ( + + ) : isErrorBuckets ? ( + + ) : ( + + + + )} + + )} + /> + + ( + + {isLoadingNamespaces && canSelectNamespace ? ( + + ) : isErrorNamespaces ? ( + + ) : ( + + + + )} + + )} + /> + + {namespace === CREATE_NEW_NAMESPACE && ( + ( + + + + + + )} + /> + )} + + ( + + Automatically retrieved from your project's{' '} + + service role key + + + } + > + +
+ ) : null + } + /> + + )} + /> + + ( + +

+ Access keys are managed in your Storage{' '} + S3 settings +

+ + {isSuccessKeys && keyNoLongerExists && ( + + Please select another key or create a new set, as this destination will + not work otherwise. S3 access keys can be managed in your{' '} + + storage settings + + . + + } + /> + )} + + {s3AccessKeyId === CREATE_NEW_KEY && ( + + )} +
+ } + > + {isLoadingKeys ? ( + + ) : isErrorKeys ? ( + + ) : ( + + + + )} + + )} + /> + + {s3AccessKeyId !== CREATE_NEW_KEY && ( + ( + + + + +