mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 09:25:06 +03:00
feat(replication): add better form to create ducklake replication (#47069)
## What kind of change does this PR introduce? Add support to use supabase projects as a pg catalog and storage when adding a ducklake replication. ## What is the current behavior? Only simple form with raw input text for custom parameters is available. ## What is the new behavior? Being able to select supabase project to directly use projects in supabase for the ducklake. I also fixed a warning we had in the console for this form (cf screenshot) ## Additional context [API Changes ](https://github.com/supabase/platform/pull/34282) https://github.com/user-attachments/assets/4ff9ee65-6ba4-4f17-9ea1-9aebad34171c <img width="862" height="228" alt="Capture d’écran 2026-06-18 à 09 58 50" src="https://github.com/user-attachments/assets/1592c3be-807e-426f-9a5a-84979e05d93c" /> ### Test scenario Follow the screencast, go to your supabase project (better if it's in ap-southeast-1) Create a test table with 1 row for example -> Database -> Replication -> New destination -> Select ducklake and use supabase option -> Keep the same current supabase project selected for both catalog and storage -> Create destination -> You'll get a warning about the storage and credentials -> Confirm creation -> Wait until it's in status Running, if it's runing then it works <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Release Notes * **New Features** * Added DuckLake replication destination with **Use Supabase** and **Custom parameters** modes. * Added DuckLake bucket selection with a **“New bucket”** creation dialog. * Added/expanded BigQuery, Analytics Bucket, and Snowflake destination configuration. * **Improvements** * Updated DuckLake create vs edit behavior: mode selection is hidden in edit mode and configuration is mapped correctly for the selected variant. * Enhanced field-level validation (including whitespace-only handling) and added clearer validation issue messages. * Added a cross-region warning for DuckLake when catalog and storage regions differ. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: Benjamin <5719034+bnjjj@users.noreply.github.com> Co-authored-by: Joshen Lim <joshenlimek@gmail.com>
This commit is contained in:
Coenen BenjaminandJoshen Lim authored and GitHub committed
1 parent
dfcacc8aac
commit
2ce01473bb
21 files changed
+3550
-1705
No files matched your search
+48
@@ -0,0 +1,48 @@
|
||||
import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from '../DestinationForm.constants'
|
||||
import { type DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
|
||||
type AnalyticsBucketFieldPath =
|
||||
| 'warehouseName'
|
||||
| 'namespace'
|
||||
| 'newNamespaceName'
|
||||
| 's3Region'
|
||||
| 's3AccessKeyId'
|
||||
| 's3SecretAccessKey'
|
||||
|
||||
export type AnalyticsBucketValidationIssue = {
|
||||
path: AnalyticsBucketFieldPath
|
||||
message: string
|
||||
}
|
||||
|
||||
// Fields that are always required regardless of the namespace / access key selections.
|
||||
const ANALYTICS_BUCKET_REQUIRED_FIELDS: AnalyticsBucketValidationIssue[] = [
|
||||
{ path: 'warehouseName', message: 'Bucket is required' },
|
||||
{ path: 's3Region', message: 'S3 Region is required' },
|
||||
{ path: 's3AccessKeyId', message: 'S3 Access Key ID is required' },
|
||||
]
|
||||
|
||||
export const getAnalyticsBucketValidationIssues = (
|
||||
data: Pick<DestinationPanelSchemaType, AnalyticsBucketFieldPath>
|
||||
): AnalyticsBucketValidationIssue[] => {
|
||||
const issues = ANALYTICS_BUCKET_REQUIRED_FIELDS.filter(({ path }) => !data[path]?.trim().length)
|
||||
|
||||
const isCreatingNewNamespace = data.namespace === CREATE_NEW_NAMESPACE
|
||||
const hasValidNamespace =
|
||||
(data.namespace?.trim().length && !isCreatingNewNamespace) ||
|
||||
(isCreatingNewNamespace && data.newNamespaceName?.trim().length)
|
||||
|
||||
if (!hasValidNamespace) {
|
||||
issues.push(
|
||||
isCreatingNewNamespace
|
||||
? { path: 'newNamespaceName', message: 'Namespace name is required' }
|
||||
: { path: 'namespace', message: 'Namespace is required' }
|
||||
)
|
||||
}
|
||||
|
||||
// Creating a new key generates the secret later, so only require it for existing keys.
|
||||
if (data.s3AccessKeyId !== CREATE_NEW_KEY && !data.s3SecretAccessKey?.trim().length) {
|
||||
issues.push({ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' })
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
+418
@@ -0,0 +1,418 @@
|
||||
import { PermissionAction } from '@supabase/shared-types/out/constants'
|
||||
import { useParams } from 'common'
|
||||
import { Eye, EyeOff, Loader2 } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
import type { UseFormReturn } from 'react-hook-form'
|
||||
import {
|
||||
Button,
|
||||
FormControl,
|
||||
FormField,
|
||||
Input,
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectGroup,
|
||||
SelectItem,
|
||||
SelectSeparator,
|
||||
SelectTrigger,
|
||||
WarningIcon,
|
||||
} from 'ui'
|
||||
import { Admonition } from 'ui-patterns'
|
||||
import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
|
||||
import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from '../DestinationForm.constants'
|
||||
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
import { InlineLink } from '@/components/ui/InlineLink'
|
||||
import { useAPIKeys } from '@/data/api-keys/api-keys-query'
|
||||
import { useAnalyticsBucketsQuery } from '@/data/storage/analytics-buckets-query'
|
||||
import { useIcebergNamespacesQuery } from '@/data/storage/iceberg-namespaces-query'
|
||||
import { useStorageCredentialsQuery } from '@/data/storage/s3-access-key-query'
|
||||
import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
|
||||
|
||||
/**
|
||||
* [Joshen] JFYI I'd foresee a possible UX friction point here regarding S3 access key IDs and secret access keys
|
||||
* - We'd allow users to select access key IDs via a dropdown here, but require a text input for secret access keys
|
||||
* - Chances are most users wouldn't have the corresponding secret access key for the selected key ID at the top of their heads
|
||||
* - So highly likely may have to default to "Create a new key" -> which from here they won't know the secret access key thereafter
|
||||
* - And it'll end up just creating more keys for each destination
|
||||
* Ideal scenario: Just select an access key ID, we then apply the secret access key in the PATCH request, so FE has no
|
||||
* context of the secret access key at any point
|
||||
*/
|
||||
export const AnalyticsBucketFields = ({
|
||||
form,
|
||||
setIsFormInteracting,
|
||||
onSelectNewBucket,
|
||||
}: {
|
||||
form: UseFormReturn<DestinationPanelSchemaType>
|
||||
setIsFormInteracting: (value: boolean) => void
|
||||
onSelectNewBucket: () => void
|
||||
}) => {
|
||||
const { warehouseName, s3AccessKeyId, namespace } = form.watch()
|
||||
const [showCatalogToken, setShowCatalogToken] = useState(false)
|
||||
const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
|
||||
|
||||
const { ref: projectRef } = useParams()
|
||||
|
||||
const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
|
||||
const { data: apiKeysData } = useAPIKeys(
|
||||
{ projectRef, reveal: true },
|
||||
{ enabled: canReadAPIKeys }
|
||||
)
|
||||
const { serviceKey } = apiKeysData ?? {}
|
||||
const serviceApiKey = serviceKey?.api_key ?? ''
|
||||
|
||||
const {
|
||||
data: keysData,
|
||||
isSuccess: isSuccessKeys,
|
||||
isPending: isLoadingKeys,
|
||||
isError: isErrorKeys,
|
||||
} = useStorageCredentialsQuery({ projectRef })
|
||||
const s3Keys = keysData?.data ?? []
|
||||
const keyNoLongerExists =
|
||||
(s3AccessKeyId ?? '').length > 0 &&
|
||||
s3AccessKeyId !== CREATE_NEW_KEY &&
|
||||
!s3Keys.find((k) => k.access_key === s3AccessKeyId)
|
||||
|
||||
const {
|
||||
data: analyticsBuckets = [],
|
||||
isPending: isLoadingBuckets,
|
||||
isError: isErrorBuckets,
|
||||
} = useAnalyticsBucketsQuery({ projectRef })
|
||||
|
||||
const canSelectNamespace = !!warehouseName && !!serviceApiKey
|
||||
|
||||
const {
|
||||
data: namespaces = [],
|
||||
isPending: isLoadingNamespaces,
|
||||
isError: isErrorNamespaces,
|
||||
} = useIcebergNamespacesQuery(
|
||||
{ projectRef, warehouse: warehouseName },
|
||||
{ enabled: !!serviceApiKey }
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">Analytics Bucket settings</p>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="warehouseName"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="Bucket"
|
||||
layout="horizontal"
|
||||
description="The Analytics Bucket where data will be stored"
|
||||
>
|
||||
{isLoadingBuckets ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving buckets
|
||||
</Button>
|
||||
) : isErrorBuckets ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve buckets
|
||||
</Button>
|
||||
) : (
|
||||
<FormControl>
|
||||
<Select
|
||||
value={field.value}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'new-bucket') {
|
||||
onSelectNewBucket()
|
||||
} else {
|
||||
setIsFormInteracting(true)
|
||||
field.onChange(value)
|
||||
// [Joshen] Ideally should select the first namespace of the selected bucket
|
||||
form.setValue('namespace', '')
|
||||
}
|
||||
}}
|
||||
>
|
||||
<SelectTrigger>{field.value || 'Select a bucket'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{analyticsBuckets.length === 0 ? (
|
||||
<SelectItem value="__no_buckets__" disabled>
|
||||
No buckets available
|
||||
</SelectItem>
|
||||
) : (
|
||||
analyticsBuckets.map((bucket) => (
|
||||
<SelectItem key={bucket.name} value={bucket.name}>
|
||||
{bucket.name}
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
<SelectSeparator />
|
||||
<SelectItem value="new-bucket">Create a new bucket</SelectItem>
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
)}
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="namespace"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="Namespace"
|
||||
layout="horizontal"
|
||||
description="The namespace within the bucket where tables will be organized"
|
||||
>
|
||||
{isLoadingNamespaces && canSelectNamespace ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving namespaces
|
||||
</Button>
|
||||
) : isErrorNamespaces ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve namespaces
|
||||
</Button>
|
||||
) : (
|
||||
<FormControl>
|
||||
<Select
|
||||
value={field.value}
|
||||
onValueChange={(value) => {
|
||||
setIsFormInteracting(true)
|
||||
field.onChange(value)
|
||||
}}
|
||||
disabled={!canSelectNamespace}
|
||||
>
|
||||
<SelectTrigger>
|
||||
{!canSelectNamespace
|
||||
? 'Select a warehouse first'
|
||||
: field.value === CREATE_NEW_NAMESPACE
|
||||
? 'Create a new namespace'
|
||||
: field.value || 'Select a namespace'}
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{namespaces.length === 0 ? (
|
||||
<SelectItem value="__no_namespaces__" disabled>
|
||||
No namespaces available
|
||||
</SelectItem>
|
||||
) : (
|
||||
namespaces.map((namespace) => (
|
||||
<SelectItem key={namespace} value={namespace}>
|
||||
{namespace}
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
<SelectSeparator />
|
||||
<SelectItem key={CREATE_NEW_NAMESPACE} value={CREATE_NEW_NAMESPACE}>
|
||||
Create a new namespace
|
||||
</SelectItem>
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
)}
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
{namespace === CREATE_NEW_NAMESPACE && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="newNamespaceName"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="New Namespace Name"
|
||||
layout="horizontal"
|
||||
description="A unique name for the new namespace"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="new_namespace" value={field.value || ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="catalogToken"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Catalog Token"
|
||||
description={
|
||||
<>
|
||||
Automatically retrieved from your project's{' '}
|
||||
<InlineLink href={`/project/${projectRef}/settings/api-keys`}>
|
||||
service role key
|
||||
</InlineLink>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<PasswordInput
|
||||
disabled
|
||||
value={field.value}
|
||||
type={showCatalogToken ? 'text' : 'password'}
|
||||
placeholder="Auto-populated"
|
||||
actions={
|
||||
serviceApiKey ? (
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
icon={showCatalogToken ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowCatalogToken(!showCatalogToken)}
|
||||
/>
|
||||
</div>
|
||||
) : null
|
||||
}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="s3AccessKeyId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Access Key ID"
|
||||
description={
|
||||
<div className="flex flex-col gap-y-2">
|
||||
<p>
|
||||
Access keys are managed in your Storage{' '}
|
||||
<InlineLink href={`/project/${projectRef}/storage/s3`}>S3 settings</InlineLink>
|
||||
</p>
|
||||
|
||||
{isSuccessKeys && keyNoLongerExists && (
|
||||
<Admonition
|
||||
type="warning"
|
||||
title="Unable to find access key ID in project"
|
||||
description={
|
||||
<>
|
||||
Please select another key or create a new set, as this destination will
|
||||
not work otherwise. S3 access keys can be managed in your{' '}
|
||||
<InlineLink href={`/project/${projectRef}/storage/files/settings`}>
|
||||
storage settings
|
||||
</InlineLink>
|
||||
.
|
||||
</>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{s3AccessKeyId === CREATE_NEW_KEY && (
|
||||
<Admonition
|
||||
type="default"
|
||||
description="A new set of S3 access keys will be created."
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
>
|
||||
{isLoadingKeys ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving keys
|
||||
</Button>
|
||||
) : isErrorKeys ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve keys
|
||||
</Button>
|
||||
) : (
|
||||
<FormControl>
|
||||
<Select value={field.value} onValueChange={field.onChange}>
|
||||
<SelectTrigger>
|
||||
{field.value === CREATE_NEW_KEY
|
||||
? 'Create a new key'
|
||||
: (field.value ?? '').length === 0
|
||||
? 'Select an access key ID'
|
||||
: field.value}
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{s3Keys.map((key) => (
|
||||
<SelectItem key={key.id} value={key.access_key}>
|
||||
{key.access_key}
|
||||
<p className="text-foreground-lighter">{key.description}</p>
|
||||
</SelectItem>
|
||||
))}
|
||||
<SelectSeparator />
|
||||
<SelectItem key={CREATE_NEW_KEY} value={CREATE_NEW_KEY}>
|
||||
Create a new key
|
||||
</SelectItem>
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
)}
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
{s3AccessKeyId !== CREATE_NEW_KEY && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="s3SecretAccessKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Secret Access Key"
|
||||
className="relative"
|
||||
description="The secret key corresponding to your selected access key ID."
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
type={showSecretAccessKey ? 'text' : 'password'}
|
||||
value={field.value ?? ''}
|
||||
placeholder="Provide the secret access key"
|
||||
/>
|
||||
</FormControl>
|
||||
<Button
|
||||
variant="default"
|
||||
icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
|
||||
className="w-7 absolute right-1 top-[4px]"
|
||||
onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import { type DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
|
||||
type BigQueryFieldPath = 'projectId' | 'datasetId' | 'serviceAccountKey'
|
||||
|
||||
export type BigQueryValidationIssue = {
|
||||
path: BigQueryFieldPath
|
||||
message: string
|
||||
}
|
||||
|
||||
const BIGQUERY_REQUIRED_FIELDS: { path: BigQueryFieldPath; message: string }[] = [
|
||||
{ path: 'projectId', message: 'Project ID is required' },
|
||||
{ path: 'datasetId', message: 'Dataset ID is required' },
|
||||
{ path: 'serviceAccountKey', message: 'Service Account Key is required' },
|
||||
]
|
||||
|
||||
export const getBigQueryValidationIssues = (
|
||||
data: Pick<DestinationPanelSchemaType, BigQueryFieldPath>
|
||||
): BigQueryValidationIssue[] =>
|
||||
BIGQUERY_REQUIRED_FIELDS.filter(({ path }) => !data[path]?.trim().length)
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
import type { UseFormReturn } from 'react-hook-form'
|
||||
import { FormControl, FormField, Input, TextArea } from 'ui'
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
|
||||
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
|
||||
export const BigQueryFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">BigQuery settings</p>
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="projectId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Project ID"
|
||||
description="The Google Cloud project ID where data will be sent"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="my-gcp-project" />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="datasetId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="Dataset ID"
|
||||
layout="horizontal"
|
||||
description="The BigQuery dataset where replicated tables will be created"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="my_dataset" />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="serviceAccountKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Service Account Key"
|
||||
description="Service account credentials JSON for authenticating with BigQuery"
|
||||
>
|
||||
<FormControl>
|
||||
<TextArea
|
||||
{...field}
|
||||
rows={5}
|
||||
maxLength={5000}
|
||||
placeholder='{"type": "service_account", "project_id": "...", ...}'
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+8
@@ -33,6 +33,14 @@ export const DestinationPanelFormSchema = z.object({
|
||||
s3SecretAccessKey: z.string().optional(),
|
||||
s3Region: z.string().optional(),
|
||||
// DuckLake fields
|
||||
// `supabase` mode picks Supabase projects for catalog + storage (managed), while `custom`
|
||||
// mode keeps the manual PostgreSQL catalog URL + S3-compatible credentials.
|
||||
ducklakeMode: z.enum(['supabase', 'custom']).optional(),
|
||||
// DuckLake "Use Supabase" fields
|
||||
ducklakeCatalogProjectRef: z.string().optional(),
|
||||
ducklakeStorageProjectRef: z.string().optional(),
|
||||
ducklakeStorageBucket: z.string().optional(),
|
||||
// DuckLake "Custom parameters" fields
|
||||
ducklakeCatalogUrl: z.string().optional(),
|
||||
ducklakeDataPath: z.string().optional(),
|
||||
ducklakePoolSize: z.number().int().min(1).max(6).optional(),
|
||||
|
||||
+239
-2
@@ -1,11 +1,14 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsBucket.utils'
|
||||
import { getBigQueryValidationIssues } from './BigQuery/BigQuery.utils'
|
||||
import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
|
||||
import {
|
||||
buildDestinationConfig,
|
||||
buildDestinationConfigForValidation,
|
||||
getDucklakeValidationIssues,
|
||||
getSnowflakeValidationIssues,
|
||||
} from './DestinationForm.utils'
|
||||
import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
|
||||
import { getSnowflakeValidationIssues } from './Snowflake/Snowflake.utils'
|
||||
|
||||
const baseDucklakeFormData = {
|
||||
name: 'DuckLake Destination',
|
||||
@@ -154,6 +157,27 @@ describe('DestinationForm.utils DuckLake', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('treats whitespace-only values as missing', () => {
|
||||
const issues = getDucklakeValidationIssues({
|
||||
ducklakeCatalogUrl: ' ',
|
||||
ducklakeDataPath: '\t',
|
||||
ducklakeS3AccessKeyId: ' ',
|
||||
ducklakeS3SecretAccessKey: ' ',
|
||||
ducklakeS3Region: '\n',
|
||||
ducklakeS3Endpoint: ' ',
|
||||
ducklakeMetadataSchema: '',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([
|
||||
{ path: 'ducklakeCatalogUrl', message: 'Catalog URL is required' },
|
||||
{ path: 'ducklakeDataPath', message: 'Data path is required' },
|
||||
{ path: 'ducklakeS3AccessKeyId', message: 'S3 Access Key ID is required' },
|
||||
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 Secret Access Key is required' },
|
||||
{ path: 'ducklakeS3Region', message: 'S3 Region is required' },
|
||||
{ path: 'ducklakeS3Endpoint', message: 'S3 Endpoint is required' },
|
||||
])
|
||||
})
|
||||
|
||||
it('returns format errors for invalid DuckLake values', () => {
|
||||
const issues = getDucklakeValidationIssues({
|
||||
ducklakeCatalogUrl: 'mysql://catalog',
|
||||
@@ -200,6 +224,100 @@ describe('DestinationForm.utils DuckLake', () => {
|
||||
})
|
||||
})
|
||||
|
||||
const baseDucklakeSupabaseFormData = {
|
||||
...baseDucklakeFormData,
|
||||
ducklakeMode: 'supabase' as const,
|
||||
ducklakeCatalogProjectRef: 'catalog-ref',
|
||||
ducklakeStorageProjectRef: 'storage-ref',
|
||||
ducklakeStorageBucket: 'ducklake-data',
|
||||
}
|
||||
|
||||
describe('DestinationForm.utils DuckLake (Use Supabase)', () => {
|
||||
it('builds DuckLake validation config from project refs in supabase mode', () => {
|
||||
const config = buildDestinationConfigForValidation({
|
||||
projectRef: 'project-ref',
|
||||
selectedType: 'DuckLake',
|
||||
data: baseDucklakeSupabaseFormData,
|
||||
})
|
||||
|
||||
expect(config).toEqual({
|
||||
ducklake: {
|
||||
catalogProjectRef: 'catalog-ref',
|
||||
storageProjectRef: 'storage-ref',
|
||||
bucket: 'ducklake-data',
|
||||
poolSize: 4,
|
||||
metadataSchema: 'ducklake_metadata',
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('builds DuckLake submit config from project refs in supabase mode', async () => {
|
||||
const createS3AccessKey = vi.fn()
|
||||
const resolveNamespace = vi.fn()
|
||||
|
||||
const config = await buildDestinationConfig({
|
||||
projectRef: 'project-ref',
|
||||
selectedType: 'DuckLake',
|
||||
data: baseDucklakeSupabaseFormData,
|
||||
createS3AccessKey,
|
||||
resolveNamespace,
|
||||
})
|
||||
|
||||
expect(config).toEqual({
|
||||
ducklake: {
|
||||
catalogProjectRef: 'catalog-ref',
|
||||
storageProjectRef: 'storage-ref',
|
||||
bucket: 'ducklake-data',
|
||||
poolSize: 4,
|
||||
metadataSchema: 'ducklake_metadata',
|
||||
},
|
||||
})
|
||||
expect(createS3AccessKey).not.toHaveBeenCalled()
|
||||
expect(resolveNamespace).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns required-field errors for missing supabase selections, ignoring custom fields', () => {
|
||||
const issues = getDucklakeValidationIssues({
|
||||
ducklakeMode: 'supabase',
|
||||
ducklakeCatalogProjectRef: '',
|
||||
ducklakeStorageProjectRef: '',
|
||||
ducklakeStorageBucket: '',
|
||||
// Custom-mode fields are intentionally blank and must not be validated in supabase mode
|
||||
ducklakeCatalogUrl: '',
|
||||
ducklakeDataPath: '',
|
||||
ducklakeS3AccessKeyId: '',
|
||||
ducklakeS3SecretAccessKey: '',
|
||||
ducklakeS3Region: '',
|
||||
ducklakeS3Endpoint: '',
|
||||
ducklakeMetadataSchema: '',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([
|
||||
{ path: 'ducklakeCatalogProjectRef', message: 'Catalog project is required' },
|
||||
{ path: 'ducklakeStorageProjectRef', message: 'Storage project is required' },
|
||||
{ path: 'ducklakeStorageBucket', message: 'Bucket is required' },
|
||||
])
|
||||
})
|
||||
|
||||
it('accepts a complete supabase configuration', () => {
|
||||
expect(
|
||||
getDucklakeValidationIssues({
|
||||
ducklakeMode: 'supabase',
|
||||
ducklakeCatalogProjectRef: 'catalog-ref',
|
||||
ducklakeStorageProjectRef: 'storage-ref',
|
||||
ducklakeStorageBucket: 'ducklake-data',
|
||||
ducklakeCatalogUrl: '',
|
||||
ducklakeDataPath: '',
|
||||
ducklakeS3AccessKeyId: '',
|
||||
ducklakeS3SecretAccessKey: '',
|
||||
ducklakeS3Region: '',
|
||||
ducklakeS3Endpoint: '',
|
||||
ducklakeMetadataSchema: '',
|
||||
})
|
||||
).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('DestinationForm.utils Snowflake', () => {
|
||||
it('builds Snowflake validation config with identifiers trimmed and secrets preserved', () => {
|
||||
const config = buildDestinationConfigForValidation({
|
||||
@@ -270,3 +388,122 @@ describe('DestinationForm.utils Snowflake', () => {
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('DestinationForm.utils BigQuery', () => {
|
||||
it('returns required-field errors for missing BigQuery settings', () => {
|
||||
const issues = getBigQueryValidationIssues({
|
||||
projectId: '',
|
||||
datasetId: '',
|
||||
serviceAccountKey: '',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([
|
||||
{ path: 'projectId', message: 'Project ID is required' },
|
||||
{ path: 'datasetId', message: 'Dataset ID is required' },
|
||||
{ path: 'serviceAccountKey', message: 'Service Account Key is required' },
|
||||
])
|
||||
})
|
||||
|
||||
it('treats whitespace-only values as missing', () => {
|
||||
const issues = getBigQueryValidationIssues({
|
||||
projectId: ' ',
|
||||
datasetId: '\t',
|
||||
serviceAccountKey: '\n',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([
|
||||
{ path: 'projectId', message: 'Project ID is required' },
|
||||
{ path: 'datasetId', message: 'Dataset ID is required' },
|
||||
{ path: 'serviceAccountKey', message: 'Service Account Key is required' },
|
||||
])
|
||||
})
|
||||
|
||||
it('returns no issues for a complete configuration', () => {
|
||||
const issues = getBigQueryValidationIssues({
|
||||
projectId: 'my-project',
|
||||
datasetId: 'my_dataset',
|
||||
serviceAccountKey: '{ "type": "service_account" }',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('DestinationForm.utils Analytics Bucket', () => {
|
||||
it('returns required-field errors for an empty configuration', () => {
|
||||
const issues = getAnalyticsBucketValidationIssues({
|
||||
warehouseName: '',
|
||||
namespace: '',
|
||||
newNamespaceName: '',
|
||||
s3Region: '',
|
||||
s3AccessKeyId: '',
|
||||
s3SecretAccessKey: '',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([
|
||||
{ path: 'warehouseName', message: 'Bucket is required' },
|
||||
{ path: 's3Region', message: 'S3 Region is required' },
|
||||
{ path: 's3AccessKeyId', message: 'S3 Access Key ID is required' },
|
||||
{ path: 'namespace', message: 'Namespace is required' },
|
||||
{ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' },
|
||||
])
|
||||
})
|
||||
|
||||
it('treats whitespace-only values as missing', () => {
|
||||
const issues = getAnalyticsBucketValidationIssues({
|
||||
warehouseName: ' ',
|
||||
namespace: ' ',
|
||||
newNamespaceName: '',
|
||||
s3Region: '\t',
|
||||
s3AccessKeyId: ' ',
|
||||
s3SecretAccessKey: ' ',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([
|
||||
{ path: 'warehouseName', message: 'Bucket is required' },
|
||||
{ path: 's3Region', message: 'S3 Region is required' },
|
||||
{ path: 's3AccessKeyId', message: 'S3 Access Key ID is required' },
|
||||
{ path: 'namespace', message: 'Namespace is required' },
|
||||
{ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' },
|
||||
])
|
||||
})
|
||||
|
||||
it('requires a name when creating a new namespace', () => {
|
||||
const issues = getAnalyticsBucketValidationIssues({
|
||||
warehouseName: 'bucket',
|
||||
namespace: CREATE_NEW_NAMESPACE,
|
||||
newNamespaceName: '',
|
||||
s3Region: 'us-east-1',
|
||||
s3AccessKeyId: 'key',
|
||||
s3SecretAccessKey: 'secret',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([{ path: 'newNamespaceName', message: 'Namespace name is required' }])
|
||||
})
|
||||
|
||||
it('skips the S3 secret when creating a new access key', () => {
|
||||
const issues = getAnalyticsBucketValidationIssues({
|
||||
warehouseName: 'bucket',
|
||||
namespace: 'analytics',
|
||||
newNamespaceName: '',
|
||||
s3Region: 'us-east-1',
|
||||
s3AccessKeyId: CREATE_NEW_KEY,
|
||||
s3SecretAccessKey: '',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([])
|
||||
})
|
||||
|
||||
it('returns no issues for a complete configuration', () => {
|
||||
const issues = getAnalyticsBucketValidationIssues({
|
||||
warehouseName: 'bucket',
|
||||
namespace: CREATE_NEW_NAMESPACE,
|
||||
newNamespaceName: 'new_namespace',
|
||||
s3Region: 'us-east-1',
|
||||
s3AccessKeyId: 'key',
|
||||
s3SecretAccessKey: 'secret',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([])
|
||||
})
|
||||
})
|
||||
+153
-192
@@ -8,13 +8,24 @@ import {
|
||||
DestinationPanelFormSchema,
|
||||
type DestinationPanelSchemaType,
|
||||
} from './DestinationForm.schema'
|
||||
import {
|
||||
DUCKLAKE_MODE_CUSTOM,
|
||||
DUCKLAKE_MODE_SUPABASE,
|
||||
type DucklakeMode,
|
||||
} from './DuckLake/DuckLake.constants'
|
||||
import { type DucklakeApiConfig } from './DuckLake/DuckLake.utils'
|
||||
import { type SnowflakeApiConfig } from './Snowflake/Snowflake.utils'
|
||||
import {
|
||||
BigQueryDestinationConfig,
|
||||
DestinationConfig,
|
||||
DucklakeDestinationConfig,
|
||||
DucklakeManualDestinationConfig,
|
||||
DucklakeSupabaseDestinationConfig,
|
||||
IcebergDestinationConfig,
|
||||
SnowflakeDestinationConfig,
|
||||
} from '@/data/replication/create-destination-pipeline-mutation'
|
||||
import { type ReplicationDestinationByIdData } from '@/data/replication/destination-by-id-query'
|
||||
import { type ReplicationPipelineByIdData } from '@/data/replication/pipeline-by-id-query'
|
||||
import { type ValidationFailure } from '@/data/replication/validate-destination-mutation'
|
||||
import {
|
||||
type CreateS3AccessKeyCredentialVariables,
|
||||
@@ -33,140 +44,149 @@ const normalizeOptionalUntrimmedString = (value?: string) => {
|
||||
return value && value.length > 0 ? value : undefined
|
||||
}
|
||||
|
||||
type DucklakeFieldPath =
|
||||
| 'ducklakeCatalogUrl'
|
||||
| 'ducklakeDataPath'
|
||||
| 'ducklakeS3AccessKeyId'
|
||||
| 'ducklakeS3SecretAccessKey'
|
||||
| 'ducklakeS3Region'
|
||||
| 'ducklakeS3Endpoint'
|
||||
| 'ducklakeMetadataSchema'
|
||||
// Builds the initial react-hook-form values for the destination form. New destinations fall back to
|
||||
// sensible defaults; existing destinations are hydrated from the destination + pipeline configs.
|
||||
export const generateDefaultValues = ({
|
||||
destinationData,
|
||||
pipelineData,
|
||||
catalogToken,
|
||||
region,
|
||||
projectRef,
|
||||
editMode,
|
||||
}: {
|
||||
destinationData?: ReplicationDestinationByIdData
|
||||
pipelineData?: ReplicationPipelineByIdData
|
||||
catalogToken: string
|
||||
region?: string
|
||||
projectRef?: string
|
||||
editMode: boolean
|
||||
}): DestinationPanelSchemaType => {
|
||||
const config = destinationData?.config
|
||||
const isBigQueryConfig = config && 'big_query' in config
|
||||
const isIcebergConfig = config && 'iceberg' in config
|
||||
const ducklakeConfigValue =
|
||||
config && 'ducklake' in (config as Record<string, unknown>)
|
||||
? (config as Record<string, unknown>).ducklake
|
||||
: undefined
|
||||
const ducklakeConfig =
|
||||
ducklakeConfigValue && typeof ducklakeConfigValue === 'object'
|
||||
? (ducklakeConfigValue as DucklakeApiConfig)
|
||||
: undefined
|
||||
const snowflakeConfigValue =
|
||||
config && 'snowflake' in (config as Record<string, unknown>)
|
||||
? (config as Record<string, unknown>).snowflake
|
||||
: undefined
|
||||
const snowflakeConfig =
|
||||
snowflakeConfigValue && typeof snowflakeConfigValue === 'object'
|
||||
? (snowflakeConfigValue as SnowflakeApiConfig)
|
||||
: undefined
|
||||
|
||||
export type DucklakeValidationIssue = {
|
||||
path: DucklakeFieldPath
|
||||
message: string
|
||||
return {
|
||||
// Common fields
|
||||
name: destinationData?.name ?? '',
|
||||
publicationName: pipelineData?.config.publication_name ?? '',
|
||||
maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? undefined,
|
||||
maxTableSyncWorkers: pipelineData?.config?.max_table_sync_workers ?? undefined,
|
||||
maxCopyConnectionsPerTable: pipelineData?.config?.max_copy_connections_per_table ?? undefined,
|
||||
invalidatedSlotBehavior:
|
||||
(pipelineData?.config as { invalidated_slot_behavior?: 'error' | 'recreate' } | undefined)
|
||||
?.invalidated_slot_behavior ?? undefined,
|
||||
// BigQuery fields
|
||||
projectId: isBigQueryConfig ? config.big_query.project_id : '',
|
||||
datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',
|
||||
serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',
|
||||
connectionPoolSize:
|
||||
(config as { big_query?: { connection_pool_size?: number } } | undefined)?.big_query
|
||||
?.connection_pool_size ?? undefined,
|
||||
maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined, // Default: null
|
||||
// Analytics Bucket fields
|
||||
warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',
|
||||
namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '',
|
||||
newNamespaceName: '',
|
||||
catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken,
|
||||
s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '',
|
||||
s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '',
|
||||
s3Region: region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''),
|
||||
// DuckLake fields
|
||||
// New destinations default to the managed "Use Supabase" mode with the current project
|
||||
// pre-selected as both catalog and storage. Existing destinations always read back as the
|
||||
// resolved/custom shape, so edit mode is locked to "Custom parameters".
|
||||
ducklakeMode: (editMode ? DUCKLAKE_MODE_CUSTOM : DUCKLAKE_MODE_SUPABASE) as DucklakeMode,
|
||||
ducklakeCatalogProjectRef: editMode ? '' : (projectRef ?? ''),
|
||||
ducklakeStorageProjectRef: editMode ? '' : (projectRef ?? ''),
|
||||
ducklakeStorageBucket: '',
|
||||
ducklakeCatalogUrl: ducklakeConfig?.catalog_url ?? '',
|
||||
ducklakeDataPath: ducklakeConfig?.data_path ?? '',
|
||||
ducklakePoolSize: ducklakeConfig?.pool_size,
|
||||
ducklakeS3AccessKeyId: ducklakeConfig?.s3_access_key_id ?? '',
|
||||
ducklakeS3SecretAccessKey: ducklakeConfig?.s3_secret_access_key ?? '',
|
||||
ducklakeS3Region: ducklakeConfig?.s3_region ?? '',
|
||||
ducklakeS3Endpoint: ducklakeConfig?.s3_endpoint ?? '',
|
||||
ducklakeS3UrlStyle: ducklakeConfig?.s3_url_style ?? 'path',
|
||||
ducklakeS3UseSsl: ducklakeConfig?.s3_use_ssl ?? true,
|
||||
ducklakeMetadataSchema: ducklakeConfig?.metadata_schema ?? 'ducklake',
|
||||
// Snowflake fields
|
||||
snowflakeAccountId: snowflakeConfig?.account_id ?? '',
|
||||
snowflakeUser: snowflakeConfig?.user ?? '',
|
||||
snowflakePrivateKey: snowflakeConfig?.private_key ?? '',
|
||||
snowflakePrivateKeyPassphrase: snowflakeConfig?.private_key_passphrase ?? '',
|
||||
snowflakeDatabase: snowflakeConfig?.database ?? '',
|
||||
snowflakeSchema: snowflakeConfig?.schema ?? '',
|
||||
snowflakeRole: snowflakeConfig?.role ?? '',
|
||||
}
|
||||
}
|
||||
|
||||
export const getDucklakeValidationIssues = (
|
||||
data: Pick<
|
||||
DestinationPanelSchemaType,
|
||||
| 'ducklakeCatalogUrl'
|
||||
| 'ducklakeDataPath'
|
||||
| 'ducklakeS3AccessKeyId'
|
||||
| 'ducklakeS3SecretAccessKey'
|
||||
| 'ducklakeS3Region'
|
||||
| 'ducklakeS3Endpoint'
|
||||
| 'ducklakeMetadataSchema'
|
||||
>
|
||||
): DucklakeValidationIssue[] => {
|
||||
const issues: DucklakeValidationIssue[] = []
|
||||
const buildBigQueryConfig = (
|
||||
data: z.infer<typeof DestinationPanelFormSchema>
|
||||
): BigQueryDestinationConfig => ({
|
||||
projectId: data.projectId ?? '',
|
||||
datasetId: data.datasetId ?? '',
|
||||
serviceAccountKey: data.serviceAccountKey ?? '',
|
||||
connectionPoolSize: data.connectionPoolSize,
|
||||
maxStalenessMins: data.maxStalenessMins,
|
||||
})
|
||||
|
||||
if (!data.ducklakeCatalogUrl?.length) {
|
||||
issues.push({ path: 'ducklakeCatalogUrl', message: 'Catalog URL is required' })
|
||||
} else if (
|
||||
!data.ducklakeCatalogUrl.startsWith('postgres://') &&
|
||||
!data.ducklakeCatalogUrl.startsWith('postgresql://')
|
||||
) {
|
||||
issues.push({
|
||||
path: 'ducklakeCatalogUrl',
|
||||
message: 'DuckLake catalog URL must be a PostgreSQL-compatible URL',
|
||||
})
|
||||
const buildSnowflakeConfig = (
|
||||
data: z.infer<typeof DestinationPanelFormSchema>
|
||||
): SnowflakeDestinationConfig => ({
|
||||
accountId: normalizeRequiredString(data.snowflakeAccountId),
|
||||
user: normalizeRequiredString(data.snowflakeUser),
|
||||
privateKey: data.snowflakePrivateKey ?? '',
|
||||
privateKeyPassphrase: normalizeOptionalUntrimmedString(data.snowflakePrivateKeyPassphrase),
|
||||
database: normalizeRequiredString(data.snowflakeDatabase),
|
||||
schema: normalizeRequiredString(data.snowflakeSchema),
|
||||
role: normalizeOptionalString(data.snowflakeRole),
|
||||
})
|
||||
|
||||
// Builds the studio-side DuckLake config from form data, picking the right shape for the
|
||||
// selected mode. The create / update / validate mutations convert this to the API payload.
|
||||
const buildDucklakeConfig = (
|
||||
data: z.infer<typeof DestinationPanelFormSchema>
|
||||
): DucklakeDestinationConfig => {
|
||||
if (data.ducklakeMode === DUCKLAKE_MODE_SUPABASE) {
|
||||
const supabaseConfig: DucklakeSupabaseDestinationConfig = {
|
||||
catalogProjectRef: normalizeRequiredString(data.ducklakeCatalogProjectRef),
|
||||
storageProjectRef: normalizeRequiredString(data.ducklakeStorageProjectRef),
|
||||
bucket: normalizeRequiredString(data.ducklakeStorageBucket),
|
||||
poolSize: data.ducklakePoolSize,
|
||||
metadataSchema: normalizeOptionalString(data.ducklakeMetadataSchema),
|
||||
}
|
||||
return supabaseConfig
|
||||
}
|
||||
|
||||
if (!data.ducklakeDataPath?.length) {
|
||||
issues.push({ path: 'ducklakeDataPath', message: 'Data path is required' })
|
||||
} else if (
|
||||
!data.ducklakeDataPath.startsWith('s3://') ||
|
||||
data.ducklakeDataPath.includes('file://')
|
||||
) {
|
||||
issues.push({
|
||||
path: 'ducklakeDataPath',
|
||||
message: 'DuckLake data path must start with s3:// and cannot contain file://',
|
||||
})
|
||||
const manualConfig: DucklakeManualDestinationConfig = {
|
||||
catalogUrl: data.ducklakeCatalogUrl ?? '',
|
||||
dataPath: data.ducklakeDataPath ?? '',
|
||||
poolSize: data.ducklakePoolSize,
|
||||
s3AccessKeyId: normalizeRequiredString(data.ducklakeS3AccessKeyId),
|
||||
s3SecretAccessKey: normalizeRequiredString(data.ducklakeS3SecretAccessKey),
|
||||
s3Region: normalizeRequiredString(data.ducklakeS3Region),
|
||||
s3Endpoint: normalizeRequiredString(data.ducklakeS3Endpoint),
|
||||
s3UrlStyle: data.ducklakeS3UrlStyle,
|
||||
s3UseSsl: data.ducklakeS3UseSsl,
|
||||
metadataSchema: normalizeOptionalString(data.ducklakeMetadataSchema),
|
||||
}
|
||||
|
||||
if (!data.ducklakeS3AccessKeyId?.length) {
|
||||
issues.push({ path: 'ducklakeS3AccessKeyId', message: 'S3 Access Key ID is required' })
|
||||
}
|
||||
|
||||
if (!data.ducklakeS3SecretAccessKey?.length) {
|
||||
issues.push({
|
||||
path: 'ducklakeS3SecretAccessKey',
|
||||
message: 'S3 Secret Access Key is required',
|
||||
})
|
||||
}
|
||||
|
||||
if (!data.ducklakeS3Region?.length) {
|
||||
issues.push({ path: 'ducklakeS3Region', message: 'S3 Region is required' })
|
||||
}
|
||||
|
||||
if (!data.ducklakeS3Endpoint?.length) {
|
||||
issues.push({ path: 'ducklakeS3Endpoint', message: 'S3 Endpoint is required' })
|
||||
} else if (
|
||||
data.ducklakeS3Endpoint.startsWith('http://') ||
|
||||
data.ducklakeS3Endpoint.startsWith('https://')
|
||||
) {
|
||||
issues.push({
|
||||
path: 'ducklakeS3Endpoint',
|
||||
message: 'S3 endpoint should not contain the protocol scheme',
|
||||
})
|
||||
}
|
||||
|
||||
if (data.ducklakeMetadataSchema && !/^[A-Za-z0-9_]+$/.test(data.ducklakeMetadataSchema)) {
|
||||
issues.push({
|
||||
path: 'ducklakeMetadataSchema',
|
||||
message: 'DuckLake metadata schema must contain only letters, numbers, and underscores',
|
||||
})
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
|
||||
type SnowflakeFieldPath =
|
||||
| 'snowflakeAccountId'
|
||||
| 'snowflakeUser'
|
||||
| 'snowflakePrivateKey'
|
||||
| 'snowflakeDatabase'
|
||||
| 'snowflakeSchema'
|
||||
|
||||
export type SnowflakeValidationIssue = {
|
||||
path: SnowflakeFieldPath
|
||||
message: string
|
||||
}
|
||||
|
||||
export const getSnowflakeValidationIssues = (
|
||||
data: Pick<
|
||||
DestinationPanelSchemaType,
|
||||
| 'snowflakeAccountId'
|
||||
| 'snowflakeUser'
|
||||
| 'snowflakePrivateKey'
|
||||
| 'snowflakeDatabase'
|
||||
| 'snowflakeSchema'
|
||||
>
|
||||
): SnowflakeValidationIssue[] => {
|
||||
const issues: SnowflakeValidationIssue[] = []
|
||||
|
||||
if (!data.snowflakeAccountId?.trim().length) {
|
||||
issues.push({ path: 'snowflakeAccountId', message: 'Account ID is required' })
|
||||
}
|
||||
|
||||
if (!data.snowflakeUser?.trim().length) {
|
||||
issues.push({ path: 'snowflakeUser', message: 'User is required' })
|
||||
}
|
||||
|
||||
if (!data.snowflakePrivateKey?.trim().length) {
|
||||
issues.push({ path: 'snowflakePrivateKey', message: 'Private key is required' })
|
||||
}
|
||||
|
||||
if (!data.snowflakeDatabase?.trim().length) {
|
||||
issues.push({ path: 'snowflakeDatabase', message: 'Database is required' })
|
||||
}
|
||||
|
||||
if (!data.snowflakeSchema?.trim().length) {
|
||||
issues.push({ path: 'snowflakeSchema', message: 'Schema is required' })
|
||||
}
|
||||
|
||||
return issues
|
||||
return manualConfig
|
||||
}
|
||||
|
||||
// Helper function to build destination config for validation
|
||||
@@ -182,15 +202,7 @@ export const buildDestinationConfigForValidation = ({
|
||||
if (!projectRef) throw new Error('Project ref is required')
|
||||
|
||||
if (selectedType === 'BigQuery') {
|
||||
return {
|
||||
bigQuery: {
|
||||
projectId: data.projectId ?? '',
|
||||
datasetId: data.datasetId ?? '',
|
||||
serviceAccountKey: data.serviceAccountKey ?? '',
|
||||
connectionPoolSize: data.connectionPoolSize,
|
||||
maxStalenessMins: data.maxStalenessMins,
|
||||
},
|
||||
}
|
||||
return { bigQuery: buildBigQueryConfig(data) }
|
||||
} else if (selectedType === 'Analytics Bucket') {
|
||||
// For validation, use the namespace as-is (even if it's CREATE_NEW_NAMESPACE)
|
||||
// The actual creation will happen later in submitPipeline
|
||||
@@ -217,32 +229,9 @@ export const buildDestinationConfigForValidation = ({
|
||||
},
|
||||
}
|
||||
} else if (selectedType === 'DuckLake') {
|
||||
return {
|
||||
ducklake: {
|
||||
catalogUrl: data.ducklakeCatalogUrl ?? '',
|
||||
dataPath: data.ducklakeDataPath ?? '',
|
||||
poolSize: data.ducklakePoolSize,
|
||||
s3AccessKeyId: normalizeRequiredString(data.ducklakeS3AccessKeyId),
|
||||
s3SecretAccessKey: normalizeRequiredString(data.ducklakeS3SecretAccessKey),
|
||||
s3Region: normalizeRequiredString(data.ducklakeS3Region),
|
||||
s3Endpoint: normalizeRequiredString(data.ducklakeS3Endpoint),
|
||||
s3UrlStyle: data.ducklakeS3UrlStyle,
|
||||
s3UseSsl: data.ducklakeS3UseSsl,
|
||||
metadataSchema: normalizeOptionalString(data.ducklakeMetadataSchema),
|
||||
},
|
||||
}
|
||||
return { ducklake: buildDucklakeConfig(data) }
|
||||
} else if (selectedType === 'Snowflake') {
|
||||
return {
|
||||
snowflake: {
|
||||
accountId: normalizeRequiredString(data.snowflakeAccountId),
|
||||
user: normalizeRequiredString(data.snowflakeUser),
|
||||
privateKey: data.snowflakePrivateKey ?? '',
|
||||
privateKeyPassphrase: normalizeOptionalUntrimmedString(data.snowflakePrivateKeyPassphrase),
|
||||
database: normalizeRequiredString(data.snowflakeDatabase),
|
||||
schema: normalizeRequiredString(data.snowflakeSchema),
|
||||
role: normalizeOptionalString(data.snowflakeRole),
|
||||
},
|
||||
}
|
||||
return { snowflake: buildSnowflakeConfig(data) }
|
||||
} else {
|
||||
throw new Error('Invalid destination type')
|
||||
}
|
||||
@@ -275,14 +264,7 @@ export const buildDestinationConfig = async ({
|
||||
let destinationConfig: DestinationConfig | undefined = undefined
|
||||
|
||||
if (selectedType === 'BigQuery') {
|
||||
const bigQueryConfig: BigQueryDestinationConfig = {
|
||||
projectId: data.projectId ?? '',
|
||||
datasetId: data.datasetId ?? '',
|
||||
serviceAccountKey: data.serviceAccountKey ?? '',
|
||||
connectionPoolSize: data.connectionPoolSize,
|
||||
maxStalenessMins: data.maxStalenessMins,
|
||||
}
|
||||
destinationConfig = { bigQuery: bigQueryConfig }
|
||||
destinationConfig = { bigQuery: buildBigQueryConfig(data) }
|
||||
} else if (selectedType === 'Analytics Bucket') {
|
||||
let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
|
||||
|
||||
@@ -308,30 +290,9 @@ export const buildDestinationConfig = async ({
|
||||
}
|
||||
destinationConfig = { iceberg: icebergConfig }
|
||||
} else if (selectedType === 'DuckLake') {
|
||||
const ducklakeConfig: DucklakeDestinationConfig = {
|
||||
catalogUrl: data.ducklakeCatalogUrl ?? '',
|
||||
dataPath: data.ducklakeDataPath ?? '',
|
||||
poolSize: data.ducklakePoolSize,
|
||||
s3AccessKeyId: normalizeRequiredString(data.ducklakeS3AccessKeyId),
|
||||
s3SecretAccessKey: normalizeRequiredString(data.ducklakeS3SecretAccessKey),
|
||||
s3Region: normalizeRequiredString(data.ducklakeS3Region),
|
||||
s3Endpoint: normalizeRequiredString(data.ducklakeS3Endpoint),
|
||||
s3UrlStyle: data.ducklakeS3UrlStyle,
|
||||
s3UseSsl: data.ducklakeS3UseSsl,
|
||||
metadataSchema: normalizeOptionalString(data.ducklakeMetadataSchema),
|
||||
}
|
||||
destinationConfig = { ducklake: ducklakeConfig }
|
||||
destinationConfig = { ducklake: buildDucklakeConfig(data) }
|
||||
} else if (selectedType === 'Snowflake') {
|
||||
const snowflakeConfig: SnowflakeDestinationConfig = {
|
||||
accountId: normalizeRequiredString(data.snowflakeAccountId),
|
||||
user: normalizeRequiredString(data.snowflakeUser),
|
||||
privateKey: data.snowflakePrivateKey ?? '',
|
||||
privateKeyPassphrase: normalizeOptionalUntrimmedString(data.snowflakePrivateKeyPassphrase),
|
||||
database: normalizeRequiredString(data.snowflakeDatabase),
|
||||
schema: normalizeRequiredString(data.snowflakeSchema),
|
||||
role: normalizeOptionalString(data.snowflakeRole),
|
||||
}
|
||||
destinationConfig = { snowflake: snowflakeConfig }
|
||||
destinationConfig = { snowflake: buildSnowflakeConfig(data) }
|
||||
}
|
||||
|
||||
return destinationConfig
|
||||
|
||||
-896
@@ -1,896 +0,0 @@
|
||||
import { PermissionAction } from '@supabase/shared-types/out/constants'
|
||||
import { useParams } from 'common'
|
||||
import { Eye, EyeOff, Loader2 } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
import type { UseFormReturn } from 'react-hook-form'
|
||||
import {
|
||||
Button,
|
||||
FormControl,
|
||||
FormField,
|
||||
Input,
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectGroup,
|
||||
SelectItem,
|
||||
SelectSeparator,
|
||||
SelectTrigger,
|
||||
TextArea,
|
||||
WarningIcon,
|
||||
} from 'ui'
|
||||
import { Admonition } from 'ui-patterns'
|
||||
import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
|
||||
import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
|
||||
import type { DestinationPanelSchemaType } from './DestinationForm.schema'
|
||||
import { InlineLink } from '@/components/ui/InlineLink'
|
||||
import { useAPIKeys } from '@/data/api-keys/api-keys-query'
|
||||
import { useAnalyticsBucketsQuery } from '@/data/storage/analytics-buckets-query'
|
||||
import { useIcebergNamespacesQuery } from '@/data/storage/iceberg-namespaces-query'
|
||||
import { useStorageCredentialsQuery } from '@/data/storage/s3-access-key-query'
|
||||
import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
|
||||
|
||||
export const BigQueryFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">BigQuery settings</p>
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="projectId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Project ID"
|
||||
description="The Google Cloud project ID where data will be sent"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="my-gcp-project" />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="datasetId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="Dataset ID"
|
||||
layout="horizontal"
|
||||
description="The BigQuery dataset where replicated tables will be created"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="my_dataset" />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="serviceAccountKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Service Account Key"
|
||||
description="Service account credentials JSON for authenticating with BigQuery"
|
||||
>
|
||||
<FormControl>
|
||||
<TextArea
|
||||
{...field}
|
||||
rows={5}
|
||||
maxLength={5000}
|
||||
placeholder='{"type": "service_account", "project_id": "...", ...}'
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const DuckLakeFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
|
||||
const [showCatalogUrl, setShowCatalogUrl] = useState(false)
|
||||
const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">DuckLake settings</p>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Catalog</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Configure the PostgreSQL-backed DuckLake catalog and the S3-compatible storage location
|
||||
for replicated data.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeCatalogUrl"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Catalog URL"
|
||||
description="A PostgreSQL connection string for the DuckLake catalog"
|
||||
>
|
||||
<FormControl>
|
||||
<PasswordInput
|
||||
value={field.value ?? ''}
|
||||
type={showCatalogUrl ? 'text' : 'password'}
|
||||
placeholder="postgres://user:pass@host:5432/ducklake_catalog"
|
||||
onChange={(event) => field.onChange(event.target.value)}
|
||||
actions={
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
icon={showCatalogUrl ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowCatalogUrl(!showCatalogUrl)}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeDataPath"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Data path"
|
||||
description="An S3 path where DuckLake data files will be written"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="s3://bucket/path" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakePoolSize"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Pool size"
|
||||
description="Optional number of concurrent DuckDB connections to use"
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={6}
|
||||
value={field.value ?? ''}
|
||||
placeholder="Default: 4"
|
||||
onChange={(event) =>
|
||||
field.onChange(
|
||||
event.target.value === '' ? undefined : Number(event.target.value)
|
||||
)
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Object storage</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Optional credentials and endpoint settings for S3-compatible storage providers.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3AccessKeyId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Access Key ID"
|
||||
description="Required access key ID for the object storage provider"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="my-access-key" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3SecretAccessKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Secret Access Key"
|
||||
description="Required secret access key for the object storage provider"
|
||||
className="relative"
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
type={showSecretAccessKey ? 'text' : 'password'}
|
||||
placeholder="my-secret-key"
|
||||
value={field.value ?? ''}
|
||||
/>
|
||||
</FormControl>
|
||||
<Button
|
||||
variant="default"
|
||||
icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
|
||||
className="w-7 absolute right-6 top-[4px]"
|
||||
onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3Region"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Region"
|
||||
description="Required region for the object storage provider"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="us-east-1" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3Endpoint"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Endpoint"
|
||||
description="Required endpoint without the protocol scheme, for example `127.0.0.1:5000/s3`"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="127.0.0.1:5000/s3" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3UrlStyle"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 URL style"
|
||||
description="Choose `path` for MinIO/Supabase-style endpoints or `vhost` for AWS-style virtual host addressing"
|
||||
>
|
||||
<FormControl>
|
||||
<Select value={field.value ?? 'path'} onValueChange={field.onChange}>
|
||||
<SelectTrigger>{field.value ?? 'path'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="path">path</SelectItem>
|
||||
<SelectItem value="vhost">vhost</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3UseSsl"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Use SSL"
|
||||
description="Whether to use SSL when connecting to the S3-compatible endpoint"
|
||||
>
|
||||
<FormControl>
|
||||
<Select
|
||||
value={field.value === false ? 'false' : 'true'}
|
||||
onValueChange={(value) => field.onChange(value === 'true')}
|
||||
>
|
||||
<SelectTrigger>{field.value === false ? 'false' : 'true'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="true">true</SelectItem>
|
||||
<SelectItem value="false">false</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Metadata</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Optional schema setting for DuckLake metadata tables.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeMetadataSchema"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Metadata schema"
|
||||
description="Schema used for DuckLake metadata tables in PostgreSQL"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ducklake" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const SnowflakeFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
|
||||
const [showPrivateKeyPassphrase, setShowPrivateKeyPassphrase] = useState(false)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">Snowflake settings</p>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Connection</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Configure the Snowflake account, user, and target namespace for replicated data.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeAccountId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Account ID"
|
||||
description="Snowflake account identifier, for example ORGNAME-ACCOUNTNAME"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="MYORG-MYACCOUNT" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeUser"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="User"
|
||||
description="Snowflake user configured for key-pair authentication"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ETL_USER" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeDatabase"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Database"
|
||||
description="Snowflake database where replicated tables will be created"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ANALYTICS" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeSchema"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Schema"
|
||||
description="Snowflake schema where replicated tables will be created"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="PUBLIC" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeRole"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Role"
|
||||
description="Optional Snowflake role to assume after connecting"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ETL_ROLE" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Authentication</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Use the RSA private key whose public key is registered on the Snowflake user.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakePrivateKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Private key"
|
||||
description="RSA private key PEM contents in PKCS#8 or PKCS#1 format"
|
||||
>
|
||||
<FormControl>
|
||||
<TextArea
|
||||
{...field}
|
||||
rows={8}
|
||||
maxLength={10000}
|
||||
placeholder={'-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----'}
|
||||
value={field.value ?? ''}
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakePrivateKeyPassphrase"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Private key passphrase"
|
||||
description="Optional passphrase for encrypted private keys"
|
||||
>
|
||||
<FormControl>
|
||||
<PasswordInput
|
||||
value={field.value ?? ''}
|
||||
type={showPrivateKeyPassphrase ? 'text' : 'password'}
|
||||
placeholder="Optional"
|
||||
onChange={(event) => field.onChange(event.target.value)}
|
||||
actions={
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
icon={showPrivateKeyPassphrase ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowPrivateKeyPassphrase(!showPrivateKeyPassphrase)}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* [Joshen] JFYI I'd foresee a possible UX friction point here regarding S3 access key IDs and secret access keys
|
||||
* - We'd allow users to select access key IDs via a dropdown here, but require a text input for secret access keys
|
||||
* - Chances are most users wouldn't have the corresponding secret access key for the selected key ID at the top of their heads
|
||||
* - So highly likely may have to default to "Create a new key" -> which from here they won't know the secret access key thereafter
|
||||
* - And it'll end up just creating more keys for each destination
|
||||
* Ideal scenario: Just select an access key ID, we then apply the secret access key in the PATCH request, so FE has no
|
||||
* context of the secret access key at any point
|
||||
*/
|
||||
export const AnalyticsBucketFields = ({
|
||||
form,
|
||||
setIsFormInteracting,
|
||||
onSelectNewBucket,
|
||||
}: {
|
||||
form: UseFormReturn<DestinationPanelSchemaType>
|
||||
setIsFormInteracting: (value: boolean) => void
|
||||
onSelectNewBucket: () => void
|
||||
}) => {
|
||||
const { warehouseName, s3AccessKeyId, namespace } = form.watch()
|
||||
const [showCatalogToken, setShowCatalogToken] = useState(false)
|
||||
const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
|
||||
|
||||
const { ref: projectRef } = useParams()
|
||||
|
||||
const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
|
||||
const { data: apiKeysData } = useAPIKeys(
|
||||
{ projectRef, reveal: true },
|
||||
{ enabled: canReadAPIKeys }
|
||||
)
|
||||
const { serviceKey } = apiKeysData ?? {}
|
||||
const serviceApiKey = serviceKey?.api_key ?? ''
|
||||
|
||||
const {
|
||||
data: keysData,
|
||||
isSuccess: isSuccessKeys,
|
||||
isPending: isLoadingKeys,
|
||||
isError: isErrorKeys,
|
||||
} = useStorageCredentialsQuery({ projectRef })
|
||||
const s3Keys = keysData?.data ?? []
|
||||
const keyNoLongerExists =
|
||||
(s3AccessKeyId ?? '').length > 0 &&
|
||||
s3AccessKeyId !== CREATE_NEW_KEY &&
|
||||
!s3Keys.find((k) => k.access_key === s3AccessKeyId)
|
||||
|
||||
const {
|
||||
data: analyticsBuckets = [],
|
||||
isPending: isLoadingBuckets,
|
||||
isError: isErrorBuckets,
|
||||
} = useAnalyticsBucketsQuery({ projectRef })
|
||||
|
||||
const canSelectNamespace = !!warehouseName && !!serviceApiKey
|
||||
|
||||
const {
|
||||
data: namespaces = [],
|
||||
isPending: isLoadingNamespaces,
|
||||
isError: isErrorNamespaces,
|
||||
} = useIcebergNamespacesQuery(
|
||||
{ projectRef, warehouse: warehouseName },
|
||||
{ enabled: !!serviceApiKey }
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">Analytics Bucket settings</p>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="warehouseName"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="Bucket"
|
||||
layout="horizontal"
|
||||
description="The Analytics Bucket where data will be stored"
|
||||
>
|
||||
{isLoadingBuckets ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving buckets
|
||||
</Button>
|
||||
) : isErrorBuckets ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve buckets
|
||||
</Button>
|
||||
) : (
|
||||
<FormControl>
|
||||
<Select
|
||||
value={field.value}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'new-bucket') {
|
||||
onSelectNewBucket()
|
||||
} else {
|
||||
setIsFormInteracting(true)
|
||||
field.onChange(value)
|
||||
// [Joshen] Ideally should select the first namespace of the selected bucket
|
||||
form.setValue('namespace', '')
|
||||
}
|
||||
}}
|
||||
>
|
||||
<SelectTrigger>{field.value || 'Select a bucket'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{analyticsBuckets.length === 0 ? (
|
||||
<SelectItem value="__no_buckets__" disabled>
|
||||
No buckets available
|
||||
</SelectItem>
|
||||
) : (
|
||||
analyticsBuckets.map((bucket) => (
|
||||
<SelectItem key={bucket.name} value={bucket.name}>
|
||||
{bucket.name}
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
<SelectSeparator />
|
||||
<SelectItem value="new-bucket">Create a new bucket</SelectItem>
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
)}
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="namespace"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="Namespace"
|
||||
layout="horizontal"
|
||||
description="The namespace within the bucket where tables will be organized"
|
||||
>
|
||||
{isLoadingNamespaces && canSelectNamespace ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving namespaces
|
||||
</Button>
|
||||
) : isErrorNamespaces ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve namespaces
|
||||
</Button>
|
||||
) : (
|
||||
<FormControl>
|
||||
<Select
|
||||
value={field.value}
|
||||
onValueChange={(value) => {
|
||||
setIsFormInteracting(true)
|
||||
field.onChange(value)
|
||||
}}
|
||||
disabled={!canSelectNamespace}
|
||||
>
|
||||
<SelectTrigger>
|
||||
{!canSelectNamespace
|
||||
? 'Select a warehouse first'
|
||||
: field.value === CREATE_NEW_NAMESPACE
|
||||
? 'Create a new namespace'
|
||||
: field.value || 'Select a namespace'}
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{namespaces.length === 0 ? (
|
||||
<SelectItem value="__no_namespaces__" disabled>
|
||||
No namespaces available
|
||||
</SelectItem>
|
||||
) : (
|
||||
namespaces.map((namespace) => (
|
||||
<SelectItem key={namespace} value={namespace}>
|
||||
{namespace}
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
<SelectSeparator />
|
||||
<SelectItem key={CREATE_NEW_NAMESPACE} value={CREATE_NEW_NAMESPACE}>
|
||||
Create a new namespace
|
||||
</SelectItem>
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
)}
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
{namespace === CREATE_NEW_NAMESPACE && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="newNamespaceName"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
label="New Namespace Name"
|
||||
layout="horizontal"
|
||||
description="A unique name for the new namespace"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="new_namespace" value={field.value || ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="catalogToken"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Catalog Token"
|
||||
description={
|
||||
<>
|
||||
Automatically retrieved from your project's{' '}
|
||||
<InlineLink href={`/project/${projectRef}/settings/api-keys`}>
|
||||
service role key
|
||||
</InlineLink>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<PasswordInput
|
||||
disabled
|
||||
value={field.value}
|
||||
type={showCatalogToken ? 'text' : 'password'}
|
||||
placeholder="Auto-populated"
|
||||
actions={
|
||||
serviceApiKey ? (
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
icon={showCatalogToken ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowCatalogToken(!showCatalogToken)}
|
||||
/>
|
||||
</div>
|
||||
) : null
|
||||
}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="s3AccessKeyId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Access Key ID"
|
||||
description={
|
||||
<div className="flex flex-col gap-y-2">
|
||||
<p>
|
||||
Access keys are managed in your Storage{' '}
|
||||
<InlineLink href={`/project/${projectRef}/storage/s3`}>S3 settings</InlineLink>
|
||||
</p>
|
||||
|
||||
{isSuccessKeys && keyNoLongerExists && (
|
||||
<Admonition
|
||||
type="warning"
|
||||
title="Unable to find access key ID in project"
|
||||
description={
|
||||
<>
|
||||
Please select another key or create a new set, as this destination will
|
||||
not work otherwise. S3 access keys can be managed in your{' '}
|
||||
<InlineLink href={`/project/${projectRef}/storage/files/settings`}>
|
||||
storage settings
|
||||
</InlineLink>
|
||||
.
|
||||
</>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{s3AccessKeyId === CREATE_NEW_KEY && (
|
||||
<Admonition
|
||||
type="default"
|
||||
description="A new set of S3 access keys will be created."
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
>
|
||||
{isLoadingKeys ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving keys
|
||||
</Button>
|
||||
) : isErrorKeys ? (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve keys
|
||||
</Button>
|
||||
) : (
|
||||
<FormControl>
|
||||
<Select value={field.value} onValueChange={field.onChange}>
|
||||
<SelectTrigger>
|
||||
{field.value === CREATE_NEW_KEY
|
||||
? 'Create a new key'
|
||||
: (field.value ?? '').length === 0
|
||||
? 'Select an access key ID'
|
||||
: field.value}
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{s3Keys.map((key) => (
|
||||
<SelectItem key={key.id} value={key.access_key}>
|
||||
{key.access_key}
|
||||
<p className="text-foreground-lighter">{key.description}</p>
|
||||
</SelectItem>
|
||||
))}
|
||||
<SelectSeparator />
|
||||
<SelectItem key={CREATE_NEW_KEY} value={CREATE_NEW_KEY}>
|
||||
Create a new key
|
||||
</SelectItem>
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
)}
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
{s3AccessKeyId !== CREATE_NEW_KEY && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="s3SecretAccessKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Secret Access Key"
|
||||
className="relative"
|
||||
description="The secret key corresponding to your selected access key ID."
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
type={showSecretAccessKey ? 'text' : 'password'}
|
||||
value={field.value ?? ''}
|
||||
placeholder="Provide the secret access key"
|
||||
/>
|
||||
</FormControl>
|
||||
<Button
|
||||
variant="default"
|
||||
icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
|
||||
className="w-7 absolute right-1 top-[4px]"
|
||||
onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
// DuckLake can either be backed by Supabase projects (catalog + storage are managed for the
|
||||
// user) or configured manually with a PostgreSQL catalog URL and S3-compatible credentials.
|
||||
export const DUCKLAKE_MODE_SUPABASE = 'supabase'
|
||||
export const DUCKLAKE_MODE_CUSTOM = 'custom'
|
||||
export type DucklakeMode = typeof DUCKLAKE_MODE_SUPABASE | typeof DUCKLAKE_MODE_CUSTOM
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
import { type DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
import { DUCKLAKE_MODE_SUPABASE } from './DuckLake.constants'
|
||||
|
||||
export type DucklakeApiConfig = {
|
||||
catalog_url: string
|
||||
data_path: string
|
||||
pool_size?: number
|
||||
s3_access_key_id?: string
|
||||
s3_secret_access_key?: string
|
||||
s3_region?: string
|
||||
s3_endpoint?: string
|
||||
s3_url_style?: 'path' | 'vhost'
|
||||
s3_use_ssl?: boolean
|
||||
metadata_schema?: string
|
||||
}
|
||||
|
||||
// Fields entered when configuring DuckLake manually (always present in the validation data).
|
||||
const DUCKLAKE_INPUT_FIELD_PATHS = [
|
||||
'ducklakeCatalogUrl',
|
||||
'ducklakeDataPath',
|
||||
'ducklakeS3AccessKeyId',
|
||||
'ducklakeS3SecretAccessKey',
|
||||
'ducklakeS3Region',
|
||||
'ducklakeS3Endpoint',
|
||||
'ducklakeMetadataSchema',
|
||||
] as const
|
||||
|
||||
// Fields entered only in "Use Supabase" mode (optional in the validation data).
|
||||
const DUCKLAKE_SUPABASE_FIELD_PATHS = [
|
||||
'ducklakeCatalogProjectRef',
|
||||
'ducklakeStorageProjectRef',
|
||||
'ducklakeStorageBucket',
|
||||
] as const
|
||||
|
||||
type DucklakeFieldPath =
|
||||
| (typeof DUCKLAKE_INPUT_FIELD_PATHS)[number]
|
||||
| (typeof DUCKLAKE_SUPABASE_FIELD_PATHS)[number]
|
||||
|
||||
export type DucklakeValidationIssue = {
|
||||
path: DucklakeFieldPath
|
||||
message: string
|
||||
}
|
||||
|
||||
type DucklakeValidationData = Pick<
|
||||
DestinationPanelSchemaType,
|
||||
(typeof DUCKLAKE_INPUT_FIELD_PATHS)[number]
|
||||
> &
|
||||
Partial<
|
||||
Pick<
|
||||
DestinationPanelSchemaType,
|
||||
'ducklakeMode' | (typeof DUCKLAKE_SUPABASE_FIELD_PATHS)[number]
|
||||
>
|
||||
>
|
||||
|
||||
// Required fields per mode. "Use Supabase" only needs project refs + a bucket; the catalog URL and
|
||||
// S3 credentials are resolved by the platform API.
|
||||
const DUCKLAKE_SUPABASE_REQUIRED_FIELDS: DucklakeValidationIssue[] = [
|
||||
{ path: 'ducklakeCatalogProjectRef', message: 'Catalog project is required' },
|
||||
{ path: 'ducklakeStorageProjectRef', message: 'Storage project is required' },
|
||||
{ path: 'ducklakeStorageBucket', message: 'Bucket is required' },
|
||||
]
|
||||
|
||||
const DUCKLAKE_CUSTOM_REQUIRED_FIELDS: DucklakeValidationIssue[] = [
|
||||
{ path: 'ducklakeCatalogUrl', message: 'Catalog URL is required' },
|
||||
{ path: 'ducklakeDataPath', message: 'Data path is required' },
|
||||
{ path: 'ducklakeS3AccessKeyId', message: 'S3 Access Key ID is required' },
|
||||
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 Secret Access Key is required' },
|
||||
{ path: 'ducklakeS3Region', message: 'S3 Region is required' },
|
||||
{ path: 'ducklakeS3Endpoint', message: 'S3 Endpoint is required' },
|
||||
]
|
||||
|
||||
// Catalog metadata schema is optional, but must be a valid Postgres identifier when set.
|
||||
const METADATA_SCHEMA_PATTERN = /^[A-Za-z0-9_]+$/
|
||||
const METADATA_SCHEMA_ISSUE: DucklakeValidationIssue = {
|
||||
path: 'ducklakeMetadataSchema',
|
||||
message: 'DuckLake metadata schema must contain only letters, numbers, and underscores',
|
||||
}
|
||||
|
||||
const getMissingRequiredFieldIssues = (
|
||||
data: DucklakeValidationData,
|
||||
requiredFields: DucklakeValidationIssue[]
|
||||
) => requiredFields.filter(({ path }) => !data[path]?.trim().length)
|
||||
|
||||
export const getDucklakeValidationIssues = (
|
||||
data: DucklakeValidationData
|
||||
): DucklakeValidationIssue[] => {
|
||||
if (data.ducklakeMode === DUCKLAKE_MODE_SUPABASE) {
|
||||
const issues = getMissingRequiredFieldIssues(data, DUCKLAKE_SUPABASE_REQUIRED_FIELDS)
|
||||
|
||||
if (data.ducklakeMetadataSchema && !METADATA_SCHEMA_PATTERN.test(data.ducklakeMetadataSchema)) {
|
||||
issues.push(METADATA_SCHEMA_ISSUE)
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
|
||||
const issues = getMissingRequiredFieldIssues(data, DUCKLAKE_CUSTOM_REQUIRED_FIELDS)
|
||||
|
||||
// Format checks only apply once a value is present; missing values are already flagged above.
|
||||
if (
|
||||
data.ducklakeCatalogUrl?.trim().length &&
|
||||
!data.ducklakeCatalogUrl.startsWith('postgres://') &&
|
||||
!data.ducklakeCatalogUrl.startsWith('postgresql://')
|
||||
) {
|
||||
issues.push({
|
||||
path: 'ducklakeCatalogUrl',
|
||||
message: 'DuckLake catalog URL must be a PostgreSQL-compatible URL',
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
data.ducklakeDataPath?.trim().length &&
|
||||
(!data.ducklakeDataPath.startsWith('s3://') || data.ducklakeDataPath.includes('file://'))
|
||||
) {
|
||||
issues.push({
|
||||
path: 'ducklakeDataPath',
|
||||
message: 'DuckLake data path must start with s3:// and cannot contain file://',
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
data.ducklakeS3Endpoint?.trim().length &&
|
||||
(data.ducklakeS3Endpoint.startsWith('http://') ||
|
||||
data.ducklakeS3Endpoint.startsWith('https://'))
|
||||
) {
|
||||
issues.push({
|
||||
path: 'ducklakeS3Endpoint',
|
||||
message: 'S3 endpoint should not contain the protocol scheme',
|
||||
})
|
||||
}
|
||||
|
||||
if (data.ducklakeMetadataSchema && !METADATA_SCHEMA_PATTERN.test(data.ducklakeMetadataSchema)) {
|
||||
issues.push(METADATA_SCHEMA_ISSUE)
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
+829
@@ -0,0 +1,829 @@
|
||||
import { Check, Database, Eye, EyeOff, Loader2, Plus, SlidersHorizontal } from 'lucide-react'
|
||||
import { useMemo, useState } from 'react'
|
||||
import type { UseFormReturn } from 'react-hook-form'
|
||||
import { toast } from 'sonner'
|
||||
import {
|
||||
Button,
|
||||
cn,
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogSection,
|
||||
DialogSectionSeparator,
|
||||
DialogTitle,
|
||||
FormControl,
|
||||
FormField,
|
||||
Input,
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectGroup,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
WarningIcon,
|
||||
} from 'ui'
|
||||
import { Admonition } from 'ui-patterns'
|
||||
import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
|
||||
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
import {
|
||||
DUCKLAKE_MODE_CUSTOM,
|
||||
DUCKLAKE_MODE_SUPABASE,
|
||||
type DucklakeMode,
|
||||
} from './DuckLake.constants'
|
||||
import { useOrgProjectsInfiniteQuery } from '@/data/projects/org-projects-infinite-query'
|
||||
import { useBucketCreateMutation } from '@/data/storage/bucket-create-mutation'
|
||||
import { usePaginatedBucketsQuery } from '@/data/storage/buckets-query'
|
||||
import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganization'
|
||||
import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject'
|
||||
import { PROJECT_STATUS } from '@/lib/constants'
|
||||
|
||||
const DUCKLAKE_MODE_OPTIONS = [
|
||||
{
|
||||
value: DUCKLAKE_MODE_SUPABASE,
|
||||
icon: Database,
|
||||
label: 'Use Supabase',
|
||||
description:
|
||||
'Create or use a DuckLake backed by your Supabase projects. Catalog and storage are managed for you.',
|
||||
},
|
||||
{
|
||||
value: DUCKLAKE_MODE_CUSTOM,
|
||||
icon: SlidersHorizontal,
|
||||
label: 'Custom parameters',
|
||||
description: 'Bring your own PostgreSQL catalog and S3-compatible object storage credentials.',
|
||||
},
|
||||
] as const
|
||||
|
||||
const DuckLakeModeSelector = ({
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
value: DucklakeMode
|
||||
onChange: (value: DucklakeMode) => void
|
||||
}) => {
|
||||
return (
|
||||
<div
|
||||
role="radiogroup"
|
||||
aria-label="DuckLake configuration mode"
|
||||
className="grid grid-cols-2 gap-3"
|
||||
>
|
||||
{DUCKLAKE_MODE_OPTIONS.map((option) => {
|
||||
const Icon = option.icon
|
||||
const selected = value === option.value
|
||||
return (
|
||||
<button
|
||||
key={option.value}
|
||||
type="button"
|
||||
role="radio"
|
||||
aria-checked={selected}
|
||||
onClick={() => onChange(option.value)}
|
||||
className={cn(
|
||||
'relative flex flex-col gap-y-3 rounded-md border p-4 text-left transition',
|
||||
'hover:border-foreground-muted',
|
||||
selected
|
||||
? 'border-foreground-muted bg-surface-300 ring-1 ring-border'
|
||||
: 'border-default bg-surface-100'
|
||||
)}
|
||||
>
|
||||
<div className="flex items-start justify-between">
|
||||
<Icon size={18} strokeWidth={1.5} className="text-foreground-light" />
|
||||
{selected ? (
|
||||
<Check size={16} className="text-brand" />
|
||||
) : (
|
||||
<span className="h-4 w-4 rounded-full border border-strong" />
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<span className="text-sm text-foreground">{option.label}</span>
|
||||
<span className="text-xs text-foreground-light">{option.description}</span>
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const DuckLakeSupabaseFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
|
||||
const { ducklakeStorageProjectRef } = form.watch()
|
||||
|
||||
const [showNewBucketDialog, setShowNewBucketDialog] = useState(false)
|
||||
const [newBucketName, setNewBucketName] = useState('')
|
||||
|
||||
const { data: organization } = useSelectedOrganizationQuery()
|
||||
const { data: sourceProject } = useSelectedProjectQuery()
|
||||
const sourceRegion = sourceProject?.region
|
||||
|
||||
const { data: projectsData } = useOrgProjectsInfiniteQuery(
|
||||
{ slug: organization?.slug, statuses: [PROJECT_STATUS.ACTIVE_HEALTHY] },
|
||||
{ enabled: !!organization?.slug }
|
||||
)
|
||||
|
||||
const projects = useMemo(
|
||||
() =>
|
||||
(projectsData?.pages.flatMap((page) => page.projects) ?? []).filter(
|
||||
(project) => !project.is_branch
|
||||
),
|
||||
[projectsData]
|
||||
)
|
||||
const projectsByRef = useMemo(
|
||||
() => new Map(projects.map((project) => [project.ref, project])),
|
||||
[projects]
|
||||
)
|
||||
|
||||
const regionForRef = (ref?: string) => {
|
||||
if (!ref) return undefined
|
||||
const region = projectsByRef.get(ref)?.region
|
||||
if (region) return region
|
||||
return ref === sourceProject?.ref ? sourceProject?.region : undefined
|
||||
}
|
||||
|
||||
const { mutate: createBucket, isPending: isCreatingBucket } = useBucketCreateMutation({
|
||||
onSuccess: (_, vars) => {
|
||||
form.setValue('ducklakeStorageBucket', vars.id)
|
||||
setNewBucketName('')
|
||||
setShowNewBucketDialog(false)
|
||||
},
|
||||
})
|
||||
|
||||
const handleCreateBucket = async () => {
|
||||
const name = newBucketName.trim()
|
||||
if (!name || !ducklakeStorageProjectRef) return
|
||||
if (name.includes('/')) {
|
||||
return toast.error('Bucket name cannot contain "/"')
|
||||
}
|
||||
|
||||
createBucket({
|
||||
projectRef: ducklakeStorageProjectRef,
|
||||
id: name,
|
||||
type: 'STANDARD',
|
||||
isPublic: false,
|
||||
})
|
||||
}
|
||||
|
||||
const renderRegionWarning = (ref?: string) => {
|
||||
const region = regionForRef(ref)
|
||||
if (!region || !sourceRegion || region === sourceRegion) return null
|
||||
return (
|
||||
<Admonition
|
||||
type="warning"
|
||||
className="mb-0"
|
||||
description={`This project is in ${region}, a different region than your source project (${sourceRegion}). Cross-region replication can add noticeable latency.`}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6">
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Catalog</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
The selected project's Postgres database is used as the PostgreSQL DuckLake catalog
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeCatalogProjectRef"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Catalog project"
|
||||
description={
|
||||
<div className="flex flex-col gap-y-2">
|
||||
{renderRegionWarning(field.value)}
|
||||
<span>
|
||||
Warehouse connects to this project's Postgres instance to store the DuckLake
|
||||
catalog
|
||||
</span>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<FormControl>
|
||||
<ProjectSelection
|
||||
value={field.value}
|
||||
onChange={field.onChange}
|
||||
placeholder="Select a project"
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakePoolSize"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Pool size"
|
||||
description="Number of concurrent DuckDB connections to the catalog"
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={6}
|
||||
value={field.value ?? ''}
|
||||
placeholder="Default: 4"
|
||||
onChange={(event) =>
|
||||
field.onChange(event.target.value === '' ? undefined : Number(event.target.value))
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeMetadataSchema"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Metadata schema"
|
||||
description="Schema used for DuckLake metadata tables in the catalog's Postgres"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ducklake" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Object storage</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Replicated data files are written to a Storage bucket in the selected project.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeStorageProjectRef"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Storage project"
|
||||
description={
|
||||
<div className="flex flex-col gap-y-2">
|
||||
{renderRegionWarning(field.value)}
|
||||
<span>The project whose object storage holds the DuckLake data files</span>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<FormControl>
|
||||
<ProjectSelection
|
||||
value={field.value}
|
||||
onChange={(value) => {
|
||||
field.onChange(value)
|
||||
// Buckets are project-scoped, so clear the selection when the project changes
|
||||
form.setValue('ducklakeStorageBucket', '')
|
||||
}}
|
||||
placeholder="Select a project"
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeStorageBucket"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Bucket"
|
||||
description="The bucket in which DuckLake data files will be stored."
|
||||
>
|
||||
<div className="flex items-center gap-x-2">
|
||||
<div className="grow">
|
||||
<FormControl>
|
||||
<BucketSelection form={form} value={field.value} onChange={field.onChange} />
|
||||
</FormControl>
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
variant="default"
|
||||
icon={<Plus />}
|
||||
disabled={!ducklakeStorageProjectRef}
|
||||
onClick={() => setShowNewBucketDialog(true)}
|
||||
>
|
||||
New bucket
|
||||
</Button>
|
||||
</div>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Dialog open={showNewBucketDialog} onOpenChange={setShowNewBucketDialog}>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Create a new file bucket</DialogTitle>
|
||||
</DialogHeader>
|
||||
<DialogSectionSeparator />
|
||||
<DialogSection className="flex flex-col gap-y-2">
|
||||
<label htmlFor="ducklake-new-bucket-name" className="text-sm text-foreground-light">
|
||||
Bucket name
|
||||
</label>
|
||||
<Input
|
||||
id="ducklake-new-bucket-name"
|
||||
value={newBucketName}
|
||||
placeholder="ducklake-data"
|
||||
onChange={(event) => setNewBucketName(event.target.value)}
|
||||
/>
|
||||
</DialogSection>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
type="button"
|
||||
variant="default"
|
||||
disabled={isCreatingBucket}
|
||||
onClick={() => setShowNewBucketDialog(false)}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
loading={isCreatingBucket}
|
||||
disabled={!newBucketName.trim()}
|
||||
onClick={handleCreateBucket}
|
||||
>
|
||||
Create bucket
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const DuckLakeCustomFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
|
||||
const [showCatalogUrl, setShowCatalogUrl] = useState(false)
|
||||
const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6">
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Catalog</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Configure the PostgreSQL DuckLake catalog and S3-compatible storage for replicated data
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeCatalogUrl"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Catalog URL"
|
||||
description="A PostgreSQL connection string for the DuckLake catalog"
|
||||
>
|
||||
<FormControl>
|
||||
<PasswordInput
|
||||
value={field.value ?? ''}
|
||||
type={showCatalogUrl ? 'text' : 'password'}
|
||||
placeholder="postgres://user:pass@host:5432/ducklake_catalog"
|
||||
onChange={(event) => field.onChange(event.target.value)}
|
||||
actions={
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
icon={showCatalogUrl ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowCatalogUrl(!showCatalogUrl)}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeDataPath"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Data path"
|
||||
description="An S3 path where DuckLake data files will be written"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="s3://bucket/path" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakePoolSize"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Pool size"
|
||||
description="Number of concurrent DuckDB connections to use"
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={6}
|
||||
value={field.value ?? ''}
|
||||
placeholder="Default: 4"
|
||||
onChange={(event) =>
|
||||
field.onChange(
|
||||
event.target.value === '' ? undefined : Number(event.target.value)
|
||||
)
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Object storage</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Optional credentials and endpoint settings for S3-compatible storage providers.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3AccessKeyId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Access Key ID"
|
||||
description="Required access key ID for the object storage provider"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="my-access-key" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3SecretAccessKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Secret Access Key"
|
||||
description="Required secret access key for the object storage provider"
|
||||
className="relative"
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
type={showSecretAccessKey ? 'text' : 'password'}
|
||||
placeholder="my-secret-key"
|
||||
value={field.value ?? ''}
|
||||
/>
|
||||
</FormControl>
|
||||
<Button
|
||||
variant="default"
|
||||
icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
|
||||
className="w-7 absolute right-6 top-[4px]"
|
||||
onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3Region"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Region"
|
||||
description="Required region for the object storage provider"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="us-east-1" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3Endpoint"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 Endpoint"
|
||||
description="Required endpoint without the protocol scheme, for example `127.0.0.1:5000/s3`"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="127.0.0.1:5000/s3" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3UrlStyle"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="S3 URL style"
|
||||
description="Choose `path` for MinIO/Supabase-style endpoints or `vhost` for AWS-style virtual host addressing"
|
||||
>
|
||||
<FormControl>
|
||||
<Select value={field.value ?? 'path'} onValueChange={field.onChange}>
|
||||
<SelectTrigger>{field.value ?? 'path'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="path">path</SelectItem>
|
||||
<SelectItem value="vhost">vhost</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeS3UseSsl"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Use SSL"
|
||||
description="Whether to use SSL when connecting to the S3-compatible endpoint"
|
||||
>
|
||||
<FormControl>
|
||||
<Select
|
||||
value={field.value === false ? 'false' : 'true'}
|
||||
onValueChange={(value) => field.onChange(value === 'true')}
|
||||
>
|
||||
<SelectTrigger>{field.value === false ? 'false' : 'true'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="true">true</SelectItem>
|
||||
<SelectItem value="false">false</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Metadata</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Optional schema setting for DuckLake metadata tables.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ducklakeMetadataSchema"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Metadata schema"
|
||||
description="Schema used for DuckLake metadata tables in PostgreSQL"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ducklake" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const DuckLakeFields = ({
|
||||
form,
|
||||
editMode,
|
||||
}: {
|
||||
form: UseFormReturn<DestinationPanelSchemaType>
|
||||
editMode: boolean
|
||||
}) => {
|
||||
const ducklakeMode = (form.watch('ducklakeMode') ?? DUCKLAKE_MODE_SUPABASE) as DucklakeMode
|
||||
// The platform API resolves "Use Supabase" config into a flat catalog URL + provisioned S3
|
||||
// credentials before persisting, so an existing destination can only be edited as custom
|
||||
// parameters — the original project selections aren't recoverable.
|
||||
const effectiveMode = editMode ? DUCKLAKE_MODE_CUSTOM : ducklakeMode
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">DuckLake settings</p>
|
||||
|
||||
{!editMode && (
|
||||
<div className="flex flex-col gap-y-3">
|
||||
<p className="text-xs uppercase tracking-wider text-foreground-lighter">
|
||||
How should this DuckLake be configured?
|
||||
</p>
|
||||
<DuckLakeModeSelector
|
||||
value={effectiveMode}
|
||||
onChange={(value) =>
|
||||
form.setValue('ducklakeMode', value, { shouldValidate: true, shouldDirty: true })
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{effectiveMode === DUCKLAKE_MODE_SUPABASE ? (
|
||||
<DuckLakeSupabaseFields form={form} />
|
||||
) : (
|
||||
<DuckLakeCustomFields form={form} />
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const ProjectSelection = ({
|
||||
value,
|
||||
onChange,
|
||||
placeholder,
|
||||
}: {
|
||||
value: string | undefined
|
||||
onChange: (value: string) => void
|
||||
placeholder: string
|
||||
}) => {
|
||||
const { data: organization } = useSelectedOrganizationQuery()
|
||||
|
||||
const {
|
||||
data: projectsData,
|
||||
isPending: isLoadingProjects,
|
||||
isError: isErrorProjects,
|
||||
} = useOrgProjectsInfiniteQuery(
|
||||
{ slug: organization?.slug, statuses: [PROJECT_STATUS.ACTIVE_HEALTHY] },
|
||||
{ enabled: !!organization?.slug }
|
||||
)
|
||||
|
||||
const projects = useMemo(
|
||||
() =>
|
||||
(projectsData?.pages.flatMap((page) => page.projects) ?? []).filter(
|
||||
(project) => !project.is_branch
|
||||
),
|
||||
[projectsData]
|
||||
)
|
||||
|
||||
const projectsByRef = useMemo(
|
||||
() => new Map(projects.map((project) => [project.ref, project])),
|
||||
[projects]
|
||||
)
|
||||
|
||||
const projectLabel = (ref?: string) => {
|
||||
if (!ref) return undefined
|
||||
const project = projectsByRef.get(ref)
|
||||
return project ? `${project.name} · ${project.ref}` : ref
|
||||
}
|
||||
|
||||
if (isLoadingProjects) {
|
||||
return (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving projects
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
if (isErrorProjects) {
|
||||
return (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve projects
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<Select value={value || ''} onValueChange={onChange}>
|
||||
<SelectTrigger>{projectLabel(value) ?? placeholder}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{projects.length === 0 ? (
|
||||
<SelectItem value="__no_projects__" disabled>
|
||||
No active projects available
|
||||
</SelectItem>
|
||||
) : (
|
||||
projects.map((project) => (
|
||||
<SelectItem key={project.ref} value={project.ref}>
|
||||
<div className="flex flex-col">
|
||||
<span>{project.name}</span>
|
||||
<span className="text-foreground-lighter">
|
||||
{project.ref} · {project.region}
|
||||
</span>
|
||||
</div>
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
)
|
||||
}
|
||||
|
||||
const BucketSelection = ({
|
||||
form,
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
form: UseFormReturn<DestinationPanelSchemaType>
|
||||
value: string | undefined
|
||||
onChange: (value: string) => void
|
||||
}) => {
|
||||
const { ducklakeStorageProjectRef } = form.watch()
|
||||
|
||||
const {
|
||||
data: bucketsData,
|
||||
isPending: isLoadingBuckets,
|
||||
isError: isErrorBuckets,
|
||||
} = usePaginatedBucketsQuery(
|
||||
{ projectRef: ducklakeStorageProjectRef },
|
||||
{ enabled: !!ducklakeStorageProjectRef }
|
||||
)
|
||||
|
||||
const buckets = useMemo(
|
||||
() =>
|
||||
(bucketsData?.pages.flat() ?? []).filter(
|
||||
(bucket) => !bucket.type || bucket.type === 'STANDARD'
|
||||
),
|
||||
[bucketsData]
|
||||
)
|
||||
|
||||
if (!ducklakeStorageProjectRef) {
|
||||
return (
|
||||
<Button disabled variant="default" className="w-full justify-start" size="small">
|
||||
Select a storage project first
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
if (isLoadingBuckets) {
|
||||
return (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-between"
|
||||
size="small"
|
||||
iconRight={<Loader2 className="animate-spin" />}
|
||||
>
|
||||
Retrieving buckets
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
if (isErrorBuckets) {
|
||||
return (
|
||||
<Button
|
||||
disabled
|
||||
variant="default"
|
||||
className="w-full justify-start"
|
||||
size="small"
|
||||
icon={<WarningIcon />}
|
||||
>
|
||||
Failed to retrieve buckets
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<Select
|
||||
value={value || ''}
|
||||
onValueChange={(e) => {
|
||||
if (e) onChange(e)
|
||||
}}
|
||||
>
|
||||
<SelectTrigger>{value || 'Select a bucket'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectGroup>
|
||||
{buckets.length === 0 ? (
|
||||
<SelectItem value="__no_buckets__" disabled>
|
||||
No buckets available
|
||||
</SelectItem>
|
||||
) : (
|
||||
buckets.map((bucket) => (
|
||||
<SelectItem key={bucket.id} value={bucket.id}>
|
||||
{bucket.name}
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
)
|
||||
}
|
||||
+5
-5
@@ -27,7 +27,7 @@ import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject'
|
||||
interface NewPublicationPanelProps {
|
||||
visible: boolean
|
||||
sourceId?: number
|
||||
onClose: () => void
|
||||
onClose: (newPublication?: string) => void
|
||||
}
|
||||
|
||||
export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicationPanelProps) => {
|
||||
@@ -38,10 +38,10 @@ export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicati
|
||||
|
||||
const { mutate: createPublication, isPending: creatingPublication } =
|
||||
useCreatePublicationMutation({
|
||||
onSuccess: () => {
|
||||
onSuccess: (_, vars) => {
|
||||
toast.success('Successfully created publication')
|
||||
form.reset(defaultValues)
|
||||
onClose()
|
||||
onClose(vars.name)
|
||||
},
|
||||
})
|
||||
|
||||
@@ -82,7 +82,7 @@ export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicati
|
||||
|
||||
return (
|
||||
<>
|
||||
<Sheet open={visible} onOpenChange={onClose}>
|
||||
<Sheet open={visible} onOpenChange={() => onClose()}>
|
||||
<SheetContent size="default">
|
||||
<div className="flex flex-col h-full">
|
||||
<SheetHeader>
|
||||
@@ -147,7 +147,7 @@ export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicati
|
||||
</Form>
|
||||
</SheetSection>
|
||||
<SheetFooter>
|
||||
<Button variant="default" disabled={creatingPublication} onClick={onClose}>
|
||||
<Button variant="default" disabled={creatingPublication} onClick={() => onClose()}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant="primary" disabled={creatingPublication} form={formId} type="submit">
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
import { Eye, EyeOff } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
import type { UseFormReturn } from 'react-hook-form'
|
||||
import { Button, FormControl, FormField, Input, TextArea } from 'ui'
|
||||
import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
|
||||
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
|
||||
export const SnowflakeFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
|
||||
const [showPrivateKeyPassphrase, setShowPrivateKeyPassphrase] = useState(false)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">Snowflake settings</p>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Connection</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Configure the Snowflake account, user, and target namespace for replicated data.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeAccountId"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Account ID"
|
||||
description="Snowflake account identifier, for example ORGNAME-ACCOUNTNAME"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="MYORG-MYACCOUNT" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeUser"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="User"
|
||||
description="Snowflake user configured for key-pair authentication"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ETL_USER" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeDatabase"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Database"
|
||||
description="Snowflake database where replicated tables will be created"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ANALYTICS" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeSchema"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Schema"
|
||||
description="Snowflake schema where replicated tables will be created"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="PUBLIC" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakeRole"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Role"
|
||||
description="Optional Snowflake role to assume after connecting"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder="ETL_ROLE" value={field.value ?? ''} />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<p className="text-sm font-medium text-foreground">Authentication</p>
|
||||
<p className="text-sm text-foreground-light">
|
||||
Use the RSA private key whose public key is registered on the Snowflake user.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakePrivateKey"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Private key"
|
||||
description="RSA private key PEM contents in PKCS#8 or PKCS#1 format"
|
||||
>
|
||||
<FormControl>
|
||||
<TextArea
|
||||
{...field}
|
||||
rows={8}
|
||||
maxLength={10000}
|
||||
placeholder={'-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----'}
|
||||
value={field.value ?? ''}
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="snowflakePrivateKeyPassphrase"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Private key passphrase"
|
||||
description="Optional passphrase for encrypted private keys"
|
||||
>
|
||||
<FormControl>
|
||||
<PasswordInput
|
||||
value={field.value ?? ''}
|
||||
type={showPrivateKeyPassphrase ? 'text' : 'password'}
|
||||
placeholder="Optional"
|
||||
onChange={(event) => field.onChange(event.target.value)}
|
||||
actions={
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
icon={showPrivateKeyPassphrase ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowPrivateKeyPassphrase(!showPrivateKeyPassphrase)}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import { type DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
|
||||
export type SnowflakeApiConfig = {
|
||||
account_id: string
|
||||
user: string
|
||||
private_key: string
|
||||
private_key_passphrase?: string
|
||||
database: string
|
||||
schema: string
|
||||
role?: string
|
||||
}
|
||||
|
||||
type SnowflakeFieldPath =
|
||||
| 'snowflakeAccountId'
|
||||
| 'snowflakeUser'
|
||||
| 'snowflakePrivateKey'
|
||||
| 'snowflakeDatabase'
|
||||
| 'snowflakeSchema'
|
||||
|
||||
export type SnowflakeValidationIssue = {
|
||||
path: SnowflakeFieldPath
|
||||
message: string
|
||||
}
|
||||
|
||||
const SNOWFLAKE_REQUIRED_FIELDS: { path: SnowflakeFieldPath; message: string }[] = [
|
||||
{ path: 'snowflakeAccountId', message: 'Account ID is required' },
|
||||
{ path: 'snowflakeUser', message: 'User is required' },
|
||||
{ path: 'snowflakePrivateKey', message: 'Private key is required' },
|
||||
{ path: 'snowflakeDatabase', message: 'Database is required' },
|
||||
{ path: 'snowflakeSchema', message: 'Schema is required' },
|
||||
]
|
||||
|
||||
export const getSnowflakeValidationIssues = (
|
||||
data: Pick<DestinationPanelSchemaType, SnowflakeFieldPath>
|
||||
): SnowflakeValidationIssue[] =>
|
||||
SNOWFLAKE_REQUIRED_FIELDS.filter(({ path }) => !data[path]?.trim().length)
|
||||
+32
-129
@@ -17,25 +17,26 @@ import {
|
||||
} from '../../useIsETLPrivateAlpha'
|
||||
import { DestinationType } from '../DestinationPanel.types'
|
||||
import { AdvancedSettings } from './AdvancedSettings'
|
||||
import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsBucket.utils'
|
||||
import { AnalyticsBucketFields } from './AnalyticsBucket/Fields'
|
||||
import { getBigQueryValidationIssues } from './BigQuery/BigQuery.utils'
|
||||
import { BigQueryFields } from './BigQuery/Fields'
|
||||
import { CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
|
||||
import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
|
||||
import {
|
||||
areValidationFailuresEqual,
|
||||
buildDestinationConfig,
|
||||
buildDestinationConfigForValidation,
|
||||
getDucklakeValidationIssues,
|
||||
getSnowflakeValidationIssues,
|
||||
generateDefaultValues,
|
||||
} from './DestinationForm.utils'
|
||||
import { DestinationNameInput } from './DestinationNameInput'
|
||||
import {
|
||||
AnalyticsBucketFields,
|
||||
BigQueryFields,
|
||||
DuckLakeFields,
|
||||
SnowflakeFields,
|
||||
} from './DestinationPanelFields'
|
||||
import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
|
||||
import { DuckLakeFields } from './DuckLake/Fields'
|
||||
import { NewPublicationPanel } from './NewPublicationPanel'
|
||||
import { NoDestinationsAvailable } from './NoDestinationsAvailable'
|
||||
import { PublicationSelection } from './PublicationSelection'
|
||||
import { SnowflakeFields } from './Snowflake/Fields'
|
||||
import { getSnowflakeValidationIssues } from './Snowflake/Snowflake.utils'
|
||||
import { ValidationFailuresSection } from './ValidationFailuresSection'
|
||||
import { ValidationWarningsDialog } from './ValidationWarningsDialog'
|
||||
import { CreateAnalyticsBucketSheet } from '@/components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'
|
||||
@@ -81,29 +82,6 @@ interface DestinationFormProps {
|
||||
onClose: () => void
|
||||
}
|
||||
|
||||
type DucklakeApiConfig = {
|
||||
catalog_url: string
|
||||
data_path: string
|
||||
pool_size?: number
|
||||
s3_access_key_id?: string
|
||||
s3_secret_access_key?: string
|
||||
s3_region?: string
|
||||
s3_endpoint?: string
|
||||
s3_url_style?: 'path' | 'vhost'
|
||||
s3_use_ssl?: boolean
|
||||
metadata_schema?: string
|
||||
}
|
||||
|
||||
type SnowflakeApiConfig = {
|
||||
account_id: string
|
||||
user: string
|
||||
private_key: string
|
||||
private_key_passphrase?: string
|
||||
database: string
|
||||
schema: string
|
||||
role?: string
|
||||
}
|
||||
|
||||
export const DestinationForm = ({
|
||||
selectedType,
|
||||
visible,
|
||||
@@ -208,75 +186,18 @@ export const DestinationForm = ({
|
||||
|
||||
const isValidating = isValidatingDestination || isValidatingPipeline
|
||||
|
||||
const defaultValues = useMemo(() => {
|
||||
const config = destinationData?.config
|
||||
const isBigQueryConfig = config && 'big_query' in config
|
||||
const isIcebergConfig = config && 'iceberg' in config
|
||||
const ducklakeConfigValue =
|
||||
config && 'ducklake' in (config as Record<string, unknown>)
|
||||
? (config as Record<string, unknown>).ducklake
|
||||
: undefined
|
||||
const ducklakeConfig =
|
||||
ducklakeConfigValue && typeof ducklakeConfigValue === 'object'
|
||||
? (ducklakeConfigValue as DucklakeApiConfig)
|
||||
: undefined
|
||||
const snowflakeConfigValue =
|
||||
config && 'snowflake' in (config as Record<string, unknown>)
|
||||
? (config as Record<string, unknown>).snowflake
|
||||
: undefined
|
||||
const snowflakeConfig =
|
||||
snowflakeConfigValue && typeof snowflakeConfigValue === 'object'
|
||||
? (snowflakeConfigValue as SnowflakeApiConfig)
|
||||
: undefined
|
||||
|
||||
return {
|
||||
// Common fields
|
||||
name: destinationData?.name ?? '',
|
||||
publicationName: pipelineData?.config.publication_name ?? '',
|
||||
maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? undefined,
|
||||
maxTableSyncWorkers: pipelineData?.config?.max_table_sync_workers ?? undefined,
|
||||
maxCopyConnectionsPerTable: pipelineData?.config?.max_copy_connections_per_table ?? undefined,
|
||||
invalidatedSlotBehavior:
|
||||
(pipelineData?.config as { invalidated_slot_behavior?: 'error' | 'recreate' } | undefined)
|
||||
?.invalidated_slot_behavior ?? undefined,
|
||||
// BigQuery fields
|
||||
projectId: isBigQueryConfig ? config.big_query.project_id : '',
|
||||
datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',
|
||||
serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',
|
||||
connectionPoolSize:
|
||||
(config as { big_query?: { connection_pool_size?: number } } | undefined)?.big_query
|
||||
?.connection_pool_size ?? undefined,
|
||||
maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined, // Default: null
|
||||
// Analytics Bucket fields
|
||||
warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',
|
||||
namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '',
|
||||
newNamespaceName: '',
|
||||
catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken,
|
||||
s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '',
|
||||
s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '',
|
||||
s3Region:
|
||||
projectSettings?.region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''),
|
||||
// DuckLake fields
|
||||
ducklakeCatalogUrl: ducklakeConfig?.catalog_url ?? '',
|
||||
ducklakeDataPath: ducklakeConfig?.data_path ?? '',
|
||||
ducklakePoolSize: ducklakeConfig?.pool_size,
|
||||
ducklakeS3AccessKeyId: ducklakeConfig?.s3_access_key_id ?? '',
|
||||
ducklakeS3SecretAccessKey: ducklakeConfig?.s3_secret_access_key ?? '',
|
||||
ducklakeS3Region: ducklakeConfig?.s3_region ?? '',
|
||||
ducklakeS3Endpoint: ducklakeConfig?.s3_endpoint ?? '',
|
||||
ducklakeS3UrlStyle: ducklakeConfig?.s3_url_style ?? 'path',
|
||||
ducklakeS3UseSsl: ducklakeConfig?.s3_use_ssl ?? true,
|
||||
ducklakeMetadataSchema: ducklakeConfig?.metadata_schema ?? 'ducklake',
|
||||
// Snowflake fields
|
||||
snowflakeAccountId: snowflakeConfig?.account_id ?? '',
|
||||
snowflakeUser: snowflakeConfig?.user ?? '',
|
||||
snowflakePrivateKey: snowflakeConfig?.private_key ?? '',
|
||||
snowflakePrivateKeyPassphrase: snowflakeConfig?.private_key_passphrase ?? '',
|
||||
snowflakeDatabase: snowflakeConfig?.database ?? '',
|
||||
snowflakeSchema: snowflakeConfig?.schema ?? '',
|
||||
snowflakeRole: snowflakeConfig?.role ?? '',
|
||||
}
|
||||
}, [destinationData, pipelineData, catalogToken, projectSettings])
|
||||
const defaultValues = useMemo(
|
||||
() =>
|
||||
generateDefaultValues({
|
||||
destinationData,
|
||||
pipelineData,
|
||||
catalogToken,
|
||||
region: projectSettings?.region,
|
||||
projectRef,
|
||||
editMode,
|
||||
}),
|
||||
[destinationData, pipelineData, catalogToken, projectSettings, projectRef, editMode]
|
||||
)
|
||||
|
||||
const form = useForm<z.infer<typeof FormSchema>>({
|
||||
mode: 'onChange',
|
||||
@@ -292,34 +213,13 @@ export const DestinationForm = ({
|
||||
}
|
||||
|
||||
if (selectedType === 'BigQuery') {
|
||||
if (!data.projectId?.length) addRequiredFieldError('projectId', 'Project ID is required')
|
||||
if (!data.datasetId?.length) addRequiredFieldError('datasetId', 'Dataset ID is required')
|
||||
if (!data.serviceAccountKey?.length)
|
||||
addRequiredFieldError('serviceAccountKey', 'Service Account Key is required')
|
||||
getBigQueryValidationIssues(data).forEach(({ path, message }) => {
|
||||
addRequiredFieldError(path, message)
|
||||
})
|
||||
} else if (selectedType === 'Analytics Bucket') {
|
||||
if (!data.warehouseName?.length)
|
||||
addRequiredFieldError('warehouseName', 'Bucket is required')
|
||||
|
||||
const hasValidNamespace =
|
||||
(data.namespace?.length && data.namespace !== 'create-new-namespace') ||
|
||||
(data.namespace === 'create-new-namespace' && data.newNamespaceName?.length)
|
||||
|
||||
if (!hasValidNamespace) {
|
||||
const isCreatingNew = data.namespace === 'create-new-namespace'
|
||||
addRequiredFieldError(
|
||||
isCreatingNew ? 'newNamespaceName' : 'namespace',
|
||||
isCreatingNew ? 'Namespace name is required' : 'Namespace is required'
|
||||
)
|
||||
}
|
||||
|
||||
if (!data.s3Region?.length) addRequiredFieldError('s3Region', 'S3 Region is required')
|
||||
|
||||
if (!data.s3AccessKeyId?.length)
|
||||
addRequiredFieldError('s3AccessKeyId', 'S3 Access Key ID is required')
|
||||
|
||||
if (data.s3AccessKeyId !== 'create-new' && !data.s3SecretAccessKey?.length) {
|
||||
addRequiredFieldError('s3SecretAccessKey', 'S3 Secret Access Key is required')
|
||||
}
|
||||
getAnalyticsBucketValidationIssues(data).forEach(({ path, message }) => {
|
||||
addRequiredFieldError(path, message)
|
||||
})
|
||||
} else if (selectedType === 'DuckLake') {
|
||||
getDucklakeValidationIssues(data).forEach(({ path, message }) => {
|
||||
addRequiredFieldError(path, message)
|
||||
@@ -650,7 +550,7 @@ export const DestinationForm = ({
|
||||
onSelectNewBucket={() => setNewBucketSheetVisible(true)}
|
||||
/>
|
||||
) : selectedType === 'DuckLake' && etlEnableDucklake ? (
|
||||
<DuckLakeFields form={form} />
|
||||
<DuckLakeFields form={form} editMode={editMode} />
|
||||
) : selectedType === 'Snowflake' && etlEnableSnowflake ? (
|
||||
<SnowflakeFields form={form} />
|
||||
) : null}
|
||||
@@ -710,7 +610,10 @@ export const DestinationForm = ({
|
||||
<NewPublicationPanel
|
||||
sourceId={sourceId}
|
||||
visible={publicationPanelVisible}
|
||||
onClose={() => setPublicationPanelVisible(false)}
|
||||
onClose={(newPublication?: string) => {
|
||||
if (newPublication) form.setValue('publicationName', newPublication)
|
||||
setPublicationPanelVisible(false)
|
||||
}}
|
||||
/>
|
||||
|
||||
<CreateAnalyticsBucketSheet
|
||||
|
||||
+1
-1
@@ -144,7 +144,7 @@ export const DestinationTypeSelection = () => {
|
||||
labelOptional="Destination type cannot be changed after creation"
|
||||
description={
|
||||
selectedOption?.isAlpha && (
|
||||
<span className="text-sm text-foreground-light mb-1">
|
||||
<span className="block text-sm text-foreground-light mb-1">
|
||||
This destination type is in alpha and may change while we iterate.{' '}
|
||||
<InlineLink href="https://github.com/orgs/supabase/discussions/39416">
|
||||
Leave feedback
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { buildDucklakeApiConfig } from './create-destination-pipeline-mutation'
|
||||
|
||||
describe('buildDucklakeApiConfig', () => {
|
||||
it('maps a "Use Supabase" config with catalog-level pool size + metadata schema', () => {
|
||||
expect(
|
||||
buildDucklakeApiConfig({
|
||||
catalogProjectRef: 'catalog-ref',
|
||||
storageProjectRef: 'storage-ref',
|
||||
bucket: 'ducklake-data',
|
||||
poolSize: 4,
|
||||
metadataSchema: 'ducklake',
|
||||
})
|
||||
).toEqual({
|
||||
ducklake: {
|
||||
catalog: {
|
||||
type: 'supabase_project',
|
||||
project_ref: 'catalog-ref',
|
||||
pool_size: 4,
|
||||
metadata_schema: 'ducklake',
|
||||
},
|
||||
storage: {
|
||||
type: 'supabase_storage',
|
||||
project_ref: 'storage-ref',
|
||||
bucket: 'ducklake-data',
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('includes the optional path prefix when provided', () => {
|
||||
const config = buildDucklakeApiConfig({
|
||||
catalogProjectRef: 'catalog-ref',
|
||||
storageProjectRef: 'storage-ref',
|
||||
bucket: 'ducklake-data',
|
||||
path: 'replication',
|
||||
})
|
||||
|
||||
expect(config).toMatchObject({
|
||||
ducklake: {
|
||||
storage: {
|
||||
type: 'supabase_storage',
|
||||
project_ref: 'storage-ref',
|
||||
bucket: 'ducklake-data',
|
||||
path: 'replication',
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('maps a "Custom parameters" config to the flat snake_case payload', () => {
|
||||
expect(
|
||||
buildDucklakeApiConfig({
|
||||
catalogUrl: 'postgres://user:pass@host:5432/catalog',
|
||||
dataPath: 's3://bucket/path',
|
||||
poolSize: 4,
|
||||
s3AccessKeyId: 'access-key',
|
||||
s3SecretAccessKey: 'secret-key',
|
||||
s3Region: 'eu-west-1',
|
||||
s3Endpoint: 's3.example.com',
|
||||
s3UrlStyle: 'path',
|
||||
s3UseSsl: true,
|
||||
metadataSchema: 'ducklake',
|
||||
})
|
||||
).toEqual({
|
||||
ducklake: {
|
||||
catalog_url: 'postgres://user:pass@host:5432/catalog',
|
||||
data_path: 's3://bucket/path',
|
||||
pool_size: 4,
|
||||
s3_access_key_id: 'access-key',
|
||||
s3_secret_access_key: 'secret-key',
|
||||
s3_region: 'eu-west-1',
|
||||
s3_endpoint: 's3.example.com',
|
||||
s3_url_style: 'path',
|
||||
s3_use_ssl: true,
|
||||
metadata_schema: 'ducklake',
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -7,18 +7,10 @@ import { handleError, post } from '@/data/fetchers'
|
||||
import type { ResponseError, UseCustomMutationOptions } from '@/types'
|
||||
|
||||
export type DestinationConfig =
|
||||
| {
|
||||
bigQuery: BigQueryDestinationConfig
|
||||
}
|
||||
| {
|
||||
iceberg: IcebergDestinationConfig
|
||||
}
|
||||
| {
|
||||
ducklake: DucklakeDestinationConfig
|
||||
}
|
||||
| {
|
||||
snowflake: SnowflakeDestinationConfig
|
||||
}
|
||||
| { bigQuery: BigQueryDestinationConfig }
|
||||
| { iceberg: IcebergDestinationConfig }
|
||||
| { ducklake: DucklakeDestinationConfig }
|
||||
| { snowflake: SnowflakeDestinationConfig }
|
||||
|
||||
export type BigQueryDestinationConfig = {
|
||||
projectId: string
|
||||
@@ -38,7 +30,9 @@ export type IcebergDestinationConfig = {
|
||||
s3Region: string
|
||||
}
|
||||
|
||||
export type DucklakeDestinationConfig = {
|
||||
// "Custom parameters" DuckLake: caller provides the PostgreSQL catalog URL and the
|
||||
// S3-compatible storage credentials directly.
|
||||
export type DucklakeManualDestinationConfig = {
|
||||
catalogUrl: string
|
||||
dataPath: string
|
||||
poolSize?: number
|
||||
@@ -51,6 +45,67 @@ export type DucklakeDestinationConfig = {
|
||||
metadataSchema?: string
|
||||
}
|
||||
|
||||
// "Use Supabase" DuckLake: caller provides Supabase project refs and a bucket; the platform
|
||||
// API resolves these into a catalog URL + provisioned S3 credentials before persisting.
|
||||
export type DucklakeSupabaseDestinationConfig = {
|
||||
catalogProjectRef: string
|
||||
storageProjectRef: string
|
||||
bucket: string
|
||||
path?: string
|
||||
poolSize?: number
|
||||
metadataSchema?: string
|
||||
}
|
||||
|
||||
export type DucklakeDestinationConfig =
|
||||
| DucklakeManualDestinationConfig
|
||||
| DucklakeSupabaseDestinationConfig
|
||||
|
||||
function isDucklakeSupabaseConfig(
|
||||
config: DucklakeDestinationConfig
|
||||
): config is DucklakeSupabaseDestinationConfig {
|
||||
return 'catalogProjectRef' in config
|
||||
}
|
||||
|
||||
// Maps the studio-side DuckLake config to the snake_case `{ ducklake: ... }` payload accepted
|
||||
// by the platform API. Shared by the create / update / validate mutations.
|
||||
export function buildDucklakeApiConfig(config: DucklakeDestinationConfig) {
|
||||
if (isDucklakeSupabaseConfig(config)) {
|
||||
return {
|
||||
ducklake: {
|
||||
// pool_size / metadata_schema live on the catalog so they apply to the selected
|
||||
// Supabase Postgres catalog (the API resolves catalog-level values over top-level).
|
||||
catalog: {
|
||||
type: 'supabase_project' as const,
|
||||
project_ref: config.catalogProjectRef,
|
||||
pool_size: config.poolSize,
|
||||
metadata_schema: config.metadataSchema,
|
||||
},
|
||||
storage: {
|
||||
type: 'supabase_storage' as const,
|
||||
project_ref: config.storageProjectRef,
|
||||
bucket: config.bucket,
|
||||
...(config.path ? { path: config.path } : {}),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ducklake: {
|
||||
catalog_url: config.catalogUrl,
|
||||
data_path: config.dataPath,
|
||||
pool_size: config.poolSize,
|
||||
s3_access_key_id: config.s3AccessKeyId,
|
||||
s3_secret_access_key: config.s3SecretAccessKey,
|
||||
s3_region: config.s3Region,
|
||||
s3_endpoint: config.s3Endpoint,
|
||||
s3_url_style: config.s3UrlStyle,
|
||||
s3_use_ssl: config.s3UseSsl,
|
||||
metadata_schema: config.metadataSchema,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export type SnowflakeDestinationConfig = {
|
||||
accountId: string
|
||||
user: string
|
||||
@@ -138,33 +193,9 @@ async function createDestinationPipeline(
|
||||
},
|
||||
}
|
||||
} else if ('ducklake' in destinationConfig) {
|
||||
const {
|
||||
catalogUrl,
|
||||
dataPath,
|
||||
poolSize,
|
||||
s3AccessKeyId,
|
||||
s3SecretAccessKey,
|
||||
s3Region,
|
||||
s3Endpoint,
|
||||
s3UrlStyle,
|
||||
s3UseSsl,
|
||||
metadataSchema,
|
||||
} = destinationConfig.ducklake
|
||||
|
||||
destination_config = {
|
||||
ducklake: {
|
||||
catalog_url: catalogUrl,
|
||||
data_path: dataPath,
|
||||
pool_size: poolSize,
|
||||
s3_access_key_id: s3AccessKeyId,
|
||||
s3_secret_access_key: s3SecretAccessKey,
|
||||
s3_region: s3Region,
|
||||
s3_endpoint: s3Endpoint,
|
||||
s3_url_style: s3UrlStyle,
|
||||
s3_use_ssl: s3UseSsl,
|
||||
metadata_schema: metadataSchema,
|
||||
},
|
||||
} as unknown as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']
|
||||
destination_config = buildDucklakeApiConfig(
|
||||
destinationConfig.ducklake
|
||||
) as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']
|
||||
} else if ('snowflake' in destinationConfig) {
|
||||
const { accountId, user, privateKey, privateKeyPassphrase, database, schema, role } =
|
||||
destinationConfig.snowflake
|
||||
|
||||
@@ -2,7 +2,11 @@ import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import type { components } from 'api-types'
|
||||
import { toast } from 'sonner'
|
||||
|
||||
import { BatchConfig, DestinationConfig } from './create-destination-pipeline-mutation'
|
||||
import {
|
||||
BatchConfig,
|
||||
buildDucklakeApiConfig,
|
||||
DestinationConfig,
|
||||
} from './create-destination-pipeline-mutation'
|
||||
import { replicationKeys } from './keys'
|
||||
import { handleError, post } from '@/data/fetchers'
|
||||
import type { ResponseError, UseCustomMutationOptions } from '@/types'
|
||||
@@ -82,32 +86,9 @@ async function updateDestinationPipeline(
|
||||
},
|
||||
}
|
||||
} else if ('ducklake' in destinationConfig) {
|
||||
const {
|
||||
catalogUrl,
|
||||
dataPath,
|
||||
poolSize,
|
||||
s3AccessKeyId,
|
||||
s3SecretAccessKey,
|
||||
s3Region,
|
||||
s3Endpoint,
|
||||
s3UrlStyle,
|
||||
s3UseSsl,
|
||||
metadataSchema,
|
||||
} = destinationConfig.ducklake
|
||||
destination_config = {
|
||||
ducklake: {
|
||||
catalog_url: catalogUrl,
|
||||
data_path: dataPath,
|
||||
pool_size: poolSize,
|
||||
s3_access_key_id: s3AccessKeyId,
|
||||
s3_secret_access_key: s3SecretAccessKey,
|
||||
s3_region: s3Region,
|
||||
s3_endpoint: s3Endpoint,
|
||||
s3_url_style: s3UrlStyle,
|
||||
s3_use_ssl: s3UseSsl,
|
||||
metadata_schema: metadataSchema,
|
||||
},
|
||||
} as unknown as components['schemas']['UpdateReplicationDestinationPipelineBody']['destination_config']
|
||||
destination_config = buildDucklakeApiConfig(
|
||||
destinationConfig.ducklake
|
||||
) as components['schemas']['UpdateReplicationDestinationPipelineBody']['destination_config']
|
||||
} else if ('snowflake' in destinationConfig) {
|
||||
const { accountId, user, privateKey, privateKeyPassphrase, database, schema, role } =
|
||||
destinationConfig.snowflake
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useMutation } from '@tanstack/react-query'
|
||||
import type { components } from 'api-types'
|
||||
|
||||
import { DestinationConfig } from './create-destination-pipeline-mutation'
|
||||
import { buildDucklakeApiConfig, DestinationConfig } from './create-destination-pipeline-mutation'
|
||||
import { handleError, post } from '@/data/fetchers'
|
||||
import type { ResponseError, UseCustomMutationOptions } from '@/types'
|
||||
|
||||
@@ -75,33 +75,9 @@ async function validateDestination(
|
||||
},
|
||||
}
|
||||
} else if ('ducklake' in destinationConfig) {
|
||||
const {
|
||||
catalogUrl,
|
||||
dataPath,
|
||||
poolSize,
|
||||
s3AccessKeyId,
|
||||
s3SecretAccessKey,
|
||||
s3Region,
|
||||
s3Endpoint,
|
||||
s3UrlStyle,
|
||||
s3UseSsl,
|
||||
metadataSchema,
|
||||
} = destinationConfig.ducklake
|
||||
|
||||
config = {
|
||||
ducklake: {
|
||||
catalog_url: catalogUrl,
|
||||
data_path: dataPath,
|
||||
pool_size: poolSize,
|
||||
s3_access_key_id: s3AccessKeyId,
|
||||
s3_secret_access_key: s3SecretAccessKey,
|
||||
s3_region: s3Region,
|
||||
s3_endpoint: s3Endpoint,
|
||||
s3_url_style: s3UrlStyle,
|
||||
s3_use_ssl: s3UseSsl,
|
||||
metadata_schema: metadataSchema,
|
||||
},
|
||||
} as unknown as components['schemas']['ValidateReplicationDestinationBody']['config']
|
||||
config = buildDucklakeApiConfig(
|
||||
destinationConfig.ducklake
|
||||
) as components['schemas']['ValidateReplicationDestinationBody']['config']
|
||||
} else if ('snowflake' in destinationConfig) {
|
||||
const { accountId, user, privateKey, privateKeyPassphrase, database, schema, role } =
|
||||
destinationConfig.snowflake
|
||||
|
||||
+1218
-385
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user