mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 09:25:06 +03:00
feat(studio): add ClickHouse replication destination (#46870)
Adds ClickHouse as a replication destination type in Studio.
- New ClickHouse option in the destination type selector, gated behind
the
`etlEnableClickHousePrivateAlpha` organization feature flag (off by
default).
- ClickHouse settings form: URL, user, password (optional), database,
and
- Client-side URL validation requires HTTPS and rejects URLs targeting
internal addresses (loopback, RFC 1918, link-local, CGNAT, IPv6
loopback/link-local/ULA, and IPv4-mapped/NAT64 forms). Server-side
validation remains authoritative.
<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit
## New Features
- Added **ClickHouse** as a replication destination option (private
alpha), including support in destination selection/panel, replication
diagram rendering, and destination icons.
- Introduced a ClickHouse destination form with fields for URL, user,
optional password (masked toggle), database, and engine selection.
- Added ClickHouse destination config handling for create/update flows,
with normalization and engine support.
## Tests
- Expanded unit tests to cover ClickHouse validation and destination
config building/normalization, including HTTPS-only and blocking
localhost/internal targets.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
---------
Co-authored-by: Joshen Lim <joshenlimek@gmail.com>
This commit is contained in:
Jordan McQueenandJoshen Lim authored and GitHub committed
1 parent
8962308215
commit
fa1e4c4bbf
30 files changed
+724
-75
No files matched your search
+12
@@ -0,0 +1,12 @@
|
||||
import * as z from 'zod'
|
||||
|
||||
// Only warehouse name and namespace are visible + editable fields
|
||||
export const AnalyticsBucketFormSchema = z.object({
|
||||
warehouseName: z.string().optional(),
|
||||
namespace: z.string().optional(),
|
||||
newNamespaceName: z.string().optional(),
|
||||
catalogToken: z.string().optional(),
|
||||
s3AccessKeyId: z.string().optional(),
|
||||
s3SecretAccessKey: z.string().optional(),
|
||||
s3Region: z.string().optional(),
|
||||
})
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
import * as z from 'zod'
|
||||
|
||||
export const BigQueryFormSchema = z.object({
|
||||
projectId: z.string().optional(),
|
||||
datasetId: z.string().optional(),
|
||||
serviceAccountKey: z.string().optional(),
|
||||
connectionPoolSize: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1, 'Connection pool size must be greater than 0')
|
||||
.optional(),
|
||||
maxStalenessMins: z
|
||||
.number()
|
||||
.int('Maximum staleness must be a whole number of minutes')
|
||||
.min(0, 'Maximum staleness must be 0 or greater')
|
||||
.optional(),
|
||||
})
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
import * as z from 'zod'
|
||||
|
||||
export const ClickHouseFormSchema = z.object({
|
||||
clickhouseUrl: z.string().optional(),
|
||||
clickhouseUser: z.string().optional(),
|
||||
clickhousePassword: z.string().optional(),
|
||||
clickhouseDatabase: z.string().optional(),
|
||||
clickhouseEngine: z.enum(['merge_tree', 'replacing_merge_tree']).optional(),
|
||||
})
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
import { type DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
|
||||
export type ClickHouseApiConfig = {
|
||||
url: string
|
||||
user: string
|
||||
password?: string
|
||||
database: string
|
||||
engine?: 'merge_tree' | 'replacing_merge_tree'
|
||||
}
|
||||
|
||||
type ClickHouseFieldPath = 'clickhouseUrl' | 'clickhouseUser' | 'clickhouseDatabase'
|
||||
|
||||
export type ClickHouseValidationIssue = {
|
||||
path: ClickHouseFieldPath
|
||||
message: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Client-side check that the URL does not target an obviously internal address.
|
||||
* This is a UX-level guard to surface mistakes before the validate round-trip;
|
||||
* server-side validation remains authoritative.
|
||||
*/
|
||||
const isClickHouseHostInternal = (host: string): boolean => {
|
||||
if (host === '' || host === 'localhost' || host.endsWith('.localhost')) return true
|
||||
|
||||
const ipv4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/)
|
||||
|
||||
if (ipv4) {
|
||||
const a = Number(ipv4[1])
|
||||
const b = Number(ipv4[2])
|
||||
|
||||
return (
|
||||
a === 10 ||
|
||||
a === 127 ||
|
||||
a === 0 ||
|
||||
(a === 100 && b >= 64 && b <= 127) || // CGNAT (RFC 6598)
|
||||
(a === 169 && b === 254) ||
|
||||
(a === 172 && b >= 16 && b <= 31) ||
|
||||
(a === 192 && b === 168) ||
|
||||
(a === 198 && (b === 18 || b === 19)) || // benchmarking (RFC 2544)
|
||||
a >= 224
|
||||
)
|
||||
}
|
||||
|
||||
if (host.includes(':')) {
|
||||
const lower = host.toLowerCase()
|
||||
|
||||
return (
|
||||
lower === '::1' ||
|
||||
lower === '::' ||
|
||||
/^fe[89ab][0-9a-f]?:/.test(lower) ||
|
||||
/^f[cd][0-9a-f]{2}:/.test(lower) ||
|
||||
lower.startsWith('::ffff:') ||
|
||||
lower.startsWith('64:ff9b:') // NAT64 (RFC 6052 well-known + RFC 8215 local-use)
|
||||
)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
export const getClickHouseValidationIssues = (
|
||||
data: Pick<DestinationPanelSchemaType, ClickHouseFieldPath>
|
||||
): ClickHouseValidationIssue[] => {
|
||||
const issues: ClickHouseValidationIssue[] = []
|
||||
|
||||
if (!data.clickhouseUrl?.length) {
|
||||
issues.push({ path: 'clickhouseUrl', message: 'URL is required' })
|
||||
} else {
|
||||
let parsed: URL | undefined
|
||||
|
||||
try {
|
||||
parsed = new URL(data.clickhouseUrl)
|
||||
} catch {
|
||||
issues.push({ path: 'clickhouseUrl', message: 'ClickHouse URL must be a valid URL' })
|
||||
}
|
||||
|
||||
if (parsed) {
|
||||
if (parsed.protocol !== 'https:') {
|
||||
issues.push({ path: 'clickhouseUrl', message: 'ClickHouse URL must use https://' })
|
||||
} else {
|
||||
const host = parsed.hostname.replace(/^\[|\]$/g, '')
|
||||
|
||||
if (isClickHouseHostInternal(host)) {
|
||||
issues.push({
|
||||
path: 'clickhouseUrl',
|
||||
message: 'ClickHouse URL must not target an internal address',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.clickhouseUser?.length) {
|
||||
issues.push({ path: 'clickhouseUser', message: 'User is required' })
|
||||
}
|
||||
|
||||
if (!data.clickhouseDatabase?.length) {
|
||||
issues.push({ path: 'clickhouseDatabase', message: 'Database is required' })
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
import { Eye, EyeOff } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
import type { UseFormReturn } from 'react-hook-form'
|
||||
import {
|
||||
Button,
|
||||
FormControl,
|
||||
FormField,
|
||||
Input,
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
} from 'ui'
|
||||
import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
|
||||
import { STORED_SECRET_PLACEHOLDER } from '../DestinationForm.constants'
|
||||
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
|
||||
export const ClickHouseFields = ({
|
||||
form,
|
||||
hasStoredPassword,
|
||||
}: {
|
||||
form: UseFormReturn<DestinationPanelSchemaType>
|
||||
hasStoredPassword: boolean
|
||||
}) => {
|
||||
const [showPassword, setShowPassword] = useState(false)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-6 p-5">
|
||||
<p className="text-sm font-medium text-foreground">ClickHouse settings</p>
|
||||
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="clickhouseUrl"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="URL"
|
||||
description="HTTPS endpoint for your ClickHouse server, including port"
|
||||
>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
value={field.value ?? ''}
|
||||
placeholder="https://your-cluster.clickhouse.cloud:8443"
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="clickhouseUser"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="User"
|
||||
description="ClickHouse user with permission to write to the target database"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} value={field.value ?? ''} placeholder="default" />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="clickhousePassword"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Password"
|
||||
description={
|
||||
hasStoredPassword
|
||||
? 'Stored password is hidden. Enter a new password to replace it.'
|
||||
: 'Omit for passwordless access'
|
||||
}
|
||||
>
|
||||
<FormControl>
|
||||
<PasswordInput
|
||||
value={field.value ?? ''}
|
||||
type={showPassword ? 'text' : 'password'}
|
||||
placeholder={hasStoredPassword ? STORED_SECRET_PLACEHOLDER : 'Optional'}
|
||||
onChange={(event) => field.onChange(event.target.value)}
|
||||
actions={
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
title={showPassword ? 'Hide password' : 'Show password'}
|
||||
aria-label={showPassword ? 'Hide password' : 'Show password'}
|
||||
icon={showPassword ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowPassword(!showPassword)}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="clickhouseDatabase"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Database"
|
||||
description="The ClickHouse database where replicated tables will be created"
|
||||
>
|
||||
<FormControl>
|
||||
<Input {...field} value={field.value ?? ''} placeholder="default" />
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="clickhouseEngine"
|
||||
render={({ field }) => (
|
||||
<FormItemLayout
|
||||
layout="horizontal"
|
||||
label="Table engine"
|
||||
description="Server defaults to replacing_merge_tree when unset"
|
||||
>
|
||||
<FormControl>
|
||||
<Select
|
||||
value={field.value ?? 'replacing_merge_tree'}
|
||||
onValueChange={field.onChange}
|
||||
>
|
||||
<SelectTrigger>{field.value ?? 'replacing_merge_tree'}</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="replacing_merge_tree">replacing_merge_tree</SelectItem>
|
||||
<SelectItem value="merge_tree">merge_tree</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+13
-58
@@ -1,9 +1,12 @@
|
||||
import * as z from 'zod'
|
||||
|
||||
// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components
|
||||
// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other
|
||||
export const DestinationPanelFormSchema = z.object({
|
||||
// Common fields
|
||||
import { AnalyticsBucketFormSchema } from './AnalyticsBucket/AnalyticsBucket.schema'
|
||||
import { BigQueryFormSchema } from './BigQuery/BigQuery.schema'
|
||||
import { ClickHouseFormSchema } from './ClickHouse/ClickHouse.schema'
|
||||
import { DuckLakeFormSchema } from './DuckLake/DuckLake.schema'
|
||||
import { SnowflakeFormSchema } from './Snowflake/Snowflake.schema'
|
||||
|
||||
const CommonFormSchema = z.object({
|
||||
name: z.string().min(1, 'Name is required'),
|
||||
publicationName: z.string().min(1, 'Publication is required'),
|
||||
maxFillMs: z
|
||||
@@ -22,60 +25,12 @@ export const DestinationPanelFormSchema = z.object({
|
||||
.min(1, 'Max copy connections per table must be greater than 0')
|
||||
.optional(),
|
||||
invalidatedSlotBehavior: z.enum(['error', 'recreate']).optional(),
|
||||
// BigQuery fields
|
||||
projectId: z.string().optional(),
|
||||
datasetId: z.string().optional(),
|
||||
serviceAccountKey: z.string().optional(),
|
||||
connectionPoolSize: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1, 'Connection pool size must be greater than 0')
|
||||
.optional(),
|
||||
maxStalenessMins: z
|
||||
.number()
|
||||
.int('Maximum staleness must be a whole number of minutes')
|
||||
.min(0, 'Maximum staleness must be 0 or greater')
|
||||
.optional(),
|
||||
// Analytics Bucket fields, only warehouse name and namespace are visible + editable fields
|
||||
warehouseName: z.string().optional(),
|
||||
namespace: z.string().optional(),
|
||||
newNamespaceName: z.string().optional(),
|
||||
catalogToken: z.string().optional(),
|
||||
s3AccessKeyId: z.string().optional(),
|
||||
s3SecretAccessKey: z.string().optional(),
|
||||
s3Region: z.string().optional(),
|
||||
// DuckLake fields
|
||||
// `supabase` mode picks Supabase projects for catalog + storage (managed), while `custom`
|
||||
// mode keeps the manual PostgreSQL catalog URL + S3-compatible credentials.
|
||||
ducklakeMode: z.enum(['supabase', 'custom']).optional(),
|
||||
// DuckLake "Use Supabase" fields
|
||||
ducklakeCatalogProjectRef: z.string().optional(),
|
||||
ducklakeStorageProjectRef: z.string().optional(),
|
||||
ducklakeStorageBucket: z.string().optional(),
|
||||
// DuckLake "Custom parameters" fields
|
||||
ducklakeCatalogUrl: z.string().optional(),
|
||||
ducklakeDataPath: z.string().optional(),
|
||||
ducklakePoolSize: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1, 'Pool size must be greater than 0')
|
||||
.max(6, 'Pool size must be 6 or less')
|
||||
.optional(),
|
||||
ducklakeS3AccessKeyId: z.string().optional(),
|
||||
ducklakeS3SecretAccessKey: z.string().optional(),
|
||||
ducklakeS3Region: z.string().optional(),
|
||||
ducklakeS3Endpoint: z.string().optional(),
|
||||
ducklakeS3UrlStyle: z.enum(['path', 'vhost']).optional(),
|
||||
ducklakeS3UseSsl: z.boolean().optional(),
|
||||
ducklakeMetadataSchema: z.string().optional(),
|
||||
// Snowflake fields
|
||||
snowflakeAccountId: z.string().optional(),
|
||||
snowflakeUser: z.string().optional(),
|
||||
snowflakePrivateKey: z.string().optional(),
|
||||
snowflakePrivateKeyPassphrase: z.string().optional(),
|
||||
snowflakeDatabase: z.string().optional(),
|
||||
snowflakeSchema: z.string().optional(),
|
||||
snowflakeRole: z.string().optional(),
|
||||
})
|
||||
|
||||
export const DestinationPanelFormSchema = CommonFormSchema.extend(BigQueryFormSchema.shape)
|
||||
.extend(AnalyticsBucketFormSchema.shape)
|
||||
.extend(DuckLakeFormSchema.shape)
|
||||
.extend(SnowflakeFormSchema.shape)
|
||||
.extend(ClickHouseFormSchema.shape)
|
||||
|
||||
export type DestinationPanelSchemaType = z.infer<typeof DestinationPanelFormSchema>
|
||||
+169
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsBucket.utils'
|
||||
import { getBigQueryValidationIssues } from './BigQuery/BigQuery.utils'
|
||||
import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils'
|
||||
import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
|
||||
import {
|
||||
buildDestinationConfig,
|
||||
@@ -79,6 +80,42 @@ const baseSnowflakeFormData = {
|
||||
snowflakeRole: ' PIPELINES_ROLE ',
|
||||
}
|
||||
|
||||
const baseClickHouseFormData = {
|
||||
name: 'ClickHouse Destination',
|
||||
publicationName: 'pub',
|
||||
maxFillMs: undefined,
|
||||
maxTableSyncWorkers: undefined,
|
||||
maxCopyConnectionsPerTable: undefined,
|
||||
invalidatedSlotBehavior: undefined,
|
||||
projectId: undefined,
|
||||
datasetId: undefined,
|
||||
serviceAccountKey: undefined,
|
||||
connectionPoolSize: undefined,
|
||||
maxStalenessMins: undefined,
|
||||
warehouseName: undefined,
|
||||
namespace: undefined,
|
||||
newNamespaceName: undefined,
|
||||
catalogToken: undefined,
|
||||
s3AccessKeyId: undefined,
|
||||
s3SecretAccessKey: undefined,
|
||||
s3Region: undefined,
|
||||
ducklakeCatalogUrl: undefined,
|
||||
ducklakeDataPath: undefined,
|
||||
ducklakePoolSize: undefined,
|
||||
ducklakeS3AccessKeyId: undefined,
|
||||
ducklakeS3SecretAccessKey: undefined,
|
||||
ducklakeS3Region: undefined,
|
||||
ducklakeS3Endpoint: undefined,
|
||||
ducklakeS3UrlStyle: undefined,
|
||||
ducklakeS3UseSsl: undefined,
|
||||
ducklakeMetadataSchema: undefined,
|
||||
clickhouseUrl: ' https://your-cluster.clickhouse.cloud:8443 ',
|
||||
clickhouseUser: ' default ',
|
||||
clickhousePassword: ' secret password ',
|
||||
clickhouseDatabase: ' analytics ',
|
||||
clickhouseEngine: 'replacing_merge_tree' as const,
|
||||
}
|
||||
|
||||
describe('DestinationForm.utils DuckLake', () => {
|
||||
it('builds DuckLake validation config with required fields trimmed and blank optionals removed', () => {
|
||||
const config = buildDestinationConfigForValidation({
|
||||
@@ -440,6 +477,138 @@ describe('DestinationForm.utils Snowflake', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('DestinationForm.utils ClickHouse', () => {
|
||||
it('builds ClickHouse validation config with required fields trimmed and blank optionals removed', () => {
|
||||
const config = buildDestinationConfigForValidation({
|
||||
projectRef: 'project-ref',
|
||||
selectedType: 'ClickHouse',
|
||||
data: {
|
||||
...baseClickHouseFormData,
|
||||
clickhousePassword: '',
|
||||
},
|
||||
})
|
||||
|
||||
expect(config).toEqual({
|
||||
clickHouse: {
|
||||
url: 'https://your-cluster.clickhouse.cloud:8443',
|
||||
user: 'default',
|
||||
password: undefined,
|
||||
database: 'analytics',
|
||||
engine: 'replacing_merge_tree',
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('builds ClickHouse submit config with normalized values', async () => {
|
||||
const createS3AccessKey = vi.fn()
|
||||
const resolveNamespace = vi.fn()
|
||||
|
||||
const config = await buildDestinationConfig({
|
||||
projectRef: 'project-ref',
|
||||
selectedType: 'ClickHouse',
|
||||
data: baseClickHouseFormData,
|
||||
createS3AccessKey,
|
||||
resolveNamespace,
|
||||
})
|
||||
|
||||
expect(config).toEqual({
|
||||
clickHouse: {
|
||||
url: 'https://your-cluster.clickhouse.cloud:8443',
|
||||
user: 'default',
|
||||
password: ' secret password ',
|
||||
database: 'analytics',
|
||||
engine: 'replacing_merge_tree',
|
||||
},
|
||||
})
|
||||
expect(createS3AccessKey).not.toHaveBeenCalled()
|
||||
expect(resolveNamespace).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns required-field errors for missing ClickHouse settings', () => {
|
||||
const issues = getClickHouseValidationIssues({
|
||||
clickhouseUrl: '',
|
||||
clickhouseUser: '',
|
||||
clickhouseDatabase: '',
|
||||
})
|
||||
|
||||
expect(issues).toEqual([
|
||||
{ path: 'clickhouseUrl', message: 'URL is required' },
|
||||
{ path: 'clickhouseUser', message: 'User is required' },
|
||||
{ path: 'clickhouseDatabase', message: 'Database is required' },
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects ClickHouse URLs that are not https or target internal addresses', () => {
|
||||
expect(
|
||||
getClickHouseValidationIssues({
|
||||
clickhouseUrl: 'http://example.clickhouse.cloud:8443',
|
||||
clickhouseUser: 'default',
|
||||
clickhouseDatabase: 'analytics',
|
||||
})
|
||||
).toEqual([{ path: 'clickhouseUrl', message: 'ClickHouse URL must use https://' }])
|
||||
|
||||
expect(
|
||||
getClickHouseValidationIssues({
|
||||
clickhouseUrl: 'https://127.0.0.1:8443',
|
||||
clickhouseUser: 'default',
|
||||
clickhouseDatabase: 'analytics',
|
||||
})
|
||||
).toEqual([
|
||||
{ path: 'clickhouseUrl', message: 'ClickHouse URL must not target an internal address' },
|
||||
])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['loopback hostname', 'https://localhost:8443'],
|
||||
['loopback subdomain', 'https://foo.localhost:8443'],
|
||||
['IPv4 loopback', 'https://127.0.0.1:8443'],
|
||||
['IPv4 unspecified', 'https://0.0.0.0:8443'],
|
||||
['RFC 1918 10.0.0.0/8', 'https://10.1.2.3:8443'],
|
||||
['RFC 1918 172.16.0.0/12', 'https://172.16.0.1:8443'],
|
||||
['RFC 1918 172.16.0.0/12 upper bound', 'https://172.31.255.254:8443'],
|
||||
['RFC 1918 192.168.0.0/16', 'https://192.168.1.1:8443'],
|
||||
['link-local 169.254.0.0/16', 'https://169.254.1.1:8443'],
|
||||
['CGNAT 100.64.0.0/10', 'https://100.64.0.1:8443'],
|
||||
['CGNAT 100.64.0.0/10 upper bound', 'https://100.127.255.254:8443'],
|
||||
['benchmarking 198.18.0.0/15', 'https://198.18.0.1:8443'],
|
||||
['multicast/reserved 224.0.0.0/4+', 'https://224.0.0.1:8443'],
|
||||
['broadcast', 'https://255.255.255.255:8443'],
|
||||
['IPv6 loopback', 'https://[::1]:8443'],
|
||||
['IPv6 unspecified', 'https://[::]:8443'],
|
||||
['IPv6 link-local', 'https://[fe80::1]:8443'],
|
||||
['IPv6 unique local (fc00::/7)', 'https://[fc00::1]:8443'],
|
||||
['IPv6 unique local (fd00::/8)', 'https://[fd12:3456::1]:8443'],
|
||||
['IPv4-mapped IPv6', 'https://[::ffff:127.0.0.1]:8443'],
|
||||
['NAT64', 'https://[64:ff9b::127.0.0.1]:8443'],
|
||||
['decimal-encoded IPv4 loopback', 'https://2130706433:8443'],
|
||||
['hex-encoded IPv4 loopback', 'https://0x7f000001:8443'],
|
||||
])('rejects ClickHouse URLs targeting an internal address: %s', (_label, clickhouseUrl) => {
|
||||
expect(
|
||||
getClickHouseValidationIssues({
|
||||
clickhouseUrl,
|
||||
clickhouseUser: 'default',
|
||||
clickhouseDatabase: 'analytics',
|
||||
})
|
||||
).toEqual([
|
||||
{ path: 'clickhouseUrl', message: 'ClickHouse URL must not target an internal address' },
|
||||
])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['public IPv4 address', 'https://8.8.8.8:8443'],
|
||||
['public hostname', 'https://your-cluster.clickhouse.cloud:8443'],
|
||||
['public IPv6 address', 'https://[2606:4700:4700::1111]:8443'],
|
||||
])('accepts ClickHouse URLs targeting a public address: %s', (_label, clickhouseUrl) => {
|
||||
expect(
|
||||
getClickHouseValidationIssues({
|
||||
clickhouseUrl,
|
||||
clickhouseUser: 'default',
|
||||
clickhouseDatabase: 'analytics',
|
||||
})
|
||||
).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('DestinationForm.utils BigQuery', () => {
|
||||
it('returns required-field errors for missing BigQuery settings', () => {
|
||||
const issues = getBigQueryValidationIssues({
|
||||
|
||||
+30
@@ -3,6 +3,7 @@ import { snakeCase } from 'lodash'
|
||||
import z from 'zod'
|
||||
|
||||
import { DestinationType } from '../DestinationPanel.types'
|
||||
import { type ClickHouseApiConfig } from './ClickHouse/ClickHouse.utils'
|
||||
import {
|
||||
CREATE_NEW_KEY,
|
||||
CREATE_NEW_NAMESPACE,
|
||||
@@ -25,6 +26,7 @@ import { type DucklakeApiConfig } from './DuckLake/DuckLake.utils'
|
||||
import { type SnowflakeApiConfig } from './Snowflake/Snowflake.utils'
|
||||
import {
|
||||
BigQueryDestinationConfig,
|
||||
ClickHouseDestinationConfig,
|
||||
DestinationConfig,
|
||||
DucklakeDestinationConfig,
|
||||
DucklakeManualDestinationConfig,
|
||||
@@ -88,6 +90,14 @@ export const generateDefaultValues = ({
|
||||
snowflakeConfigValue && typeof snowflakeConfigValue === 'object'
|
||||
? (snowflakeConfigValue as SnowflakeApiConfig)
|
||||
: undefined
|
||||
const clickhouseConfigValue =
|
||||
config && 'clickhouse' in (config as Record<string, unknown>)
|
||||
? (config as Record<string, unknown>).clickhouse
|
||||
: undefined
|
||||
const clickhouseConfig =
|
||||
clickhouseConfigValue && typeof clickhouseConfigValue === 'object'
|
||||
? (clickhouseConfigValue as ClickHouseApiConfig)
|
||||
: undefined
|
||||
|
||||
return {
|
||||
// Common fields
|
||||
@@ -144,6 +154,12 @@ export const generateDefaultValues = ({
|
||||
snowflakeDatabase: snowflakeConfig?.database ?? '',
|
||||
snowflakeSchema: snowflakeConfig?.schema ?? '',
|
||||
snowflakeRole: snowflakeConfig?.role ?? '',
|
||||
// ClickHouse fields
|
||||
clickhouseUrl: clickhouseConfig?.url ?? '',
|
||||
clickhouseUser: clickhouseConfig?.user ?? '',
|
||||
clickhousePassword: clickhouseConfig?.password ?? '',
|
||||
clickhouseDatabase: clickhouseConfig?.database ?? '',
|
||||
clickhouseEngine: clickhouseConfig?.engine ?? 'replacing_merge_tree',
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +185,16 @@ const buildSnowflakeConfig = (
|
||||
role: normalizeOptionalString(data.snowflakeRole),
|
||||
})
|
||||
|
||||
const buildClickHouseConfig = (
|
||||
data: z.infer<typeof DestinationPanelFormSchema>
|
||||
): ClickHouseDestinationConfig => ({
|
||||
url: normalizeRequiredString(data.clickhouseUrl),
|
||||
user: normalizeRequiredString(data.clickhouseUser),
|
||||
password: normalizeOptionalUntrimmedString(data.clickhousePassword),
|
||||
database: normalizeRequiredString(data.clickhouseDatabase),
|
||||
engine: data.clickhouseEngine,
|
||||
})
|
||||
|
||||
// Builds the studio-side DuckLake config from form data, picking the right shape for the
|
||||
// selected mode. The create / update / validate mutations convert this to the API payload.
|
||||
const buildDucklakeConfig = (
|
||||
@@ -243,6 +269,8 @@ export const buildDestinationConfigForValidation = ({
|
||||
return { ducklake: buildDucklakeConfig(data) }
|
||||
} else if (selectedType === 'Snowflake') {
|
||||
return { snowflake: buildSnowflakeConfig(data) }
|
||||
} else if (selectedType === 'ClickHouse') {
|
||||
return { clickHouse: buildClickHouseConfig(data) }
|
||||
} else {
|
||||
throw new Error('Invalid destination type')
|
||||
}
|
||||
@@ -302,6 +330,8 @@ export const buildDestinationConfig = async ({
|
||||
destinationConfig = { ducklake: buildDucklakeConfig(data) }
|
||||
} else if (selectedType === 'Snowflake') {
|
||||
destinationConfig = { snowflake: buildSnowflakeConfig(data) }
|
||||
} else if (selectedType === 'ClickHouse') {
|
||||
destinationConfig = { clickHouse: buildClickHouseConfig(data) }
|
||||
}
|
||||
|
||||
return destinationConfig
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
import * as z from 'zod'
|
||||
|
||||
// `supabase` mode picks Supabase projects for catalog + storage (managed), while `custom`
|
||||
// mode keeps the manual PostgreSQL catalog URL + S3-compatible credentials.
|
||||
export const DuckLakeFormSchema = z.object({
|
||||
ducklakeMode: z.enum(['supabase', 'custom']).optional(),
|
||||
// DuckLake "Use Supabase" fields
|
||||
ducklakeCatalogProjectRef: z.string().optional(),
|
||||
ducklakeStorageProjectRef: z.string().optional(),
|
||||
ducklakeStorageBucket: z.string().optional(),
|
||||
// DuckLake "Custom parameters" fields
|
||||
ducklakeCatalogUrl: z.string().optional(),
|
||||
ducklakeDataPath: z.string().optional(),
|
||||
ducklakePoolSize: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1, 'Pool size must be greater than 0')
|
||||
.max(6, 'Pool size must be 6 or less')
|
||||
.optional(),
|
||||
ducklakeS3AccessKeyId: z.string().optional(),
|
||||
ducklakeS3SecretAccessKey: z.string().optional(),
|
||||
ducklakeS3Region: z.string().optional(),
|
||||
ducklakeS3Endpoint: z.string().optional(),
|
||||
ducklakeS3UrlStyle: z.enum(['path', 'vhost']).optional(),
|
||||
ducklakeS3UseSsl: z.boolean().optional(),
|
||||
ducklakeMetadataSchema: z.string().optional(),
|
||||
})
|
||||
+1
-1
@@ -150,7 +150,7 @@ export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicati
|
||||
<Button variant="default" disabled={creatingPublication} onClick={() => onClose()}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant="primary" disabled={creatingPublication} form={formId} type="submit">
|
||||
<Button variant="primary" loading={creatingPublication} form={formId} type="submit">
|
||||
Create publication
|
||||
</Button>
|
||||
</SheetFooter>
|
||||
|
||||
+10
-2
@@ -11,9 +11,11 @@ import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
export const SnowflakeFields = ({
|
||||
form,
|
||||
editMode,
|
||||
hasStoredPrivateKeyPassphrase,
|
||||
}: {
|
||||
form: UseFormReturn<DestinationPanelSchemaType>
|
||||
editMode: boolean
|
||||
hasStoredPrivateKeyPassphrase: boolean
|
||||
}) => {
|
||||
const [showPrivateKeyPassphrase, setShowPrivateKeyPassphrase] = useState(false)
|
||||
|
||||
@@ -157,7 +159,7 @@ export const SnowflakeFields = ({
|
||||
layout="horizontal"
|
||||
label="Private key passphrase"
|
||||
description={
|
||||
editMode
|
||||
hasStoredPrivateKeyPassphrase
|
||||
? 'Stored passphrase setting is hidden. Enter a new passphrase to replace it.'
|
||||
: 'Optional passphrase for encrypted private keys'
|
||||
}
|
||||
@@ -166,13 +168,19 @@ export const SnowflakeFields = ({
|
||||
<PasswordInput
|
||||
value={field.value ?? ''}
|
||||
type={showPrivateKeyPassphrase ? 'text' : 'password'}
|
||||
placeholder={editMode ? STORED_SECRET_PLACEHOLDER : 'Optional'}
|
||||
placeholder={
|
||||
hasStoredPrivateKeyPassphrase ? STORED_SECRET_PLACEHOLDER : 'Optional'
|
||||
}
|
||||
onChange={(event) => field.onChange(event.target.value)}
|
||||
actions={
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-7"
|
||||
title={showPrivateKeyPassphrase ? 'Hide passphrase' : 'Show passphrase'}
|
||||
aria-label={
|
||||
showPrivateKeyPassphrase ? 'Hide passphrase' : 'Show passphrase'
|
||||
}
|
||||
icon={showPrivateKeyPassphrase ? <Eye /> : <EyeOff />}
|
||||
onClick={() => setShowPrivateKeyPassphrase(!showPrivateKeyPassphrase)}
|
||||
/>
|
||||
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
import * as z from 'zod'
|
||||
|
||||
export const SnowflakeFormSchema = z.object({
|
||||
snowflakeAccountId: z.string().optional(),
|
||||
snowflakeUser: z.string().optional(),
|
||||
snowflakePrivateKey: z.string().optional(),
|
||||
snowflakePrivateKeyPassphrase: z.string().optional(),
|
||||
snowflakeDatabase: z.string().optional(),
|
||||
snowflakeSchema: z.string().optional(),
|
||||
snowflakeRole: z.string().optional(),
|
||||
})
|
||||
+28
-2
@@ -26,6 +26,7 @@ import * as z from 'zod'
|
||||
|
||||
import {
|
||||
useIsETLBigQueryPrivateAlpha,
|
||||
useIsETLClickHousePrivateAlpha,
|
||||
useIsETLDucklakePrivateAlpha,
|
||||
useIsETLIcebergPrivateAlpha,
|
||||
useIsETLSnowflakePrivateAlpha,
|
||||
@@ -36,6 +37,8 @@ import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsB
|
||||
import { AnalyticsBucketFields } from './AnalyticsBucket/Fields'
|
||||
import { getBigQueryValidationIssues } from './BigQuery/BigQuery.utils'
|
||||
import { BigQueryFields } from './BigQuery/Fields'
|
||||
import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils'
|
||||
import { ClickHouseFields } from './ClickHouse/Fields'
|
||||
import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
|
||||
import { areValidationFailuresEqual, generateDefaultValues } from './DestinationForm.utils'
|
||||
import { DestinationNameInput } from './DestinationNameInput'
|
||||
@@ -86,6 +89,7 @@ export const DestinationForm = ({
|
||||
const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
|
||||
const etlEnableDucklake = useIsETLDucklakePrivateAlpha()
|
||||
const etlEnableSnowflake = useIsETLSnowflakePrivateAlpha()
|
||||
const etlEnableClickHouse = useIsETLClickHousePrivateAlpha()
|
||||
const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
|
||||
|
||||
const [showValidationWarningsDialog, setShowValidationWarningsDialog] = useState(false)
|
||||
@@ -108,8 +112,15 @@ export const DestinationForm = ({
|
||||
destinations.push({ value: 'Analytics Bucket', label: 'Analytics Bucket' })
|
||||
if (etlEnableDucklake) destinations.push({ value: 'DuckLake', label: 'DuckLake' })
|
||||
if (etlEnableSnowflake) destinations.push({ value: 'Snowflake', label: 'Snowflake' })
|
||||
if (etlEnableClickHouse) destinations.push({ value: 'ClickHouse', label: 'ClickHouse' })
|
||||
return destinations
|
||||
}, [etlEnableBigQuery, etlEnableDucklake, etlEnableIceberg, etlEnableSnowflake])
|
||||
}, [
|
||||
etlEnableBigQuery,
|
||||
etlEnableDucklake,
|
||||
etlEnableIceberg,
|
||||
etlEnableSnowflake,
|
||||
etlEnableClickHouse,
|
||||
])
|
||||
const hasNoAvailableDestinations = availableDestinations.length === 0
|
||||
|
||||
const { data: sourcesData } = useReplicationSourcesQuery({ projectRef })
|
||||
@@ -204,6 +215,10 @@ export const DestinationForm = ({
|
||||
addRequiredFieldError(path, message)
|
||||
}
|
||||
)
|
||||
} else if (selectedType === 'ClickHouse') {
|
||||
getClickHouseValidationIssues(data).forEach(({ path, message }) => {
|
||||
addRequiredFieldError(path, message)
|
||||
})
|
||||
}
|
||||
})
|
||||
),
|
||||
@@ -404,7 +419,18 @@ export const DestinationForm = ({
|
||||
) : selectedType === 'DuckLake' && etlEnableDucklake ? (
|
||||
<DuckLakeFields form={form} editMode={editMode} />
|
||||
) : selectedType === 'Snowflake' && etlEnableSnowflake ? (
|
||||
<SnowflakeFields form={form} editMode={editMode} />
|
||||
<SnowflakeFields
|
||||
form={form}
|
||||
editMode={editMode}
|
||||
hasStoredPrivateKeyPassphrase={
|
||||
editMode && !!defaultValues.snowflakePrivateKeyPassphrase
|
||||
}
|
||||
/>
|
||||
) : selectedType === 'ClickHouse' && etlEnableClickHouse ? (
|
||||
<ClickHouseFields
|
||||
form={form}
|
||||
hasStoredPassword={editMode && !!defaultValues.clickhousePassword}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
<DialogSectionSeparator />
|
||||
|
||||
+1
@@ -43,6 +43,7 @@ export const DestinationPanel = ({ onSuccessCreateReadReplica }: DestinationPane
|
||||
'Analytics Bucket',
|
||||
'DuckLake',
|
||||
'Snowflake',
|
||||
'ClickHouse',
|
||||
]).withOptions({
|
||||
history: 'push',
|
||||
clearOnDefault: true,
|
||||
|
||||
+1
@@ -4,6 +4,7 @@ export type DestinationType =
|
||||
| 'Analytics Bucket'
|
||||
| 'DuckLake'
|
||||
| 'Snowflake'
|
||||
| 'ClickHouse'
|
||||
|
||||
export type ExistingDestination = {
|
||||
sourceId?: number
|
||||
|
||||
+8
@@ -20,12 +20,14 @@ const mockBigQueryEnabled = vi.fn()
|
||||
const mockIcebergEnabled = vi.fn()
|
||||
const mockDucklakeEnabled = vi.fn()
|
||||
const mockSnowflakeEnabled = vi.fn()
|
||||
const mockClickHouseEnabled = vi.fn()
|
||||
|
||||
vi.mock('../useIsETLPrivateAlpha', () => ({
|
||||
useIsETLBigQueryPrivateAlpha: () => mockBigQueryEnabled(),
|
||||
useIsETLIcebergPrivateAlpha: () => mockIcebergEnabled(),
|
||||
useIsETLDucklakePrivateAlpha: () => mockDucklakeEnabled(),
|
||||
useIsETLSnowflakePrivateAlpha: () => mockSnowflakeEnabled(),
|
||||
useIsETLClickHousePrivateAlpha: () => mockClickHouseEnabled(),
|
||||
}))
|
||||
|
||||
vi.mock('@/hooks/misc/useIsFeatureEnabled', () => ({
|
||||
@@ -57,6 +59,7 @@ describe('DestinationTypeSelection', () => {
|
||||
mockIcebergEnabled.mockReturnValue(false)
|
||||
mockDucklakeEnabled.mockReturnValue(false)
|
||||
mockSnowflakeEnabled.mockReturnValue(false)
|
||||
mockClickHouseEnabled.mockReturnValue(false)
|
||||
addBackgroundMocks()
|
||||
|
||||
customRender(<DestinationTypeSelection />)
|
||||
@@ -69,6 +72,7 @@ describe('DestinationTypeSelection', () => {
|
||||
mockIcebergEnabled.mockReturnValue(false)
|
||||
mockDucklakeEnabled.mockReturnValue(false)
|
||||
mockSnowflakeEnabled.mockReturnValue(false)
|
||||
mockClickHouseEnabled.mockReturnValue(false)
|
||||
addBackgroundMocks()
|
||||
|
||||
customRender(<DestinationTypeSelection />)
|
||||
@@ -84,6 +88,7 @@ describe('DestinationTypeSelection', () => {
|
||||
mockIcebergEnabled.mockReturnValue(false)
|
||||
mockDucklakeEnabled.mockReturnValue(false)
|
||||
mockSnowflakeEnabled.mockReturnValue(false)
|
||||
mockClickHouseEnabled.mockReturnValue(false)
|
||||
addBackgroundMocks()
|
||||
|
||||
customRender(<DestinationTypeSelection />)
|
||||
@@ -99,6 +104,7 @@ describe('DestinationTypeSelection', () => {
|
||||
mockIcebergEnabled.mockReturnValue(false)
|
||||
mockDucklakeEnabled.mockReturnValue(false)
|
||||
mockSnowflakeEnabled.mockReturnValue(false)
|
||||
mockClickHouseEnabled.mockReturnValue(false)
|
||||
addBackgroundMocks()
|
||||
|
||||
customRender(<DestinationTypeSelection />)
|
||||
@@ -118,6 +124,7 @@ describe('DestinationTypeSelection', () => {
|
||||
mockIcebergEnabled.mockReturnValue(false)
|
||||
mockDucklakeEnabled.mockReturnValue(false)
|
||||
mockSnowflakeEnabled.mockReturnValue(false)
|
||||
mockClickHouseEnabled.mockReturnValue(false)
|
||||
addBackgroundMocks()
|
||||
|
||||
customRender(<DestinationTypeSelection />)
|
||||
@@ -133,6 +140,7 @@ describe('DestinationTypeSelection', () => {
|
||||
mockIcebergEnabled.mockReturnValue(false)
|
||||
mockDucklakeEnabled.mockReturnValue(false)
|
||||
mockSnowflakeEnabled.mockReturnValue(false)
|
||||
mockClickHouseEnabled.mockReturnValue(false)
|
||||
addBackgroundMocks()
|
||||
// Edit mode triggers useDestinationInformation({ id: 1 }) which fires destination-by-id
|
||||
addAPIMock({
|
||||
|
||||
+12
-1
@@ -1,4 +1,4 @@
|
||||
import { AnalyticsBucket, BigQuery, Database } from 'icons'
|
||||
import { AnalyticsBucket, BigQuery, ClickHouse, Database } from 'icons'
|
||||
import { Snowflake } from 'lucide-react'
|
||||
import { parseAsInteger, parseAsStringEnum, useQueryState } from 'nuqs'
|
||||
import {
|
||||
@@ -16,6 +16,7 @@ import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
import { useDestinationInformation } from '../useDestinationInformation'
|
||||
import {
|
||||
useIsETLBigQueryPrivateAlpha,
|
||||
useIsETLClickHousePrivateAlpha,
|
||||
useIsETLDucklakePrivateAlpha,
|
||||
useIsETLIcebergPrivateAlpha,
|
||||
useIsETLSnowflakePrivateAlpha,
|
||||
@@ -43,6 +44,7 @@ export const DestinationTypeSelection = () => {
|
||||
const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
|
||||
const etlEnableDucklake = useIsETLDucklakePrivateAlpha()
|
||||
const etlEnableSnowflake = useIsETLSnowflakePrivateAlpha()
|
||||
const etlEnableClickHouse = useIsETLClickHousePrivateAlpha()
|
||||
const { infrastructureReadReplicas } = useIsFeatureEnabled(['infrastructure:read_replicas'])
|
||||
|
||||
const [urlDestinationType, setDestinationType] = useQueryState(
|
||||
@@ -53,6 +55,7 @@ export const DestinationTypeSelection = () => {
|
||||
'Analytics Bucket',
|
||||
'DuckLake',
|
||||
'Snowflake',
|
||||
'ClickHouse',
|
||||
]).withOptions({
|
||||
history: 'push',
|
||||
clearOnDefault: true,
|
||||
@@ -124,6 +127,14 @@ export const DestinationTypeSelection = () => {
|
||||
stage: 'Early Access',
|
||||
enabled: isOptionVisible('Snowflake', etlEnableSnowflake),
|
||||
},
|
||||
{
|
||||
value: 'ClickHouse',
|
||||
label: 'ClickHouse',
|
||||
description: 'Stream changes to a ClickHouse cluster for fast columnar analytics',
|
||||
icon: ClickHouse,
|
||||
stage: 'Early Access',
|
||||
enabled: isOptionVisible('ClickHouse', etlEnableClickHouse),
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useParams } from 'common'
|
||||
import { AnalyticsBucket, BigQuery, Database } from 'icons'
|
||||
import { AnalyticsBucket, BigQuery, ClickHouse, Database } from 'icons'
|
||||
import { Minus, Snowflake } from 'lucide-react'
|
||||
import Link from 'next/link'
|
||||
import { useEffect, useState } from 'react'
|
||||
@@ -158,6 +158,8 @@ export const DestinationRow = ({ destinationId }: DestinationRowProps) => {
|
||||
<Database size={18} className="text-foreground-light" />
|
||||
) : type === 'Snowflake' ? (
|
||||
<Snowflake size={18} className="text-foreground-light" />
|
||||
) : type === 'ClickHouse' ? (
|
||||
<ClickHouse size={18} className="text-foreground-light" />
|
||||
) : (
|
||||
<Database size={18} className="text-foreground-light" />
|
||||
)}
|
||||
|
||||
@@ -32,6 +32,7 @@ import { DisablePipelinesDialog } from './DisablePipelinesDialog'
|
||||
import { ReadReplicaRow } from './ReadReplicas/ReadReplicaRow'
|
||||
import {
|
||||
useIsETLBigQueryPrivateAlpha,
|
||||
useIsETLClickHousePrivateAlpha,
|
||||
useIsETLDucklakePrivateAlpha,
|
||||
useIsETLIcebergPrivateAlpha,
|
||||
useIsETLSnowflakePrivateAlpha,
|
||||
@@ -61,6 +62,7 @@ export const Destinations = () => {
|
||||
const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
|
||||
const etlEnableDucklake = useIsETLDucklakePrivateAlpha()
|
||||
const etlEnableSnowflake = useIsETLSnowflakePrivateAlpha()
|
||||
const etlEnableClickHouse = useIsETLClickHousePrivateAlpha()
|
||||
const { infrastructureReadReplicas } = useIsFeatureEnabled(['infrastructure:read_replicas'])
|
||||
|
||||
const newDestinationDefaultType = infrastructureReadReplicas
|
||||
@@ -73,7 +75,9 @@ export const Destinations = () => {
|
||||
? 'DuckLake'
|
||||
: etlEnableSnowflake
|
||||
? 'Snowflake'
|
||||
: null
|
||||
: etlEnableClickHouse
|
||||
? 'ClickHouse'
|
||||
: null
|
||||
|
||||
const prefetchedRef = useRef(false)
|
||||
const searchInputRef = useRef<HTMLInputElement>(null)
|
||||
@@ -89,6 +93,7 @@ export const Destinations = () => {
|
||||
'Analytics Bucket',
|
||||
'DuckLake',
|
||||
'Snowflake',
|
||||
'ClickHouse',
|
||||
]).withOptions({
|
||||
history: 'push',
|
||||
clearOnDefault: true,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Handle, Position } from '@xyflow/react'
|
||||
import { useParams } from 'common'
|
||||
import { AnalyticsBucket, BigQuery, Database } from 'icons'
|
||||
import { AnalyticsBucket, BigQuery, ClickHouse, Database } from 'icons'
|
||||
import { Snowflake } from 'lucide-react'
|
||||
import { ComponentType, PropsWithChildren, useMemo } from 'react'
|
||||
import { AWS_REGIONS } from 'shared-data'
|
||||
@@ -28,6 +28,7 @@ const destinationIconByType: Record<
|
||||
'Analytics Bucket': AnalyticsBucket,
|
||||
DuckLake: Database,
|
||||
Snowflake,
|
||||
ClickHouse,
|
||||
}
|
||||
|
||||
const NodeContainer = ({ className, children }: PropsWithChildren<{ className?: string }>) => {
|
||||
|
||||
+4
@@ -19,6 +19,10 @@ describe('getReplicationDestinationType', () => {
|
||||
expect(getReplicationDestinationType({ snowflake: {} })).toBe('Snowflake')
|
||||
})
|
||||
|
||||
it('returns ClickHouse for clickhouse configs', () => {
|
||||
expect(getReplicationDestinationType({ clickhouse: {} })).toBe('ClickHouse')
|
||||
})
|
||||
|
||||
it('returns undefined for unknown or missing configs', () => {
|
||||
expect(getReplicationDestinationType({})).toBeUndefined()
|
||||
expect(getReplicationDestinationType(undefined)).toBeUndefined()
|
||||
|
||||
+7
-1
@@ -1,4 +1,9 @@
|
||||
export type ReplicationDestinationType = 'BigQuery' | 'Analytics Bucket' | 'DuckLake' | 'Snowflake'
|
||||
export type ReplicationDestinationType =
|
||||
| 'BigQuery'
|
||||
| 'Analytics Bucket'
|
||||
| 'DuckLake'
|
||||
| 'Snowflake'
|
||||
| 'ClickHouse'
|
||||
|
||||
export const getReplicationDestinationType = (
|
||||
config?: Record<string, unknown>
|
||||
@@ -8,5 +13,6 @@ export const getReplicationDestinationType = (
|
||||
if ('iceberg' in config) return 'Analytics Bucket'
|
||||
if ('ducklake' in config) return 'DuckLake'
|
||||
if ('snowflake' in config) return 'Snowflake'
|
||||
if ('clickhouse' in config) return 'ClickHouse'
|
||||
return undefined
|
||||
}
|
||||
@@ -39,16 +39,22 @@ export const useIsETLSnowflakePrivateAlpha = () => {
|
||||
return useIsCurrentOrgInFlagList('etlEnableSnowflakePrivateAlpha')
|
||||
}
|
||||
|
||||
export const useIsETLClickHousePrivateAlpha = () => {
|
||||
return useIsCurrentOrgInFlagList('etlEnableClickHousePrivateAlpha')
|
||||
}
|
||||
|
||||
export const useIsETLPrivateAlpha = () => {
|
||||
const hasAccessToETLBigQuery = useIsCurrentOrgInFlagList('etlEnableBigQueryPrivateAlpha')
|
||||
const hasAccessToETLIceberg = useIsCurrentOrgInFlagList('etlEnableIcebergPrivateAlpha')
|
||||
const hasAccessToETLDucklake = useIsCurrentOrgInFlagList('etlEnableDucklakePrivateAlpha')
|
||||
const hasAccessToETLSnowflake = useIsCurrentOrgInFlagList('etlEnableSnowflakePrivateAlpha')
|
||||
const hasAccessToETLClickHouse = useIsCurrentOrgInFlagList('etlEnableClickHousePrivateAlpha')
|
||||
|
||||
return (
|
||||
hasAccessToETLBigQuery ||
|
||||
hasAccessToETLIceberg ||
|
||||
hasAccessToETLDucklake ||
|
||||
hasAccessToETLSnowflake
|
||||
hasAccessToETLSnowflake ||
|
||||
hasAccessToETLClickHouse
|
||||
)
|
||||
}
|
||||
@@ -12,6 +12,7 @@ export type DestinationConfig =
|
||||
| { iceberg: IcebergDestinationConfig }
|
||||
| { ducklake: DucklakeDestinationConfig }
|
||||
| { snowflake: SnowflakeDestinationConfig }
|
||||
| { clickHouse: ClickHouseDestinationConfig }
|
||||
|
||||
export type BigQueryDestinationConfig = {
|
||||
projectId: string
|
||||
@@ -128,6 +129,14 @@ export type SnowflakeDestinationConfig = {
|
||||
role?: string
|
||||
}
|
||||
|
||||
export type ClickHouseDestinationConfig = {
|
||||
url: string
|
||||
user: string
|
||||
password?: string
|
||||
database: string
|
||||
engine?: 'merge_tree' | 'replacing_merge_tree'
|
||||
}
|
||||
|
||||
export type BatchConfig = {
|
||||
maxFillMs?: number
|
||||
}
|
||||
@@ -222,10 +231,22 @@ async function createDestinationPipeline(
|
||||
schema,
|
||||
role,
|
||||
},
|
||||
} as unknown as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']
|
||||
} as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']
|
||||
} else if ('clickHouse' in destinationConfig) {
|
||||
const { url, user, password, database, engine } = destinationConfig.clickHouse
|
||||
|
||||
destination_config = {
|
||||
clickhouse: {
|
||||
url,
|
||||
user,
|
||||
password,
|
||||
database,
|
||||
engine,
|
||||
},
|
||||
} as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']
|
||||
} else {
|
||||
throw new Error(
|
||||
'Invalid destination config: must specify bigQuery, iceberg, ducklake, or snowflake'
|
||||
'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -108,10 +108,21 @@ async function updateDestinationPipeline(
|
||||
schema,
|
||||
role,
|
||||
},
|
||||
} as unknown as UpdateDestinationConfig
|
||||
} as UpdateDestinationConfig
|
||||
} else if ('clickHouse' in destinationConfig) {
|
||||
const { url, user, password, database, engine } = destinationConfig.clickHouse
|
||||
destination_config = {
|
||||
clickhouse: {
|
||||
url,
|
||||
user,
|
||||
password: optionalSecret(password),
|
||||
database,
|
||||
engine,
|
||||
},
|
||||
} as UpdateDestinationConfig
|
||||
} else {
|
||||
throw new Error(
|
||||
'Invalid destination config: must specify bigQuery, iceberg, ducklake, or snowflake'
|
||||
'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -92,10 +92,22 @@ async function validateDestination(
|
||||
schema,
|
||||
role,
|
||||
},
|
||||
} as unknown as components['schemas']['ValidateReplicationDestinationBody']['config']
|
||||
} as components['schemas']['ValidateReplicationDestinationBody']['config']
|
||||
} else if ('clickHouse' in destinationConfig) {
|
||||
const { url, user, password, database, engine } = destinationConfig.clickHouse
|
||||
|
||||
config = {
|
||||
clickhouse: {
|
||||
url,
|
||||
user,
|
||||
password,
|
||||
database,
|
||||
engine,
|
||||
},
|
||||
} as components['schemas']['ValidateReplicationDestinationBody']['config']
|
||||
} else {
|
||||
throw new Error(
|
||||
'Invalid destination config: must specify bigQuery, iceberg, ducklake, or snowflake'
|
||||
'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -114,6 +114,16 @@ export const Index: Record<string, any> = [
|
||||
svg: "<svg width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"none\" xmlns=\"http://www.w3.org/2000/svg\">\n<path d=\"M5.92381 15.2988L9.85798 13.0912L9.92381 12.8988L9.85798 12.7924H9.66558L9.00735 12.7519L6.75925 12.6912L4.80988 12.6102L2.92127 12.5089L2.44533 12.4076L1.99976 11.8203L2.04533 11.5266L2.44533 11.2583L3.01748 11.3089L4.2833 11.395L6.18203 11.5266L7.55925 11.6076L9.59976 11.8203H9.92381L9.96938 11.6886L9.85798 11.6076L9.77191 11.5266L7.80735 10.195L5.68077 8.78737L4.56684 7.97724L3.96431 7.56711L3.66052 7.1823L3.52887 6.3418L4.07571 5.73927L4.80988 5.7899L4.99722 5.84053L5.74153 6.41268L7.3314 7.64306L9.40735 9.17218L9.71115 9.42534L9.83267 9.33927L9.84786 9.27851L9.71115 9.05066L8.58203 7.01015L7.37697 4.9342L6.84026 4.07344L6.69849 3.55699C6.64786 3.34433 6.61241 3.16712 6.61241 2.94939L7.2352 2.10382L7.5795 1.99243L8.40988 2.10382L8.75925 2.40762L9.27571 3.58737L10.1111 5.4456L11.4074 7.97218L11.7871 8.72154L11.9896 9.41522L12.0656 9.62787H12.1972V9.50635L12.3036 8.08357L12.501 6.33673L12.6934 4.08863L12.7592 3.45572L13.0732 2.69623L13.696 2.2861L14.182 2.51901L14.582 3.09117L14.5263 3.46079L14.2884 5.00509L13.8225 7.42534L13.5187 9.0456H13.696L13.8985 8.84306L14.7187 7.75446L16.096 6.03294L16.7036 5.34939L17.4124 4.59496L17.8681 4.23547H18.7289L19.3618 5.17724L19.0782 6.14939L18.1922 7.27344L17.458 8.22534L16.4048 9.64306L15.7466 10.7772L15.8074 10.8684L15.9643 10.8532L18.3441 10.3469L19.6301 10.114L21.1643 9.85066L21.858 10.1747L21.9339 10.5038L21.6605 11.1772L20.02 11.5823L18.096 11.9671L15.2301 12.6456L15.1947 12.6709L15.2352 12.7215L16.5263 12.8431L17.0782 12.8734H18.4301L20.9466 13.0608L21.6048 13.4962L21.9998 14.0279L21.9339 14.4329L20.9213 14.9494L19.5542 14.6253L16.3643 13.8658L15.2706 13.5924H15.1187V13.6836L16.0301 14.5747L17.701 16.0836L19.7922 18.0279L19.8985 18.5089L19.6301 18.8886L19.3466 18.8481L17.5086 17.4658L16.7998 16.8431L15.1947 15.4912H15.0884V15.6329L15.458 16.1747L17.4124 19.1114L17.5137 20.0127L17.3719 20.3064L16.8656 20.4836L16.3086 20.3823L15.1643 18.7772L13.9846 16.9696L13.0327 15.3494L12.9162 15.4152L12.3542 21.4658L12.0909 21.7747L11.4833 22.0076L10.977 21.6228L10.7086 21L10.977 19.7696L11.301 18.1646L11.5643 16.8886L11.8023 15.3038L11.9441 14.7772L11.9339 14.7418L11.8175 14.757L10.6225 16.3975L8.80482 18.8532L7.36684 20.3924L7.02254 20.5291L6.42507 20.2203L6.48077 19.6684L6.81495 19.1772L8.80482 16.6456L10.0048 15.076L10.7795 14.1696L10.7744 14.038H10.7289L5.44279 17.4709L4.50102 17.5924L4.09596 17.2127L4.14659 16.5899L4.339 16.3874L5.92887 15.2937L5.92381 15.2988Z\" fill=\"currentColor\"/>\n</svg>\n",
|
||||
jsx: "import { Claude } from \"icons\"\n <Claude/>\n "
|
||||
},
|
||||
{
|
||||
name: "click-house",
|
||||
componentName: "ClickHouse",
|
||||
deprecated: false,
|
||||
raw: "import createSupabaseIcon from '../createSupabaseIcon';\n\n/**\n * @component @name ClickHouse\n * @description Supabase SVG icon component, renders SVG Element with children.\n *\n * @preview \n *\n * @param {Object} props - Supabase icons props and any valid SVG attribute\n * @returns {JSX.Element} JSX Element\n *\n */\nconst ClickHouse = createSupabaseIcon(\n 'ClickHouse',\n [\n [\n 'path',\n {\n d: 'M3 3V21M7.5 3V21M12 3V21M16.5 3V21M21 10V14',\n stroke: 'currentColor',\n 'stroke-width': '1.5',\n 'stroke-linecap': 'round',\n key: '1ho44o',\n },\n ],\n ],\n { fill: 'none' },\n);\n\nexport default ClickHouse;\n",
|
||||
component: React.lazy(() => import('icons/src/icons/click-house')),
|
||||
import: "import { ClickHouse } from 'icons'",
|
||||
svg: "<svg width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" xmlns=\"http://www.w3.org/2000/svg\">\n<path d=\"M3 3V21M7.5 3V21M12 3V21M16.5 3V21M21 10V14\" stroke=\"currentColor\" stroke-width=\"1.5\" stroke-linecap=\"round\"/>\n</svg>\n",
|
||||
jsx: "import { ClickHouse } from \"icons\"\n <ClickHouse/>\n "
|
||||
},
|
||||
{
|
||||
name: "database",
|
||||
componentName: "Database",
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import createSupabaseIcon from '../createSupabaseIcon';
|
||||
|
||||
/**
|
||||
* @component @name ClickHouse
|
||||
* @description Supabase SVG icon component, renders SVG Element with children.
|
||||
*
|
||||
* @preview 
|
||||
*
|
||||
* @param {Object} props - Supabase icons props and any valid SVG attribute
|
||||
* @returns {JSX.Element} JSX Element
|
||||
*
|
||||
*/
|
||||
const ClickHouse = createSupabaseIcon(
|
||||
'ClickHouse',
|
||||
[
|
||||
[
|
||||
'path',
|
||||
{
|
||||
d: 'M3 3V21M7.5 3V21M12 3V21M16.5 3V21M21 10V14',
|
||||
stroke: 'currentColor',
|
||||
'stroke-width': '1.5',
|
||||
'stroke-linecap': 'round',
|
||||
key: '1ho44o',
|
||||
},
|
||||
],
|
||||
],
|
||||
{ fill: 'none' },
|
||||
);
|
||||
|
||||
export default ClickHouse;
|
||||
@@ -9,6 +9,7 @@ export { default as BoxPlus } from './box-plus';
|
||||
export { default as BucketPlus } from './bucket-plus';
|
||||
export { default as Chatgpt } from './chatgpt';
|
||||
export { default as Claude } from './claude';
|
||||
export { default as ClickHouse } from './click-house';
|
||||
export { default as Database } from './database';
|
||||
export { default as Datadog } from './datadog';
|
||||
export { default as EdgeFunctions } from './edge-functions';
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M3 3V21M7.5 3V21M12 3V21M16.5 3V21M21 10V14" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 223 B |
Reference in new issue
Block a user