Files
Jordan McQueenandJoshen Lim fa1e4c4bbf feat(studio): add ClickHouse replication destination (#46870)
Adds ClickHouse as a replication destination type in Studio.

- New ClickHouse option in the destination type selector, gated behind
the
`etlEnableClickHousePrivateAlpha` organization feature flag (off by
default).
- ClickHouse settings form: URL, user, password (optional), database,
and
  - Client-side URL validation requires HTTPS and rejects URLs targeting
    internal addresses (loopback, RFC 1918, link-local, CGNAT, IPv6
    loopback/link-local/ULA, and IPv4-mapped/NAT64 forms). Server-side
    validation remains authoritative.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## New Features
- Added **ClickHouse** as a replication destination option (private
alpha), including support in destination selection/panel, replication
diagram rendering, and destination icons.
- Introduced a ClickHouse destination form with fields for URL, user,
optional password (masked toggle), database, and engine selection.
- Added ClickHouse destination config handling for create/update flows,
with normalization and engine support.

## Tests
- Expanded unit tests to cover ClickHouse validation and destination
config building/normalization, including HTTPS-only and blocking
localhost/internal targets.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Joshen Lim <joshenlimek@gmail.com>
2026-07-20 12:09:13 +08:00

178 lines
6.6 KiB
TypeScript

import { Handle, Position } from '@xyflow/react'
import { useParams } from 'common'
import { AnalyticsBucket, BigQuery, ClickHouse, Database } from 'icons'
import { Snowflake } from 'lucide-react'
import { ComponentType, PropsWithChildren, useMemo } from 'react'
import { AWS_REGIONS } from 'shared-data'
import { cn, Tooltip, TooltipContent, TooltipTrigger } from 'ui'
import { getStatusName } from '../Pipeline.utils'
import { getStatusLabel } from '../ReadReplicas/ReadReplicas.utils'
import { STATUS_REFRESH_FREQUENCY_MS } from '../Replication.constants'
import { getReplicationDestinationType, type ReplicationDestinationType } from './Nodes.utils'
import { useReadReplicasQuery } from '@/data/read-replicas/replicas-query'
import { formatDatabaseID } from '@/data/read-replicas/replicas.utils'
import { useReplicationDestinationsQuery } from '@/data/replication/destinations-query'
import { useReplicationPipelineStatusQuery } from '@/data/replication/pipeline-status-query'
import { useReplicationPipelinesQuery } from '@/data/replication/pipelines-query'
import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject'
import { BASE_PATH } from '@/lib/constants'
export const NODE_WIDTH = 480
const destinationIconByType: Record<
ReplicationDestinationType,
ComponentType<{ className?: string; size?: string | number }>
> = {
BigQuery,
'Analytics Bucket': AnalyticsBucket,
DuckLake: Database,
Snowflake,
ClickHouse,
}
const NodeContainer = ({ className, children }: PropsWithChildren<{ className?: string }>) => {
return (
<div
style={{ width: NODE_WIDTH / 2 + 55 }}
className={cn(
'flex items-start justify-between p-3 rounded-sm bg-surface-100 border border-default',
className
)}
>
{children}
</div>
)
}
export const PrimaryDatabaseNode = () => {
const { ref: projectRef } = useParams()
const { data: project } = useSelectedProjectQuery()
const { data: databases = [] } = useReadReplicasQuery({ projectRef })
const hasReadReplicas = databases.some((x) => x.identifier !== projectRef)
const { data: destinationsData } = useReplicationDestinationsQuery({ projectRef })
const hasDestinations = (destinationsData?.destinations ?? []).length > 0
const region = Object.values(AWS_REGIONS).find((x) => x.code === project?.region)
const hasReplication = hasReadReplicas || hasDestinations
return (
<NodeContainer>
<div className="text-sm flex flex-col gap-y-0.5">
<p>Primary Database</p>
<p className="text-foreground-light">{region?.displayName}</p>
<p className="text-foreground-light">{region?.code}</p>
</div>
{!!project && (
<img
alt="region icon"
className="w-8 rounded-xs mt-0.5"
src={`${BASE_PATH}/img/regions/${project?.region}.svg`}
/>
)}
<Handle
type="source"
position={Position.Right}
className={hasReplication ? 'opacity-25' : 'opacity-0'}
/>
</NodeContainer>
)
}
export const ReplicationNode = ({ id }: { id: string }) => {
const { ref: projectRef } = useParams()
const { data: destinationsData } = useReplicationDestinationsQuery({ projectRef })
const destination = (destinationsData?.destinations ?? []).find((x) => x.id.toString() === id)
const { data: pipelinesData } = useReplicationPipelinesQuery({
projectRef,
})
const pipeline = (pipelinesData?.pipelines ?? []).find((x) => x.destination_id.toString() === id)
const { data: pipelineStatusData } = useReplicationPipelineStatusQuery(
{ projectRef, pipelineId: pipeline?.id },
{ refetchInterval: STATUS_REFRESH_FREQUENCY_MS }
)
const statusName = getStatusName(pipelineStatusData?.status)
const type = getReplicationDestinationType(destination?.config)
const DestinationIcon = type ? destinationIconByType[type] : undefined
return (
<NodeContainer className="justify-start gap-x-3">
{DestinationIcon ? <DestinationIcon size={20} className="text-foreground-light" /> : null}
<div className="text-sm flex flex-col gap-y-0.5">
<div className="flex items-center">
<p>{type}</p>
{(statusName === 'started' || statusName === 'failed') && (
<Tooltip>
<TooltipTrigger>
<div className="w-6 h-full flex items-center justify-center">
<div
className={cn(
'w-2 h-2 rounded-full',
statusName === 'started' ? 'bg-brand' : 'bg-destructive'
)}
/>
</div>
</TooltipTrigger>
<TooltipContent side="bottom" className="capitalize">
{statusName}
</TooltipContent>
</Tooltip>
)}
</div>
<p className="text-foreground-light">{destination?.name}</p>
<p className="text-foreground-light">ID: {destination?.id}</p>
</div>
<Handle type="target" position={Position.Left} className="opacity-25" />
</NodeContainer>
)
}
export const ReadReplicaNode = ({ id }: { id: string }) => {
const { ref: projectRef } = useParams()
const { data: databases = [] } = useReadReplicasQuery({ projectRef })
const database = databases.find((x) => x.identifier === id)
const region = Object.values(AWS_REGIONS).find((x) => x.code === database?.region)
const formattedId = formatDatabaseID(database?.identifier ?? '')
const statusLabel = useMemo(
() => getStatusLabel({ status: database?.status }),
[database?.status]
)
return (
<NodeContainer className="justify-start gap-x-3">
<Database size={20} className="text-foreground-light" />
<div className="flex flex-col gap-y-0.5">
<div className="flex items-center">
<p className="text-sm">Read Replica</p>
<Tooltip>
<TooltipTrigger>
<div className="w-6 h-full flex items-center justify-center">
<div
className={cn(
'w-2 h-2 rounded-full',
database?.status === 'ACTIVE_HEALTHY' ? 'bg-brand' : 'bg-selection'
)}
/>
</div>
</TooltipTrigger>
<TooltipContent side="bottom">{statusLabel}</TooltipContent>
</Tooltip>
</div>
<p className="text-sm text-foreground-light">{region?.displayName}</p>
<div className="flex gap-x-2 items-center text-sm text-foreground-light">
<span>ID: {formattedId}</span>
<span>•</span>
<span>{region?.code}</span>
</div>
</div>
<Handle type="target" position={Position.Left} className="opacity-25" />
</NodeContainer>
)
}