mirror of
https://github.com/supabase/supabase.git
synced 2026-10-18 23:59:08 +03:00
feat(studio): configure BigQuery table layouts
This commit is contained in:
Danny White committed
1 parent
77559fb3c0
commit
230cc3b6da
7 files changed
+1425
No files matched your search
+12
@@ -19,6 +19,7 @@ import {
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
|
||||
import { DestinationType } from '../DestinationPanel.types'
|
||||
import { TableOptions } from './BigQuery/TableOptions'
|
||||
import {
|
||||
DEFAULT_CONNECTION_POOL_SIZE,
|
||||
DEFAULT_MAX_COPY_CONNECTIONS_PER_TABLE,
|
||||
@@ -237,6 +238,17 @@ export const AdvancedSettings = ({
|
||||
</FormItemLayout>
|
||||
)}
|
||||
/>
|
||||
|
||||
<div className="flex flex-col gap-y-3">
|
||||
<div className="flex flex-col gap-y-1">
|
||||
<span className="text-sm text-foreground">Table layout</span>
|
||||
<p className="text-sm text-foreground-lighter">
|
||||
Partitioning and clustering for each BigQuery table. Applied when a
|
||||
destination table is first created or reset.
|
||||
</p>
|
||||
</div>
|
||||
<TableOptions control={form.control} />
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</AccordionContent>
|
||||
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
import type { ChangeEvent } from 'react'
|
||||
import { Input, Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from 'ui'
|
||||
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
|
||||
import { MultiSelector } from 'ui-patterns/multi-select'
|
||||
import { SelectionListState } from 'ui-patterns/SelectionListState'
|
||||
|
||||
import { defaultPartitionByForKind, parseIntegerInput, shortenPgType } from './BigQuery.utils'
|
||||
import type { BigQueryPartitionKind } from './BigQuery.utils'
|
||||
import { isPartitionColumnTypeCompatible } from './TableOptions.utils'
|
||||
import {
|
||||
BIGQUERY_MAX_CLUSTERING_COLUMNS,
|
||||
BIGQUERY_TIME_PARTITION_GRANULARITIES,
|
||||
type BigQueryPartitionBy,
|
||||
type BigQueryTimePartitionGranularity,
|
||||
} from '@/data/replication/create-destination-pipeline-mutation'
|
||||
|
||||
const PARTITION_KIND_LABELS: Record<BigQueryPartitionKind, string> = {
|
||||
none: 'None',
|
||||
time_column: 'Time column',
|
||||
integer_range: 'Integer range',
|
||||
ingestion_time: 'Ingestion time',
|
||||
}
|
||||
|
||||
const GRANULARITY_LABELS: Record<BigQueryTimePartitionGranularity, string> = {
|
||||
hour: 'Hour',
|
||||
day: 'Day',
|
||||
month: 'Month',
|
||||
year: 'Year',
|
||||
}
|
||||
|
||||
const ColumnOption = ({
|
||||
name,
|
||||
type,
|
||||
isUnavailable = false,
|
||||
}: {
|
||||
name: string
|
||||
type?: string
|
||||
isUnavailable?: boolean
|
||||
}) => (
|
||||
<span
|
||||
className={
|
||||
isUnavailable
|
||||
? 'flex items-center gap-x-2 min-w-0 text-destructive-600'
|
||||
: 'flex items-center gap-x-2 min-w-0'
|
||||
}
|
||||
>
|
||||
<span className="truncate min-w-0">{name}</span>
|
||||
{isUnavailable && <span className="sr-only"> (no longer available)</span>}
|
||||
{type && (
|
||||
<span
|
||||
className={
|
||||
isUnavailable
|
||||
? 'font-mono text-xs shrink-0 text-destructive-600'
|
||||
: 'font-mono text-xs shrink-0 text-foreground-lighter'
|
||||
}
|
||||
>
|
||||
{shortenPgType(type)}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
)
|
||||
|
||||
export interface ColumnSelectionState {
|
||||
availableColumnNames: string[]
|
||||
columnTypeByName: ReadonlyMap<string, string>
|
||||
isError: boolean
|
||||
isLoading: boolean
|
||||
onOpenChange: (isOpen: boolean) => void
|
||||
unavailableColumnSet: ReadonlySet<string>
|
||||
}
|
||||
|
||||
interface PartitioningFieldsProps {
|
||||
columnSelection: ColumnSelectionState
|
||||
errors: {
|
||||
column?: string
|
||||
end?: string
|
||||
interval?: string
|
||||
start?: string
|
||||
}
|
||||
onChange: (partitionBy: BigQueryPartitionBy | undefined) => void
|
||||
onNeedsColumns: () => void
|
||||
partitionBy: BigQueryPartitionBy | undefined
|
||||
}
|
||||
|
||||
export const PartitioningFields = ({
|
||||
columnSelection,
|
||||
errors,
|
||||
onChange,
|
||||
onNeedsColumns,
|
||||
partitionBy,
|
||||
}: PartitioningFieldsProps) => {
|
||||
const partitionKind: BigQueryPartitionKind = partitionBy?.kind ?? 'none'
|
||||
|
||||
const compatibleColumnNames =
|
||||
partitionBy && (partitionBy.kind === 'time_column' || partitionBy.kind === 'integer_range')
|
||||
? columnSelection.availableColumnNames.filter((column) =>
|
||||
isPartitionColumnTypeCompatible(
|
||||
partitionBy.kind,
|
||||
columnSelection.columnTypeByName.get(column)
|
||||
)
|
||||
)
|
||||
: columnSelection.availableColumnNames
|
||||
const compatibleColumnEmptyLabel =
|
||||
partitionBy?.kind === 'time_column'
|
||||
? 'No published date or timestamp columns'
|
||||
: 'No published integer columns'
|
||||
|
||||
return (
|
||||
<>
|
||||
<FormItemLayout isReactForm={false} label="Partition by" layout="vertical">
|
||||
<Select
|
||||
value={partitionKind}
|
||||
onValueChange={(kind: BigQueryPartitionKind) => {
|
||||
if (kind === 'time_column' || kind === 'integer_range') onNeedsColumns()
|
||||
onChange(defaultPartitionByForKind(kind))
|
||||
}}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent side="bottom" collisionPadding={16}>
|
||||
{Object.entries(PARTITION_KIND_LABELS).map(([kind, label]) => (
|
||||
<SelectItem key={kind} value={kind}>
|
||||
{label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItemLayout>
|
||||
|
||||
{partitionBy && 'column' in partitionBy && (
|
||||
<FormItemLayout
|
||||
isReactForm={false}
|
||||
label="Partition column"
|
||||
layout="vertical"
|
||||
error={errors.column}
|
||||
>
|
||||
<Select
|
||||
value={partitionBy.column || undefined}
|
||||
onOpenChange={columnSelection.onOpenChange}
|
||||
onValueChange={(column: string) => onChange({ ...partitionBy, column })}
|
||||
>
|
||||
<SelectTrigger
|
||||
className={
|
||||
columnSelection.unavailableColumnSet.has(partitionBy.column)
|
||||
? 'border-destructive-600'
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{partitionBy.column ? (
|
||||
<ColumnOption
|
||||
name={partitionBy.column}
|
||||
type={columnSelection.columnTypeByName.get(partitionBy.column)}
|
||||
isUnavailable={columnSelection.unavailableColumnSet.has(partitionBy.column)}
|
||||
/>
|
||||
) : (
|
||||
<SelectValue placeholder="Select a column" />
|
||||
)}
|
||||
</SelectTrigger>
|
||||
<SelectContent side="bottom" collisionPadding={16}>
|
||||
<SelectionListState
|
||||
isLoading={columnSelection.isLoading}
|
||||
isError={columnSelection.isError}
|
||||
isEmpty={
|
||||
!columnSelection.isLoading &&
|
||||
!columnSelection.isError &&
|
||||
compatibleColumnNames.length === 0
|
||||
}
|
||||
emptyLabel={compatibleColumnEmptyLabel}
|
||||
errorLabel="Unable to load columns. Open the list again to retry."
|
||||
/>
|
||||
{compatibleColumnNames.map((column) => (
|
||||
<SelectItem key={column} value={column}>
|
||||
<ColumnOption name={column} type={columnSelection.columnTypeByName.get(column)} />
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
|
||||
{partitionBy && 'granularity' in partitionBy && (
|
||||
<FormItemLayout isReactForm={false} label="Granularity" layout="vertical">
|
||||
<Select
|
||||
value={partitionBy.granularity ?? 'day'}
|
||||
onValueChange={(granularity: BigQueryTimePartitionGranularity) =>
|
||||
onChange({ ...partitionBy, granularity })
|
||||
}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent side="bottom" collisionPadding={16}>
|
||||
{BIGQUERY_TIME_PARTITION_GRANULARITIES.map((granularity) => (
|
||||
<SelectItem key={granularity} value={granularity}>
|
||||
{GRANULARITY_LABELS[granularity]}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItemLayout>
|
||||
)}
|
||||
|
||||
{partitionBy?.kind === 'integer_range' && (
|
||||
<div className="grid grid-cols-[repeat(auto-fit,minmax(10rem,1fr))] gap-2">
|
||||
<FormItemLayout
|
||||
isReactForm={false}
|
||||
label="Start"
|
||||
layout="vertical"
|
||||
description="Inclusive"
|
||||
error={errors.start}
|
||||
>
|
||||
<Input
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={partitionBy.start}
|
||||
onChange={(event: ChangeEvent<HTMLInputElement>) =>
|
||||
onChange({
|
||||
...partitionBy,
|
||||
start: parseIntegerInput(event.target.value, partitionBy.start),
|
||||
})
|
||||
}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
<FormItemLayout
|
||||
isReactForm={false}
|
||||
label="End"
|
||||
layout="vertical"
|
||||
description="Exclusive"
|
||||
error={errors.end}
|
||||
>
|
||||
<Input
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={partitionBy.end}
|
||||
onChange={(event: ChangeEvent<HTMLInputElement>) =>
|
||||
onChange({
|
||||
...partitionBy,
|
||||
end: parseIntegerInput(event.target.value, partitionBy.end),
|
||||
})
|
||||
}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
<FormItemLayout
|
||||
isReactForm={false}
|
||||
label="Interval"
|
||||
layout="vertical"
|
||||
description="Range width"
|
||||
error={errors.interval}
|
||||
>
|
||||
<Input
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={partitionBy.interval}
|
||||
onChange={(event: ChangeEvent<HTMLInputElement>) =>
|
||||
onChange({
|
||||
...partitionBy,
|
||||
interval: parseIntegerInput(event.target.value, partitionBy.interval),
|
||||
})
|
||||
}
|
||||
/>
|
||||
</FormItemLayout>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
interface ClusteringFieldsProps {
|
||||
clusterBy: string[]
|
||||
columnSelection: ColumnSelectionState
|
||||
error?: string
|
||||
onChange: (columns: string[]) => void
|
||||
}
|
||||
|
||||
export const ClusteringFields = ({
|
||||
clusterBy,
|
||||
columnSelection,
|
||||
error,
|
||||
onChange,
|
||||
}: ClusteringFieldsProps) => {
|
||||
const hasReachedColumnLimit = clusterBy.length >= BIGQUERY_MAX_CLUSTERING_COLUMNS
|
||||
|
||||
return (
|
||||
<FormItemLayout
|
||||
isReactForm={false}
|
||||
label="Cluster by"
|
||||
layout="vertical"
|
||||
description={`Selection order sets the clustering order. Maximum ${BIGQUERY_MAX_CLUSTERING_COLUMNS} columns.`}
|
||||
error={error}
|
||||
>
|
||||
<MultiSelector
|
||||
values={clusterBy}
|
||||
onValuesChange={(columns) => {
|
||||
if (columns.length <= BIGQUERY_MAX_CLUSTERING_COLUMNS) onChange(columns)
|
||||
}}
|
||||
onOpenChange={columnSelection.onOpenChange}
|
||||
>
|
||||
<MultiSelector.Trigger
|
||||
aria-label="Select clustering columns"
|
||||
badgeLimit="wrap"
|
||||
label="Select columns..."
|
||||
renderValue={(column) => (
|
||||
<ColumnOption
|
||||
name={column}
|
||||
type={columnSelection.columnTypeByName.get(column)}
|
||||
isUnavailable={columnSelection.unavailableColumnSet.has(column)}
|
||||
/>
|
||||
)}
|
||||
/>
|
||||
<MultiSelector.Content>
|
||||
<MultiSelector.List
|
||||
emptyLabel="No published columns available"
|
||||
error={columnSelection.isError}
|
||||
errorLabel="Unable to load columns. Open the list again to retry."
|
||||
loading={columnSelection.isLoading}
|
||||
>
|
||||
{columnSelection.availableColumnNames.map((column) => (
|
||||
<MultiSelector.Item
|
||||
key={column}
|
||||
value={column}
|
||||
disabled={hasReachedColumnLimit && !clusterBy.includes(column)}
|
||||
>
|
||||
<ColumnOption name={column} type={columnSelection.columnTypeByName.get(column)} />
|
||||
</MultiSelector.Item>
|
||||
))}
|
||||
</MultiSelector.List>
|
||||
</MultiSelector.Content>
|
||||
</MultiSelector>
|
||||
</FormItemLayout>
|
||||
)
|
||||
}
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
import { useParams } from 'common'
|
||||
import { RotateCcw } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
import { get, useController, useFormState, type Control, type FieldErrors } from 'react-hook-form'
|
||||
import { Button } from 'ui'
|
||||
import { Admonition } from 'ui-patterns/Admonition'
|
||||
|
||||
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
import {
|
||||
isMetadataListErrorVisible,
|
||||
isMetadataListLoading,
|
||||
useRefreshOnOpen,
|
||||
} from '../useRefreshOnOpen'
|
||||
import {
|
||||
ClusteringFields,
|
||||
PartitioningFields,
|
||||
type ColumnSelectionState,
|
||||
} from './TableOptionFields'
|
||||
import { useReplicationSourceId } from '@/data/replication/sources-query'
|
||||
import { useReplicationTableColumnsQuery } from '@/data/replication/table-columns-query'
|
||||
|
||||
type TableOption = NonNullable<DestinationPanelSchemaType['tableOptions']>[number]
|
||||
type PartitionBy = TableOption['partitionBy']
|
||||
|
||||
const getFieldErrorMessage = (errors: FieldErrors<DestinationPanelSchemaType>, path: string) => {
|
||||
const error = get(errors, path)
|
||||
return typeof error?.message === 'string' ? error.message : undefined
|
||||
}
|
||||
|
||||
interface TableOptionErrors {
|
||||
partitionBy?: string
|
||||
column?: string
|
||||
start?: string
|
||||
end?: string
|
||||
interval?: string
|
||||
clusterBy?: string
|
||||
}
|
||||
|
||||
interface TableOptionEditorProps {
|
||||
partitionBy: PartitionBy
|
||||
clusterBy: string[]
|
||||
onPartitionByChange: (partitionBy: PartitionBy) => void
|
||||
onClusterByChange: (clusterBy: string[]) => void
|
||||
errors?: TableOptionErrors
|
||||
isPublicationColumnsError?: boolean
|
||||
isPublicationColumnsPending?: boolean
|
||||
publishedColumnNames?: ReadonlySet<string> | null
|
||||
tableId: number
|
||||
onClear?: () => void
|
||||
}
|
||||
|
||||
const TableOptionEditor = ({
|
||||
partitionBy,
|
||||
clusterBy,
|
||||
onPartitionByChange,
|
||||
onClusterByChange,
|
||||
errors = {},
|
||||
isPublicationColumnsError = false,
|
||||
isPublicationColumnsPending = false,
|
||||
publishedColumnNames,
|
||||
tableId,
|
||||
onClear,
|
||||
}: TableOptionEditorProps) => {
|
||||
const { ref: projectRef } = useParams()
|
||||
const sourceId = useReplicationSourceId({ projectRef })
|
||||
|
||||
const shouldInitiallyLoadColumns =
|
||||
partitionBy?.kind === 'time_column' ||
|
||||
partitionBy?.kind === 'integer_range' ||
|
||||
clusterBy.length > 0
|
||||
const [shouldLoadColumns, setShouldLoadColumns] = useState(shouldInitiallyLoadColumns)
|
||||
const {
|
||||
data: columns = [],
|
||||
isPending,
|
||||
isFetching,
|
||||
isError,
|
||||
isSuccess,
|
||||
refetch: refetchColumns,
|
||||
} = useReplicationTableColumnsQuery(
|
||||
{ projectRef, sourceId, tableId },
|
||||
{ enabled: shouldLoadColumns }
|
||||
)
|
||||
|
||||
const isLoadingColumns =
|
||||
(shouldLoadColumns && isMetadataListLoading(isPending || isFetching, columns.length)) ||
|
||||
isPublicationColumnsPending
|
||||
const sourceColumnNames = columns.map((column) => column.name)
|
||||
const sourceColumnNameSet = new Set(sourceColumnNames)
|
||||
const canUseColumns = !isPublicationColumnsPending && !isPublicationColumnsError
|
||||
const availableColumnNames = canUseColumns
|
||||
? sourceColumnNames.filter(
|
||||
(column) =>
|
||||
publishedColumnNames === undefined ||
|
||||
(publishedColumnNames !== null && publishedColumnNames.has(column))
|
||||
)
|
||||
: []
|
||||
const columnTypeByName = new Map(columns.map((column) => [column.name, column.type]))
|
||||
const areColumnsVerified = isSuccess && canUseColumns
|
||||
const partitionColumn = partitionBy && 'column' in partitionBy ? partitionBy.column : undefined
|
||||
const configuredColumns = [partitionColumn, ...clusterBy].filter(
|
||||
(column, columnIndex, allConfiguredColumns): column is string =>
|
||||
typeof column === 'string' &&
|
||||
column.trim().length > 0 &&
|
||||
allConfiguredColumns.indexOf(column) === columnIndex
|
||||
)
|
||||
const unavailableColumns = configuredColumns.filter(
|
||||
(column) =>
|
||||
areColumnsVerified &&
|
||||
(!sourceColumnNameSet.has(column) ||
|
||||
(publishedColumnNames != null && !publishedColumnNames.has(column)))
|
||||
)
|
||||
const unavailableColumnSet = new Set(unavailableColumns)
|
||||
const isColumnSelectionError =
|
||||
isPublicationColumnsError || isMetadataListErrorVisible(isError, columns.length)
|
||||
|
||||
const { handleOpenChange: handleRefreshColumnsOnOpen } = useRefreshOnOpen({
|
||||
isEnabled: shouldLoadColumns,
|
||||
refetch: refetchColumns,
|
||||
})
|
||||
const handleColumnPickerOpenChange = (isOpen: boolean) => {
|
||||
if (isOpen && !shouldLoadColumns) {
|
||||
setShouldLoadColumns(true)
|
||||
return
|
||||
}
|
||||
handleRefreshColumnsOnOpen(isOpen)
|
||||
}
|
||||
|
||||
const columnSelection: ColumnSelectionState = {
|
||||
availableColumnNames,
|
||||
columnTypeByName,
|
||||
isError: isColumnSelectionError,
|
||||
isLoading: isLoadingColumns,
|
||||
onOpenChange: handleColumnPickerOpenChange,
|
||||
unavailableColumnSet,
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-4 border-t px-3 py-4">
|
||||
<PartitioningFields
|
||||
partitionBy={partitionBy}
|
||||
onChange={onPartitionByChange}
|
||||
onNeedsColumns={() => setShouldLoadColumns(true)}
|
||||
columnSelection={columnSelection}
|
||||
errors={{
|
||||
column: errors.column,
|
||||
start: errors.start,
|
||||
end: errors.end,
|
||||
interval: errors.interval,
|
||||
}}
|
||||
/>
|
||||
|
||||
{errors.partitionBy && <p className="text-sm text-destructive-600">{errors.partitionBy}</p>}
|
||||
|
||||
<ClusteringFields
|
||||
clusterBy={clusterBy}
|
||||
onChange={onClusterByChange}
|
||||
columnSelection={columnSelection}
|
||||
error={errors.clusterBy}
|
||||
/>
|
||||
|
||||
{shouldLoadColumns && isColumnSelectionError && (
|
||||
<Admonition
|
||||
type="warning"
|
||||
title="Columns could not be verified"
|
||||
description="Refresh the page before changing or saving this table’s layout."
|
||||
/>
|
||||
)}
|
||||
|
||||
{unavailableColumns.length > 0 && (
|
||||
<Admonition
|
||||
type="destructive"
|
||||
title="Some selected columns are no longer available"
|
||||
description="Choose columns that are still in the source and publication."
|
||||
/>
|
||||
)}
|
||||
|
||||
{onClear !== undefined && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="default"
|
||||
className="self-start"
|
||||
icon={<RotateCcw size={14} />}
|
||||
onClick={onClear}
|
||||
>
|
||||
Clear
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
type SharedRowProps = Pick<
|
||||
TableOptionEditorProps,
|
||||
'isPublicationColumnsError' | 'isPublicationColumnsPending' | 'publishedColumnNames' | 'tableId'
|
||||
>
|
||||
|
||||
interface TableOptionRowProps extends SharedRowProps {
|
||||
control: Control<DestinationPanelSchemaType>
|
||||
index: number
|
||||
onClear: () => void
|
||||
}
|
||||
|
||||
export const TableOptionRow = ({
|
||||
control,
|
||||
index,
|
||||
onClear,
|
||||
...sharedProps
|
||||
}: TableOptionRowProps) => {
|
||||
const { errors } = useFormState({ control })
|
||||
|
||||
const { field: partitionByField } = useController({
|
||||
control,
|
||||
name: `tableOptions.${index}.partitionBy`,
|
||||
})
|
||||
const { field: clusterByField } = useController({
|
||||
control,
|
||||
name: `tableOptions.${index}.clusterBy`,
|
||||
})
|
||||
|
||||
const fieldPath = `tableOptions.${index}`
|
||||
|
||||
return (
|
||||
<TableOptionEditor
|
||||
partitionBy={partitionByField.value}
|
||||
clusterBy={clusterByField.value ?? []}
|
||||
onPartitionByChange={partitionByField.onChange}
|
||||
onClusterByChange={clusterByField.onChange}
|
||||
errors={{
|
||||
partitionBy: getFieldErrorMessage(errors, `${fieldPath}.partitionBy`),
|
||||
column: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.column`),
|
||||
start: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.start`),
|
||||
end: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.end`),
|
||||
interval: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.interval`),
|
||||
clusterBy: getFieldErrorMessage(errors, `${fieldPath}.clusterBy`),
|
||||
}}
|
||||
onClear={onClear}
|
||||
{...sharedProps}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
interface TableOptionDraftRowProps extends SharedRowProps {
|
||||
onCreate: (option: Pick<TableOption, 'partitionBy' | 'clusterBy'>) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Rendered while a table has no entry in the tableOptions field array. Expanding a row must not
|
||||
* touch form state, otherwise merely looking at a table leaves the form dirty and triggers the
|
||||
* unsaved-changes prompt. The first edit creates the entry, after which TableOptionRow takes over.
|
||||
*/
|
||||
export const TableOptionDraftRow = ({ onCreate, ...sharedProps }: TableOptionDraftRowProps) => (
|
||||
<TableOptionEditor
|
||||
partitionBy={undefined}
|
||||
clusterBy={[]}
|
||||
onPartitionByChange={(partitionBy) => onCreate({ partitionBy, clusterBy: [] })}
|
||||
onClusterByChange={(clusterBy) => onCreate({ partitionBy: undefined, clusterBy })}
|
||||
{...sharedProps}
|
||||
/>
|
||||
)
|
||||
+505
@@ -0,0 +1,505 @@
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
import { fireEvent, screen } from '@testing-library/react'
|
||||
import type { components } from 'api-types'
|
||||
import { HttpResponse } from 'msw'
|
||||
import { useForm } from 'react-hook-form'
|
||||
import { Form } from 'ui'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
DestinationPanelFormSchema,
|
||||
type DestinationPanelSchemaType,
|
||||
} from '../DestinationForm.schema'
|
||||
import { TableOptions } from './TableOptions'
|
||||
import { customRender } from '@/tests/lib/custom-render'
|
||||
import { addAPIMock } from '@/tests/lib/msw'
|
||||
|
||||
type PublicationDetailsResponse = components['schemas']['PublicationDetailsResponse_Output']
|
||||
type ReadColumnsResponse = components['schemas']['ReadColumnsResponse_Output']
|
||||
type ReadTablesResponse = components['schemas']['ReadTablesResponse_Output']
|
||||
type SourcesResponse = components['schemas']['SourcesResponse_Output']
|
||||
|
||||
const publicationTable = {
|
||||
id: 101,
|
||||
schema: 'public',
|
||||
name: 'Orders',
|
||||
kind: 'table' as const,
|
||||
partition_parent_id: null,
|
||||
}
|
||||
|
||||
const publication: PublicationDetailsResponse = {
|
||||
name: 'analytics',
|
||||
config: {
|
||||
type: 'tables',
|
||||
tables: [
|
||||
{
|
||||
id: publicationTable.id,
|
||||
schema: publicationTable.schema,
|
||||
name: publicationTable.name,
|
||||
columns: null,
|
||||
row_filter: null,
|
||||
},
|
||||
],
|
||||
operations: ['insert'],
|
||||
publish_via_partition_root: false,
|
||||
},
|
||||
tables: [publicationTable],
|
||||
}
|
||||
|
||||
const mockSources = () => {
|
||||
const response: SourcesResponse = {
|
||||
sources: [
|
||||
{
|
||||
id: 1,
|
||||
name: 'default',
|
||||
tenant_id: 'tenant',
|
||||
config: { host: 'db.internal', name: 'main-db', port: 5432, username: 'etl_user' },
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
addAPIMock({
|
||||
method: 'get',
|
||||
path: '/platform/replication/:ref/sources',
|
||||
response: () => HttpResponse.json<SourcesResponse>(response),
|
||||
})
|
||||
}
|
||||
|
||||
const mockPublication = (response = publication) => {
|
||||
addAPIMock({
|
||||
method: 'get',
|
||||
path: '/platform/replication/v2/:ref/sources/:source_id/publications/:publication_name',
|
||||
response: () => HttpResponse.json<PublicationDetailsResponse>(response),
|
||||
})
|
||||
}
|
||||
|
||||
const mockColumns = (tableId: number, columns: ReadColumnsResponse['columns']) => {
|
||||
addAPIMock({
|
||||
method: 'get',
|
||||
path: '/platform/replication/v2/:ref/sources/:source_id/tables/:table_id/columns',
|
||||
response: ({ params }) => {
|
||||
if (Number(params.table_id) !== tableId) {
|
||||
return HttpResponse.json<ReadColumnsResponse>({ columns: [] })
|
||||
}
|
||||
return HttpResponse.json<ReadColumnsResponse>({ columns })
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const mockTables = (tables: ReadTablesResponse['tables']) => {
|
||||
addAPIMock({
|
||||
method: 'get',
|
||||
path: '/platform/replication/v2/:ref/sources/:source_id/tables',
|
||||
response: () => HttpResponse.json<ReadTablesResponse>({ tables }),
|
||||
})
|
||||
}
|
||||
|
||||
const TableOptionsHarness = ({
|
||||
tableOptions,
|
||||
publicationName = 'analytics',
|
||||
}: {
|
||||
tableOptions: NonNullable<DestinationPanelSchemaType['tableOptions']>
|
||||
publicationName?: string
|
||||
}) => {
|
||||
const form = useForm<DestinationPanelSchemaType>({
|
||||
mode: 'onChange',
|
||||
resolver: zodResolver(DestinationPanelFormSchema),
|
||||
defaultValues: {
|
||||
name: 'Warehouse',
|
||||
publicationName,
|
||||
tableSyncCopyMode: 'include_all_tables',
|
||||
tableSyncCopyTableIds: [],
|
||||
tableOptions,
|
||||
},
|
||||
})
|
||||
|
||||
return (
|
||||
<Form {...form}>
|
||||
<TableOptions control={form.control} />
|
||||
<p>{form.formState.isDirty ? 'Form is dirty' : 'Form is pristine'}</p>
|
||||
<button type="button" tabIndex={0} onClick={() => void form.trigger()}>
|
||||
Validate form
|
||||
</button>
|
||||
</Form>
|
||||
)
|
||||
}
|
||||
|
||||
describe('TableOptions source reconciliation', () => {
|
||||
it('explains why a publication is required before showing table settings', async () => {
|
||||
mockSources()
|
||||
|
||||
customRender(<TableOptionsHarness publicationName="" tableOptions={[]} />)
|
||||
|
||||
expect(await screen.findByText('Select a publication')).toBeInTheDocument()
|
||||
expect(
|
||||
screen.getByText(
|
||||
'Choose the publication whose destination tables you want to partition or cluster.'
|
||||
)
|
||||
).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('discloses table settings without implying that the table is excluded', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
|
||||
customRender(<TableOptionsHarness tableOptions={[]} />)
|
||||
|
||||
const tableTrigger = await screen.findByRole('button', {
|
||||
name: 'public.Orders: Not configured',
|
||||
})
|
||||
expect(screen.queryByText('Partition by')).not.toBeInTheDocument()
|
||||
|
||||
fireEvent.click(tableTrigger)
|
||||
expect(await screen.findByText('Partition by')).toBeInTheDocument()
|
||||
expect(screen.queryByRole('button', { name: 'Clear' })).not.toBeInTheDocument()
|
||||
|
||||
fireEvent.click(tableTrigger)
|
||||
expect(screen.queryByText('Partition by')).not.toBeInTheDocument()
|
||||
expect(screen.getByText('Not configured')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('keeps the form pristine when a table row is only expanded', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
|
||||
customRender(<TableOptionsHarness tableOptions={[]} />)
|
||||
|
||||
const tableTrigger = await screen.findByRole('button', {
|
||||
name: 'public.Orders: Not configured',
|
||||
})
|
||||
expect(screen.getByText('Form is pristine')).toBeInTheDocument()
|
||||
|
||||
fireEvent.click(tableTrigger)
|
||||
expect(await screen.findByText('Partition by')).toBeInTheDocument()
|
||||
expect(screen.getByText('Form is pristine')).toBeInTheDocument()
|
||||
|
||||
fireEvent.click(tableTrigger)
|
||||
expect(screen.getByText('Form is pristine')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('clears a configured layout without removing its table row', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
mockColumns(101, [{ name: 'region', type: 'text', nullable: false, primary_key: false }])
|
||||
|
||||
customRender(<TableOptionsHarness tableOptions={[{ tableId: 101, clusterBy: ['region'] }]} />)
|
||||
|
||||
fireEvent.click(
|
||||
await screen.findByRole('button', { name: 'public.Orders: 1 clustering column' })
|
||||
)
|
||||
fireEvent.click(await screen.findByRole('button', { name: 'Clear' }))
|
||||
|
||||
expect(
|
||||
screen.getByRole('button', { name: 'public.Orders: Not configured' })
|
||||
).toBeInTheDocument()
|
||||
expect(screen.queryByText('Partition by')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows integer range validation errors beside the invalid fields', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
mockColumns(101, [{ name: 'id', type: 'int8', nullable: false, primary_key: true }])
|
||||
|
||||
customRender(
|
||||
<TableOptionsHarness
|
||||
tableOptions={[
|
||||
{
|
||||
tableId: 101,
|
||||
partitionBy: {
|
||||
kind: 'integer_range',
|
||||
column: 'id',
|
||||
start: 0,
|
||||
end: 0,
|
||||
interval: 0,
|
||||
},
|
||||
},
|
||||
]}
|
||||
/>
|
||||
)
|
||||
|
||||
fireEvent.click(await screen.findByRole('button', { name: /public\.Orders: Integer range/ }))
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Validate form' }))
|
||||
expect(await screen.findByText('Interval must be greater than 0.')).toBeInTheDocument()
|
||||
// Once beside the End field, and once on the trigger so a collapsed row still explains itself.
|
||||
expect(screen.getAllByText('End must be greater than start.')).toHaveLength(2)
|
||||
expect(
|
||||
screen.getByRole('button', { name: 'public.Orders: End must be greater than start.' })
|
||||
).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('blocks an incomplete partition instead of dropping it silently on save', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
mockColumns(101, [
|
||||
{ name: 'CreatedAt', type: 'timestamptz', nullable: false, primary_key: true },
|
||||
])
|
||||
|
||||
customRender(
|
||||
<TableOptionsHarness
|
||||
tableOptions={[{ tableId: 101, partitionBy: { kind: 'time_column', column: '' } }]}
|
||||
/>
|
||||
)
|
||||
|
||||
const tableTrigger = await screen.findByRole('button', {
|
||||
name: 'public.Orders: Time column partitioning',
|
||||
})
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Validate form' }))
|
||||
expect(await screen.findByText('Select a partition column')).toBeInTheDocument()
|
||||
|
||||
// The row can be collapsed when the save is attempted, so the trigger has to explain it too.
|
||||
expect(
|
||||
await screen.findByRole('button', { name: 'public.Orders: Select a partition column' })
|
||||
).toBe(tableTrigger)
|
||||
})
|
||||
|
||||
it('marks configured columns that no longer exist while preserving valid names and types', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
mockColumns(101, [
|
||||
{ name: 'region', type: 'text', nullable: false, primary_key: false },
|
||||
{ name: 'CreatedAt', type: 'timestamptz', nullable: false, primary_key: false },
|
||||
])
|
||||
|
||||
customRender(
|
||||
<TableOptionsHarness
|
||||
tableOptions={[
|
||||
{
|
||||
tableId: 101,
|
||||
partitionBy: { kind: 'time_column', column: 'DroppedAt', granularity: 'day' },
|
||||
clusterBy: ['region', 'LegacySegment'],
|
||||
},
|
||||
]}
|
||||
/>
|
||||
)
|
||||
|
||||
fireEvent.click(
|
||||
await screen.findByRole('button', { name: /public\.Orders: Daily by DroppedAt/ })
|
||||
)
|
||||
expect(
|
||||
await screen.findByText('Some selected columns are no longer available')
|
||||
).toBeInTheDocument()
|
||||
expect(screen.getAllByText('region')[0]).toBeInTheDocument()
|
||||
expect(screen.getByText('text')).toBeInTheDocument()
|
||||
expect(screen.getByText('DroppedAt').parentElement).toHaveClass('text-destructive-600')
|
||||
expect(screen.getByText('LegacySegment').parentElement).toHaveClass('text-destructive-600')
|
||||
})
|
||||
|
||||
it('marks existing columns that are no longer included in the publication', async () => {
|
||||
mockSources()
|
||||
mockPublication({
|
||||
...publication,
|
||||
config: {
|
||||
type: 'tables',
|
||||
tables: [
|
||||
{
|
||||
id: publicationTable.id,
|
||||
schema: publicationTable.schema,
|
||||
name: publicationTable.name,
|
||||
columns: ['region'],
|
||||
row_filter: null,
|
||||
},
|
||||
],
|
||||
operations: ['insert'],
|
||||
publish_via_partition_root: false,
|
||||
},
|
||||
})
|
||||
mockColumns(101, [
|
||||
{ name: 'region', type: 'text', nullable: false, primary_key: false },
|
||||
{ name: 'PrivateNote', type: 'text', nullable: true, primary_key: false },
|
||||
])
|
||||
|
||||
customRender(
|
||||
<TableOptionsHarness tableOptions={[{ tableId: 101, clusterBy: ['PrivateNote'] }]} />
|
||||
)
|
||||
|
||||
fireEvent.click(
|
||||
await screen.findByRole('button', { name: /public\.Orders: 1 clustering column/ })
|
||||
)
|
||||
expect(
|
||||
await screen.findByText('Some selected columns are no longer available')
|
||||
).toBeInTheDocument()
|
||||
expect(screen.getByText('PrivateNote').parentElement).toHaveClass('text-destructive-600')
|
||||
})
|
||||
|
||||
it('inherits explicit column filters through nested partition ancestors', async () => {
|
||||
mockSources()
|
||||
mockPublication({
|
||||
...publication,
|
||||
config: {
|
||||
type: 'tables',
|
||||
tables: [
|
||||
{
|
||||
id: 10,
|
||||
schema: 'public',
|
||||
name: 'Events',
|
||||
columns: ['AllowedColumn'],
|
||||
row_filter: null,
|
||||
},
|
||||
],
|
||||
operations: ['insert'],
|
||||
publish_via_partition_root: false,
|
||||
},
|
||||
tables: [
|
||||
{
|
||||
id: 12,
|
||||
schema: 'public',
|
||||
name: 'Events2026August',
|
||||
kind: 'table',
|
||||
partition_parent_id: 11,
|
||||
},
|
||||
],
|
||||
})
|
||||
mockTables([
|
||||
{
|
||||
id: 10,
|
||||
schema: 'public',
|
||||
name: 'Events',
|
||||
kind: 'partitioned_table',
|
||||
partition_parent_id: null,
|
||||
},
|
||||
{
|
||||
id: 11,
|
||||
schema: 'public',
|
||||
name: 'Events2026',
|
||||
kind: 'partitioned_table',
|
||||
partition_parent_id: 10,
|
||||
},
|
||||
{
|
||||
id: 12,
|
||||
schema: 'public',
|
||||
name: 'Events2026August',
|
||||
kind: 'table',
|
||||
partition_parent_id: 11,
|
||||
},
|
||||
])
|
||||
mockColumns(12, [
|
||||
{ name: 'AllowedColumn', type: 'text', nullable: false, primary_key: false },
|
||||
{ name: 'ExcludedColumn', type: 'text', nullable: false, primary_key: false },
|
||||
])
|
||||
|
||||
customRender(
|
||||
<TableOptionsHarness tableOptions={[{ tableId: 12, clusterBy: ['ExcludedColumn'] }]} />
|
||||
)
|
||||
|
||||
fireEvent.click(
|
||||
await screen.findByRole('button', {
|
||||
name: /public\.Events2026August: 1 clustering column/,
|
||||
})
|
||||
)
|
||||
expect(
|
||||
await screen.findByText('Some selected columns are no longer available')
|
||||
).toBeInTheDocument()
|
||||
expect(screen.getByText('ExcludedColumn').parentElement).toHaveClass('text-destructive-600')
|
||||
})
|
||||
|
||||
it('does not treat an unresolved partition ancestry as publishing every column', async () => {
|
||||
mockSources()
|
||||
mockPublication({
|
||||
...publication,
|
||||
config: {
|
||||
type: 'tables',
|
||||
tables: [
|
||||
{
|
||||
id: 10,
|
||||
schema: 'public',
|
||||
name: 'Events',
|
||||
columns: ['AllowedColumn'],
|
||||
row_filter: null,
|
||||
},
|
||||
],
|
||||
operations: ['insert'],
|
||||
publish_via_partition_root: false,
|
||||
},
|
||||
tables: [
|
||||
{
|
||||
id: 12,
|
||||
schema: 'public',
|
||||
name: 'Events2026August',
|
||||
kind: 'table',
|
||||
partition_parent_id: 11,
|
||||
},
|
||||
],
|
||||
})
|
||||
mockTables([
|
||||
{
|
||||
id: 12,
|
||||
schema: 'public',
|
||||
name: 'Events2026August',
|
||||
kind: 'table',
|
||||
partition_parent_id: 11,
|
||||
},
|
||||
])
|
||||
mockColumns(12, [
|
||||
{ name: 'AllowedColumn', type: 'text', nullable: false, primary_key: false },
|
||||
{ name: 'UnverifiedColumn', type: 'text', nullable: false, primary_key: false },
|
||||
])
|
||||
|
||||
customRender(
|
||||
<TableOptionsHarness tableOptions={[{ tableId: 12, clusterBy: ['UnverifiedColumn'] }]} />
|
||||
)
|
||||
|
||||
fireEvent.click(
|
||||
await screen.findByRole('button', {
|
||||
name: /public\.Events2026August: 1 clustering column/,
|
||||
})
|
||||
)
|
||||
expect(await screen.findByText('Columns could not be verified')).toBeInTheDocument()
|
||||
expect(
|
||||
screen.queryByText('Some selected columns are no longer available')
|
||||
).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('hides settings for a table removed from the publication without exposing its id', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
mockTables([
|
||||
publicationTable,
|
||||
{
|
||||
id: 202,
|
||||
schema: 'Billing',
|
||||
name: 'Invoices',
|
||||
kind: 'table',
|
||||
partition_parent_id: null,
|
||||
},
|
||||
])
|
||||
customRender(<TableOptionsHarness tableOptions={[{ tableId: 202, clusterBy: ['Region'] }]} />)
|
||||
|
||||
expect(await screen.findByText('Billing.Invoices')).toBeInTheDocument()
|
||||
expect(screen.getByText('No longer in publication')).toHaveClass('text-destructive-600')
|
||||
expect(screen.getByText('Some tables are no longer in the publication')).toBeInTheDocument()
|
||||
expect(screen.queryByText('Region')).not.toBeInTheDocument()
|
||||
expect(document.body).not.toHaveTextContent('202')
|
||||
})
|
||||
|
||||
it('does not match a dropped and recreated table by name instead of id', async () => {
|
||||
mockSources()
|
||||
mockPublication()
|
||||
mockTables([
|
||||
publicationTable,
|
||||
{
|
||||
id: 1000,
|
||||
schema: 'public',
|
||||
name: 'PreviouslyConfigured',
|
||||
kind: 'table',
|
||||
partition_parent_id: null,
|
||||
},
|
||||
])
|
||||
|
||||
customRender(
|
||||
<TableOptionsHarness
|
||||
tableOptions={[
|
||||
{
|
||||
tableId: 999,
|
||||
partitionBy: { kind: 'time_column', column: 'OccurredAt', granularity: 'day' },
|
||||
},
|
||||
]}
|
||||
/>
|
||||
)
|
||||
|
||||
expect(await screen.findByText('Previously configured table')).toBeInTheDocument()
|
||||
expect(screen.queryByText('OccurredAt')).not.toBeInTheDocument()
|
||||
expect(screen.getByText('Some tables are no longer in the publication')).toBeInTheDocument()
|
||||
expect(document.body).not.toHaveTextContent('999')
|
||||
})
|
||||
})
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
import { useParams } from 'common'
|
||||
import { Table2, Trash2 } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
import { useFieldArray, useFormState, useWatch, type Control } from 'react-hook-form'
|
||||
import { Accordion, AccordionContent, AccordionItem, AccordionTrigger, Button, cn } from 'ui'
|
||||
import { Admonition } from 'ui-patterns/Admonition'
|
||||
import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader'
|
||||
|
||||
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
|
||||
import { isMetadataValueLoading } from '../useRefreshOnOpen'
|
||||
import { TableOptionDraftRow, TableOptionRow } from './TableOptionRow'
|
||||
import { findFirstErrorMessage, resolvePublishedColumnNames } from './TableOptions.utils'
|
||||
import { useReplicationPublicationQuery } from '@/data/replication/publication-query'
|
||||
import { useReplicationSourceId } from '@/data/replication/sources-query'
|
||||
import { useReplicationTablesQuery } from '@/data/replication/tables-query'
|
||||
|
||||
const tableLabel = (table: { schema: string; name: string }) => `${table.schema}.${table.name}`
|
||||
|
||||
const GRANULARITY_SUMMARIES = {
|
||||
hour: 'Hourly',
|
||||
day: 'Daily',
|
||||
month: 'Monthly',
|
||||
year: 'Yearly',
|
||||
} as const
|
||||
|
||||
// Returns undefined when a table has nothing applied, so the caller can both render and
|
||||
// de-emphasise the placeholder rather than comparing against its wording.
|
||||
const tableOptionSummary = (
|
||||
option: NonNullable<DestinationPanelSchemaType['tableOptions']>[number] | undefined
|
||||
): string | undefined => {
|
||||
if (option === undefined) return undefined
|
||||
|
||||
const parts: string[] = []
|
||||
const partitionBy = option.partitionBy
|
||||
|
||||
if (partitionBy?.kind === 'time_column') {
|
||||
parts.push(
|
||||
partitionBy.column
|
||||
? `${GRANULARITY_SUMMARIES[partitionBy.granularity ?? 'day']} by ${partitionBy.column}`
|
||||
: 'Time column partitioning'
|
||||
)
|
||||
} else if (partitionBy?.kind === 'integer_range') {
|
||||
parts.push(
|
||||
partitionBy.column ? `Integer range by ${partitionBy.column}` : 'Integer range partitioning'
|
||||
)
|
||||
} else if (partitionBy?.kind === 'ingestion_time') {
|
||||
parts.push(
|
||||
`${GRANULARITY_SUMMARIES[partitionBy.granularity ?? 'day']} ingestion-time partitioning`
|
||||
)
|
||||
}
|
||||
|
||||
const clusteringColumnCount = option.clusterBy?.length ?? 0
|
||||
if (clusteringColumnCount > 0) {
|
||||
parts.push(
|
||||
`${clusteringColumnCount} clustering ${clusteringColumnCount === 1 ? 'column' : 'columns'}`
|
||||
)
|
||||
}
|
||||
|
||||
return parts.length > 0 ? parts.join(' · ') : undefined
|
||||
}
|
||||
|
||||
// Mirrors the row list so the list does not resize when the publication resolves.
|
||||
const TableOptionsSkeleton = () => (
|
||||
<div className="overflow-hidden rounded-md border" aria-hidden="true">
|
||||
{['w-2/3', 'w-1/2', 'w-3/4'].map((width, index) => (
|
||||
<div key={width} className="flex flex-col gap-1.5 px-3.5 py-3.5 border-t first:border-t-0">
|
||||
<ShimmeringLoader className={cn('h-3 py-0', width)} delayIndex={index} />
|
||||
<ShimmeringLoader className="h-3 w-1/4 py-0" delayIndex={index} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
|
||||
interface TableOptionsProps {
|
||||
control: Control<DestinationPanelSchemaType>
|
||||
}
|
||||
|
||||
export const TableOptions = ({ control }: TableOptionsProps) => {
|
||||
const { ref: projectRef } = useParams()
|
||||
const sourceId = useReplicationSourceId({ projectRef })
|
||||
const publicationName = useWatch({ control, name: 'publicationName' })
|
||||
const tableOptions = useWatch({ control, name: 'tableOptions' }) ?? []
|
||||
const [expandedTableId, setExpandedTableId] = useState<number>()
|
||||
|
||||
const {
|
||||
data: selectedPublication,
|
||||
isPending,
|
||||
isFetching,
|
||||
isError: isPublicationError,
|
||||
isSuccess: isPublicationSuccess,
|
||||
} = useReplicationPublicationQuery({
|
||||
projectRef,
|
||||
sourceId,
|
||||
publicationName,
|
||||
})
|
||||
const isLoadingPublicationTables = isMetadataValueLoading(
|
||||
isPending || isFetching,
|
||||
selectedPublication
|
||||
)
|
||||
const publicationTables = [...(selectedPublication?.tables ?? [])].sort((a, b) =>
|
||||
tableLabel(a).localeCompare(tableLabel(b))
|
||||
)
|
||||
|
||||
const { fields, append, remove } = useFieldArray({ control, name: 'tableOptions' })
|
||||
const { errors } = useFormState({ control, name: 'tableOptions' })
|
||||
const publicationTableIds = new Set(publicationTables.map(({ id }) => id))
|
||||
const publicationTablesById = new Map(publicationTables.map((table) => [table.id, table]))
|
||||
const configuredTables =
|
||||
selectedPublication?.config.type === 'tables' ? selectedPublication.config.tables : []
|
||||
const configuredTablesById = new Map(configuredTables.map((table) => [table.id, table]))
|
||||
const unavailableTableOptions = isPublicationSuccess
|
||||
? fields
|
||||
.map((field, index) => ({ field, index }))
|
||||
.filter(({ field }) => !publicationTableIds.has(field.tableId))
|
||||
: []
|
||||
// An expanded row needs its published columns resolved even before it has an entry, so that
|
||||
// the first column picker it opens is already filtered correctly.
|
||||
const activeTableIds = new Set([
|
||||
...fields.map(({ tableId }) => tableId),
|
||||
...(expandedTableId === undefined ? [] : [expandedTableId]),
|
||||
])
|
||||
const tablesNeedingPartitionAncestry = new Set(
|
||||
[...activeTableIds].flatMap((tableId) => {
|
||||
if (!publicationTableIds.has(tableId) || configuredTablesById.has(tableId)) return []
|
||||
|
||||
const parentId = publicationTablesById.get(tableId)?.partition_parent_id
|
||||
return parentId !== null && parentId !== undefined && !configuredTablesById.has(parentId)
|
||||
? [tableId]
|
||||
: []
|
||||
})
|
||||
)
|
||||
const shouldResolvePartitionAncestry =
|
||||
selectedPublication?.config.type === 'tables' && tablesNeedingPartitionAncestry.size > 0
|
||||
const { data: sourceTables = [], isPending: isSourceTablesPending } = useReplicationTablesQuery(
|
||||
{ projectRef, sourceId },
|
||||
{
|
||||
enabled: unavailableTableOptions.length > 0 || shouldResolvePartitionAncestry,
|
||||
}
|
||||
)
|
||||
const sourceTablesById = new Map(sourceTables.map((table) => [table.id, table]))
|
||||
const knownSourceTablesById = new Map(
|
||||
[...publicationTables, ...sourceTables].map((table) => [table.id, table])
|
||||
)
|
||||
|
||||
if (!publicationName) {
|
||||
return (
|
||||
<Admonition
|
||||
type="note"
|
||||
title="Select a publication"
|
||||
description="Choose the publication whose destination tables you want to partition or cluster."
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
if (isLoadingPublicationTables) {
|
||||
return <TableOptionsSkeleton />
|
||||
}
|
||||
|
||||
if (isPublicationError) {
|
||||
return (
|
||||
<Admonition
|
||||
type="warning"
|
||||
title="Table settings could not be loaded"
|
||||
description="Refresh the page before changing or saving the table layout."
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
if (publicationTables.length === 0 && unavailableTableOptions.length === 0) {
|
||||
return <p className="text-sm text-foreground-lighter">This publication has no tables.</p>
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-3">
|
||||
<div className="overflow-hidden rounded-md border">
|
||||
<Accordion
|
||||
type="single"
|
||||
collapsible
|
||||
value={expandedTableId?.toString() ?? ''}
|
||||
onValueChange={(value) => setExpandedTableId(value === '' ? undefined : Number(value))}
|
||||
>
|
||||
{publicationTables.map((table) => {
|
||||
const index = fields.findIndex((field) => field.tableId === table.id)
|
||||
const isConfigured = index !== -1
|
||||
const summary = tableOptionSummary(isConfigured ? tableOptions[index] : undefined)
|
||||
// Errors live inside the collapsed row, so the trigger has to carry them or a
|
||||
// blocked save looks like nothing happened at all.
|
||||
const rowError = isConfigured
|
||||
? findFirstErrorMessage(errors.tableOptions?.[index])
|
||||
: undefined
|
||||
const summaryLabel = rowError ?? summary ?? 'Not configured'
|
||||
const publishedColumnNames =
|
||||
selectedPublication?.config.type === 'tables'
|
||||
? resolvePublishedColumnNames(table.id, configuredTablesById, knownSourceTablesById)
|
||||
: undefined
|
||||
const isPublicationColumnsPending =
|
||||
tablesNeedingPartitionAncestry.has(table.id) && isSourceTablesPending
|
||||
const sharedProps = {
|
||||
isPublicationColumnsError:
|
||||
selectedPublication?.config.type === 'tables' &&
|
||||
publishedColumnNames === null &&
|
||||
!isPublicationColumnsPending,
|
||||
isPublicationColumnsPending,
|
||||
publishedColumnNames,
|
||||
tableId: table.id,
|
||||
}
|
||||
|
||||
return (
|
||||
<AccordionItem key={table.id} value={table.id.toString()} className="last:border-b-0">
|
||||
<AccordionTrigger
|
||||
aria-label={`${tableLabel(table)}: ${summaryLabel}`}
|
||||
className="px-3.5 py-3 font-normal hover:bg-surface-200 hover:no-underline [&>svg]:text-foreground-lighter"
|
||||
>
|
||||
<span className="flex min-w-0 items-center gap-3.5">
|
||||
<Table2
|
||||
size={16}
|
||||
strokeWidth={1.5}
|
||||
className="shrink-0 text-foreground-lighter"
|
||||
/>
|
||||
<span className="flex min-w-0 flex-col gap-0.5">
|
||||
<span className="truncate text-sm font-medium text-foreground">
|
||||
{tableLabel(table)}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
'truncate text-sm',
|
||||
rowError && 'text-destructive-600',
|
||||
!rowError && summary && 'text-foreground-lighter',
|
||||
// Most rows in a large publication are unconfigured, so keep the
|
||||
// placeholder quieter than the summaries worth scanning for.
|
||||
!rowError && !summary && 'text-foreground-muted'
|
||||
)}
|
||||
>
|
||||
{summaryLabel}
|
||||
</span>
|
||||
</span>
|
||||
</span>
|
||||
</AccordionTrigger>
|
||||
{/* Radix unmounts closed content after its exit animation, so this must not be
|
||||
gated on the expanded state as well or the row collapses as an empty box. */}
|
||||
<AccordionContent className="[&>div]:p-0">
|
||||
{isConfigured ? (
|
||||
<TableOptionRow
|
||||
control={control}
|
||||
index={index}
|
||||
onClear={() => {
|
||||
remove(index)
|
||||
setExpandedTableId(undefined)
|
||||
}}
|
||||
{...sharedProps}
|
||||
/>
|
||||
) : (
|
||||
<TableOptionDraftRow
|
||||
onCreate={(option) => append({ tableId: table.id, ...option })}
|
||||
{...sharedProps}
|
||||
/>
|
||||
)}
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
)
|
||||
})}
|
||||
</Accordion>
|
||||
|
||||
{unavailableTableOptions.map(({ field, index }) => {
|
||||
const sourceTable = sourceTablesById.get(field.tableId)
|
||||
|
||||
return (
|
||||
<div
|
||||
key={field.id}
|
||||
className="flex items-center justify-between gap-3 border-t px-3.5 py-3"
|
||||
>
|
||||
<span className="flex min-w-0 items-center gap-3.5">
|
||||
<Table2 size={16} strokeWidth={1.5} className="shrink-0 text-foreground-lighter" />
|
||||
<span className="flex min-w-0 flex-col gap-0.5">
|
||||
<span className="truncate text-sm font-medium text-foreground">
|
||||
{sourceTable ? tableLabel(sourceTable) : 'Previously configured table'}
|
||||
</span>
|
||||
<span className="text-sm text-destructive-600">No longer in publication</span>
|
||||
</span>
|
||||
</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="default"
|
||||
icon={<Trash2 size={14} />}
|
||||
onClick={() => remove(index)}
|
||||
>
|
||||
Remove
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
|
||||
{unavailableTableOptions.length > 0 && (
|
||||
<Admonition
|
||||
type="destructive"
|
||||
title="Some tables are no longer in the publication"
|
||||
description="Remove their configuration before saving."
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+4
@@ -59,6 +59,10 @@ export const PublicationSelection = ({
|
||||
shouldDirty: true,
|
||||
shouldValidate: true,
|
||||
})
|
||||
form.setValue('tableOptions', [], {
|
||||
shouldDirty: true,
|
||||
shouldValidate: true,
|
||||
})
|
||||
}
|
||||
field.onChange(value)
|
||||
},
|
||||
|
||||
+10
@@ -34,6 +34,7 @@ import {
|
||||
buildTableSyncCopyConfig,
|
||||
generateDefaultValues,
|
||||
pruneStaleSelectedTableIds,
|
||||
pruneStaleTableOptions,
|
||||
} from './DestinationForm.utils'
|
||||
import { DestinationNameInput } from './DestinationNameInput'
|
||||
import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
|
||||
@@ -314,6 +315,11 @@ export const DestinationForm = ({
|
||||
publication: selectedPublication,
|
||||
publicationName: rawData.publicationName,
|
||||
}),
|
||||
tableOptions: pruneStaleTableOptions({
|
||||
tableOptions: rawData.tableOptions,
|
||||
publication: selectedPublication,
|
||||
publicationName: rawData.publicationName,
|
||||
}),
|
||||
}
|
||||
|
||||
if (
|
||||
@@ -546,6 +552,10 @@ export const DestinationForm = ({
|
||||
shouldDirty: true,
|
||||
shouldValidate: true,
|
||||
})
|
||||
form.setValue('tableOptions', [], {
|
||||
shouldDirty: true,
|
||||
shouldValidate: true,
|
||||
})
|
||||
form.setValue('publicationName', newPublication, {
|
||||
shouldDirty: true,
|
||||
shouldValidate: true,
|
||||
|
||||
Reference in new issue
Block a user