From 230cc3b6dad0535fc51c8802ff23e91d2d98a84c Mon Sep 17 00:00:00 2001 From: Danny White <3104761+dnywh@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:45:07 +1000 Subject: [PATCH] feat(studio): configure BigQuery table layouts --- .../DestinationForm/AdvancedSettings.tsx | 12 + .../BigQuery/TableOptionFields.tsx | 332 ++++++++++++ .../BigQuery/TableOptionRow.tsx | 259 +++++++++ .../BigQuery/TableOptions.test.tsx | 505 ++++++++++++++++++ .../DestinationForm/BigQuery/TableOptions.tsx | 303 +++++++++++ .../DestinationForm/PublicationSelection.tsx | 4 + .../DestinationForm/index.tsx | 10 + 7 files changed, 1425 insertions(+) create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionFields.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionRow.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.test.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.tsx diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx index 23b147e9699..ce485f3183d 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx @@ -19,6 +19,7 @@ import { import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' import { DestinationType } from '../DestinationPanel.types' +import { TableOptions } from './BigQuery/TableOptions' import { DEFAULT_CONNECTION_POOL_SIZE, DEFAULT_MAX_COPY_CONNECTIONS_PER_TABLE, @@ -237,6 +238,17 @@ export const AdvancedSettings = ({ )} /> + +
+
+ Table layout +

+ Partitioning and clustering for each BigQuery table. Applied when a + destination table is first created or reset. +

+
+ +
)} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionFields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionFields.tsx new file mode 100644 index 00000000000..a07820b38dd --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionFields.tsx @@ -0,0 +1,332 @@ +import type { ChangeEvent } from 'react' +import { Input, Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from 'ui' +import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' +import { MultiSelector } from 'ui-patterns/multi-select' +import { SelectionListState } from 'ui-patterns/SelectionListState' + +import { defaultPartitionByForKind, parseIntegerInput, shortenPgType } from './BigQuery.utils' +import type { BigQueryPartitionKind } from './BigQuery.utils' +import { isPartitionColumnTypeCompatible } from './TableOptions.utils' +import { + BIGQUERY_MAX_CLUSTERING_COLUMNS, + BIGQUERY_TIME_PARTITION_GRANULARITIES, + type BigQueryPartitionBy, + type BigQueryTimePartitionGranularity, +} from '@/data/replication/create-destination-pipeline-mutation' + +const PARTITION_KIND_LABELS: Record = { + none: 'None', + time_column: 'Time column', + integer_range: 'Integer range', + ingestion_time: 'Ingestion time', +} + +const GRANULARITY_LABELS: Record = { + hour: 'Hour', + day: 'Day', + month: 'Month', + year: 'Year', +} + +const ColumnOption = ({ + name, + type, + isUnavailable = false, +}: { + name: string + type?: string + isUnavailable?: boolean +}) => ( + + {name} + {isUnavailable && (no longer available)} + {type && ( + + {shortenPgType(type)} + + )} + +) + +export interface ColumnSelectionState { + availableColumnNames: string[] + columnTypeByName: ReadonlyMap + isError: boolean + isLoading: boolean + onOpenChange: (isOpen: boolean) => void + unavailableColumnSet: ReadonlySet +} + +interface PartitioningFieldsProps { + columnSelection: ColumnSelectionState + errors: { + column?: string + end?: string + interval?: string + start?: string + } + onChange: (partitionBy: BigQueryPartitionBy | undefined) => void + onNeedsColumns: () => void + partitionBy: BigQueryPartitionBy | undefined +} + +export const PartitioningFields = ({ + columnSelection, + errors, + onChange, + onNeedsColumns, + partitionBy, +}: PartitioningFieldsProps) => { + const partitionKind: BigQueryPartitionKind = partitionBy?.kind ?? 'none' + + const compatibleColumnNames = + partitionBy && (partitionBy.kind === 'time_column' || partitionBy.kind === 'integer_range') + ? columnSelection.availableColumnNames.filter((column) => + isPartitionColumnTypeCompatible( + partitionBy.kind, + columnSelection.columnTypeByName.get(column) + ) + ) + : columnSelection.availableColumnNames + const compatibleColumnEmptyLabel = + partitionBy?.kind === 'time_column' + ? 'No published date or timestamp columns' + : 'No published integer columns' + + return ( + <> + + + + + {partitionBy && 'column' in partitionBy && ( + + + + )} + + {partitionBy && 'granularity' in partitionBy && ( + + + + )} + + {partitionBy?.kind === 'integer_range' && ( +
+ + ) => + onChange({ + ...partitionBy, + start: parseIntegerInput(event.target.value, partitionBy.start), + }) + } + /> + + + ) => + onChange({ + ...partitionBy, + end: parseIntegerInput(event.target.value, partitionBy.end), + }) + } + /> + + + ) => + onChange({ + ...partitionBy, + interval: parseIntegerInput(event.target.value, partitionBy.interval), + }) + } + /> + +
+ )} + + ) +} + +interface ClusteringFieldsProps { + clusterBy: string[] + columnSelection: ColumnSelectionState + error?: string + onChange: (columns: string[]) => void +} + +export const ClusteringFields = ({ + clusterBy, + columnSelection, + error, + onChange, +}: ClusteringFieldsProps) => { + const hasReachedColumnLimit = clusterBy.length >= BIGQUERY_MAX_CLUSTERING_COLUMNS + + return ( + + { + if (columns.length <= BIGQUERY_MAX_CLUSTERING_COLUMNS) onChange(columns) + }} + onOpenChange={columnSelection.onOpenChange} + > + ( + + )} + /> + + + {columnSelection.availableColumnNames.map((column) => ( + + + + ))} + + + + + ) +} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionRow.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionRow.tsx new file mode 100644 index 00000000000..f1f7d91caec --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionRow.tsx @@ -0,0 +1,259 @@ +import { useParams } from 'common' +import { RotateCcw } from 'lucide-react' +import { useState } from 'react' +import { get, useController, useFormState, type Control, type FieldErrors } from 'react-hook-form' +import { Button } from 'ui' +import { Admonition } from 'ui-patterns/Admonition' + +import type { DestinationPanelSchemaType } from '../DestinationForm.schema' +import { + isMetadataListErrorVisible, + isMetadataListLoading, + useRefreshOnOpen, +} from '../useRefreshOnOpen' +import { + ClusteringFields, + PartitioningFields, + type ColumnSelectionState, +} from './TableOptionFields' +import { useReplicationSourceId } from '@/data/replication/sources-query' +import { useReplicationTableColumnsQuery } from '@/data/replication/table-columns-query' + +type TableOption = NonNullable[number] +type PartitionBy = TableOption['partitionBy'] + +const getFieldErrorMessage = (errors: FieldErrors, path: string) => { + const error = get(errors, path) + return typeof error?.message === 'string' ? error.message : undefined +} + +interface TableOptionErrors { + partitionBy?: string + column?: string + start?: string + end?: string + interval?: string + clusterBy?: string +} + +interface TableOptionEditorProps { + partitionBy: PartitionBy + clusterBy: string[] + onPartitionByChange: (partitionBy: PartitionBy) => void + onClusterByChange: (clusterBy: string[]) => void + errors?: TableOptionErrors + isPublicationColumnsError?: boolean + isPublicationColumnsPending?: boolean + publishedColumnNames?: ReadonlySet | null + tableId: number + onClear?: () => void +} + +const TableOptionEditor = ({ + partitionBy, + clusterBy, + onPartitionByChange, + onClusterByChange, + errors = {}, + isPublicationColumnsError = false, + isPublicationColumnsPending = false, + publishedColumnNames, + tableId, + onClear, +}: TableOptionEditorProps) => { + const { ref: projectRef } = useParams() + const sourceId = useReplicationSourceId({ projectRef }) + + const shouldInitiallyLoadColumns = + partitionBy?.kind === 'time_column' || + partitionBy?.kind === 'integer_range' || + clusterBy.length > 0 + const [shouldLoadColumns, setShouldLoadColumns] = useState(shouldInitiallyLoadColumns) + const { + data: columns = [], + isPending, + isFetching, + isError, + isSuccess, + refetch: refetchColumns, + } = useReplicationTableColumnsQuery( + { projectRef, sourceId, tableId }, + { enabled: shouldLoadColumns } + ) + + const isLoadingColumns = + (shouldLoadColumns && isMetadataListLoading(isPending || isFetching, columns.length)) || + isPublicationColumnsPending + const sourceColumnNames = columns.map((column) => column.name) + const sourceColumnNameSet = new Set(sourceColumnNames) + const canUseColumns = !isPublicationColumnsPending && !isPublicationColumnsError + const availableColumnNames = canUseColumns + ? sourceColumnNames.filter( + (column) => + publishedColumnNames === undefined || + (publishedColumnNames !== null && publishedColumnNames.has(column)) + ) + : [] + const columnTypeByName = new Map(columns.map((column) => [column.name, column.type])) + const areColumnsVerified = isSuccess && canUseColumns + const partitionColumn = partitionBy && 'column' in partitionBy ? partitionBy.column : undefined + const configuredColumns = [partitionColumn, ...clusterBy].filter( + (column, columnIndex, allConfiguredColumns): column is string => + typeof column === 'string' && + column.trim().length > 0 && + allConfiguredColumns.indexOf(column) === columnIndex + ) + const unavailableColumns = configuredColumns.filter( + (column) => + areColumnsVerified && + (!sourceColumnNameSet.has(column) || + (publishedColumnNames != null && !publishedColumnNames.has(column))) + ) + const unavailableColumnSet = new Set(unavailableColumns) + const isColumnSelectionError = + isPublicationColumnsError || isMetadataListErrorVisible(isError, columns.length) + + const { handleOpenChange: handleRefreshColumnsOnOpen } = useRefreshOnOpen({ + isEnabled: shouldLoadColumns, + refetch: refetchColumns, + }) + const handleColumnPickerOpenChange = (isOpen: boolean) => { + if (isOpen && !shouldLoadColumns) { + setShouldLoadColumns(true) + return + } + handleRefreshColumnsOnOpen(isOpen) + } + + const columnSelection: ColumnSelectionState = { + availableColumnNames, + columnTypeByName, + isError: isColumnSelectionError, + isLoading: isLoadingColumns, + onOpenChange: handleColumnPickerOpenChange, + unavailableColumnSet, + } + + return ( +
+ setShouldLoadColumns(true)} + columnSelection={columnSelection} + errors={{ + column: errors.column, + start: errors.start, + end: errors.end, + interval: errors.interval, + }} + /> + + {errors.partitionBy &&

{errors.partitionBy}

} + + + + {shouldLoadColumns && isColumnSelectionError && ( + + )} + + {unavailableColumns.length > 0 && ( + + )} + + {onClear !== undefined && ( + + )} +
+ ) +} + +type SharedRowProps = Pick< + TableOptionEditorProps, + 'isPublicationColumnsError' | 'isPublicationColumnsPending' | 'publishedColumnNames' | 'tableId' +> + +interface TableOptionRowProps extends SharedRowProps { + control: Control + index: number + onClear: () => void +} + +export const TableOptionRow = ({ + control, + index, + onClear, + ...sharedProps +}: TableOptionRowProps) => { + const { errors } = useFormState({ control }) + + const { field: partitionByField } = useController({ + control, + name: `tableOptions.${index}.partitionBy`, + }) + const { field: clusterByField } = useController({ + control, + name: `tableOptions.${index}.clusterBy`, + }) + + const fieldPath = `tableOptions.${index}` + + return ( + + ) +} + +interface TableOptionDraftRowProps extends SharedRowProps { + onCreate: (option: Pick) => void +} + +/** + * Rendered while a table has no entry in the tableOptions field array. Expanding a row must not + * touch form state, otherwise merely looking at a table leaves the form dirty and triggers the + * unsaved-changes prompt. The first edit creates the entry, after which TableOptionRow takes over. + */ +export const TableOptionDraftRow = ({ onCreate, ...sharedProps }: TableOptionDraftRowProps) => ( + onCreate({ partitionBy, clusterBy: [] })} + onClusterByChange={(clusterBy) => onCreate({ partitionBy: undefined, clusterBy })} + {...sharedProps} + /> +) diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.test.tsx new file mode 100644 index 00000000000..711757428c3 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.test.tsx @@ -0,0 +1,505 @@ +import { zodResolver } from '@hookform/resolvers/zod' +import { fireEvent, screen } from '@testing-library/react' +import type { components } from 'api-types' +import { HttpResponse } from 'msw' +import { useForm } from 'react-hook-form' +import { Form } from 'ui' +import { describe, expect, it } from 'vitest' + +import { + DestinationPanelFormSchema, + type DestinationPanelSchemaType, +} from '../DestinationForm.schema' +import { TableOptions } from './TableOptions' +import { customRender } from '@/tests/lib/custom-render' +import { addAPIMock } from '@/tests/lib/msw' + +type PublicationDetailsResponse = components['schemas']['PublicationDetailsResponse_Output'] +type ReadColumnsResponse = components['schemas']['ReadColumnsResponse_Output'] +type ReadTablesResponse = components['schemas']['ReadTablesResponse_Output'] +type SourcesResponse = components['schemas']['SourcesResponse_Output'] + +const publicationTable = { + id: 101, + schema: 'public', + name: 'Orders', + kind: 'table' as const, + partition_parent_id: null, +} + +const publication: PublicationDetailsResponse = { + name: 'analytics', + config: { + type: 'tables', + tables: [ + { + id: publicationTable.id, + schema: publicationTable.schema, + name: publicationTable.name, + columns: null, + row_filter: null, + }, + ], + operations: ['insert'], + publish_via_partition_root: false, + }, + tables: [publicationTable], +} + +const mockSources = () => { + const response: SourcesResponse = { + sources: [ + { + id: 1, + name: 'default', + tenant_id: 'tenant', + config: { host: 'db.internal', name: 'main-db', port: 5432, username: 'etl_user' }, + }, + ], + } + + addAPIMock({ + method: 'get', + path: '/platform/replication/:ref/sources', + response: () => HttpResponse.json(response), + }) +} + +const mockPublication = (response = publication) => { + addAPIMock({ + method: 'get', + path: '/platform/replication/v2/:ref/sources/:source_id/publications/:publication_name', + response: () => HttpResponse.json(response), + }) +} + +const mockColumns = (tableId: number, columns: ReadColumnsResponse['columns']) => { + addAPIMock({ + method: 'get', + path: '/platform/replication/v2/:ref/sources/:source_id/tables/:table_id/columns', + response: ({ params }) => { + if (Number(params.table_id) !== tableId) { + return HttpResponse.json({ columns: [] }) + } + return HttpResponse.json({ columns }) + }, + }) +} + +const mockTables = (tables: ReadTablesResponse['tables']) => { + addAPIMock({ + method: 'get', + path: '/platform/replication/v2/:ref/sources/:source_id/tables', + response: () => HttpResponse.json({ tables }), + }) +} + +const TableOptionsHarness = ({ + tableOptions, + publicationName = 'analytics', +}: { + tableOptions: NonNullable + publicationName?: string +}) => { + const form = useForm({ + mode: 'onChange', + resolver: zodResolver(DestinationPanelFormSchema), + defaultValues: { + name: 'Warehouse', + publicationName, + tableSyncCopyMode: 'include_all_tables', + tableSyncCopyTableIds: [], + tableOptions, + }, + }) + + return ( +
+ +

{form.formState.isDirty ? 'Form is dirty' : 'Form is pristine'}

+ + + ) +} + +describe('TableOptions source reconciliation', () => { + it('explains why a publication is required before showing table settings', async () => { + mockSources() + + customRender() + + expect(await screen.findByText('Select a publication')).toBeInTheDocument() + expect( + screen.getByText( + 'Choose the publication whose destination tables you want to partition or cluster.' + ) + ).toBeInTheDocument() + }) + + it('discloses table settings without implying that the table is excluded', async () => { + mockSources() + mockPublication() + + customRender() + + const tableTrigger = await screen.findByRole('button', { + name: 'public.Orders: Not configured', + }) + expect(screen.queryByText('Partition by')).not.toBeInTheDocument() + + fireEvent.click(tableTrigger) + expect(await screen.findByText('Partition by')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Clear' })).not.toBeInTheDocument() + + fireEvent.click(tableTrigger) + expect(screen.queryByText('Partition by')).not.toBeInTheDocument() + expect(screen.getByText('Not configured')).toBeInTheDocument() + }) + + it('keeps the form pristine when a table row is only expanded', async () => { + mockSources() + mockPublication() + + customRender() + + const tableTrigger = await screen.findByRole('button', { + name: 'public.Orders: Not configured', + }) + expect(screen.getByText('Form is pristine')).toBeInTheDocument() + + fireEvent.click(tableTrigger) + expect(await screen.findByText('Partition by')).toBeInTheDocument() + expect(screen.getByText('Form is pristine')).toBeInTheDocument() + + fireEvent.click(tableTrigger) + expect(screen.getByText('Form is pristine')).toBeInTheDocument() + }) + + it('clears a configured layout without removing its table row', async () => { + mockSources() + mockPublication() + mockColumns(101, [{ name: 'region', type: 'text', nullable: false, primary_key: false }]) + + customRender() + + fireEvent.click( + await screen.findByRole('button', { name: 'public.Orders: 1 clustering column' }) + ) + fireEvent.click(await screen.findByRole('button', { name: 'Clear' })) + + expect( + screen.getByRole('button', { name: 'public.Orders: Not configured' }) + ).toBeInTheDocument() + expect(screen.queryByText('Partition by')).not.toBeInTheDocument() + }) + + it('shows integer range validation errors beside the invalid fields', async () => { + mockSources() + mockPublication() + mockColumns(101, [{ name: 'id', type: 'int8', nullable: false, primary_key: true }]) + + customRender( + + ) + + fireEvent.click(await screen.findByRole('button', { name: /public\.Orders: Integer range/ })) + fireEvent.click(screen.getByRole('button', { name: 'Validate form' })) + expect(await screen.findByText('Interval must be greater than 0.')).toBeInTheDocument() + // Once beside the End field, and once on the trigger so a collapsed row still explains itself. + expect(screen.getAllByText('End must be greater than start.')).toHaveLength(2) + expect( + screen.getByRole('button', { name: 'public.Orders: End must be greater than start.' }) + ).toBeInTheDocument() + }) + + it('blocks an incomplete partition instead of dropping it silently on save', async () => { + mockSources() + mockPublication() + mockColumns(101, [ + { name: 'CreatedAt', type: 'timestamptz', nullable: false, primary_key: true }, + ]) + + customRender( + + ) + + const tableTrigger = await screen.findByRole('button', { + name: 'public.Orders: Time column partitioning', + }) + + fireEvent.click(screen.getByRole('button', { name: 'Validate form' })) + expect(await screen.findByText('Select a partition column')).toBeInTheDocument() + + // The row can be collapsed when the save is attempted, so the trigger has to explain it too. + expect( + await screen.findByRole('button', { name: 'public.Orders: Select a partition column' }) + ).toBe(tableTrigger) + }) + + it('marks configured columns that no longer exist while preserving valid names and types', async () => { + mockSources() + mockPublication() + mockColumns(101, [ + { name: 'region', type: 'text', nullable: false, primary_key: false }, + { name: 'CreatedAt', type: 'timestamptz', nullable: false, primary_key: false }, + ]) + + customRender( + + ) + + fireEvent.click( + await screen.findByRole('button', { name: /public\.Orders: Daily by DroppedAt/ }) + ) + expect( + await screen.findByText('Some selected columns are no longer available') + ).toBeInTheDocument() + expect(screen.getAllByText('region')[0]).toBeInTheDocument() + expect(screen.getByText('text')).toBeInTheDocument() + expect(screen.getByText('DroppedAt').parentElement).toHaveClass('text-destructive-600') + expect(screen.getByText('LegacySegment').parentElement).toHaveClass('text-destructive-600') + }) + + it('marks existing columns that are no longer included in the publication', async () => { + mockSources() + mockPublication({ + ...publication, + config: { + type: 'tables', + tables: [ + { + id: publicationTable.id, + schema: publicationTable.schema, + name: publicationTable.name, + columns: ['region'], + row_filter: null, + }, + ], + operations: ['insert'], + publish_via_partition_root: false, + }, + }) + mockColumns(101, [ + { name: 'region', type: 'text', nullable: false, primary_key: false }, + { name: 'PrivateNote', type: 'text', nullable: true, primary_key: false }, + ]) + + customRender( + + ) + + fireEvent.click( + await screen.findByRole('button', { name: /public\.Orders: 1 clustering column/ }) + ) + expect( + await screen.findByText('Some selected columns are no longer available') + ).toBeInTheDocument() + expect(screen.getByText('PrivateNote').parentElement).toHaveClass('text-destructive-600') + }) + + it('inherits explicit column filters through nested partition ancestors', async () => { + mockSources() + mockPublication({ + ...publication, + config: { + type: 'tables', + tables: [ + { + id: 10, + schema: 'public', + name: 'Events', + columns: ['AllowedColumn'], + row_filter: null, + }, + ], + operations: ['insert'], + publish_via_partition_root: false, + }, + tables: [ + { + id: 12, + schema: 'public', + name: 'Events2026August', + kind: 'table', + partition_parent_id: 11, + }, + ], + }) + mockTables([ + { + id: 10, + schema: 'public', + name: 'Events', + kind: 'partitioned_table', + partition_parent_id: null, + }, + { + id: 11, + schema: 'public', + name: 'Events2026', + kind: 'partitioned_table', + partition_parent_id: 10, + }, + { + id: 12, + schema: 'public', + name: 'Events2026August', + kind: 'table', + partition_parent_id: 11, + }, + ]) + mockColumns(12, [ + { name: 'AllowedColumn', type: 'text', nullable: false, primary_key: false }, + { name: 'ExcludedColumn', type: 'text', nullable: false, primary_key: false }, + ]) + + customRender( + + ) + + fireEvent.click( + await screen.findByRole('button', { + name: /public\.Events2026August: 1 clustering column/, + }) + ) + expect( + await screen.findByText('Some selected columns are no longer available') + ).toBeInTheDocument() + expect(screen.getByText('ExcludedColumn').parentElement).toHaveClass('text-destructive-600') + }) + + it('does not treat an unresolved partition ancestry as publishing every column', async () => { + mockSources() + mockPublication({ + ...publication, + config: { + type: 'tables', + tables: [ + { + id: 10, + schema: 'public', + name: 'Events', + columns: ['AllowedColumn'], + row_filter: null, + }, + ], + operations: ['insert'], + publish_via_partition_root: false, + }, + tables: [ + { + id: 12, + schema: 'public', + name: 'Events2026August', + kind: 'table', + partition_parent_id: 11, + }, + ], + }) + mockTables([ + { + id: 12, + schema: 'public', + name: 'Events2026August', + kind: 'table', + partition_parent_id: 11, + }, + ]) + mockColumns(12, [ + { name: 'AllowedColumn', type: 'text', nullable: false, primary_key: false }, + { name: 'UnverifiedColumn', type: 'text', nullable: false, primary_key: false }, + ]) + + customRender( + + ) + + fireEvent.click( + await screen.findByRole('button', { + name: /public\.Events2026August: 1 clustering column/, + }) + ) + expect(await screen.findByText('Columns could not be verified')).toBeInTheDocument() + expect( + screen.queryByText('Some selected columns are no longer available') + ).not.toBeInTheDocument() + }) + + it('hides settings for a table removed from the publication without exposing its id', async () => { + mockSources() + mockPublication() + mockTables([ + publicationTable, + { + id: 202, + schema: 'Billing', + name: 'Invoices', + kind: 'table', + partition_parent_id: null, + }, + ]) + customRender() + + expect(await screen.findByText('Billing.Invoices')).toBeInTheDocument() + expect(screen.getByText('No longer in publication')).toHaveClass('text-destructive-600') + expect(screen.getByText('Some tables are no longer in the publication')).toBeInTheDocument() + expect(screen.queryByText('Region')).not.toBeInTheDocument() + expect(document.body).not.toHaveTextContent('202') + }) + + it('does not match a dropped and recreated table by name instead of id', async () => { + mockSources() + mockPublication() + mockTables([ + publicationTable, + { + id: 1000, + schema: 'public', + name: 'PreviouslyConfigured', + kind: 'table', + partition_parent_id: null, + }, + ]) + + customRender( + + ) + + expect(await screen.findByText('Previously configured table')).toBeInTheDocument() + expect(screen.queryByText('OccurredAt')).not.toBeInTheDocument() + expect(screen.getByText('Some tables are no longer in the publication')).toBeInTheDocument() + expect(document.body).not.toHaveTextContent('999') + }) +}) diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.tsx new file mode 100644 index 00000000000..f1fca4970c4 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.tsx @@ -0,0 +1,303 @@ +import { useParams } from 'common' +import { Table2, Trash2 } from 'lucide-react' +import { useState } from 'react' +import { useFieldArray, useFormState, useWatch, type Control } from 'react-hook-form' +import { Accordion, AccordionContent, AccordionItem, AccordionTrigger, Button, cn } from 'ui' +import { Admonition } from 'ui-patterns/Admonition' +import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader' + +import type { DestinationPanelSchemaType } from '../DestinationForm.schema' +import { isMetadataValueLoading } from '../useRefreshOnOpen' +import { TableOptionDraftRow, TableOptionRow } from './TableOptionRow' +import { findFirstErrorMessage, resolvePublishedColumnNames } from './TableOptions.utils' +import { useReplicationPublicationQuery } from '@/data/replication/publication-query' +import { useReplicationSourceId } from '@/data/replication/sources-query' +import { useReplicationTablesQuery } from '@/data/replication/tables-query' + +const tableLabel = (table: { schema: string; name: string }) => `${table.schema}.${table.name}` + +const GRANULARITY_SUMMARIES = { + hour: 'Hourly', + day: 'Daily', + month: 'Monthly', + year: 'Yearly', +} as const + +// Returns undefined when a table has nothing applied, so the caller can both render and +// de-emphasise the placeholder rather than comparing against its wording. +const tableOptionSummary = ( + option: NonNullable[number] | undefined +): string | undefined => { + if (option === undefined) return undefined + + const parts: string[] = [] + const partitionBy = option.partitionBy + + if (partitionBy?.kind === 'time_column') { + parts.push( + partitionBy.column + ? `${GRANULARITY_SUMMARIES[partitionBy.granularity ?? 'day']} by ${partitionBy.column}` + : 'Time column partitioning' + ) + } else if (partitionBy?.kind === 'integer_range') { + parts.push( + partitionBy.column ? `Integer range by ${partitionBy.column}` : 'Integer range partitioning' + ) + } else if (partitionBy?.kind === 'ingestion_time') { + parts.push( + `${GRANULARITY_SUMMARIES[partitionBy.granularity ?? 'day']} ingestion-time partitioning` + ) + } + + const clusteringColumnCount = option.clusterBy?.length ?? 0 + if (clusteringColumnCount > 0) { + parts.push( + `${clusteringColumnCount} clustering ${clusteringColumnCount === 1 ? 'column' : 'columns'}` + ) + } + + return parts.length > 0 ? parts.join(' · ') : undefined +} + +// Mirrors the row list so the list does not resize when the publication resolves. +const TableOptionsSkeleton = () => ( + +) + +interface TableOptionsProps { + control: Control +} + +export const TableOptions = ({ control }: TableOptionsProps) => { + const { ref: projectRef } = useParams() + const sourceId = useReplicationSourceId({ projectRef }) + const publicationName = useWatch({ control, name: 'publicationName' }) + const tableOptions = useWatch({ control, name: 'tableOptions' }) ?? [] + const [expandedTableId, setExpandedTableId] = useState() + + const { + data: selectedPublication, + isPending, + isFetching, + isError: isPublicationError, + isSuccess: isPublicationSuccess, + } = useReplicationPublicationQuery({ + projectRef, + sourceId, + publicationName, + }) + const isLoadingPublicationTables = isMetadataValueLoading( + isPending || isFetching, + selectedPublication + ) + const publicationTables = [...(selectedPublication?.tables ?? [])].sort((a, b) => + tableLabel(a).localeCompare(tableLabel(b)) + ) + + const { fields, append, remove } = useFieldArray({ control, name: 'tableOptions' }) + const { errors } = useFormState({ control, name: 'tableOptions' }) + const publicationTableIds = new Set(publicationTables.map(({ id }) => id)) + const publicationTablesById = new Map(publicationTables.map((table) => [table.id, table])) + const configuredTables = + selectedPublication?.config.type === 'tables' ? selectedPublication.config.tables : [] + const configuredTablesById = new Map(configuredTables.map((table) => [table.id, table])) + const unavailableTableOptions = isPublicationSuccess + ? fields + .map((field, index) => ({ field, index })) + .filter(({ field }) => !publicationTableIds.has(field.tableId)) + : [] + // An expanded row needs its published columns resolved even before it has an entry, so that + // the first column picker it opens is already filtered correctly. + const activeTableIds = new Set([ + ...fields.map(({ tableId }) => tableId), + ...(expandedTableId === undefined ? [] : [expandedTableId]), + ]) + const tablesNeedingPartitionAncestry = new Set( + [...activeTableIds].flatMap((tableId) => { + if (!publicationTableIds.has(tableId) || configuredTablesById.has(tableId)) return [] + + const parentId = publicationTablesById.get(tableId)?.partition_parent_id + return parentId !== null && parentId !== undefined && !configuredTablesById.has(parentId) + ? [tableId] + : [] + }) + ) + const shouldResolvePartitionAncestry = + selectedPublication?.config.type === 'tables' && tablesNeedingPartitionAncestry.size > 0 + const { data: sourceTables = [], isPending: isSourceTablesPending } = useReplicationTablesQuery( + { projectRef, sourceId }, + { + enabled: unavailableTableOptions.length > 0 || shouldResolvePartitionAncestry, + } + ) + const sourceTablesById = new Map(sourceTables.map((table) => [table.id, table])) + const knownSourceTablesById = new Map( + [...publicationTables, ...sourceTables].map((table) => [table.id, table]) + ) + + if (!publicationName) { + return ( + + ) + } + + if (isLoadingPublicationTables) { + return + } + + if (isPublicationError) { + return ( + + ) + } + + if (publicationTables.length === 0 && unavailableTableOptions.length === 0) { + return

This publication has no tables.

+ } + + return ( +
+
+ setExpandedTableId(value === '' ? undefined : Number(value))} + > + {publicationTables.map((table) => { + const index = fields.findIndex((field) => field.tableId === table.id) + const isConfigured = index !== -1 + const summary = tableOptionSummary(isConfigured ? tableOptions[index] : undefined) + // Errors live inside the collapsed row, so the trigger has to carry them or a + // blocked save looks like nothing happened at all. + const rowError = isConfigured + ? findFirstErrorMessage(errors.tableOptions?.[index]) + : undefined + const summaryLabel = rowError ?? summary ?? 'Not configured' + const publishedColumnNames = + selectedPublication?.config.type === 'tables' + ? resolvePublishedColumnNames(table.id, configuredTablesById, knownSourceTablesById) + : undefined + const isPublicationColumnsPending = + tablesNeedingPartitionAncestry.has(table.id) && isSourceTablesPending + const sharedProps = { + isPublicationColumnsError: + selectedPublication?.config.type === 'tables' && + publishedColumnNames === null && + !isPublicationColumnsPending, + isPublicationColumnsPending, + publishedColumnNames, + tableId: table.id, + } + + return ( + + + + + + + {tableLabel(table)} + + + {summaryLabel} + + + + + {/* Radix unmounts closed content after its exit animation, so this must not be + gated on the expanded state as well or the row collapses as an empty box. */} + + {isConfigured ? ( + { + remove(index) + setExpandedTableId(undefined) + }} + {...sharedProps} + /> + ) : ( + append({ tableId: table.id, ...option })} + {...sharedProps} + /> + )} + + + ) + })} + + + {unavailableTableOptions.map(({ field, index }) => { + const sourceTable = sourceTablesById.get(field.tableId) + + return ( +
+ + + + + {sourceTable ? tableLabel(sourceTable) : 'Previously configured table'} + + No longer in publication + + + +
+ ) + })} +
+ + {unavailableTableOptions.length > 0 && ( + + )} +
+ ) +} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx index d8005623ba0..4b732594581 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx @@ -59,6 +59,10 @@ export const PublicationSelection = ({ shouldDirty: true, shouldValidate: true, }) + form.setValue('tableOptions', [], { + shouldDirty: true, + shouldValidate: true, + }) } field.onChange(value) }, diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx index 7f6202b1423..80d904bab70 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx @@ -34,6 +34,7 @@ import { buildTableSyncCopyConfig, generateDefaultValues, pruneStaleSelectedTableIds, + pruneStaleTableOptions, } from './DestinationForm.utils' import { DestinationNameInput } from './DestinationNameInput' import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils' @@ -314,6 +315,11 @@ export const DestinationForm = ({ publication: selectedPublication, publicationName: rawData.publicationName, }), + tableOptions: pruneStaleTableOptions({ + tableOptions: rawData.tableOptions, + publication: selectedPublication, + publicationName: rawData.publicationName, + }), } if ( @@ -546,6 +552,10 @@ export const DestinationForm = ({ shouldDirty: true, shouldValidate: true, }) + form.setValue('tableOptions', [], { + shouldDirty: true, + shouldValidate: true, + }) form.setValue('publicationName', newPublication, { shouldDirty: true, shouldValidate: true,