feat(studio): add pipeline data and custom settings review (#51635)

## Problem

The pipeline wizard in #49243 needs publication, initial sync and custom
settings reviewed before creation.

## Solution

Extend the preview-enabled review from #51633 with Data, custom
connection settings and configured BigQuery table layouts. Values update
as fields change. Unchanged advanced defaults and unconfigured table
layouts are omitted; an explicit maximum staleness of zero is shown.

Preview off and editing keep their existing layout. Validation and
creation behaviour stay the same; no wizard routing is added. Stacked on
#51633.

| This PR: Simple Case | This PR: Extreme Case | Reference: Final Wizard
|
| --- | --- | --- |
| <img width="1244" height="1994" alt="CleanShot 2026-10-14 at 11 22
16@2x"
src="https://github.com/user-attachments/assets/5c2c9ea4-7b2b-4f0b-8ebe-0cfc2ac27059"
/> | <img width="1226" height="3568" alt="CleanShot 2026-10-14 at 12 02
16@2x"
src="https://github.com/user-attachments/assets/dc556d4a-5130-4b6c-80b5-8644603d9a55"
/> | <img width="1486" height="2364" alt="CleanShot 2026-10-14 at 10 50
30@2x"
src="https://github.com/user-attachments/assets/982b7d23-9449-4841-b002-9e30fbbabfe9"
/> |

## Review instructions

1. Open the [Pipelines
preview](https://studio-staging-git-dnywh-featpipeline-data-review-supabase.vercel.app/dashboard/project/_/database/pipelines),
select a test project with Pipelines enabled, enable **Pipeline creation
wizard** in Feature Previews and click **Add pipeline**. Confirm
**Data** review follows the destination review.
2. Select a publication. Confirm its name, table count and initial-sync
choice appear as read-only fields. Choose selected tables to include or
exclude and confirm their schema-qualified names appear. Change
publication and confirm old tables are not shown.
3. Under Data’s **Advanced settings**, change batch wait time, table
sync workers or initial sync connections. Confirm only custom values
appear in review. Restore defaults and confirm those rows disappear. For
BigQuery, configure partitioning or clustering and confirm the table
name and wrapping layout summary appear in the main column under **Table
layout**; unconfigured layouts should be omitted.
4. Change BigQuery connection pool size or maximum staleness and confirm
they appear within **Connection**. If available, check DuckLake pool
size and Snowflake role too. Check desktop and phone widths, then cancel
and discard changes; no credentials or creation are needed.
5. Turn the preview off and reopen **Add pipeline**: confirm no review.
With the preview on, edit an existing pipeline and confirm no review
there either.
This commit is contained in:
Danny White authored and GitHub committed 2026-10-15 11:17:34 +11:00
1 parent 17a7d2a00f
commit f20277377b
8 files changed
+480 -10

No files matched your search

@@ -0,0 +1,69 @@
import { useWatch, type Control } from 'react-hook-form'
import { Input } from 'ui'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import {
DEFAULT_CONNECTION_POOL_SIZE,
DEFAULT_DUCKLAKE_POOL_SIZE,
} from '../DestinationPanel/DestinationForm/DestinationForm.constants'
import type { DestinationPanelSchemaType } from '../DestinationPanel/DestinationForm/DestinationForm.schema'
import type { DestinationType } from '../DestinationPanel/DestinationPanel.types'
export const PipelineConnectionReview = ({
type,
control,
}: {
type: DestinationType
control: Control<DestinationPanelSchemaType>
}) => {
const [pool, staleness, lakePool, role] = useWatch({
control,
name: ['connectionPoolSize', 'maxStalenessMins', 'ducklakePoolSize', 'snowflakeRole'],
})
const rows: { label: string; value: string; description: string }[] = []
if (type === 'BigQuery') {
if (typeof pool === 'number' && pool !== DEFAULT_CONNECTION_POOL_SIZE)
rows.push({
label: 'Connection pool size',
value: String(pool),
description: 'Number of BigQuery connections used for destination writes.',
})
if (typeof staleness === 'number')
rows.push({
label: 'Maximum staleness',
value: `${staleness} minutes`,
description: 'Maximum age of BigQuery query results for newly created or recreated tables.',
})
}
if (
type === 'DuckLake' &&
typeof lakePool === 'number' &&
lakePool !== DEFAULT_DUCKLAKE_POOL_SIZE
)
rows.push({
label: 'Pool size',
value: String(lakePool),
description: 'Maximum concurrent connections this pipeline opens to the catalog.',
})
if (type === 'Snowflake' && role?.trim())
rows.push({
label: 'Role',
value: role,
description: 'Snowflake role used for SQL requests.',
})
return (
<>
{rows.map(({ label, value, description }) => (
<FormItemLayout
key={label}
isReactForm={false}
layout="horizontal"
label={label}
description={description}
>
<Input readOnly aria-label={label} value={value} />
</FormItemLayout>
))}
</>
)
}
@@ -0,0 +1,198 @@
import { fireEvent, screen } from '@testing-library/react'
import { useForm } from 'react-hook-form'
import { Button } from 'ui'
import { describe, expect, it } from 'vitest'
import type { DestinationPanelSchemaType } from '../DestinationPanel/DestinationForm/DestinationForm.schema'
import type { DestinationType } from '../DestinationPanel/DestinationPanel.types'
import { PipelineConnectionReview } from './PipelineConnectionReview'
import { PipelineDataReview } from './PipelineDataReview'
import { describeBigQueryTableLayout } from './PipelineReviewSummary.utils'
import type { ReplicationPublicationData } from '@/data/replication/publication-query'
import { customRender } from '@/tests/lib/custom-render'
const publication: ReplicationPublicationData = {
name: 'analytics',
config: { type: 'all_tables', operations: ['insert'], publish_via_partition_root: false },
tables: [{ id: 101, schema: 'public', name: 'orders', kind: 'table', partition_parent_id: null }],
}
const TestReview = ({
type = 'BigQuery',
values = {},
available = true,
}: {
type?: DestinationType
values?: Partial<DestinationPanelSchemaType>
available?: boolean
}) => {
const form = useForm<DestinationPanelSchemaType>({
defaultValues: {
publicationName: 'analytics',
tableSyncCopyMode: 'include_all_tables',
tableSyncCopyTableIds: [],
maxFillMs: 10000,
maxTableSyncWorkers: 4,
maxCopyConnectionsPerTable: 4,
connectionPoolSize: 4,
ducklakePoolSize: 4,
invalidatedSlotBehavior: 'error',
...values,
},
})
return (
<>
<PipelineConnectionReview type={type} control={form.control} />
<PipelineDataReview
type={type}
control={form.control}
publication={available ? publication : undefined}
/>
<Button onClick={() => form.setValue('maxTableSyncWorkers', 8)}>Change workers</Button>
<Button onClick={() => form.setValue('publicationName', 'other')}>Change publication</Button>
</>
)
}
describe('PipelineDataReview', () => {
it('shows data defaults without unchanged advanced settings and reacts to changes', () => {
customRender(<TestReview />)
expect(screen.getByLabelText('Publication')).toHaveValue('analytics')
expect(screen.getByLabelText('Publication tables')).toHaveValue('1 table')
expect(screen.getByLabelText('Initial sync')).toHaveValue('All tables')
expect(screen.queryByLabelText('Table sync workers')).not.toBeInTheDocument()
expect(screen.queryByText('Custom connection settings')).not.toBeInTheDocument()
fireEvent.click(screen.getByText('Change workers'))
expect(screen.getByLabelText('Table sync workers')).toHaveValue('8')
expect(
screen.getByText('Maximum number of tables synced at the same time.')
).toBeInTheDocument()
fireEvent.click(screen.getByText('Change publication'))
expect(screen.getByLabelText('Publication tables')).toHaveValue('Unavailable')
})
it.each(['include_tables', 'skip_tables'] as const)(
'lists selected publication tables for %s',
(mode) => {
customRender(
<TestReview values={{ tableSyncCopyMode: mode, tableSyncCopyTableIds: ['101', '999'] }} />
)
expect(
screen.getByLabelText(mode === 'include_tables' ? 'Tables to include' : 'Tables to exclude')
).toHaveValue('public.orders')
}
)
it('shows custom replication and BigQuery settings and only configured layouts', () => {
customRender(
<TestReview
values={{
maxFillMs: 500,
maxCopyConnectionsPerTable: 8,
invalidatedSlotBehavior: 'recreate',
connectionPoolSize: 6,
maxStalenessMins: 0,
tableOptions: [{ tableId: 101, clusterBy: ['id'] }, { tableId: 999 }],
}}
/>
)
expect(screen.getByLabelText('Batch wait time')).toHaveValue('500 milliseconds')
expect(
screen.getByText('Maximum time before sending a partially filled batch.')
).toBeInTheDocument()
expect(
screen.getByText(
'Maximum number of source connections used to sync existing rows for each table.'
)
).toBeInTheDocument()
expect(
screen.getByText(
'If the replication slot becomes invalid, the pipeline replaces destination tables and runs a new, billable initial sync.'
)
).toBeInTheDocument()
expect(screen.getByLabelText('Connection pool size')).toHaveValue('6')
expect(screen.getByLabelText('Maximum staleness')).toHaveValue('0 minutes')
expect(screen.getByText('public.orders')).toBeInTheDocument()
expect(screen.getByText('No partitioning; Cluster by id')).toBeInTheDocument()
expect(screen.queryByText('Table 999')).not.toBeInTheDocument()
})
it.each([
{ type: 'DuckLake', values: { ducklakePoolSize: 6 }, label: 'Pool size', value: '6' },
{ type: 'Snowflake', values: { snowflakeRole: 'ANALYST' }, label: 'Role', value: 'ANALYST' },
] satisfies {
type: DestinationType
values: Partial<DestinationPanelSchemaType>
label: string
value: string
}[])('shows custom settings for $type', ({ type, values, label, value }) => {
customRender(<TestReview type={type} values={values} />)
expect(screen.getByLabelText(label)).toHaveValue(value)
expect(screen.queryByLabelText('Connection pool size')).not.toBeInTheDocument()
})
it('marks unavailable publication data without inventing a table count', () => {
customRender(<TestReview available={false} />)
expect(screen.getByLabelText('Publication tables')).toHaveValue('Unavailable')
})
it('puts the table name and incomplete partition summary in the value column', () => {
customRender(
<TestReview
values={{
tableOptions: [
{
tableId: 101,
partitionBy: { kind: 'time_column', column: '', granularity: 'month' },
clusterBy: ['table_id'],
},
],
}}
/>
)
expect(screen.getByText('Table layout')).toBeInTheDocument()
expect(
screen.getByText(
'Partitioning and clustering applied when destination tables are first created or reset.'
)
).toBeInTheDocument()
expect(screen.getByText('public.orders')).toHaveClass('break-all')
expect(screen.getByText('Partition by Not selected (month); Cluster by table_id')).toHaveClass(
'break-words'
)
expect(screen.queryByText('Table layout: public.orders')).not.toBeInTheDocument()
})
it('does not describe unavailable selections as empty or show stale layouts', () => {
const view = customRender(
<TestReview
available={false}
values={{
tableSyncCopyMode: 'include_tables',
tableSyncCopyTableIds: ['101'],
tableOptions: [{ tableId: 999, clusterBy: ['id'] }],
}}
/>
)
expect(screen.getByLabelText('Tables to include')).toHaveValue('Unavailable')
expect(screen.queryByText('Table 999')).not.toBeInTheDocument()
view.unmount()
customRender(<TestReview values={{ tableOptions: [{ tableId: 999, clusterBy: ['id'] }] }} />)
expect(screen.queryByText('Table layout')).not.toBeInTheDocument()
})
it('describes each partition mode', () => {
expect(
describeBigQueryTableLayout({
tableId: 1,
partitionBy: { kind: 'ingestion_time', granularity: 'day' },
})
).toBe('Partition by ingestion time (day); No clustering')
expect(
describeBigQueryTableLayout({
tableId: 1,
partitionBy: { kind: 'time_column', column: 'created_at', granularity: 'day' },
})
).toBe('Partition by created_at (day); No clustering')
expect(
describeBigQueryTableLayout({
tableId: 1,
partitionBy: { kind: 'integer_range', column: 'id', start: 0, end: 100, interval: 10 },
})
).toBe('Partition by id (range 0 to 100, interval 10); No clustering')
})
})
@@ -0,0 +1,157 @@
import { useWatch, type Control } from 'react-hook-form'
import { Input } from 'ui'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import * as defaults from '../DestinationPanel/DestinationForm/DestinationForm.constants'
import type { DestinationPanelSchemaType } from '../DestinationPanel/DestinationForm/DestinationForm.schema'
import {
INITIAL_SYNC_FIELD_COPY,
INITIAL_SYNC_LABELS,
PUBLICATION_FIELD_COPY,
} from '../DestinationPanel/DestinationForm/DestinationFormFieldCopy'
import type { DestinationType } from '../DestinationPanel/DestinationPanel.types'
import { describeBigQueryTableLayout } from './PipelineReviewSummary.utils'
import type { ReplicationPublicationData } from '@/data/replication/publication-query'
export const PipelineDataReview = ({
type,
control,
publication,
}: {
type: DestinationType
control: Control<DestinationPanelSchemaType>
publication?: ReplicationPublicationData
}) => {
const [publicationName, mode, ids, workers, copies, slot, batch, options] = useWatch({
control,
name: [
'publicationName',
'tableSyncCopyMode',
'tableSyncCopyTableIds',
'maxTableSyncWorkers',
'maxCopyConnectionsPerTable',
'invalidatedSlotBehavior',
'maxFillMs',
'tableOptions',
],
})
const tables = publication?.name === publicationName ? publication.tables : undefined
const rows: { label: string; value: string; description: string }[] = [
{ ...PUBLICATION_FIELD_COPY, value: publicationName || 'None selected' },
{
label: 'Publication tables',
description: 'Tables whose ongoing changes will be replicated to this destination.',
value: tables
? `${tables.length} ${tables.length === 1 ? 'table' : 'tables'}`
: 'Unavailable',
},
{
...INITIAL_SYNC_FIELD_COPY,
description:
'Controls which publication tables copy their existing rows. Ongoing replication includes new changes from every publication table, even when initial sync is skipped.',
value: INITIAL_SYNC_LABELS[mode] ?? 'Not set',
},
]
if (mode === 'include_tables' || mode === 'skip_tables') {
const selected = tables?.filter((table) => ids?.includes(String(table.id)))
rows.push({
label: mode === 'include_tables' ? 'Tables to include' : 'Tables to exclude',
description:
mode === 'include_tables'
? 'Only these tables copy their existing rows during initial sync. Ongoing changes from every publication table are replicated.'
: 'These tables skip copying existing rows during initial sync. Ongoing changes from every publication table are replicated.',
value: tables
? selected?.map(({ schema, name }) => `${schema}.${name}`).join(', ') || 'None selected'
: 'Unavailable',
})
}
const addNumber = (
label: string,
value: number | '' | undefined,
defaultValue: number,
description: string,
unit = ''
) => {
if (typeof value === 'number' && value !== defaultValue)
rows.push({ label, value: `${value}${unit}`, description })
}
addNumber(
'Batch wait time',
batch,
defaults.DEFAULT_MAX_FILL_MS,
'Maximum time before sending a partially filled batch.',
' milliseconds'
)
addNumber(
'Table sync workers',
workers,
defaults.DEFAULT_MAX_TABLE_SYNC_WORKERS,
'Maximum number of tables synced at the same time.'
)
addNumber(
'Initial sync connections per table',
copies,
defaults.DEFAULT_MAX_COPY_CONNECTIONS_PER_TABLE,
'Maximum number of source connections used to sync existing rows for each table.'
)
if (slot === 'recreate')
rows.push({
label: 'Invalidated slot behavior',
value: 'Recreate slot',
description:
'If the replication slot becomes invalid, the pipeline replaces destination tables and runs a new, billable initial sync.',
})
const layouts =
type === 'BigQuery'
? (options ?? []).filter(
(option) =>
tables?.some(({ id }) => id === option.tableId) &&
(option.partitionBy || option.clusterBy?.length)
)
: []
return (
<section aria-label="Data review" className="space-y-6 p-5">
<h3 className="text-sm font-medium text-foreground">Data</h3>
{rows.map((row) => (
<ReviewField key={row.label} {...row} />
))}
{layouts.length > 0 && (
<FormItemLayout
isReactForm={false}
layout="horizontal"
label="Table layout"
description="Partitioning and clustering applied when destination tables are first created or reset."
>
<div className="min-w-0 space-y-3">
{layouts.map((option) => {
const table = tables?.find(({ id }) => id === option.tableId)
const name = table ? `${table.schema}.${table.name}` : `Table ${option.tableId}`
return (
<div key={option.tableId} className="min-w-0 rounded-md border bg-surface-200 p-3">
<p className="break-all text-sm text-foreground">{name}</p>
<p className="mt-1 break-words text-sm text-foreground-light">
{describeBigQueryTableLayout(option)}
</p>
</div>
)
})}
</div>
</FormItemLayout>
)}
</section>
)
}
const ReviewField = ({
label,
value,
description,
}: {
label: string
value: string
description: string
}) => (
<FormItemLayout isReactForm={false} layout="horizontal" label={label} description={description}>
<Input readOnly aria-label={label} value={value} />
</FormItemLayout>
)
@@ -114,3 +114,21 @@ describe('PipelineDestinationReview', () => {
expect(screen.queryByText('Catalog URL')).not.toBeInTheDocument()
})
})
it('includes custom settings in Connection without a separate heading', () => {
customRender(
<TestReview
type="BigQuery"
values={{ ...defaults, connectionPoolSize: 3, maxStalenessMins: 10 }}
/>
)
expect(screen.getByLabelText('Connection pool size')).toHaveValue('3')
expect(screen.getByLabelText('Maximum staleness')).toHaveValue('10 minutes')
expect(
screen.getByText('Number of BigQuery connections used for destination writes.')
).toBeInTheDocument()
expect(
screen.getByText('Maximum age of BigQuery query results for newly created or recreated tables.')
).toBeInTheDocument()
expect(screen.queryByText('Custom connection settings')).not.toBeInTheDocument()
})
@@ -1,4 +1,4 @@
import { Eye, EyeOff } from 'lucide-react'
import { EyeIcon, EyeOffIcon } from 'lucide-react'
import { useState } from 'react'
import { useWatch, type Control } from 'react-hook-form'
import { Button, Input, Separator } from 'ui'
@@ -23,11 +23,9 @@ import {
SNOWFLAKE_DATABASE_FIELD_COPY,
SNOWFLAKE_SCHEMA_FIELD_COPY,
} from '../DestinationPanel/DestinationForm/DestinationFormFieldCopy'
import {
getPipelineRegionDescription,
PipelineRegionReadonly,
} from '../DestinationPanel/DestinationForm/PipelineRegionField'
import { PipelineRegionReadonly } from '../DestinationPanel/DestinationForm/PipelineRegionField'
import type { DestinationType } from '../DestinationPanel/DestinationPanel.types'
import { PipelineConnectionReview } from './PipelineConnectionReview'
const MaskedUrl = ({ value, label }: { value: string; label: string }) => {
const [isVisible, setIsVisible] = useState(false)
@@ -41,7 +39,7 @@ const MaskedUrl = ({ value, label }: { value: string; label: string }) => {
size="tiny"
aria-label={`${isVisible ? 'Hide' : 'Show'} ${label.replace(/^[A-Z][a-z]+/, (word) => word.toLowerCase())}`}
aria-pressed={isVisible}
icon={isVisible ? <EyeOff size={14} /> : <Eye size={14} />}
icon={isVisible ? <EyeOffIcon size={14} /> : <EyeIcon size={14} />}
onClick={() => setIsVisible(!isVisible)}
/>
</div>
@@ -93,7 +91,7 @@ export const PipelineDestinationReview = ({
'newNamespaceName',
],
})
const rows: { label: string; value?: string; description?: string; isMasked?: boolean }[] = []
const rows: { label: string; value?: string; description: string; isMasked?: boolean }[] = []
switch (type) {
case 'BigQuery':
rows.push(
@@ -106,7 +104,7 @@ export const PipelineDestinationReview = ({
rows.push(
{
...DUCKLAKE_CATALOG_URL_FIELD_COPY,
description: DUCKLAKE_CATALOG_URL_FIELD_COPY.createDescription,
description: 'Postgres URL of the database used for the DuckLake catalog.',
value: catalogUrl,
isMasked: true,
},
@@ -177,11 +175,12 @@ export const PipelineDestinationReview = ({
)}
</FormItemLayout>
))}
<PipelineConnectionReview type={type} control={control} />
<FormItemLayout
isReactForm={false}
layout="horizontal"
label="Pipeline region"
description={getPipelineRegionDescription(type)}
description={`All pipelines run from this region. ${type} connections originate here.`}
>
<PipelineRegionReadonly />
</FormItemLayout>
@@ -0,0 +1,21 @@
import type { DestinationPanelSchemaType } from '../DestinationPanel/DestinationForm/DestinationForm.schema'
export const describeBigQueryTableLayout = (
option: NonNullable<DestinationPanelSchemaType['tableOptions']>[number]
) => {
const partition = option.partitionBy
let partitionDescription = 'No partitioning'
if (partition?.kind === 'time_column') {
partitionDescription = `Partition by ${partition.column || 'Not selected'} (${partition.granularity ?? 'day'})`
}
if (partition?.kind === 'ingestion_time') {
partitionDescription = `Partition by ingestion time (${partition.granularity ?? 'day'})`
}
if (partition?.kind === 'integer_range') {
partitionDescription = `Partition by ${partition.column || 'Not selected'} (range ${partition.start} to ${partition.end}, interval ${partition.interval})`
}
const clustering = option.clusterBy?.length
? `Cluster by ${option.clusterBy.join(', ')}`
: 'No clustering'
return `${partitionDescription}; ${clustering}`
}
@@ -320,6 +320,7 @@ describe('DestinationForm edit submission', () => {
expect(screen.queryByRole('region', { name: 'Destination review' }) !== null).toBe(
shouldShowSection
)
expect(screen.queryByRole('region', { name: 'Data review' }) !== null).toBe(shouldShowSection)
expect(screen.queryByRole('region', { name: 'Data' }) !== null).toBe(shouldShowSection)
expect(screen.queryByRole('region', { name: 'Destination connection' }) !== null).toBe(
shouldShowSection
@@ -327,7 +328,7 @@ describe('DestinationForm edit submission', () => {
if (shouldShowSection) {
const section = screen.getByRole('region', { name: 'Destination connection' })
expect(
section.compareDocumentPosition(screen.getByText('Data')) &
section.compareDocumentPosition(screen.getByRole('region', { name: 'Data' })) &
Node.DOCUMENT_POSITION_FOLLOWING
).toBeTruthy()
}
@@ -11,6 +11,7 @@ import { Admonition } from 'ui-patterns/Admonition'
import * as z from 'zod'
import { PipelineConnectionSection } from '../../CreatePipeline/PipelineConnectionSection'
import { PipelineDataReview } from '../../CreatePipeline/PipelineDataReview'
import { PipelineDataSection } from '../../CreatePipeline/PipelineDataSection'
import { PipelineDestinationReview } from '../../CreatePipeline/PipelineDestinationReview'
import {
@@ -542,6 +543,12 @@ export const DestinationForm = ({
<>
<DialogSectionSeparator />
<PipelineDestinationReview type={selectedType} control={form.control} />
<DialogSectionSeparator />
<PipelineDataReview
type={selectedType}
control={form.control}
publication={selectedPublication}
/>
</>
)}