feat(pipelines): Add new cost estimation dialog (#47915)

This commit is contained in:
Riccardo Busetti authored and GitHub committed 2026-07-15 06:20:59 +00:00
1 parent c97bc6282d
commit c9eb33abc7
18 files changed
+723 -192

No files matched your search

@@ -22,8 +22,8 @@ type AnalyticsBucketValidationOptions = {
// Fields that are always required regardless of the namespace / access key selections.
const ANALYTICS_BUCKET_REQUIRED_FIELDS: AnalyticsBucketValidationIssue[] = [
{ path: 'warehouseName', message: 'Bucket is required' },
{ path: 's3Region', message: 'S3 Region is required' },
{ path: 's3AccessKeyId', message: 'S3 Access Key ID is required' },
{ path: 's3Region', message: 'S3 region is required' },
{ path: 's3AccessKeyId', message: 'S3 access key ID is required' },
]
export const getAnalyticsBucketValidationIssues = (
@@ -53,7 +53,7 @@ export const getAnalyticsBucketValidationIssues = (
data.s3SecretAccessKey?.trim().length &&
!data.s3AccessKeyId?.trim().length
) {
issues.push({ path: 's3AccessKeyId', message: 'S3 Access Key ID is required' })
issues.push({ path: 's3AccessKeyId', message: 'S3 access key ID is required' })
}
const currentS3AccessKeyId = data.s3AccessKeyId?.trim()
@@ -69,7 +69,7 @@ export const getAnalyticsBucketValidationIssues = (
(!options.secretsOptional || hasChangedStoredS3AccessKey) &&
!data.s3SecretAccessKey?.trim().length
) {
issues.push({ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' })
issues.push({ path: 's3SecretAccessKey', message: 'S3 secret access key is required' })
}
return issues
@@ -255,7 +255,7 @@ export const AnalyticsBucketFields = ({
name="newNamespaceName"
render={({ field }) => (
<FormItemLayout
label="New Namespace Name"
label="New namespace name"
layout="horizontal"
description="A unique name for the new namespace"
>
@@ -273,7 +273,7 @@ export const AnalyticsBucketFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="Catalog Token"
label="Catalog token"
description={
editMode ? (
'Stored catalog token is hidden and kept automatically.'
@@ -315,7 +315,7 @@ export const AnalyticsBucketFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="S3 Access Key ID"
label="S3 access key ID"
description={
<div className="flex flex-col gap-y-2">
<p>
@@ -403,7 +403,7 @@ export const AnalyticsBucketFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="S3 Secret Access Key"
label="S3 secret access key"
className="relative"
description={
editMode
@@ -10,7 +10,7 @@ export type BigQueryValidationIssue = {
const BIGQUERY_REQUIRED_FIELDS: { path: BigQueryFieldPath; message: string }[] = [
{ path: 'projectId', message: 'Project ID is required' },
{ path: 'datasetId', message: 'Dataset ID is required' },
{ path: 'serviceAccountKey', message: 'Service Account Key is required' },
{ path: 'serviceAccountKey', message: 'Service account key is required' },
]
export const getBigQueryValidationIssues = (
@@ -54,7 +54,7 @@ export const BigQueryFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="Service Account Key"
label="Service account key"
description={
editMode
? 'Stored credentials are hidden. Enter new credentials to replace them.'
@@ -6,23 +6,27 @@ export const DestinationPanelFormSchema = z.object({
// Common fields
name: z.string().min(1, 'Name is required'),
publicationName: z.string().min(1, 'Publication is required'),
maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(),
maxFillMs: z.number().min(1, 'Max fill milliseconds must be greater than 0').int().optional(),
maxTableSyncWorkers: z
.number()
.min(1, 'Max table sync workers should be greater than 0')
.min(1, 'Max table sync workers must be greater than 0')
.int()
.optional(),
maxCopyConnectionsPerTable: z
.number()
.int()
.min(1, 'Max copy connections per table should be greater than 0')
.min(1, 'Max copy connections per table must be greater than 0')
.optional(),
invalidatedSlotBehavior: z.enum(['error', 'recreate']).optional(),
// BigQuery fields
projectId: z.string().optional(),
datasetId: z.string().optional(),
serviceAccountKey: z.string().optional(),
connectionPoolSize: z.number().int().min(1).optional(),
connectionPoolSize: z
.number()
.int()
.min(1, 'Connection pool size must be greater than 0')
.optional(),
maxStalenessMins: z.number().nonnegative().optional(),
// Analytics Bucket fields, only warehouse name and namespace are visible + editable fields
warehouseName: z.string().optional(),
@@ -43,7 +47,12 @@ export const DestinationPanelFormSchema = z.object({
// DuckLake "Custom parameters" fields
ducklakeCatalogUrl: z.string().optional(),
ducklakeDataPath: z.string().optional(),
ducklakePoolSize: z.number().int().min(1).max(6).optional(),
ducklakePoolSize: z
.number()
.int()
.min(1, 'Pool size must be greater than 0')
.max(6, 'Pool size must be 6 or less')
.optional(),
ducklakeS3AccessKeyId: z.string().optional(),
ducklakeS3SecretAccessKey: z.string().optional(),
ducklakeS3Region: z.string().optional(),
@@ -150,10 +150,10 @@ describe('DestinationForm.utils DuckLake', () => {
expect(issues).toEqual([
{ path: 'ducklakeCatalogUrl', message: 'Catalog URL is required' },
{ path: 'ducklakeDataPath', message: 'Data path is required' },
{ path: 'ducklakeS3AccessKeyId', message: 'S3 Access Key ID is required' },
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 Secret Access Key is required' },
{ path: 'ducklakeS3Region', message: 'S3 Region is required' },
{ path: 'ducklakeS3Endpoint', message: 'S3 Endpoint is required' },
{ path: 'ducklakeS3AccessKeyId', message: 'S3 access key ID is required' },
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 secret access key is required' },
{ path: 'ducklakeS3Region', message: 'S3 region is required' },
{ path: 'ducklakeS3Endpoint', message: 'S3 endpoint is required' },
])
})
@@ -189,7 +189,7 @@ describe('DestinationForm.utils DuckLake', () => {
)
expect(issues).toEqual([
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 Secret Access Key is required' },
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 secret access key is required' },
])
})
@@ -207,10 +207,10 @@ describe('DestinationForm.utils DuckLake', () => {
expect(issues).toEqual([
{ path: 'ducklakeCatalogUrl', message: 'Catalog URL is required' },
{ path: 'ducklakeDataPath', message: 'Data path is required' },
{ path: 'ducklakeS3AccessKeyId', message: 'S3 Access Key ID is required' },
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 Secret Access Key is required' },
{ path: 'ducklakeS3Region', message: 'S3 Region is required' },
{ path: 'ducklakeS3Endpoint', message: 'S3 Endpoint is required' },
{ path: 'ducklakeS3AccessKeyId', message: 'S3 access key ID is required' },
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 secret access key is required' },
{ path: 'ducklakeS3Region', message: 'S3 region is required' },
{ path: 'ducklakeS3Endpoint', message: 'S3 endpoint is required' },
])
})
@@ -236,7 +236,7 @@ describe('DestinationForm.utils DuckLake', () => {
},
{
path: 'ducklakeS3Endpoint',
message: 'S3 endpoint should not contain the protocol scheme',
message: 'S3 endpoint must not contain the protocol scheme',
},
{
path: 'ducklakeMetadataSchema',
@@ -451,7 +451,7 @@ describe('DestinationForm.utils BigQuery', () => {
expect(issues).toEqual([
{ path: 'projectId', message: 'Project ID is required' },
{ path: 'datasetId', message: 'Dataset ID is required' },
{ path: 'serviceAccountKey', message: 'Service Account Key is required' },
{ path: 'serviceAccountKey', message: 'Service account key is required' },
])
})
@@ -465,7 +465,7 @@ describe('DestinationForm.utils BigQuery', () => {
expect(issues).toEqual([
{ path: 'projectId', message: 'Project ID is required' },
{ path: 'datasetId', message: 'Dataset ID is required' },
{ path: 'serviceAccountKey', message: 'Service Account Key is required' },
{ path: 'serviceAccountKey', message: 'Service account key is required' },
])
})
@@ -506,10 +506,10 @@ describe('DestinationForm.utils Analytics Bucket', () => {
expect(issues).toEqual([
{ path: 'warehouseName', message: 'Bucket is required' },
{ path: 's3Region', message: 'S3 Region is required' },
{ path: 's3AccessKeyId', message: 'S3 Access Key ID is required' },
{ path: 's3Region', message: 'S3 region is required' },
{ path: 's3AccessKeyId', message: 'S3 access key ID is required' },
{ path: 'namespace', message: 'Namespace is required' },
{ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' },
{ path: 's3SecretAccessKey', message: 'S3 secret access key is required' },
])
})
@@ -525,10 +525,10 @@ describe('DestinationForm.utils Analytics Bucket', () => {
expect(issues).toEqual([
{ path: 'warehouseName', message: 'Bucket is required' },
{ path: 's3Region', message: 'S3 Region is required' },
{ path: 's3AccessKeyId', message: 'S3 Access Key ID is required' },
{ path: 's3Region', message: 'S3 region is required' },
{ path: 's3AccessKeyId', message: 'S3 access key ID is required' },
{ path: 'namespace', message: 'Namespace is required' },
{ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' },
{ path: 's3SecretAccessKey', message: 'S3 secret access key is required' },
])
})
@@ -604,7 +604,7 @@ describe('DestinationForm.utils Analytics Bucket', () => {
)
expect(issues).toEqual([
{ path: 's3SecretAccessKey', message: 'S3 Secret Access Key is required' },
{ path: 's3SecretAccessKey', message: 'S3 secret access key is required' },
])
})
@@ -63,10 +63,10 @@ const DUCKLAKE_SUPABASE_REQUIRED_FIELDS: DucklakeValidationIssue[] = [
const DUCKLAKE_CUSTOM_REQUIRED_FIELDS: DucklakeValidationIssue[] = [
{ path: 'ducklakeCatalogUrl', message: 'Catalog URL is required' },
{ path: 'ducklakeDataPath', message: 'Data path is required' },
{ path: 'ducklakeS3AccessKeyId', message: 'S3 Access Key ID is required' },
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 Secret Access Key is required' },
{ path: 'ducklakeS3Region', message: 'S3 Region is required' },
{ path: 'ducklakeS3Endpoint', message: 'S3 Endpoint is required' },
{ path: 'ducklakeS3AccessKeyId', message: 'S3 access key ID is required' },
{ path: 'ducklakeS3SecretAccessKey', message: 'S3 secret access key is required' },
{ path: 'ducklakeS3Region', message: 'S3 region is required' },
{ path: 'ducklakeS3Endpoint', message: 'S3 endpoint is required' },
]
const DUCKLAKE_CUSTOM_SECRET_FIELDS = new Set<DucklakeFieldPath>([
@@ -113,7 +113,7 @@ export const getDucklakeValidationIssues = (
) {
issues.push({
path: 'ducklakeS3AccessKeyId',
message: 'S3 Access Key ID is required',
message: 'S3 access key ID is required',
})
}
@@ -124,7 +124,7 @@ export const getDucklakeValidationIssues = (
) {
issues.push({
path: 'ducklakeS3SecretAccessKey',
message: 'S3 Secret Access Key is required',
message: 'S3 secret access key is required',
})
}
@@ -157,7 +157,7 @@ export const getDucklakeValidationIssues = (
) {
issues.push({
path: 'ducklakeS3Endpoint',
message: 'S3 endpoint should not contain the protocol scheme',
message: 'S3 endpoint must not contain the protocol scheme',
})
}
@@ -476,7 +476,7 @@ const DuckLakeCustomFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="S3 Access Key ID"
label="S3 access key ID"
description={
editMode
? 'Stored access key ID is hidden. Enter a new key ID to replace it.'
@@ -500,7 +500,7 @@ const DuckLakeCustomFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="S3 Secret Access Key"
label="S3 secret access key"
description={
editMode
? 'Stored secret access key is hidden. Enter a new secret to replace it.'
@@ -532,7 +532,7 @@ const DuckLakeCustomFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="S3 Region"
label="S3 region"
description="Required region for the object storage provider"
>
<FormControl>
@@ -548,7 +548,7 @@ const DuckLakeCustomFields = ({
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="S3 Endpoint"
label="S3 endpoint"
description="Required endpoint without the protocol scheme, for example `127.0.0.1:5000/s3`"
>
<FormControl>
@@ -0,0 +1,248 @@
import { useEffect } from 'react'
import {
Button,
Card,
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogSection,
DialogSectionSeparator,
DialogTitle,
Table,
TableBody,
TableCell,
TableFooter,
TableHead,
TableHeader,
TableRow,
} from 'ui'
import { GenericSkeletonLoader } from 'ui-patterns/ShimmeringLoader'
import { useReplicationCostEstimateQuery } from '@/data/replication/cost-estimate-query'
import { useLatest } from '@/hooks/misc/useLatest'
import { formatBytes, formatCurrency } from '@/lib/helpers'
const MAX_VISIBLE_TABLES = 10
interface PipelineCostDialogProps {
open: boolean
isConfirming: boolean
projectRef?: string
sourceId?: number
publicationName?: string
onOpenChange: (open: boolean) => void
onConfirm: () => void
}
/**
* Shows an estimate of what the pipeline will cost (one-time initial copy, hourly pipeline
* fee, and the usage-based streaming rate).
*
* This should be a non-blocking enhancement - so if there's an error while fetching the
* pricing estimate, we skip this gate rather than block the user from creating pipeline
*/
export const PipelineCostDialog = ({
open,
isConfirming,
projectRef,
sourceId,
publicationName,
onOpenChange,
onConfirm,
}: PipelineCostDialogProps) => {
const onConfirmRef = useLatest(onConfirm)
const {
data: estimate,
isLoading,
isError,
isSuccess,
} = useReplicationCostEstimateQuery({ projectRef, sourceId, publicationName }, { enabled: open })
const tables = estimate?.table_copy.tables ?? []
const tableCount = tables.length
const visibleTables = tables.slice(0, MAX_VISIBLE_TABLES)
const hiddenTableCount = tableCount - visibleTables.length
const hasRowFilteredTables = tables.some((table) => table.is_row_filtered)
const firstMonthTotal =
(estimate?.table_copy.total_cost ?? 0) + (estimate?.pipeline.monthly_cost ?? 0)
useEffect(() => {
if (open && isError) onConfirmRef.current()
}, [open, isError, onConfirmRef])
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent size="large">
<DialogHeader>
<DialogTitle>Confirm to create and start pipeline</DialogTitle>
<DialogDescription>
Review the estimated costs before you create and start the pipeline.
</DialogDescription>
</DialogHeader>
<DialogSectionSeparator />
{/* Review these costs before you create and start the pipeline. */}
{isLoading || isError ? (
<DialogSection className="py-6">
<GenericSkeletonLoader className="w-full" />
</DialogSection>
) : (
isSuccess &&
estimate && (
<>
<DialogSection className="flex flex-col gap-y-5">
<p className="text-sm text-foreground-light">
This pipeline will replicate{' '}
<span className="text-foreground">
{tableCount} {tableCount === 1 ? 'table' : 'tables'}
</span>
{publicationName ? (
<>
{' '}
from the{' '}
<span className="text-foreground" translate="no">
{publicationName}
</span>{' '}
publication
</>
) : null}
.
</p>
<div className="flex flex-col gap-y-2">
<p className="text-sm font-medium text-foreground">Initial table copy</p>
{tableCount > 0 ? (
<Card>
<Table>
<TableHeader className="[&_th]:h-auto [&_th]:py-2">
<TableRow>
<TableHead>Table</TableHead>
<TableHead className="text-right">Est. size</TableHead>
<TableHead className="text-right" translate="no">
Est. cost ({formatCurrency(estimate.table_copy.rate_per_gb)}/GB)
</TableHead>
</TableRow>
</TableHeader>
<TableBody className="[&_td]:py-2">
{visibleTables.map((table) => (
<TableRow key={`${table.schema}.${table.name}`}>
<TableCell className="font-mono text-xs" translate="no">
{table.schema}.{table.name}
</TableCell>
<TableCell className="text-right text-xs">
{formatBytes(table.estimated_bytes)}
</TableCell>
<TableCell className="text-right font-mono text-xs" translate="no">
{formatCurrency(table.estimated_cost)}
</TableCell>
</TableRow>
))}
{hiddenTableCount > 0 && (
<TableRow>
<TableCell colSpan={3} className="text-xs text-foreground-lighter">
+{hiddenTableCount} more{' '}
{hiddenTableCount === 1 ? 'table' : 'tables'}
</TableCell>
</TableRow>
)}
</TableBody>
<TableFooter>
<TableRow>
<TableCell className="py-2">Total</TableCell>
<TableCell className="text-right py-2 text-xs">
{formatBytes(estimate.table_copy.total_bytes)}
</TableCell>
<TableCell className="text-right py-2 font-mono" translate="no">
{formatCurrency(estimate.table_copy.total_cost)}
</TableCell>
</TableRow>
</TableFooter>
</Table>
</Card>
) : (
<p className="text-sm text-foreground-light">
This publication has no tables to copy.
</p>
)}
</div>
</DialogSection>
<DialogSectionSeparator />
<DialogSection className="flex flex-col gap-y-5">
<div className="flex flex-col gap-y-2">
<p className="text-sm font-medium text-foreground">Ongoing</p>
<div className="flex items-center justify-between gap-x-4 text-sm">
<span className="text-foreground-light">Active pipeline</span>
<span className="shrink-0 text-right font-mono text-foreground" translate="no">
${estimate.pipeline.hourly_cost}/hour{' '}
<span className="text-foreground-lighter">
(~{formatCurrency(estimate.pipeline.monthly_cost)}/month)
</span>
</span>
</div>
<div className="flex items-center justify-between gap-x-4 text-sm">
<span className="text-foreground-light">Streaming changes</span>
<span className="shrink-0 text-right font-mono text-foreground" translate="no">
{formatCurrency(estimate.streaming.rate_per_gb)}/GB
</span>
</div>
<p className="text-xs text-foreground-lighter">
Streaming is billed on the volume of changes replicated after the initial copy,
so the total depends on how often your data changes.
</p>
</div>
<div className="flex flex-col gap-y-2">
<div className="flex items-center justify-between gap-x-6 rounded-md border bg-surface-100 px-4 py-3">
<div className="min-w-0">
<p className="text-sm font-medium text-foreground">
Estimated first month total
</p>
<p className="text-xs text-foreground-lighter">
Initial copy + first pipeline fee, excluding usage-based streaming
</p>
</div>
<span
className="shrink-0 text-right font-mono text-lg font-semibold text-foreground"
translate="no"
>
{formatCurrency(firstMonthTotal)}
{hasRowFilteredTables ? '*' : null}
</span>
</div>
{hasRowFilteredTables && (
<p className="text-xs text-foreground-lighter">
*Tables with row filters may cost less than shown.
</p>
)}
</div>
</DialogSection>
</>
)
)}
<DialogFooter>
<Button variant="default" disabled={isConfirming} onClick={() => onOpenChange(false)}>
Cancel
</Button>
<Button
variant="primary"
loading={isConfirming}
disabled={isConfirming || isLoading || isError}
onClick={onConfirm}
>
Create and start pipeline
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
)
}
@@ -40,6 +40,7 @@ import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
import { DuckLakeFields } from './DuckLake/Fields'
import { NewPublicationPanel } from './NewPublicationPanel'
import { NoDestinationsAvailable } from './NoDestinationsAvailable'
import { PipelineCostDialog } from './PipelineCostDialog'
import { PublicationSelection } from './PublicationSelection'
import { SnowflakeFields } from './Snowflake/Fields'
import { getSnowflakeValidationIssues } from './Snowflake/Snowflake.utils'
@@ -84,6 +85,7 @@ export const DestinationForm = ({
const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
const [showValidationWarningsDialog, setShowValidationWarningsDialog] = useState(false)
const [showCostDialog, setShowCostDialog] = useState(false)
const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)
const [newBucketSheetVisible, setNewBucketSheetVisible] = useState(false)
const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(
@@ -231,65 +233,77 @@ export const DestinationForm = ({
}
}
// Stages the form values and opens the cost-estimation dialog, which is the final gate before
// a pipeline is created and started.
const openCostDialog = (data: z.infer<typeof FormSchema>) => {
setPendingFormValues(data)
setShowCostDialog(true)
}
const onSubmit = async (data: z.infer<typeof FormSchema>) => {
if (!editMode) {
const previousValidationFailures = allValidationFailures
const previousWarnings = previousValidationFailures.filter(
(f) => f.failure_type === 'warning'
)
const previousFailuresAreOnlyWarnings =
hasRunValidation &&
previousValidationFailures.length > 0 &&
previousValidationFailures.every((f) => f.failure_type === 'warning')
const validationResult = await validateConfiguration({
// Editing an existing pipeline doesn't incur a new initial copy, so it skips the cost gate.
if (editMode) {
await submitPipeline({
data,
onValidationFail: () => {
setTimeout(() => {
validationSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'start' })
}, 100)
},
existingDestination,
onSuccess: () => form.reset(defaultValues),
onClose,
})
if (!validationResult.canContinue) {
// Critical failures shown inline — stop so user can fix them
return
}
const hasWarnings = validationResult.warnings.length > 0
const warningsUnchanged =
previousFailuresAreOnlyWarnings &&
areValidationFailuresEqual(previousWarnings, validationResult.warnings)
// Open the confirmation dialog when validation is clean, or when warnings are unchanged on
// resubmit. New/changed warnings are shown inline so the user can review and submit again.
if (hasWarnings) {
if (warningsUnchanged) {
setPendingFormValues(data)
setShowValidationWarningsDialog(true)
}
return
}
return
}
await submitPipeline({
const previousValidationFailures = allValidationFailures
const previousWarnings = previousValidationFailures.filter((f) => f.failure_type === 'warning')
const previousFailuresAreOnlyWarnings =
hasRunValidation &&
previousValidationFailures.length > 0 &&
previousValidationFailures.every((f) => f.failure_type === 'warning')
const validationResult = await validateConfiguration({
data,
existingDestination,
onSuccess: () => form.reset(defaultValues),
onClose,
onValidationFail: () => {
setTimeout(() => {
validationSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'start' })
}, 100)
},
})
if (!validationResult.canContinue) {
// Critical failures shown inline — stop so user can fix them
return
}
const hasWarnings = validationResult.warnings.length > 0
const warningsUnchanged =
previousFailuresAreOnlyWarnings &&
areValidationFailuresEqual(previousWarnings, validationResult.warnings)
// Open the warnings confirmation when there are warnings (and they're unchanged on resubmit),
// otherwise go straight to the cost dialog. New/changed warnings are shown inline so the user
// can review and submit again.
if (hasWarnings) {
if (warningsUnchanged) {
setPendingFormValues(data)
setShowValidationWarningsDialog(true)
}
return
}
openCostDialog(data)
}
const handleValidationWarningsDialogChange = (open: boolean) => {
setShowValidationWarningsDialog(open)
if (!open) setPendingFormValues(null)
// Confirming the warnings advances to the cost dialog rather than submitting directly, so the
// cost estimate is always the last thing shown before the pipeline starts.
const handleValidationWarningsConfirm = () => {
if (!pendingFormValues) return
setShowValidationWarningsDialog(false)
openCostDialog(pendingFormValues)
}
const handleValidationWarningsConfirm = async () => {
const handleCostConfirm = async () => {
if (!pendingFormValues) return
const values = pendingFormValues
setPendingFormValues(null)
setShowValidationWarningsDialog(false)
setShowCostDialog(false)
await submitPipeline({
data: values,
@@ -452,11 +466,21 @@ export const DestinationForm = ({
<ValidationWarningsDialog
open={showValidationWarningsDialog}
onOpenChange={handleValidationWarningsDialogChange}
onOpenChange={setShowValidationWarningsDialog}
isLoading={isSaving}
warningCount={validationWarnings.length}
onConfirm={handleValidationWarningsConfirm}
/>
<PipelineCostDialog
open={showCostDialog}
isConfirming={isSaving}
projectRef={projectRef}
sourceId={sourceId}
publicationName={pendingFormValues?.publicationName}
onOpenChange={setShowCostDialog}
onConfirm={handleCostConfirm}
/>
</>
)
}
@@ -1,4 +1,5 @@
import {
Card,
cn,
Dialog,
DialogContent,
@@ -57,48 +58,52 @@ export const ReadReplicaPricingDialog = () => {
Read replicas will match the compute size of your primary database and will include
25% more disk size than the primary database to accommodate WAL files.
</p>
<p className="text-foreground-light text-sm">
The additional cost for the replica breaks down to:
</p>
<Table>
<TableHeader className="font-mono uppercase text-xs [&_th]:h-auto [&_th]:pb-2 [&_th]:pt-4">
<TableRow>
<TableHead className="w-[140px] pl-0">Item</TableHead>
<TableHead>Description</TableHead>
<TableHead className="text-right pr-0">Cost (/month)</TableHead>
</TableRow>
</TableHeader>
<TableBody className="[&_td]:py-0 [&_tr]:h-[50px] [&_tr]:border-dotted">
<TableRow>
<TableCell className="pl-0">Compute size</TableCell>
<TableCell>{compute.label}</TableCell>
<TableCell className="text-right font-mono pr-0" translate="no">
{compute.cost}
</TableCell>
</TableRow>
<TableRow>
<TableCell className="pl-0">Disk size</TableCell>
<TableCell>{disk.label}</TableCell>
<TableCell className="text-right font-mono pr-0" translate="no">
{disk.cost}
</TableCell>
</TableRow>
<TableRow>
<TableCell className="pl-0">IOPS</TableCell>
<TableCell>{iops.label}</TableCell>
<TableCell className="text-right font-mono pr-0" translate="no">
{iops.cost}
</TableCell>
</TableRow>
{disk.type === 'gp3' && (
<Card className="mt-2">
<Table>
<TableHeader>
<TableRow>
<TableCell className="pl-0">Throughput</TableCell>
<TableCell>{throughput.label}</TableCell>
<TableCell className="text-right font-mono pr-0">{throughput.cost}</TableCell>
<TableHead className="w-[140px]">Item</TableHead>
<TableHead>Description</TableHead>
<TableHead className="text-right">Cost (/month)</TableHead>
</TableRow>
)}
</TableBody>
</Table>
</TableHeader>
<TableBody className="[&_td]:py-0 [&_tr]:h-[50px] [&_tr]:border-dotted">
<TableRow>
<TableCell>Compute size</TableCell>
<TableCell>{compute.label}</TableCell>
<TableCell className="text-right font-mono" translate="no">
{compute.cost}
</TableCell>
</TableRow>
<TableRow>
<TableCell>Disk size</TableCell>
<TableCell>{disk.label}</TableCell>
<TableCell className="text-right font-mono" translate="no">
{disk.cost}
</TableCell>
</TableRow>
<TableRow>
<TableCell>IOPS</TableCell>
<TableCell>{iops.label}</TableCell>
<TableCell className="text-right font-mono" translate="no">
{iops.cost}
</TableCell>
</TableRow>
{disk.type === 'gp3' && (
<TableRow>
<TableCell>Throughput</TableCell>
<TableCell>{throughput.label}</TableCell>
<TableCell className="text-right font-mono">{throughput.cost}</TableCell>
</TableRow>
)}
</TableBody>
</Table>
</Card>
</>
) : (
<p className="text-foreground-light text-sm">
@@ -167,11 +167,15 @@ export const DestinationRow = ({ destinationId }: DestinationRowProps) => {
{isPipelineLoading ? (
<ShimmeringLoader />
) : (
<div>
<p>
{type} (Pipeline ID: {pipeline?.id})
<div className="flex flex-col gap-y-0.5">
<p className="text-sm font-medium text-foreground truncate">
{destinationName || type}
</p>
<p className="text-foreground-lighter">{destinationName}</p>
<div className="flex items-center gap-x-1.5 text-xs text-foreground-lighter">
<span className="font-mono">#{pipeline?.id}</span>
<span aria-hidden>&middot;</span>
<span>{type}</span>
</div>
</div>
)}
</TableCell>
@@ -1,6 +1,7 @@
import { useQueryClient } from '@tanstack/react-query'
import { useParams } from 'common'
import { MoreVertical, Plus, Search, X } from 'lucide-react'
import { ChartArea, MoreVertical, Plus, Search, X } from 'lucide-react'
import Link from 'next/link'
import { parseAsStringEnum, useQueryState } from 'nuqs'
import { useEffect, useMemo, useRef, useState } from 'react'
import {
@@ -46,6 +47,7 @@ import { useReplicationPipelinesQuery } from '@/data/replication/pipelines-query
import { useReplicationSourcesQuery } from '@/data/replication/sources-query'
import { checkLocalETLNotSetUp } from '@/data/replication/utils'
import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled'
import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganization'
import { DOCS_URL } from '@/lib/constants'
import { SHORTCUT_IDS } from '@/state/shortcuts/registry'
import { useShortcut } from '@/state/shortcuts/useShortcut'
@@ -53,6 +55,7 @@ import { useShortcut } from '@/state/shortcuts/useShortcut'
export const Destinations = () => {
const queryClient = useQueryClient()
const { ref: projectRef } = useParams()
const { data: organization } = useSelectedOrganizationQuery()
const etlEnableBigQuery = useIsETLBigQueryPrivateAlpha()
const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
@@ -232,6 +235,7 @@ export const Destinations = () => {
actions={
filterString.length > 0 && (
<Button
aria-label="Clear filter"
variant="text"
icon={<X />}
className="p-0 h-5 w-5"
@@ -258,11 +262,21 @@ export const Destinations = () => {
Add destination
</Button>
</Shortcut>
{organization?.slug && (
<Button asChild variant="default" icon={<ChartArea />}>
<Link href={`/org/${organization.slug}/usage#pipeline-backfill-data`}>Usage</Link>
</Button>
)}
<DocsButton href={`${DOCS_URL}/guides/database/replication`} />
{canDisablePipelines && (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="default" icon={<MoreVertical />} className="w-7" />
<Button
aria-label="More actions"
variant="default"
icon={<MoreVertical />}
className="w-7"
/>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-52">
<DropdownMenuItem onClick={() => setShowDisablePipelinesDialog(true)}>
@@ -418,7 +418,12 @@ export const USAGE_CATEGORIES: (subscription?: OrgSubscription) => CategoryMeta[
description:
'Total amount of backfill/initial copy data processed across all projects.\nBilling is based on the total amount of backfill/initial copy data processed in Gigabyte.',
chartDescription: 'The data refreshes every hour.',
links: [],
links: [
{
name: 'Pipelines',
url: `${DOCS_URL}/guides/database/replication/pipelines`,
},
],
},
{
anchor: 'pipeline-replicated-data',
@@ -429,7 +434,12 @@ export const USAGE_CATEGORIES: (subscription?: OrgSubscription) => CategoryMeta[
description:
'Total amount of replicated data processed across all projects.\nBilling is based on the total amount of replicated data processed in Gigabyte.',
chartDescription: 'The data refreshes every hour.',
links: [],
links: [
{
name: 'Pipelines',
url: `${DOCS_URL}/guides/database/replication/pipelines`,
},
],
},
],
},
@@ -1,7 +1,6 @@
import MotionNumber from '@number-flow/react'
import Link from 'next/link'
import { useMemo } from 'react'
import { Alert, AlertDescription, AlertTitle, Button, CriticalIcon } from 'ui'
import { Alert, AlertDescription, AlertTitle, Button, cn, CriticalIcon } from 'ui'
import { InfoTooltip } from 'ui-patterns/info-tooltip'
import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader'
@@ -119,8 +118,8 @@ export const DiskUsage = ({
<p className="text-xs">8 GB GP3 disk per project</p>
</div>
<div className="flex items-center justify-between">
<p className="text-xs text-foreground-light">Overages in period</p>
<div className="flex items-center justify-between py-1">
<p className="text-xs text-foreground-light">Overage in period</p>
<p className="text-xs">
{(gp3UsageInPeriod?.usage ?? 0).toLocaleString()} GP3 GB-Hrs
{io2UsageInPeriod?.usage
@@ -153,65 +152,62 @@ export const DiskUsage = ({
</Panel>
)}
{relevantProjects.map((project, idx) => {
const primaryDiskUsage = project.databases
.filter((it) => it.type === 'PRIMARY')
.reduce((acc, curr) => acc + (curr.disk_volume_size_gb ?? 8), 0)
const replicaDbs = project.databases.filter((it) => it.type !== 'PRIMARY')
const replicaDiskUsage = replicaDbs.reduce(
(acc, curr) => acc + (curr.disk_volume_size_gb ?? 8),
0
)
<div className="flex flex-col gap-y-2">
{relevantProjects.map((project, idx) => {
const primaryDiskUsage = project.databases
.filter((it) => it.type === 'PRIMARY')
.reduce((acc, curr) => acc + (curr.disk_volume_size_gb ?? 8), 0)
const replicaDbs = project.databases.filter((it) => it.type !== 'PRIMARY')
const replicaDiskUsage = replicaDbs.reduce(
(acc, curr) => acc + (curr.disk_volume_size_gb ?? 8),
0
)
const totalDiskUsage = primaryDiskUsage + replicaDiskUsage
const totalDiskUsage = primaryDiskUsage + replicaDiskUsage
return (
<div
key={`usage-project-${project.ref}`}
className={idx !== relevantProjects.length - 1 ? 'border-b pb-2' : ''}
>
<div className="flex justify-between">
<span className="text-foreground-light flex items-center gap-2">
{project.name}
</span>
<Button asChild variant="default" size={'tiny'}>
return (
<div
key={`usage-project-${project.ref}`}
className={cn(
'flex items-center justify-between',
idx !== relevantProjects.length - 1 && 'border-b pb-2'
)}
>
<div>
<p className="text-foreground-light text-sm flex items-center gap-2">
{project.name}
</p>
<div className="flex items-center gap-x-2">
<span className="text-foreground-lighter text-sm flex items-center gap-2">
{totalDiskUsage} GB Disk provisioned
</span>
<InfoTooltip side="right">
<p>{primaryDiskUsage} GB for Primary Database</p>
{replicaDbs.length > 0 && (
<>
<p>
{replicaDiskUsage} GB for {replicaDbs.length} Read{' '}
{replicaDbs.length === 1 ? 'Replica' : 'Replicas'}
</p>
<p className="mt-1">
Read replicas have their own disk and use 25% more disk to
account for WAL files.
</p>
</>
)}
</InfoTooltip>
</div>
</div>
<Button asChild variant="default" size="tiny">
<Link href={`/project/${project.ref}/settings/compute-and-disk`}>
Manage Disk
</Link>
</Button>
</div>
<div className="flex flex-col gap-2">
<div className="flex items-center h-6 gap-3">
<span className="text-foreground-light text-sm font-mono flex items-center gap-2">
<span className="text-foreground font-semibold mt-[-2px]">
<MotionNumber
value={totalDiskUsage}
style={{ lineHeight: 0.8 }}
className="font-mono"
/>
</span>{' '}
GB Disk provisioned
</span>
<InfoTooltip side="top">
<p>{primaryDiskUsage} GB for Primary Database</p>
{replicaDbs.length > 0 && (
<>
<p>
{replicaDiskUsage} GB for {replicaDbs.length} Read{' '}
{replicaDbs.length === 1 ? 'Replica' : 'Replicas'}
</p>
<p className="mt-1">
Read replicas have their own disk and use 25% more disk to account
for WAL files.
</p>
</>
)}
</InfoTooltip>
</div>
</div>
</div>
)
})}
)
})}
</div>
</div>
) : (
<Panel>
@@ -0,0 +1,57 @@
import { useQuery } from '@tanstack/react-query'
import { replicationKeys } from './keys'
import { get, handleError } from '@/data/fetchers'
import type { ResponseError, UseCustomQueryOptions } from '@/types'
type ReplicationCostEstimateParams = {
projectRef?: string
sourceId?: number
publicationName?: string
}
async function fetchReplicationCostEstimate(
{ projectRef, sourceId, publicationName }: ReplicationCostEstimateParams,
signal?: AbortSignal
) {
if (!projectRef) throw new Error('projectRef is required')
if (!sourceId) throw new Error('sourceId is required')
if (!publicationName) throw new Error('publicationName is required')
const { data, error } = await get(
'/platform/replication/{ref}/sources/{source_id}/publications/{publication_name}/cost-estimate',
{
params: {
path: { ref: projectRef, source_id: sourceId, publication_name: publicationName },
},
signal,
}
)
if (error) {
handleError(error)
}
return data
}
export type ReplicationCostEstimateData = Awaited<ReturnType<typeof fetchReplicationCostEstimate>>
export const useReplicationCostEstimateQuery = <TData = ReplicationCostEstimateData>(
{ projectRef, sourceId, publicationName }: ReplicationCostEstimateParams,
{
enabled = true,
...options
}: UseCustomQueryOptions<ReplicationCostEstimateData, ResponseError, TData> = {}
) =>
useQuery<ReplicationCostEstimateData, ResponseError, TData>({
queryKey: replicationKeys.costEstimate(projectRef, sourceId, publicationName),
queryFn: ({ signal }) =>
fetchReplicationCostEstimate({ projectRef, sourceId, publicationName }, signal),
enabled:
enabled &&
typeof projectRef !== 'undefined' &&
typeof sourceId !== 'undefined' &&
typeof publicationName !== 'undefined' &&
publicationName.length > 0,
...options,
})
+14
View File
@@ -8,6 +8,20 @@ export const replicationKeys = {
['projects', projectRef, 'sources', source_id, 'publications'] as const,
tables: (projectRef: string | undefined, source_id: number | undefined) =>
['projects', projectRef, 'sources', source_id, 'tables'] as const,
costEstimate: (
projectRef: string | undefined,
source_id: number | undefined,
publicationName: string | undefined
) =>
[
'projects',
projectRef,
'sources',
source_id,
'publications',
publicationName,
'cost-estimate',
] as const,
pipelines: (projectRef: string | undefined) => ['projects', projectRef, 'pipelines'] as const,
pipelineById: (projectRef: string | undefined, pipelineId: number | undefined) =>
['projects', projectRef, 'pipelines', pipelineId] as const,
+150
View File
@@ -4075,6 +4075,26 @@ export interface paths {
patch?: never
trace?: never
}
'/platform/replication/{ref}/sources/{source_id}/publications/{publication_name}/cost-estimate': {
parameters: {
query?: never
header?: never
path?: never
cookie?: never
}
/**
* Estimate replication cost for a publication
* @description Estimate the cost of replicating a publication's tables. Returns the flat per-pipeline fee, a per-table breakdown of the one-time initial-copy cost derived from the tables’ on-disk size, and the usage-based streaming rate. Requires bearer auth and an active, healthy project.
*/
get: operations['ReplicationSourcesController_getCostEstimate']
put?: never
post?: never
delete?: never
options?: never
head?: never
patch?: never
trace?: never
}
'/platform/replication/{ref}/sources/{source_id}/tables': {
parameters: {
query?: never
@@ -10879,6 +10899,81 @@ export interface components {
name: string
}
}
CostEstimateResponse: {
/**
* @description Currency of all amounts
* @example usd
* @enum {string}
*/
currency: 'usd'
/** @description Recurring per-pipeline cost */
pipeline: {
/**
* @description Hourly rate charged per active pipeline
* @example 0.053
*/
hourly_cost: number
/**
* @description Projected monthly cost for an active pipeline, based on an average 730-hour month. Pipelines are billed hourly, so this is an estimate, not a metered amount.
* @example 38.69
*/
monthly_cost: number
}
/** @description Usage-based streaming cost, expressed as a rate */
streaming: {
/**
* @description Usage-based streaming rate per GB. Actual cost depends on the change volume.
* @example 3
*/
rate_per_gb: number
}
/** @description One-time cost for the initial table copy */
table_copy: {
/**
* @description One-time initial-copy rate per GB
* @example 0.6
*/
rate_per_gb: number
/** @description Per-table initial-copy cost estimate */
tables: {
/**
* @description Estimated on-disk size of the table in bytes
* @example 10960896
*/
estimated_bytes: number
/**
* @description Estimated one-time initial-copy cost for the table, in the response currency
* @example 0.01
*/
estimated_cost: number
/**
* @description Whether this table has a row filter. The estimate does not account for how many rows the filter excludes, so the actual replicated volume may be lower than shown.
* @example false
*/
is_row_filtered: boolean
/**
* @description Table name
* @example orders
*/
name: string
/**
* @description Table schema
* @example public
*/
schema: string
}[]
/**
* @description Total estimated bytes across all tables
* @example 11911168
*/
total_bytes: number
/**
* @description Total estimated one-time initial-copy cost
* @example 0.01
*/
total_cost: number
}
}
ReplicationPublicationsResponse: {
/** @description List of publications */
publications: {
@@ -28217,6 +28312,61 @@ export interface operations {
}
}
}
ReplicationSourcesController_getCostEstimate: {
parameters: {
query?: never
header?: never
path: {
/** @description Publication name */
publication_name: string
/** @description Project ref */
ref: string
/** @description Source id */
source_id: number
}
cookie?: never
}
requestBody?: never
responses: {
/** @description Cost estimate for replicating the publication. */
200: {
headers: {
[name: string]: unknown
}
content: {
'application/json': components['schemas']['CostEstimateResponse']
}
}
/** @description Unauthorized */
401: {
headers: {
[name: string]: unknown
}
content?: never
}
/** @description Forbidden action */
403: {
headers: {
[name: string]: unknown
}
content?: never
}
/** @description Rate limit exceeded */
429: {
headers: {
[name: string]: unknown
}
content?: never
}
/** @description Unexpected error while estimating replication cost. */
500: {
headers: {
[name: string]: unknown
}
content?: never
}
}
}
ReplicationSourcesController_createPublication: {
parameters: {
query?: never