From 94bfd454bee6520ea7bda169c74ea2d5caa56016 Mon Sep 17 00:00:00 2001 From: Benjamin <5719034+bnjjj@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:31:53 +0200 Subject: [PATCH] feat(studio): wire warehouse UI to real API hooks Replace the prototype's local valtio demo store (warehouseDemoStore) with React Query hooks in data/warehouse/ that call new /platform/warehouse/{ref}/... endpoints (designed in data/warehouse/WAREHOUSE_API_PLAN.md; they 404 until the backend ships): - useWarehouseTablesQuery / useWarehouseTableState(s): per-table storage state - useWarehouseLinkTableMutation: "Copy to Warehouse" - useWarehouseDetachTableMutation: "Detach Warehouse copy" - useWarehouseCatalogQuery / useUpdateWarehouseCatalogMutation: catalog access + creds Rewires the storage panel (now keyed by schema/name), table list, table-detail layout/overview/settings, the Warehouse Catalog integration cards, and the Connect-sheet catalog panel. Deletes warehouseDemoStore and the unused WarehouseModeChip / WarehouseStorageDetailsDialog. Also fixes latent type errors carried in by the prototype (dangling imports; adds 'table_settings' to the index-advisor telemetry origin union). Warehouse Catalog credentials/snippets remain Iceberg-REST-shaped pending the DuckLake external-access design (see WAREHOUSE_API_PLAN.md). --- .../interfaces/ConnectSheet/ConnectSheet.tsx | 2 +- .../ConnectSheet/WarehouseCatalogPanel.tsx | 11 +- .../Tables/TableDetailOverviewMetrics.tsx | 11 +- .../Tables/TableDetailSettingsTab.tsx | 13 +- .../interfaces/Database/Tables/TableList.tsx | 22 +- .../Warehouse/WarehouseDetachModal.tsx | 44 ++-- .../Warehouse/WarehouseEnablementModal.tsx | 51 ++--- .../Database/Warehouse/WarehouseModeChip.tsx | 30 --- .../Warehouse/WarehouseProgressSteps.tsx | 16 +- .../WarehouseStorageDetailsDialog.tsx | 128 ----------- .../Database/Warehouse/WarehouseSyncChip.tsx | 2 +- .../Warehouse/WarehouseTableStoragePanel.tsx | 25 +-- .../Database/Warehouse/warehouseDemoStore.ts | 71 ------ .../WarehouseCatalogCards.tsx | 47 ++-- .../WarehouseCatalogCredentials.tsx | 41 ++-- .../warehouseCatalog.constants.ts | 22 +- .../DatabaseLayout/TableDetailLayout.tsx | 13 +- .../data/warehouse/WAREHOUSE_API_PLAN.md | 208 ++++++++++++++++++ .../data/warehouse/detach-table-mutation.ts | 72 ++++++ apps/studio/data/warehouse/keys.ts | 6 + .../data/warehouse/link-table-mutation.ts | 64 ++++++ .../warehouse/warehouse-catalog-mutation.ts | 65 ++++++ .../data/warehouse/warehouse-catalog-query.ts | 68 ++++++ .../data/warehouse/warehouse-tables-query.ts | 73 ++++++ apps/studio/data/warehouse/warehouse-types.ts | 74 +++++++ packages/common/telemetry-constants.ts | 2 +- 26 files changed, 821 insertions(+), 360 deletions(-) delete mode 100644 apps/studio/components/interfaces/Database/Warehouse/WarehouseModeChip.tsx delete mode 100644 apps/studio/components/interfaces/Database/Warehouse/WarehouseStorageDetailsDialog.tsx delete mode 100644 apps/studio/components/interfaces/Database/Warehouse/warehouseDemoStore.ts create mode 100644 apps/studio/data/warehouse/WAREHOUSE_API_PLAN.md create mode 100644 apps/studio/data/warehouse/detach-table-mutation.ts create mode 100644 apps/studio/data/warehouse/keys.ts create mode 100644 apps/studio/data/warehouse/link-table-mutation.ts create mode 100644 apps/studio/data/warehouse/warehouse-catalog-mutation.ts create mode 100644 apps/studio/data/warehouse/warehouse-catalog-query.ts create mode 100644 apps/studio/data/warehouse/warehouse-tables-query.ts create mode 100644 apps/studio/data/warehouse/warehouse-types.ts diff --git a/apps/studio/components/interfaces/ConnectSheet/ConnectSheet.tsx b/apps/studio/components/interfaces/ConnectSheet/ConnectSheet.tsx index 77506753a97..c833f5ce60c 100644 --- a/apps/studio/components/interfaces/ConnectSheet/ConnectSheet.tsx +++ b/apps/studio/components/interfaces/ConnectSheet/ConnectSheet.tsx @@ -2,7 +2,7 @@ import { PermissionAction } from '@supabase/shared-types/out/constants' import { useParams } from 'common' import { parseAsBoolean, useQueryState } from 'nuqs' import { useEffect, useMemo, useRef } from 'react' -import { cn, Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from 'ui' +import { Sheet, SheetContent, SheetHeader, SheetTitle } from 'ui' import type { ConnectMode, ProjectKeys } from './Connect.types' import { ConnectConfigSection, ModeSelector } from './ConnectConfigSection' diff --git a/apps/studio/components/interfaces/ConnectSheet/WarehouseCatalogPanel.tsx b/apps/studio/components/interfaces/ConnectSheet/WarehouseCatalogPanel.tsx index 09a1e44488d..658c0f76ccc 100644 --- a/apps/studio/components/interfaces/ConnectSheet/WarehouseCatalogPanel.tsx +++ b/apps/studio/components/interfaces/ConnectSheet/WarehouseCatalogPanel.tsx @@ -2,11 +2,10 @@ import { useParams } from 'common' import Link from 'next/link' import { Button } from 'ui' import { Admonition } from 'ui-patterns' -import { useSnapshot } from 'valtio' import type { WarehouseCatalogEngine } from '@/components/interfaces/Integrations/WarehouseCatalog/warehouseCatalog.constants' import { WarehouseCatalogCredentials } from '@/components/interfaces/Integrations/WarehouseCatalog/WarehouseCatalogCredentials' -import { warehouseDemoStore } from '@/components/interfaces/Database/Warehouse/warehouseDemoStore' +import { useWarehouseCatalogQuery } from '@/data/warehouse/warehouse-catalog-query' interface WarehouseCatalogPanelProps { queryEngine?: WarehouseCatalogEngine @@ -14,7 +13,8 @@ interface WarehouseCatalogPanelProps { export function WarehouseCatalogPanel({ queryEngine = 'env' }: WarehouseCatalogPanelProps) { const { ref: projectRef } = useParams() - const { catalogEnabled } = useSnapshot(warehouseDemoStore) + const { data: catalog } = useWarehouseCatalogQuery({ projectRef }) + const catalogEnabled = catalog?.enabled ?? false return (
@@ -34,7 +34,10 @@ export function WarehouseCatalogPanel({ queryEngine = 'env' }: WarehouseCatalogP ]} /> ) : ( - + )}
diff --git a/apps/studio/components/interfaces/Database/Tables/TableDetailOverviewMetrics.tsx b/apps/studio/components/interfaces/Database/Tables/TableDetailOverviewMetrics.tsx index 5cfc255b36c..9570abbf564 100644 --- a/apps/studio/components/interfaces/Database/Tables/TableDetailOverviewMetrics.tsx +++ b/apps/studio/components/interfaces/Database/Tables/TableDetailOverviewMetrics.tsx @@ -3,7 +3,6 @@ import { useMemo } from 'react' import type { ChartConfig } from 'ui' import { Card, CardContent } from 'ui' import { Chart, ChartContent, ChartLine, ChartMetric, type ChartLineTick } from 'ui-patterns/Chart' -import { useSnapshot } from 'valtio' import { buildTableOverviewSparklineData, @@ -12,11 +11,9 @@ import { TABLE_ROW_COUNT_CHART_CONFIG, TABLE_SIZE_CHART_CONFIG, } from './TableDetailOverview.utils' -import { - getWarehouseStorageSummaryLabel, - warehouseDemoStore, -} from '@/components/interfaces/Database/Warehouse/warehouseDemoStore' import type { TableLike } from '@/data/table-editor/table-editor-types' +import { useWarehouseTableStates } from '@/data/warehouse/warehouse-tables-query' +import { getWarehouseStorageSummaryLabel } from '@/data/warehouse/warehouse-types' import { formatBytes } from '@/lib/helpers' const DATE_TIME_FORMAT = 'MMM D' @@ -69,9 +66,9 @@ interface TableDetailOverviewMetricsProps { } export function TableDetailOverviewMetrics({ table }: TableDetailOverviewMetricsProps) { - const warehouseSnap = useSnapshot(warehouseDemoStore) + const warehouseStates = useWarehouseTableStates() const tableKey = `${table.schema}.${table.name}` - const warehouseState = warehouseSnap.tables[tableKey] + const warehouseState = warehouseStates.get(tableKey) const rowCount = table.live_rows_estimate ?? 0 const columnCount = table.columns?.length ?? 0 diff --git a/apps/studio/components/interfaces/Database/Tables/TableDetailSettingsTab.tsx b/apps/studio/components/interfaces/Database/Tables/TableDetailSettingsTab.tsx index a3fbfc3b3ef..98a23d44e61 100644 --- a/apps/studio/components/interfaces/Database/Tables/TableDetailSettingsTab.tsx +++ b/apps/studio/components/interfaces/Database/Tables/TableDetailSettingsTab.tsx @@ -10,9 +10,9 @@ import { PageSectionTitle, } from 'ui-patterns/PageSection' -import { TableDetailGeneralSettingsCard } from '@/components/interfaces/Database/Tables/TableDetailGeneralSettingsCard' import { TableDetailDataApiSection } from '@/components/interfaces/Database/Tables/TableDetailDataApiSection' import { TableDetailDeleteTableSection } from '@/components/interfaces/Database/Tables/TableDetailDeleteTableSection' +import { TableDetailGeneralSettingsCard } from '@/components/interfaces/Database/Tables/TableDetailGeneralSettingsCard' import { TableDetailIndexAdvisorSection } from '@/components/interfaces/Database/Tables/TableDetailIndexAdvisorSection' import { WarehouseTableStoragePanel } from '@/components/interfaces/Database/Warehouse/WarehouseTableStoragePanel' import { RealtimeToggleDialog } from '@/components/interfaces/TableGridEditor/RealtimeToggleDialog' @@ -30,7 +30,6 @@ export function TableDetailSettingsTab({ table }: TableDetailSettingsTabProps) { const { realtimeAll: isRealtimeFeatureEnabled } = useIsFeatureEnabled(['realtime:all']) const isRealtimeEnabled = useIsTableRealtimeEnabled({ id: table.id }) const showStorage = table.entity_type === ENTITY_TYPE.TABLE - const tableKey = `${table.schema}.${table.name}` return ( <> @@ -54,7 +53,11 @@ export function TableDetailSettingsTab({ table }: TableDetailSettingsTabProps) { - + )} @@ -77,7 +80,9 @@ export function TableDetailSettingsTab({ table }: TableDetailSettingsTabProps) { > @@ -61,12 +70,13 @@ export const WarehouseCatalogEnableCard = () => { * Catalog credentials card. Only rendered once the catalog is enabled. */ export const WarehouseCatalogCredentialsCard = () => { - const { catalogEnabled } = useSnapshot(warehouseDemoStore) + const { ref: projectRef } = useParams() + const { data: catalog } = useWarehouseCatalogQuery({ projectRef }) const [queryEngine, setQueryEngine] = useState( WAREHOUSE_CATALOG_ENGINES[0]?.key ?? 'env' ) - if (!catalogEnabled) return null + if (!catalog?.enabled) return null return ( @@ -80,6 +90,7 @@ export const WarehouseCatalogCredentialsCard = () => { diff --git a/apps/studio/components/interfaces/Integrations/WarehouseCatalog/WarehouseCatalogCredentials.tsx b/apps/studio/components/interfaces/Integrations/WarehouseCatalog/WarehouseCatalogCredentials.tsx index 74bba874676..456806d0b2a 100644 --- a/apps/studio/components/interfaces/Integrations/WarehouseCatalog/WarehouseCatalogCredentials.tsx +++ b/apps/studio/components/interfaces/Integrations/WarehouseCatalog/WarehouseCatalogCredentials.tsx @@ -2,29 +2,42 @@ import { useCallback } from 'react' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from 'ui' import { CodeBlock } from 'ui-patterns/CodeBlock' import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' +import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader' import { buildWarehouseCatalogEnv, getWarehouseCatalogEngineContent, - WAREHOUSE_CATALOG_CREDENTIALS, WAREHOUSE_CATALOG_ENGINES, WAREHOUSE_CATALOG_ENV_VARS, type WarehouseCatalogEngine, } from './warehouseCatalog.constants' import { EnvRow } from '@/components/interfaces/ConnectSheet/content/server/common/EnvRow' import CopyButton from '@/components/ui/CopyButton' +import type { WarehouseCatalogCredentials as CatalogCredentials } from '@/data/warehouse/warehouse-catalog-query' interface WarehouseCatalogCredentialsProps { queryEngine: WarehouseCatalogEngine onQueryEngineChange?: (engine: WarehouseCatalogEngine) => void + credentials?: CatalogCredentials +} + +function maskToken(token: string): string { + return `${token.slice(0, 4)}${'•'.repeat(16)}` } export function WarehouseCatalogCredentials({ queryEngine, onQueryEngineChange, + credentials, }: WarehouseCatalogCredentialsProps) { - const buildCatalogEnv = useCallback(() => buildWarehouseCatalogEnv(), []) - const engineContent = getWarehouseCatalogEngineContent(queryEngine) + const buildCatalogEnv = useCallback( + () => (credentials ? buildWarehouseCatalogEnv(credentials) : ''), + [credentials] + ) + + const engineContent = credentials + ? getWarehouseCatalogEngineContent(queryEngine, credentials) + : undefined return (
@@ -51,7 +64,12 @@ export function WarehouseCatalogCredentials({ )} - {queryEngine === 'env' ? ( + {!credentials || !engineContent ? ( +
+ + +
+ ) : queryEngine === 'env' ? (
catalog.env @@ -63,40 +81,37 @@ export function WarehouseCatalogCredentials({ />
- +
diff --git a/apps/studio/components/interfaces/Integrations/WarehouseCatalog/warehouseCatalog.constants.ts b/apps/studio/components/interfaces/Integrations/WarehouseCatalog/warehouseCatalog.constants.ts index 16938a4b73c..9729d929907 100644 --- a/apps/studio/components/interfaces/Integrations/WarehouseCatalog/warehouseCatalog.constants.ts +++ b/apps/studio/components/interfaces/Integrations/WarehouseCatalog/warehouseCatalog.constants.ts @@ -1,3 +1,5 @@ +import type { WarehouseCatalogCredentials } from '@/data/warehouse/warehouse-catalog-query' + export type WarehouseCatalogEngine = 'env' | 'duckdb' | 'spark' | 'trino' | 'pyiceberg' export const WAREHOUSE_CATALOG_ENGINES: { key: WarehouseCatalogEngine; label: string }[] = [ @@ -8,20 +10,13 @@ export const WAREHOUSE_CATALOG_ENGINES: { key: WarehouseCatalogEngine; label: st { key: 'pyiceberg', label: 'PyIceberg' }, ] -export const WAREHOUSE_CATALOG_CREDENTIALS = { - catalogUri: 'https://catalog.supabase.example', - accessToken: 'sbw_demo_token_abc123xyz', - accessTokenMasked: 'sbw_••••••••••••••••', - warehouseId: 'wh_demo123', -} as const - export const WAREHOUSE_CATALOG_ENV_VARS = { catalogUri: 'CATALOG_URI', accessToken: 'ACCESS_TOKEN', warehouseIdentifier: 'WAREHOUSE_IDENTIFIER', } as const -export function buildWarehouseCatalogEnv(creds = WAREHOUSE_CATALOG_CREDENTIALS): string { +export function buildWarehouseCatalogEnv(creds: WarehouseCatalogCredentials): string { return [ `${WAREHOUSE_CATALOG_ENV_VARS.catalogUri}=${creds.catalogUri}`, `${WAREHOUSE_CATALOG_ENV_VARS.accessToken}=${creds.accessToken}`, @@ -29,9 +24,18 @@ export function buildWarehouseCatalogEnv(creds = WAREHOUSE_CATALOG_CREDENTIALS): ].join('\n') } +/** + * Per-engine connection snippets for the Warehouse catalog. + * + * NOTE: these templates are currently written against an Iceberg REST catalog shape + * (`uri` / `token` / `warehouse`). The Warehouse product is backed by DuckLake, whose external + * access model differs (a Postgres catalog connection + object storage). The credential triple + * and these snippets should be revisited with the DuckLake catalog design before GA — see the + * warehouse endpoints plan. + */ export function getWarehouseCatalogEngineContent( engine: WarehouseCatalogEngine, - creds = WAREHOUSE_CATALOG_CREDENTIALS + creds: WarehouseCatalogCredentials ): { headerLabel: string; language: 'sql' | 'toml' | 'python'; value: string } { const { catalogUri, accessToken, warehouseId } = creds diff --git a/apps/studio/components/layouts/DatabaseLayout/TableDetailLayout.tsx b/apps/studio/components/layouts/DatabaseLayout/TableDetailLayout.tsx index 34bf145aaf8..bffa210980a 100644 --- a/apps/studio/components/layouts/DatabaseLayout/TableDetailLayout.tsx +++ b/apps/studio/components/layouts/DatabaseLayout/TableDetailLayout.tsx @@ -4,7 +4,7 @@ import { TableEditor } from 'icons' import { Check, Edit, X } from 'lucide-react' import Link from 'next/link' import { useRouter } from 'next/router' -import React, { type PropsWithChildren } from 'react' +import { type PropsWithChildren } from 'react' import { BreadcrumbItem, BreadcrumbLink, @@ -26,14 +26,9 @@ import { PageHeaderTitle, } from 'ui-patterns/PageHeader' import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader' -import { useSnapshot } from 'valtio' import DatabaseLayout from './DatabaseLayout' import { buildTableEditorUrl } from '@/components/grid/SupabaseGrid.utils' -import { - getWarehouseStorageSummaryLabel, - warehouseDemoStore, -} from '@/components/interfaces/Database/Warehouse/warehouseDemoStore' import { WarehouseSyncChip } from '@/components/interfaces/Database/Warehouse/WarehouseSyncChip' import DeleteConfirmationDialogs from '@/components/interfaces/TableGridEditor/DeleteConfirmationDialogs' import { SidePanelEditor } from '@/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor' @@ -41,6 +36,8 @@ import { EntityTypeIcon } from '@/components/ui/EntityTypeIcon' import { useDatabasePublicationsQuery } from '@/data/database-publications/database-publications-query' import { useTableEditorQuery } from '@/data/table-editor/table-editor-query' import { isTableLike } from '@/data/table-editor/table-editor-types' +import { useWarehouseTableStates } from '@/data/warehouse/warehouse-tables-query' +import { getWarehouseStorageSummaryLabel } from '@/data/warehouse/warehouse-types' import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions' import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject' import { useTableEditorStateSnapshot } from '@/state/table-editor' @@ -79,12 +76,12 @@ export function TableDetailLayout({ selectedTable?.id !== undefined && (realtimePublication?.tables ?? []).some((table) => table.id === selectedTable.id) - const warehouseSnap = useSnapshot(warehouseDemoStore) + const warehouseStates = useWarehouseTableStates() const tableKey = selectedTable?.schema !== undefined && selectedTable?.name !== undefined ? `${selectedTable.schema}.${selectedTable.name}` : undefined - const warehouseState = tableKey ? warehouseSnap.tables[tableKey] : undefined + const warehouseState = tableKey ? warehouseStates.get(tableKey) : undefined const { can: canUpdateTables } = useAsyncCheckPermissions( PermissionAction.TENANT_SQL_ADMIN_WRITE, diff --git a/apps/studio/data/warehouse/WAREHOUSE_API_PLAN.md b/apps/studio/data/warehouse/WAREHOUSE_API_PLAN.md new file mode 100644 index 00000000000..eaf945eee9e --- /dev/null +++ b/apps/studio/data/warehouse/WAREHOUSE_API_PLAN.md @@ -0,0 +1,208 @@ +# Warehouse — required new platform endpoints + +This document specifies the **new platform API endpoints** the Studio Warehouse feature calls. None +of them exist yet — the Studio hooks in this folder (`data/warehouse/*`) call them and will 404 +until the backend ships. The goal here is to design the payloads and enumerate the work for the +platform team. **No platform code has been written.** + +The endpoints live in the existing platform API repo (NestJS) alongside replication, e.g. +`/Users/bnj/supabase/platform/api/apps/mgmt-api/src/routes/platform/`. + +## Architecture: Warehouse is a thin layer over Replication + DuckLake + +The Warehouse product reuses the existing replication/ETL machinery. Conceptually: + +- Each project has **at most one** "warehouse pipeline": a DuckLake destination plugged into the + project itself, plus a pipeline and a publication. Naming convention: + - pipeline/destination name: `supabase_warehouse_pipeline` + - publication name: `supabase_warehouse_publication` +- **"Copy to Warehouse"** for a table = ensure that pipeline exists (create it lazily on first use), + add the table to its publication, and ensure the pipeline is running. +- **"Detach Warehouse copy"** = remove the table from the publication (and drop its DuckLake copy). +- **Catalog access** = a project-level toggle that exposes catalog credentials so external query + engines can read the DuckLake tables without touching Postgres. + +The warehouse endpoints are a **convenience facade** so Studio never has to orchestrate +sources/destinations/pipelines/publications directly. Internally they call the existing replication +service (see `replication.client.ts`) and the DuckLake config resolver +(`ducklake-supabase-config.ts`). + +### Existing building blocks reused (already in the platform API) + +| Concern | Existing endpoint / module | +| ----------------------------- | ------------------------------------------------------------------------------------- | +| Create tenant + source | `POST /platform/replication/{ref}/tenants-sources` | +| Create destination + pipeline | `POST /platform/replication/{ref}/destinations-pipelines` | +| Start / stop pipeline | `POST /platform/replication/{ref}/pipelines/{id}/start` \| `/stop` | +| Pipeline status | `GET /platform/replication/{ref}/pipelines/{id}/status` | +| Per-table replication status | `GET /platform/replication/{ref}/pipelines/{id}/replication-status` | +| Create / update publication | `POST /platform/replication/{ref}/sources/{source_id}/publications[/{name}]` | +| DuckLake destination config | `ReplicationDucklakeConfigSchema` in `destinations.dto.ts` (Supabase-managed variant) | + +The DuckLake destination for the warehouse pipeline uses the **Supabase-managed** variant, pointed +at the project itself: + +```jsonc +{ + "ducklake": { + "catalog": { "type": "supabase_project", "project_ref": "" }, + "storage": { "type": "supabase_storage", "project_ref": "", "bucket": "warehouse" }, + }, +} +``` + +--- + +## New endpoints + +Base path: `/platform/warehouse/{ref}` (mirrors `/platform/replication/{ref}`). +Auth: `@AuthWithProjectRef(PermissionAction.REPLICATION_ADMIN_READ | WRITE, '*')` (or a dedicated +`WAREHOUSE_ADMIN_*` action). Gate writes behind the same entitlement as ETL +(`@ApiEntitlementRequired('replication.etl')` or a new `warehouse` entitlement). + +### 1. List linked tables — `GET /platform/warehouse/{ref}/tables` + +Returns every table that currently has a Warehouse copy, with live sync status. Drives the Tables +list chips, the table-detail header badge, and the Storage panel. + +Response `200`: + +```jsonc +{ + "tables": [ + { + "schema": "public", + "name": "orders", + "state": "live", // 'syncing' | 'live' | 'error' (derived from replication-status) + "lag_ms": 12000, // optional + "last_synced_at": "2026-06-23T17:48:00Z", // optional ISO-8601 + "copy_name": "warehouse.orders", + "warehouse_size_bytes": 197912092672, // optional + }, + ], +} +``` + +Server behavior: if no warehouse pipeline exists, return `{ "tables": [] }`. Otherwise read the +publication's table list and join with `GET .../pipelines/{id}/replication-status` to populate +`state` / `lag_ms` / `last_synced_at`. Tables not in the publication are simply absent (Studio +treats absent = Postgres-only). + +Studio hook: `useWarehouseTablesQuery` (`warehouse-tables-query.ts`). + +### 2. Link a table ("Copy to Warehouse") — `POST /platform/warehouse/{ref}/tables` + +Request: + +```jsonc +{ "schema": "public", "name": "orders" } +``` + +Response `202`: the new linked-table row (same shape as an item in endpoint #1). `state` will +usually be `"syncing"` initially. + +Server behavior (idempotent): + +1. Ensure the warehouse tenant + source exist (`tenants-sources`). +2. Ensure the `supabase_warehouse_pipeline` destination + pipeline exist. If not, create the + DuckLake destination (Supabase-managed, pointed at this project) + pipeline + + `supabase_warehouse_publication` via `destinations-pipelines`. +3. Add `{schema, name}` to `supabase_warehouse_publication` (create-or-update publication). +4. Ensure the pipeline is running (`/start` if stopped). + +Errors: `409` if the table can't be replicated (e.g. no primary key / replica identity), `402` if +the entitlement is missing. + +Studio hook: `useWarehouseLinkTableMutation` (`link-table-mutation.ts`). + +### 3. Detach a table ("Detach Warehouse copy") — `DELETE /platform/warehouse/{ref}/tables/{schema}/{name}` + +Response `204`. + +Server behavior: remove `{schema, name}` from `supabase_warehouse_publication` (update publication) +and drop the table's DuckLake copy. Leave the pipeline running for the remaining tables; if it was +the last table, the pipeline may be stopped (implementation detail). The Postgres source table is +untouched. + +Studio hook: `useWarehouseDetachTableMutation` (`detach-table-mutation.ts`). + +### 4. Get catalog access — `GET /platform/warehouse/{ref}/catalog` + +Returns whether external catalog access is enabled and, if so, the credentials external query +engines use. + +Response `200`: + +```jsonc +{ + "enabled": true, + "credentials": { + // present only when enabled + "catalog_uri": "https://catalog..supabase.co", + "access_token": "sbw_...", // secret; only returned to project admins + "warehouse_id": "wh_...", + }, +} +``` + +Studio hook: `useWarehouseCatalogQuery` (`warehouse-catalog-query.ts`). + +### 5. Enable / disable catalog access — `POST /platform/warehouse/{ref}/catalog` + +Request: + +```jsonc +{ "enabled": true } +``` + +Response `200`: same shape as endpoint #4 (on enable, includes freshly provisioned `credentials`). + +Server behavior: on enable, provision the catalog endpoint + access token for external readers; on +disable, revoke them. Access is managed independently from the project's database connection +settings. + +Studio hook: `useUpdateWarehouseCatalogMutation` (`warehouse-catalog-mutation.ts`). + +### (Optional, not yet called by Studio) + +- `POST /platform/warehouse/{ref}/catalog/token/rotate` → rotate the access token. +- `GET /platform/warehouse/{ref}` → project-level overview (`{ provisioned, pipeline_id, +pipeline_status, catalog: { enabled } }`) if a single status call is preferred over #1 + #4. + +--- + +## ⚠️ Open question: catalog credentials shape (DuckLake vs Iceberg REST) + +The credentials triple above (`catalog_uri` / `access_token` / `warehouse_id`) and the per-engine +snippets in `warehouseCatalog.constants.ts` are currently written for an **Iceberg REST catalog** +(the prototype mocked it that way). The Warehouse product is backed by **DuckLake**, whose external +access model is different — typically a **Postgres catalog connection** plus **object-storage +credentials** rather than a REST endpoint + token. + +Before GA, decide one of: + +1. Expose DuckLake directly → credentials become `{ catalog_postgres_url, storage_endpoint, +storage_access_key_id, storage_secret_access_key, ... }`, and the DuckDB/Spark/Trino/PyIceberg + snippets change to `ATTACH 'ducklake:postgres:...'`-style configs. +2. Front DuckLake with an Iceberg-REST-compatible catalog → keep the current triple + snippets. + +The Studio data layer isolates this: only `WarehouseCatalogCredentials` (`warehouse-catalog-query.ts`) +and `warehouseCatalog.constants.ts` need to change when the shape is finalized. + +--- + +## Hook → endpoint → underlying replication mapping + +| Studio hook | New warehouse endpoint | Underlying replication work | +| ----------------------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `useWarehouseTablesQuery` | `GET .../warehouse/{ref}/tables` | read publication tables + `replication-status` | +| `useWarehouseLinkTableMutation` | `POST .../warehouse/{ref}/tables` | ensure pipeline (`tenants-sources`, `destinations-pipelines`) + update publication + start | +| `useWarehouseDetachTableMutation` | `DELETE .../warehouse/{ref}/tables/{schema}/{name}` | update publication (remove table) | +| `useWarehouseCatalogQuery` | `GET .../warehouse/{ref}/catalog` | read catalog access + credentials | +| `useUpdateWarehouseCatalogMutation` | `POST .../warehouse/{ref}/catalog` | provision/revoke catalog credentials | + +## Excluded (deferred upstream) + +- **Move to Warehouse** (fully migrating a table off the Postgres heap) — already deferred to a + separate branch; no endpoint designed here. +- **Snapshots / time-travel** — only surfaced after a Move in the prototype; out of scope. diff --git a/apps/studio/data/warehouse/detach-table-mutation.ts b/apps/studio/data/warehouse/detach-table-mutation.ts new file mode 100644 index 00000000000..b71a6c90332 --- /dev/null +++ b/apps/studio/data/warehouse/detach-table-mutation.ts @@ -0,0 +1,72 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { toast } from 'sonner' + +import { warehouseKeys } from './keys' +import { constructHeaders, fetchHandler } from '@/data/fetchers' +import { API_URL } from '@/lib/constants' +import { ResponseError, type UseCustomMutationOptions } from '@/types' + +export type WarehouseDetachTableVariables = { + projectRef: string + schema: string + name: string +} + +/** + * Detaches a table's Warehouse copy ("Detach Warehouse copy"). Server-side this removes the table + * from the `supabase_warehouse_pipeline` publication and drops its DuckLake copy. The source table + * in Postgres is unaffected. + */ +export async function detachWarehouseTable({ + projectRef, + schema, + name, +}: WarehouseDetachTableVariables) { + if (!projectRef) throw new Error('projectRef is required') + + const headers = await constructHeaders({ 'Content-Type': 'application/json' }) + const response = await fetchHandler( + `${API_URL}/platform/warehouse/${projectRef}/tables/${encodeURIComponent(schema)}/${encodeURIComponent(name)}`, + { method: 'DELETE', headers, credentials: 'include' } + ) + + if (!response.ok) { + let message = `Failed to detach Warehouse copy (${response.status})` + try { + const body = await response.json() + message = body?.message ?? body?.msg ?? message + } catch { + // response had no JSON body; keep the default message + } + throw new ResponseError(message, response.status) + } +} + +type WarehouseDetachTableData = Awaited> + +export const useWarehouseDetachTableMutation = ({ + onSuccess, + onError, + ...options +}: Omit< + UseCustomMutationOptions, + 'mutationFn' +> = {}) => { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: detachWarehouseTable, + async onSuccess(data, variables, context) { + await queryClient.invalidateQueries({ queryKey: warehouseKeys.tables(variables.projectRef) }) + await onSuccess?.(data, variables, context) + }, + async onError(data, variables, context) { + if (onError === undefined) { + toast.error(`Failed to detach Warehouse copy: ${data.message}`) + } else { + onError(data, variables, context) + } + }, + ...options, + }) +} diff --git a/apps/studio/data/warehouse/keys.ts b/apps/studio/data/warehouse/keys.ts new file mode 100644 index 00000000000..c7458317771 --- /dev/null +++ b/apps/studio/data/warehouse/keys.ts @@ -0,0 +1,6 @@ +export const warehouseKeys = { + tables: (projectRef: string | undefined) => + ['projects', projectRef, 'warehouse', 'tables'] as const, + catalog: (projectRef: string | undefined) => + ['projects', projectRef, 'warehouse', 'catalog'] as const, +} diff --git a/apps/studio/data/warehouse/link-table-mutation.ts b/apps/studio/data/warehouse/link-table-mutation.ts new file mode 100644 index 00000000000..1435edd0558 --- /dev/null +++ b/apps/studio/data/warehouse/link-table-mutation.ts @@ -0,0 +1,64 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { toast } from 'sonner' + +import { warehouseKeys } from './keys' +import type { WarehouseLinkedTable } from './warehouse-types' +import { fetchPost } from '@/data/fetchers' +import { API_URL } from '@/lib/constants' +import { ResponseError, type UseCustomMutationOptions } from '@/types' + +export type WarehouseLinkTableVariables = { + projectRef: string + schema: string + name: string +} + +/** + * Links a table into Warehouse ("Copy to Warehouse"). Server-side this ensures the project's + * `supabase_warehouse_pipeline` (DuckLake destination) exists, adds the table to its publication, + * and starts the pipeline. Returns the new linked-table status. + */ +export async function linkWarehouseTable({ + projectRef, + schema, + name, +}: WarehouseLinkTableVariables) { + if (!projectRef) throw new Error('projectRef is required') + + const result = await fetchPost( + `${API_URL}/platform/warehouse/${projectRef}/tables`, + { schema, name }, + { credentials: 'include' } + ) + if (result instanceof ResponseError) throw result + return result +} + +type WarehouseLinkTableData = Awaited> + +export const useWarehouseLinkTableMutation = ({ + onSuccess, + onError, + ...options +}: Omit< + UseCustomMutationOptions, + 'mutationFn' +> = {}) => { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: linkWarehouseTable, + async onSuccess(data, variables, context) { + await queryClient.invalidateQueries({ queryKey: warehouseKeys.tables(variables.projectRef) }) + await onSuccess?.(data, variables, context) + }, + async onError(data, variables, context) { + if (onError === undefined) { + toast.error(`Failed to create Warehouse copy: ${data.message}`) + } else { + onError(data, variables, context) + } + }, + ...options, + }) +} diff --git a/apps/studio/data/warehouse/warehouse-catalog-mutation.ts b/apps/studio/data/warehouse/warehouse-catalog-mutation.ts new file mode 100644 index 00000000000..91ca93fb185 --- /dev/null +++ b/apps/studio/data/warehouse/warehouse-catalog-mutation.ts @@ -0,0 +1,65 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { toast } from 'sonner' + +import { warehouseKeys } from './keys' +import { fetchPost } from '@/data/fetchers' +import { API_URL } from '@/lib/constants' +import { ResponseError, type UseCustomMutationOptions } from '@/types' + +export type UpdateWarehouseCatalogVariables = { + projectRef: string + enabled: boolean +} + +/** + * Enables or disables external Warehouse Catalog access for the project. Enabling provisions the + * catalog credentials that external query engines (DuckDB, Spark, Trino, PyIceberg) use to read + * Warehouse tables without touching the primary database. + */ +export async function updateWarehouseCatalog({ + projectRef, + enabled, +}: UpdateWarehouseCatalogVariables) { + if (!projectRef) throw new Error('projectRef is required') + + const result = await fetchPost( + `${API_URL}/platform/warehouse/${projectRef}/catalog`, + { enabled }, + { credentials: 'include' } + ) + if (result instanceof ResponseError) throw result + return result +} + +type UpdateWarehouseCatalogData = Awaited> + +export const useUpdateWarehouseCatalogMutation = ({ + onSuccess, + onError, + ...options +}: Omit< + UseCustomMutationOptions< + UpdateWarehouseCatalogData, + ResponseError, + UpdateWarehouseCatalogVariables + >, + 'mutationFn' +> = {}) => { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: updateWarehouseCatalog, + async onSuccess(data, variables, context) { + await queryClient.invalidateQueries({ queryKey: warehouseKeys.catalog(variables.projectRef) }) + await onSuccess?.(data, variables, context) + }, + async onError(data, variables, context) { + if (onError === undefined) { + toast.error(`Failed to update Warehouse Catalog: ${data.message}`) + } else { + onError(data, variables, context) + } + }, + ...options, + }) +} diff --git a/apps/studio/data/warehouse/warehouse-catalog-query.ts b/apps/studio/data/warehouse/warehouse-catalog-query.ts new file mode 100644 index 00000000000..e2529c85dce --- /dev/null +++ b/apps/studio/data/warehouse/warehouse-catalog-query.ts @@ -0,0 +1,68 @@ +import { useQuery, type UseQueryOptions } from '@tanstack/react-query' + +import { warehouseKeys } from './keys' +import { fetchGet } from '@/data/fetchers' +import { API_URL } from '@/lib/constants' +import { ResponseError } from '@/types' + +export interface WarehouseCatalogCredentials { + catalogUri: string + accessToken: string + warehouseId: string +} + +export interface WarehouseCatalog { + enabled: boolean + credentials?: WarehouseCatalogCredentials +} + +type WarehouseCatalogApiResponse = { + enabled: boolean + credentials?: { + catalog_uri: string + access_token: string + warehouse_id: string + } +} + +export type WarehouseCatalogVariables = { projectRef?: string } +export type WarehouseCatalogData = WarehouseCatalog +export type WarehouseCatalogError = ResponseError + +export async function getWarehouseCatalog( + { projectRef }: WarehouseCatalogVariables, + signal?: AbortSignal +): Promise { + if (!projectRef) throw new Error('projectRef is required') + + const result = await fetchGet( + `${API_URL}/platform/warehouse/${projectRef}/catalog`, + { abortSignal: signal, credentials: 'include' } + ) + if (result instanceof ResponseError) throw result + + return { + enabled: result.enabled, + credentials: result.credentials + ? { + catalogUri: result.credentials.catalog_uri, + accessToken: result.credentials.access_token, + warehouseId: result.credentials.warehouse_id, + } + : undefined, + } +} + +export const useWarehouseCatalogQuery = ( + { projectRef }: WarehouseCatalogVariables, + { + enabled = true, + ...options + }: Omit, 'queryKey'> = {} +) => + useQuery({ + queryKey: warehouseKeys.catalog(projectRef), + queryFn: ({ signal }) => getWarehouseCatalog({ projectRef }, signal), + enabled: enabled && typeof projectRef !== 'undefined', + ...options, + }) diff --git a/apps/studio/data/warehouse/warehouse-tables-query.ts b/apps/studio/data/warehouse/warehouse-tables-query.ts new file mode 100644 index 00000000000..c99a4a917b1 --- /dev/null +++ b/apps/studio/data/warehouse/warehouse-tables-query.ts @@ -0,0 +1,73 @@ +import { useQuery, type UseQueryOptions } from '@tanstack/react-query' +import { useMemo } from 'react' + +import { warehouseKeys } from './keys' +import { + POSTGRES_TABLE_STATE, + tableKeyOf, + tableStateFromLinkedTable, + type WarehouseLinkedTable, + type WarehouseTableState, +} from './warehouse-types' +import { fetchGet } from '@/data/fetchers' +import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject' +import { API_URL } from '@/lib/constants' +import { ResponseError } from '@/types' + +export type WarehouseTablesVariables = { projectRef?: string } +type WarehouseTablesResponse = { tables: WarehouseLinkedTable[] } +export type WarehouseTablesData = WarehouseLinkedTable[] +export type WarehouseTablesError = ResponseError + +export async function getWarehouseTables( + { projectRef }: WarehouseTablesVariables, + signal?: AbortSignal +): Promise { + if (!projectRef) throw new Error('projectRef is required') + + const result = await fetchGet( + `${API_URL}/platform/warehouse/${projectRef}/tables`, + { abortSignal: signal, credentials: 'include' } + ) + if (result instanceof ResponseError) throw result + return result.tables ?? [] +} + +export const useWarehouseTablesQuery = ( + { projectRef }: WarehouseTablesVariables, + { + enabled = true, + ...options + }: Omit, 'queryKey'> = {} +) => + useQuery({ + queryKey: warehouseKeys.tables(projectRef), + queryFn: ({ signal }) => getWarehouseTables({ projectRef }, signal), + enabled: enabled && typeof projectRef !== 'undefined', + ...options, + }) + +/** Per-table storage state for the selected project. Falls back to Postgres-only when not linked. */ +export function useWarehouseTableState(tableKey: string): WarehouseTableState { + const { data: project } = useSelectedProjectQuery() + const { data } = useWarehouseTablesQuery({ projectRef: project?.ref }) + + return useMemo(() => { + const match = (data ?? []).find((table) => tableKeyOf(table.schema, table.name) === tableKey) + return match ? tableStateFromLinkedTable(match) : POSTGRES_TABLE_STATE + }, [data, tableKey]) +} + +/** Lookup of every linked table for the selected project, keyed by `schema.name`. */ +export function useWarehouseTableStates(): Map { + const { data: project } = useSelectedProjectQuery() + const { data } = useWarehouseTablesQuery({ projectRef: project?.ref }) + + return useMemo(() => { + const map = new Map() + for (const table of data ?? []) { + map.set(tableKeyOf(table.schema, table.name), tableStateFromLinkedTable(table)) + } + return map + }, [data]) +} diff --git a/apps/studio/data/warehouse/warehouse-types.ts b/apps/studio/data/warehouse/warehouse-types.ts new file mode 100644 index 00000000000..a15a7ee7c66 --- /dev/null +++ b/apps/studio/data/warehouse/warehouse-types.ts @@ -0,0 +1,74 @@ +/** + * Studio-facing model for the Warehouse product. + * + * Under the hood a "Warehouse copy" is a table linked into the project's DuckLake replication + * pipeline (`supabase_warehouse_pipeline`) and added to its publication. These types describe what + * the table surfaces need, decoupled from the lower-level replication/destination payloads. + */ + +export type WarehouseMode = 'postgres' | 'has_warehouse_copy' +export type WarehouseSyncState = 'syncing' | 'live' | 'error' + +/** Row returned by `GET /platform/warehouse/{ref}/tables` (snake_case, mirrors the API). */ +export interface WarehouseLinkedTable { + schema: string + name: string + state: WarehouseSyncState + lag_ms?: number + last_synced_at?: string + copy_name: string + warehouse_size_bytes?: number +} + +/** UI-facing per-table storage state consumed by the table surfaces. */ +export interface WarehouseTableState { + mode: WarehouseMode + syncState?: WarehouseSyncState + lagSeconds?: number + lastSyncedAt?: string + copyName?: string + warehouseSizeBytes?: number +} + +/** Default state for a table that has not been linked to Warehouse. */ +export const POSTGRES_TABLE_STATE: WarehouseTableState = { mode: 'postgres' } + +export const tableKeyOf = (schema: string, name: string) => `${schema}.${name}` + +/** Maps an API linked-table row onto the UI-facing state. A listed table always has a copy. */ +export function tableStateFromLinkedTable(table: WarehouseLinkedTable): WarehouseTableState { + return { + mode: 'has_warehouse_copy', + syncState: table.state, + lagSeconds: table.lag_ms !== undefined ? Math.round(table.lag_ms / 1000) : undefined, + lastSyncedAt: table.last_synced_at, + copyName: table.copy_name, + warehouseSizeBytes: table.warehouse_size_bytes, + } +} + +export function formatWarehouseSize(bytes: number | undefined): string { + if (bytes === undefined) return '—' + const gb = bytes / 1024 ** 3 + return `${Math.round(gb)} GB` +} + +/** + * Inline storage summary shown next to a table (e.g. "112 kB · Copy · 184 GB"). Returns null when + * the table lives only in the Postgres heap so callers can fall back to the plain Postgres size. + */ +export function getWarehouseStorageSummaryLabel( + state: Pick | undefined, + postgresSize?: string +): string | null { + const mode = state?.mode ?? 'postgres' + if (mode === 'postgres') return null + + const hasSize = state?.warehouseSizeBytes !== undefined + const warehouseSize = formatWarehouseSize(state?.warehouseSizeBytes) + + if (postgresSize && hasSize) return `${postgresSize} · Copy · ${warehouseSize}` + if (postgresSize) return `${postgresSize} · Copy` + if (hasSize) return `Copy · ${warehouseSize}` + return 'Copy' +} diff --git a/packages/common/telemetry-constants.ts b/packages/common/telemetry-constants.ts index a7fb3969c7b..b5e6a79199b 100644 --- a/packages/common/telemetry-constants.ts +++ b/packages/common/telemetry-constants.ts @@ -1404,7 +1404,7 @@ export interface UnifiedLogsBannerDismissButtonClickedEvent { export interface IndexAdvisorEnableButtonClickedEvent { action: 'index_advisor_enable_button_clicked' properties: { - origin: 'banner' | 'dialog' + origin: 'banner' | 'dialog' | 'table_settings' } groups: TelemetryGroups }