From 28401c6beac655cb59ef53ecf8ac6bb08f248b87 Mon Sep 17 00:00:00 2001 From: Ivan Vasilov Date: Mon, 21 Jul 2025 17:10:51 +0200 Subject: [PATCH] Initlal work on DuckDB wrapper. --- .../Wrappers/CreateDuckDbWrapperSheet.tsx | 485 ++++++++++++++++++ .../Wrappers/Wrappers.constants.ts | 162 +++++- apps/studio/public/img/icons/duckdb-icon.svg | 5 + .../integrations/duckdb_wrapper/overview.md | 3 + 4 files changed, 654 insertions(+), 1 deletion(-) create mode 100644 apps/studio/components/interfaces/Integrations/Wrappers/CreateDuckDbWrapperSheet.tsx create mode 100644 apps/studio/public/img/icons/duckdb-icon.svg create mode 100644 apps/studio/static-data/integrations/duckdb_wrapper/overview.md diff --git a/apps/studio/components/interfaces/Integrations/Wrappers/CreateDuckDbWrapperSheet.tsx b/apps/studio/components/interfaces/Integrations/Wrappers/CreateDuckDbWrapperSheet.tsx new file mode 100644 index 00000000000..347db10ef8b --- /dev/null +++ b/apps/studio/components/interfaces/Integrations/Wrappers/CreateDuckDbWrapperSheet.tsx @@ -0,0 +1,485 @@ +import { isEmpty } from 'lodash' +import { useMemo, useState } from 'react' +import { toast } from 'sonner' + +import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext' +import { FormSection, FormSectionContent, FormSectionLabel } from 'components/ui/Forms/FormSection' +import { useSchemaCreateMutation } from 'data/database/schema-create-mutation' +import { useSchemasQuery } from 'data/database/schemas-query' +import { useFDWCreateMutation } from 'data/fdw/fdw-create-mutation' +import { useSendEventMutation } from 'data/telemetry/send-event-mutation' +import { useSelectedOrganizationQuery } from 'hooks/misc/useSelectedOrganization' +import { + Button, + Form, + Input, + RadioGroupStacked, + RadioGroupStackedItem, + Separator, + SheetFooter, + SheetHeader, + SheetTitle, +} from 'ui' +import ConfirmationModal from 'ui-patterns/Dialogs/ConfirmationModal' +import { CreateWrapperSheetProps } from './CreateWrapperSheet' +import InputField from './InputField' +import { makeValidateRequired } from './Wrappers.utils' + +const FORM_ID = 'create-wrapper-form' + +const requiredFields: Record = { + s3: [ + { name: 'vault_key_id', required: true }, + { name: 'vault_secret', required: true }, + { name: 'region', required: false }, + { name: 'endpoint', required: false }, + { name: 'session_token', required: false }, + { name: 'url_compatibility_mode', required: false }, + { name: 'url_style', required: false }, + { name: 'use_ssl', required: false }, + { name: 'kms_key_id', required: false }, + ], + s3_tables: [ + { name: 'vault_key_id', required: true }, + { name: 'vault_secret', required: true }, + { name: 's3_tables_arn', required: true }, + { name: 'region', required: false }, + ], + r2: [ + { name: 'vault_key_id', required: true }, + { name: 'vault_secret', required: true }, + { name: 'account_id', required: true }, + ], + r2_catalog: [ + { name: 'vault_token', required: true }, + { name: 'warehouse', required: true }, + { name: 'catalog_uri', required: true }, + ], + polaris: [ + { name: 'vault_client_id', required: true }, + { name: 'vault_client_secret', required: true }, + { name: 'warehouse', required: true }, + { name: 'catalog_uri', required: true }, + ], + lakekeeper: [ + { name: 'vault_client_id', required: true }, + { name: 'vault_client_secret', required: true }, + { name: 'oauth2_scope', required: true }, + { name: 'oauth2_server_uri', required: true }, + { name: 'warehouse', required: true }, + { name: 'catalog_uri', required: true }, + ], + iceberg: [ + { name: 'vault_client_id', required: false }, + { name: 'vault_client_secret', required: false }, + { name: 'vault_token', required: false }, + { name: 'oauth2_scope', required: false }, + { name: 'oauth2_server_uri', required: false }, + { name: 'warehouse', required: true }, + { name: 'catalog_uri', required: true }, + ], +} as const + +type Target = 's3' | 's3_tables' | 'r2' | 'r2_catalog' | 'polaris' | 'lakekeeper' | 'iceberg' + +export const CreateDuckDbWrapperSheet = ({ + wrapperMeta: wrapperMetaOriginal, + isClosing, + setIsClosing, + onClose, +}: CreateWrapperSheetProps) => { + const { project } = useProjectContext() + const { data: org } = useSelectedOrganizationQuery() + const { mutate: sendEvent } = useSendEventMutation() + + const [selectedTarget, setSelectedTarget] = useState('s3') + + const [formErrors, setFormErrors] = useState<{ [k: string]: string }>({}) + + const { mutateAsync: createFDW, isLoading: isCreatingWrapper } = useFDWCreateMutation({ + onSuccess: () => { + toast.success(`Successfully created ${wrapperMeta?.label} foreign data wrapper`) + onClose() + }, + }) + + const wrapperMeta = useMemo(() => { + const fields = requiredFields[selectedTarget] + + return { + ...wrapperMetaOriginal, + server: { + options: wrapperMetaOriginal.server.options + // when the target changes, change the wrapperMeta field + .filter((option) => fields.find((field) => field.name === option.name)) + .map((option) => { + const field = fields.find((field) => field.name === option.name) + return { + ...option, + required: field?.required ?? false, + } + }), + }, + } + }, [wrapperMetaOriginal, selectedTarget]) + + const { data: schemas } = useSchemasQuery({ + projectRef: project?.ref!, + connectionString: project?.connectionString, + }) + + const initialValues = { + wrapper_name: '', + server_name: '', + source_schema: wrapperMeta.sourceSchemaOption?.defaultValue ?? '', + target_schema: '', + ...Object.fromEntries( + wrapperMeta.server.options.map((option) => [option.name, option.defaultValue ?? '']) + ), + } + + const { mutateAsync: createSchema, isLoading: isCreatingSchema } = useSchemaCreateMutation() + + const onSubmit = async (values: any) => { + const validate = makeValidateRequired(wrapperMeta.server.options) + const errors: any = validate(values) + + if (selectedTarget === 's3') { + if ( + values.url_compatibility_mode && + values.url_compatibility_mode !== 'true' && + values.url_compatibility_mode !== 'false' + ) { + errors.url_compatibility_mode = 'Please set it true or false' + } + + if (values.url_style && values.url_style !== 'vhost' && values.url_style !== 'path') { + errors.url_style = 'Please set it vhost or path' + } + + if (values.use_ssl && values.use_ssl !== 'true' && values.use_ssl !== 'false') { + errors.use_ssl = 'Please set it true or false' + } + } + + if (values.source_schema.length === 0) { + errors.source_schema = 'Please provide a namespace name' + } + if (values.wrapper_name.length === 0) { + errors.wrapper_name = 'Please provide a name for your wrapper' + } + + if (values.target_schema.length === 0) { + errors.target_schema = 'Please provide an unique target schema' + } + const foundSchema = schemas?.find((s) => s.name === values.target_schema) + if (foundSchema) { + errors.target_schema = 'This schema already exists. Please specify a unique schema name.' + } + + setFormErrors(errors) + if (!isEmpty(errors)) { + return + } + + try { + await createSchema({ + projectRef: project?.ref, + connectionString: project?.connectionString, + name: values.target_schema, + }) + + await createFDW({ + projectRef: project?.ref, + connectionString: project?.connectionString, + wrapperMeta, + formState: { + type: selectedTarget, + ...values, + server_name: `${values.wrapper_name}_server`, + }, + mode: 'schema', + tables: [], + sourceSchema: values.source_schema, + targetSchema: values.target_schema, + }) + + sendEvent({ + action: 'foreign_data_wrapper_created', + properties: { + wrapperType: wrapperMeta.label, + }, + groups: { + project: project?.ref ?? 'Unknown', + organization: org?.slug ?? 'Unknown', + }, + }) + } catch (error) { + console.error(error) + // The error will be handled by the mutation onError callback (toast.error) + } + } + + const isLoading = isCreatingWrapper || isCreatingSchema + + return ( + <> +
+
+ {({ values, initialValues, setFieldValue }: any) => { + const hasChanges = JSON.stringify(values) !== JSON.stringify(initialValues) + + const onClosePanel = () => { + if (hasChanges) { + setIsClosing(true) + } else { + onClose() + } + } + + // if the form hasn't been touched and the user clicked esc or the backdrop, close the sheet + if (!hasChanges && isClosing) { + onClose() + } + + return ( + <> + + Create a {wrapperMeta.label} wrapper + +
+ Wrapper Configuration}> + + 0 ? ( + <> + Your wrapper's server name will be{' '} + {values.wrapper_name}_server + + ) : ( + '' + ) + } + /> + + + + Data target}> + + setSelectedTarget(value as Target)} + > + +
+
+

+ AWS S3 storage that's optimized for analytics workloads. +

+
+
+
+ +
+
+

+ AWS S3 storage that's optimized for analytics workloads. +

+
+
+
+ +
+
+

+ Managed Apache Iceberg built directly into your R2 bucket. +

+
+
+
+ +
+
+

+ Managed Apache Iceberg built directly into your R2 bucket. +

+
+
+
+ +
+
+

+ Something about Polaris. +

+
+
+
+ +
+
+

+ Can be used with any S3-compatible storage. +

+
+
+
+ +
+
+

+ Can be used with any S3-compatible storage. +

+
+
+
+
+
+
+ {wrapperMeta.label} Configuration} + > + + {wrapperMeta.server.options + .filter((option) => !option.hidden) + .map((option) => ( + + ))} + + + + +

Foreign Schema

+

+ All wrapper tables will be created in the specified target schema. +

+ + } + > + + {wrapperMeta.sourceSchemaOption && ( +
+ +

+ {wrapperMeta.sourceSchemaOption.description} +

+
+ )} +
+ +

+ A new schema will be created. All wrapper tables will be created in the + specified target schema. +

+
+
+
+
+ + + + + + + ) + }} +
+
+ setIsClosing(false)} + onConfirm={() => onClose()} + > +

+ There are unsaved changes. Are you sure you want to close the panel? Your changes will be + lost. +

+
+ + ) +} diff --git a/apps/studio/components/interfaces/Integrations/Wrappers/Wrappers.constants.ts b/apps/studio/components/interfaces/Integrations/Wrappers/Wrappers.constants.ts index 76deeca5c7c..fb888e6cd7b 100644 --- a/apps/studio/components/interfaces/Integrations/Wrappers/Wrappers.constants.ts +++ b/apps/studio/components/interfaces/Integrations/Wrappers/Wrappers.constants.ts @@ -14,6 +14,7 @@ export const WRAPPER_HANDLERS = { MSSQL: 'mssql_fdw_handler', REDIS: 'redis_fdw_handler', ICEBERG: 'iceberg_fdw_handler', + DUCKDB: 'duckdb_fdw_handler', PADDLE: 'wasm_fdw_handler', SNOWFLAKE: 'wasm_fdw_handler', CAL: 'wasm_fdw_handler', @@ -1960,6 +1961,165 @@ export const WRAPPERS: WrapperMeta[] = [ }, ], }, + { + name: 'duckdb_wrapper', + handlerName: WRAPPER_HANDLERS.DUCKDB, + validatorName: 'duckdb_fdw_validator', + icon: `${BASE_PATH}/img/icons/duckdb-icon.svg`, + description: "Query and transform your data anywhere using DuckDB's feature-rich SQL dialect.", + extensionName: 'duckdbFdw', + label: 'DuckDB', + docsUrl: 'https://supabase.com/docs/guides/database/extensions/wrappers/duckdb', + minimumExtensionVersion: '0.5.3', + customComponent: true, + server: { + options: [ + { + name: 'vault_key_id', + required: false, + encrypted: true, + secureEntry: true, + label: 'Key ID', + }, + { + name: 'vault_secret', + required: false, + encrypted: true, + secureEntry: true, + label: 'Secret', + }, + { + name: 'region', + required: false, + encrypted: false, + secureEntry: false, + defaultValue: 'us-east-1', + label: 'Region', + }, + { + name: 'endpoint', + required: false, + encrypted: false, + secureEntry: false, + defaultValue: 's3.amazonaws.com', + label: 'Endpoint', + }, + { + name: 'session_token', + required: false, + encrypted: false, + secureEntry: false, + label: 'Session Token', + }, + { + name: 'url_compatibility_mode', + required: false, + encrypted: false, + secureEntry: false, + defaultValue: 'true', + label: 'URL Compatibility Mode', + }, + { + name: 'url_style', + required: false, + encrypted: false, + secureEntry: false, + defaultValue: 'vhost', + label: 'URL Style', + }, + + { + name: 'use_ssl', + required: false, + encrypted: false, + secureEntry: false, + defaultValue: 'true', + label: 'Use SSL', + }, + { + name: 'kms_key_id', + required: false, + encrypted: false, + secureEntry: false, + label: 'KMS Key ID', + }, + { + name: 's3_tables_arn', + required: false, + encrypted: false, + secureEntry: false, + label: 'S3 Tables ARN', + }, + { + name: 'account_id', + required: false, + encrypted: false, + secureEntry: false, + label: 'Account ID', + }, + { + name: 'vault_token', + required: false, + encrypted: true, + secureEntry: true, + label: 'Token', + }, + { + name: 'warehouse', + required: false, + encrypted: false, + secureEntry: false, + label: 'Warehouse', + }, + { + name: 'catalog_uri', + required: false, + encrypted: false, + secureEntry: false, + label: 'Catalog URI', + }, + { + name: 'vault_client_id', + required: false, + encrypted: true, + secureEntry: true, + label: 'Client ID', + }, + { + name: 'vault_client_secret', + required: false, + encrypted: true, + secureEntry: true, + label: 'Client Secret', + }, + { + name: 'oauth2_scope', + required: false, + encrypted: false, + secureEntry: false, + label: 'OAuth2 Scope', + }, + { + name: 'oauth2_server_uri', + required: false, + encrypted: false, + secureEntry: false, + label: 'OAuth2 Server URI', + }, + ], + }, + canTargetSchema: true, + sourceSchemaOption: { + name: 'source_schema', + label: 'Source Schema', + description: 'It should match the namespace of the Iceberg catalog.', + required: true, + encrypted: false, + secureEntry: false, + defaultValue: '', + }, + tables: [], + }, { name: 'paddle_wrapper', handlerName: WRAPPER_HANDLERS.PADDLE, @@ -2169,7 +2329,7 @@ export const WRAPPERS: WrapperMeta[] = [ handlerName: WRAPPER_HANDLERS.ICEBERG, validatorName: 'iceberg_fdw_validator', icon: `${BASE_PATH}/img/icons/iceberg-icon.svg`, - description: 'Iceberg is a data warehouse', + description: 'Connect to Iceberg-compatible data lakes.', extensionName: 'icebergFdw', label: 'Iceberg', docsUrl: `${DOCS_URL}/guides/database/extensions/wrappers/iceberg`, diff --git a/apps/studio/public/img/icons/duckdb-icon.svg b/apps/studio/public/img/icons/duckdb-icon.svg new file mode 100644 index 00000000000..22f6baf2ee3 --- /dev/null +++ b/apps/studio/public/img/icons/duckdb-icon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/apps/studio/static-data/integrations/duckdb_wrapper/overview.md b/apps/studio/static-data/integrations/duckdb_wrapper/overview.md new file mode 100644 index 00000000000..41182adf6cf --- /dev/null +++ b/apps/studio/static-data/integrations/duckdb_wrapper/overview.md @@ -0,0 +1,3 @@ +DuckDB is an open-source column-oriented Relational Database Management System. Unlike traditional databases that require a separate server process, DuckDB runs inside your database and works directly on data in memory or on disk, making it ideal for embedded analytics, data science workflows, and interactive analysis. It supports standard SQL syntax and can query data from multiple formats like CSV, Parquet, and even Pandas DataFrames or Arrow tables without needing to import them into a separate database. + +Its architecture is optimized for analytical (OLAP) workloads rather than transactional (OLTP) ones. DuckDB uses a columnar storage format and vectorized execution engine, which allows it to perform complex aggregations and scans efficiently. Thanks to its lightweight nature and minimal setup, it's widely adopted in environments like Python, R, or even browser-based tools, serving as a powerful alternative to tools like SQLite for analytics tasks.