feat: Create new schema when creating a FDW with the schema option (#37205)

* Change the import foreign schema to always create a schema.

* Fix the Iceberg wrapper.

* Refactor the create wrapper to always create a new schema.

* Remove unneeded props.

* Smol fixes

* Prevent double error toasts

* Smol fix

* Fix the wrapper creation to include the correct api key.

* Fix a bug with the new api keys.

* Handle both types of keys when fetching the iceberg namespaces.

* Add a field to all wrappers to hold the schema name which can import foreign schema.

* sq

* When importing a foreign schema, save the schema in a special field.

* Fix a type error.

* Fix importing foreign schema overriding wrapper server options with unencrypted values

* Add comment

* Handle duplicate and empty schemas when importing a foreign schema.

* Update the copy in the foreign schema wrappers.

* Remove unnecessary code.

---------

Co-authored-by: Joshen Lim <joshenlimek@gmail.com>
This commit is contained in:
Ivan VasilovandJoshen Lim authored and GitHub committed 2025-07-25 10:54:17 +02:00
1 parent 1332d30f2f
commit 32c3eeb389
18 files changed
+539 -296

No files matched your search

@@ -42,7 +42,7 @@ export const MarkdownContent = ({ integrationId }: { integrationId: string }) =>
/>
)}
{supportExpanding && (
<div className={cn('bottom-0 z-10', !isExpanded ? 'absolute' : 'relative mt-3')}>
<div className={cn('bottom-0', !isExpanded ? 'absolute' : 'relative mt-3')}>
<button
className="text-foreground-light hover:text-foreground underline text-sm"
onClick={() => setIsExpanded(!isExpanded)}
@@ -2,18 +2,17 @@ import { isEmpty } from 'lodash'
import { useMemo, useState } from 'react'
import { toast } from 'sonner'
import SchemaEditor from 'components/interfaces/TableGridEditor/SidePanelEditor/SchemaEditor'
import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext'
import { FormSection, FormSectionContent, FormSectionLabel } from 'components/ui/Forms/FormSection'
import SchemaSelector from 'components/ui/SchemaSelector'
import { useSchemaCreateMutation } from 'data/database/schema-create-mutation'
import { useSchemasQuery } from 'data/database/schemas-query'
import { useFDWCreateMutation } from 'data/fdw/fdw-create-mutation'
import { useSendEventMutation } from 'data/telemetry/send-event-mutation'
import { useSelectedOrganization } from 'hooks/misc/useSelectedOrganization'
import {
Button,
Form,
Input,
Label_Shadcn_,
RadioGroupStacked,
RadioGroupStackedItem,
Separator,
@@ -25,7 +24,6 @@ import ConfirmationModal from 'ui-patterns/Dialogs/ConfirmationModal'
import { CreateWrapperSheetProps } from './CreateWrapperSheet'
import InputField from './InputField'
import { makeValidateRequired } from './Wrappers.utils'
import { useSelectedOrganization } from 'hooks/misc/useSelectedOrganization'
const FORM_ID = 'create-wrapper-form'
@@ -68,12 +66,11 @@ export const CreateIcebergWrapperSheet = ({
const org = useSelectedOrganization()
const { mutate: sendEvent } = useSendEventMutation()
const [createSchemaSheetOpen, setCreateSchemaSheetOpen] = useState(false)
const [selectedTarget, setSelectedTarget] = useState<Target>('S3Tables')
const [formErrors, setFormErrors] = useState<{ [k: string]: string }>({})
const { mutate: createFDW, isLoading: isCreating } = useFDWCreateMutation({
const { mutateAsync: createFDW, isLoading: isCreatingWrapper } = useFDWCreateMutation({
onSuccess: () => {
toast.success(`Successfully created ${wrapperMeta?.label} foreign data wrapper`)
onClose()
@@ -100,8 +97,10 @@ export const CreateIcebergWrapperSheet = ({
}
}, [wrapperMetaOriginal, selectedTarget])
// prefetch schemas to make sure the schema selector is populated
useSchemasQuery({ projectRef: project?.ref, connectionString: project?.connectionString })
const { data: schemas } = useSchemasQuery({
projectRef: project?.ref!,
connectionString: project?.connectionString,
})
const initialValues = {
wrapper_name: '',
@@ -113,41 +112,72 @@ export const CreateIcebergWrapperSheet = ({
),
}
const { mutateAsync: createSchema, isLoading: isCreatingSchema } = useSchemaCreateMutation()
const onSubmit = async (values: any) => {
const validate = makeValidateRequired(wrapperMeta.server.options)
const errors: any = validate(values)
if (values.source_schema.length === 0) {
errors.source_schema = 'Please provide a source schema'
errors.source_schema = 'Please provide a namespace name'
}
if (values.wrapper_name.length === 0) {
errors.wrapper_name = 'Please provide a name for your wrapper'
}
if (!isEmpty(errors)) return setFormErrors(errors)
createFDW({
projectRef: project?.ref,
connectionString: project?.connectionString,
wrapperMeta,
formState: { ...values, server_name: `${values.wrapper_name}_server` },
mode: 'schema',
tables: [],
sourceSchema: values.source_schema,
targetSchema: values.target_schema,
})
if (values.target_schema.length === 0) {
errors.target_schema = 'Please provide an unique target schema'
}
const foundSchema = schemas?.find((s) => s.name === values.target_schema)
if (foundSchema) {
errors.target_schema = 'This schema already exists. Please specify a unique schema name.'
}
sendEvent({
action: 'foreign_data_wrapper_created',
properties: {
wrapperType: wrapperMeta.label,
},
groups: {
project: project?.ref ?? 'Unknown',
organization: org?.slug ?? 'Unknown',
},
})
setFormErrors(errors)
if (!isEmpty(errors)) {
return
}
try {
await createSchema({
projectRef: project?.ref,
connectionString: project?.connectionString,
name: values.target_schema,
})
await createFDW({
projectRef: project?.ref,
connectionString: project?.connectionString,
wrapperMeta,
formState: {
...values,
server_name: `${values.wrapper_name}_server`,
supabase_target_schema: values.target_schema,
},
mode: 'schema',
tables: [],
sourceSchema: values.source_schema,
targetSchema: values.target_schema,
})
sendEvent({
action: 'foreign_data_wrapper_created',
properties: {
wrapperType: wrapperMeta.label,
},
groups: {
project: project?.ref ?? 'Unknown',
organization: org?.slug ?? 'Unknown',
},
})
} catch (error) {
console.error(error)
// The error will be handled by the mutation onError callback (toast.error)
}
}
const isLoading = isCreatingWrapper || isCreatingSchema
return (
<>
<div className="h-full" tabIndex={-1}>
@@ -272,7 +302,8 @@ export const CreateIcebergWrapperSheet = ({
<FormSectionLabel>
<p>Foreign Schema</p>
<p className="text-foreground-light mt-2 w-[90%]">
All wrapper tables will be created in the specified target schema.
You can query your data from the foreign tables in the specified schema
after the wrapper is created.
</p>
</FormSectionLabel>
}
@@ -292,18 +323,21 @@ export const CreateIcebergWrapperSheet = ({
</div>
)}
<div className="flex flex-col gap-2">
<Label_Shadcn_ className="text-foreground-light">
Target Schema
</Label_Shadcn_>
<SchemaSelector
portal={false}
size="small"
selectedSchemaName={values.target_schema}
onSelectSchema={(schema) => setFieldValue('target_schema', schema)}
onSelectCreateSchema={() => setCreateSchemaSheetOpen(true)}
<InputField
key="target_schema"
option={{
name: 'target_schema',
label: 'Specify a new schema to create all wrapper tables in',
required: true,
encrypted: false,
secureEntry: false,
}}
loading={false}
error={formErrors['target_schema']}
/>
<p className="text-foreground-lighter text-sm">
Be careful not to use an API exposed schema.
A new schema will be created. For security purposes, the wrapper tables
from the foreign schema cannot be created within an existing schema
</p>
</div>
</FormSectionContent>
@@ -316,7 +350,7 @@ export const CreateIcebergWrapperSheet = ({
type="default"
htmlType="button"
onClick={onClosePanel}
disabled={isCreating}
disabled={isLoading}
>
Cancel
</Button>
@@ -325,20 +359,11 @@ export const CreateIcebergWrapperSheet = ({
type="primary"
form={FORM_ID}
htmlType="submit"
disabled={isCreating}
loading={isCreating}
loading={isLoading}
>
Create wrapper
</Button>
</SheetFooter>
<SchemaEditor
visible={createSchemaSheetOpen}
closePanel={() => setCreateSchemaSheetOpen(false)}
onSuccess={(schema) => {
setFieldValue('target_schema', schema)
setCreateSchemaSheetOpen(false)
}}
/>
</>
)
}}
@@ -4,19 +4,18 @@ import { Edit, Trash } from 'lucide-react'
import { useState } from 'react'
import { toast } from 'sonner'
import SchemaEditor from 'components/interfaces/TableGridEditor/SidePanelEditor/SchemaEditor'
import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext'
import { FormSection, FormSectionContent, FormSectionLabel } from 'components/ui/Forms/FormSection'
import SchemaSelector from 'components/ui/SchemaSelector'
import { useDatabaseExtensionsQuery } from 'data/database-extensions/database-extensions-query'
import { useSchemaCreateMutation } from 'data/database/schema-create-mutation'
import { invalidateSchemasQuery, useSchemasQuery } from 'data/database/schemas-query'
import { useFDWCreateMutation } from 'data/fdw/fdw-create-mutation'
import { useSendEventMutation } from 'data/telemetry/send-event-mutation'
import { useSelectedOrganization } from 'hooks/misc/useSelectedOrganization'
import {
Button,
Form,
Input,
Label_Shadcn_,
RadioGroupStacked,
RadioGroupStackedItem,
Separator,
@@ -30,7 +29,6 @@ import InputField from './InputField'
import { WrapperMeta } from './Wrappers.types'
import { makeValidateRequired } from './Wrappers.utils'
import WrapperTableEditor from './WrapperTableEditor'
import { useSelectedOrganization } from 'hooks/misc/useSelectedOrganization'
export interface CreateWrapperSheetProps {
isClosing: boolean
@@ -48,6 +46,7 @@ export const CreateWrapperSheet = ({
onClose,
}: CreateWrapperSheetProps) => {
const queryClient = useQueryClient()
const { project } = useProjectContext()
const org = useSelectedOrganization()
const { mutate: sendEvent } = useSendEventMutation()
@@ -73,7 +72,7 @@ export const CreateWrapperSheet = ({
const [formErrors, setFormErrors] = useState<{ [k: string]: string }>({})
const { mutate: createFDW, isLoading: isCreating } = useFDWCreateMutation({
const { mutateAsync: createFDW, isLoading: isCreatingWrapper } = useFDWCreateMutation({
onSuccess: () => {
toast.success(`Successfully created ${wrapperMeta?.label} foreign data wrapper`)
setNewTables([])
@@ -85,8 +84,10 @@ export const CreateWrapperSheet = ({
},
})
// prefetch schemas to make sure the schema selector is populated
useSchemasQuery({ projectRef: project?.ref, connectionString: project?.connectionString })
const { data: schemas } = useSchemasQuery({
projectRef: project?.ref!,
connectionString: project?.connectionString,
})
const initialValues = {
wrapper_name: '',
@@ -98,6 +99,8 @@ export const CreateWrapperSheet = ({
),
}
const { mutateAsync: createSchema, isLoading: isCreatingSchema } = useSchemaCreateMutation()
const onUpdateTable = (values: any) => {
setNewTables((prev) => {
// if the new values have tableIndex, we are editing an existing table
@@ -121,37 +124,70 @@ export const CreateWrapperSheet = ({
if (values.wrapper_name.length === 0) {
errors.wrapper_name = 'Please provide a name for your wrapper'
}
if (selectedMode === 'tables' && newTables.length === 0) {
errors.tables = 'Please add at least one table'
}
if (selectedMode === 'schema' && values.source_schema.length === 0) {
errors.source_schema = 'Please provide a source schema'
}
if (!isEmpty(errors)) return setFormErrors(errors)
createFDW({
projectRef: project?.ref,
connectionString: project?.connectionString,
wrapperMeta,
formState: { ...values, server_name: `${values.wrapper_name}_server` },
mode: selectedMode,
tables: newTables,
sourceSchema: values.source_schema,
targetSchema: values.target_schema,
})
if (selectedMode === 'tables') {
if (newTables.length === 0) {
errors.tables = 'Please provide at least one table'
}
}
if (selectedMode === 'schema') {
if (values.source_schema.length === 0) {
errors.source_schema = 'Please provide a source schema'
}
if (values.target_schema.length === 0) {
errors.target_schema = 'Please provide an unique target schema'
}
const foundSchema = schemas?.find((s) => s.name === values.target_schema)
if (foundSchema) {
errors.target_schema = 'This schema already exists. Please specify a unique schema name.'
}
}
sendEvent({
action: 'foreign_data_wrapper_created',
properties: {
wrapperType: wrapperMeta.label,
},
groups: {
project: project?.ref ?? 'Unknown',
organization: org?.slug ?? 'Unknown',
},
})
setFormErrors(errors)
if (!isEmpty(errors)) {
return
}
try {
await createSchema({
projectRef: project?.ref,
connectionString: project?.connectionString,
name: values.target_schema,
})
await createFDW({
projectRef: project?.ref,
connectionString: project?.connectionString,
wrapperMeta,
formState: {
...values,
server_name: `${values.wrapper_name}_server`,
supabase_target_schema: selectedMode === 'schema' ? values.target_schema : undefined,
},
mode: selectedMode,
tables: newTables,
sourceSchema: values.source_schema,
targetSchema: values.target_schema,
})
sendEvent({
action: 'foreign_data_wrapper_created',
properties: {
wrapperType: wrapperMeta.label,
},
groups: {
project: project?.ref ?? 'Unknown',
organization: org?.slug ?? 'Unknown',
},
})
} catch (error) {
console.error(error)
// The error will be handled by the mutation onError callback (toast.error)
}
}
const isLoading = isCreatingWrapper || isCreatingSchema
return (
<>
<div className="h-full" tabIndex={-1}>
@@ -161,7 +197,7 @@ export const CreateWrapperSheet = ({
onSubmit={onSubmit}
className="flex-grow flex flex-col h-full"
>
{({ handleReset, values, initialValues, setFieldValue }: any) => {
{({ values, initialValues, setFieldValue }: any) => {
const hasChanges = JSON.stringify(values) !== JSON.stringify(initialValues)
const onClosePanel = () => {
@@ -236,7 +272,7 @@ export const CreateWrapperSheet = ({
<div className="flex gap-x-5">
<div className="flex flex-col">
<p className="text-foreground-light text-left">
Define tables where the wrapper data will be shown.
Create foreign tables to query data from {wrapperMeta.label}.
</p>
</div>
</div>
@@ -261,7 +297,8 @@ export const CreateWrapperSheet = ({
<div className="flex gap-x-5">
<div className="flex flex-col">
<p className="text-foreground-light text-left">
Specify schema in which the wrapper will create tables.
Create all foreign tables from {wrapperMeta.label} in a specified
schema.
</p>
</div>
</div>
@@ -366,38 +403,44 @@ export const CreateWrapperSheet = ({
<FormSectionLabel>
<p>Foreign Schema</p>
<p className="text-foreground-light mt-2 w-[90%]">
All wrapper tables will be created in the specified target schema.
You can query your data from the foreign tables in the specified schema
after the wrapper is created.
</p>
</FormSectionLabel>
}
>
<FormSectionContent loading={false}>
{wrapperMeta.sourceSchemaOption && (
<div>
<InputField
key="source_schema"
option={wrapperMeta.sourceSchemaOption}
loading={false}
error={formErrors['source_schema']}
/>
<p className="text-foreground-lighter text-sm">
{wrapperMeta.sourceSchemaOption.description}
</p>
</div>
)}
{wrapperMeta.sourceSchemaOption &&
!wrapperMeta.sourceSchemaOption?.readOnly && (
// Hide the field if the source schema is read-only
<div>
<InputField
key="source_schema"
option={wrapperMeta.sourceSchemaOption}
loading={false}
error={formErrors['source_schema']}
/>
<p className="text-foreground-lighter text-sm">
{wrapperMeta.sourceSchemaOption.description}
</p>
</div>
)}
<div className="flex flex-col gap-2">
<Label_Shadcn_ className="text-foreground-light">
Target Schema
</Label_Shadcn_>
<SchemaSelector
portal={false}
size="small"
selectedSchemaName={values.target_schema}
onSelectSchema={(schema) => setFieldValue('target_schema', schema)}
onSelectCreateSchema={() => setCreateSchemaSheetOpen(true)}
<InputField
key="target_schema"
option={{
name: 'target_schema',
label: 'Specify a new schema to create all wrapper tables in',
required: true,
encrypted: false,
secureEntry: false,
}}
loading={false}
error={formErrors['target_schema']}
/>
<p className="text-foreground-lighter text-sm">
Be careful not to use an API exposed schema.
A new schema will be created. For security purposes, the wrapper tables
from the foreign schema cannot be created within an existing schema
</p>
</div>
</FormSectionContent>
@@ -411,7 +454,7 @@ export const CreateWrapperSheet = ({
type="default"
htmlType="button"
onClick={onClosePanel}
disabled={isCreating}
disabled={isLoading}
>
Cancel
</Button>
@@ -420,20 +463,11 @@ export const CreateWrapperSheet = ({
type="primary"
form={FORM_ID}
htmlType="submit"
disabled={isCreating}
loading={isCreating}
loading={isLoading}
>
Create wrapper
</Button>
</SheetFooter>
<SchemaEditor
visible={createSchemaSheetOpen}
closePanel={() => setCreateSchemaSheetOpen(false)}
onSuccess={(schema) => {
setFieldValue('target_schema', schema)
setCreateSchemaSheetOpen(false)
}}
/>
</>
)
}}
@@ -107,37 +107,39 @@ const WrapperRow = ({ wrapper }: WrapperRowProps) => {
</div>
))}
</TableCell>
<TableCell className="space-x-2 flex-nowrap">
<ButtonTooltip
disabled={!canManageWrappers}
type="default"
icon={<Edit strokeWidth={1.5} />}
className="px-1.5"
onClick={() => setEditWrapperShown(true)}
tooltip={{
content: {
side: 'bottom',
text: !canManageWrappers
? 'You need additional permissions to edit wrappers'
: 'Edit wrapper',
},
}}
/>
<ButtonTooltip
type="default"
disabled={!canManageWrappers}
icon={<Trash strokeWidth={1.5} />}
className="px-1.5"
onClick={() => setDeleteWrapperShown(true)}
tooltip={{
content: {
side: 'bottom',
text: !canManageWrappers
? 'You need additional permissions to delete wrappers'
: 'Delete wrapper',
},
}}
/>
<TableCell className="flex-nowrap">
<div className="flex items-center gap-x-2">
<ButtonTooltip
disabled={!canManageWrappers}
type="default"
icon={<Edit strokeWidth={1.5} />}
className="px-1.5"
onClick={() => setEditWrapperShown(true)}
tooltip={{
content: {
side: 'bottom',
text: !canManageWrappers
? 'You need additional permissions to edit wrappers'
: 'Edit wrapper',
},
}}
/>
<ButtonTooltip
type="default"
disabled={!canManageWrappers}
icon={<Trash strokeWidth={1.5} />}
className="px-1.5"
onClick={() => setDeleteWrapperShown(true)}
tooltip={{
content: {
side: 'bottom',
text: !canManageWrappers
? 'You need additional permissions to delete wrappers'
: 'Delete wrapper',
},
}}
/>
</div>
</TableCell>
</TableRow>
<Sheet open={editWrapperShown} onOpenChange={() => setIsClosingEditWrapper(true)}>
@@ -1,6 +1,6 @@
import { BASE_PATH } from 'lib/constants'
import { CreateIcebergWrapperSheet } from './CreateIcebergWrapperSheet'
import type { WrapperMeta } from './Wrappers.types'
import type { ServerOption, WrapperMeta } from './Wrappers.types'
export const WRAPPER_HANDLERS = {
STRIPE: 'stripe_fdw_handler',
@@ -27,6 +27,16 @@ export const WRAPPER_HANDLERS = {
ORB: 'wasm_fdw_handler',
}
const SUPABASE_TARGET_SCHEMA_OPTION: ServerOption = {
name: 'supabase_target_schema',
label: 'Target Schema',
required: false,
encrypted: false,
secureEntry: false,
readOnly: true,
hidden: true,
}
export const WRAPPERS: WrapperMeta[] = [
{
name: 'stripe_wrapper',
@@ -55,6 +65,7 @@ export const WRAPPERS: WrapperMeta[] = [
encrypted: false,
secureEntry: false,
},
SUPABASE_TARGET_SCHEMA_OPTION,
],
},
tables: [
@@ -1775,6 +1786,7 @@ export const WRAPPERS: WrapperMeta[] = [
encrypted: false,
secureEntry: false,
},
SUPABASE_TARGET_SCHEMA_OPTION,
],
},
canTargetSchema: true,
@@ -2224,12 +2236,13 @@ export const WRAPPERS: WrapperMeta[] = [
encrypted: false,
secureEntry: false,
},
SUPABASE_TARGET_SCHEMA_OPTION,
],
},
canTargetSchema: true,
sourceSchemaOption: {
name: 'source_schema',
label: 'Source Schema',
label: 'Namespace',
description: 'It should match the namespace of the Iceberg catalog.',
required: true,
encrypted: false,
@@ -12,7 +12,7 @@ export const makeValidateRequired = (options: { name: string; required: boolean
)
const requiredArrayOptions = Array.from(requiredArrayOptionsSet)
return (values: any) => {
return (values: Record<string, any>) => {
const errors = Object.fromEntries(
Object.entries(values)
.flatMap(([key, value]) =>
@@ -1,8 +1,10 @@
import { ExternalLink, Eye, EyeOff, Loader } from 'lucide-react'
import { useState } from 'react'
import { useParams } from 'common'
import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext'
import { useVaultSecretDecryptedValueQuery } from 'data/vault/vault-secret-decrypted-value-query'
import { Eye, EyeOff, Loader } from 'lucide-react'
import { useState } from 'react'
import { Button, Input } from 'ui'
import { Button, Input, Tooltip, TooltipContent, TooltipTrigger } from 'ui'
export const DecryptedReadOnlyInput = ({
value,
@@ -15,10 +17,11 @@ export const DecryptedReadOnlyInput = ({
descriptionText: string
label: string
}) => {
const [showHidden, setShowHidden] = useState(false)
const { ref } = useParams()
const { project } = useProjectContext()
const [showHidden, setShowHidden] = useState(false)
const { isLoading: isDecryptedValueLoading, data: decryptedValue } =
const { data: decryptedValue, isLoading: isDecryptedValueLoading } =
useVaultSecretDecryptedValueQuery(
{
projectRef: project?.ref,
@@ -29,22 +32,40 @@ export const DecryptedReadOnlyInput = ({
)
const isLoading = isDecryptedValueLoading && showHidden
const renderedValue = secureEntry
? isLoading
? 'Fetching value from Vault...'
: showHidden
? decryptedValue
: value
: value
return (
<Input
label={label}
readOnly
copy
disabled
value={
secureEntry
? isLoading
? 'Fetching value from Vault...'
: showHidden
? decryptedValue
: value
: value
label={
<div className="flex items-center gap-x-2">
<span>{label}</span>
<Tooltip>
<TooltipTrigger asChild>
<a
target="_blank"
rel="noreferrer noopener"
href={`/project/${ref}/integrations/vault/secrets?search=${value}`}
>
<ExternalLink
size={14}
className="text-foreground-lighter hover:text-foreground-light transition"
/>
</a>
</TooltipTrigger>
<TooltipContent side="bottom">View parameter in Vault</TooltipContent>
</Tooltip>
</div>
}
value={renderedValue}
type={secureEntry ? (isLoading ? 'text' : showHidden ? 'text' : 'password') : 'text'}
descriptionText={descriptionText}
layout="horizontal"
@@ -1,6 +1,7 @@
import { RefreshCw, SquareArrowOutUpRight } from 'lucide-react'
import { useMemo, useState } from 'react'
import type { WrapperMeta } from 'components/interfaces/Integrations/Wrappers/Wrappers.types'
import { FormattedWrapperTable } from 'components/interfaces/Integrations/Wrappers/Wrappers.utils'
import { ImportForeignSchemaDialog } from 'components/interfaces/Storage/ImportForeignSchemaDialog'
import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext'
@@ -9,34 +10,32 @@ import { useFDWImportForeignSchemaMutation } from 'data/fdw/fdw-import-foreign-s
import { FDW } from 'data/fdw/fdws-query'
import { useIcebergNamespaceTablesQuery } from 'data/storage/iceberg-namespace-tables-query'
import { BASE_PATH } from 'lib/constants'
import { Badge, Button, TableCell, TableRow } from 'ui'
import { Button, cn, TableCell, TableRow } from 'ui'
type NamespaceRowProps = {
bucketName: string
namespace: string
schema: string
excludedSchemas: string[]
tables: (FormattedWrapperTable & { id: number })[]
token: string
wrapperInstance: FDW
wrapperValues: Record<string, string>
wrapperMeta: WrapperMeta
}
export const NamespaceRow = ({
bucketName,
namespace,
schema,
excludedSchemas,
tables,
token,
wrapperInstance,
wrapperValues,
wrapperMeta,
}: NamespaceRowProps) => {
const { project } = useProjectContext()
const [importForeignSchemaShown, setImportForeignSchemaShown] = useState(false)
const targetSchema = tables[0]?.schema_name ?? ''
const { data: tablesData, isLoading: isLoadingNamespaceTables } = useIcebergNamespaceTablesQuery(
{
catalogUri: wrapperValues.catalog_uri,
@@ -56,7 +55,7 @@ export const NamespaceRow = ({
connectionString: project?.connectionString,
serverName: wrapperInstance.server_name,
sourceSchema: namespace,
targetSchema: targetSchema,
targetSchema: schema,
})
}
@@ -76,11 +75,11 @@ export const NamespaceRow = ({
return (
<TableRow key={namespace}>
<TableCell>
<Badge variant="brand">{namespace}</Badge>
<TableCell>{namespace}</TableCell>
<TableCell className={cn(!schema ? 'text-foreground-lighter' : '')}>
{schema ?? 'No schema'}
</TableCell>
<TableCell className="text-foreground-light">{schema && <Badge>{schema}</Badge>}</TableCell>
<TableCell className="text-foreground-light text-center">
<TableCell>
{tablesData ? `${tables.length}/${tablesData.length} connected tables` : ``}
</TableCell>
<TableCell className="text-right">
@@ -96,15 +95,15 @@ export const NamespaceRow = ({
Sync
</ButtonTooltip>
{schema ? (
<a
href={`${BASE_PATH}/project/${project?.ref}/editor?schema=${schema}`}
target="_blank"
rel="noreferrer"
>
<Button type="default" icon={<SquareArrowOutUpRight />}>
Table Editor
</Button>
</a>
<Button asChild type="default" icon={<SquareArrowOutUpRight />}>
<a
target="_blank"
rel="noreferrer"
href={`${BASE_PATH}/project/${project?.ref}/editor?schema=${schema}`}
>
Open in Table Editor
</a>
</Button>
) : (
<ButtonTooltip
type="default"
@@ -114,7 +113,7 @@ export const NamespaceRow = ({
content: { text: 'There are no tables connected.' },
}}
>
Table Editor
Open in Table Editor
</ButtonTooltip>
)}
</div>
@@ -122,8 +121,7 @@ export const NamespaceRow = ({
<ImportForeignSchemaDialog
bucketName={bucketName}
namespace={namespace}
excludedSchemas={excludedSchemas}
wrapperValues={wrapperValues}
wrapperMeta={wrapperMeta}
visible={importForeignSchemaShown}
onClose={() => setImportForeignSchemaShown(false)}
/>
@@ -127,7 +127,6 @@ export const AnalyticBucketDetails = ({ bucket }: { bucket: Bucket }) => {
}
})
}, [wrapperTables, namespacesData])
const excludedSchemas = uniq(namespaces.map((n) => n.schema))
const wrappersExtension = extensionsData?.find((ext) => ext.name === 'wrappers')
@@ -196,10 +195,10 @@ export const AnalyticBucketDetails = ({ bucket }: { bucket: Bucket }) => {
<Table>
<TableHeader>
<TableRow>
<TableHead className="text-center">Namespace</TableHead>
<TableHead className="text-center">Schema</TableHead>
<TableHead className="text-center">Tables</TableHead>
<TableHead className="text-center">Actions</TableHead>
<TableHead>Namespace</TableHead>
<TableHead>Schema</TableHead>
<TableHead>Tables</TableHead>
<TableHead />
</TableRow>
</TableHeader>
<TableBody>
@@ -209,11 +208,11 @@ export const AnalyticBucketDetails = ({ bucket }: { bucket: Bucket }) => {
bucketName={bucket.name}
namespace={namespace}
schema={schema}
excludedSchemas={excludedSchemas}
tables={tables as any}
token={token!}
wrapperInstance={wrapperInstance}
wrapperValues={wrapperValues}
wrapperMeta={wrapperMeta}
/>
))}
</TableBody>
@@ -11,10 +11,7 @@ import z from 'zod'
import { useParams } from 'common'
import { useIcebergWrapperExtension } from 'components/interfaces/Storage/AnalyticBucketDetails/useIcebergWrapper'
import { StorageSizeUnits } from 'components/interfaces/Storage/StorageSettings/StorageSettings.constants'
import {
convertFromBytes,
convertToBytes,
} from 'components/interfaces/Storage/StorageSettings/StorageSettings.utils'
import { InlineLink } from 'components/ui/InlineLink'
import { useProjectStorageConfigQuery } from 'data/config/project-storage-config-query'
import { useBucketCreateMutation } from 'data/storage/bucket-create-mutation'
import { useIcebergWrapperCreateMutation } from 'data/storage/iceberg-wrapper-create-mutation'
@@ -42,6 +39,7 @@ import {
} from 'ui'
import { Admonition } from 'ui-patterns/admonition'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import { convertFromBytes, convertToBytes } from './StorageSettings/StorageSettings.utils'
export interface CreateBucketModalProps {
visible: boolean
@@ -80,7 +78,10 @@ const CreateBucketModal = ({ visible, onClose }: CreateBucketModalProps) => {
const { mutate: sendEvent } = useSendEventMutation()
const router = useRouter()
const { mutateAsync: createBucket, isLoading: isCreating } = useBucketCreateMutation()
const { mutateAsync: createBucket, isLoading: isCreating } = useBucketCreateMutation({
// [Joshen] Silencing the error here as it's being handled in onSubmit
onError: () => {},
})
const { mutateAsync: createIcebergWrapper, isLoading: isCreatingIcebergWrapper } =
useIcebergWrapperCreateMutation()
@@ -150,9 +151,8 @@ const CreateBucketModal = ({ visible, onClose }: CreateBucketModalProps) => {
toast.success(`Successfully created bucket ${values.name}`)
router.push(`/project/${ref}/storage/buckets/${values.name}`)
onClose()
} catch (error) {
console.error(error)
toast.error('Failed to create bucket')
} catch (error: any) {
toast.error(`Failed to create bucket: ${error.message}`)
}
}
@@ -235,20 +235,16 @@ const CreateBucketModal = ({ visible, onClose }: CreateBucketModalProps) => {
</p>
</div>
</div>
{icebergCatalogEnabled ? null : (
{!icebergCatalogEnabled && (
<div className="w-full flex gap-x-2 py-2 items-center">
<WarningIcon />
<span className="text-xs text-left">
This feature is currently in alpha and not yet enabled for your
project. Sign up{' '}
<a
className="underline"
target="_blank"
rel="noreferrer"
href="https://forms.supabase.com/analytics-buckets"
>
<span className="text-xs text-left text-foreground-lighter">
This is currently in alpha and not enabled for your project. Sign
up{' '}
<InlineLink href="https://forms.supabase.com/analytics-buckets">
here
</a>
</InlineLink>
.
</span>
</div>
)}
@@ -1,5 +1,5 @@
import { zodResolver } from '@hookform/resolvers/zod'
import { snakeCase } from 'lodash'
import { snakeCase, uniq } from 'lodash'
import { useEffect, useState } from 'react'
import { SubmitHandler, useForm } from 'react-hook-form'
import { toast } from 'sonner'
@@ -7,41 +7,30 @@ import z from 'zod'
import { useParams } from 'common'
import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext'
import SchemaSelector from 'components/ui/SchemaSelector'
import { useSchemaCreateMutation } from 'data/database/schema-create-mutation'
import { useSchemasQuery } from 'data/database/schemas-query'
import { useFDWImportForeignSchemaMutation } from 'data/fdw/fdw-import-foreign-schema-mutation'
import {
Button,
Form_Shadcn_,
FormControl_Shadcn_,
FormField_Shadcn_,
Input_Shadcn_,
Modal,
} from 'ui'
import { useFDWUpdateMutation } from 'data/fdw/fdw-update-mutation'
import { getFDWs } from 'data/fdw/fdws-query'
import { Button, Form_Shadcn_, FormField_Shadcn_, Input_Shadcn_, Modal } from 'ui'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import type { WrapperMeta } from '../Integrations/Wrappers/Wrappers.types'
import { formatWrapperTables } from '../Integrations/Wrappers/Wrappers.utils'
import SchemaEditor from '../TableGridEditor/SidePanelEditor/SchemaEditor'
import { getDecryptedParameters } from './ImportForeignSchemaDialog.utils'
export interface ImportForeignSchemaDialogProps {
bucketName: string
namespace: string
excludedSchemas: string[]
wrapperValues: Record<string, string>
wrapperMeta: WrapperMeta
visible: boolean
onClose: () => void
}
const FormSchema = z.object({
bucketName: z.string().trim(),
sourceNamespace: z.string().trim(),
targetSchema: z.string().trim(),
})
export type ImportForeignSchemaForm = z.infer<typeof FormSchema>
export const ImportForeignSchemaDialog = ({
bucketName,
namespace,
excludedSchemas,
wrapperValues,
wrapperMeta,
visible,
onClose,
}: ImportForeignSchemaDialogProps) => {
@@ -50,14 +39,34 @@ export const ImportForeignSchemaDialog = ({
const [loading, setLoading] = useState(false)
const [createSchemaSheetOpen, setCreateSchemaSheetOpen] = useState(false)
const { mutateAsync: importForeignSchema } = useFDWImportForeignSchemaMutation({
const { mutateAsync: importForeignSchema } = useFDWImportForeignSchemaMutation({})
const { mutateAsync: updateFDW } = useFDWUpdateMutation({
onSuccess: () => {
toast.success(`Successfully connected ${bucketName} to the database.`)
onClose()
},
})
const form = useForm<ImportForeignSchemaForm>({
const { data: schemas } = useSchemasQuery({ projectRef: project?.ref! })
const FormSchema = z.object({
bucketName: z.string().trim(),
sourceNamespace: z.string().trim(),
targetSchema: z
.string()
.trim()
.min(1, 'Schema name is required')
.refine(
(val) => {
return !schemas?.find((s) => s.name === val)
},
{
message: 'This schema already exists. Please specify a unique schema name.',
}
),
})
const form = useForm<z.infer<typeof FormSchema>>({
resolver: zodResolver(FormSchema),
defaultValues: {
bucketName,
@@ -66,18 +75,67 @@ export const ImportForeignSchemaDialog = ({
},
})
const onSubmit: SubmitHandler<ImportForeignSchemaForm> = async (values) => {
const { mutateAsync: createSchema } = useSchemaCreateMutation()
const onSubmit: SubmitHandler<z.infer<typeof FormSchema>> = async (values) => {
const serverName = `${snakeCase(values.bucketName)}_fdw_server`
if (!ref) return console.error('Project ref is required')
setLoading(true)
try {
await createSchema({
projectRef: ref,
connectionString: project?.connectionString,
name: values.targetSchema,
})
await importForeignSchema({
projectRef: ref,
connectionString: project?.connectionString,
serverName: `${snakeCase(values.bucketName)}_fdw_server`,
serverName: serverName,
sourceSchema: values.sourceNamespace,
targetSchema: values.targetSchema,
})
const FDWs = await getFDWs({ projectRef: ref, connectionString: project?.connectionString })
const wrapper = FDWs.find((fdw) => fdw.server_name === serverName)
if (!wrapper) {
throw new Error(`Foreign data wrapper with server name ${serverName} not found`)
}
const serverOptions = await getDecryptedParameters({
ref: project?.ref,
connectionString: project?.connectionString ?? undefined,
serverName,
})
const formValues: Record<string, string> = {
wrapper_name: wrapper.name,
server_name: wrapper.server_name,
...serverOptions,
}
const targetSchemas = (formValues['supabase_target_schema'] || '')
.split(',')
.map((s) => s.trim())
const wrapperTables = formatWrapperTables(wrapper, wrapperMeta)
await updateFDW({
projectRef: project?.ref,
connectionString: project?.connectionString,
wrapper: wrapper,
wrapperMeta: wrapperMeta,
formState: {
...formValues,
server_name: serverName,
supabase_target_schema: uniq([...targetSchemas, values.targetSchema])
.filter(Boolean)
.join(','),
},
tables: wrapperTables,
})
} catch (error: any) {
// error will be handled by the mutation onError callback
} finally {
@@ -110,35 +168,16 @@ export const ImportForeignSchemaDialog = ({
<Form_Shadcn_ {...form}>
<form onSubmit={form.handleSubmit(onSubmit)}>
<Modal.Content className="flex flex-col gap-y-4">
<FormField_Shadcn_
control={form.control}
name="sourceNamespace"
render={({ field }) => (
<FormItemLayout label="Namespace" layout="vertical">
<FormControl_Shadcn_>
<Input_Shadcn_ {...field} placeholder="Enter namespace name" disabled />
</FormControl_Shadcn_>
</FormItemLayout>
)}
/>
<FormField_Shadcn_
control={form.control}
name="targetSchema"
render={({ field }) => (
<FormItemLayout
label="Target Schema"
description="Select the database schema where the Iceberg data will be accessible. Each schema can only be connected to one namespace."
description="Enter a schema name under which the Iceberg data will be accessible. The schema will be created."
layout="vertical"
>
<SchemaSelector
portal={false}
size="small"
selectedSchemaName={field.value}
excludedSchemas={excludedSchemas}
onSelectSchema={(schema) => field.onChange(schema)}
onSelectCreateSchema={() => setCreateSchemaSheetOpen(true)}
/>
<Input_Shadcn_ {...field} placeholder="Enter schema name" />
</FormItemLayout>
)}
/>
@@ -0,0 +1,51 @@
import { getFDWs } from 'data/fdw/fdws-query'
import { getDecryptedValues } from 'data/vault/vault-secret-decrypted-value-query'
import { INTEGRATIONS } from '../Integrations/Landing/Integrations.constants'
import { WrapperMeta } from '../Integrations/Wrappers/Wrappers.types'
import { convertKVStringArrayToJson } from '../Integrations/Wrappers/Wrappers.utils'
export const getDecryptedParameters = async ({
ref,
connectionString,
serverName,
}: {
ref?: string
connectionString?: string
serverName: string
}) => {
const integration = INTEGRATIONS.find((i) => i.id === 'iceberg_wrapper' && i.type === 'wrapper')
const wrapperMeta = (integration?.type === 'wrapper' && integration.meta) as WrapperMeta
const wrapperServerOptions = wrapperMeta.server.options
const FDWs = await getFDWs({ projectRef: ref, connectionString: connectionString })
const wrapper = FDWs.find((fdw) => fdw.server_name === serverName)
const serverOptions = convertKVStringArrayToJson(wrapper?.server_options ?? [])
const paramsToBeDecrypted = Object.fromEntries(
new Map(
Object.entries(serverOptions).filter(([key, value]) => {
return wrapperServerOptions.find((option) => option.name === key)?.encrypted
})
)
)
const decryptedValues = await getDecryptedValues({
projectRef: ref,
connectionString: connectionString,
ids: Object.values(paramsToBeDecrypted),
})
const paramsWithDecryptedValues = Object.fromEntries(
new Map(
Object.entries(paramsToBeDecrypted).map(([name, id]) => {
const decryptedValue = decryptedValues[id]
return [name, decryptedValue]
})
)
)
return {
...serverOptions,
...paramsWithDecryptedValues,
}
}
@@ -18,10 +18,20 @@ async function createIcebergNamespace({
token,
namespace,
}: CreateIcebergNamespaceVariables) {
const headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
let headers = new Headers()
// handle both secret key and service role key
if (token.startsWith('sb_secret_')) {
headers = await constructHeaders({
'Content-Type': 'application/json',
apikey: `${token}`,
})
headers.delete('Authorization')
} else {
headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
}
const url = `${catalogUri}/v1/${warehouse}/namespaces`.replaceAll(/(?<!:)\/\//g, '/')
@@ -17,10 +17,20 @@ async function checkNamespaceExists({
token,
namespace,
}: ExistsNamespaceVariables) {
const headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
let headers = new Headers()
// handle both secret key and service role key
if (token.startsWith('sb_secret_')) {
headers = await constructHeaders({
'Content-Type': 'application/json',
apikey: `${token}`,
})
headers.delete('Authorization')
} else {
headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
}
const url = `${catalogUri}/v1/${warehouse}/namespaces/${namespace}`.replaceAll(/(?<!:)\/\//g, '/')
@@ -17,10 +17,20 @@ async function getNamespaceTables({
token,
namespace,
}: GetNamespaceTablesVariables) {
const headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
let headers = new Headers()
// handle both secret key and service role key
if (token.startsWith('sb_secret_')) {
headers = await constructHeaders({
'Content-Type': 'application/json',
apikey: `${token}`,
})
headers.delete('Authorization')
} else {
headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
}
const url = `${catalogUri}/v1/${warehouse}/namespaces/${namespace}/tables`.replaceAll(
/(?<!:)\/\//g,
@@ -11,10 +11,20 @@ type GetNamespacesVariables = {
}
async function getNamespaces({ catalogUri, warehouse, token }: GetNamespacesVariables) {
const headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
let headers = new Headers()
// handle both secret key and service role key
if (token.startsWith('sb_secret_')) {
headers = await constructHeaders({
'Content-Type': 'application/json',
apikey: `${token}`,
})
headers.delete('Authorization')
} else {
headers = await constructHeaders({
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
})
}
const url = `${catalogUri}/v1/${warehouse}/namespaces`.replaceAll(/(?<!:)\/\//g, '/')
@@ -5,10 +5,7 @@ import {
getCatalogURI,
getConnectionURL,
} from 'components/interfaces/Storage/StorageSettings/StorageSettings.utils'
import {
useIsProjectActive,
useProjectContext,
} from 'components/layouts/ProjectLayout/ProjectContext'
import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext'
import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'
import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'
import { useProjectStorageConfigQuery } from 'data/config/project-storage-config-query'
@@ -20,24 +17,20 @@ import { useS3AccessKeyCreateMutation } from './s3-access-key-create-mutation'
export const useIcebergWrapperCreateMutation = () => {
const { project } = useProjectContext()
const { data: apiKeys } = useAPIKeysQuery({ projectRef: project?.ref })
const { serviceKey } = getKeys(apiKeys)
const { data: apiKeys } = useAPIKeysQuery({ projectRef: project?.ref, reveal: true })
const { secretKey, serviceKey } = getKeys(apiKeys)
const { data: settings } = useProjectSettingsV2Query({ projectRef: project?.ref })
const protocol = settings?.app_config?.protocol ?? 'https'
const endpoint = settings?.app_config?.endpoint
const serviceApiKey = serviceKey?.api_key ?? 'SUPABASE_CLIENT_SERVICE_KEY'
const apiKey = secretKey?.api_key ?? serviceKey?.api_key ?? 'SUPABASE_CLIENT_API_KEY'
const wrapperMeta = WRAPPERS.find((wrapper) => wrapper.name === 'iceberg_wrapper')
const isProjectActive = useIsProjectActive()
const canCreateCredentials = useCheckPermissions(PermissionAction.STORAGE_ADMIN_WRITE, '*')
const { data: config } = useProjectStorageConfigQuery({ projectRef: project?.ref })
const isS3ConnectionEnabled = config?.features.s3Protocol.enabled
const disableCreation = !isProjectActive || !canCreateCredentials || !isS3ConnectionEnabled
const { mutateAsync: createS3AccessKey, isLoading: isCreatingS3AccessKey } =
useS3AccessKeyCreateMutation()
@@ -61,7 +54,7 @@ export const useIcebergWrapperCreateMutation = () => {
server_name: `${wrapperName}_server`,
vault_aws_access_key_id: createS3KeyData?.access_key,
vault_aws_secret_access_key: createS3KeyData?.secret_key,
vault_token: serviceApiKey,
vault_token: apiKey,
warehouse: bucketName,
's3.endpoint': getConnectionURL(project?.ref ?? '', protocol, endpoint),
catalog_uri: getCatalogURI(project?.ref ?? '', protocol, endpoint),
@@ -3,7 +3,7 @@ import { UseQueryOptions, useQuery } from '@tanstack/react-query'
import { executeSql } from '../sql/execute-sql-query'
import { vaultSecretsKeys } from './keys'
export const vaultSecretDecryptedValueQuery = (id: string) => {
const vaultSecretDecryptedValueQuery = (id: string) => {
const sql = new Query()
.from('decrypted_secrets', 'vault')
.select('decrypted_secret')
@@ -13,6 +13,16 @@ export const vaultSecretDecryptedValueQuery = (id: string) => {
return sql
}
const vaultSecretDecryptedValuesQuery = (ids: string[]) => {
const sql = new Query()
.from('decrypted_secrets', 'vault')
.select('id,decrypted_secret')
.filter('id', 'in', ids)
.toSql()
return sql
}
export type VaultSecretsDecryptedValueVariables = {
projectRef?: string
connectionString?: string | null
@@ -58,3 +68,25 @@ export const useVaultSecretDecryptedValueQuery = <TData = string>(
...options,
}
)
// [Joshen] Considering to consolidate fetching single and multiple decrypted values by just passing in a string array
// This is currently used in ImportForeignSchemaDialog, but reckon EditWrapperSheet can use this too to replace the useEffect on L153
// which fetches all the decrypted secrets
export const getDecryptedValues = async (
{
projectRef,
connectionString,
ids,
}: {
projectRef?: string
connectionString?: string | null
ids: string[]
},
signal?: AbortSignal
) => {
const sql = vaultSecretDecryptedValuesQuery(ids)
const { result } = await executeSql({ projectRef, connectionString, sql }, signal)
return result.reduce((a: any, b: any) => {
return { ...a, [b.id]: b.decrypted_secret }
}, {})
}