From 1d203f6c93f008fef6d94d97feb13a70c5e2f166 Mon Sep 17 00:00:00 2001 From: Joshen Lim Date: Mon, 1 Jun 2026 22:02:22 +0800 Subject: [PATCH] feat: Support CLI for Vector buckets (#46381) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Context > [!IMPORTANT] > Will open up for review once CLI PR is merged and deployed so that it's easier to test Related PR: https://github.com/supabase/cli/pull/5230 Adding support for vector buckets for local CLI - will need to be tested locally via `pnpm run dev:studio-local` ## To test There's a bit of testing instructions in the linear ticket [here](https://linear.app/supabase/issue/FE-3474/show-vector-buckets-in-local-admin-studio) as it involves using a branch of CLI - otherwise do reach out to Fabrizio if any help might be needed, but generally: ### Local CLI You might need to manually set `isCli` to `true` in `StorageMenuV2` if the "Vectors" nav item isn't showing up on the storage UI given we're testing via `pnpm run dev:studio-local` - [x] Can create bucket - [x] Can delete bucket - [x] Can create indexes - [x] Can insert data into indexes (via FDW) - [x] Can delete indexes Known issues (that aren't directly solvable from FE end) Reach out to Fabrizio for context as we were both investigating this - PG database needs to be on 17.6 (otherwise there's no S3 vectors FDW) - Storage version needs to be on 1.59.0 ### Self-hosted (This might be tricky to actually test, but just ensure that the code satisfies this) - [x] Cannot see vector buckets ### Hosted - [x] Everything works status quo ## Summary by CodeRabbit * **New Features** * Vector bucket management UI and platform APIs (create/list/delete buckets & indexes) * Local S3 credentials endpoint and client-side hook for self‑hosted/CLI use * **Bug Fixes** * Improved S3 vector setup notifications and clearer error guidance for manual installation * **Refactor** * Deployment-mode gating: platform vs CLI/self‑hosted now controls feature visibility and page behavior * **Tests** * Added suites covering deployment-mode gates and vector bucket error/usage scenarios * **Chores** * Build env updated to expose local S3 credential vars [![Review Change Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/supabase/supabase/pull/46381?utm_source=github_walkthrough&utm_medium=github&utm_campaign=change_stack) --------- Co-authored-by: Ali Waseem --- .../interfaces/Storage/StorageMenuV2.tsx | 13 +- .../CreateVectorBucketDialog.tsx | 15 +- .../VectorBuckets/CreateVectorTableSheet.tsx | 5 +- .../VectorBucketsErrorState.test.tsx | 52 ++++++ .../VectorBuckets/VectorBucketsErrorState.tsx | 16 ++ .../VectorBucketsLocalDisabledState.tsx | 28 ++++ .../Storage/VectorBuckets/index.tsx | 6 +- .../project-storage-config-query.test.tsx | 113 +++++++++++++ .../config/project-storage-config-query.ts | 5 +- apps/studio/data/misc/keys.ts | 1 + apps/studio/data/misc/local-s3-keys-query.ts | 32 ++++ .../s3-vectors-wrapper-create-mutation.ts | 52 +++++- .../lib/api/self-hosted/settings.test.ts | 2 +- apps/studio/lib/api/self-hosted/settings.ts | 2 +- apps/studio/pages/api/get-s3-keys.ts | 10 ++ .../[ref]/vector-buckets/[id]/index.ts | 37 +++++ .../[id]/indexes/[indexName].ts | 32 ++++ .../vector-buckets/[id]/indexes/index.ts | 58 +++++++ .../storage/[ref]/vector-buckets/index.ts | 38 +++++ .../project/[ref]/storage/vectors/index.tsx | 8 +- .../[ref]/storage/vectors/index.test.tsx | 155 ++++++++++++++++++ apps/studio/turbo.jsonc | 2 + 22 files changed, 654 insertions(+), 28 deletions(-) create mode 100644 apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.test.tsx create mode 100644 apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.tsx create mode 100644 apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsLocalDisabledState.tsx create mode 100644 apps/studio/data/config/project-storage-config-query.test.tsx create mode 100644 apps/studio/data/misc/local-s3-keys-query.ts create mode 100644 apps/studio/pages/api/get-s3-keys.ts create mode 100644 apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/index.ts create mode 100644 apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/[indexName].ts create mode 100644 apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/index.ts create mode 100644 apps/studio/pages/api/platform/storage/[ref]/vector-buckets/index.ts create mode 100644 apps/studio/tests/pages/project/[ref]/storage/vectors/index.test.tsx diff --git a/apps/studio/components/interfaces/Storage/StorageMenuV2.tsx b/apps/studio/components/interfaces/Storage/StorageMenuV2.tsx index 9ea0cbe55e5..65d3746b816 100644 --- a/apps/studio/components/interfaces/Storage/StorageMenuV2.tsx +++ b/apps/studio/components/interfaces/Storage/StorageMenuV2.tsx @@ -1,4 +1,4 @@ -import { IS_PLATFORM, useParams } from 'common' +import { useParams } from 'common' import Link from 'next/link' import { useRouter } from 'next/router' import { Badge, Menu } from 'ui' @@ -10,6 +10,7 @@ import { useIsAnalyticsBucketsEnabled, useIsVectorBucketsEnabled, } from '@/data/config/project-storage-config-query' +import { useDeploymentMode } from '@/hooks/misc/useDeploymentMode' import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled' import { SHORTCUT_IDS, type ShortcutId } from '@/state/shortcuts/registry' import { useShortcut } from '@/state/shortcuts/useShortcut' @@ -25,6 +26,8 @@ export const StorageMenuV2 = () => { const { ref } = useParams() const page = useStorageV2Page() + const { isCli, isPlatform } = useDeploymentMode() + const { storageAnalytics, storageVectors } = useIsFeatureEnabled([ 'storage:analytics', 'storage:vectors', @@ -33,8 +36,8 @@ export const StorageMenuV2 = () => { const isAnalyticsBucketsEnabled = useIsAnalyticsBucketsEnabled({ projectRef: ref }) const isVectorBucketsEnabled = useIsVectorBucketsEnabled({ projectRef: ref }) - const showAnalytics = IS_PLATFORM && storageAnalytics - const showVectors = IS_PLATFORM && storageVectors + const showAnalytics = isPlatform && storageAnalytics + const showVectors = (isPlatform && storageVectors) || isCli useShortcut(SHORTCUT_IDS.NAV_STORAGE_FILES, () => router.push(`/project/${ref}/storage/files`)) useShortcut( @@ -48,7 +51,7 @@ export const StorageMenuV2 = () => { { enabled: showVectors } ) useShortcut(SHORTCUT_IDS.NAV_STORAGE_S3, () => router.push(`/project/${ref}/storage/s3`), { - enabled: IS_PLATFORM, + enabled: isPlatform, }) const bucketTypes = Object.entries(BUCKET_TYPES).filter(([key]) => { @@ -95,7 +98,7 @@ export const StorageMenuV2 = () => { })} - {IS_PLATFORM && ( + {isPlatform && ( <>
diff --git a/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorBucketDialog.tsx b/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorBucketDialog.tsx index c3099f00e31..fb0f8c40c42 100644 --- a/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorBucketDialog.tsx +++ b/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorBucketDialog.tsx @@ -141,15 +141,22 @@ export const CreateVectorBucketDialog = ({ } await createS3VectorsWrapper({ bucketName: values.name }) + toast.success(`Successfully created vector bucket ${values.name}`) } catch (error: any) { - toast.warning( - `Failed to create vector bucket integration: ${error.message}. The bucket will be created but you will need to manually install the integration.` + toast.success( +
+

Successfully created vector bucket {values.name}

+

+ However, bucket integration will need to be manually installed as we ran into an error: +

+

{error.message}

+
, + { duration: 8000 } ) } - setIsLoading(false) + setIsLoading(false) track('storage_bucket_created', { bucketType: 'vector' }) - toast.success(`Successfully created vector bucket ${values.name}`) form.reset() setVisible(false) } diff --git a/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorTableSheet.tsx b/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorTableSheet.tsx index af0a1b2e206..a8ff4e49bd9 100644 --- a/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorTableSheet.tsx +++ b/apps/studio/components/interfaces/Storage/VectorBuckets/CreateVectorTableSheet.tsx @@ -34,6 +34,7 @@ import { DocsButton } from '@/components/ui/DocsButton' import { useFDWImportForeignSchemaMutation } from '@/data/fdw/fdw-import-foreign-schema-mutation' import { useVectorBucketIndexCreateMutation } from '@/data/storage/vector-bucket-index-create-mutation' import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions' +import { useDeploymentMode } from '@/hooks/misc/useDeploymentMode' import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject' import { DOCS_URL } from '@/lib/constants' @@ -104,6 +105,7 @@ interface CreateVectorTableSheetProps { export const CreateVectorTableSheet = ({ bucketName }: CreateVectorTableSheetProps) => { const { data: project } = useSelectedProjectQuery() + const { isCli } = useDeploymentMode() const [visible, setVisible] = useQueryState( 'newTable', @@ -117,7 +119,8 @@ export const CreateVectorTableSheet = ({ bucketName }: CreateVectorTableSheetPro ?.split('supabase_target_schema=')[1] // [Joshen] Can remove this once this restriction is removed - const showIndexCreationNotice = isStagingLocal && !!project && project?.region !== 'us-east-1' + const showIndexCreationNotice = + isStagingLocal && !isCli && !!project && project?.region !== 'us-east-1' const defaultValues = { name: '', diff --git a/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.test.tsx b/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.test.tsx new file mode 100644 index 00000000000..296c90edbfd --- /dev/null +++ b/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.test.tsx @@ -0,0 +1,52 @@ +import { screen } from '@testing-library/react' +import { beforeEach, describe, expect, test, vi } from 'vitest' + +import { VectorBucketsErrorState } from './VectorBucketsErrorState' +import type { DeploymentMode } from '@/hooks/misc/useDeploymentMode' +import { customRender } from '@/tests/lib/custom-render' +import { ResponseError } from '@/types' + +const { mockUseDeploymentMode } = vi.hoisted(() => ({ + mockUseDeploymentMode: vi.fn<() => DeploymentMode>(), +})) + +vi.mock('@/hooks/misc/useDeploymentMode', () => ({ + useDeploymentMode: mockUseDeploymentMode, +})) + +vi.mock('@/lib/telemetry/track', () => ({ + useTrack: () => vi.fn(), +})) + +const deploymentMode = (overrides: Partial): DeploymentMode => ({ + isPlatform: false, + isCli: false, + isSelfHosted: false, + ...overrides, +}) + +describe('VectorBucketsErrorState', () => { + beforeEach(() => { + mockUseDeploymentMode.mockReset() + }) + + test('CLI: shows the config.toml enable guidance, not the support error', () => { + mockUseDeploymentMode.mockReturnValue(deploymentMode({ isCli: true })) + + customRender() + + expect(screen.getByText('Vector buckets are not enabled')).toBeInTheDocument() + expect(screen.getByText('supabase/config.toml')).toBeInTheDocument() + expect(screen.queryByText('Contact support')).not.toBeInTheDocument() + }) + + test('platform: shows the generic support error', () => { + mockUseDeploymentMode.mockReturnValue(deploymentMode({ isPlatform: true })) + + customRender() + + expect(screen.getByText('Failed to retrieve vector buckets')).toBeInTheDocument() + expect(screen.getByText('Contact support')).toBeInTheDocument() + expect(screen.queryByText('Vector buckets are not enabled')).not.toBeInTheDocument() + }) +}) diff --git a/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.tsx b/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.tsx new file mode 100644 index 00000000000..6bd95e679e3 --- /dev/null +++ b/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsErrorState.tsx @@ -0,0 +1,16 @@ +import { VectorBucketsLocalDisabledState } from './VectorBucketsLocalDisabledState' +import AlertError from '@/components/ui/AlertError' +import { useDeploymentMode } from '@/hooks/misc/useDeploymentMode' +import type { ResponseError } from '@/types' + +interface VectorBucketsErrorStateProps { + error: ResponseError | null +} + +export const VectorBucketsErrorState = ({ error }: VectorBucketsErrorStateProps) => { + const { isCli } = useDeploymentMode() + + if (isCli) return + + return +} diff --git a/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsLocalDisabledState.tsx b/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsLocalDisabledState.tsx new file mode 100644 index 00000000000..b58bd8fd8fb --- /dev/null +++ b/apps/studio/components/interfaces/Storage/VectorBuckets/VectorBucketsLocalDisabledState.tsx @@ -0,0 +1,28 @@ +import { Admonition } from 'ui-patterns/admonition' +import { CodeBlock } from 'ui-patterns/CodeBlock' + +const CONFIG_SNIPPET = `[storage.vector] +enabled = true +max_buckets = 10 +max_indexes = 5` + +/** + * Shown on the local CLI when listing vector buckets fails — most commonly + * because `[storage.vector]` is not enabled in `config.toml`. Studio can't read + * `config.toml` directly, so we surface the snippet to enable the feature rather + * than the generic "contact support" error. + */ +export const VectorBucketsLocalDisabledState = () => { + return ( + +

+ To use vector buckets locally, enable them in your{' '} + supabase/config.toml and restart with{' '} + supabase start. +

+ + {CONFIG_SNIPPET} + + + ) +} diff --git a/apps/studio/components/interfaces/Storage/VectorBuckets/index.tsx b/apps/studio/components/interfaces/Storage/VectorBuckets/index.tsx index 0f628ec1b70..4003cc2db9c 100644 --- a/apps/studio/components/interfaces/Storage/VectorBuckets/index.tsx +++ b/apps/studio/components/interfaces/Storage/VectorBuckets/index.tsx @@ -14,7 +14,7 @@ import { TimestampInfo } from 'ui-patterns/TimestampInfo' import { EmptyBucketState } from '../EmptyBucketState' import { CreateBucketButton } from '../NewBucketButton' import { CreateVectorBucketDialog } from './CreateVectorBucketDialog' -import AlertError from '@/components/ui/AlertError' +import { VectorBucketsErrorState } from './VectorBucketsErrorState' import { AlphaNotice } from '@/components/ui/AlphaNotice' import { useVectorBucketsQuery } from '@/data/storage/vector-buckets-query' import { createNavigationHandler } from '@/lib/navigation' @@ -62,9 +62,7 @@ export const VectorsBuckets = () => { {isLoadingBuckets && } - {isErrorBuckets && ( - - )} + {isErrorBuckets && } {isSuccessBuckets && ( <> diff --git a/apps/studio/data/config/project-storage-config-query.test.tsx b/apps/studio/data/config/project-storage-config-query.test.tsx new file mode 100644 index 00000000000..b0bdc6469c0 --- /dev/null +++ b/apps/studio/data/config/project-storage-config-query.test.tsx @@ -0,0 +1,113 @@ +import { waitFor } from '@testing-library/react' +import { HttpResponse } from 'msw' +import { beforeEach, describe, expect, test, vi } from 'vitest' + +import { + useIsVectorBucketsEnabled, + type ProjectStorageConfigData, +} from './project-storage-config-query' +import type { DeploymentMode } from '@/hooks/misc/useDeploymentMode' +import { customRenderHook } from '@/tests/lib/custom-render' +import { addAPIMock } from '@/tests/lib/msw' + +const { mockIsPlatform, mockUseDeploymentMode } = vi.hoisted(() => ({ + mockIsPlatform: { value: false }, + mockUseDeploymentMode: vi.fn<() => DeploymentMode>(), +})) + +// `useProjectStorageConfigQuery` (same module as the hook under test) gates its +// fetch on the build-time `IS_PLATFORM` constant — mock it so the query fires in +// the platform cases. +vi.mock('@/lib/constants', async () => { + const actual = await vi.importActual>('@/lib/constants') + return { + ...actual, + get IS_PLATFORM() { + return mockIsPlatform.value + }, + } +}) + +vi.mock('@/hooks/misc/useDeploymentMode', () => ({ + useDeploymentMode: mockUseDeploymentMode, +})) + +const createStorageConfig = (vectorBucketsEnabled: boolean): ProjectStorageConfigData => + ({ + capabilities: { iceberg_catalog: false, list_v2: false }, + databasePoolMode: 'transaction', + external: { upstreamTarget: 'main' }, + features: { + icebergCatalog: { enabled: false, maxCatalogs: 0, maxNamespaces: 0, maxTables: 0 }, + imageTransformation: { enabled: false }, + s3Protocol: { enabled: false }, + vectorBuckets: { enabled: vectorBucketsEnabled, maxBuckets: 0, maxIndexes: 0 }, + }, + fileSizeLimit: 0, + migrationVersion: 'v1', + }) as ProjectStorageConfigData + +const deploymentMode = (overrides: Partial): DeploymentMode => ({ + isPlatform: false, + isCli: false, + isSelfHosted: false, + ...overrides, +}) + +describe('useIsVectorBucketsEnabled', () => { + beforeEach(() => { + mockIsPlatform.value = false + mockUseDeploymentMode.mockReset() + }) + + test('platform + storage config flag enabled: true', async () => { + mockIsPlatform.value = true + mockUseDeploymentMode.mockReturnValue(deploymentMode({ isPlatform: true })) + addAPIMock({ + method: 'get', + path: '/platform/projects/:ref/config/storage', + response: () => HttpResponse.json(createStorageConfig(true)), + }) + + const { result } = customRenderHook(() => useIsVectorBucketsEnabled({ projectRef: 'default' })) + + await waitFor(() => expect(result.current).toBe(true)) + }) + + test('platform + storage config flag disabled: false', async () => { + mockIsPlatform.value = true + mockUseDeploymentMode.mockReturnValue(deploymentMode({ isPlatform: true })) + + let configRequested = false + addAPIMock({ + method: 'get', + path: '/platform/projects/:ref/config/storage', + response: () => { + configRequested = true + return HttpResponse.json(createStorageConfig(false)) + }, + }) + + const { result } = customRenderHook(() => useIsVectorBucketsEnabled({ projectRef: 'default' })) + + await waitFor(() => expect(configRequested).toBe(true)) + expect(result.current).toBe(false) + }) + + test('CLI: true regardless of the storage config flag', () => { + // Query is disabled off-platform, so no storage config endpoint is needed. + mockUseDeploymentMode.mockReturnValue(deploymentMode({ isCli: true })) + + const { result } = customRenderHook(() => useIsVectorBucketsEnabled({ projectRef: 'default' })) + + expect(result.current).toBe(true) + }) + + test('self-hosted: false', () => { + mockUseDeploymentMode.mockReturnValue(deploymentMode({ isSelfHosted: true })) + + const { result } = customRenderHook(() => useIsVectorBucketsEnabled({ projectRef: 'default' })) + + expect(result.current).toBe(false) + }) +}) diff --git a/apps/studio/data/config/project-storage-config-query.ts b/apps/studio/data/config/project-storage-config-query.ts index 37e21ec49e5..987a0bbe364 100644 --- a/apps/studio/data/config/project-storage-config-query.ts +++ b/apps/studio/data/config/project-storage-config-query.ts @@ -3,6 +3,7 @@ import { useQuery } from '@tanstack/react-query' import { configKeys } from './keys' import { components } from '@/data/api' import { get, handleError } from '@/data/fetchers' +import { useDeploymentMode } from '@/hooks/misc/useDeploymentMode' import { IS_PLATFORM } from '@/lib/constants' import type { ResponseError, UseCustomQueryOptions } from '@/types' @@ -60,6 +61,8 @@ export const useIsAnalyticsBucketsEnabled = ({ projectRef }: { projectRef?: stri export const useIsVectorBucketsEnabled = ({ projectRef }: { projectRef?: string }) => { const { data } = useProjectStorageConfigQuery({ projectRef }) - const isVectorBucketsEnabled = !!data?.features.vectorBuckets?.enabled + const { isCli, isPlatform } = useDeploymentMode() + + const isVectorBucketsEnabled = isCli || (isPlatform && !!data?.features.vectorBuckets?.enabled) return isVectorBucketsEnabled } diff --git a/apps/studio/data/misc/keys.ts b/apps/studio/data/misc/keys.ts index 02ec349d343..36d2eed504e 100644 --- a/apps/studio/data/misc/keys.ts +++ b/apps/studio/data/misc/keys.ts @@ -5,4 +5,5 @@ export const miscKeys = { ipAddress: () => ['ip-address'] as const, clockSkew: () => ['clock-skew'] as const, enabledFeaturesOverride: () => ['enabled-features-override'] as const, + localS3Keys: () => ['local-s3-keys'] as const, } diff --git a/apps/studio/data/misc/local-s3-keys-query.ts b/apps/studio/data/misc/local-s3-keys-query.ts new file mode 100644 index 00000000000..1ca08be065d --- /dev/null +++ b/apps/studio/data/misc/local-s3-keys-query.ts @@ -0,0 +1,32 @@ +import { useQuery } from '@tanstack/react-query' + +import { miscKeys } from './keys' +import { fetchHandler } from '@/data/fetchers' +import { BASE_PATH, IS_PLATFORM } from '@/lib/constants' +import type { ResponseError, UseCustomQueryOptions } from '@/types' + +export async function getLocalS3Keys() { + try { + const data = await fetchHandler(`${BASE_PATH}/api/get-s3-keys`).then((res) => res.json()) + return data as { accessKey?: string; secretKey?: string } + } catch (error) { + throw error + } +} + +export type LocalS3KeysData = Awaited> +export type LocalS3KeysError = ResponseError + +/** + * Specifically only for local CLI - to use the S3 keys as defined in the env file + */ +export const useLocalS3KeysQuery = ({ + enabled = true, + ...options +}: UseCustomQueryOptions = {}) => + useQuery({ + queryKey: miscKeys.localS3Keys(), + queryFn: () => getLocalS3Keys(), + enabled: enabled && !IS_PLATFORM, + ...options, + }) diff --git a/apps/studio/data/storage/s3-vectors-wrapper-create-mutation.ts b/apps/studio/data/storage/s3-vectors-wrapper-create-mutation.ts index da9f22f595b..c77b8c329a6 100644 --- a/apps/studio/data/storage/s3-vectors-wrapper-create-mutation.ts +++ b/apps/studio/data/storage/s3-vectors-wrapper-create-mutation.ts @@ -1,5 +1,7 @@ import { PermissionAction } from '@supabase/shared-types/out/constants' +import { IS_PLATFORM } from 'common' +import { useLocalS3KeysQuery } from '../misc/local-s3-keys-query' import { useS3AccessKeyCreateMutation } from './s3-access-key-create-mutation' import { WRAPPERS } from '@/components/interfaces/Integrations/Wrappers/Wrappers.constants' import { getVectorURI } from '@/components/interfaces/Storage/StorageSettings/StorageSettings.utils' @@ -11,14 +13,29 @@ import { import { useProjectSettingsV2Query } from '@/data/config/project-settings-v2-query' import { FDWCreateVariables, useFDWCreateMutation } from '@/data/fdw/fdw-create-mutation' import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions' +import { useDeploymentMode } from '@/hooks/misc/useDeploymentMode' import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject' export const useS3VectorsWrapperCreateMutation = () => { const { data: project } = useSelectedProjectQuery() + const { data: localKeys } = useLocalS3KeysQuery() + const { isPlatform } = useDeploymentMode() const { data: settings } = useProjectSettingsV2Query({ projectRef: project?.ref }) const protocol = settings?.app_config?.protocol ?? 'https' - const endpoint = settings?.app_config?.storage_endpoint || settings?.app_config?.endpoint + + /** + * [Joshen] Endpoint for vectors FDW needs to use docker domain + * Not sure if this affects other areas of the dashboard hence why conditionally rendering here + * instead of updating `lib/api/self-hosted/settings -> api_config` + */ + const port = + !isPlatform && !!settings + ? new URL(`${protocol}://${settings.app_config?.endpoint}`).port + : null + const endpoint = !isPlatform + ? `host.docker.internal:${port}` + : settings?.app_config?.storage_endpoint || settings?.app_config?.endpoint const wrapperMeta = WRAPPERS.find((wrapper) => wrapper.name === 's3_vectors_wrapper') @@ -30,13 +47,32 @@ export const useS3VectorsWrapperCreateMutation = () => { const { mutateAsync: createS3AccessKey, isPending: isCreatingS3AccessKey } = useS3AccessKeyCreateMutation() - const { mutateAsync: createFDW, isPending: isCreatingFDW } = useFDWCreateMutation() + // [Joshen] Silence the error, handled upstream + const { mutateAsync: createFDW, isPending: isCreatingFDW } = useFDWCreateMutation({ + onError: () => {}, + }) const mutateAsync = async ({ bucketName }: { bucketName: string }) => { - const createS3KeyData = await createS3AccessKey({ - projectRef: project?.ref, - description: getVectorBucketS3KeyName(bucketName), - }) + let accessKey: string | undefined + let secretKey: string | undefined + + if (IS_PLATFORM) { + const createS3KeyData = await createS3AccessKey({ + projectRef: project?.ref, + description: getVectorBucketS3KeyName(bucketName), + }) + accessKey = createS3KeyData.access_key + secretKey = createS3KeyData.secret_key + } else { + accessKey = localKeys?.accessKey + secretKey = localKeys?.secretKey + } + + if (!accessKey || !secretKey) { + throw new Error( + IS_PLATFORM ? 'Failed to obtain S3 keys from the API' : 'Local S3 keys are not configured' + ) + } const wrapperName = getVectorBucketFDWName(bucketName) const serverName = getVectorBucketFDWServerName(bucketName) @@ -48,8 +84,8 @@ export const useS3VectorsWrapperCreateMutation = () => { formState: { wrapper_name: wrapperName, server_name: serverName, - vault_access_key_id: createS3KeyData?.access_key, - vault_secret_access_key: createS3KeyData?.secret_key, + vault_access_key_id: accessKey, + vault_secret_access_key: secretKey, aws_region: settings!.region, endpoint_url: getVectorURI(project?.ref ?? '', protocol, endpoint), }, diff --git a/apps/studio/lib/api/self-hosted/settings.test.ts b/apps/studio/lib/api/self-hosted/settings.test.ts index d4458dc34c8..4903332909c 100644 --- a/apps/studio/lib/api/self-hosted/settings.test.ts +++ b/apps/studio/lib/api/self-hosted/settings.test.ts @@ -51,7 +51,7 @@ describe('api/self-hosted/settings', () => { expect(settings.db_port).toBe(5432) expect(settings.db_user).toBe('postgres') expect(settings.ref).toBe('default') - expect(settings.region).toBe('ap-southeast-1') + expect(settings.region).toBe('local') expect(settings.status).toBe('ACTIVE_HEALTHY') expect(settings.ssl_enforced).toBe(false) }) diff --git a/apps/studio/lib/api/self-hosted/settings.ts b/apps/studio/lib/api/self-hosted/settings.ts index f7b8a40ae06..4a35bb2fbb8 100644 --- a/apps/studio/lib/api/self-hosted/settings.ts +++ b/apps/studio/lib/api/self-hosted/settings.ts @@ -40,7 +40,7 @@ export function getProjectSettings() { process.env.AUTH_JWT_SECRET ?? 'super-secret-jwt-token-with-at-least-32-characters-long', name: process.env.DEFAULT_PROJECT_NAME || 'Default Project', ref: 'default', - region: 'ap-southeast-1', + region: 'local', service_api_keys: [ { api_key: process.env.SUPABASE_SERVICE_KEY ?? '', diff --git a/apps/studio/pages/api/get-s3-keys.ts b/apps/studio/pages/api/get-s3-keys.ts new file mode 100644 index 00000000000..f6331fbf0bf --- /dev/null +++ b/apps/studio/pages/api/get-s3-keys.ts @@ -0,0 +1,10 @@ +import { NextApiRequest, NextApiResponse } from 'next' + +const accessKey = process.env.S3_PROTOCOL_ACCESS_KEY_ID +const secretKey = process.env.S3_PROTOCOL_ACCESS_KEY_SECRET + +const handler = async (_req: NextApiRequest, res: NextApiResponse) => { + return res.status(200).json({ accessKey: accessKey, secretKey }) +} + +export default handler diff --git a/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/index.ts b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/index.ts new file mode 100644 index 00000000000..da26596bbe3 --- /dev/null +++ b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/index.ts @@ -0,0 +1,37 @@ +import { createClient } from '@supabase/supabase-js' +import { NextApiRequest, NextApiResponse } from 'next' + +import apiWrapper from '@/lib/api/apiWrapper' + +const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_KEY!) + +// eslint-disable-next-line import/no-anonymous-default-export +export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler) + +async function handler(req: NextApiRequest, res: NextApiResponse) { + const { method } = req + + switch (method) { + case 'GET': + return handleGet(req, res) + case 'DELETE': + return handleDelete(req, res) + default: + res.setHeader('Allow', ['GET', 'PATCH', 'DELETE']) + res.status(405).json({ data: null, error: { message: `Method ${method} Not Allowed` } }) + } +} + +const handleGet = async (req: NextApiRequest, res: NextApiResponse) => { + const { id } = req.query + const { data, error } = await supabase.storage.vectors.getBucket(id as string) + if (error) return res.status(400).json({ error: { message: error.message } }) + return res.status(200).json(data.vectorBucket) +} + +const handleDelete = async (req: NextApiRequest, res: NextApiResponse) => { + const { id } = req.query + const { data, error } = await supabase.storage.vectors.deleteBucket(id as string) + if (error) return res.status(400).json({ error: { message: error.message } }) + return res.status(200).json(data) +} diff --git a/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/[indexName].ts b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/[indexName].ts new file mode 100644 index 00000000000..671361ae94f --- /dev/null +++ b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/[indexName].ts @@ -0,0 +1,32 @@ +import { createClient } from '@supabase/supabase-js' +import { NextApiRequest, NextApiResponse } from 'next' + +import apiWrapper from '@/lib/api/apiWrapper' + +const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_KEY!) + +// eslint-disable-next-line import/no-anonymous-default-export +export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler) + +async function handler(req: NextApiRequest, res: NextApiResponse) { + const { method } = req + + switch (method) { + case 'DELETE': + return handleDelete(req, res) + + default: + res.setHeader('Allow', ['GET', 'POST']) + res.status(405).json({ data: null, error: { message: `Method ${method} Not Allowed` } }) + } +} +const handleDelete = async (req: NextApiRequest, res: NextApiResponse) => { + const { id, indexName } = req.query + + const { data, error } = await supabase.storage.vectors + .from(id as string) + .deleteIndex(indexName as string) + + if (error) return res.status(400).json({ error: { message: error.message } }) + return res.status(200).json(data) +} diff --git a/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/index.ts b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/index.ts new file mode 100644 index 00000000000..272fdc74a9d --- /dev/null +++ b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/[id]/indexes/index.ts @@ -0,0 +1,58 @@ +import { createClient } from '@supabase/supabase-js' +import { NextApiRequest, NextApiResponse } from 'next' + +import apiWrapper from '@/lib/api/apiWrapper' + +const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_KEY!) + +// eslint-disable-next-line import/no-anonymous-default-export +export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler) + +async function handler(req: NextApiRequest, res: NextApiResponse) { + const { method } = req + + switch (method) { + case 'GET': + return handleGet(req, res) + case 'POST': + return handlePost(req, res) + + default: + res.setHeader('Allow', ['GET', 'POST']) + res.status(405).json({ data: null, error: { message: `Method ${method} Not Allowed` } }) + } +} + +const handleGet = async (req: NextApiRequest, res: NextApiResponse) => { + const { id } = req.query + + const { data, error } = await supabase.storage.vectors + .from(id as string) + .listIndexes({ maxResults: 100 }) + + if (error) return res.status(500).json({ error: { message: error.message } }) + + const indexes = await Promise.all( + data.indexes.map(async ({ indexName }) => { + return (await supabase.storage.vectors.from(id as string).getIndex(indexName)).data?.index + }) + ) + + return res.status(200).json({ indexes, nextToken: data.nextToken }) +} + +const handlePost = async (req: NextApiRequest, res: NextApiResponse) => { + const { id } = req.query + const { indexName, dataType, dimension, distanceMetric, metadataKeys } = req.body + const payload = { + indexName, + dataType, + dimension, + distanceMetric, + metadataConfiguration: { nonFilterableMetadataKeys: metadataKeys }, + } + + const { data, error } = await supabase.storage.vectors.from(id as string).createIndex(payload) + if (error) return res.status(400).json({ error: { message: error.message } }) + return res.status(200).json(data) +} diff --git a/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/index.ts b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/index.ts new file mode 100644 index 00000000000..2d1b91ee053 --- /dev/null +++ b/apps/studio/pages/api/platform/storage/[ref]/vector-buckets/index.ts @@ -0,0 +1,38 @@ +import { createClient } from '@supabase/supabase-js' +import { NextApiRequest, NextApiResponse } from 'next' + +import apiWrapper from '@/lib/api/apiWrapper' + +const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_KEY!) + +// eslint-disable-next-line import/no-anonymous-default-export +export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler) + +async function handler(req: NextApiRequest, res: NextApiResponse) { + const { method } = req + + switch (method) { + case 'GET': + return handleGet(req, res) + case 'POST': + return handlePost(req, res) + + default: + res.setHeader('Allow', ['GET', 'POST']) + res.status(405).json({ data: null, error: { message: `Method ${method} Not Allowed` } }) + } +} + +const handleGet = async (_req: NextApiRequest, res: NextApiResponse) => { + const { data, error } = await supabase.storage.vectors.listBuckets() + if (error) return res.status(500).json({ error: { message: error.message } }) + + return res.status(200).json(data) +} + +const handlePost = async (req: NextApiRequest, res: NextApiResponse) => { + const { bucketName } = req.body + const { data, error } = await supabase.storage.vectors.createBucket(bucketName) + if (error) return res.status(400).json({ error: { message: error.message } }) + return res.status(200).json(data) +} diff --git a/apps/studio/pages/project/[ref]/storage/vectors/index.tsx b/apps/studio/pages/project/[ref]/storage/vectors/index.tsx index edc2cd6a79e..8a277b627ae 100644 --- a/apps/studio/pages/project/[ref]/storage/vectors/index.tsx +++ b/apps/studio/pages/project/[ref]/storage/vectors/index.tsx @@ -1,4 +1,4 @@ -import { useParams } from 'common' +import { IS_PLATFORM, useParams } from 'common' import { BucketsUpgradePlan } from '@/components/interfaces/Storage/BucketsUpgradePlan' import { VectorsBuckets } from '@/components/interfaces/Storage/VectorBuckets' @@ -23,10 +23,12 @@ const StorageVectorsPage: NextPageWithLayout = () => { project?.region ?? '' ) - if (!isAvailableInProjectRegion) { + if (IS_PLATFORM && !isAvailableInProjectRegion) { return - } else if (!isVectorBucketsEnabled) { + } else if (IS_PLATFORM && !isVectorBucketsEnabled) { return + } else if (!isVectorBucketsEnabled) { + return null } else { return } diff --git a/apps/studio/tests/pages/project/[ref]/storage/vectors/index.test.tsx b/apps/studio/tests/pages/project/[ref]/storage/vectors/index.test.tsx new file mode 100644 index 00000000000..f5c0cd53e3e --- /dev/null +++ b/apps/studio/tests/pages/project/[ref]/storage/vectors/index.test.tsx @@ -0,0 +1,155 @@ +import { screen, waitFor } from '@testing-library/react' +import { http, HttpResponse } from 'msw' +import { beforeEach, describe, expect, test, vi } from 'vitest' + +import type { ProjectStorageConfigData } from '@/data/config/project-storage-config-query' +import type { ProjectDetail } from '@/data/projects/project-detail-query' +import { API_URL } from '@/lib/constants' +import StorageVectorsPage from '@/pages/project/[ref]/storage/vectors' +import { customRender } from '@/tests/lib/custom-render' +import { addAPIMock, mswServer } from '@/tests/lib/msw' + +const { mockIsPlatform } = vi.hoisted(() => ({ + mockIsPlatform: { value: true }, +})) + +// `IS_PLATFORM` is a build-time constant, so it can't be driven over the +// network — mock it in both modules that read it (the page imports from +// `common`; the data hooks read from `@/lib/constants`). Everything else the +// page branches on comes from real queries via MSW. +vi.mock('common', async (importOriginal) => { + const actual = await importOriginal>() + return { + ...actual, + useParams: () => ({ ref: 'default' }), + get IS_PLATFORM() { + return mockIsPlatform.value + }, + } +}) + +vi.mock('@/lib/constants', async (importOriginal) => { + const actual = await importOriginal>() + return { + ...actual, + get IS_PLATFORM() { + return mockIsPlatform.value + }, + } +}) + +// Stub the heavy leaf children — each fires its own queries and renders a large +// tree. We only assert which branch the page picks. Keep the real +// `VECTOR_BUCKETS_AVAILABLE_REGIONS` so the region gate runs for real. +vi.mock( + '@/components/interfaces/Storage/VectorBuckets/RegionLimitation', + async (importOriginal) => ({ + ...(await importOriginal>()), + RegionLimitation: () =>
region-limitation
, + }) +) + +vi.mock('@/components/interfaces/Storage/VectorBuckets', () => ({ + VectorsBuckets: () =>
vectors-buckets
, +})) + +vi.mock('@/components/interfaces/Storage/BucketsUpgradePlan', () => ({ + BucketsUpgradePlan: ({ type }: { type: string }) =>
buckets-upgrade-plan-{type}
, +})) + +const AVAILABLE_REGION = 'us-east-1' +const UNAVAILABLE_REGION = 'ap-south-1' + +const mockProject = (region: string) => { + addAPIMock({ + method: 'get', + path: '/platform/projects/:ref', + // The page only reads `region` off the project + response: () => HttpResponse.json({ region } as unknown as ProjectDetail), + }) +} + +const mockStorageConfig = (vectorBucketsEnabled: boolean) => { + addAPIMock({ + method: 'get', + path: '/platform/projects/:ref/config/storage', + response: () => + HttpResponse.json({ + capabilities: { iceberg_catalog: false, list_v2: false }, + databasePoolMode: 'transaction', + external: { upstreamTarget: 'main' }, + features: { + icebergCatalog: { enabled: false, maxCatalogs: 0, maxNamespaces: 0, maxTables: 0 }, + imageTransformation: { enabled: false }, + s3Protocol: { enabled: false }, + vectorBuckets: { enabled: vectorBucketsEnabled, maxBuckets: 0, maxIndexes: 0 }, + }, + fileSizeLimit: 0, + migrationVersion: 'v1', + } as ProjectStorageConfigData), + }) +} + +const mockDeploymentMode = (isCli: boolean) => { + mswServer.use( + http.get(`${API_URL}/platform/deployment-mode`, () => HttpResponse.json({ is_cli_mode: isCli })) + ) +} + +describe('StorageVectorsPage', () => { + beforeEach(() => { + mockIsPlatform.value = true + mockProject(AVAILABLE_REGION) + }) + + test('platform + region not supported: shows region limitation', async () => { + mockProject(UNAVAILABLE_REGION) + mockStorageConfig(true) + + customRender() + + expect(await screen.findByText('region-limitation')).toBeInTheDocument() + expect(screen.queryByText('vectors-buckets')).not.toBeInTheDocument() + }) + + test('platform + supported region + not enabled: shows upgrade plan', async () => { + mockStorageConfig(false) + + customRender() + + expect(await screen.findByText('buckets-upgrade-plan-vector')).toBeInTheDocument() + expect(screen.queryByText('vectors-buckets')).not.toBeInTheDocument() + }) + + test('platform + supported region + enabled: shows vector buckets', async () => { + mockStorageConfig(true) + + customRender() + + expect(await screen.findByText('vectors-buckets')).toBeInTheDocument() + }) + + test('CLI (non-platform, enabled): shows vector buckets, skips region/upgrade gates', async () => { + mockIsPlatform.value = false + // Even an unsupported region shouldn't gate off-platform + mockProject(UNAVAILABLE_REGION) + mockDeploymentMode(true) + + customRender() + + expect(await screen.findByText('vectors-buckets')).toBeInTheDocument() + expect(screen.queryByText('region-limitation')).not.toBeInTheDocument() + }) + + test('self-hosted (non-platform): renders nothing', async () => { + mockIsPlatform.value = false + mockDeploymentMode(false) + + const { container } = customRender() + + // `useDeploymentMode` defaults to CLI during its loading window, so the page + // briefly renders before resolving to self-hosted — wait for it to settle. + await waitFor(() => expect(container).toBeEmptyDOMElement()) + expect(screen.queryByText('vectors-buckets')).not.toBeInTheDocument() + }) +}) diff --git a/apps/studio/turbo.jsonc b/apps/studio/turbo.jsonc index 483593c5843..53b62b1c357 100644 --- a/apps/studio/turbo.jsonc +++ b/apps/studio/turbo.jsonc @@ -107,6 +107,8 @@ "VERCEL_GIT_COMMIT_SHA", "SNIPPETS_MANAGEMENT_FOLDER", "EDGE_FUNCTIONS_MANAGEMENT_FOLDER", + "S3_PROTOCOL_ACCESS_KEY_ID", + "S3_PROTOCOL_ACCESS_KEY_SECRET", ], "outputs": [".next/**", "!.next/cache/**", "!.next/dev/**/*"], },