feat(etl): Add new flags per org (#43292)

This PR adds new feature flags for controlling etl destinations
visibility based on the org slug.

---------

Co-authored-by: Joshen Lim <joshenlimek@gmail.com>

apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx

@@ -1,6 +1,6 @@
 import { zodResolver } from '@hookform/resolvers/zod'
 import { PermissionAction } from '@supabase/shared-types/out/constants'
-import { useFlag, useParams } from 'common'
+import { useParams } from 'common'
 import { CreateAnalyticsBucketSheet } from 'components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'
 import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'
 import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'
@@ -35,6 +35,10 @@ import {
 import { Button, DialogSectionSeparator, Form_Shadcn_, SheetFooter, SheetSection } from 'ui'
 import * as z from 'zod'
 
+import {
+  useIsETLBigQueryPrivateAlpha,
+  useIsETLIcebergPrivateAlpha,
+} from '../../useIsETLPrivateAlpha'
 import { DestinationType } from '../DestinationPanel.types'
 import { AdvancedSettings } from './AdvancedSettings'
 import { CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
@@ -75,8 +79,8 @@ export const DestinationForm = ({
   const { ref: projectRef } = useParams()
   const { setRequestStatus } = usePipelineRequestStatus()
 
-  const etlEnableBigQuery = useFlag('etlEnableBigQuery')
-  const etlEnableIceberg = useFlag('etlEnableIceberg')
+  const etlEnableBigQuery = useIsETLBigQueryPrivateAlpha()
+  const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
   const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
 
   const [isFormInteracting, setIsFormInteracting] = useState(false)

apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx

@@ -7,6 +7,7 @@ import { useEffect } from 'react'
 import { toast } from 'sonner'
 import {
   Button,
+  cn,
   DialogSectionSeparator,
   Sheet,
   SheetContent,
@@ -14,7 +15,6 @@ import {
   SheetHeader,
   SheetSection,
   SheetTitle,
-  cn,
 } from 'ui'
 
 import { EnableReplicationCallout } from '../EnableReplicationCallout'

apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx

@@ -1,18 +1,17 @@
 import { useFlag } from 'common'
 import { AnalyticsBucket, BigQuery, Database } from 'icons'
 import { parseAsInteger, parseAsStringEnum, useQueryState } from 'nuqs'
-import { Badge, RadioGroupStacked, RadioGroupStackedItem, cn } from 'ui'
+import { Badge, cn, RadioGroupStacked, RadioGroupStackedItem } from 'ui'
 
 import { useDestinationInformation } from '../useDestinationInformation'
-import { useIsETLPrivateAlpha } from '../useIsETLPrivateAlpha'
+import { useIsETLBigQueryPrivateAlpha, useIsETLIcebergPrivateAlpha } from '../useIsETLPrivateAlpha'
 import { DestinationType } from './DestinationPanel.types'
 import { InlineLink } from '@/components/ui/InlineLink'
 
 export const DestinationTypeSelection = () => {
-  const enablePgReplicate = useIsETLPrivateAlpha()
   const unifiedReplication = useFlag('unifiedReplication')
-  const etlEnableBigQuery = useFlag('etlEnableBigQuery')
-  const etlEnableIceberg = useFlag('etlEnableIceberg')
+  const etlEnableBigQuery = useIsETLBigQueryPrivateAlpha()
+  const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
 
   const numberOfTypes = [unifiedReplication, etlEnableBigQuery, etlEnableIceberg].filter(
     Boolean
@@ -52,8 +51,8 @@ export const DestinationTypeSelection = () => {
         value={destinationType}
         onValueChange={(value) => setDestinationType(value as DestinationType)}
         className={cn(
-          'grid [&>button>div]:py-4',
-          numberOfTypes === 3 ? 'grid-cols-3' : numberOfTypes === 2 ? 'grid-cols-2' : 'grid-cols-1',
+          'grid [&>button>div]:py-4 grid-cols-3',
+          numberOfTypes === 3 && !editMode ? 'grid-cols-3' : 'grid-cols-2',
           '[&>button:first-of-type]:rounded-none [&>button:last-of-type]:rounded-none',
           '[&>button:first-of-type]:!rounded-l-lg [&>button:last-of-type]:!rounded-r-lg'
         )}
@@ -121,7 +120,7 @@ export const DestinationTypeSelection = () => {
         )}
       </RadioGroupStacked>
 
-      {destinationType !== 'Read Replica' && enablePgReplicate && (
+      {destinationType !== 'Read Replica' && (
         <p className="mt-3 text-sm text-foreground-light">
           Replication is in alpha. Expect rapid changes and possible breaking updates.{' '}
           <InlineLink href="https://github.com/orgs/supabase/discussions/39416">

apps/studio/components/interfaces/Database/Replication/useIsETLPrivateAlpha.ts

@@ -2,18 +2,37 @@ import { useFlag } from 'common'
 import { useSelectedOrganizationQuery } from 'hooks/misc/useSelectedOrganization'
 
 /**
- * Organization level opt in for ETL private alpha
+ * Organization level opt in for ETL private alpha, there's 2 flags we're using which controls
+ * the individual destination types. Access to the ETL UI (`useIsETLPrivateAlpha`) will just
+ * check if the org has access to at least one of the destination types
  */
-export const useIsETLPrivateAlpha = () => {
+const useIsCurrentOrgInFlagList = (flag: string) => {
+  const flagValue = useFlag(flag)
   const { data: organization } = useSelectedOrganizationQuery()
 
-  const etlPrivateAlpha = useFlag('etlPrivateAlpha')
-  const privateAlphaOrgSlugs =
-    typeof etlPrivateAlpha === 'string'
-      ? (etlPrivateAlpha as string).split(',').map((x) => x.trim())
+  const allowedOrgSlugs =
+    typeof flagValue === 'string'
+      ? (flagValue as string).split(',').map((x: string) => x.trim())
       : []
 
-  const etlShowForAllProjects = useFlag('etlPrivateAlphaOverride')
+  // [Joshen] Override for to enable for all organizations by setting the flag value as `all`
+  if (allowedOrgSlugs.includes('all')) return true
 
-  return etlShowForAllProjects || privateAlphaOrgSlugs.includes(organization?.slug ?? '')
+  // [Joshen] Otherwise fallback to checking against org slug
+  return allowedOrgSlugs.includes(organization?.slug ?? '')
+}
+
+export const useIsETLBigQueryPrivateAlpha = () => {
+  return useIsCurrentOrgInFlagList('etlEnableBigQueryPrivateAlpha')
+}
+
+export const useIsETLIcebergPrivateAlpha = () => {
+  return useIsCurrentOrgInFlagList('etlEnableIcebergPrivateAlpha')
+}
+
+export const useIsETLPrivateAlpha = () => {
+  const hasAccessToETLBigQuery = useIsCurrentOrgInFlagList('etlEnableBigQueryPrivateAlpha')
+  const hasAccessToETLIceberg = useIsCurrentOrgInFlagList('etlEnableIcebergPrivateAlpha')
+
+  return hasAccessToETLBigQuery || hasAccessToETLIceberg
 }

apps/studio/components/layouts/DatabaseLayout/DatabaseLayout.tsx

@@ -1,60 +1,21 @@
+import { ProductMenu } from 'components/ui/ProductMenu'
+import { withAuth } from 'hooks/misc/withAuth'
 import { useRouter } from 'next/router'
 import { PropsWithChildren } from 'react'
 
-import { useIsColumnLevelPrivilegesEnabled } from 'components/interfaces/App/FeaturePreview/FeaturePreviewContext'
-import { useIsETLPrivateAlpha } from 'components/interfaces/Database/Replication/useIsETLPrivateAlpha'
-import { ProductMenu } from 'components/ui/ProductMenu'
-import { useDatabaseExtensionsQuery } from 'data/database-extensions/database-extensions-query'
-import { useProjectAddonsQuery } from 'data/subscriptions/project-addons-query'
-import { useIsFeatureEnabled } from 'hooks/misc/useIsFeatureEnabled'
-import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
-import { withAuth } from 'hooks/misc/withAuth'
 import { ProjectLayout } from '../ProjectLayout'
-import { generateDatabaseMenu } from './DatabaseMenu.utils'
+import { useGenerateDatabaseMenu } from './DatabaseMenu.utils'
 
 export interface DatabaseLayoutProps {
   title?: string
 }
 
 const DatabaseProductMenu = () => {
-  const { data: project } = useSelectedProjectQuery()
-
   const router = useRouter()
   const page = router.pathname.split('/')[4]
+  const menu = useGenerateDatabaseMenu()
 
-  const { data } = useDatabaseExtensionsQuery({
-    projectRef: project?.ref,
-    connectionString: project?.connectionString,
-  })
-  const { data: addons } = useProjectAddonsQuery({ projectRef: project?.ref })
-
-  const pgNetExtensionExists = (data ?? []).find((ext) => ext.name === 'pg_net') !== undefined
-  const pitrEnabled = addons?.selected_addons.find((addon) => addon.type === 'pitr') !== undefined
-  const columnLevelPrivileges = useIsColumnLevelPrivilegesEnabled()
-  const enablePgReplicate = useIsETLPrivateAlpha()
-
-  const {
-    databaseReplication: showPgReplicate,
-    databaseRoles: showRoles,
-    integrationsWrappers: showWrappers,
-  } = useIsFeatureEnabled(['database:replication', 'database:roles', 'integrations:wrappers'])
-
-  return (
-    <>
-      <ProductMenu
-        page={page}
-        menu={generateDatabaseMenu(project, {
-          pgNetExtensionExists,
-          pitrEnabled,
-          columnLevelPrivileges,
-          showPgReplicate,
-          enablePgReplicate,
-          showRoles,
-          showWrappers,
-        })}
-      />
-    </>
-  )
+  return <ProductMenu page={page} menu={menu} />
 }
 
 const DatabaseLayout = ({ children }: PropsWithChildren<DatabaseLayoutProps>) => {

apps/studio/components/layouts/DatabaseLayout/DatabaseMenu.utils.tsx

@@ -1,163 +1,104 @@
+import { useParams } from 'common'
+import type {
+  ProductMenuGroup,
+  ProductMenuGroupItem,
+} from 'components/ui/ProductMenu/ProductMenu.types'
+import { IS_PLATFORM } from 'lib/constants'
 import { ArrowUpRight } from 'lucide-react'
 
-import type { ProductMenuGroup } from 'components/ui/ProductMenu/ProductMenu.types'
-import type { Project } from 'data/projects/project-detail-query'
-import { IS_PLATFORM } from 'lib/constants'
+import { useIsColumnLevelPrivilegesEnabled } from '@/components/interfaces/App/FeaturePreview/FeaturePreviewContext'
+import { useIsETLPrivateAlpha } from '@/components/interfaces/Database/Replication/useIsETLPrivateAlpha'
+import { useDatabaseExtensionsQuery } from '@/data/database-extensions/database-extensions-query'
+import { useProjectAddonsQuery } from '@/data/subscriptions/project-addons-query'
+import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled'
+import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject'
+
+const ExternalLinkIcon = <ArrowUpRight strokeWidth={1} className="h-4 w-4" />
+
+export const useGenerateDatabaseMenu = (): ProductMenuGroup[] => {
+  const { ref } = useParams()
+  const { data: project } = useSelectedProjectQuery()
 
-export const generateDatabaseMenu = (
-  project?: Project,
-  flags?: {
-    pgNetExtensionExists: boolean
-    pitrEnabled: boolean
-    columnLevelPrivileges: boolean
-    showPgReplicate: boolean
-    enablePgReplicate: boolean
-    showRoles: boolean
-    showWrappers: boolean
-  }
-): ProductMenuGroup[] => {
-  const ref = project?.ref ?? 'default'
   const {
-    pgNetExtensionExists,
-    pitrEnabled,
-    columnLevelPrivileges,
-    showPgReplicate,
-    enablePgReplicate,
-    showRoles,
-    showWrappers,
-  } = flags || {}
+    databaseReplication: showPgReplicate,
+    databaseRoles: showRoles,
+    integrationsWrappers: showWrappers,
+  } = useIsFeatureEnabled(['database:replication', 'database:roles', 'integrations:wrappers'])
+
+  const { data } = useDatabaseExtensionsQuery({
+    projectRef: project?.ref,
+    connectionString: project?.connectionString,
+  })
+  const { data: addons } = useProjectAddonsQuery({ projectRef: project?.ref })
+
+  const pgNetExtensionExists = (data ?? []).some((ext) => ext.name === 'pg_net')
+  const pitrEnabled = addons?.selected_addons.some((addon) => addon.type === 'pitr') ?? false
+  const columnLevelPrivileges = useIsColumnLevelPrivilegesEnabled()
+  const enablePgReplicate = useIsETLPrivateAlpha()
+
+  const getDatabaseURL = (path: string) => `/project/${ref}/database/${path}`
 
   return [
     {
       title: 'Database Management',
       items: [
-        {
-          name: 'Schema Visualizer',
-          key: 'schemas',
-          url: `/project/${ref}/database/schemas`,
-          items: [],
-        },
-        { name: 'Tables', key: 'tables', url: `/project/${ref}/database/tables`, items: [] },
-        {
-          name: 'Functions',
-          key: 'functions',
-          url: `/project/${ref}/database/functions`,
-          items: [],
-        },
-        {
-          name: 'Triggers',
-          key: 'triggers',
-          url: `/project/${ref}/database/triggers/data`,
-          items: [],
-        },
-        {
-          name: 'Enumerated Types',
-          key: 'types',
-          url: `/project/${ref}/database/types`,
-
-          items: [],
-        },
-        {
-          name: 'Extensions',
-          key: 'extensions',
-          url: `/project/${ref}/database/extensions`,
-          items: [],
-        },
-        {
-          name: 'Indexes',
-          key: 'indexes',
-          url: `/project/${ref}/database/indexes`,
-          items: [],
-        },
-        {
-          name: 'Publications',
-          key: 'publications',
-          url: `/project/${ref}/database/publications`,
-          items: [],
-        },
+        { name: 'Schema Visualizer', key: 'schemas', url: getDatabaseURL('schemas') },
+        { name: 'Tables', key: 'tables', url: getDatabaseURL('tables') },
+        { name: 'Functions', key: 'functions', url: getDatabaseURL('functions') },
+        { name: 'Triggers', key: 'triggers', url: getDatabaseURL('triggers/data') },
+        { name: 'Enumerated Types', key: 'types', url: getDatabaseURL('types') },
+        { name: 'Extensions', key: 'extensions', url: getDatabaseURL('extensions') },
+        { name: 'Indexes', key: 'indexes', url: getDatabaseURL('indexes') },
+        { name: 'Publications', key: 'publications', url: getDatabaseURL('publications') },
       ],
     },
     {
       title: 'Configuration',
       items: [
-        ...(showRoles
-          ? [{ name: 'Roles', key: 'roles', url: `/project/${ref}/database/roles`, items: [] }]
-          : []),
-        ...(columnLevelPrivileges
-          ? [
-              {
-                name: 'Column Privileges',
-                key: 'column-privileges',
-                url: `/project/${ref}/database/column-privileges`,
-                items: [],
-              },
-            ]
-          : []),
+        showRoles && { name: 'Roles', key: 'roles', url: getDatabaseURL('roles') },
+        columnLevelPrivileges && {
+          name: 'Column Privileges',
+          key: 'column-privileges',
+          url: getDatabaseURL('column-privileges'),
+        },
         {
           name: 'Policies',
           key: 'policies',
           url: `/project/${ref}/auth/policies`,
-          rightIcon: <ArrowUpRight strokeWidth={1} className="h-4 w-4" />,
-          items: [],
+          rightIcon: ExternalLinkIcon,
         },
-        { name: 'Settings', key: 'settings', url: `/project/${ref}/database/settings`, items: [] },
-      ],
+        { name: 'Settings', key: 'settings', url: getDatabaseURL('settings') },
+      ].filter(Boolean) as ProductMenuGroupItem[],
     },
     {
       title: 'Platform',
       items: [
-        ...(IS_PLATFORM && showPgReplicate
-          ? [
-              {
-                name: 'Replication',
-                key: 'replication',
-                url: `/project/${ref}/database/replication`,
-                label: enablePgReplicate ? 'New' : undefined,
-                items: [],
-              },
-            ]
-          : []),
-        ...(IS_PLATFORM
-          ? [
-              {
-                name: 'Backups',
-                key: 'backups',
-                url: pitrEnabled
-                  ? `/project/${ref}/database/backups/pitr`
-                  : `/project/${ref}/database/backups/scheduled`,
-                items: [],
-              },
-            ]
-          : []),
-        {
-          name: 'Migrations',
-          key: 'migrations',
-          url: `/project/${ref}/database/migrations`,
-          items: [],
+        IS_PLATFORM &&
+          showPgReplicate && {
+            name: 'Replication',
+            key: 'replication',
+            url: getDatabaseURL('replication'),
+            label: enablePgReplicate ? 'New' : undefined,
+          },
+        IS_PLATFORM && {
+          name: 'Backups',
+          key: 'backups',
+          url: pitrEnabled ? getDatabaseURL('backups/pitr') : getDatabaseURL('backups/scheduled'),
         },
-        ...(showWrappers
-          ? [
-              {
-                name: 'Wrappers',
-                key: 'wrappers',
-                url: `/project/${ref}/integrations?category=wrapper`,
-                rightIcon: <ArrowUpRight strokeWidth={1} className="h-4 w-4" />,
-                items: [],
-              },
-            ]
-          : []),
-        ...(!!pgNetExtensionExists
-          ? [
-              {
-                name: 'Webhooks',
-                key: 'hooks',
-                url: `/project/${ref}/integrations/webhooks/overview`,
-                rightIcon: <ArrowUpRight strokeWidth={1} className="h-4 w-4" />,
-                items: [],
-              },
-            ]
-          : []),
-      ],
+        { name: 'Migrations', key: 'migrations', url: getDatabaseURL('migrations') },
+        showWrappers && {
+          name: 'Wrappers',
+          key: 'wrappers',
+          url: `/project/${ref}/integrations?category=wrapper`,
+          rightIcon: ExternalLinkIcon,
+        },
+        pgNetExtensionExists && {
+          name: 'Webhooks',
+          key: 'hooks',
+          url: `/project/${ref}/integrations/webhooks/overview`,
+          rightIcon: ExternalLinkIcon,
+        },
+      ].filter(Boolean) as ProductMenuGroupItem[],
     },
     {
       title: 'Tools',
@@ -166,22 +107,19 @@ export const generateDatabaseMenu = (
           name: 'Security Advisor',
           key: 'security-advisor',
           url: `/project/${ref}/advisors/security`,
-          rightIcon: <ArrowUpRight strokeWidth={1} className="h-4 w-4" />,
-          items: [],
+          rightIcon: ExternalLinkIcon,
         },
         {
           name: 'Performance Advisor',
           key: 'performance-advisor',
           url: `/project/${ref}/advisors/performance`,
-          rightIcon: <ArrowUpRight strokeWidth={1} className="h-4 w-4" />,
-          items: [],
+          rightIcon: ExternalLinkIcon,
         },
         {
           name: 'Query Performance',
           key: 'query-performance',
           url: `/project/${ref}/observability/query-performance`,
-          rightIcon: <ArrowUpRight strokeWidth={1} className="h-4 w-4" />,
-          items: [],
+          rightIcon: ExternalLinkIcon,
         },
       ],
     },

apps/studio/components/layouts/ProjectLayout/NavigationBar/NavigationBar.utils.tsx

@@ -1,6 +1,4 @@
 import { ICON_SIZE, ICON_STROKE_WIDTH } from 'components/interfaces/Sidebar'
-import { generateAuthMenu } from 'components/layouts/AuthLayout/AuthLayout.utils'
-import { generateDatabaseMenu } from 'components/layouts/DatabaseLayout/DatabaseMenu.utils'
 import { generateSettingsMenu } from 'components/layouts/ProjectSettingsLayout/SettingsMenu.utils'
 import type { Route } from 'components/ui/ui.types'
 import { EditorIndexPageLink } from 'data/prefetchers/project.$ref.editor'
@@ -54,9 +52,6 @@ export const generateProductRoutes = (
   const realtimeEnabled = features?.realtime ?? true
   const authOverviewPageEnabled = features?.authOverviewPage ?? false
 
-  const databaseMenu = generateDatabaseMenu(project)
-  const authMenu = generateAuthMenu(ref as string)
-
   return [
     {
       key: 'database',
@@ -70,7 +65,6 @@ export const generateProductRoutes = (
           : isProjectActive
             ? `/project/${ref}/database/schemas`
             : `/project/${ref}/database/backups/scheduled`),
-      items: databaseMenu,
     },
     ...(authEnabled
       ? [
@@ -86,7 +80,6 @@ export const generateProductRoutes = (
                 : authOverviewPageEnabled
                   ? `/project/${ref}/auth/overview`
                   : `/project/${ref}/auth/users`),
-            items: authMenu,
           },
         ]
       : []),