feat: Add terraform federated content and data (#48010)

apps/docs/.gitignore

@@ -44,6 +44,8 @@ public/docs/
 # scripts/federated-content/fetch-federated-content.ts
 /content/guides/graphql/
 /content/guides/graphql.mdx
+/content/guides/deployment/terraform.mdx
+/content/guides/deployment/terraform/tutorial.mdx
 
 # Downloaded TypeDoc dumps under spec/reference/<lib>/<ver>/. Regenerated by
 # `cd apps/docs/spec && make download.tsdoc.v2`. Hand-authored files in the

apps/docs/app/guides/deployment/terraform/[[...slug]]/page.tsx

@@ -1,152 +1,26 @@
-import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
-import { genGuideMeta, removeRedundantH1 } from '~/features/docs/GuidesMdx.utils'
+import { GuideTemplate } from '~/features/docs/GuidesMdx.template'
+import {
+  genGuideMeta,
+  genGuidesStaticParams,
+  getGuidesMarkdown,
+} from '~/features/docs/GuidesMdx.utils'
 import { getEmptyArray } from '~/features/helpers.fn'
 import { IS_DEV } from '~/lib/constants'
-import { isValidGuideFrontmatter } from '~/lib/docs'
-import { linkTransform, UrlTransformFunction } from '~/lib/mdx/plugins/rehypeLinkTransform'
-import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
-import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle'
-import remarkPyMdownTabs from '~/lib/mdx/plugins/remarkTabs'
-import { getGitHubFileContents } from '~/lib/octokit'
-import { SerializeOptions } from '~/types/next-mdx-remote-serialize'
-import matter from 'gray-matter'
-import { notFound } from 'next/navigation'
-import rehypeSlug from 'rehype-slug'
 
-import {
-  terraformDocsBranch,
-  terraformDocsDocsDir,
-  terraformDocsOrg,
-  terraformDocsRepo,
-} from '../terraformConstants'
-
-// Each external docs page is mapped to a local page
-const pageMap = [
-  {
-    remoteFile: 'README.md',
-    meta: {
-      title: 'Terraform Provider',
-    },
-    useRoot: true,
-  },
-  {
-    slug: 'tutorial',
-    remoteFile: 'tutorial.md',
-    meta: {
-      title: 'Using the Supabase Terraform Provider',
-    },
-  },
-]
-
-interface Params {
-  slug?: string[]
-}
+type Params = { slug?: string[] }
 
 const TerraformDocs = async (props: { params: Promise<Params> }) => {
   const params = await props.params
-  const { meta, ...data } = await getContent(params)
+  const slug = ['deployment', 'terraform', ...(params.slug ?? [])]
+  const data = await getGuidesMarkdown(slug)
 
-  const options = {
-    mdxOptions: {
-      remarkPlugins: [remarkMkDocsAdmonition, remarkPyMdownTabs, [removeTitle, meta.title]],
-      rehypePlugins: [[linkTransform, urlTransform], rehypeSlug],
-    },
-  } as SerializeOptions
-
-  return <GuideTemplate mdxOptions={options} meta={meta} {...data} />
+  return <GuideTemplate {...data!} />
 }
 
-/**
- * The GitHub repo uses relative links, which don't lead to the right locations
- * in docs.
- *
- * @param url The original link, as written in the Markdown file
- * @returns The rewritten link
- */
-const urlTransform: UrlTransformFunction = (url: string) => {
-  try {
-    const placeholderHostname = 'placeholder'
-    const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`)
-
-    // Don't modify a url with a FQDN or a url that's only a hash
-    if (hostname !== placeholderHostname || pathname === '/') {
-      return url
-    }
-
-    const getBasename = (pathname: string) =>
-      pathname.endsWith('.md') ? pathname.replace(/\.md$/, '') : pathname
-    const stripLeadingPrefix = (pathname: string) => pathname.replace(/^\//, '')
-    const stripLeadingDocs = (pathname: string) => pathname.replace(/^docs\//, '')
-
-    const relativePath = stripLeadingPrefix(getBasename(pathname))
-
-    const page = pageMap.find(
-      ({ remoteFile, useRoot }) =>
-        (useRoot && `${relativePath}.md` === remoteFile) ||
-        (!useRoot && `${stripLeadingDocs(relativePath)}.md` === remoteFile)
-    )
-
-    if (page) {
-      return 'terraform' + `/${page.slug}` + hash
-    }
-
-    // If we don't have this page in our docs, link to GitHub repo
-    return `https://github.com/${terraformDocsOrg}/${terraformDocsRepo}/blob/${terraformDocsBranch}${pathname}${hash}`
-  } catch (err) {
-    console.error('Error transforming markdown URL', err)
-    return url
-  }
-}
-
-/**
- * Fetch markdown from external repo
- */
-const getContent = async ({ slug }: Params) => {
-  const [requestedSlug] = slug ?? []
-  const page = pageMap.find((page) => page.slug === requestedSlug)
-
-  if (!page) {
-    notFound()
-  }
-
-  const { meta, remoteFile, useRoot } = page
-
-  const editLink = newEditLink(
-    `${terraformDocsOrg}/${terraformDocsRepo}/blob/${terraformDocsBranch}/${useRoot ? '' : `${terraformDocsDocsDir}/`}${remoteFile}`
-  )
-
-  let rawContent = await getGitHubFileContents({
-    org: terraformDocsOrg,
-    repo: terraformDocsRepo,
-    path: useRoot ? remoteFile : `${terraformDocsDocsDir}/${remoteFile}`,
-    branch: terraformDocsBranch,
-  })
-  // Strip out HTML comments
-  rawContent = rawContent.replace(/<!--.*?-->/, '')
-  let { content, data } = matter(rawContent)
-
-  // Remove the title from the content so it isn't duplicated in the final display
-  content = removeRedundantH1(content)
-
-  Object.assign(meta, data)
-
-  if (!isValidGuideFrontmatter(meta)) {
-    throw Error('Guide frontmatter is invalid.')
-  }
-
-  return {
-    pathname:
-      `/guides/deployment/terraform${slug?.length ? `/${slug.join('/')}` : ''}` satisfies `/${string}`,
-    meta,
-    content,
-    editLink,
-  }
-}
-
-const generateStaticParams = !IS_DEV
-  ? async () => pageMap.map(({ slug }) => ({ slug: slug ? [slug] : [] }))
-  : getEmptyArray
-const generateMetadata = genGuideMeta(getContent)
+const generateStaticParams = !IS_DEV ? genGuidesStaticParams('deployment/terraform') : getEmptyArray
+const generateMetadata = genGuideMeta((params: { slug?: string[] }) =>
+  getGuidesMarkdown(['deployment', 'terraform', ...(params.slug ?? [])])
+)
 
 export default TerraformDocs
-export { generateMetadata, generateStaticParams }
+export { generateStaticParams, generateMetadata }

apps/docs/app/guides/deployment/terraform/terraformConstants.ts

@@ -1,9 +0,0 @@
-/**
- * Information on where to fetch docs content
- */
-const terraformDocsOrg = 'supabase'
-const terraformDocsRepo = 'terraform-provider-supabase'
-const terraformDocsBranch = 'v1.1.3'
-const terraformDocsDocsDir = 'docs'
-
-export { terraformDocsOrg, terraformDocsRepo, terraformDocsBranch, terraformDocsDocsDir }

apps/docs/components/TerraformProviderSchema.tsx

@@ -1,31 +1,13 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { TabPanel, Tabs } from '~/features/ui/Tabs'
+import { GENERATED_DIRECTORY } from '~/lib/docs'
 import { codeBlock } from 'common-tags'
 import { Check, PlusCircle } from 'lucide-react'
-import Link from 'next/link'
 import ReactMarkdown from 'react-markdown'
 import { Heading, Popover, PopoverContent, PopoverTrigger } from 'ui'
 import { CodeBlock } from 'ui-patterns/CodeBlock'
 
-import {
-  terraformDocsBranch,
-  terraformDocsDocsDir,
-  terraformDocsOrg,
-  terraformDocsRepo,
-} from '../terraformConstants'
-import { GuideTemplate, newEditLink } from '@/features/docs/GuidesMdx.template'
-import { genGuideMeta } from '@/features/docs/GuidesMdx.utils'
-import { TabPanel, Tabs } from '@/features/ui/Tabs'
-import { getGitHubFileContents } from '@/lib/octokit'
-
-const meta = {
-  title: 'Terraform Provider reference',
-  subtitle: 'Resources and data sources available through the Terraform Provider',
-}
-
-const generateMetadata = genGuideMeta(() => ({
-  pathname: '/guides/deployment/terraform/reference',
-  meta,
-}))
-
 function ProviderSettings({ schema }: { schema: any }) {
   const attributes = schema.block.attributes
 
@@ -346,65 +328,16 @@ function DataSources({ schema }: { schema: any }) {
   )
 }
 
-const TerraformReferencePage = async () => {
-  const { schema } = await getSchema()
-
-  const editLink = newEditLink('supabase/terraform-provider-supabase')
+export async function TerraformProviderSchema() {
+  const raw = await readFile(join(GENERATED_DIRECTORY, 'terraform.schema.json'), 'utf-8')
+  const schema = JSON.parse(raw)
+  const providerSchema = schema.provider_schemas['registry.terraform.io/supabase/supabase']
 
   return (
-    <GuideTemplate
-      meta={meta}
-      editLink={editLink}
-      pathname="/guides/deployment/terraform/reference"
-    >
-      The Terraform Provider provides access to{' '}
-      <Link
-        href="https://developer.hashicorp.com/terraform/language/resources"
-        rel="noopener noreferrer"
-      >
-        resources
-      </Link>{' '}
-      and{' '}
-      <Link
-        href="https://developer.hashicorp.com/terraform/language/data-sources"
-        rel="noreferrer noopener"
-      >
-        data sources
-      </Link>
-      . Resources are infrastructure objects, such as a Supabase project, that you can declaratively
-      configure. Data sources are sources of information about your Supabase instances.
-      <ProviderSettings
-        schema={schema.provider_schemas['registry.terraform.io/supabase/supabase'].provider}
-      />
-      <Resources
-        schema={schema.provider_schemas['registry.terraform.io/supabase/supabase'].resource_schemas}
-      />
-      <DataSources
-        schema={
-          schema.provider_schemas['registry.terraform.io/supabase/supabase'].data_source_schemas
-        }
-      />
-    </GuideTemplate>
+    <>
+      <ProviderSettings schema={providerSchema.provider} />
+      <Resources schema={providerSchema.resource_schemas} />
+      <DataSources schema={providerSchema.data_source_schemas} />
+    </>
   )
 }
-
-/**
- * Fetch JSON schema from external repo
- */
-const getSchema = async () => {
-  const schema = JSON.parse(
-    await getGitHubFileContents({
-      org: terraformDocsOrg,
-      repo: terraformDocsRepo,
-      path: `${terraformDocsDocsDir}/schema.json`,
-      branch: terraformDocsBranch,
-    })
-  )
-
-  return {
-    schema,
-  }
-}
-
-export default TerraformReferencePage
-export { generateMetadata }

apps/docs/content/guides/deployment/terraform/reference.mdx

@@ -0,0 +1,8 @@
+---
+title: Terraform Provider reference
+subtitle: Resources and data sources available through the Terraform Provider
+---
+
+The Terraform Provider provides access to [resources](https://developer.hashicorp.com/terraform/language/resources) and [data sources](https://developer.hashicorp.com/terraform/language/data-sources). Resources are infrastructure objects, such as a Supabase project, that you can declaratively configure. Data sources are sources of information about your Supabase instances.
+
+<TerraformProviderSchema />

apps/docs/features/docs/MdxBase.shared.tsx

@@ -19,6 +19,7 @@ import { RealtimeLimitsEstimator } from '~/components/RealtimeLimitsEstimator'
 import { RegionsList, SmartRegionsList } from '~/components/RegionsList'
 import { SharedData } from '~/components/SharedData'
 import StepHikeCompact from '~/components/StepHikeCompact'
+import { TerraformProviderSchema } from '~/components/TerraformProviderSchema'
 import { CodeSampleDummy, CodeSampleWrapper } from '~/features/directives/CodeSample.client'
 import { NamedCodeBlock } from '~/features/directives/CodeTabs.components'
 import { MdxAnchor } from '~/features/docs/MdxAnchor'
@@ -111,6 +112,7 @@ const components = {
   StepHikeCompact,
   Tabs,
   TabPanel,
+  TerraformProviderSchema,
   InfoTooltip,
   a: MdxAnchor,
   h2: (props: ComponentPropsWithoutRef<'h2'>) => (

apps/docs/internals/generate-guides-markdown.ts

@@ -34,6 +34,7 @@ import { RegionsList, SmartRegionsList } from './markdown-schema/RegionsList'
 import { SharedData } from './markdown-schema/SharedData'
 import { StepHike } from './markdown-schema/StepHike'
 import { TabPanel } from './markdown-schema/TabPanel'
+import { TerraformProviderSchema } from './markdown-schema/TerraformProviderSchema'
 import {
   collectMarkdownSources,
   type FrontmatterFormat,
@@ -187,6 +188,7 @@ const SCHEMA: ComponentSchema = {
   ContentListings,
   NavData,
   SharedData,
+  TerraformProviderSchema,
 }
 
 function parseFrontmatter(raw: string, frontmatter: FrontmatterFormat) {

apps/docs/internals/markdown-schema/TerraformProviderSchema.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from 'node:fs'
+import path from 'node:path'
+
+const SCHEMA_PATH = path.join(process.cwd(), 'features/docs/generated/terraform.schema.json')
+
+function attributesTable(attributes: Record<string, any>, extraColumns: string[]): string {
+  const columns = ['Description', ...extraColumns]
+  const header = `| Attribute | ${columns.join(' | ')} |`
+  const divider = `| --- | ${columns.map(() => '---').join(' | ')} |`
+  const rows = Object.entries(attributes).map(
+    ([name, attribute]) =>
+      `| \`${name}\` | ${columns.map((column) => String(attribute[column.toLowerCase()] ?? '')).join(' | ')} |`
+  )
+  return [header, divider, ...rows].join('\n')
+}
+
+export const TerraformProviderSchema = (): string => {
+  const schema = JSON.parse(readFileSync(SCHEMA_PATH, 'utf-8'))
+  const provider = schema.provider_schemas['registry.terraform.io/supabase/supabase']
+
+  const resources = Object.entries(provider.resource_schemas)
+    .map(
+      ([name, resource]: [string, any]) =>
+        `### ${name}\n\n${attributesTable(resource.block.attributes, ['Type', 'Required', 'Optional'])}`
+    )
+    .join('\n\n')
+
+  const dataSources = Object.entries(provider.data_source_schemas)
+    .map(
+      ([name, dataSource]: [string, any]) =>
+        `### ${name}\n\n${attributesTable(dataSource.block.attributes, ['Type', 'Required', 'Optional'])}`
+    )
+    .join('\n\n')
+
+  return [
+    `## Provider settings\n\n${attributesTable(provider.provider.block.attributes, ['Type', 'Optional', 'Sensitive'])}`,
+    `## Resources\n\n${resources}`,
+    `## Data sources\n\n${dataSources}`,
+  ].join('\n\n')
+}

apps/docs/lib/docs.ts

@@ -16,6 +16,7 @@ export const CONTENT_DIRECTORY = join(DOCS_DIRECTORY, 'content')
 export const EXAMPLES_DIRECTORY = join(DOCS_DIRECTORY, 'examples')
 export const GUIDES_DIRECTORY = join(CONTENT_DIRECTORY, 'guides')
 export const PARTIALS_DIRECTORY = join(CONTENT_DIRECTORY, '_partials')
+export const GENERATED_DIRECTORY = join(DOCS_DIRECTORY, 'features/docs/generated')
 export const REF_DOCS_DIRECTORY = join(DOCS_DIRECTORY, 'docs/ref')
 export const SPEC_DIRECTORY = join(DOCS_DIRECTORY, 'spec')
 

apps/docs/scripts/federated-content/fetch-federated-content.ts

@@ -4,7 +4,7 @@ import { mkdir, readdir, writeFile } from 'node:fs/promises'
 import { dirname, isAbsolute, join, relative } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { BASE_PATH } from '~/lib/constants'
-import { GUIDES_DIRECTORY } from '~/lib/docs'
+import { GENERATED_DIRECTORY, GUIDES_DIRECTORY } from '~/lib/docs'
 import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
 import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle'
 import remarkPyMdownTabs from '~/lib/mdx/plugins/remarkTabs'
@@ -49,6 +49,13 @@ async function loadSources(): Promise<FederatedContentSource[]> {
   )
 }
 
+/**
+ * Path of a page's remote file, relative to the repo root.
+ */
+function remotePath(source: FederatedContentSource, page: FederatedPage): string {
+  return page.useRoot ? page.remoteFile : `${source.docsDir}/${page.remoteFile}`
+}
+
 /**
  * Rewrites a link URL: pages mapped in `source.pageMap` go to their local
  * `/guides/<section>` route, everything else falls back to `externalSite`.
@@ -70,13 +77,19 @@ function transformUrl(source: FederatedContentSource, url: string): string {
           ? relative(new URL(source.externalSite).pathname, pathname)
           : pathname
     ).replace(/^\//, '')
+    const docsRelative = relativePath.replace(new RegExp(`^${source.docsDir}/`), '')
 
-    const mapped = source.pageMap.find(({ remoteFile }) => `${relativePath}.md` === remoteFile)
+    const mapped = source.pageMap.find(({ remoteFile, useRoot }) =>
+      useRoot ? `${relativePath}.md` === remoteFile : `${docsRelative}.md` === remoteFile
+    )
 
     // If we have a mapping for this page, use the mapped path; otherwise
     // link to the original docs
-    return mapped
-      ? `${BASE_PATH}/guides/${source.section}${mapped.slug ? `/${mapped.slug}` : ''}${hash}`
+    if (mapped) {
+      return `${BASE_PATH}/guides/${source.section}${mapped.slug ? `/${mapped.slug}` : ''}${hash}`
+    }
+    return source.rawFallback
+      ? `${source.externalSite}${pathname}${hash}`
       : `${source.externalSite}/${relativePath}${hash}`
   } catch (err) {
     throw Error('[DOCS] fetch-federated-content: Error transforming markdown URL', { cause: err })
@@ -87,14 +100,19 @@ async function fetchPage(source: FederatedContentSource, page: FederatedPage): P
   const raw = await getGitHubFileContents({
     org: source.org,
     repo: source.repo,
-    path: `${source.docsDir}/${page.remoteFile}`,
+    path: remotePath(source, page),
     branch: source.branch,
   })
 
   const tree = fromMarkdown(raw, PARSE_OPTIONS)
   remarkMkDocsAdmonition()(tree)
   remarkPyMdownTabs()(tree)
-  removeTitle(page.meta.title)(tree)
+  if (page.dropLeadingHeading) {
+    const [firstNode] = tree.children
+    if (firstNode?.type === 'heading' && firstNode.depth === 1) tree.children.splice(0, 1)
+  } else {
+    removeTitle(page.meta.title)(tree)
+  }
   visit(tree, ['link', 'image', 'definition'], (node: any) => {
     node.url = transformUrl(source, node.url)
   })
@@ -104,18 +122,33 @@ async function fetchPage(source: FederatedContentSource, page: FederatedPage): P
     title: page.meta.title,
     // Points the "Edit this page on GitHub" link back at the source repo
     // instead of this generated file.
-    editLink: `${source.org}/${source.repo}/blob/${source.branch}/${source.docsDir}/${page.remoteFile}`,
+    editLink: `${source.org}/${source.repo}/blob/${source.branch}/${remotePath(source, page)}`,
   }
   if (page.meta.subtitle) frontmatter.subtitle = page.meta.subtitle
 
   return matter.stringify(`${content}\n`, frontmatter)
 }
 
+async function fetchRawFile(
+  source: FederatedContentSource,
+  rawFile: NonNullable<FederatedContentSource['rawFiles']>[number]
+): Promise<void> {
+  const content = await getGitHubFileContents({
+    org: source.org,
+    repo: source.repo,
+    path: `${source.docsDir}/${rawFile.remoteFile}`,
+    branch: source.branch,
+  })
+
+  await mkdir(GENERATED_DIRECTORY, { recursive: true })
+  await writeFile(join(GENERATED_DIRECTORY, rawFile.outFile), content)
+}
+
 async function fetchSource(source: FederatedContentSource): Promise<void> {
   await mkdir(join(GUIDES_DIRECTORY, source.section), { recursive: true })
 
-  await Promise.all(
-    source.pageMap.map(async (page) => {
+  await Promise.all([
+    ...source.pageMap.map(async (page) => {
       const output = await fetchPage(source, page)
 
       const outPath = page.slug
@@ -123,8 +156,9 @@ async function fetchSource(source: FederatedContentSource): Promise<void> {
         : join(GUIDES_DIRECTORY, `${source.section}.mdx`)
 
       await writeFile(outPath, output)
-    })
-  )
+    }),
+    ...(source.rawFiles ?? []).map((rawFile) => fetchRawFile(source, rawFile)),
+  ])
 }
 
 async function fetchFederatedContent() {

apps/docs/scripts/federated-content/sources/terraform.ts

@@ -0,0 +1,32 @@
+import type { FederatedContentSource } from '../types'
+
+// We fetch these docs at build time from an external repo
+const terraform: FederatedContentSource = {
+  section: 'deployment/terraform',
+  org: 'supabase',
+  repo: 'terraform-provider-supabase',
+  branch: 'v1.1.3',
+  docsDir: 'docs',
+  externalSite: 'https://github.com/supabase/terraform-provider-supabase/blob/v1.1.3',
+  rawFallback: true,
+  pageMap: [
+    {
+      meta: {
+        title: 'Terraform Provider',
+      },
+      remoteFile: 'README.md',
+      useRoot: true,
+      dropLeadingHeading: true,
+    },
+    {
+      slug: 'tutorial',
+      meta: {
+        title: 'Using the Supabase Terraform Provider',
+      },
+      remoteFile: 'tutorial.md',
+    },
+  ],
+  rawFiles: [{ remoteFile: 'schema.json', outFile: 'terraform.schema.json' }],
+}
+
+export default terraform

apps/docs/scripts/federated-content/types.ts

@@ -10,6 +10,14 @@ export interface FederatedPage {
   }
   /** Path of the file in the remote repo, relative to `docsDir`. */
   remoteFile: string
+  /** Fetch `remoteFile` from the repo root instead of `docsDir`. */
+  useRoot?: boolean
+  /**
+   * Unconditionally drop the first heading, regardless of whether its text
+   * matches `meta.title`. Use when the curated title intentionally differs
+   * from the source file's own heading text.
+   */
+  dropLeadingHeading?: boolean
 }
 
 /**
@@ -29,5 +37,18 @@ export interface FederatedContentSource {
   docsDir: string
   /** Public site the remote docs are also published to, used to resolve unmapped links. */
   externalSite: string
+  /**
+   * Unmapped links fall back to `${externalSite}/${relativePath}${hash}`. Set
+   * this when `externalSite` is a source-controlled host (e.g. a GitHub blob
+   * URL) that needs the file extension kept, rather than a docs site that
+   * serves clean, extensionless URLs.
+   */
+  rawFallback?: boolean
   pageMap: FederatedPage[]
+  /**
+   * Non-guide files fetched verbatim (no Markdown transform), e.g. JSON data
+   * consumed by a custom MDX component. Written to
+   * `features/docs/generated/<outFile>`.
+   */
+  rawFiles?: { remoteFile: string; outFile: string }[]
 }