feat: Add federated content script. Resolve graphql routes. (#47934)

apps/docs/.gitignore

@@ -40,6 +40,11 @@ public/docs/
 # Generated reference content (built by scripts/build-reference-content.ts)
 /content/reference/
 
+# Federated guide content, fetched from an external repo by
+# scripts/federated-content/fetch-federated-content.ts
+/content/guides/graphql/
+/content/guides/graphql.mdx
+
 # Downloaded TypeDoc dumps under spec/reference/<lib>/<ver>/. Regenerated by
 # `cd apps/docs/spec && make download.tsdoc.v2`. Hand-authored files in the
 # same folders (config.json, partials/) stay tracked.

apps/docs/app/guides/graphql/[[...slug]]/page.tsx

@@ -1,198 +1,26 @@
-import { isAbsolute, relative } from 'path'
-import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
-import { genGuideMeta } from '~/features/docs/GuidesMdx.utils'
+import { GuideTemplate } from '~/features/docs/GuidesMdx.template'
+import {
+  genGuideMeta,
+  genGuidesStaticParams,
+  getGuidesMarkdown,
+} from '~/features/docs/GuidesMdx.utils'
 import { getEmptyArray } from '~/features/helpers.fn'
 import { IS_DEV } from '~/lib/constants'
-import { linkTransform, UrlTransformFunction } from '~/lib/mdx/plugins/rehypeLinkTransform'
-import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
-import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle'
-import remarkPyMdownTabs from '~/lib/mdx/plugins/remarkTabs'
-import { getGitHubFileContents } from '~/lib/octokit'
-import { SerializeOptions } from '~/types/next-mdx-remote-serialize'
-import { notFound } from 'next/navigation'
-import rehypeSlug from 'rehype-slug'
 
-// We fetch these docs at build time from an external repo
-const org = 'supabase'
-const repo = 'pg_graphql'
-const branch = 'master'
-const docsDir = 'docs'
-const externalSite = 'https://supabase.github.io/pg_graphql'
-
-// Each external docs page is mapped to a local page
-const pageMap = [
-  {
-    meta: {
-      id: 'graphql-overview',
-      title: 'GraphQL',
-      subtitle: 'Autogenerated GraphQL APIs with Postgres.',
-    },
-    remoteFile: 'supabase.md',
-  },
-  {
-    slug: 'api',
-    meta: {
-      id: 'graphql-api',
-      title: 'GraphQL API',
-      subtitle: 'Understanding the core concepts of the GraphQL API.',
-    },
-    remoteFile: 'api.md',
-  },
-  {
-    slug: 'views',
-    meta: {
-      id: 'graphql-views',
-      title: 'Views',
-      subtitle: 'Using Postgres Views with GraphQL.',
-    },
-    remoteFile: 'views.md',
-  },
-  {
-    slug: 'functions',
-    meta: {
-      id: 'graphql-functions',
-      title: 'Functions',
-      subtitle: 'Using Postgres Functions with GraphQL.',
-    },
-    remoteFile: 'functions.md',
-  },
-  {
-    slug: 'computed-fields',
-    meta: {
-      id: 'graphql-computed-fields',
-      title: 'Computed Fields',
-      subtitle: 'Using Postgres Computed Fields with GraphQL.',
-    },
-    remoteFile: 'computed_fields.md',
-  },
-  {
-    slug: 'configuration',
-    meta: {
-      id: 'graphql-configuration',
-      title: 'Configuration & Customization',
-      subtitle: 'Extra configuration options can be set on SQL entities using comment directives.',
-    },
-    remoteFile: 'configuration.md',
-  },
-  {
-    slug: 'security',
-    meta: {
-      id: 'graphql-security',
-      title: 'Security',
-      subtitle: 'Securing your GraphQL API.',
-    },
-    remoteFile: 'security.md',
-  },
-  {
-    slug: 'with-apollo',
-    meta: {
-      id: 'graphql-with-apollo',
-      title: 'With Apollo',
-      subtitle: 'Using pg_grapqhl with Apollo.',
-    },
-    remoteFile: 'usage_with_apollo.md',
-  },
-  {
-    slug: 'with-relay',
-    meta: {
-      id: 'graphql-with-relay',
-      title: 'With Relay',
-      subtitle: 'Using pg_grapqhl with Relay.',
-    },
-    remoteFile: 'usage_with_relay.md',
-  },
-]
-
-interface Params {
-  slug?: string[]
-}
+type Params = { slug?: string[] }
 
 const PGGraphQLDocs = async (props: { params: Promise<Params> }) => {
   const params = await props.params
-  const { meta, ...data } = await getContent(params)
+  const slug = ['graphql', ...(params.slug ?? [])]
+  const data = await getGuidesMarkdown(slug)
 
-  const options = {
-    mdxOptions: {
-      remarkPlugins: [remarkMkDocsAdmonition, remarkPyMdownTabs, [removeTitle, meta.title]],
-      rehypePlugins: [[linkTransform, urlTransform], rehypeSlug],
-    },
-  } as SerializeOptions
-
-  return <GuideTemplate mdxOptions={options} meta={meta} {...data} />
+  return <GuideTemplate {...data!} />
 }
 
-/**
- * Fetch markdown from external repo and transform links
- */
-const getContent = async ({ slug }: Params) => {
-  const page = pageMap.find((page) => page.slug === slug?.at(0))
-
-  if (!page) {
-    notFound()
-  }
-
-  const { remoteFile, meta } = page
-
-  const editLink = newEditLink(`${org}/${repo}/blob/${branch}/${docsDir}/${remoteFile}`)
-
-  const content = await getGitHubFileContents({
-    org,
-    repo,
-    path: `${docsDir}/${remoteFile}`,
-    branch,
-  })
-
-  return {
-    pathname: `/guides/graphql${slug?.length ? `/${slug.join('/')}` : ''}` satisfies `/${string}`,
-    meta,
-    content,
-    editLink,
-  }
-}
-
-const urlTransform: UrlTransformFunction = (url) => {
-  try {
-    const externalSiteUrl = new URL(externalSite)
-
-    const placeholderHostname = 'placeholder'
-    const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`)
-
-    // Don't modify a url with a FQDN or a url that's only a hash
-    if (hostname !== placeholderHostname || pathname === '/') {
-      return url
-    }
-
-    const getRelativePath = () => {
-      if (pathname.endsWith('.md')) {
-        return pathname.replace(/\.md$/, '')
-      }
-      if (isAbsolute(url)) {
-        return relative(externalSiteUrl.pathname, pathname)
-      }
-      return pathname
-    }
-
-    const relativePath = getRelativePath().replace(/^\//, '')
-
-    const page = pageMap.find(({ remoteFile }) => `${relativePath}.md` === remoteFile)
-
-    // If we have a mapping for this page, use the mapped path
-    if (page) {
-      return '/docs/guides/graphql/' + page.slug + hash
-    }
-
-    // If we don't have this page in our docs, link to original docs
-    return `${externalSite}/${relativePath}${hash}`
-  } catch (err) {
-    console.error('Error transforming markdown URL', err)
-    return url
-  }
-}
-
-const generateStaticParams = !IS_DEV
-  ? async () => pageMap.map(({ slug }) => ({ slug: slug ? [slug] : [] }))
-  : getEmptyArray
-const generateMetadata = genGuideMeta(getContent)
+const generateStaticParams = !IS_DEV ? genGuidesStaticParams('graphql') : getEmptyArray
+const generateMetadata = genGuideMeta((params: { slug?: string[] }) =>
+  getGuidesMarkdown(['graphql', ...(params.slug ?? [])])
+)
 
 export default PGGraphQLDocs
-export { generateMetadata, generateStaticParams }
+export { generateStaticParams, generateMetadata }

apps/docs/features/docs/GuidesMdx.utils.tsx

@@ -30,7 +30,7 @@ const PUBLISHED_SECTIONS = [
   'deployment',
   'functions',
   'getting-started',
-  // 'graphql', -- technically published, but completely federated
+  'graphql',
   'integrations',
   'local-development',
   'platform',
@@ -77,13 +77,14 @@ const getGuidesMarkdownInternal = async (slug: string[]) => {
       throw Error(`Type of frontmatter is not valid for path: ${fullPath}`)
     }
 
+    const { editLink: editLinkOverride, ...restMeta } = meta
     const editLink = newEditLink(
-      `supabase/supabase/blob/master/apps/docs/content/guides/${relPath}.mdx`
+      editLinkOverride ?? `supabase/supabase/blob/master/apps/docs/content/guides/${relPath}.mdx`
     )
 
     return {
       pathname: `/guides/${slug.join('/')}` satisfies `/${string}`,
-      meta,
+      meta: restMeta,
       content,
       editLink,
     }

apps/docs/lib/docs.ts

@@ -28,6 +28,12 @@ export type GuideFrontmatter = {
   /** @deprecated */
   hide_table_of_contents?: boolean
   tocVideo?: string
+  /**
+   * Overrides the "Edit this page on GitHub" link. Used for federated
+   * content, whose source of truth lives in an external repo rather than
+   * this generated file.
+   */
+  editLink?: string
 }
 
 /**
@@ -64,6 +70,9 @@ export function isValidGuideFrontmatter(obj: object): obj is GuideFrontmatter {
   if ('tocVideo' in obj && typeof obj.tocVideo !== 'string') {
     throw Error(`Invalid guide frontmatter: tocVideo must be a string. Received ${obj.tocVideo}`)
   }
+  if ('editLink' in obj && typeof obj.editLink !== 'string') {
+    throw Error(`Invalid guide frontmatter: editLink must be a string. Received: ${obj.editLink}`)
+  }
   return true
 }
 

apps/docs/package.json

@@ -7,6 +7,7 @@
     "build": "next build",
     "build:analyze": "ANALYZE=true next build",
     "build:guides-markdown": "tsx ./internals/generate-guides-markdown.ts",
+    "build:federated-content": "tsx --conditions=react-server ./scripts/federated-content/fetch-federated-content.ts",
     "prebuild:reference-markdown": "pnpm run codegen:references",
     "build:reference-markdown": "tsx ./internals/generate-reference-markdown.ts",
     "build:markdown": "pnpm build:guides-markdown && pnpm build:reference-markdown",
@@ -35,8 +36,8 @@
     "lint": "eslint .",
     "lint:mdx": "supa-mdx-lint content --config ../../supa-mdx-lint.config.toml",
     "postbuild": "pnpm run build:sitemap && ./../../scripts/upload-static-assets.sh",
-    "prebuild": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm run build:markdown && pnpm run build:gz-archive",
-    "predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm run build:markdown",
+    "prebuild": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm build:federated-content && pnpm run build:markdown && pnpm run build:gz-archive",
+    "predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm build:federated-content && pnpm run build:markdown",
     "preembeddings": "pnpm run codegen:references",
     "preinstall": "npx only-allow pnpm",
     "presync": "pnpm run codegen:graphql",

apps/docs/scripts/federated-content/fetch-federated-content.ts

@@ -0,0 +1,143 @@
+import '../utils/dotenv'
+
+import { mkdir, readdir, writeFile } from 'node:fs/promises'
+import { dirname, isAbsolute, join, relative } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { BASE_PATH } from '~/lib/constants'
+import { GUIDES_DIRECTORY } from '~/lib/docs'
+import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
+import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle'
+import remarkPyMdownTabs from '~/lib/mdx/plugins/remarkTabs'
+import { getGitHubFileContents } from '~/lib/octokit'
+import matter from 'gray-matter'
+import { fromMarkdown } from 'mdast-util-from-markdown'
+import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm'
+import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx'
+import { toMarkdown } from 'mdast-util-to-markdown'
+import { gfm } from 'micromark-extension-gfm'
+import { mdxjs } from 'micromark-extension-mdxjs'
+import { visit } from 'unist-util-visit'
+
+import type { FederatedContentSource, FederatedPage } from './types'
+
+const SOURCES_DIR = join(dirname(fileURLToPath(import.meta.url)), 'sources')
+
+const PARSE_OPTIONS = {
+  extensions: [mdxjs(), gfm()],
+  mdastExtensions: [mdxFromMarkdown(), gfmFromMarkdown()],
+}
+// `fences: true` keeps code blocks as ``` rather than indented, so they
+// aren't misread differently (e.g. as MDX/JSX) than they were authored.
+const STRINGIFY_OPTIONS = {
+  extensions: [mdxToMarkdown(), gfmToMarkdown()],
+  bullet: '-' as const,
+  listItemIndent: 'one' as const,
+  fences: true,
+}
+
+/**
+ * Discovers every `FederatedContentSource` under `./sources`.
+ */
+async function loadSources(): Promise<FederatedContentSource[]> {
+  const files = (await readdir(SOURCES_DIR)).filter((file) => /\.tsx?$/.test(file))
+
+  return Promise.all(
+    files.map(async (file) => {
+      const mod = await import(join(SOURCES_DIR, file))
+      return mod.default as FederatedContentSource
+    })
+  )
+}
+
+/**
+ * Rewrites a link URL: pages mapped in `source.pageMap` go to their local
+ * `/guides/<section>` route, everything else falls back to `externalSite`.
+ */
+function transformUrl(source: FederatedContentSource, url: string): string {
+  try {
+    const placeholderHostname = 'placeholder'
+    const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`)
+
+    // Don't modify a url with a FQDN or a url that's only a hash
+    if (hostname !== placeholderHostname || pathname === '/') {
+      return url
+    }
+
+    const relativePath = (
+      pathname.endsWith('.md')
+        ? pathname.replace(/\.md$/, '')
+        : isAbsolute(url)
+          ? relative(new URL(source.externalSite).pathname, pathname)
+          : pathname
+    ).replace(/^\//, '')
+
+    const mapped = source.pageMap.find(({ remoteFile }) => `${relativePath}.md` === remoteFile)
+
+    // If we have a mapping for this page, use the mapped path; otherwise
+    // link to the original docs
+    return mapped
+      ? `${BASE_PATH}/guides/${source.section}${mapped.slug ? `/${mapped.slug}` : ''}${hash}`
+      : `${source.externalSite}/${relativePath}${hash}`
+  } catch (err) {
+    throw Error('[DOCS] fetch-federated-content: Error transforming markdown URL', { cause: err })
+  }
+}
+
+async function fetchPage(source: FederatedContentSource, page: FederatedPage): Promise<string> {
+  const raw = await getGitHubFileContents({
+    org: source.org,
+    repo: source.repo,
+    path: `${source.docsDir}/${page.remoteFile}`,
+    branch: source.branch,
+  })
+
+  const tree = fromMarkdown(raw, PARSE_OPTIONS)
+  remarkMkDocsAdmonition()(tree)
+  remarkPyMdownTabs()(tree)
+  removeTitle(page.meta.title)(tree)
+  visit(tree, ['link', 'image', 'definition'], (node: any) => {
+    node.url = transformUrl(source, node.url)
+  })
+  const content = toMarkdown(tree, STRINGIFY_OPTIONS).trim()
+
+  const frontmatter: Record<string, string> = {
+    title: page.meta.title,
+    // Points the "Edit this page on GitHub" link back at the source repo
+    // instead of this generated file.
+    editLink: `${source.org}/${source.repo}/blob/${source.branch}/${source.docsDir}/${page.remoteFile}`,
+  }
+  if (page.meta.subtitle) frontmatter.subtitle = page.meta.subtitle
+
+  return matter.stringify(`${content}\n`, frontmatter)
+}
+
+async function fetchSource(source: FederatedContentSource): Promise<void> {
+  await mkdir(join(GUIDES_DIRECTORY, source.section), { recursive: true })
+
+  await Promise.all(
+    source.pageMap.map(async (page) => {
+      const output = await fetchPage(source, page)
+
+      const outPath = page.slug
+        ? join(GUIDES_DIRECTORY, source.section, `${page.slug}.mdx`)
+        : join(GUIDES_DIRECTORY, `${source.section}.mdx`)
+
+      await writeFile(outPath, output)
+    })
+  )
+}
+
+async function fetchFederatedContent() {
+  const sources = await loadSources()
+
+  await Promise.all(sources.map(fetchSource))
+
+  const pageCount = sources.reduce((sum, source) => sum + source.pageMap.length, 0)
+  console.log(
+    `Fetched ${pageCount} federated page(s) across ${sources.length} source(s) into content/guides/`
+  )
+}
+
+fetchFederatedContent().catch((error) => {
+  throw error
+})

apps/docs/scripts/federated-content/sources/graphql.ts

@@ -0,0 +1,87 @@
+import type { FederatedContentSource } from '../types'
+
+// We fetch these docs at build time from an external repo
+const graphql: FederatedContentSource = {
+  section: 'graphql',
+  org: 'supabase',
+  repo: 'pg_graphql',
+  branch: 'master',
+  docsDir: 'docs',
+  externalSite: 'https://supabase.github.io/pg_graphql',
+  pageMap: [
+    {
+      meta: {
+        title: 'GraphQL',
+        subtitle: 'Autogenerated GraphQL APIs with Postgres.',
+      },
+      remoteFile: 'supabase.md',
+    },
+    {
+      slug: 'api',
+      meta: {
+        title: 'GraphQL API',
+        subtitle: 'Understanding the core concepts of the GraphQL API.',
+      },
+      remoteFile: 'api.md',
+    },
+    {
+      slug: 'views',
+      meta: {
+        title: 'Views',
+        subtitle: 'Using Postgres Views with GraphQL.',
+      },
+      remoteFile: 'views.md',
+    },
+    {
+      slug: 'functions',
+      meta: {
+        title: 'Functions',
+        subtitle: 'Using Postgres Functions with GraphQL.',
+      },
+      remoteFile: 'functions.md',
+    },
+    {
+      slug: 'computed-fields',
+      meta: {
+        title: 'Computed Fields',
+        subtitle: 'Using Postgres Computed Fields with GraphQL.',
+      },
+      remoteFile: 'computed_fields.md',
+    },
+    {
+      slug: 'configuration',
+      meta: {
+        title: 'Configuration & Customization',
+        subtitle:
+          'Extra configuration options can be set on SQL entities using comment directives.',
+      },
+      remoteFile: 'configuration.md',
+    },
+    {
+      slug: 'security',
+      meta: {
+        title: 'Security',
+        subtitle: 'Securing your GraphQL API.',
+      },
+      remoteFile: 'security.md',
+    },
+    {
+      slug: 'with-apollo',
+      meta: {
+        title: 'With Apollo',
+        subtitle: 'Using pg_grapqhl with Apollo.',
+      },
+      remoteFile: 'usage_with_apollo.md',
+    },
+    {
+      slug: 'with-relay',
+      meta: {
+        title: 'With Relay',
+        subtitle: 'Using pg_grapqhl with Relay.',
+      },
+      remoteFile: 'usage_with_relay.md',
+    },
+  ],
+}
+
+export default graphql

apps/docs/scripts/federated-content/types.ts

@@ -0,0 +1,33 @@
+/**
+ * A single external page that gets federated into a local guide.
+ */
+export interface FederatedPage {
+  /** Local slug, relative to `section`. Omit for the section's index page. */
+  slug?: string
+  meta: {
+    title: string
+    subtitle?: string
+  }
+  /** Path of the file in the remote repo, relative to `docsDir`. */
+  remoteFile: string
+}
+
+/**
+ * Describes where to fetch a set of external docs pages from, and how they
+ * map onto local `/guides/<section>` routes.
+ *
+ * Add a new entry point by exporting a `FederatedContentSource` as the
+ * default export of a file under `./sources`.
+ */
+export interface FederatedContentSource {
+  /** Guide section these pages are mounted under, e.g. 'graphql' -> content/guides/graphql */
+  section: string
+  org: string
+  repo: string
+  branch: string
+  /** Directory in the remote repo containing the docs, e.g. 'docs'. */
+  docsDir: string
+  /** Public site the remote docs are also published to, used to resolve unmapped links. */
+  externalSite: string
+  pageMap: FederatedPage[]
+}