From a72a58eeae85afdae0bcceeb354483f6d1ae6865 Mon Sep 17 00:00:00 2001 From: Jeremias Menichelli Date: Thu, 16 Jul 2026 12:53:47 +0200 Subject: [PATCH] feat: Add federated content script. Resolve graphql routes. (#47934) --- apps/docs/.gitignore | 5 + .../app/guides/graphql/[[...slug]]/page.tsx | 202 ++---------------- apps/docs/features/docs/GuidesMdx.utils.tsx | 7 +- apps/docs/lib/docs.ts | 9 + apps/docs/package.json | 5 +- .../fetch-federated-content.ts | 143 +++++++++++++ .../federated-content/sources/graphql.ts | 87 ++++++++ apps/docs/scripts/federated-content/types.ts | 33 +++ 8 files changed, 299 insertions(+), 192 deletions(-) create mode 100644 apps/docs/scripts/federated-content/fetch-federated-content.ts create mode 100644 apps/docs/scripts/federated-content/sources/graphql.ts create mode 100644 apps/docs/scripts/federated-content/types.ts diff --git a/apps/docs/.gitignore b/apps/docs/.gitignore index 5dbb96adacc..ca8ec6b88b9 100644 --- a/apps/docs/.gitignore +++ b/apps/docs/.gitignore @@ -40,6 +40,11 @@ public/docs/ # Generated reference content (built by scripts/build-reference-content.ts) /content/reference/ +# Federated guide content, fetched from an external repo by +# scripts/federated-content/fetch-federated-content.ts +/content/guides/graphql/ +/content/guides/graphql.mdx + # Downloaded TypeDoc dumps under spec/reference///. Regenerated by # `cd apps/docs/spec && make download.tsdoc.v2`. Hand-authored files in the # same folders (config.json, partials/) stay tracked. diff --git a/apps/docs/app/guides/graphql/[[...slug]]/page.tsx b/apps/docs/app/guides/graphql/[[...slug]]/page.tsx index eb0c01b48f5..66ac2c7fd6b 100644 --- a/apps/docs/app/guides/graphql/[[...slug]]/page.tsx +++ b/apps/docs/app/guides/graphql/[[...slug]]/page.tsx @@ -1,198 +1,26 @@ -import { isAbsolute, relative } from 'path' -import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template' -import { genGuideMeta } from '~/features/docs/GuidesMdx.utils' +import { GuideTemplate } from '~/features/docs/GuidesMdx.template' +import { + genGuideMeta, + genGuidesStaticParams, + getGuidesMarkdown, +} from '~/features/docs/GuidesMdx.utils' import { getEmptyArray } from '~/features/helpers.fn' import { IS_DEV } from '~/lib/constants' -import { linkTransform, UrlTransformFunction } from '~/lib/mdx/plugins/rehypeLinkTransform' -import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition' -import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle' -import remarkPyMdownTabs from '~/lib/mdx/plugins/remarkTabs' -import { getGitHubFileContents } from '~/lib/octokit' -import { SerializeOptions } from '~/types/next-mdx-remote-serialize' -import { notFound } from 'next/navigation' -import rehypeSlug from 'rehype-slug' -// We fetch these docs at build time from an external repo -const org = 'supabase' -const repo = 'pg_graphql' -const branch = 'master' -const docsDir = 'docs' -const externalSite = 'https://supabase.github.io/pg_graphql' - -// Each external docs page is mapped to a local page -const pageMap = [ - { - meta: { - id: 'graphql-overview', - title: 'GraphQL', - subtitle: 'Autogenerated GraphQL APIs with Postgres.', - }, - remoteFile: 'supabase.md', - }, - { - slug: 'api', - meta: { - id: 'graphql-api', - title: 'GraphQL API', - subtitle: 'Understanding the core concepts of the GraphQL API.', - }, - remoteFile: 'api.md', - }, - { - slug: 'views', - meta: { - id: 'graphql-views', - title: 'Views', - subtitle: 'Using Postgres Views with GraphQL.', - }, - remoteFile: 'views.md', - }, - { - slug: 'functions', - meta: { - id: 'graphql-functions', - title: 'Functions', - subtitle: 'Using Postgres Functions with GraphQL.', - }, - remoteFile: 'functions.md', - }, - { - slug: 'computed-fields', - meta: { - id: 'graphql-computed-fields', - title: 'Computed Fields', - subtitle: 'Using Postgres Computed Fields with GraphQL.', - }, - remoteFile: 'computed_fields.md', - }, - { - slug: 'configuration', - meta: { - id: 'graphql-configuration', - title: 'Configuration & Customization', - subtitle: 'Extra configuration options can be set on SQL entities using comment directives.', - }, - remoteFile: 'configuration.md', - }, - { - slug: 'security', - meta: { - id: 'graphql-security', - title: 'Security', - subtitle: 'Securing your GraphQL API.', - }, - remoteFile: 'security.md', - }, - { - slug: 'with-apollo', - meta: { - id: 'graphql-with-apollo', - title: 'With Apollo', - subtitle: 'Using pg_grapqhl with Apollo.', - }, - remoteFile: 'usage_with_apollo.md', - }, - { - slug: 'with-relay', - meta: { - id: 'graphql-with-relay', - title: 'With Relay', - subtitle: 'Using pg_grapqhl with Relay.', - }, - remoteFile: 'usage_with_relay.md', - }, -] - -interface Params { - slug?: string[] -} +type Params = { slug?: string[] } const PGGraphQLDocs = async (props: { params: Promise }) => { const params = await props.params - const { meta, ...data } = await getContent(params) + const slug = ['graphql', ...(params.slug ?? [])] + const data = await getGuidesMarkdown(slug) - const options = { - mdxOptions: { - remarkPlugins: [remarkMkDocsAdmonition, remarkPyMdownTabs, [removeTitle, meta.title]], - rehypePlugins: [[linkTransform, urlTransform], rehypeSlug], - }, - } as SerializeOptions - - return + return } -/** - * Fetch markdown from external repo and transform links - */ -const getContent = async ({ slug }: Params) => { - const page = pageMap.find((page) => page.slug === slug?.at(0)) - - if (!page) { - notFound() - } - - const { remoteFile, meta } = page - - const editLink = newEditLink(`${org}/${repo}/blob/${branch}/${docsDir}/${remoteFile}`) - - const content = await getGitHubFileContents({ - org, - repo, - path: `${docsDir}/${remoteFile}`, - branch, - }) - - return { - pathname: `/guides/graphql${slug?.length ? `/${slug.join('/')}` : ''}` satisfies `/${string}`, - meta, - content, - editLink, - } -} - -const urlTransform: UrlTransformFunction = (url) => { - try { - const externalSiteUrl = new URL(externalSite) - - const placeholderHostname = 'placeholder' - const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`) - - // Don't modify a url with a FQDN or a url that's only a hash - if (hostname !== placeholderHostname || pathname === '/') { - return url - } - - const getRelativePath = () => { - if (pathname.endsWith('.md')) { - return pathname.replace(/\.md$/, '') - } - if (isAbsolute(url)) { - return relative(externalSiteUrl.pathname, pathname) - } - return pathname - } - - const relativePath = getRelativePath().replace(/^\//, '') - - const page = pageMap.find(({ remoteFile }) => `${relativePath}.md` === remoteFile) - - // If we have a mapping for this page, use the mapped path - if (page) { - return '/docs/guides/graphql/' + page.slug + hash - } - - // If we don't have this page in our docs, link to original docs - return `${externalSite}/${relativePath}${hash}` - } catch (err) { - console.error('Error transforming markdown URL', err) - return url - } -} - -const generateStaticParams = !IS_DEV - ? async () => pageMap.map(({ slug }) => ({ slug: slug ? [slug] : [] })) - : getEmptyArray -const generateMetadata = genGuideMeta(getContent) +const generateStaticParams = !IS_DEV ? genGuidesStaticParams('graphql') : getEmptyArray +const generateMetadata = genGuideMeta((params: { slug?: string[] }) => + getGuidesMarkdown(['graphql', ...(params.slug ?? [])]) +) export default PGGraphQLDocs -export { generateMetadata, generateStaticParams } +export { generateStaticParams, generateMetadata } diff --git a/apps/docs/features/docs/GuidesMdx.utils.tsx b/apps/docs/features/docs/GuidesMdx.utils.tsx index 1cd8e04a9eb..fc47f2664ae 100644 --- a/apps/docs/features/docs/GuidesMdx.utils.tsx +++ b/apps/docs/features/docs/GuidesMdx.utils.tsx @@ -30,7 +30,7 @@ const PUBLISHED_SECTIONS = [ 'deployment', 'functions', 'getting-started', - // 'graphql', -- technically published, but completely federated + 'graphql', 'integrations', 'local-development', 'platform', @@ -77,13 +77,14 @@ const getGuidesMarkdownInternal = async (slug: string[]) => { throw Error(`Type of frontmatter is not valid for path: ${fullPath}`) } + const { editLink: editLinkOverride, ...restMeta } = meta const editLink = newEditLink( - `supabase/supabase/blob/master/apps/docs/content/guides/${relPath}.mdx` + editLinkOverride ?? `supabase/supabase/blob/master/apps/docs/content/guides/${relPath}.mdx` ) return { pathname: `/guides/${slug.join('/')}` satisfies `/${string}`, - meta, + meta: restMeta, content, editLink, } diff --git a/apps/docs/lib/docs.ts b/apps/docs/lib/docs.ts index bfd0b1e426c..d070cc84c47 100644 --- a/apps/docs/lib/docs.ts +++ b/apps/docs/lib/docs.ts @@ -28,6 +28,12 @@ export type GuideFrontmatter = { /** @deprecated */ hide_table_of_contents?: boolean tocVideo?: string + /** + * Overrides the "Edit this page on GitHub" link. Used for federated + * content, whose source of truth lives in an external repo rather than + * this generated file. + */ + editLink?: string } /** @@ -64,6 +70,9 @@ export function isValidGuideFrontmatter(obj: object): obj is GuideFrontmatter { if ('tocVideo' in obj && typeof obj.tocVideo !== 'string') { throw Error(`Invalid guide frontmatter: tocVideo must be a string. Received ${obj.tocVideo}`) } + if ('editLink' in obj && typeof obj.editLink !== 'string') { + throw Error(`Invalid guide frontmatter: editLink must be a string. Received: ${obj.editLink}`) + } return true } diff --git a/apps/docs/package.json b/apps/docs/package.json index 8f2b9580222..737d46427f8 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -7,6 +7,7 @@ "build": "next build", "build:analyze": "ANALYZE=true next build", "build:guides-markdown": "tsx ./internals/generate-guides-markdown.ts", + "build:federated-content": "tsx --conditions=react-server ./scripts/federated-content/fetch-federated-content.ts", "prebuild:reference-markdown": "pnpm run codegen:references", "build:reference-markdown": "tsx ./internals/generate-reference-markdown.ts", "build:markdown": "pnpm build:guides-markdown && pnpm build:reference-markdown", @@ -35,8 +36,8 @@ "lint": "eslint .", "lint:mdx": "supa-mdx-lint content --config ../../supa-mdx-lint.config.toml", "postbuild": "pnpm run build:sitemap && ./../../scripts/upload-static-assets.sh", - "prebuild": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm run build:markdown && pnpm run build:gz-archive", - "predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm run build:markdown", + "prebuild": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm build:federated-content && pnpm run build:markdown && pnpm run build:gz-archive", + "predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples && pnpm build:federated-content && pnpm run build:markdown", "preembeddings": "pnpm run codegen:references", "preinstall": "npx only-allow pnpm", "presync": "pnpm run codegen:graphql", diff --git a/apps/docs/scripts/federated-content/fetch-federated-content.ts b/apps/docs/scripts/federated-content/fetch-federated-content.ts new file mode 100644 index 00000000000..91b1df9c4e4 --- /dev/null +++ b/apps/docs/scripts/federated-content/fetch-federated-content.ts @@ -0,0 +1,143 @@ +import '../utils/dotenv' + +import { mkdir, readdir, writeFile } from 'node:fs/promises' +import { dirname, isAbsolute, join, relative } from 'node:path' +import { fileURLToPath } from 'node:url' +import { BASE_PATH } from '~/lib/constants' +import { GUIDES_DIRECTORY } from '~/lib/docs' +import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition' +import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle' +import remarkPyMdownTabs from '~/lib/mdx/plugins/remarkTabs' +import { getGitHubFileContents } from '~/lib/octokit' +import matter from 'gray-matter' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm' +import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx' +import { toMarkdown } from 'mdast-util-to-markdown' +import { gfm } from 'micromark-extension-gfm' +import { mdxjs } from 'micromark-extension-mdxjs' +import { visit } from 'unist-util-visit' + +import type { FederatedContentSource, FederatedPage } from './types' + +const SOURCES_DIR = join(dirname(fileURLToPath(import.meta.url)), 'sources') + +const PARSE_OPTIONS = { + extensions: [mdxjs(), gfm()], + mdastExtensions: [mdxFromMarkdown(), gfmFromMarkdown()], +} +// `fences: true` keeps code blocks as ``` rather than indented, so they +// aren't misread differently (e.g. as MDX/JSX) than they were authored. +const STRINGIFY_OPTIONS = { + extensions: [mdxToMarkdown(), gfmToMarkdown()], + bullet: '-' as const, + listItemIndent: 'one' as const, + fences: true, +} + +/** + * Discovers every `FederatedContentSource` under `./sources`. + */ +async function loadSources(): Promise { + const files = (await readdir(SOURCES_DIR)).filter((file) => /\.tsx?$/.test(file)) + + return Promise.all( + files.map(async (file) => { + const mod = await import(join(SOURCES_DIR, file)) + return mod.default as FederatedContentSource + }) + ) +} + +/** + * Rewrites a link URL: pages mapped in `source.pageMap` go to their local + * `/guides/
` route, everything else falls back to `externalSite`. + */ +function transformUrl(source: FederatedContentSource, url: string): string { + try { + const placeholderHostname = 'placeholder' + const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`) + + // Don't modify a url with a FQDN or a url that's only a hash + if (hostname !== placeholderHostname || pathname === '/') { + return url + } + + const relativePath = ( + pathname.endsWith('.md') + ? pathname.replace(/\.md$/, '') + : isAbsolute(url) + ? relative(new URL(source.externalSite).pathname, pathname) + : pathname + ).replace(/^\//, '') + + const mapped = source.pageMap.find(({ remoteFile }) => `${relativePath}.md` === remoteFile) + + // If we have a mapping for this page, use the mapped path; otherwise + // link to the original docs + return mapped + ? `${BASE_PATH}/guides/${source.section}${mapped.slug ? `/${mapped.slug}` : ''}${hash}` + : `${source.externalSite}/${relativePath}${hash}` + } catch (err) { + throw Error('[DOCS] fetch-federated-content: Error transforming markdown URL', { cause: err }) + } +} + +async function fetchPage(source: FederatedContentSource, page: FederatedPage): Promise { + const raw = await getGitHubFileContents({ + org: source.org, + repo: source.repo, + path: `${source.docsDir}/${page.remoteFile}`, + branch: source.branch, + }) + + const tree = fromMarkdown(raw, PARSE_OPTIONS) + remarkMkDocsAdmonition()(tree) + remarkPyMdownTabs()(tree) + removeTitle(page.meta.title)(tree) + visit(tree, ['link', 'image', 'definition'], (node: any) => { + node.url = transformUrl(source, node.url) + }) + const content = toMarkdown(tree, STRINGIFY_OPTIONS).trim() + + const frontmatter: Record = { + title: page.meta.title, + // Points the "Edit this page on GitHub" link back at the source repo + // instead of this generated file. + editLink: `${source.org}/${source.repo}/blob/${source.branch}/${source.docsDir}/${page.remoteFile}`, + } + if (page.meta.subtitle) frontmatter.subtitle = page.meta.subtitle + + return matter.stringify(`${content}\n`, frontmatter) +} + +async function fetchSource(source: FederatedContentSource): Promise { + await mkdir(join(GUIDES_DIRECTORY, source.section), { recursive: true }) + + await Promise.all( + source.pageMap.map(async (page) => { + const output = await fetchPage(source, page) + + const outPath = page.slug + ? join(GUIDES_DIRECTORY, source.section, `${page.slug}.mdx`) + : join(GUIDES_DIRECTORY, `${source.section}.mdx`) + + await writeFile(outPath, output) + }) + ) +} + +async function fetchFederatedContent() { + const sources = await loadSources() + + await Promise.all(sources.map(fetchSource)) + + const pageCount = sources.reduce((sum, source) => sum + source.pageMap.length, 0) + console.log( + `Fetched ${pageCount} federated page(s) across ${sources.length} source(s) into content/guides/` + ) +} + +fetchFederatedContent().catch((error) => { + throw error +}) diff --git a/apps/docs/scripts/federated-content/sources/graphql.ts b/apps/docs/scripts/federated-content/sources/graphql.ts new file mode 100644 index 00000000000..f3fdfc57f6a --- /dev/null +++ b/apps/docs/scripts/federated-content/sources/graphql.ts @@ -0,0 +1,87 @@ +import type { FederatedContentSource } from '../types' + +// We fetch these docs at build time from an external repo +const graphql: FederatedContentSource = { + section: 'graphql', + org: 'supabase', + repo: 'pg_graphql', + branch: 'master', + docsDir: 'docs', + externalSite: 'https://supabase.github.io/pg_graphql', + pageMap: [ + { + meta: { + title: 'GraphQL', + subtitle: 'Autogenerated GraphQL APIs with Postgres.', + }, + remoteFile: 'supabase.md', + }, + { + slug: 'api', + meta: { + title: 'GraphQL API', + subtitle: 'Understanding the core concepts of the GraphQL API.', + }, + remoteFile: 'api.md', + }, + { + slug: 'views', + meta: { + title: 'Views', + subtitle: 'Using Postgres Views with GraphQL.', + }, + remoteFile: 'views.md', + }, + { + slug: 'functions', + meta: { + title: 'Functions', + subtitle: 'Using Postgres Functions with GraphQL.', + }, + remoteFile: 'functions.md', + }, + { + slug: 'computed-fields', + meta: { + title: 'Computed Fields', + subtitle: 'Using Postgres Computed Fields with GraphQL.', + }, + remoteFile: 'computed_fields.md', + }, + { + slug: 'configuration', + meta: { + title: 'Configuration & Customization', + subtitle: + 'Extra configuration options can be set on SQL entities using comment directives.', + }, + remoteFile: 'configuration.md', + }, + { + slug: 'security', + meta: { + title: 'Security', + subtitle: 'Securing your GraphQL API.', + }, + remoteFile: 'security.md', + }, + { + slug: 'with-apollo', + meta: { + title: 'With Apollo', + subtitle: 'Using pg_grapqhl with Apollo.', + }, + remoteFile: 'usage_with_apollo.md', + }, + { + slug: 'with-relay', + meta: { + title: 'With Relay', + subtitle: 'Using pg_grapqhl with Relay.', + }, + remoteFile: 'usage_with_relay.md', + }, + ], +} + +export default graphql diff --git a/apps/docs/scripts/federated-content/types.ts b/apps/docs/scripts/federated-content/types.ts new file mode 100644 index 00000000000..eddd7f9cb26 --- /dev/null +++ b/apps/docs/scripts/federated-content/types.ts @@ -0,0 +1,33 @@ +/** + * A single external page that gets federated into a local guide. + */ +export interface FederatedPage { + /** Local slug, relative to `section`. Omit for the section's index page. */ + slug?: string + meta: { + title: string + subtitle?: string + } + /** Path of the file in the remote repo, relative to `docsDir`. */ + remoteFile: string +} + +/** + * Describes where to fetch a set of external docs pages from, and how they + * map onto local `/guides/
` routes. + * + * Add a new entry point by exporting a `FederatedContentSource` as the + * default export of a file under `./sources`. + */ +export interface FederatedContentSource { + /** Guide section these pages are mounted under, e.g. 'graphql' -> content/guides/graphql */ + section: string + org: string + repo: string + branch: string + /** Directory in the remote repo containing the docs, e.g. 'docs'. */ + docsDir: string + /** Public site the remote docs are also published to, used to resolve unmapped links. */ + externalSite: string + pageMap: FederatedPage[] +}