import '../utils/dotenv' import { mkdir, readdir, writeFile } from 'node:fs/promises' import { dirname, isAbsolute, join, relative } from 'node:path' import { fileURLToPath } from 'node:url' import { BASE_PATH } from '~/lib/constants' import { GENERATED_DIRECTORY, GUIDES_DIRECTORY } from '~/lib/docs' import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition' import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle' import remarkPyMdownTabs from '~/lib/mdx/plugins/remarkTabs' import { getGitHubFileContents, octokit, OCTOKIT_RETRY_OPTIONS } from '~/lib/octokit' import matter from 'gray-matter' import { fromMarkdown } from 'mdast-util-from-markdown' import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm' import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx' import { toMarkdown } from 'mdast-util-to-markdown' import { gfm } from 'micromark-extension-gfm' import { mdxjs } from 'micromark-extension-mdxjs' import emoji from 'remark-emoji' import { visit } from 'unist-util-visit' import type { FederatedContentSource, FederatedPage } from './types' const SOURCES_DIR = join(dirname(fileURLToPath(import.meta.url)), 'sources') const PARSE_OPTIONS = { extensions: [mdxjs(), gfm()], mdastExtensions: [mdxFromMarkdown(), gfmFromMarkdown()], } // `fences: true` keeps code blocks as ``` rather than indented, so they // aren't misread differently (e.g. as MDX/JSX) than they were authored. const STRINGIFY_OPTIONS = { extensions: [mdxToMarkdown(), gfmToMarkdown()], bullet: '-' as const, listItemIndent: 'one' as const, fences: true, } // remark-emoji's attacher is typed with a unified `this: Processor` context // it never actually uses; cast it to the plain transformer factory it is. const emojiTransform = (emoji as unknown as () => (tree: any) => void)() /** * Discovers every `FederatedContentSource` under `./sources`. */ async function loadSources(): Promise { const files = (await readdir(SOURCES_DIR)).filter((file) => /\.tsx?$/.test(file)) return Promise.all( files.map(async (file) => { const mod = await import(join(SOURCES_DIR, file)) return mod.default as FederatedContentSource }) ) } /** * Path of a page's remote file, relative to the repo root. */ function remotePath(source: FederatedContentSource, page: FederatedPage): string { return page.useRoot ? page.remoteFile : `${source.docsDir}/${page.remoteFile}` } type LatestTagQueryResponse = { repository: { refs: { nodes: { name: string }[] | null pageInfo: { hasNextPage: boolean; endCursor: string | null } } } } const LATEST_TAG_QUERY = ` query LatestTagQuery($owner: String!, $name: String!, $after: String) { repository(owner: $owner, name: $name) { refs( refPrefix: "refs/tags/", orderBy: { field: TAG_COMMIT_DATE, direction: DESC }, first: 20, after: $after ) { nodes { name } pageInfo { hasNextPage endCursor } } } } ` /** * Resolves a source's `latestTag.pattern` to the newest matching tag name. * GraphQL is required here since the REST API can't order tags by date. */ async function resolveLatestTag( source: FederatedContentSource, after: string | null = null ): Promise { const pattern = new RegExp(source.latestTag!.pattern) const { repository: { refs: { nodes, pageInfo: { hasNextPage, endCursor }, }, }, } = await octokit().graphql(LATEST_TAG_QUERY, { owner: source.org, name: source.repo, after, }) const tag = nodes?.find(({ name }) => pattern.test(name))?.name if (tag) return tag if (hasNextPage && endCursor) return resolveLatestTag(source, endCursor) throw new Error( `No tag matching ${source.latestTag!.pattern} found for ${source.org}/${source.repo}` ) } /** * Rewrites a link URL: pages mapped in `source.pageMap` go to their local * `/guides/
` route, everything else falls back to `externalSite`. */ function transformUrl(source: FederatedContentSource, url: string): string { const assetPattern = /(\.\.\/)+assets\// if (source.assetsDir && assetPattern.test(url)) { return url.replace( assetPattern, `https://raw.githubusercontent.com/${source.org}/${source.repo}/${source.branch}/${source.assetsDir}/` ) } try { const placeholderHostname = 'placeholder' const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`) // Don't modify a url with a FQDN or a url that's only a hash if (hostname !== placeholderHostname || pathname === '/') { return url } const relativePath = ( pathname.endsWith('.md') ? pathname.replace(/\.md$/, '') : isAbsolute(url) ? relative(new URL(source.externalSite).pathname, pathname) : pathname ).replace(/^\//, '') const docsRelative = relativePath.replace(new RegExp(`^${source.docsDir}/`), '') const mapped = source.pageMap.find(({ remoteFile, useRoot }) => useRoot ? `${relativePath}.md` === remoteFile : `${docsRelative}.md` === remoteFile ) // If we have a mapping for this page, use the mapped path; otherwise // link to the original docs if (mapped) { return `${BASE_PATH}/guides/${source.section}${mapped.slug ? `/${mapped.slug}` : ''}${hash}` } return source.rawFallback ? `${source.externalSite}${pathname}${hash}` : `${source.externalSite}/${relativePath}${hash}` } catch (err) { throw Error('[DOCS] fetch-federated-content: Error transforming markdown URL', { cause: err }) } } async function fetchPage(source: FederatedContentSource, page: FederatedPage): Promise { const raw = await getGitHubFileContents({ org: source.org, repo: source.repo, path: remotePath(source, page), branch: source.branch, }) // Strip the source file's own frontmatter, if any; we build our own from // `page.meta` below rather than merging it in. const tree = fromMarkdown(matter(raw).content, PARSE_OPTIONS) remarkMkDocsAdmonition()(tree) remarkPyMdownTabs()(tree) emojiTransform(tree) if (page.dropLeadingHeading) { const [firstNode] = tree.children if (firstNode?.type === 'heading' && firstNode.depth === 1) tree.children.splice(0, 1) } else { removeTitle(page.meta.title)(tree) } visit(tree, ['link', 'image', 'definition'], (node: any) => { node.url = transformUrl(source, node.url) }) let content = toMarkdown(tree, STRINGIFY_OPTIONS).trim() if (page.meta.dashboardIntegrationPath) { content = `\n\n${content}` } const frontmatter: Record = { title: page.meta.title, // Points the "Edit this page on GitHub" link back at the source repo // instead of this generated file. editLink: `${source.org}/${source.repo}/blob/${source.branch}/${remotePath(source, page)}`, } if (page.meta.subtitle) frontmatter.subtitle = page.meta.subtitle if (page.meta.description) frontmatter.description = page.meta.description if (page.meta.tocVideo) frontmatter.tocVideo = page.meta.tocVideo return matter.stringify(`${content}\n`, frontmatter) } async function fetchRawFile( source: FederatedContentSource, rawFile: NonNullable[number] ): Promise { const content = await getGitHubFileContents({ org: source.org, repo: source.repo, path: `${source.docsDir}/${rawFile.remoteFile}`, branch: source.branch, }) await mkdir(GENERATED_DIRECTORY, { recursive: true }) await writeFile(join(GENERATED_DIRECTORY, rawFile.outFile), content) } async function fetchSource(baseSource: FederatedContentSource): Promise { const source = baseSource.latestTag ? { ...baseSource, branch: await resolveLatestTag(baseSource) } : baseSource await mkdir(join(GUIDES_DIRECTORY, source.section), { recursive: true }) await Promise.all([ ...source.pageMap.map(async (page) => { const output = await fetchPage(source, page) const outPath = page.slug ? join(GUIDES_DIRECTORY, source.section, `${page.slug}.mdx`) : join(GUIDES_DIRECTORY, `${source.section}.mdx`) await writeFile(outPath, output) }), ...(source.rawFiles ?? []).map((rawFile) => fetchRawFile(source, rawFile)), ]) } const AI_SKILLS_REPO = { org: 'supabase', repo: 'agent-skills', branch: 'main', path: 'skills', } /** * Lists the skill directories in the agent-skills repo, fetches each * `SKILL.md`'s frontmatter, and writes the summary to * `features/docs/generated/ai-skills.json` for `AiSkills.utils.ts` to read. */ async function fetchAiSkills(): Promise { const { data: contents } = await octokit().request('GET /repos/{owner}/{repo}/contents/{path}', { owner: AI_SKILLS_REPO.org, repo: AI_SKILLS_REPO.repo, path: AI_SKILLS_REPO.path, ref: AI_SKILLS_REPO.branch, request: OCTOKIT_RETRY_OPTIONS, }) if (!Array.isArray(contents)) { throw new Error('Expected directory listing from GitHub agent skills repo') } const skillDirs = contents.filter((item) => item.type === 'dir') const skills = await Promise.all( skillDirs.map(async (item) => { const rawContent = await getGitHubFileContents({ org: AI_SKILLS_REPO.org, repo: AI_SKILLS_REPO.repo, branch: AI_SKILLS_REPO.branch, path: `${AI_SKILLS_REPO.path}/${item.name}/SKILL.md`, }) const { data } = matter(rawContent) as { data: { description?: string } } return { name: item.name, description: data.description || '', installCommand: `npx skills add supabase/agent-skills --skill ${item.name}`, } }) ) skills.sort((a, b) => a.name.localeCompare(b.name)) await mkdir(GENERATED_DIRECTORY, { recursive: true }) await writeFile(join(GENERATED_DIRECTORY, 'ai-skills.json'), JSON.stringify(skills, null, 2)) } const SPLINTER_REPO = { org: 'supabase', repo: 'splinter', branch: 'main', docsDir: 'docs', } /** * Rewrites a splinter lint doc's link: cross-references to other lints * become `?lint=` (matching `Tabs`'s `queryGroup` tab-switching), and * everything else falls back to viewing the file on GitHub. */ function splinterUrlTransform(lintPaths: string[]) { return (url: string): string => { try { const placeholderHostname = 'placeholder' const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`) if (hostname !== placeholderHostname || pathname === '/') { return url } const basename = pathname.split('/').at(-1)!.replace(/\.md$/, '') if (lintPaths.includes(basename)) { return `?lint=${basename}${hash}` } return `https://github.com/${SPLINTER_REPO.org}/${SPLINTER_REPO.repo}/blob/${SPLINTER_REPO.branch}${pathname}${hash}` } catch (err) { console.error('Error transforming markdown URL', err) return url } } } /** * Lists the numbered lint docs in the splinter repo, transforms each the * same way as a guide page, and writes them to * `features/docs/generated/database-advisors.json` for the * `DatabaseAdvisorsIndex` component to read. */ async function fetchDatabaseAdvisors(): Promise { const { data: contents } = await octokit().request('GET /repos/{owner}/{repo}/contents/{path}', { owner: SPLINTER_REPO.org, repo: SPLINTER_REPO.repo, path: SPLINTER_REPO.docsDir, ref: SPLINTER_REPO.branch, request: OCTOKIT_RETRY_OPTIONS, }) if (!Array.isArray(contents)) { throw new Error('Expected directory listing from GitHub splinter repo') } const lintFiles = contents.filter(({ path }) => /docs\/\d+.+\.md$/.test(path)) const lintPaths = lintFiles.map(({ path }) => path.split('/').at(-1)!.replace(/\.md$/, '')) const urlTransform = splinterUrlTransform(lintPaths) const lints = await Promise.all( lintFiles.map(async ({ path }) => { const raw = await getGitHubFileContents({ org: SPLINTER_REPO.org, repo: SPLINTER_REPO.repo, path, branch: SPLINTER_REPO.branch, }) const tree = fromMarkdown(matter(raw).content, PARSE_OPTIONS) remarkMkDocsAdmonition()(tree) remarkPyMdownTabs()(tree) visit(tree, ['link', 'image', 'definition'], (node: any) => { node.url = urlTransform(node.url) }) return { path: path.split('/').at(-1)!.replace(/\.md$/, ''), content: toMarkdown(tree, STRINGIFY_OPTIONS).trim(), } }) ) await mkdir(GENERATED_DIRECTORY, { recursive: true }) await writeFile( join(GENERATED_DIRECTORY, 'database-advisors.json'), JSON.stringify(lints, null, 2) ) } async function fetchFederatedContent() { const sources = await loadSources() await Promise.all([...sources.map(fetchSource), fetchAiSkills(), fetchDatabaseAdvisors()]) const pageCount = sources.reduce((sum, source) => sum + source.pageMap.length, 0) console.log( `Fetched ${pageCount} federated page(s) across ${sources.length} source(s) into content/guides/` ) } fetchFederatedContent().catch((error) => { throw error })