mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 09:25:06 +03:00
## Summary Since the guides UA-redirect shipped (GROWTH-811), named LLM bots requesting `/docs/guides/*` get rewritten to the markdown handler, which returns a 404 when no `.md` file exists. About 90K of those 404s per day land on real pages that serve HTML 200 fine: the bot gets nothing on a page that works. The root cause is that the docs middleware hardcoded `hasMarkdownVariant: true` for every guide path, so it never checked whether a `.md` actually existed. I fixed it in two layers: 1. A build-time slug manifest makes `hasMarkdownVariant` truthful. Guide pages with no `.md` now fall through to HTML 200 instead of a 404. This is content-source-agnostic and future-proof: a new content source can never silently regress to a 404. 2. A second generator pass emits real markdown for the troubleshooting collection (the largest source, ~70% of the 404 volume), so those bots get clean markdown rather than just HTML. ## Changes - Add a shared `markdown-sources` module: a single source of truth for which slugs get a `.md` (guides + troubleshooting), so the generator output and the manifest cannot drift. - Generate markdown for the troubleshooting collection (196 pages, TOML frontmatter parsed via `smol-toml`), written under `public/markdown/guides/troubleshooting/`. - Emit a build-time slug manifest (a gitignored generated `.ts` module, regenerated in `prebuild`, `predev`, and `pretypecheck`, mirroring the existing `__generated__/graphql.ts` lifecycle). - Gate the middleware's `hasMarkdownVariant` on the manifest: serve HTML 200 instead of a 404 for guide paths with no markdown variant. This PR intentionally does not generate markdown for the ai-prompts, YAML config, and externally-fetched (splinter) sources. The HTML fallback covers them now; generating their markdown is follow-up work. ## Testing Local verification (deterministic, against the real manifest and the real negotiation function): - Manifest invariant holds: 744 manifest slugs equal 744 generated `.md` files. - Generator emits 196 troubleshooting files with zero warnings, frontmatter stripped, no leaked delimiters. - Negotiation decision matrix, 6/6: covered slug + bot UA to markdown; uncovered real page + bot UA to pass (HTML 200); nonexistent + bot UA to pass; browser to HTML; covered + `.md` suffix to markdown; uncovered + `.md` suffix to pass. Verified on the Vercel preview deploy: - [x] `User-Agent: ChatGPT-User` on a troubleshooting page returns `200 text/markdown` (real markdown body, frontmatter stripped). - [x] `User-Agent: ChatGPT-User` on an uncovered real page (`ai-tools/ai-prompts/code-format-sql`) returns `200 text/html` (was 404). - [x] Browser request to the same uncovered page returns `200 text/html` (unchanged for humans). - [x] `User-Agent: ChatGPT-User` on a covered standard guide returns `200 text/markdown` (no regression). - [x] `User-Agent: ChatGPT-User` on a nonexistent guide URL returns `404` (correct). Known limitation: an explicit `.md`-suffix request on an uncovered page still 404s by design (an explicit markdown request for a page that has no markdown). The ~90K/day volume is plain-URL UA-based, so it is unaffected. Post-deploy, I will re-run the request-grain 404 reclassification in the GROWTH-915 BQ workspace to confirm fixable guide markdown 404s drop to near zero. ## Linear - fixes GROWTH-946 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added generated markdown slug tracking for docs guides, improving markdown availability detection. * Added automated manifest generation and validation during docs build and CI workflows. * **Bug Fixes** * Improved guide markdown negotiation so only supported guide slugs are treated as having a markdown variant. * Standardized markdown source handling for guides and troubleshooting pages. * **Tests** * Added coverage for guide and troubleshooting slug generation. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Alaister Young <10985857+alaister@users.noreply.github.com>
91 lines
2.9 KiB
TypeScript
91 lines
2.9 KiB
TypeScript
import { clientSdkIds } from '~/content/navigation.references'
|
|
import { BASE_PATH } from '~/lib/constants'
|
|
import { MARKDOWN_SLUGS } from '~/lib/markdown-manifest'
|
|
import { negotiateMarkdown } from 'common/markdown-negotiation'
|
|
import { isbot } from 'isbot'
|
|
import { NextResponse, type NextRequest } from 'next/server'
|
|
|
|
const REFERENCE_PATH = `${BASE_PATH ?? ''}/reference`
|
|
const GUIDES_PATH = `${BASE_PATH ?? ''}/guides`
|
|
const GUIDES_MARKDOWN_SLUGS = new Set(MARKDOWN_SLUGS)
|
|
|
|
export function middleware(request: NextRequest) {
|
|
const url = new URL(request.url)
|
|
const { pathname } = url
|
|
|
|
if (pathname.startsWith(GUIDES_PATH + '/')) {
|
|
const isMdSuffix = pathname.endsWith('.md')
|
|
const slug = pathname.replace(`${GUIDES_PATH}/`, '').replace(/\.md$/, '')
|
|
const decision = negotiateMarkdown(
|
|
{
|
|
acceptHeader: request.headers.get('accept') ?? '',
|
|
userAgent: request.headers.get('user-agent') ?? '',
|
|
},
|
|
{ hasMarkdownVariant: GUIDES_MARKDOWN_SLUGS.has(slug), isMarkdownSuffix: isMdSuffix }
|
|
)
|
|
|
|
if (decision === 'not-acceptable') {
|
|
return new NextResponse('Not Acceptable', {
|
|
status: 406,
|
|
headers: { 'Cache-Control': 'no-store', Vary: 'Accept' },
|
|
})
|
|
}
|
|
|
|
if (decision === 'markdown') {
|
|
const rewriteUrl = new URL(url)
|
|
rewriteUrl.pathname = `${BASE_PATH ?? ''}/api/guides-md/${slug}`
|
|
return NextResponse.rewrite(rewriteUrl)
|
|
}
|
|
}
|
|
|
|
if (!pathname.startsWith(REFERENCE_PATH)) {
|
|
return NextResponse.next()
|
|
}
|
|
|
|
if (isbot(request.headers.get('user-agent'))) {
|
|
let [, lib, maybeVersion, ...slug] = pathname.replace(REFERENCE_PATH, '').split('/')
|
|
|
|
if (clientSdkIds.includes(lib)) {
|
|
const version = /v\d+/.test(maybeVersion) ? maybeVersion : undefined
|
|
if (!version) {
|
|
slug = [maybeVersion, ...slug]
|
|
}
|
|
|
|
if (slug.length > 0) {
|
|
const rewriteUrl = new URL(url)
|
|
rewriteUrl.pathname = (BASE_PATH ?? '') + '/api/crawlers'
|
|
return NextResponse.rewrite(rewriteUrl)
|
|
}
|
|
}
|
|
}
|
|
|
|
const [, lib, maybeVersion] = pathname.replace(REFERENCE_PATH, '').split('/')
|
|
|
|
if (lib === 'cli') {
|
|
const rewritePath = [REFERENCE_PATH, 'cli'].join('/')
|
|
return NextResponse.rewrite(new URL(rewritePath, request.url))
|
|
}
|
|
|
|
if (lib === 'api') {
|
|
const rewritePath = [REFERENCE_PATH, 'api'].join('/')
|
|
return NextResponse.rewrite(new URL(rewritePath, request.url))
|
|
}
|
|
|
|
if (lib?.startsWith('self-hosting-')) {
|
|
const rewritePath = [REFERENCE_PATH, lib].join('/')
|
|
return NextResponse.rewrite(new URL(rewritePath, request.url))
|
|
}
|
|
|
|
if (clientSdkIds.includes(lib)) {
|
|
const version = /v\d+/.test(maybeVersion) ? maybeVersion : null
|
|
const rewritePath = [REFERENCE_PATH, lib, version].filter(Boolean).join('/')
|
|
return NextResponse.rewrite(new URL(rewritePath, request.url))
|
|
}
|
|
|
|
return NextResponse.next()
|
|
}
|
|
|
|
export const config = {
|
|
matcher: ['/reference/:path*', '/guides/:path*'],
|
|
}
|