diff --git a/apps/docs/app/api/guides-md/[...slug]/route.ts b/apps/docs/app/api/guides-md/[...slug]/route.ts index 51dea28525c..f29c8258538 100644 --- a/apps/docs/app/api/guides-md/[...slug]/route.ts +++ b/apps/docs/app/api/guides-md/[...slug]/route.ts @@ -18,6 +18,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ slug headers: { 'Content-Type': 'text/markdown; charset=utf-8', 'Cache-Control': 'public, max-age=86400, stale-while-revalidate=3600', + Vary: 'Accept', }, }) } catch { diff --git a/apps/docs/middleware.test.ts b/apps/docs/middleware.test.ts index 9e447e4afb8..2c0c111e939 100644 --- a/apps/docs/middleware.test.ts +++ b/apps/docs/middleware.test.ts @@ -84,14 +84,7 @@ describe('docs middleware — /guides/* content negotiation', () => { expect(res.headers.get('Vary')).toBe('Accept') }) - it('does not 406 for LLM UAs or .md suffix paths even with a probe Accept', () => { - const llm = makeRequest('/docs/guides/auth', { - accept: 'application/x-content-negotiation-probe', - userAgent: 'Claude-User/1.0', - }) - expect(middleware(llm).status).not.toBe(406) - expect(middleware(llm).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth')) - + it('does not 406 for .md suffix paths even with a probe Accept', () => { const md = makeRequest('/docs/guides/auth.md', { accept: 'application/x-content-negotiation-probe', }) @@ -99,6 +92,14 @@ describe('docs middleware — /guides/* content negotiation', () => { expect(middleware(md).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth')) }) + it('returns 406 for a probe Accept header regardless of user agent', () => { + const req = makeRequest('/docs/guides/auth', { + accept: 'application/x-content-negotiation-probe', + userAgent: 'Claude-User/1.0', + }) + expect(middleware(req).status).toBe(406) + }) + it('does not 406 on /reference/* (negotiation contract is /guides/* only)', () => { const req = makeRequest('/docs/reference/javascript/introduction', { accept: 'application/x-content-negotiation-probe', @@ -106,32 +107,40 @@ describe('docs middleware — /guides/* content negotiation', () => { expect(middleware(req).status).not.toBe(406) }) - it('rewrites for each LLM user agent', () => { + it('does not rewrite for agent user agents without a markdown Accept preference', () => { for (const ua of [ 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)', 'Claude-Web/1.0', - 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)', 'PerplexityBot/1.0', ]) { const req = makeRequest('/docs/guides/auth', { userAgent: ua }) - expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth')) + expect(middleware(req).headers.get(REWRITE_HEADER)).toBeNull() } }) - it('LLM UA overrides an Accept header that prefers HTML', () => { + it('still serves ChatGPT-User markdown when Accept asks for it', () => { const req = makeRequest('/docs/guides/auth', { - accept: 'text/html;q=1.0, text/markdown;q=0.1', - userAgent: 'Claude-User/1.0', + accept: 'text/markdown', + userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)', }) expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth')) }) - it('falls through for non-LLM UAs (browsers, training crawlers, substring embeds)', () => { + it('serves HTML when Accept prefers HTML, regardless of user agent', () => { + const req = makeRequest('/docs/guides/auth', { + accept: 'text/html;q=1.0, text/markdown;q=0.1', + userAgent: 'Claude-User/1.0', + }) + expect(middleware(req).headers.get(REWRITE_HEADER)).toBeNull() + }) + + it('falls through for non-LLM UAs (browsers, training crawlers, excluded agents, substring embeds)', () => { for (const ua of [ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', 'GPTBot/1.0', 'ClaudeBot/1.0', 'CCBot/2.0', + 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)', 'chatgpt-userscript/2.0', 'NotPerplexityBot', ]) { diff --git a/apps/docs/middleware.ts b/apps/docs/middleware.ts index 4bbeeed8a65..fd885725c23 100644 --- a/apps/docs/middleware.ts +++ b/apps/docs/middleware.ts @@ -17,10 +17,7 @@ export function middleware(request: NextRequest) { const isMdSuffix = pathname.endsWith('.md') const slug = pathname.replace(`${GUIDES_PATH}/`, '').replace(/\.md$/, '') const decision = negotiateMarkdown( - { - acceptHeader: request.headers.get('accept') ?? '', - userAgent: request.headers.get('user-agent') ?? '', - }, + { acceptHeader: request.headers.get('accept') ?? '' }, { hasMarkdownVariant: GUIDES_MARKDOWN_SLUGS.has(slug), isMarkdownSuffix: isMdSuffix } ) diff --git a/apps/www/middleware.test.ts b/apps/www/middleware.test.ts index dbeed5d70ec..6950e7c6c7c 100644 --- a/apps/www/middleware.test.ts +++ b/apps/www/middleware.test.ts @@ -114,6 +114,30 @@ describe('www middleware', () => { expect(res.headers.get('x-middleware-rewrite')).toBeNull() }) + + it('rewrites changelog entry .md requests without doubling the suffix', () => { + const req = makeRequest('/changelog/100.md', { accept: 'text/markdown' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md') + }) + + it('serves markdown for explicit changelog .md requests even when Accept excludes it', () => { + const req = makeRequest('/changelog/100.md', { + accept: 'application/x-content-negotiation-probe', + }) + const res = middleware(req) + + expect(res.status).not.toBe(406) + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md') + }) + + it('rewrites the changelog index .md request without doubling the suffix', () => { + const req = makeRequest('/changelog.md', { accept: 'text/markdown' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog.md') + }) }) describe('Accept: text/markdown content negotiation', () => { @@ -153,6 +177,20 @@ describe('www middleware', () => { expect(res.headers.get('x-middleware-rewrite')).toBeNull() }) + + it('rewrites changelog entries to their static .md file', () => { + const req = makeRequest('/changelog/100', { accept: 'text/markdown' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md') + }) + + it('rewrites the bare changelog index to its static .md file', () => { + const req = makeRequest('/changelog', { accept: 'text/markdown' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog.md') + }) }) describe('Accept header q-value parsing', () => { @@ -234,15 +272,14 @@ describe('www middleware', () => { expect(res.status).not.toBe(406) }) - it('does not return 406 for LLM UAs even with a probe Accept header', () => { + it('returns 406 for a probe Accept header regardless of user agent', () => { const req = makeRequest('/pricing', { accept: 'application/x-content-negotiation-probe', userAgent: 'Claude-User/1.0', }) const res = middleware(req) - expect(res.status).not.toBe(406) - expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing') + expect(res.status).toBe(406) }) it('returns 406 on changelog entries when Accept excludes every type', () => { @@ -264,67 +301,38 @@ describe('www middleware', () => { }) }) - describe('LLM user-agent routing', () => { - it('rewrites for Claude-User', () => { - const req = makeRequest('/pricing', { + describe('user-agent independence', () => { + it('serves HTML to agent and bot user agents that send no markdown Accept preference', () => { + for (const ua of [ + 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)', + 'Claude-Web/1.0', + 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)', + 'PerplexityBot/1.0', + 'GPTBot/1.0', + 'ClaudeBot/1.0', + 'CCBot/2.0', + 'chatgpt-userscript/2.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + ]) { + const req = makeRequest('/auth', { userAgent: ua }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBeNull() + } + }) + + it('negotiates by Accept as usual when an agent user agent is present', () => { + const req = makeRequest('/auth', { + accept: 'text/markdown', userAgent: 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)', }) const res = middleware(req) - expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing') - }) - - it('rewrites for ChatGPT-User', () => { - const req = makeRequest('/auth', { userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)' }) - const res = middleware(req) - expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth') }) - it('rewrites for Claude-Web', () => { - const req = makeRequest('/pricing', { userAgent: 'Claude-Web/1.0' }) - const res = middleware(req) - - expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing') - }) - - it('rewrites for PerplexityBot', () => { - const req = makeRequest('/auth/', { userAgent: 'PerplexityBot/1.0' }) - const res = middleware(req) - - expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth') - }) - - it('falls through for UAs that embed a match as a substring', () => { - for (const ua of ['chatgpt-userscript/2.0', 'NotPerplexityBot']) { - const req = makeRequest('/auth', { userAgent: ua }) - const res = middleware(req) - - expect(res.headers.get('x-middleware-rewrite')).toBeNull() - } - }) - - it('falls through for browser user agents', () => { - const req = makeRequest('/auth', { - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', - }) - const res = middleware(req) - - expect(res.headers.get('x-middleware-rewrite')).toBeNull() - }) - - it('falls through for training crawlers (GPTBot, ClaudeBot, CCBot)', () => { - for (const ua of ['GPTBot/1.0', 'ClaudeBot/1.0', 'CCBot/2.0']) { - const req = makeRequest('/auth', { userAgent: ua }) - const res = middleware(req) - - expect(res.headers.get('x-middleware-rewrite')).toBeNull() - } - }) - it('falls through when slug is not in the allowlist', () => { - const req = makeRequest('/not-a-page', { userAgent: 'Claude-User' }) + const req = makeRequest('/not-a-page', { accept: 'text/markdown' }) const res = middleware(req) expect(res.headers.get('x-middleware-rewrite')).toBeNull() diff --git a/apps/www/middleware.ts b/apps/www/middleware.ts index cef09208638..ab96127d5ee 100644 --- a/apps/www/middleware.ts +++ b/apps/www/middleware.ts @@ -7,25 +7,18 @@ import { MD_PAGES } from './app/api-v2/md/content.generated' export function middleware(request: NextRequest) { const { pathname } = request.nextUrl - if (pathname.endsWith('.md')) { - const slug = pathname.slice(1, -3) - if (MD_PAGES.has(slug)) { - return NextResponse.rewrite(new URL(`/api-v2/md/${slug}`, request.nextUrl)) - } - } + const isMarkdownSuffix = pathname.endsWith('.md') + const basePathname = isMarkdownSuffix ? pathname.slice(0, -3) : pathname // Strip trailing slash so /auth/ and /auth resolve to the same allowlist // entry — NextURL preserves trailing-slash style on rewrite targets. - const slug = (pathname === '/' ? 'homepage' : pathname.slice(1)).replace(/\/$/, '') + const slug = (basePathname === '/' ? 'homepage' : basePathname.slice(1)).replace(/\/$/, '') const isMdEligible = MD_PAGES.has(slug) const isChangelogEntry = slug === 'changelog' || /^changelog\/\d+/.test(slug) const decision = negotiateMarkdown( - { - acceptHeader: request.headers.get('accept') ?? '', - userAgent: request.headers.get('user-agent') ?? '', - }, - { hasMarkdownVariant: isMdEligible || isChangelogEntry } + { acceptHeader: request.headers.get('accept') ?? '' }, + { hasMarkdownVariant: isMdEligible || isChangelogEntry, isMarkdownSuffix } ) if (decision === 'not-acceptable') { diff --git a/packages/common/markdown-negotiation.test.ts b/packages/common/markdown-negotiation.test.ts index cc43407abb9..8d18d77841c 100644 --- a/packages/common/markdown-negotiation.test.ts +++ b/packages/common/markdown-negotiation.test.ts @@ -9,7 +9,7 @@ describe('negotiateMarkdown', () => { it('passes when the route has no markdown variant, regardless of other signals', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/markdown', userAgent: 'Claude-User/1.0' }, + { acceptHeader: 'text/markdown' }, { hasMarkdownVariant: false, isMarkdownSuffix: true } ) ).toBe('pass') @@ -17,60 +17,28 @@ describe('negotiateMarkdown', () => { }) describe('forced markdown', () => { - it('returns markdown for LLM user agents even when Accept rejects everything', () => { - expect( - negotiateMarkdown( - { acceptHeader: 'application/x-content-negotiation-probe', userAgent: 'Claude-User/1.0' }, - { hasMarkdownVariant: true } - ) - ).toBe('markdown') - }) - it('returns markdown for an explicit .md suffix even with an HTML-only Accept', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/html', userAgent: '' }, + { acceptHeader: 'text/html' }, { hasMarkdownVariant: true, isMarkdownSuffix: true } ) ).toBe('markdown') }) - it.each([ - 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)', - 'Claude-Web/1.0', - 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)', - 'PerplexityBot/1.0', - ])('treats %s as an LLM agent', (userAgent) => { - expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe( - 'markdown' - ) - }) - - it.each([ - 'GPTBot/1.0', - 'ClaudeBot/1.0', - 'CCBot/2.0', - 'chatgpt-userscript/2.0', - 'NotPerplexityBot', - ])('does not treat %s (training crawler / substring embed) as an LLM agent', (userAgent) => { - expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe( - 'pass' - ) - }) - - it('caps user-agent length before matching', () => { - const padded = 'x'.repeat(600) + 'Claude-User' + it('returns markdown for an explicit .md suffix even when Accept rejects everything', () => { expect( - negotiateMarkdown({ acceptHeader: '', userAgent: padded }, { hasMarkdownVariant: true }) - ).toBe('pass') + negotiateMarkdown( + { acceptHeader: 'application/x-content-negotiation-probe' }, + { hasMarkdownVariant: true, isMarkdownSuffix: true } + ) + ).toBe('markdown') }) }) describe('no Accept header', () => { it('passes (serves HTML) when no Accept header is sent', () => { - expect( - negotiateMarkdown({ acceptHeader: '', userAgent: '' }, { hasMarkdownVariant: true }) - ).toBe('pass') + expect(negotiateMarkdown({ acceptHeader: '' }, { hasMarkdownVariant: true })).toBe('pass') }) }) @@ -78,42 +46,34 @@ describe('negotiateMarkdown', () => { it('returns not-acceptable when Accept excludes every type we serve', () => { expect( negotiateMarkdown( - { acceptHeader: 'application/x-content-negotiation-probe', userAgent: '' }, + { acceptHeader: 'application/x-content-negotiation-probe' }, { hasMarkdownVariant: true } ) ).toBe('not-acceptable') }) it('does not 406 for bare */*', () => { - expect( - negotiateMarkdown({ acceptHeader: '*/*', userAgent: '' }, { hasMarkdownVariant: true }) - ).toBe('pass') + expect(negotiateMarkdown({ acceptHeader: '*/*' }, { hasMarkdownVariant: true })).toBe('pass') }) }) describe('q-value negotiation', () => { it('serves HTML for browser-style Accept', () => { expect( - negotiateMarkdown( - { acceptHeader: BROWSER_ACCEPT, userAgent: '' }, - { hasMarkdownVariant: true } - ) + negotiateMarkdown({ acceptHeader: BROWSER_ACCEPT }, { hasMarkdownVariant: true }) ).toBe('pass') }) it('serves markdown when explicitly requested', () => { expect( - negotiateMarkdown( - { acceptHeader: 'text/markdown', userAgent: '' }, - { hasMarkdownVariant: true } - ) + negotiateMarkdown({ acceptHeader: 'text/markdown' }, { hasMarkdownVariant: true }) ).toBe('markdown') }) it('serves markdown when its q-value beats html', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0', userAgent: '' }, + { acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0' }, { hasMarkdownVariant: true } ) ).toBe('markdown') @@ -122,7 +82,7 @@ describe('negotiateMarkdown', () => { it('serves HTML when its q-value beats markdown', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5', userAgent: '' }, + { acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5' }, { hasMarkdownVariant: true } ) ).toBe('pass') @@ -131,7 +91,7 @@ describe('negotiateMarkdown', () => { it('breaks an explicit md/html tie toward markdown', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/markdown, text/html, */*', userAgent: '' }, + { acceptHeader: 'text/markdown, text/html, */*' }, { hasMarkdownVariant: true } ) ).toBe('markdown') @@ -140,7 +100,7 @@ describe('negotiateMarkdown', () => { it('does not serve markdown when the client rejects it (q=0)', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/markdown;q=0, text/html;q=1.0', userAgent: '' }, + { acceptHeader: 'text/markdown;q=0, text/html;q=1.0' }, { hasMarkdownVariant: true } ) ).toBe('pass') @@ -149,7 +109,7 @@ describe('negotiateMarkdown', () => { it('tolerates OWS around the q parameter (RFC 9110)', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5', userAgent: '' }, + { acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5' }, { hasMarkdownVariant: true } ) ).toBe('pass') @@ -158,7 +118,7 @@ describe('negotiateMarkdown', () => { it('ignores out-of-range q-values (falls back to 1.0; tie -> markdown)', () => { expect( negotiateMarkdown( - { acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0', userAgent: '' }, + { acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0' }, { hasMarkdownVariant: true } ) ).toBe('markdown') diff --git a/packages/common/markdown-negotiation.ts b/packages/common/markdown-negotiation.ts index 28f8416815a..c9373d282e7 100644 --- a/packages/common/markdown-negotiation.ts +++ b/packages/common/markdown-negotiation.ts @@ -1,17 +1,9 @@ -// Live-fetch agents only. Training crawlers (GPTBot, ClaudeBot, CCBot) are -// governed by robots.txt; serving them content that differs from the HTML -// page risks SEO and cloaking penalties. -const LLM_USER_AGENT = /\bClaude-User\b|\bClaude-Web\b|\bChatGPT-User\b|\bPerplexityBot\b/i - // Media ranges (RFC 9110 §5.3.2) ordered most to least specific. const RANGES = ['text/markdown', 'text/html', 'text/*', '*/*'] as const type Range = (typeof RANGES)[number] const Q_PARAM = /^\s*q\s*=\s*([\d.]+)\s*$/i -// Cap UA length before the regex test to bound CPU on the edge hot path. -const MAX_UA_LENGTH = 512 - function isRange(s: string): s is Range { return (RANGES as readonly string[]).includes(s) } @@ -54,12 +46,17 @@ export type MarkdownDecision = 'markdown' | 'not-acceptable' | 'pass' /** * Content negotiation for routes that can serve either HTML or markdown. * - * `hasMarkdownVariant` is false for paths with no markdown representation (they - * never negotiate). `isMarkdownSuffix` forces markdown for an explicit `.md` - * request; callers that handle `.md` upstream can leave it false. + * Markdown is served only on an explicit client signal: a `.md` request + * (`isMarkdownSuffix`) or an Accept header preferring text/markdown. There is + * deliberately no user-agent detection: responses that vary by UA poison + * UA-blind CDN caches, and at least one major agent's reader hard-fails on + * markdown it did not explicitly ask for. + * + * `hasMarkdownVariant` is false for paths with no markdown representation + * (they never negotiate). */ export function negotiateMarkdown( - { acceptHeader, userAgent }: { acceptHeader: string; userAgent: string }, + { acceptHeader }: { acceptHeader: string }, { hasMarkdownVariant, isMarkdownSuffix = false, @@ -67,18 +64,14 @@ export function negotiateMarkdown( ): MarkdownDecision { if (!hasMarkdownVariant) return 'pass' - // LLM agents and an explicit `.md` request always get markdown. - if (LLM_USER_AGENT.test(userAgent.slice(0, MAX_UA_LENGTH)) || isMarkdownSuffix) { - return 'markdown' - } + if (isMarkdownSuffix) return 'markdown' // No Accept header = browser/default client: serve HTML, never 406. if (!acceptHeader) return 'pass' const accept = parseAccept(acceptHeader) - // 406 when Accept rejects every type this route can produce. Only reached for - // non-LLM, non-`.md` clients that sent an Accept header (guards above), so a + // 406 when Accept rejects every type this route can produce, so a // deliberate `Accept: application/json` gets a clean 406 instead of HTML. if (accept.markdown === 0 && accept.html === 0) return 'not-acceptable'