diff --git a/apps/www/middleware.test.ts b/apps/www/middleware.test.ts index 53310810985..dbeed5d70ec 100644 --- a/apps/www/middleware.test.ts +++ b/apps/www/middleware.test.ts @@ -155,6 +155,115 @@ describe('www middleware', () => { }) }) + describe('Accept header q-value parsing', () => { + it('serves markdown for Cursor-style Accept (markdown preferred, plain fallback)', () => { + const req = makeRequest('/auth', { + accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8', + }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth') + }) + + it('serves markdown when md and html have equal q-values', () => { + const req = makeRequest('/auth', { accept: 'text/markdown, text/html, */*' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth') + }) + + it('serves HTML when html q-value beats markdown q-value', () => { + const req = makeRequest('/auth', { accept: 'text/html;q=1.0, text/markdown;q=0.5' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBeNull() + }) + + it('serves HTML for browser-style Accept (html with */* fallback)', () => { + const req = makeRequest('/auth', { + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBeNull() + }) + + it('serves markdown when md q-value beats html q-value', () => { + const req = makeRequest('/auth', { accept: 'text/html;q=0.5, text/markdown;q=1.0' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth') + }) + + it('tolerates OWS around the q parameter (per RFC 9110)', () => { + const req = makeRequest('/auth', { accept: 'text/html ; q = 1.0, text/markdown ; q = 0.5' }) + const res = middleware(req) + + expect(res.headers.get('x-middleware-rewrite')).toBeNull() + }) + + it('ignores out-of-range q-values rather than treating them as preference', () => { + const req = makeRequest('/auth', { accept: 'text/html;q=2.0, text/markdown;q=1.0' }) + const res = middleware(req) + + // text/html's q=2.0 is invalid and falls back to default 1.0; tie -> markdown. + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth') + }) + }) + + describe('406 Not Acceptable', () => { + it('returns 406 on MD-eligible page when Accept excludes every type we serve', () => { + const req = makeRequest('/pricing', { accept: 'application/x-content-negotiation-probe' }) + const res = middleware(req) + + expect(res.status).toBe(406) + expect(res.headers.get('x-middleware-rewrite')).toBeNull() + }) + + it('does not return 406 on non-MD pages (no negotiation contract there)', () => { + const req = makeRequest('/not-a-page', { accept: 'application/x-content-negotiation-probe' }) + const res = middleware(req) + + expect(res.status).not.toBe(406) + }) + + it('does not return 406 when Accept includes */*', () => { + const req = makeRequest('/pricing', { accept: '*/*' }) + const res = middleware(req) + + expect(res.status).not.toBe(406) + }) + + it('does not return 406 for LLM UAs even with a probe Accept header', () => { + const req = makeRequest('/pricing', { + accept: 'application/x-content-negotiation-probe', + userAgent: 'Claude-User/1.0', + }) + const res = middleware(req) + + expect(res.status).not.toBe(406) + expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing') + }) + + it('returns 406 on changelog entries when Accept excludes every type', () => { + const req = makeRequest('/changelog/100', { + accept: 'application/x-content-negotiation-probe', + }) + const res = middleware(req) + + expect(res.status).toBe(406) + }) + + it('sets Cache-Control: no-store and Vary: Accept on 406 responses', () => { + const req = makeRequest('/pricing', { accept: 'application/x-content-negotiation-probe' }) + const res = middleware(req) + + expect(res.status).toBe(406) + expect(res.headers.get('Cache-Control')).toBe('no-store') + expect(res.headers.get('Vary')).toBe('Accept') + }) + }) + describe('LLM user-agent routing', () => { it('rewrites for Claude-User', () => { const req = makeRequest('/pricing', { diff --git a/apps/www/middleware.ts b/apps/www/middleware.ts index e83020bf965..095fa5f3a67 100644 --- a/apps/www/middleware.ts +++ b/apps/www/middleware.ts @@ -3,40 +3,100 @@ import { NextResponse, type NextRequest } from 'next/server' import { MD_PAGES } from './app/api-v2/md/content.generated' -// Live-fetch LLM agents that retrieve pages on behalf of a user prompt. -// Training crawlers (GPTBot, CCBot, ClaudeBot, Anthropic-AI) are intentionally -// excluded; they are governed by robots.txt and serving them content that -// differs from the human HTML page would risk SEO and cloaking penalties. +// Live-fetch agents only. Training crawlers (GPTBot, ClaudeBot, CCBot) are +// governed by robots.txt; serving them content that differs from the HTML +// page risks SEO and cloaking penalties. const LLM_USER_AGENT = /\bClaude-User\b|\bClaude-Web\b|\bChatGPT-User\b|\bPerplexityBot\b/i +// Media ranges (RFC 9110 §5.3.2) ordered most to least specific. +const RANGES = ['text/markdown', 'text/html', 'text/*', '*/*'] as const +type Range = (typeof RANGES)[number] + +const Q_PARAM = /^\s*q\s*=\s*([\d.]+)\s*$/i + +function isRange(s: string): s is Range { + return (RANGES as readonly string[]).includes(s) +} + +function parseQ(params: string[]): number { + for (const p of params) { + const q = parseFloat(p.match(Q_PARAM)?.[1] ?? '') + if (Number.isFinite(q) && q >= 0 && q <= 1) return q + } + return 1 +} + +// `markdownExplicit` lets the caller avoid flipping a bare `Accept: */*` to +// markdown — generic clients sending */* aren't expressing a preference. +function parseAccept(header: string) { + const seen = new Map() + + for (const entry of header.toLowerCase().split(',')) { + const [rawType, ...params] = entry.trim().split(';') + const range = rawType.trim() + if (!isRange(range)) continue + seen.set(range, Math.max(seen.get(range) ?? -1, parseQ(params))) + } + + return { + html: seen.get('text/html') ?? seen.get('text/*') ?? seen.get('*/*') ?? 0, + markdown: seen.get('text/markdown') ?? seen.get('text/*') ?? seen.get('*/*') ?? 0, + markdownExplicit: seen.has('text/markdown') || seen.has('text/*'), + } +} + +function shouldServeMarkdown(accept: ReturnType): boolean { + if (accept.markdown === 0) return false + if (accept.markdown > accept.html) return true + return accept.markdown === accept.html && accept.markdownExplicit +} + export function middleware(request: NextRequest) { const { pathname } = request.nextUrl - // Handle /.md suffix: /pricing.md -> /api-v2/md/pricing if (pathname.endsWith('.md')) { - const slug = pathname.slice(1, -3) // strip leading / and trailing .md + const slug = pathname.slice(1, -3) if (MD_PAGES.has(slug)) { return NextResponse.rewrite(new URL(`/api-v2/md/${slug}`, request.nextUrl)) } } - // Serve markdown to known LLM clients (Accept header or UA match). - // Cache-key safety: rewriting to /api-v2/md/ partitions the response - // by path, so no Vary: User-Agent is needed. - const accept = (request.headers.get('accept') ?? '').toLowerCase() + const acceptHeader = request.headers.get('accept') ?? '' // Cap UA length before regex test to bound CPU on the edge hot path. const userAgent = (request.headers.get('user-agent') ?? '').slice(0, 512) - if (accept.includes('text/markdown') || LLM_USER_AGENT.test(userAgent)) { - // Strip trailing slash so /auth/ and /auth resolve to the same allowlist entry. - // (NextURL's pathname setter preserves the trailing-slash style of the cloned - // origin, which would otherwise leak through to the rewrite target.) - const slug = (pathname === '/' ? 'homepage' : pathname.slice(1)).replace(/\/$/, '') - if (MD_PAGES.has(slug)) { + const isLlmAgent = LLM_USER_AGENT.test(userAgent) + const accept = acceptHeader ? parseAccept(acceptHeader) : null + + // Strip trailing slash so /auth/ and /auth resolve to the same allowlist + // entry — NextURL preserves trailing-slash style on rewrite targets. + const slug = (pathname === '/' ? 'homepage' : pathname.slice(1)).replace(/\/$/, '') + const isMdEligible = MD_PAGES.has(slug) + const isChangelogEntry = slug === 'changelog' || /^changelog\/\d+/.test(slug) + const hasMdVariant = isMdEligible || isChangelogEntry + + // 406 when Accept rejects every type we can produce. Skip for LLM UAs + // (always served markdown) and clients with no Accept (browser default). + if ( + hasMdVariant && + !isLlmAgent && + accept !== null && + accept.markdown === 0 && + accept.html === 0 + ) { + return new NextResponse('Not Acceptable', { + status: 406, + headers: { 'Cache-Control': 'no-store', Vary: 'Accept' }, + }) + } + + const wantsMarkdown = isLlmAgent || (accept !== null && shouldServeMarkdown(accept)) + + if (wantsMarkdown) { + if (isMdEligible) { return NextResponse.rewrite(new URL(`/api-v2/md/${slug}`, request.nextUrl)) } - // Individual changelog entries are served as static .md files from public/; - // rewrite directly to the static path. The slug always starts with the number. - if (slug === 'changelog' || /^changelog\/\d+/.test(slug)) { + // Changelog entries are static .md files in public/, not API routes. + if (isChangelogEntry) { return NextResponse.rewrite(new URL(`/${slug}.md`, request.nextUrl)) } } @@ -48,7 +108,6 @@ export function middleware(request: NextRequest) { export const config = { matcher: [ - // Match all paths except Next.js internals and static files. // MUST exclude _next/data to prevent full page reloads in multi-zone apps. '/((?!api|_next/static|_next/image|_next/data|favicon.ico|__nextjs).*)', ],