fix(www): respect Accept q-values and 406 unsupported types (#45394)

apps/www/middleware.test.ts

@@ -155,6 +155,115 @@ describe('www middleware', () => {
     })
   })
 
+  describe('Accept header q-value parsing', () => {
+    it('serves markdown for Cursor-style Accept (markdown preferred, plain fallback)', () => {
+      const req = makeRequest('/auth', {
+        accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8',
+      })
+      const res = middleware(req)
+
+      expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
+    })
+
+    it('serves markdown when md and html have equal q-values', () => {
+      const req = makeRequest('/auth', { accept: 'text/markdown, text/html, */*' })
+      const res = middleware(req)
+
+      expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
+    })
+
+    it('serves HTML when html q-value beats markdown q-value', () => {
+      const req = makeRequest('/auth', { accept: 'text/html;q=1.0, text/markdown;q=0.5' })
+      const res = middleware(req)
+
+      expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+    })
+
+    it('serves HTML for browser-style Accept (html with */* fallback)', () => {
+      const req = makeRequest('/auth', {
+        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+      })
+      const res = middleware(req)
+
+      expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+    })
+
+    it('serves markdown when md q-value beats html q-value', () => {
+      const req = makeRequest('/auth', { accept: 'text/html;q=0.5, text/markdown;q=1.0' })
+      const res = middleware(req)
+
+      expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
+    })
+
+    it('tolerates OWS around the q parameter (per RFC 9110)', () => {
+      const req = makeRequest('/auth', { accept: 'text/html ; q = 1.0, text/markdown ; q = 0.5' })
+      const res = middleware(req)
+
+      expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+    })
+
+    it('ignores out-of-range q-values rather than treating them as preference', () => {
+      const req = makeRequest('/auth', { accept: 'text/html;q=2.0, text/markdown;q=1.0' })
+      const res = middleware(req)
+
+      // text/html's q=2.0 is invalid and falls back to default 1.0; tie -> markdown.
+      expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
+    })
+  })
+
+  describe('406 Not Acceptable', () => {
+    it('returns 406 on MD-eligible page when Accept excludes every type we serve', () => {
+      const req = makeRequest('/pricing', { accept: 'application/x-content-negotiation-probe' })
+      const res = middleware(req)
+
+      expect(res.status).toBe(406)
+      expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+    })
+
+    it('does not return 406 on non-MD pages (no negotiation contract there)', () => {
+      const req = makeRequest('/not-a-page', { accept: 'application/x-content-negotiation-probe' })
+      const res = middleware(req)
+
+      expect(res.status).not.toBe(406)
+    })
+
+    it('does not return 406 when Accept includes */*', () => {
+      const req = makeRequest('/pricing', { accept: '*/*' })
+      const res = middleware(req)
+
+      expect(res.status).not.toBe(406)
+    })
+
+    it('does not return 406 for LLM UAs even with a probe Accept header', () => {
+      const req = makeRequest('/pricing', {
+        accept: 'application/x-content-negotiation-probe',
+        userAgent: 'Claude-User/1.0',
+      })
+      const res = middleware(req)
+
+      expect(res.status).not.toBe(406)
+      expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')
+    })
+
+    it('returns 406 on changelog entries when Accept excludes every type', () => {
+      const req = makeRequest('/changelog/100', {
+        accept: 'application/x-content-negotiation-probe',
+      })
+      const res = middleware(req)
+
+      expect(res.status).toBe(406)
+    })
+
+    it('sets Cache-Control: no-store and Vary: Accept on 406 responses', () => {
+      const req = makeRequest('/pricing', { accept: 'application/x-content-negotiation-probe' })
+      const res = middleware(req)
+
+      expect(res.status).toBe(406)
+      expect(res.headers.get('Cache-Control')).toBe('no-store')
+      expect(res.headers.get('Vary')).toBe('Accept')
+    })
+  })
+
   describe('LLM user-agent routing', () => {
     it('rewrites for Claude-User', () => {
       const req = makeRequest('/pricing', {

apps/www/middleware.ts

@@ -3,40 +3,100 @@ import { NextResponse, type NextRequest } from 'next/server'
 
 import { MD_PAGES } from './app/api-v2/md/content.generated'
 
-// Live-fetch LLM agents that retrieve pages on behalf of a user prompt.
-// Training crawlers (GPTBot, CCBot, ClaudeBot, Anthropic-AI) are intentionally
-// excluded; they are governed by robots.txt and serving them content that
-// differs from the human HTML page would risk SEO and cloaking penalties.
+// Live-fetch agents only. Training crawlers (GPTBot, ClaudeBot, CCBot) are
+// governed by robots.txt; serving them content that differs from the HTML
+// page risks SEO and cloaking penalties.
 const LLM_USER_AGENT = /\bClaude-User\b|\bClaude-Web\b|\bChatGPT-User\b|\bPerplexityBot\b/i
 
+// Media ranges (RFC 9110 §5.3.2) ordered most to least specific.
+const RANGES = ['text/markdown', 'text/html', 'text/*', '*/*'] as const
+type Range = (typeof RANGES)[number]
+
+const Q_PARAM = /^\s*q\s*=\s*([\d.]+)\s*$/i
+
+function isRange(s: string): s is Range {
+  return (RANGES as readonly string[]).includes(s)
+}
+
+function parseQ(params: string[]): number {
+  for (const p of params) {
+    const q = parseFloat(p.match(Q_PARAM)?.[1] ?? '')
+    if (Number.isFinite(q) && q >= 0 && q <= 1) return q
+  }
+  return 1
+}
+
+// `markdownExplicit` lets the caller avoid flipping a bare `Accept: */*` to
+// markdown — generic clients sending */* aren't expressing a preference.
+function parseAccept(header: string) {
+  const seen = new Map<Range, number>()
+
+  for (const entry of header.toLowerCase().split(',')) {
+    const [rawType, ...params] = entry.trim().split(';')
+    const range = rawType.trim()
+    if (!isRange(range)) continue
+    seen.set(range, Math.max(seen.get(range) ?? -1, parseQ(params)))
+  }
+
+  return {
+    html: seen.get('text/html') ?? seen.get('text/*') ?? seen.get('*/*') ?? 0,
+    markdown: seen.get('text/markdown') ?? seen.get('text/*') ?? seen.get('*/*') ?? 0,
+    markdownExplicit: seen.has('text/markdown') || seen.has('text/*'),
+  }
+}
+
+function shouldServeMarkdown(accept: ReturnType<typeof parseAccept>): boolean {
+  if (accept.markdown === 0) return false
+  if (accept.markdown > accept.html) return true
+  return accept.markdown === accept.html && accept.markdownExplicit
+}
+
 export function middleware(request: NextRequest) {
   const { pathname } = request.nextUrl
 
-  // Handle /<page>.md suffix: /pricing.md -> /api-v2/md/pricing
   if (pathname.endsWith('.md')) {
-    const slug = pathname.slice(1, -3) // strip leading / and trailing .md
+    const slug = pathname.slice(1, -3)
     if (MD_PAGES.has(slug)) {
       return NextResponse.rewrite(new URL(`/api-v2/md/${slug}`, request.nextUrl))
     }
   }
 
-  // Serve markdown to known LLM clients (Accept header or UA match).
-  // Cache-key safety: rewriting to /api-v2/md/<slug> partitions the response
-  // by path, so no Vary: User-Agent is needed.
-  const accept = (request.headers.get('accept') ?? '').toLowerCase()
+  const acceptHeader = request.headers.get('accept') ?? ''
   // Cap UA length before regex test to bound CPU on the edge hot path.
   const userAgent = (request.headers.get('user-agent') ?? '').slice(0, 512)
-  if (accept.includes('text/markdown') || LLM_USER_AGENT.test(userAgent)) {
-    // Strip trailing slash so /auth/ and /auth resolve to the same allowlist entry.
-    // (NextURL's pathname setter preserves the trailing-slash style of the cloned
-    // origin, which would otherwise leak through to the rewrite target.)
-    const slug = (pathname === '/' ? 'homepage' : pathname.slice(1)).replace(/\/$/, '')
-    if (MD_PAGES.has(slug)) {
+  const isLlmAgent = LLM_USER_AGENT.test(userAgent)
+  const accept = acceptHeader ? parseAccept(acceptHeader) : null
+
+  // Strip trailing slash so /auth/ and /auth resolve to the same allowlist
+  // entry — NextURL preserves trailing-slash style on rewrite targets.
+  const slug = (pathname === '/' ? 'homepage' : pathname.slice(1)).replace(/\/$/, '')
+  const isMdEligible = MD_PAGES.has(slug)
+  const isChangelogEntry = slug === 'changelog' || /^changelog\/\d+/.test(slug)
+  const hasMdVariant = isMdEligible || isChangelogEntry
+
+  // 406 when Accept rejects every type we can produce. Skip for LLM UAs
+  // (always served markdown) and clients with no Accept (browser default).
+  if (
+    hasMdVariant &&
+    !isLlmAgent &&
+    accept !== null &&
+    accept.markdown === 0 &&
+    accept.html === 0
+  ) {
+    return new NextResponse('Not Acceptable', {
+      status: 406,
+      headers: { 'Cache-Control': 'no-store', Vary: 'Accept' },
+    })
+  }
+
+  const wantsMarkdown = isLlmAgent || (accept !== null && shouldServeMarkdown(accept))
+
+  if (wantsMarkdown) {
+    if (isMdEligible) {
       return NextResponse.rewrite(new URL(`/api-v2/md/${slug}`, request.nextUrl))
     }
-    // Individual changelog entries are served as static .md files from public/;
-    // rewrite directly to the static path. The slug always starts with the number.
-    if (slug === 'changelog' || /^changelog\/\d+/.test(slug)) {
+    // Changelog entries are static .md files in public/, not API routes.
+    if (isChangelogEntry) {
       return NextResponse.rewrite(new URL(`/${slug}.md`, request.nextUrl))
     }
   }
@@ -48,7 +108,6 @@ export function middleware(request: NextRequest) {
 
 export const config = {
   matcher: [
-    // Match all paths except Next.js internals and static files.
     // MUST exclude _next/data to prevent full page reloads in multi-zone apps.
     '/((?!api|_next/static|_next/image|_next/data|favicon.ico|__nextjs).*)',
   ],