feat(docs): add revalidation api route (#29378)

Add a route for manually revalidating cache contents by tag.

The route is protected by authentication to prevent abuse. Automated
actions in CI should be set up with a basic API key, which has a rate
limit of 6 hours between changes. Overriding is possible with an
override key, which should be used as an escape hatch.

Usage:
- API key provided in header `Authorization: Bearer <KEY>`
- Body has shape `{ tags: string[] }`

apps/docs/app/api/revalidate/route.test.ts

@@ -0,0 +1,188 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+
+import { createClient } from '@supabase/supabase-js'
+import { revalidateTag } from 'next/cache'
+import { headers } from 'next/headers'
+import { NextRequest } from 'next/server'
+import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest'
+
+import { _handleRevalidateRequest } from './route'
+
+// Mock Next.js modules
+vi.mock('next/cache', () => ({
+  revalidateTag: vi.fn(),
+}))
+
+vi.mock('next/headers', () => ({
+  headers: vi.fn(),
+}))
+
+// Mock Supabase client
+vi.mock('@supabase/supabase-js', () => ({
+  createClient: vi.fn(),
+}))
+
+describe('_handleRevalidateRequest', () => {
+  let mockDate: Date
+  let originalEnv: NodeJS.ProcessEnv
+  let mockSupabaseClient: {
+    rpc: Mock
+    from: Mock
+  }
+
+  beforeEach(() => {
+    // Store the original environment
+    originalEnv = { ...process.env }
+
+    // Mock environment variables
+    process.env.DOCS_REVALIDATION_KEYS = 'basic_key'
+    process.env.DOCS_REVALIDATION_OVERRIDE_KEYS = 'override_key,other_override_key'
+    process.env.NEXT_PUBLIC_SUPABASE_URL = 'http://localhost:3000'
+    process.env.SUPABASE_SECRET_KEY = 'secret_key'
+
+    // Mock current date
+    mockDate = new Date('2023-01-01T12:00:00Z')
+    vi.setSystemTime(mockDate)
+
+    // Setup mock Supabase client
+    mockSupabaseClient = {
+      rpc: vi.fn(),
+      from: vi.fn(() => ({
+        insert: vi.fn().mockResolvedValue({ error: null }),
+      })),
+    }
+    vi.mocked(createClient).mockReturnValue(mockSupabaseClient as any)
+  })
+
+  afterEach(() => {
+    process.env = originalEnv
+    vi.clearAllMocks()
+    vi.useRealTimers()
+  })
+
+  it('should return 401 if Authorization header is missing', async () => {
+    const request = new NextRequest('https://example.com', {
+      method: 'POST',
+    })
+
+    vi.mocked(headers).mockReturnValue(new Headers(request.headers))
+
+    const response = await _handleRevalidateRequest(request)
+    expect(response.status).toBe(401)
+    expect(await response.text()).toBe('Missing Authorization header')
+  })
+
+  it('should return 401 if Authorization header is invalid', async () => {
+    const request = new NextRequest('https://example.com', {
+      method: 'POST',
+      headers: {
+        Authorization: 'Bearer invalid_token',
+      },
+    })
+
+    vi.mocked(headers).mockReturnValue(new Headers(request.headers))
+
+    const response = await _handleRevalidateRequest(request)
+    expect(response.status).toBe(401)
+    expect(await response.text()).toBe('Invalid Authorization header')
+  })
+
+  it('should return 400 if request body is malformed', async () => {
+    const request = new NextRequest('https://example.com', {
+      method: 'POST',
+      headers: {
+        Authorization: 'Bearer basic_key',
+      },
+      body: JSON.stringify({ invalid: 'body' }),
+    })
+
+    vi.mocked(headers).mockReturnValue(new Headers(request.headers))
+
+    const response = await _handleRevalidateRequest(request)
+    expect(response.status).toBe(400)
+    expect(await response.text()).toContain('Malformed request body')
+  })
+
+  it('should revalidate tags if request is valid with basic permissions', async () => {
+    const request = new NextRequest('https://example.com', {
+      method: 'POST',
+      headers: {
+        Authorization: 'Bearer basic_key',
+      },
+      body: JSON.stringify({ tags: ['tag1', 'tag2'] }),
+    })
+
+    vi.mocked(headers).mockReturnValue(new Headers(request.headers))
+
+    mockSupabaseClient.rpc.mockResolvedValue({ data: [] })
+
+    const response = await _handleRevalidateRequest(request)
+    expect(response.status).toBe(204)
+    expect(revalidateTag).toHaveBeenCalledTimes(2)
+    expect(revalidateTag).toHaveBeenCalledWith('tag1')
+    expect(revalidateTag).toHaveBeenCalledWith('tag2')
+  })
+
+  it('should return 429 if last revalidation was less than 6 hours ago with basic permissions', async () => {
+    const request = new NextRequest('https://example.com', {
+      method: 'POST',
+      headers: {
+        Authorization: 'Bearer basic_key',
+      },
+      body: JSON.stringify({ tags: ['tag1'] }),
+    })
+
+    vi.mocked(headers).mockReturnValue(new Headers(request.headers))
+
+    const fiveHoursAgo = new Date(mockDate.getTime() - 5 * 60 * 60 * 1000)
+    mockSupabaseClient.rpc.mockResolvedValue({
+      data: [{ created_at: fiveHoursAgo.toISOString() }],
+    })
+
+    const response = await _handleRevalidateRequest(request)
+    expect(response.status).toBe(429)
+    expect(await response.text()).toContain('revalidated within the last 6 hours')
+  })
+
+  it('should revalidate if last revalidation was more than 6 hours ago with basic permissions', async () => {
+    const request = new NextRequest('https://example.com', {
+      method: 'POST',
+      headers: {
+        Authorization: 'Bearer basic_key',
+      },
+      body: JSON.stringify({ tags: ['tag1'] }),
+    })
+
+    vi.mocked(headers).mockReturnValue(new Headers(request.headers))
+
+    const sevenHoursAgo = new Date(mockDate.getTime() - 7 * 60 * 60 * 1000)
+    mockSupabaseClient.rpc.mockResolvedValue({
+      data: [{ created_at: sevenHoursAgo.toISOString() }],
+    })
+
+    const response = await _handleRevalidateRequest(request)
+    expect(response.status).toBe(204)
+    expect(revalidateTag).toHaveBeenCalledWith('tag1')
+  })
+
+  it('should revalidate regardless of last revalidation time with override permissions', async () => {
+    const request = new NextRequest('https://example.com', {
+      method: 'POST',
+      headers: {
+        Authorization: 'Bearer override_key',
+      },
+      body: JSON.stringify({ tags: ['tag1'] }),
+    })
+
+    vi.mocked(headers).mockReturnValue(new Headers(request.headers))
+
+    const oneHourAgo = new Date(mockDate.getTime() - 1 * 60 * 60 * 1000)
+    mockSupabaseClient.rpc.mockResolvedValue({
+      data: [{ created_at: oneHourAgo.toISOString() }],
+    })
+
+    const response = await _handleRevalidateRequest(request)
+    expect(response.status).toBe(204)
+    expect(revalidateTag).toHaveBeenCalledWith('tag1')
+  })
+})

apps/docs/app/api/revalidate/route.ts

@@ -0,0 +1,116 @@
+import { createClient } from '@supabase/supabase-js'
+import { revalidateTag } from 'next/cache'
+import { headers } from 'next/headers'
+import { type NextRequest } from 'next/server'
+import { z } from 'zod'
+
+import { type Database } from 'common'
+
+enum AuthorizationLevel {
+  Unauthorized,
+  Basic,
+  Override,
+}
+
+const requestBodySchema = z.object({
+  tags: z.array(z.string()),
+})
+
+export const POST = handleError(_handleRevalidateRequest)
+
+export async function _handleRevalidateRequest(request: NextRequest) {
+  const requestHeaders = headers()
+  const authorization = requestHeaders.get('Authorization')
+  if (!authorization) {
+    return new Response('Missing Authorization header', { status: 401 })
+  }
+
+  const basicKeys = process.env.DOCS_REVALIDATION_KEYS?.split(/\s*,\s*/) ?? []
+  const overrideKeys = process.env.DOCS_REVALIDATION_OVERRIDE_KEYS?.split(/\s*,\s*/) ?? []
+  if (basicKeys.length === 0 && overrideKeys.length === 0) {
+    console.error('No keys configured for revalidation')
+    return new Response('Internal server error', {
+      status: 500,
+    })
+  }
+
+  let authorizationLevel = AuthorizationLevel.Unauthorized
+
+  const token = authorization.replace(/^Bearer\s+/, '')
+  if (overrideKeys.includes(token)) {
+    authorizationLevel = AuthorizationLevel.Override
+  } else if (basicKeys.includes(token)) {
+    authorizationLevel = AuthorizationLevel.Basic
+  }
+
+  if (authorizationLevel === AuthorizationLevel.Unauthorized) {
+    return new Response('Invalid Authorization header', { status: 401 })
+  }
+
+  const result = requestBodySchema.safeParse(await request.json())
+  if (!result.success) {
+    return new Response(
+      'Malformed request body: should be a JSON object with a "tags" array of strings.',
+      { status: 400 }
+    )
+  }
+
+  const supabaseAdmin = createClient<Database>(
+    process.env.NEXT_PUBLIC_SUPABASE_URL!,
+    process.env.SUPABASE_SECRET_KEY!
+  )
+
+  if (authorizationLevel === AuthorizationLevel.Basic) {
+    const { data: lastRevalidation, error } = await supabaseAdmin.rpc(
+      'get_last_revalidation_for_tags',
+      {
+        tags: result.data.tags,
+      }
+    )
+    if (error) {
+      console.error(error)
+      return new Response('Internal server error', { status: 500 })
+    }
+
+    const sixHoursAgo = new Date()
+    sixHoursAgo.setHours(sixHoursAgo.getHours() - 6)
+    if (lastRevalidation?.some((revalidation) => new Date(revalidation.created_at) > sixHoursAgo)) {
+      return new Response(
+        'Your request includes a tag that has been revalidated within the last 6 hours. You can override this limit by authenticating with Override permissions.',
+        {
+          status: 429,
+        }
+      )
+    }
+  }
+
+  const { error } = await supabaseAdmin
+    .from('validation_history')
+    .insert(result.data.tags.map((tag) => ({ tag })))
+  if (error) {
+    console.error('Failed to update revalidation table: %o', error)
+  }
+
+  result.data.tags.forEach((tag) => {
+    revalidateTag(tag)
+  })
+
+  return new Response(null, {
+    status: 204,
+    headers: {
+      'Cache-Control': 'no-cache',
+    },
+  })
+}
+
+function handleError(handleRequest: (request: NextRequest) => Promise<Response>) {
+  return async function (request: NextRequest) {
+    try {
+      const response = await handleRequest(request)
+      return response
+    } catch (error) {
+      console.error(error)
+      return new Response('Internal server error', { status: 500 })
+    }
+  }
+}

packages/common/database-types.ts

@@ -58,6 +58,101 @@ export type Database = {
         }
         Relationships: []
       }
+      last_changed: {
+        Row: {
+          checksum: string
+          heading: string
+          id: number
+          last_checked: string
+          last_updated: string
+          parent_page: string
+        }
+        Insert: {
+          checksum: string
+          heading: string
+          id?: never
+          last_checked?: string
+          last_updated?: string
+          parent_page: string
+        }
+        Update: {
+          checksum?: string
+          heading?: string
+          id?: never
+          last_checked?: string
+          last_updated?: string
+          parent_page?: string
+        }
+        Relationships: []
+      }
+      launch_weeks: {
+        Row: {
+          created_at: string
+          end_date: string | null
+          id: string
+          start_date: string | null
+        }
+        Insert: {
+          created_at?: string
+          end_date?: string | null
+          id: string
+          start_date?: string | null
+        }
+        Update: {
+          created_at?: string
+          end_date?: string | null
+          id?: string
+          start_date?: string | null
+        }
+        Relationships: []
+      }
+      meetups: {
+        Row: {
+          country: string | null
+          created_at: string
+          display_info: string | null
+          id: string
+          is_live: boolean
+          is_published: boolean
+          launch_week: string
+          link: string | null
+          start_at: string | null
+          title: string | null
+        }
+        Insert: {
+          country?: string | null
+          created_at?: string
+          display_info?: string | null
+          id?: string
+          is_live?: boolean
+          is_published?: boolean
+          launch_week: string
+          link?: string | null
+          start_at?: string | null
+          title?: string | null
+        }
+        Update: {
+          country?: string | null
+          created_at?: string
+          display_info?: string | null
+          id?: string
+          is_live?: boolean
+          is_published?: boolean
+          launch_week?: string
+          link?: string | null
+          start_at?: string | null
+          title?: string | null
+        }
+        Relationships: [
+          {
+            foreignKeyName: "meetups_launch_week_fkey"
+            columns: ["launch_week"]
+            isOneToOne: false
+            referencedRelation: "launch_weeks"
+            referencedColumns: ["id"]
+          },
+        ]
+      }
       page: {
         Row: {
           checksum: string | null
@@ -141,11 +236,175 @@ export type Database = {
           },
         ]
       }
+      tickets: {
+        Row: {
+          company: string | null
+          created_at: string
+          email: string | null
+          game_won_at: string | null
+          id: string
+          launch_week: string
+          location: string | null
+          metadata: Json | null
+          name: string | null
+          referred_by: string | null
+          role: string | null
+          shared_on_linkedin: string | null
+          shared_on_twitter: string | null
+          ticket_number: number
+          user_id: string
+          username: string | null
+        }
+        Insert: {
+          company?: string | null
+          created_at?: string
+          email?: string | null
+          game_won_at?: string | null
+          id?: string
+          launch_week: string
+          location?: string | null
+          metadata?: Json | null
+          name?: string | null
+          referred_by?: string | null
+          role?: string | null
+          shared_on_linkedin?: string | null
+          shared_on_twitter?: string | null
+          ticket_number?: number
+          user_id: string
+          username?: string | null
+        }
+        Update: {
+          company?: string | null
+          created_at?: string
+          email?: string | null
+          game_won_at?: string | null
+          id?: string
+          launch_week?: string
+          location?: string | null
+          metadata?: Json | null
+          name?: string | null
+          referred_by?: string | null
+          role?: string | null
+          shared_on_linkedin?: string | null
+          shared_on_twitter?: string | null
+          ticket_number?: number
+          user_id?: string
+          username?: string | null
+        }
+        Relationships: [
+          {
+            foreignKeyName: "public_tickets_id_fkey"
+            columns: ["user_id"]
+            isOneToOne: false
+            referencedRelation: "users"
+            referencedColumns: ["id"]
+          },
+          {
+            foreignKeyName: "tickets_launch_week_fkey"
+            columns: ["launch_week"]
+            isOneToOne: false
+            referencedRelation: "launch_weeks"
+            referencedColumns: ["id"]
+          },
+          {
+            foreignKeyName: "tickets_user_id_fkey"
+            columns: ["user_id"]
+            isOneToOne: false
+            referencedRelation: "users"
+            referencedColumns: ["id"]
+          },
+        ]
+      }
+      troubleshooting_entries: {
+        Row: {
+          api: Json | null
+          date_created: string
+          date_updated: string
+          errors: Json[] | null
+          github_url: string
+          id: string
+          keywords: string[] | null
+          title: string
+          topics: string[]
+        }
+        Insert: {
+          api?: Json | null
+          date_created?: string
+          date_updated?: string
+          errors?: Json[] | null
+          github_url: string
+          id?: string
+          keywords?: string[] | null
+          title: string
+          topics: string[]
+        }
+        Update: {
+          api?: Json | null
+          date_created?: string
+          date_updated?: string
+          errors?: Json[] | null
+          github_url?: string
+          id?: string
+          keywords?: string[] | null
+          title?: string
+          topics?: string[]
+        }
+        Relationships: []
+      }
+      validation_history: {
+        Row: {
+          created_at: string
+          id: number
+          tag: string
+        }
+        Insert: {
+          created_at?: string
+          id?: never
+          tag: string
+        }
+        Update: {
+          created_at?: string
+          id?: never
+          tag?: string
+        }
+        Relationships: []
+      }
     }
     Views: {
-      [_ in never]: never
+      tickets_view: {
+        Row: {
+          company: string | null
+          created_at: string | null
+          id: string | null
+          launch_week: string | null
+          location: string | null
+          metadata: Json | null
+          name: string | null
+          platinum: boolean | null
+          referrals: number | null
+          role: string | null
+          secret: boolean | null
+          shared_on_linkedin: string | null
+          shared_on_twitter: string | null
+          ticket_number: number | null
+          username: string | null
+        }
+        Relationships: [
+          {
+            foreignKeyName: "tickets_launch_week_fkey"
+            columns: ["launch_week"]
+            isOneToOne: false
+            referencedRelation: "launch_weeks"
+            referencedColumns: ["id"]
+          },
+        ]
+      }
     }
     Functions: {
+      cleanup_last_changed_pages: {
+        Args: Record<PropertyKey, never>
+        Returns: number
+      }
       docs_search_embeddings: {
         Args: {
           embedding: string
@@ -175,15 +434,13 @@ export type Database = {
           description: string
         }[]
       }
-      get_page_parents: {
+      get_last_revalidation_for_tags: {
         Args: {
-          page_id: number
+          tags: string[]
         }
         Returns: {
-          id: number
-          parent_page_id: number
-          path: string
-          meta: Json
+          tag: string
+          created_at: string
         }[]
       }
       hnswhandler: {
@@ -207,22 +464,6 @@ export type Database = {
         }
         Returns: unknown
       }
-      match_page_sections: {
-        Args: {
-          embedding: string
-          match_threshold: number
-          match_count: number
-          min_content_length: number
-        }
-        Returns: {
-          id: number
-          page_id: number
-          slug: string
-          heading: string
-          content: string
-          similarity: number
-        }[]
-      }
       match_page_sections_v2: {
         Args: {
           embedding: string
@@ -240,6 +481,16 @@ export type Database = {
           token_count: number | null
         }[]
       }
+      update_last_changed_checksum: {
+        Args: {
+          new_parent_page: string
+          new_heading: string
+          new_checksum: string
+          git_update_time: string
+          check_time: string
+        }
+        Returns: string
+      }
       vector_avg: {
         Args: {
           "": number[]

packages/common/package.json

@@ -5,6 +5,7 @@
   "types": "./index.tsx",
   "license": "MIT",
   "scripts": {
+    "gen:types": "supabase gen types typescript --local >| ./database-types.ts",
     "typecheck_CURRENTLY_IGNORED": "tsc --noEmit"
   },
   "dependencies": {

supabase/migrations/20240918220938_validation_history.sql

@@ -0,0 +1,24 @@
+create table validation_history (
+  id bigint generated always as identity primary key,
+  tag text not null,
+  created_at timestamp with time zone not null default now()
+);
+
+create index validation_history_tag_created_at_idx on validation_history (tag, created_at desc);
+
+alter table validation_history enable row level security;
+
+create or replace function get_last_revalidation_for_tags(tags text[])
+returns table (
+  tag text,
+  created_at timestamp with time zone
+)
+language sql
+as $$
+  select
+    tag,
+    max(created_at) as created_at
+  from validation_history
+  where tag = any(tags)
+  group by tag;
+$$;

turbo.json

@@ -13,7 +13,15 @@
     },
     "docs#build": {
       "dependsOn": ["^build"],
-      "env": ["ANALYZE", "NEXT_PUBLIC_*", "NODE_ENV", "OPENAI_API_KEY"],
+      "env": [
+        "ANALYZE",
+        "DOCS_REVALIDATION_KEYS",
+        "DOCS_REVALIDATION_OVERRIDE_KEYS",
+        "NEXT_PUBLIC_*",
+        "NODE_ENV",
+        "OPENAI_API_KEY",
+        "SUPABASE_SECRET_KEY"
+      ],
       "outputs": [".next/**", "!.next/cache/**"]
     },
     "studio#build": {