ci(docs): sync script (#36001)

Add a script for syncing error codes from the repo to the database. This
is part of the newly created rootSync script, where all sync scripts
should be moved eventually.

.github/workflows/docs-sync.yml

@@ -0,0 +1,49 @@
+# All syncs from the repo to the DB will eventually be consolidated in a single
+# script run by this workflow.
+
+name: docs_sync
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      # Resync if the content changes
+      - 'apps/docs/content/**'
+      # Resync if the resource definition or sync scripts change
+      - 'apps/docs/resources/**'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+
+    env:
+      NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }}
+      SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            apps/docs
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        with:
+          run_install: false
+
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'pnpm'
+
+      - name: Install dependencies
+        run: pnpm i
+
+      - name: Run sync script
+        run: pnpm run -F docs sync

apps/docs/app/api/utils.ts

@@ -1,5 +1,6 @@
 import { type PostgrestError } from '@supabase/supabase-js'
 import { type ZodError } from 'zod'
+import { isObject } from '~/features/helpers.misc'
 
 type ObjectOrNever = object | never
 
@@ -63,6 +64,26 @@ export class NoDataError<Details extends ObjectOrNever = never> extends ApiError
   }
 }
 
+export class MultiError<ErrorType = unknown, Details extends ObjectOrNever = never> extends Error {
+  constructor(
+    message: string,
+    cause?: Array<ErrorType>,
+    public details?: Details
+  ) {
+    super(message, { cause })
+  }
+
+  get totalErrors(): number {
+    return (this.cause as Array<ErrorType>)?.length || 0
+  }
+
+  appendError(message: string, error: ErrorType): this {
+    this.message = `${this.message}\n\t${message}`
+    ;((this.cause ?? (this.cause = [])) as Array<ErrorType>).push(error)
+    return this
+  }
+}
+
 export function convertUnknownToApiError(error: unknown): ApiError {
   return new ApiError('Unknown error', error)
 }
@@ -82,3 +103,10 @@ export function convertZodToInvalidRequestError(
 
   return new InvalidRequestError(message, error)
 }
+
+export function extractMessageFromAnyError(error: unknown): string {
+  if (isObject(error) && 'message' in error && typeof error.message === 'string') {
+    return error.message
+  }
+  return String(error)
+}

apps/docs/features/helpers.fn.ts

@@ -1,3 +1,5 @@
+import { extractMessageFromAnyError, MultiError } from '~/app/api/utils'
+
 const EMPTY_ARRAY = new Array(0)
 export function getEmptyArray() {
   return EMPTY_ARRAY
@@ -40,6 +42,30 @@ export class Result<Ok, Error> {
     return new Result<Ok, Error>({ data: null, error })
   }
 
+  static tryCatchSync<Ok, Error, Args extends Array<unknown>>(
+    fn: (...args: Args) => Ok,
+    onError: (error: unknown) => Error,
+    ...args: Args
+  ): Result<Ok, Error> {
+    try {
+      return Result.ok(fn(...args))
+    } catch (error: unknown) {
+      return Result.error(onError(error))
+    }
+  }
+
+  static async tryCatch<Ok, Error, Args extends Array<unknown>>(
+    fn: (...args: Args) => Promise<Ok>,
+    onError: (error: unknown) => Error,
+    ...args: Args
+  ): Promise<Result<Ok, Error>> {
+    try {
+      return Result.ok(await fn(...args))
+    } catch (error: unknown) {
+      return Result.error(onError(error))
+    }
+  }
+
   static async tryCatchFlat<
     Ok,
     Args extends Array<unknown> = [],
@@ -57,6 +83,27 @@ export class Result<Ok, Error> {
     }
   }
 
+  static transposeArray<Ok, Error>(
+    array: Array<Result<Ok, Error>>
+  ): Result<Array<Ok>, MultiError<Error>> {
+    let data: Array<Ok> = new Array(array.length)
+    let error: MultiError | null = null
+
+    for (const result of array) {
+      if (result.isOk()) {
+        data.push(result.internal.data!)
+      } else {
+        ;(error ??= new MultiError('MultiError:')).appendError(
+          extractMessageFromAnyError(error),
+          result.internal.error
+        )
+      }
+    }
+
+    if (error) return Result.error(error)
+    return Result.ok(data)
+  }
+
   isOk(): this is Result<Ok, never> {
     return this.internal.error == null
   }

apps/docs/features/helpers.misc.ts

@@ -1,3 +1,7 @@
+export function isObject(value: unknown): value is object {
+  return !!value && typeof value === 'object' && !Array.isArray(value)
+}
+
 export function isPlainObject(value: unknown): value is Record<string, unknown> {
   if (!value || typeof value !== 'object' || Array.isArray(value)) {
     return false

apps/docs/features/ui/AuthErrorCodes.tsx

@@ -1,11 +1,7 @@
 import Link from 'next/link'
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from 'ui'
 import _errorCodes from '~/content/errorCodes/authErrorCodes.toml'
-
-interface ErrorCodeDefinition {
-  description: string
-  references: Array<{ href: string; description: string }>
-}
+import { type ErrorCodeDefinition } from '~/resources/error/errorTypes'
 
 const errorCodes: Record<string, ErrorCodeDefinition> = _errorCodes
 

apps/docs/lib/docs.ts

@@ -6,16 +6,16 @@ import { basename, extname, join, sep } from 'node:path'
 import rehypeKatex from 'rehype-katex'
 import remarkGfm from 'remark-gfm'
 import remarkMath from 'remark-math'
-
-import { SerializeOptions } from '~/types/next-mdx-remote-serialize'
+import { type SerializeOptions } from '~/types/next-mdx-remote-serialize'
 
 // MUST be process.cwd() here, not import.meta.url, or files that are added
 // with outputFileTracingIncludes (not auto-traced) will not be found at
 // runtime.
 export const DOCS_DIRECTORY = process.cwd()
+export const CONTENT_DIRECTORY = join(DOCS_DIRECTORY, 'content')
 export const EXAMPLES_DIRECTORY = join(DOCS_DIRECTORY, 'examples')
-export const GUIDES_DIRECTORY = join(DOCS_DIRECTORY, 'content/guides')
-export const PARTIALS_DIRECTORY = join(DOCS_DIRECTORY, 'content/_partials')
+export const GUIDES_DIRECTORY = join(CONTENT_DIRECTORY, 'guides')
+export const PARTIALS_DIRECTORY = join(CONTENT_DIRECTORY, '_partials')
 export const REF_DOCS_DIRECTORY = join(DOCS_DIRECTORY, 'docs/ref')
 export const SPEC_DIRECTORY = join(DOCS_DIRECTORY, 'spec')
 

apps/docs/package.json

@@ -29,6 +29,7 @@
     "pretest": "pnpm run codegen:examples",
     "pretypecheck": "pnpm run codegen:graphql",
     "start": "next start",
+    "sync": "tsx --conditions=react-server ./resources/rootSync.ts",
     "test": "pnpm supabase start && pnpm run test:local && pnpm supabase stop",
     "test:local": "vitest --exclude \"**/*.smoke.test.ts\"",
     "test:smoke": "pnpm run codegen:references && vitest -t \"prod smoke test\"",

apps/docs/resources/error/errorSync.ts

@@ -0,0 +1,190 @@
+import { type PostgrestError } from '@supabase/supabase-js'
+import { readFile } from 'node:fs/promises'
+import path from 'node:path'
+import util, { styleText } from 'node:util'
+import { parse } from 'smol-toml'
+import { Service } from '../../__generated__/graphql'
+import { extractMessageFromAnyError, MultiError } from '../../app/api/utils'
+import { Result } from '../../features/helpers.fn'
+import { CONTENT_DIRECTORY } from '../../lib/docs'
+import { DatabaseCorrected } from '../../lib/supabase'
+import { supabaseAdmin } from '../../lib/supabaseAdmin'
+import { type ErrorCodeDefinition } from './errorTypes'
+
+type ErrorCodeUploadParameters =
+  DatabaseCorrected['content']['Functions']['update_error_code']['Args']
+
+const ERROR_CODES_DIRECTORY = path.join(CONTENT_DIRECTORY, 'errorCodes')
+
+async function doFetchErrorCodes(
+  file: string,
+  service: Service
+): Promise<Result<Array<ErrorCodeUploadParameters>, Error>> {
+  return (
+    await Result.tryCatch(
+      () => readFile(path.join(ERROR_CODES_DIRECTORY, file), 'utf8'),
+      (error) =>
+        new Error(`Failed to read error code file ${file}: ${extractMessageFromAnyError(error)}`, {
+          cause: error,
+        })
+    )
+  )
+    .flatMap((toml) =>
+      Result.tryCatchSync(
+        () => parse(toml) as unknown as Record<string, ErrorCodeDefinition>,
+        (error) =>
+          new Error(
+            `Failed to parse error code file ${file}: ${extractMessageFromAnyError(error)}`,
+            { cause: error }
+          )
+      )
+    )
+    .map((data) =>
+      Object.entries(data).map(([code, definition]) => ({
+        code,
+        service,
+        message: definition.description,
+        metadata: {
+          references: definition.references,
+        },
+      }))
+    )
+}
+
+async function fetchErrorCodes(): Promise<Result<Array<ErrorCodeUploadParameters>, MultiError>> {
+  const arrayOfResults = await Promise.all([doFetchErrorCodes('authErrorCodes.toml', Service.Auth)])
+  return Result.transposeArray(arrayOfResults).map((result) => result.flat())
+}
+
+/**
+ * Uploading the error codes to the database results in an array of Results.
+ *
+ * Handles each result and check whether it is a success or failure.
+ * - If success, increment the count of upserted rows.
+ * - If failure, add to a list of errors.
+ */
+function handleErrorCodeUploadErrors(
+  results: Array<Result<boolean, PostgrestError>>,
+  errorCodes: Array<{ code: string }>
+): [number, MultiError | undefined] {
+  return results.reduce(
+    ([numberUpsertedRows, error], current, index) => {
+      return current.match(
+        (wasUpserted) =>
+          wasUpserted ? [numberUpsertedRows + 1, error] : [numberUpsertedRows, error],
+        (currentError) => [
+          numberUpsertedRows,
+          (
+            error ?? new MultiError('All errors encountered when uploading error codes:')
+          ).appendError(
+            util.format(
+              'Error uploading error code %s: %s',
+              errorCodes[index].code,
+              currentError.message
+            ),
+            currentError
+          ),
+        ]
+      )
+    },
+    [0, undefined] as [number, MultiError | undefined]
+  )
+}
+
+/**
+ * Reads the error codes from the content directory and syncs them with the
+ * database.
+ *
+ * Returns a result:
+ * - Ok(numberUpsertedRows): The number of error codes that were successfully upserted.
+ * - Err(error): An error that occurred during the upload process.
+ */
+async function uploadErrorCodes(
+  errorCodes: Array<ErrorCodeUploadParameters>
+): Promise<[number, MultiError<never> | undefined]> {
+  return Promise.all(
+    errorCodes.map(async (errorCode) => {
+      return new Result(await supabaseAdmin().schema('content').rpc('update_error_code', errorCode))
+    })
+  )
+    .then((data) => handleErrorCodeUploadErrors(data, errorCodes))
+    .catch((error) => [
+      0,
+      new MultiError(`Error uploading error codes: ${extractMessageFromAnyError(error)}`, [error]),
+    ])
+}
+
+/**
+ * Deletes error codes that are no longer used.
+ *
+ * @returns A promise that resolves to a result:
+ * - Ok(numberDeletedRows): The number of error codes that were successfully deleted.
+ * - Err(error): An error that occurred during the deletion process.
+ */
+async function deleteUnusedErrorCodes(
+  errorCodes: Array<ErrorCodeUploadParameters>
+): Promise<Result<number, Error>> {
+  const retainedErrorCodes = errorCodes.map((code) => ({
+    error_code: code.code,
+    service: code.service,
+  }))
+  return new Result(
+    await supabaseAdmin()
+      .schema('content')
+      .rpc('delete_error_codes_except', { skip_codes: retainedErrorCodes })
+  )
+    .map((data) => data)
+    .mapError(
+      (error) =>
+        new Error(util.format('Error deleting removed error codes: %s', error.message), {
+          cause: error,
+        })
+    )
+}
+
+/**
+ * Syncs error codes from the content files to the database.
+ */
+export async function syncErrorCodes(): Promise<Result<any, true>> {
+  const TAG = '[Sync error codes]'
+  const LOG_TAG = styleText('blue', TAG)
+  const ERROR_TAG = styleText('red', TAG)
+  function logWithTag(message: string) {
+    console.log(`${LOG_TAG} ${message}`)
+  }
+  function errorWithTag(message: string) {
+    console.error(`${ERROR_TAG} ${message}`)
+  }
+
+  logWithTag('Starting...')
+
+  logWithTag('Fetching error codes...')
+  return (await fetchErrorCodes())
+    .mapError((error) => {
+      errorWithTag(`Error syncing error codes: ${error.message}`)
+      return true as const
+    })
+    .flatMapAsync(async (errorCodes) => {
+      logWithTag(`Finished fetching ${errorCodes.length} error codes`)
+
+      logWithTag('Uploading error codes...')
+      const [numberUpserts, uploadError] = await uploadErrorCodes(errorCodes)
+      logWithTag(`Upserted data for ${numberUpserts} error code(s)`)
+      if (uploadError) {
+        errorWithTag(
+          `${uploadError.totalErrors} error(s) uploading error codes: ${uploadError.message}`
+        )
+      }
+
+      logWithTag('Deleting unused error codes...')
+      const deleteError = (await deleteUnusedErrorCodes(errorCodes)).match(
+        (numberDeleted) => logWithTag(`Deleted ${numberDeleted} unused error code(s)`),
+        (error) => {
+          errorWithTag(error.message)
+          return error
+        }
+      )
+
+      return uploadError || deleteError ? Result.error(true) : Result.ok(undefined)
+    })
+}

apps/docs/resources/error/errorTypes.ts

@@ -0,0 +1,4 @@
+export interface ErrorCodeDefinition {
+  description: string
+  references?: Array<{ href: string; description: string }>
+}

apps/docs/resources/rootSync.ts

@@ -0,0 +1,20 @@
+import '../scripts/utils/dotenv'
+
+import { fileURLToPath } from 'node:url'
+import { styleText } from 'node:util'
+import { syncErrorCodes } from './error/errorSync'
+
+async function sync(): Promise<void> {
+  console.log(styleText('magenta', 'Starting sync to database...'))
+  const allSyncResults = await Promise.all([syncErrorCodes()])
+  if (allSyncResults.some((result) => !result.isOk)) {
+    console.error(styleText('bold', styleText('red', 'Sync failed')))
+    process.exit(1)
+  } else {
+    console.log(styleText('bold', styleText('green', 'Sync successful')))
+  }
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  sync()
+}

packages/common/database-types.ts

@@ -10,6 +10,7 @@ export type Database = {
           deleted_at: string | null
           http_status_code: number | null
           message: string | null
+          metadata: Json | null
           service: string
           updated_at: string | null
         }
@@ -19,6 +20,7 @@ export type Database = {
           deleted_at?: string | null
           http_status_code?: number | null
           message?: string | null
+          metadata?: Json | null
           service: string
           updated_at?: string | null
         }
@@ -28,6 +30,7 @@ export type Database = {
           deleted_at?: string | null
           http_status_code?: number | null
           message?: string | null
+          metadata?: Json | null
           service?: string
           updated_at?: string | null
         }
@@ -74,7 +77,7 @@ export type Database = {
         Args: {
           skip_codes: Json
         }
-        Returns: undefined
+        Returns: number
       }
       update_error_code: {
         Args: {
@@ -82,6 +85,7 @@ export type Database = {
           service: string
           http_status_code?: number
           message?: string
+          metadata?: Json
         }
         Returns: boolean
       }

supabase/migrations/20250527221007_service_role_grants_error_tables.sql

@@ -0,0 +1,4 @@
+-- service_role needs access to content tables to run sync scripts
+grant usage on schema content to service_role;
+grant all on table content.service to service_role;
+grant all on table content.error to service_role;

supabase/migrations/20250529214621_store_error_metadata.sql

@@ -0,0 +1,89 @@
+alter table content.error
+add column metadata jsonb;
+
+alter table content.error
+add constraint constraint_content_error_metadata_schema check (
+    jsonb_matches_schema(
+        '{
+            "type": "object",
+            "properties": {
+                "references": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "href": {
+                                "type": "string"
+                            },
+                            "description": {
+                                "type": "string"
+                            }
+                        },
+                        "required": ["href", "description"]
+                    }
+                }
+            }
+        }',
+        metadata
+    )
+);
+
+drop function content.update_error_code;
+
+-- Recreate function also taking a metadata field
+-- See comments for what is new
+create function content.update_error_code(
+    code text,
+    service text,
+    http_status_code smallint default null,
+    message text default null,
+    -- Only new parameter
+    metadata jsonb default null
+)
+returns boolean
+set search_path = ''
+language plpgsql
+as $$
+#variable_conflict use_variable
+declare
+    service_id uuid;
+    result boolean;
+begin
+    insert into content.service (name)
+    values (service)
+    on conflict (name) do nothing;
+
+    select id into service_id
+    from content.service
+    where name = service;
+
+    insert into content.error (
+        service,
+        code,
+        http_status_code,
+        message,
+        -- Added metadata here
+        metadata
+    )
+    -- Added metadata here
+    values (service_id, code, http_status_code, message, metadata)
+    on conflict on constraint error_pkey do
+        update set
+            http_status_code = excluded.http_status_code,
+            message = excluded.message,
+            -- Added metadata here
+            metadata = excluded.metadata
+        where
+            error.service = service_id
+            and error.code = code
+            and (
+                error.http_status_code is distinct from excluded.http_status_code
+                or error.message is distinct from excluded.message
+                -- Added metadata here
+                or error.metadata is distinct from excluded.metadata
+            )
+        returning true into result;
+
+    return coalesce(result, false);
+end;
+$$;

supabase/migrations/20250529220759_error_code_delete_return_value.sql

@@ -0,0 +1,26 @@
+drop function content.delete_error_codes_except(jsonb);
+
+-- Recreating function to return the number of rows deleted
+create or replace function content.delete_error_codes_except(
+    skip_codes jsonb
+)
+returns integer
+set search_path = ''
+language sql
+as $$
+    with updated as (
+        update content.error
+        set deleted_at = now()
+        where
+            deleted_at is null
+            and not exists (
+                select 1
+                from jsonb_array_elements(skip_codes) skipped
+                join content.service on service.name = (skipped ->> 'service')
+                where service.id = error.service
+                and error.code = (skipped ->> 'error_code')
+            )
+        returning *
+    )
+    select count(*) from updated;
+$$;