From f89c362b2654e7831d49b627d2eebbcd6428a2bf Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Thu, 13 Aug 2026 18:26:15 +0200 Subject: [PATCH] fix(docs): accept a GitHub token for docs content reads (#48364) ## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. YES ## What kind of change does this PR introduce? Bug fix. Complete App configurations produce the same auth options as before. ## What is the current behavior? Without the docs GitHub App private key, two things fail for a contributor: - `pnpm run embeddings` aborts before doing any work. The lint warnings source throws, and every source shares one `Promise.all` in [`fetchAllSources()`](https://github.com/supabase/supabase/blob/master/apps/docs/scripts/search/sources/index.ts). - `pnpm --filter docs build` exits 1 in prebuild, so the `npm run build` pre-flight CONTRIBUTING.md asks for cannot run either: ``` Error: DOCS_GITHUB_APP_PRIVATE_KEY environment variable is required at octokit (apps/docs/lib/octokit.ts:21:13) at fetchAiSkills (apps/docs/scripts/federated-content/fetch-federated-content.ts:258:36) ``` Both read public content, so this is a rate-limit guard rather than access control: App auth landed in #43015 because unauthenticated calls (60 req/hr per IP) went flaky on shared runners. ## What is the new behavior? `apps/docs/lib/octokit.auth.ts` adds one rung below the App: a token from `GH_TOKEN`, then `GITHUB_TOKEN` (the precedence [`gh help environment`](https://cli.github.com/manual/gh_help_environment) documents), so `export GH_TOKEN=$(gh auth token)` is enough to build locally. Still authenticated, so #43015's fix holds, and still an authenticated Octokit client, so #44274 holds. A partially configured App is now an error naming the missing vars, rather than falling through to a token. Used by the lint warnings loader and `lib/octokit.ts`. The two token vars are declared in `apps/docs/turbo.jsonc` for `turbo/no-undeclared-env-vars`. ## Additional context With only `GH_TOKEN` set, `turbo run build --filter=docs --force` passes 4/4 and search-index source loading completes. `pnpm test` passes (20 files, 164 tests), and `tsc --noEmit` plus `pnpm run lint` match `origin/master`. For a complete App config the auth options are identical to before. Happy to post the fuller verification as a comment. ## Summary by CodeRabbit - **New Features** - Added flexible GitHub authentication for documentation services, supporting GitHub App credentials or personal access tokens. - GitHub App authentication is preferred when fully configured, with token-based fallback when unavailable. - Added support for both `GH_TOKEN` and `GITHUB_TOKEN`, with clear precedence rules. - **Bug Fixes** - Improved configuration validation with clear errors for missing or incomplete authentication settings. - Standardized authentication across GitHub content and lint-warning retrieval. --- apps/docs/lib/octokit.auth.test.ts | 88 +++++++++++++++++++ apps/docs/lib/octokit.auth.ts | 70 +++++++++++++++ apps/docs/lib/octokit.ts | 23 +---- .../search/sources/lint-warnings-guide.ts | 21 +---- apps/docs/turbo.jsonc | 2 + 5 files changed, 165 insertions(+), 39 deletions(-) create mode 100644 apps/docs/lib/octokit.auth.test.ts create mode 100644 apps/docs/lib/octokit.auth.ts diff --git a/apps/docs/lib/octokit.auth.test.ts b/apps/docs/lib/octokit.auth.test.ts new file mode 100644 index 00000000000..e34d0e43875 --- /dev/null +++ b/apps/docs/lib/octokit.auth.test.ts @@ -0,0 +1,88 @@ +import crypto from 'node:crypto' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { githubAuthOptions } from './octokit.auth.js' + +const APP_ID = '123456' +const INSTALLATION_ID = '7890' +// PKCS1 on purpose: the App rung has to convert it, and universal-github-app-jwt +// only accepts PKCS8. +const PKCS1_PRIVATE_KEY = crypto + .generateKeyPairSync('rsa', { modulusLength: 2048 }) + .privateKey.export({ type: 'pkcs1', format: 'pem' }) + .toString() + +function stubEnv(env: Record) { + // Empty string rather than `undefined`: some Vitest versions stringify the + // latter to 'undefined', which is truthy and would silently defeat these tests. + for (const name of [ + 'DOCS_GITHUB_APP_ID', + 'DOCS_GITHUB_APP_INSTALLATION_ID', + 'DOCS_GITHUB_APP_PRIVATE_KEY', + 'GH_TOKEN', + 'GITHUB_TOKEN', + ]) { + vi.stubEnv(name, env[name] ?? '') + } +} + +const APP_ENV = { + DOCS_GITHUB_APP_ID: APP_ID, + DOCS_GITHUB_APP_INSTALLATION_ID: INSTALLATION_ID, + DOCS_GITHUB_APP_PRIVATE_KEY: PKCS1_PRIVATE_KEY, +} + +describe('githubAuthOptions', () => { + afterEach(() => vi.unstubAllEnvs()) + + it('authenticates as the App and converts the key to PKCS8', () => { + stubEnv(APP_ENV) + const options = githubAuthOptions() + if (!('authStrategy' in options)) throw new Error('expected App auth') + expect(options.auth).toMatchObject({ appId: APP_ID, installationId: INSTALLATION_ID }) + expect(options.auth.privateKey).toMatch(/^-----BEGIN PRIVATE KEY-----/) + }) + + it('prefers the App when a token is also present', () => { + stubEnv({ ...APP_ENV, GITHUB_TOKEN: 'ghp_example' }) + expect(githubAuthOptions()).toHaveProperty('authStrategy') + }) + + it.each(['GH_TOKEN', 'GITHUB_TOKEN'])( + 'authenticates with a token from %s when the App is not configured', + (name) => { + stubEnv({ [name]: 'ghp_example' }) + expect(githubAuthOptions()).toEqual({ auth: 'ghp_example' }) + } + ) + + it('gives GH_TOKEN precedence over GITHUB_TOKEN, as the gh CLI documents', () => { + stubEnv({ GH_TOKEN: 'ghp_from_gh', GITHUB_TOKEN: 'ghp_from_actions' }) + expect(githubAuthOptions()).toEqual({ auth: 'ghp_from_gh' }) + }) + + it('refuses a partially configured App instead of masking it with a token', () => { + stubEnv({ DOCS_GITHUB_APP_ID: APP_ID, GITHUB_TOKEN: 'ghp_example' }) + expect(githubAuthOptions).toThrow(/Incomplete GitHub App configuration/) + // Names what is missing, and not what is already set. + expect(githubAuthOptions).toThrow(/DOCS_GITHUB_APP_INSTALLATION_ID/) + expect(githubAuthOptions).toThrow(/DOCS_GITHUB_APP_PRIVATE_KEY/) + expect(githubAuthOptions).not.toThrow(/DOCS_GITHUB_APP_ID\b/) + }) + + it('reports only the missing App var when one is absent', () => { + stubEnv({ + DOCS_GITHUB_APP_ID: APP_ID, + DOCS_GITHUB_APP_INSTALLATION_ID: INSTALLATION_ID, + GITHUB_TOKEN: 'ghp_example', + }) + expect(githubAuthOptions).toThrow(/DOCS_GITHUB_APP_PRIVATE_KEY not set\. Set all three/) + }) + + it('names every credential option when none is set', () => { + stubEnv({}) + expect(githubAuthOptions).toThrow(/DOCS_GITHUB_APP_ID/) + expect(githubAuthOptions).toThrow(/GH_TOKEN/) + expect(githubAuthOptions).toThrow(/GITHUB_TOKEN/) + }) +}) diff --git a/apps/docs/lib/octokit.auth.ts b/apps/docs/lib/octokit.auth.ts new file mode 100644 index 00000000000..0ce042ce336 --- /dev/null +++ b/apps/docs/lib/octokit.auth.ts @@ -0,0 +1,70 @@ +import { createAppAuth } from '@octokit/auth-app' +import crypto from 'node:crypto' + +type AppAuth = { appId: string; installationId: string; privateKey: string } + +/** + * Octokit auth options for reading public content from GitHub. + * + * Prefers the docs GitHub App (CI and production). Falls back to a personal + * access token so contributors can run the search index build locally without + * the App's private key: `GH_TOKEN` then `GITHUB_TOKEN`, matching the + * precedence the GitHub CLI documents (`gh help environment`), so an + * already-exported token just works. + * + * Both rungs authenticate on purpose: unauthenticated calls are limited to + * 60 req/hr per IP, which is what caused the flaky CI failures fixed in #43015, + * and callers here fetch one file per request. Env is read on each call rather + * than at module scope so the choice reflects the environment at call time. + * + * A partially configured App is an error rather than a token fall-back: a + * rotated-out or misnamed secret would otherwise be masked by whatever token + * happens to be in the environment, quietly reading as the wrong identity. + * + * Deliberately free of `server-only` imports: the search index scripts use this + * too, and they run outside Next. + */ +export function githubAuthOptions(): + | { authStrategy: typeof createAppAuth; auth: AppAuth } + | { auth: string } { + const appId = process.env.DOCS_GITHUB_APP_ID + const installationId = process.env.DOCS_GITHUB_APP_INSTALLATION_ID + const privateKey = process.env.DOCS_GITHUB_APP_PRIVATE_KEY + + if (appId && installationId && privateKey) { + return { + authStrategy: createAppAuth, + auth: { + appId, + installationId, + // https://github.com/gr2m/universal-github-app-jwt?tab=readme-ov-file#converting-pkcs1-to-pkcs8 + privateKey: crypto + .createPrivateKey(privateKey) + .export({ type: 'pkcs8', format: 'pem' }) + .toString(), + }, + } + } + + const appVars: Array<[string, string | undefined]> = [ + ['DOCS_GITHUB_APP_ID', appId], + ['DOCS_GITHUB_APP_INSTALLATION_ID', installationId], + ['DOCS_GITHUB_APP_PRIVATE_KEY', privateKey], + ] + const missing = appVars.filter(([, value]) => !value).map(([name]) => name) + const partiallyConfigured = missing.length < appVars.length + if (partiallyConfigured) { + throw new Error( + `Incomplete GitHub App configuration: ${missing.join(', ')} not set. Set all three, or unset the others to authenticate with GH_TOKEN / GITHUB_TOKEN instead.` + ) + } + + const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN + if (token) { + return { auth: token } + } + + throw new Error( + 'Missing GitHub credentials. Set DOCS_GITHUB_APP_ID, DOCS_GITHUB_APP_INSTALLATION_ID, and DOCS_GITHUB_APP_PRIVATE_KEY, or set GH_TOKEN / GITHUB_TOKEN for a local run (export GH_TOKEN=$(gh auth token)).' + ) +} diff --git a/apps/docs/lib/octokit.ts b/apps/docs/lib/octokit.ts index 0c488b0f1c6..5ddf874a379 100644 --- a/apps/docs/lib/octokit.ts +++ b/apps/docs/lib/octokit.ts @@ -1,11 +1,10 @@ import 'server-only' -import { createAppAuth } from '@octokit/auth-app' import { Octokit } from '@octokit/core' import { retry } from '@octokit/plugin-retry' -import crypto from 'node:crypto' import { fetchRevalidatePerDay } from '~/features/helpers.fetch' +import { githubAuthOptions } from './octokit.auth' import { OCTOKIT_RETRY_OPTIONS } from './octokit.constants' export { OCTOKIT_RETRY_OPTIONS } @@ -16,25 +15,7 @@ let octokitInstance: InstanceType export function octokit() { if (!octokitInstance) { - const privateKey = process.env.DOCS_GITHUB_APP_PRIVATE_KEY - if (!privateKey) { - throw new Error('DOCS_GITHUB_APP_PRIVATE_KEY environment variable is required') - } - - // https://github.com/gr2m/universal-github-app-jwt?tab=readme-ov-file#converting-pkcs1-to-pkcs8 - const privateKeyPkcs8 = crypto.createPrivateKey(privateKey).export({ - type: 'pkcs8', - format: 'pem', - }) - - octokitInstance = new RetryOctokit({ - authStrategy: createAppAuth, - auth: { - appId: process.env.DOCS_GITHUB_APP_ID, - installationId: process.env.DOCS_GITHUB_APP_INSTALLATION_ID, - privateKey: privateKeyPkcs8, - }, - }) + octokitInstance = new RetryOctokit(githubAuthOptions()) } return octokitInstance diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts index d28a57a27fd..43cee19ab7f 100644 --- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts +++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts @@ -1,16 +1,12 @@ -import { createAppAuth } from '@octokit/auth-app' import { Octokit } from '@octokit/core' import { retry } from '@octokit/plugin-retry' -import crypto, { createHash } from 'node:crypto' +import { createHash } from 'node:crypto' +import { githubAuthOptions } from '../../../lib/octokit.auth.js' import { OCTOKIT_RETRY_OPTIONS } from '../../../lib/octokit.constants.js' import { BaseLoader, BaseSource } from './base.js' const RetryOctokit = Octokit.plugin(retry) -const appId = process.env.DOCS_GITHUB_APP_ID -const installationId = process.env.DOCS_GITHUB_APP_INSTALLATION_ID -const privateKey = process.env.DOCS_GITHUB_APP_PRIVATE_KEY - const getBasename = (path: string) => path.split('/').at(-1)!.replace(/\.md$/, '') export class LintWarningsGuideLoader extends BaseLoader { @@ -28,18 +24,7 @@ export class LintWarningsGuideLoader extends BaseLoader { } async load() { - if (!appId || !installationId || !privateKey) { - throw new Error('Missing DOCS_GITHUB_APP_* environment variables') - } - - const octokit = new RetryOctokit({ - authStrategy: createAppAuth, - auth: { - appId, - installationId, - privateKey: crypto.createPrivateKey(privateKey).export({ type: 'pkcs8', format: 'pem' }), - }, - }) + const octokit = new RetryOctokit(githubAuthOptions()) const response = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', { owner: this.org, diff --git a/apps/docs/turbo.jsonc b/apps/docs/turbo.jsonc index b9400112caa..9d663a7ff71 100644 --- a/apps/docs/turbo.jsonc +++ b/apps/docs/turbo.jsonc @@ -64,7 +64,9 @@ "DOCS_REVALIDATION_KEYS", "DOCS_REVALIDATION_OVERRIDE_KEYS", "ENABLED_FEATURES_OVERRIDE_DISABLE_ALL", + "GH_TOKEN", "GITHUB_ACTIONS", + "GITHUB_TOKEN", "FORCE_ASSET_CDN", "LOGFLARE_INGESTION_API_KEY", "LOGFLARE_SOURCE_TOKEN",