From 6d3a4bcc48ce03914ce5993b5a4c46956a630c9e Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Tue, 11 Aug 2026 15:06:51 -0700 Subject: [PATCH] feat(www) Add scaffolding for WWW E2E tests and CI check (#48861) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes DOCS-1278 ## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. YES ## What kind of change does this PR introduce? Feature. Adds E2E test scaffolding and a CI check for the marketing site. ## What is the current behavior? Closes [FE-4047](https://linear.app/supabase/issue/FE-4047). The marketing site has no E2E coverage. Docs has a suite in `e2e/docs`, but its runner, git helpers and axe reporting are private to that package, so a second site cannot reuse them. ## What is the new behavior? * **A www suite scoped to changed content.** Changed `.mdx` files in `_blog`, `_events`, `_customers` and `_alternatives` map to the URLs they render. Pages with `disable_page_build: true` are skipped because they 404 by design. Capped at 20 pages. Enforces `heading-order` and `page-has-heading-one`, matching docs. * **`e2e/shared` The docs site is also static with similar needs. This folder shares the docs logic with www. * **A CI check that is safe to mark required.** Path scoping lives in a `Detect changed paths` step rather than a `paths:` trigger, so the check reports on every pull request instead of being skipped. `waitForVercelDocsPreview.js` becomes `waitForVercelPreview.js`, shared by both workflows. ## How the check behaves The job always reports a check run, so it is safe to mark required. Path scoping happens in a step rather than a `paths:` trigger, which would leave non-www pull requests waiting on a check that never reports. | Case | Behavior | | --- | --- | | Fork pull request adds new pages | Passes without testing. The Vercel wait is gated on `head.repo.full_name == github.repository`, so forks resolve no preview URL. The job emits a `::warning` and a job summary containing a ready-to-run `gh workflow run www-e2e.yml` command with the resolved page paths, so a maintainer can run it against the preview. | | Vercel preview times out or fails | Passes without testing. The wait step is `continue-on-error: true`, so a 900s timeout or a failed deployment leaves the URL unset and the suite skips. Vercel's own `Vercel – zone-www-dot-com` check already reports the failure. | | Draft pull request | Job does not run at all, gated at the job level on `pull_request.draft == false`. `ready_for_review` is in the trigger's `types`, so marking it ready runs the check. | | Another app changed, www untouched | Job runs and every step skips. The `www` filter matches only the four content directories, `e2e/www`, `e2e/shared`, the lockfile, and this workflow. | | Only the harness changed | Passes without testing. Scope resolves to zero pages, and the Vercel wait is additionally gated on `www_app`, so it does not wait for a preview Vercel skipped. | | No preview resolves, any reason | Skips rather than falling back to production. Production does not serve pages the pull request adds, so testing it would fail a valid change. | ### Not covered Changes to `apps/www` components and routes do not trigger this check — only the four content directories do. A follow-up can check global components such as the navigation and the footer. ## Manual testing 1. Start the site: `pnpm dev:www` 2. Run `pnpm e2e:www` with no www content changed. It should resolve zero pages and skip Playwright, not fail. 3. Touch a post, then run `pnpm e2e:www` again: `echo "" >> apps/www/_blog/2024-01-01-some-post.mdx`. The resolved `/blog/...` path should be listed before Playwright starts. 4. Run against production with no local server: `PLAYWRIGHT_BASE_URL=https://supabase.com WWW_E2E_PAGE_PATHS=/blog/postgres-language-server pnpm e2e:www` 5. Point step 4 at a page with a known heading problem. The failure should name the rule, the CSS selector and the markup. 6. Confirm docs still passes on the shared runner: `pnpm dev:docs`, then `pnpm e2e:docs` ## Summary by CodeRabbit * **New Features** * Added WWW end-to-end testing for affected content pages, including accessibility checks. * Added standard and full-site test commands, configurable preview testing, and failure reports. * Added shared utilities for page discovery, accessibility scanning, and test execution. * **Documentation** * Documented WWW test setup, coverage, debugging, CI behavior, and running checks against production or preview environments. * **Improvements** * Updated documentation test workflows to better identify affected changes and handle preview environments. --------- Co-authored-by: Claude Opus 5 --- .github/workflows/docs-e2e.yml | 14 +- .github/workflows/www-e2e.yml | 208 ++++++++++++++++++ apps/www/README.md | 20 ++ e2e/docs/features/docs-pages.spec.ts | 12 +- e2e/docs/playwright.local-smoke.config.ts | 6 +- e2e/docs/scripts/resolve-docs-scope.ts | 81 +------ e2e/docs/scripts/run-e2e-docs.ts | 181 +-------------- e2e/docs/utils/axe-helpers.ts | 57 +---- e2e/docs/utils/docs-links.ts | 30 +-- e2e/docs/utils/resolve-docs-scope.ts | 52 +---- e2e/shared/axe.ts | 85 +++++++ e2e/shared/git.ts | 38 ++++ e2e/shared/package.json | 14 ++ e2e/shared/paths.ts | 22 ++ e2e/shared/resolve-scope-cli.ts | 50 +++++ e2e/shared/run-suite.ts | 108 +++++++++ e2e/www/.gitignore | 5 + e2e/www/README.md | 169 ++++++++++++++ e2e/www/features/www-pages.spec.ts | 36 +++ e2e/www/package.json | 18 ++ e2e/www/playwright.config.ts | 31 +++ e2e/www/scripts/resolve-www-scope.ts | 15 ++ e2e/www/scripts/run-e2e-www.ts | 20 ++ e2e/www/tsconfig.json | 12 + e2e/www/utils/resolve-www-scope.ts | 91 ++++++++ package.json | 2 + pnpm-lock.yaml | 32 +++ ...DocsPreview.js => waitForVercelPreview.js} | 24 +- 28 files changed, 1031 insertions(+), 402 deletions(-) create mode 100644 .github/workflows/www-e2e.yml create mode 100644 e2e/shared/axe.ts create mode 100644 e2e/shared/git.ts create mode 100644 e2e/shared/package.json create mode 100644 e2e/shared/paths.ts create mode 100644 e2e/shared/resolve-scope-cli.ts create mode 100644 e2e/shared/run-suite.ts create mode 100644 e2e/www/.gitignore create mode 100644 e2e/www/README.md create mode 100644 e2e/www/features/www-pages.spec.ts create mode 100644 e2e/www/package.json create mode 100644 e2e/www/playwright.config.ts create mode 100644 e2e/www/scripts/resolve-www-scope.ts create mode 100644 e2e/www/scripts/run-e2e-www.ts create mode 100644 e2e/www/tsconfig.json create mode 100644 e2e/www/utils/resolve-www-scope.ts rename scripts/{waitForVercelDocsPreview.js => waitForVercelPreview.js} (77%) diff --git a/.github/workflows/docs-e2e.yml b/.github/workflows/docs-e2e.yml index 0cc745fad25..c608a8418af 100644 --- a/.github/workflows/docs-e2e.yml +++ b/.github/workflows/docs-e2e.yml @@ -55,12 +55,8 @@ jobs: - 'apps/docs/content/guides/**/*.mdx' - 'apps/docs/content/troubleshooting/**/*.mdx' - 'apps/docs/content/_partials/**' - - 'e2e/docs/features/**' - - 'e2e/docs/utils/**' - - 'e2e/docs/scripts/**' - - 'e2e/docs/playwright.config.ts' - - 'e2e/docs/package.json' - - 'e2e/docs/tsconfig.json' + - 'e2e/docs/**' + - 'e2e/shared/**' - 'pnpm-lock.yaml' - '.github/workflows/docs-e2e.yml' docs_app: @@ -75,6 +71,7 @@ jobs: fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }} sparse-checkout: | e2e/docs + e2e/shared scripts patches apps/docs/content/guides @@ -138,17 +135,18 @@ jobs: # times out polling that API even though the preview builds fine. # Poll the "Vercel – docs" commit status instead — Vercel keeps posting # those — then resolve the deployment it points to via Vercel's own API - # to get the actual preview URL. See scripts/waitForVercelDocsPreview.js. + # to get the actual preview URL. See scripts/waitForVercelPreview.js. # A Vercel failure or timeout is not the author's problem, and the required # "Vercel – docs" check already reports it. Resolve no URL and skip below. - name: Wait for Vercel docs preview if: steps.scope.outputs.skip == 'false' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && steps.changes.outputs.docs_app == 'true' id: deployment continue-on-error: true - run: node scripts/waitForVercelDocsPreview.js + run: node scripts/waitForVercelPreview.js env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} + VERCEL_STATUS_CONTEXT: 'Vercel – docs' VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} diff --git a/.github/workflows/www-e2e.yml b/.github/workflows/www-e2e.yml new file mode 100644 index 00000000000..7859ca0033c --- /dev/null +++ b/.github/workflows/www-e2e.yml @@ -0,0 +1,208 @@ +name: WWW E2E Tests + +# Path scoping lives in "Detect changed paths" rather than a `paths` trigger, so +# this reports a check run on every pull request and stays safe to mark required. +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] + branches: ['master'] + workflow_dispatch: + inputs: + base_url: + description: 'Base URL to test against' + required: false + default: 'https://supabase.com' + type: string + page_paths: + description: 'Comma-separated site-relative paths to test (required for manual runs)' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + statuses: read + pull-requests: read + +env: + CI: true + +jobs: + e2e: + name: WWW E2E + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false + timeout-minutes: 30 + runs-on: blacksmith-4vcpu-ubuntu-2404 + + steps: + - name: Detect changed paths + id: changes + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + with: + filters: | + www: + - 'apps/www/_blog/**/*.mdx' + - 'apps/www/_events/**/*.mdx' + - 'apps/www/_customers/**/*.mdx' + - 'apps/www/_alternatives/**/*.mdx' + - 'e2e/www/**' + - 'e2e/shared/**' + - 'pnpm-lock.yaml' + - '.github/workflows/www-e2e.yml' + www_app: + - 'apps/www/**' + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.www == 'true' + with: + persist-credentials: false + # String '0' — numeric 0 is falsy in Actions expressions. + fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }} + sparse-checkout: | + e2e/www + e2e/shared + scripts + patches + apps/www/_blog + apps/www/_events + apps/www/_customers + apps/www/_alternatives + + - name: Use Node.js + if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.www == 'true' + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: '.nvmrc' + + - name: Resolve www E2E scope + id: scope + if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.www == 'true' + env: + EVENT_NAME: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + PAGE_PATHS_INPUT: ${{ inputs.page_paths }} + run: | + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + if [ -z "$PAGE_PATHS_INPUT" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "paths=" >> "$GITHUB_OUTPUT" + echo "Manual run requires the page_paths input." + exit 0 + fi + echo "skip=false" >> "$GITHUB_OUTPUT" + printf 'paths=%s\n' "$PAGE_PATHS_INPUT" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git diff --name-only --diff-filter=ACMR "origin/$BASE_REF"...HEAD \ + | node --experimental-strip-types e2e/www/scripts/resolve-www-scope.ts + + - name: Skip Playwright (no in-scope pages) + if: steps.scope.outputs.skip == 'true' + run: echo "No in-scope www pages changed; skipping Playwright suite." + + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + if: steps.scope.outputs.skip == 'false' + name: Install pnpm + with: + run_install: false + + - name: Enable pnpm store cache + if: steps.scope.outputs.skip == 'false' + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + + # Vercel skips the preview when only the harness changed, so wait for one + # only when apps/www changed. See scripts/waitForVercelPreview.js. + - name: Wait for Vercel www preview + if: steps.scope.outputs.skip == 'false' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && steps.changes.outputs.www_app == 'true' + id: deployment + continue-on-error: true + run: node scripts/waitForVercelPreview.js + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + VERCEL_STATUS_CONTEXT: 'Vercel – zone-www-dot-com' + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + + - name: Resolve base URL + if: steps.scope.outputs.skip == 'false' + id: base-url + env: + EVENT_NAME: ${{ github.event_name }} + BASE_URL_INPUT: ${{ inputs.base_url }} + DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} + PAGE_PATHS: ${{ steps.scope.outputs.paths }} + run: | + set -euo pipefail + + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + printf 'url=%s\n' "$BASE_URL_INPUT" >> "$GITHUB_OUTPUT" + if [ "$BASE_URL_INPUT" = "https://supabase.com" ]; then + echo "use_bypass=false" >> "$GITHUB_OUTPUT" + else + echo "use_bypass=true" >> "$GITHUB_OUTPUT" + fi + echo "should_test=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ -n "$DEPLOYMENT_URL" ]; then + printf 'url=%s\n' "$DEPLOYMENT_URL" >> "$GITHUB_OUTPUT" + echo "use_bypass=true" >> "$GITHUB_OUTPUT" + echo "should_test=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Production lacks pages this pull request adds, so it is not a fallback. + echo "url=" >> "$GITHUB_OUTPUT" + echo "use_bypass=false" >> "$GITHUB_OUTPUT" + echo "should_test=false" >> "$GITHUB_OUTPUT" + echo "::warning::No Vercel www preview URL for this pull request. Skipping Playwright rather than testing production, which does not have pages this pull request adds." + { + echo "### WWW E2E skipped: no preview to test against" + echo + echo "Fork pull requests reach this path because they run without repository" + echo "secrets. A maintainer can run the suite against the preview manually:" + echo + echo '```' + echo "gh workflow run www-e2e.yml \\" + echo " -f base_url= \\" + echo " -f page_paths=$PAGE_PATHS" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Install dependencies + if: steps.base-url.outputs.should_test == 'true' + run: pnpm install --frozen-lockfile --filter=e2e-www... + + - name: Install Playwright Chromium + if: steps.base-url.outputs.should_test == 'true' + run: pnpm -C e2e/www exec playwright install chromium --with-deps --only-shell + + - name: Run www E2E + if: steps.base-url.outputs.should_test == 'true' + working-directory: e2e/www + run: pnpm run e2e:www + env: + PLAYWRIGHT_BASE_URL: ${{ steps.base-url.outputs.url }} + WWW_E2E_PAGE_PATHS: ${{ steps.scope.outputs.paths }} + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ steps.base-url.outputs.use_bypass == 'true' && secrets.VERCEL_AUTOMATION_BYPASS_WWW || '' }} + + - name: Upload Playwright report + if: failure() && steps.scope.outputs.skip == 'false' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: www-playwright-report + path: | + e2e/www/playwright-report/ + e2e/www/test-results/ + retention-days: 7 diff --git a/apps/www/README.md b/apps/www/README.md index f2f7df15d90..100fe836789 100644 --- a/apps/www/README.md +++ b/apps/www/README.md @@ -130,6 +130,26 @@ og_image: /images/events/2025-01-meetup/custom-og.png # Optional override **Note**: The `og_image` field is optional. If not provided, OG images are generated automatically via the Edge Function. +## End-to-end checks + +Content pages are loaded and scanned with axe-core by the Playwright suite in +`e2e/www`. Pull requests test the pages your change affects. Today the suite +enforces one accessibility rule, `page-has-heading-one`. + +To test the pages your current branch changes: + +```bash +PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:www +``` + +That resolves which pages to test from your branch, but reads them from +production, so it won't see your edits and will 404 on a page you just added. +Point `PLAYWRIGHT_BASE_URL` at your pull request's preview to test your own +content. + +See [`e2e/www/README.md`](https://github.com/supabase/supabase/blob/master/e2e/www/README.md) +for coverage and limits. + ## Go pages (`/go/*`) `/go/` is a system for building standalone campaign landing pages (lead generation, legal, thank-you flows). The name is intentionally generic — these pages are typically linked from ads, emails, or partner campaigns and are not part of the main site navigation. diff --git a/e2e/docs/features/docs-pages.spec.ts b/e2e/docs/features/docs-pages.spec.ts index 3a58f155021..a8a9890d951 100644 --- a/e2e/docs/features/docs-pages.spec.ts +++ b/e2e/docs/features/docs-pages.spec.ts @@ -1,6 +1,7 @@ import { expect, test } from '@playwright/test' import type { TestInfo } from '@playwright/test' +import { parsePagePaths } from '../../shared/paths.ts' import { attachScanReport, blockingViolations, @@ -11,15 +12,15 @@ import { settleForAxe, shouldEnforceAll, unloadedResult, + violationIds, } from '../utils/axe-helpers.js' import { articleSelectorForPagePath, browserLikeUserAgent, collectDocsOwnedLinks, - parseDocsE2EPagePaths, } from '../utils/docs-links.js' -const pagePaths = parseDocsE2EPagePaths(process.env.DOCS_E2E_PAGE_PATHS) +const pagePaths = parsePagePaths(process.env.DOCS_E2E_PAGE_PATHS) function annotate(testInfo: TestInfo, description: string) { testInfo.annotations.push({ type: 'warning', description }) @@ -27,10 +28,7 @@ function annotate(testInfo: TestInfo, description: string) { } test.describe('Docs owned pages', () => { - // playwright.config.ts sets fullyParallel: false, and Playwright shards - // work by file rather than by test in that mode — without this, every test - // in this single spec file runs on one worker no matter what --workers is - // passed. Opt this describe block into parallel scheduling explicitly. + // Without this, every test in this file runs on one worker. test.describe.configure({ mode: 'parallel' }) test('resolved page list must not be empty', () => { @@ -140,7 +138,7 @@ test.describe('Docs owned pages', () => { const enforced = shouldEnforceAll() ? 'all WCAG 2.1 A/AA rules' : ENFORCED_RULES.join(', ') expect( - blocking, + violationIds(blocking), `${pagePath} has blocking a11y violations (${enforced}):\n${formatViolations(blocking)}` ).toEqual([]) }) diff --git a/e2e/docs/playwright.local-smoke.config.ts b/e2e/docs/playwright.local-smoke.config.ts index 9d28020de51..7a97bea6beb 100644 --- a/e2e/docs/playwright.local-smoke.config.ts +++ b/e2e/docs/playwright.local-smoke.config.ts @@ -20,8 +20,6 @@ export default defineConfig({ baseURL: `http://localhost:${WEB_SERVER_PORT}`, browserName: 'chromium', headless: true, - // Higher than usual: each run boots a fresh dev server, so the first - // request to a route pays Next.js's on-demand compile cost. navigationTimeout: 60_000, screenshot: 'only-on-failure', trace: 'retain-on-failure', @@ -35,13 +33,11 @@ export default defineConfig({ ['json', { outputFile: './test-results/local-smoke-results.json' }], ], outputDir: './test-results', - // Explicitly cleared so this suite is credential-free even if the local - // .env.local has real secrets, and never reused so a server already - // running on this port (possibly with real credentials) can't be reused. webServer: { command: 'pnpm --workspace-root run dev:docs', port: WEB_SERVER_PORT, timeout: 4 * 60 * 1000, + // Never reuse a running server: it may hold real credentials from .env.local. reuseExistingServer: false, env: { DOCS_GITHUB_APP_PRIVATE_KEY: '', diff --git a/e2e/docs/scripts/resolve-docs-scope.ts b/e2e/docs/scripts/resolve-docs-scope.ts index c026527ce38..b0653d76ffb 100644 --- a/e2e/docs/scripts/resolve-docs-scope.ts +++ b/e2e/docs/scripts/resolve-docs-scope.ts @@ -1,82 +1,15 @@ #!/usr/bin/env node -/** - * Resolve docs E2E page scope from changed files. - * - * Usage: - * git diff --name-only origin/master...HEAD | node --experimental-strip-types scripts/resolve-docs-scope.ts - * node --experimental-strip-types scripts/resolve-docs-scope.ts --files a.mdx,b.mdx - * - * Outputs (GitHub Actions friendly): - * skip=true|false - * paths= - * Also prints each path on its own line to stderr for debugging. - */ import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' -import { parseChangedFilesList, resolveDocsScope } from '../utils/resolve-docs-scope.ts' +import { runResolveScopeCli } from '../../shared/resolve-scope-cli.ts' +import { resolveDocsScope } from '../utils/resolve-docs-scope.ts' -const __dirname = dirname(fileURLToPath(import.meta.url)) -const REPO_ROOT = join(__dirname, '../../..') - -function readChangedFilesFromArgv(argv: string[]): string[] | null { - const filesIdx = argv.indexOf('--files') - if (filesIdx !== -1 && argv[filesIdx + 1]) { - return parseChangedFilesList(argv[filesIdx + 1]) - } - return null -} - -async function readStdin(): Promise { - const chunks: Buffer[] = [] - for await (const chunk of process.stdin) { - chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk) - } - return Buffer.concat(chunks).toString('utf8') -} - -async function main() { - const argv = process.argv.slice(2) - let changedFiles = readChangedFilesFromArgv(argv) - - if (!changedFiles) { - if (process.stdin.isTTY) { - console.error('Pass changed files via stdin or --files path1,path2') - process.exit(2) - } - changedFiles = parseChangedFilesList(await readStdin()) - } - - const result = await resolveDocsScope({ - changedFiles, - repoRoot: REPO_ROOT, - }) - - // GitHub Actions step outputs - const githubOutput = process.env.GITHUB_OUTPUT - const skipLine = `skip=${result.skip}` - const pathsLine = `paths=${result.pages.join(',')}` - - if (githubOutput) { - // appendFile via sync to keep the CLI dependency-free - const { appendFileSync } = await import('node:fs') - appendFileSync(githubOutput, `${skipLine}\n${pathsLine}\n`) - } else { - console.log(skipLine) - console.log(pathsLine) - } - - if (result.pages.length > 0) { - console.error(`Resolved ${result.pages.length} docs page(s):`) - for (const page of result.pages) { - console.error(` ${page}`) - } - } else { - console.error('No in-scope docs pages — skipping Playwright suite.') - } -} - -main().catch((error) => { +runResolveScopeCli({ + label: 'docs', + repoRoot: join(dirname(fileURLToPath(import.meta.url)), '../../..'), + resolveScope: resolveDocsScope, +}).catch((error) => { console.error(error instanceof Error ? error.message : error) process.exit(1) }) diff --git a/e2e/docs/scripts/run-e2e-docs.ts b/e2e/docs/scripts/run-e2e-docs.ts index c8883aa90bb..b256d1fe13d 100644 --- a/e2e/docs/scripts/run-e2e-docs.ts +++ b/e2e/docs/scripts/run-e2e-docs.ts @@ -1,177 +1,20 @@ #!/usr/bin/env node -/** - * Default entry for `pnpm e2e:docs`. - * - * If DOCS_E2E_PAGE_PATHS is already set (CI, or an explicit local override), - * runs Playwright with that list. Otherwise resolves pages from files changed - * vs DOCS_E2E_BASE_REF (default origin/master), including the working tree. - * - * Pass `--all` to test every in-scope guide and troubleshooting page instead - * (hundreds of pages — expect a long run against a deployed site). - * - * Extra CLI args are forwarded to Playwright (e.g. --ui, a spec file path). - */ -import { spawn, spawnSync } from 'node:child_process' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' -import { - parseChangedFilesList, - resolveAllDocsPages, - resolveDocsScope, -} from '../utils/resolve-docs-scope.ts' +import { runSuite } from '../../shared/run-suite.ts' +import { resolveAllDocsPages, resolveDocsScope } from '../utils/resolve-docs-scope.ts' -const __dirname = dirname(fileURLToPath(import.meta.url)) -const E2E_DOCS_ROOT = join(__dirname, '..') - -// Mirrors the default in playwright.config.ts. -const DEFAULT_BASE_URL = 'http://localhost:3001' -const PREFLIGHT_TIMEOUT_MS = 3_000 - -async function isBaseUrlReachable(baseUrl: string): Promise { - const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), PREFLIGHT_TIMEOUT_MS) - try { - await fetch(baseUrl, { signal: controller.signal }) - return true - } catch { - return false - } finally { - clearTimeout(timeout) - } -} - -function git(args: string[], cwd: string): string { - const result = spawnSync('git', args, { - cwd, - encoding: 'utf8', - env: process.env, - }) - if (result.status !== 0) { - const detail = (result.stderr || result.stdout || '').trim() - throw new Error(`git ${args.join(' ')} failed${detail ? `: ${detail}` : ''}`) - } - return result.stdout -} - -function repoRootFromCwd(): string { - return git(['rev-parse', '--show-toplevel'], E2E_DOCS_ROOT).trim() -} - -function collectChangedFiles(repoRoot: string, baseRef: string): string[] { - const ranges: string[][] = [ - // Commits on this branch since diverging from the base - ['diff', '--name-only', '--diff-filter=ACMR', `${baseRef}...HEAD`], - // Unstaged working tree - ['diff', '--name-only', '--diff-filter=ACMR'], - // Staged working tree - ['diff', '--name-only', '--diff-filter=ACMR', '--cached'], - ] - - const files = new Set() - for (const args of ranges) { - try { - for (const file of parseChangedFilesList(git([...args], repoRoot))) { - files.add(file) - } - } catch (error) { - if (args.includes(`${baseRef}...HEAD`)) { - throw error - } - // Working-tree diffs can be empty / fail in odd git states; ignore those. - } - } - return [...files].sort() -} - -async function resolveAllPagePaths(): Promise { - const repoRoot = repoRootFromCwd() - const pages = await resolveAllDocsPages(repoRoot) - console.error(`Resolved all ${pages.length} in-scope docs page(s) (guides + troubleshooting).`) - return pages -} - -async function resolvePagePaths(): Promise { - const existing = process.env.DOCS_E2E_PAGE_PATHS?.trim() - if (existing) { - return existing - .split(/[\n,]/) - .map((p) => p.trim()) - .filter(Boolean) - } - - const baseRef = process.env.DOCS_E2E_BASE_REF?.trim() || 'origin/master' - const repoRoot = repoRootFromCwd() - const changedFiles = collectChangedFiles(repoRoot, baseRef) - const result = await resolveDocsScope({ changedFiles, repoRoot }) - - if (result.skip) { - console.error( - `No in-scope docs pages changed vs ${baseRef} (including working tree). Skipping Playwright.` - ) - return null - } - - console.error(`Resolved ${result.pages.length} docs page(s) from changes vs ${baseRef}:`) - for (const page of result.pages) { - console.error(` ${page}`) - } - return result.pages -} - -async function main() { - const rawArgs = process.argv.slice(2) - const runAll = rawArgs.includes('--all') - const playwrightArgs = rawArgs.filter((arg) => arg !== '--all' && arg !== '--') - - const pages = runAll ? await resolveAllPagePaths() : await resolvePagePaths() - if (pages === null) { - process.exit(0) - } - - // playwright.config.ts sets a global maxFailures: 3, which would otherwise - // abort an exhaustive --all run after just 3 failing pages out of hundreds. - // -x is Playwright's shorthand for --max-failures=1. - const hasMaxFailuresArg = playwrightArgs.some( - (arg) => arg === '-x' || arg.startsWith('--max-failures') - ) - const finalPlaywrightArgs = - runAll && !hasMaxFailuresArg ? [...playwrightArgs, '--max-failures=0'] : playwrightArgs - - const baseUrl = process.env.PLAYWRIGHT_BASE_URL?.trim() || DEFAULT_BASE_URL - if (!(await isBaseUrlReachable(baseUrl))) { - console.error(`No docs server responding at ${baseUrl}.`) - if (!process.env.PLAYWRIGHT_BASE_URL) { - console.error('Start it with `pnpm dev:docs`, or point at a deployed site:') - console.error(' PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:docs') - } else { - console.error('Check that the URL is correct and reachable.') - } - process.exit(1) - } - - const env = { - ...process.env, - DOCS_E2E_PAGE_PATHS: pages.join(','), - } - - const child = spawn('pnpm', ['exec', 'playwright', 'test', ...finalPlaywrightArgs], { - cwd: E2E_DOCS_ROOT, - env, - stdio: 'inherit', - shell: process.platform === 'win32', - }) - - child.on('exit', (code, signal) => { - if (signal) { - process.kill(process.pid, signal) - return - } - process.exit(code ?? 1) - }) -} - -main().catch((error) => { +runSuite({ + root: join(dirname(fileURLToPath(import.meta.url)), '..'), + label: 'docs', + devCommand: 'pnpm dev:docs', + pagePathsEnv: 'DOCS_E2E_PAGE_PATHS', + baseRefEnv: 'DOCS_E2E_BASE_REF', + defaultBaseUrl: 'http://localhost:3001', + resolveScope: resolveDocsScope, + resolveAllPages: resolveAllDocsPages, +}).catch((error) => { console.error(error instanceof Error ? error.message : error) process.exit(1) }) diff --git a/e2e/docs/utils/axe-helpers.ts b/e2e/docs/utils/axe-helpers.ts index ba82e3d0a96..afc86ac8c1e 100644 --- a/e2e/docs/utils/axe-helpers.ts +++ b/e2e/docs/utils/axe-helpers.ts @@ -1,7 +1,8 @@ -import { AxeBuilder } from '@axe-core/playwright' import type { Page, TestInfo } from '@playwright/test' import type { Result } from 'axe-core' +import { scan } from '../../shared/axe.ts' + export const WCAG_TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'] export const ENFORCED_RULES = ['heading-order', 'page-has-heading-one'] @@ -33,50 +34,15 @@ export function shouldEnforceAll(): boolean { return !!process.env.A11Y_ENFORCE_ALL } -export async function settleForAxe(page: Page): Promise { - await page.waitForLoadState('domcontentloaded') - await page.waitForLoadState('networkidle', { timeout: 15_000 }).catch(() => {}) - - await page - .evaluate( - ({ quietMs, capMs }) => - new Promise((resolve) => { - let timer: ReturnType - const observer = new MutationObserver(() => { - clearTimeout(timer) - timer = setTimeout(finish, quietMs) - }) - - function finish() { - clearTimeout(cap) - clearTimeout(timer) - observer.disconnect() - resolve() - } - - const cap = setTimeout(finish, capMs) - timer = setTimeout(finish, quietMs) - observer.observe(document.body, { subtree: true, childList: true, attributes: true }) - }), - { quietMs: 500, capMs: 5_000 } - ) - .catch(() => {}) -} - export async function scanArticle( page: Page, surface: string, include: string ): Promise { - const scan = () => new AxeBuilder({ page }).setLegacyMode(true).include(include) + const reported = await scan(page, { tags: WCAG_TAGS, excludeRules: EXCLUDED_RULES, include }) + const enforced = await scan(page, { rules: ENFORCED_RULES, include }) - const reported = await scan().withTags(WCAG_TAGS).disableRules(EXCLUDED_RULES).analyze() - - const enforced = await scan().withRules(ENFORCED_RULES).analyze() - - const byRule = new Map( - [...reported.violations, ...enforced.violations].map((violation) => [violation.id, violation]) - ) + const byRule = new Map([...reported, ...enforced].map((violation) => [violation.id, violation])) const elementCount = await page.evaluate( (selector) => document.querySelector(selector)?.querySelectorAll('*').length ?? 0, @@ -134,15 +100,4 @@ export function blockingViolations(result: A11yScanResult): Result[] { return result.violations.filter((violation) => ENFORCED_RULES.includes(violation.id)) } -export function formatViolations(violations: Result[]): string { - return violations - .map( - (violation) => - `${violation.id} (${violation.impact}, ${violation.nodes.length} node(s)): ${violation.help}\n` + - violation.nodes - .slice(0, 5) - .map((node) => ` ${node.target.join(' ')}\n ${node.html.slice(0, 200)}`) - .join('\n') - ) - .join('\n') -} +export { formatViolations, settleForAxe, violationIds } from '../../shared/axe.ts' diff --git a/e2e/docs/utils/docs-links.ts b/e2e/docs/utils/docs-links.ts index 7e483ba5a87..58942279c39 100644 --- a/e2e/docs/utils/docs-links.ts +++ b/e2e/docs/utils/docs-links.ts @@ -6,10 +6,6 @@ export const TROUBLESHOOTING_ARTICLE_SELECTOR = const DOCS_PATH_PREFIX = '/docs' const TROUBLESHOOTING_PATH_PREFIX = '/docs/guides/troubleshooting/' -/** - * Pick the main article selector for a docs page path. - * Both guides and troubleshooting entries expose a stable data-testid. - */ export function articleSelectorForPagePath(pagePath: string): string { const pathname = pagePath.startsWith('http') ? new URL(pagePath).pathname : pagePath @@ -23,12 +19,6 @@ export function articleSelectorForPagePath(pagePath: string): string { return GUIDE_ARTICLE_SELECTOR } -/** - * Collect unique docs-owned links from the main article. - * - * Cross-app paths such as `/ui` and `/dashboard` are excluded because the - * docs preview does not own those routes. - */ export async function collectDocsOwnedLinks( page: Page, baseURL: string, @@ -65,26 +55,8 @@ export async function collectDocsOwnedLinks( return [...links].sort() } -/** - * Playwright's headless Chromium reports a `HeadlessChrome` UA string, which - * Vercel's bot protection blocks on some routes (notably /docs/reference/*) - * even though the same page loads fine for a real browser. Stripping - * `Headless` avoids that false positive when checking links out-of-band via - * page.request rather than an actual navigation. - */ +// Vercel bot protection blocks the HeadlessChrome UA on some routes; strip it. export async function browserLikeUserAgent(page: Page): Promise { const userAgent = await page.evaluate(() => navigator.userAgent) return userAgent.replace('HeadlessChrome', 'Chrome') } - -/** - * Parse DOCS_E2E_PAGE_PATHS (comma- or newline-separated /docs/... paths). - */ -export function parseDocsE2EPagePaths(raw: string | undefined): string[] { - if (!raw?.trim()) return [] - return raw - .split(/[\n,]/) - .map((path) => path.trim()) - .filter(Boolean) - .map((path) => (path.startsWith('/') ? path : `/${path}`)) -} diff --git a/e2e/docs/utils/resolve-docs-scope.ts b/e2e/docs/utils/resolve-docs-scope.ts index acaa2db73c2..2c5325cbac9 100644 --- a/e2e/docs/utils/resolve-docs-scope.ts +++ b/e2e/docs/utils/resolve-docs-scope.ts @@ -1,11 +1,8 @@ import { readdir, readFile } from 'node:fs/promises' -import { basename, join, relative, sep } from 'node:path' +import { basename, join, relative } from 'node:path' + +import { normalizeRepoPath } from '../../shared/paths.ts' -/** - * Federated guide section prefixes — mirrors - * apps/docs/scripts/federated-content/sources/*.ts. Checked automatically - * against those files by assertFederatedSectionsInSync below. - */ export const FEDERATED_SECTIONS = [ 'graphql', 'database/extensions/wrappers', @@ -26,12 +23,6 @@ const FEDERATED_CONTENT_SOURCES_DIR = 'apps/docs/scripts/federated-content/sourc const PARTIAL_PATH_RE = /<\$Partial\b[\s\S]*?\bpath\s*=\s*"([^"]+)"[\s\S]*?\/?>/g const SOURCE_SECTION_RE = /\bsection:\s*'([^']+)'/g -/** - * Compares FEDERATED_SECTIONS against the `section:` field declared in each - * apps/docs/scripts/federated-content/sources/*.ts file, so a section added - * or removed there can't silently drift from what this suite treats as - * out-of-scope. - */ async function assertFederatedSectionsInSync(repoRoot: string): Promise { const sourcesDir = join(repoRoot, FEDERATED_CONTENT_SOURCES_DIR) const sourceFiles = (await readdir(sourcesDir)).filter((file) => file.endsWith('.ts')) @@ -70,11 +61,8 @@ async function assertFederatedSectionsInSync(repoRoot: string): Promise { } export type ResolveDocsScopeOptions = { - /** Repo-root-relative changed file paths */ changedFiles: string[] - /** Absolute path to the monorepo root */ repoRoot: string - /** Max pages to test before the rest are truncated (default MAX_SCOPED_PAGES) */ maxPages?: number } @@ -83,10 +71,6 @@ export type ResolveDocsScopeResult = { skip: boolean } -function normalizeRepoPath(filePath: string): string { - return filePath.replaceAll('\\', '/') -} - function isFederatedGuideSlug(slug: string): boolean { return FEDERATED_SECTIONS.some((section) => slug === section || slug.startsWith(`${section}/`)) } @@ -95,9 +79,6 @@ function isHiddenMdx(filePath: string): boolean { return basename(filePath).startsWith('_') } -/** - * Map a changed content file to a docs URL, or null if not a testable page. - */ export function changedFileToPagePath(filePath: string): string | null { const normalized = normalizeRepoPath(filePath) @@ -129,8 +110,6 @@ function partialRelPathFromChangedFile(filePath: string): string | null { } function normalizePartialRef(pathAttr: string): string | null { - // $Partial paths are relative to content/_partials (see Partial.ts). - // Leading-slash example-code paths are not real partials. if (!pathAttr || pathAttr.startsWith('/') || pathAttr.startsWith('http')) { return null } @@ -172,9 +151,7 @@ async function walkMdxFiles(dir: string): Promise { } type PartialIndex = { - /** partial rel path → set of partial rel paths that include it */ includedByPartials: Map> - /** page URL → set of direct partial rel paths */ pagePartials: Map> } @@ -225,10 +202,6 @@ async function buildPartialIndex(repoRoot: string): Promise { return { includedByPartials, pagePartials } } -/** - * Expand a changed partial to all partials that transitively include it - * (including itself). - */ function expandPartialClosure( seed: string, includedByPartials: Map> @@ -266,11 +239,6 @@ function pagesUsingPartials( return pages } -/** - * Resolve which docs pages to E2E-test from a list of changed repo files. - * Truncates to maxPages (sorted) so a widely shared partial can't blow up - * runtime; use resolveAllDocsPages / `pnpm e2e:docs:all` to cover everything. - */ export async function resolveDocsScope( options: ResolveDocsScopeOptions ): Promise { @@ -311,11 +279,6 @@ export async function resolveDocsScope( } } -/** - * List every in-scope docs page — all guides (excluding federated sections) - * and all troubleshooting entries — regardless of what changed. Used for - * full-suite runs rather than the default changed-files scope. - */ export async function resolveAllDocsPages(repoRoot: string): Promise { await assertFederatedSectionsInSync(repoRoot) @@ -342,12 +305,3 @@ export async function resolveAllDocsPages(repoRoot: string): Promise { return [...pages].sort() } - -/** Parse changed-file list from stdin or newline/comma-separated string. */ -export function parseChangedFilesList(input: string): string[] { - return input - .split(/[\n,]/) - .map((line) => line.trim()) - .filter(Boolean) - .map((line) => normalizeRepoPath(line.split(sep).join('/'))) -} diff --git a/e2e/shared/axe.ts b/e2e/shared/axe.ts new file mode 100644 index 00000000000..667782dcd07 --- /dev/null +++ b/e2e/shared/axe.ts @@ -0,0 +1,85 @@ +import { AxeBuilder } from '@axe-core/playwright' +import type { Page } from '@playwright/test' +import type { Result } from 'axe-core' + +export const MAX_REPORTED_NODES = 5 + +export const MAX_REPORTED_HTML = 200 + +export type ScanOptions = { + rules?: string[] + tags?: string[] + excludeRules?: string[] + include?: string +} + +export async function settleForAxe(page: Page): Promise { + await page.waitForLoadState('domcontentloaded') + await page.waitForLoadState('networkidle', { timeout: 15_000 }).catch(() => {}) + + await page + .evaluate( + ({ quietMs, capMs }) => + new Promise((resolve) => { + let timer: ReturnType + const observer = new MutationObserver(() => { + clearTimeout(timer) + timer = setTimeout(finish, quietMs) + }) + + function finish() { + clearTimeout(cap) + clearTimeout(timer) + observer.disconnect() + resolve() + } + + const cap = setTimeout(finish, capMs) + timer = setTimeout(finish, quietMs) + observer.observe(document.body, { subtree: true, childList: true, attributes: true }) + }), + { quietMs: 500, capMs: 5_000 } + ) + .catch(() => {}) +} + +// Legacy mode keeps cross-origin embeds out of the scan. Rules and tags are +// mutually exclusive in one axe run, so `rules` wins when both are given. +export async function scan(page: Page, options: ScanOptions): Promise { + let builder = new AxeBuilder({ page }).setLegacyMode(true) + + if (options.include) builder = builder.include(options.include) + if (options.rules) builder = builder.withRules(options.rules) + else if (options.tags) builder = builder.withTags(options.tags) + if (options.excludeRules?.length) builder = builder.disableRules(options.excludeRules) + + const { violations } = await builder.analyze() + return violations +} + +export function violationIds(violations: Result[]): string[] { + return violations.map((violation) => violation.id) +} + +export function formatViolations( + violations: Result[], + maxNodes: number = MAX_REPORTED_NODES +): string { + return violations + .map((violation) => { + const shown = violation.nodes.slice(0, maxNodes) + const hidden = violation.nodes.length - shown.length + const nodes = shown + .map( + (node) => ` ${node.target.join(' ')}\n ${node.html.slice(0, MAX_REPORTED_HTML)}` + ) + .join('\n') + const more = hidden > 0 ? `\n … +${hidden} more node(s)` : '' + + return ( + `${violation.id} (${violation.impact ?? 'unknown'}, ${violation.nodes.length} node(s)): ` + + `${violation.help}\n${nodes}${more}` + ) + }) + .join('\n') +} diff --git a/e2e/shared/git.ts b/e2e/shared/git.ts new file mode 100644 index 00000000000..785b9f5f564 --- /dev/null +++ b/e2e/shared/git.ts @@ -0,0 +1,38 @@ +import { spawnSync } from 'node:child_process' + +import { parseChangedFilesList } from './paths.ts' + +export function git(args: string[], cwd: string): string { + const result = spawnSync('git', args, { cwd, encoding: 'utf8', env: process.env }) + if (result.status !== 0) { + const detail = (result.stderr || result.stdout || '').trim() + throw new Error(`git ${args.join(' ')} failed${detail ? `: ${detail}` : ''}`) + } + return result.stdout +} + +export function repoRootFrom(cwd: string): string { + return git(['rev-parse', '--show-toplevel'], cwd).trim() +} + +export function collectChangedFiles(repoRoot: string, baseRef: string): string[] { + const branchRange = `${baseRef}...HEAD` + const ranges = [ + ['diff', '--name-only', '--diff-filter=ACMR', branchRange], + ['diff', '--name-only', '--diff-filter=ACMR'], + ['diff', '--name-only', '--diff-filter=ACMR', '--cached'], + ] + + const files = new Set() + for (const args of ranges) { + try { + for (const file of parseChangedFilesList(git(args, repoRoot))) { + files.add(file) + } + } catch (error) { + if (args.includes(branchRange)) throw error + } + } + + return [...files].sort() +} diff --git a/e2e/shared/package.json b/e2e/shared/package.json new file mode 100644 index 00000000000..650c71994e6 --- /dev/null +++ b/e2e/shared/package.json @@ -0,0 +1,14 @@ +{ + "name": "e2e-shared", + "version": "1.0.0", + "private": true, + "type": "module", + "dependencies": { + "@axe-core/playwright": "^4.12.1", + "@playwright/test": "^1.59.1", + "axe-core": "^4.12.1" + }, + "devDependencies": { + "@types/node": "catalog:" + } +} diff --git a/e2e/shared/paths.ts b/e2e/shared/paths.ts new file mode 100644 index 00000000000..86c8ad8030e --- /dev/null +++ b/e2e/shared/paths.ts @@ -0,0 +1,22 @@ +import { sep } from 'node:path' + +export function normalizeRepoPath(filePath: string): string { + return filePath.replaceAll('\\', '/') +} + +export function parseChangedFilesList(input: string): string[] { + return input + .split(/[\n,]/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => normalizeRepoPath(line.split(sep).join('/'))) +} + +export function parsePagePaths(raw: string | undefined): string[] { + if (!raw?.trim()) return [] + return raw + .split(/[\n,]/) + .map((path) => path.trim()) + .filter(Boolean) + .map((path) => (path.startsWith('/') ? path : `/${path}`)) +} diff --git a/e2e/shared/resolve-scope-cli.ts b/e2e/shared/resolve-scope-cli.ts new file mode 100644 index 00000000000..f780e384ecd --- /dev/null +++ b/e2e/shared/resolve-scope-cli.ts @@ -0,0 +1,50 @@ +import { parseChangedFilesList } from './paths.ts' +import type { ScopeResult } from './run-suite.ts' + +export type ResolveScopeCliConfig = { + label: string + repoRoot: string + resolveScope: (options: { changedFiles: string[]; repoRoot: string }) => Promise +} + +async function readStdin(): Promise { + const chunks: Buffer[] = [] + for await (const chunk of process.stdin) { + chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk) + } + return Buffer.concat(chunks).toString('utf8') +} + +export async function runResolveScopeCli(config: ResolveScopeCliConfig): Promise { + const argv = process.argv.slice(2) + const filesIdx = argv.indexOf('--files') + + let changedFiles: string[] + if (filesIdx !== -1 && argv[filesIdx + 1]) { + changedFiles = parseChangedFilesList(argv[filesIdx + 1]) + } else if (process.stdin.isTTY) { + console.error('Pass changed files via stdin or --files path1,path2') + process.exit(2) + } else { + changedFiles = parseChangedFilesList(await readStdin()) + } + + const result = await config.resolveScope({ changedFiles, repoRoot: config.repoRoot }) + const output = `skip=${result.skip}\npaths=${result.pages.join(',')}\n` + + if (process.env.GITHUB_OUTPUT) { + const { appendFileSync } = await import('node:fs') + appendFileSync(process.env.GITHUB_OUTPUT, output) + } else { + process.stdout.write(output) + } + + if (result.pages.length > 0) { + console.error(`Resolved ${result.pages.length} ${config.label} page(s):`) + for (const page of result.pages) { + console.error(` ${page}`) + } + } else { + console.error(`No in-scope ${config.label} pages — skipping Playwright suite.`) + } +} diff --git a/e2e/shared/run-suite.ts b/e2e/shared/run-suite.ts new file mode 100644 index 00000000000..64e8daff42c --- /dev/null +++ b/e2e/shared/run-suite.ts @@ -0,0 +1,108 @@ +import { spawn } from 'node:child_process' + +import { collectChangedFiles, repoRootFrom } from './git.ts' + +export type ScopeResult = { pages: string[]; skip: boolean } + +export type SuiteConfig = { + root: string + label: string + devCommand: string + pagePathsEnv: string + baseRefEnv: string + defaultBaseUrl: string + resolveScope: (options: { changedFiles: string[]; repoRoot: string }) => Promise + resolveAllPages?: (repoRoot: string) => Promise +} + +const PREFLIGHT_TIMEOUT_MS = 3_000 + +async function isBaseUrlReachable(baseUrl: string): Promise { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), PREFLIGHT_TIMEOUT_MS) + try { + await fetch(baseUrl, { signal: controller.signal }) + return true + } catch { + return false + } finally { + clearTimeout(timeout) + } +} + +async function resolvePagePaths(config: SuiteConfig): Promise { + const explicit = process.env[config.pagePathsEnv]?.trim() + if (explicit) { + return explicit + .split(/[\n,]/) + .map((path) => path.trim()) + .filter(Boolean) + } + + const baseRef = process.env[config.baseRefEnv]?.trim() || 'origin/master' + const repoRoot = repoRootFrom(config.root) + const changedFiles = collectChangedFiles(repoRoot, baseRef) + const result = await config.resolveScope({ changedFiles, repoRoot }) + + if (result.skip) { + console.error( + `No in-scope ${config.label} pages changed vs ${baseRef} (including working tree). Skipping Playwright.` + ) + return null + } + + console.error( + `Resolved ${result.pages.length} ${config.label} page(s) from changes vs ${baseRef}:` + ) + for (const page of result.pages) { + console.error(` ${page}`) + } + return result.pages +} + +export async function runSuite(config: SuiteConfig): Promise { + const rawArgs = process.argv.slice(2) + const runAll = rawArgs.includes('--all') + const playwrightArgs = rawArgs.filter((arg) => arg !== '--all' && arg !== '--') + + let pages: string[] | null + if (runAll && config.resolveAllPages) { + pages = await config.resolveAllPages(repoRootFrom(config.root)) + console.error(`Resolved all ${pages.length} in-scope ${config.label} page(s).`) + } else { + pages = await resolvePagePaths(config) + } + if (pages === null) process.exit(0) + + const hasMaxFailuresArg = playwrightArgs.some( + (arg) => arg === '-x' || arg.startsWith('--max-failures') + ) + const finalArgs = + runAll && !hasMaxFailuresArg ? [...playwrightArgs, '--max-failures=0'] : playwrightArgs + + const baseUrl = process.env.PLAYWRIGHT_BASE_URL?.trim() || config.defaultBaseUrl + if (!(await isBaseUrlReachable(baseUrl))) { + console.error(`No ${config.label} server responding at ${baseUrl}.`) + if (!process.env.PLAYWRIGHT_BASE_URL) { + console.error(`Start it with \`${config.devCommand}\`, or point at a deployed site.`) + } else { + console.error('Check that the URL is correct and reachable.') + } + process.exit(1) + } + + const child = spawn('pnpm', ['exec', 'playwright', 'test', ...finalArgs], { + cwd: config.root, + env: { ...process.env, [config.pagePathsEnv]: pages.join(',') }, + stdio: 'inherit', + shell: process.platform === 'win32', + }) + + child.on('exit', (code, signal) => { + if (signal) { + process.kill(process.pid, signal) + return + } + process.exit(code ?? 1) + }) +} diff --git a/e2e/www/.gitignore b/e2e/www/.gitignore new file mode 100644 index 00000000000..68c5d18f00d --- /dev/null +++ b/e2e/www/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/e2e/www/README.md b/e2e/www/README.md new file mode 100644 index 00000000000..7d197910a79 --- /dev/null +++ b/e2e/www/README.md @@ -0,0 +1,169 @@ +# WWW E2E tests + +This guide explains how to run Playwright end-to-end checks against marketing +site content pages. + +Use this suite when you change blog posts, events, customer stories, or +alternatives pages under `apps/www`. It loads each in-scope page, checks that it +returns a successful status, and scans it for a single accessibility rule. + +This page covers: + +- [Set up](#set-up) — install the browser once +- [Run the tests](#run-the-tests) — the usual local command +- [Choose a target URL](#choose-a-target-url) — production, preview, or local www +- [Override which pages run](#override-which-pages-run) — when the default git + scope is wrong +- [What the suite covers](#what-the-suite-covers) — in-scope paths and limits +- [Debug failures](#debug-failures) — reports and traces +- [How CI uses this suite](#how-ci-uses-this-suite) — pull request behavior + +## Set up + +1. From this directory, install the Playwright Chromium browser once: + + ```bash + cd e2e/www + pnpm exec playwright install chromium + ``` + +## Run the tests + +By default, `pnpm e2e:www` tests pages affected by your current changes: commits +since `origin/master`, plus staged and unstaged working-tree files. If nothing +in scope changed, the command exits successfully without starting Playwright. + +1. From the repository root, point the suite at a deployed site and run it: + + ```bash + PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:www + ``` + +Extra arguments pass through to Playwright, so `pnpm e2e:www --ui` opens UI mode +and `pnpm e2e:www --grep @a11y` runs only the accessibility assertions. + +### Run every in-scope page + +To test every content page instead of a changed-files scope — for example, a +periodic full-site check — run: + +```bash +PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:www:all +``` + +This ignores `WWW_E2E_PAGE_PATHS` and the 20-page cap described in +[Limits](#limits), and tests every page across all four content directories — +around 480 as of this writing. `--all` runs also default to +`--max-failures=0`, so a full run isn't cut short by `playwright.config.ts`'s +global `maxFailures: 3`. The suite runs one worker by default, so pass +`--workers` to parallelize it: + +```bash +PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:www:all -- --workers=4 +``` + +Against production, raising workers does not pay off: a serial full run finishes +in about 17 minutes with no failures, while four workers took longer and timed +out on 34 of 480 navigations. Those timeouts are load, not page defects. Prefer +the default single worker unless you are pointed at a preview or a local server. + +## Choose a target URL + +Tests use `PLAYWRIGHT_BASE_URL`. When unset, they default to the local www dev +server at `http://localhost:3000`. + +Prefer a deployed site for day-to-day checks. Use the local server only when you +need unpublished content that production does not serve yet. + +For a protected Vercel preview, also set `VERCEL_AUTOMATION_BYPASS_SECRET`. + +To use the local server, start it with `pnpm dev:www` from the repository root +and run the suite without `PLAYWRIGHT_BASE_URL`. + +## Override which pages run + +Leave `WWW_E2E_PAGE_PATHS` unset to keep the default changed-files scope. + +To test specific pages instead of the git diff: + +```bash +WWW_E2E_PAGE_PATHS=/blog/supabase-steve-chavez \ + PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:www +``` + +`WWW_E2E_PAGE_PATHS` accepts a comma- or newline-separated list of +site-relative paths. `WWW_E2E_BASE_REF` overrides the base ref the git diff +compares against. + +## What the suite covers + +### In scope + +| Changed path | Behavior | +| ------------------------------ | --------------------------- | +| `apps/www/_blog/*.mdx` | Test `/blog/` | +| `apps/www/_events/*.mdx` | Test `/events/` | +| `apps/www/_customers/*.mdx` | Test `/customers/` | +| `apps/www/_alternatives/*.mdx` | Test `/alternatives/` | + +Slugs come from the filename, matching `getAllPostSlugs` in +`apps/www/lib/posts.tsx`. Blog and event filenames drop their `YYYY-MM-DD-` +prefix; customers and alternatives use the filename as-is. + +Each page gets one test: it must return a successful status, and an axe scan +must report no `page-has-heading-one` violations. That is the only rule enforced +today — add more to `ENFORCED_RULES` in `features/www-pages.spec.ts` once a class +of issue reaches zero across the site. + +`ENFORCED_RULES` is deliberately separate from the docs suite's list. Docs +enforces `heading-order` as well; www cannot yet. A full-site scan found +`heading-order` violations on the large majority of content pages, almost all +from the same two shared components — the related-posts card (`h4` under an `h2`) +and a trailing `h6`. Enforcing it here would fail nearly every pull request. + +### Out of scope + +- Events with `disable_page_build: true`, which return a 404 by design +- Static marketing routes under `apps/www/pages` and `apps/www/app` +- Index and listing pages such as `/blog` and `/customers` +- Link checking, and every accessibility rule other than the one above + +### Limits + +Resolved scope is capped at 20 pages so a large content drop cannot explode +runtime. If a change resolves to more pages than that, only the first 20 in +sorted order are tested. To test beyond the cap, use `pnpm e2e:www:all` or set +`WWW_E2E_PAGE_PATHS` explicitly. + +## Debug failures + +1. Open the HTML report after a run: + + ```bash + pnpm -C e2e/www exec playwright show-report + ``` + +2. Inspect traces and screenshots under `test-results/` for failed runs. + +## How CI uses this suite + +The workflow at `.github/workflows/www-e2e.yml` runs on pull requests that touch +owned www content, `e2e/www`, or `e2e/shared`. + +1. Diff the pull request against its base branch and resolve in-scope page paths. +2. Skip Playwright when nothing in scope changed. +3. When `apps/www` changed, wait for the Vercel www preview and set + `PLAYWRIGHT_BASE_URL` to that preview. When no preview resolves, skip rather + than test against production, which does not have pages the pull request adds. +4. Run the suite with `WWW_E2E_PAGE_PATHS` set to the resolved list. + +Draft pull requests stay skipped until you mark them ready for review. Manual +`workflow_dispatch` runs require a `page_paths` input and accept an optional +`base_url`, which defaults to production. + +## Shared helpers + +`e2e/shared` holds the pieces this suite and `e2e/docs` both use: git diff +collection, page-path parsing, the runner, and the scope-resolver CLI. Suite +directories keep only what is specific to them — for www, that is the +content-file-to-URL mapping in `utils/resolve-www-scope.ts`. diff --git a/e2e/www/features/www-pages.spec.ts b/e2e/www/features/www-pages.spec.ts new file mode 100644 index 00000000000..c00a3e16890 --- /dev/null +++ b/e2e/www/features/www-pages.spec.ts @@ -0,0 +1,36 @@ +import { expect, test } from '@playwright/test' + +import { formatViolations, scan, violationIds } from '../../shared/axe.ts' +import { parsePagePaths } from '../../shared/paths.ts' + +const ENFORCED_RULES = ['page-has-heading-one'] + +const pagePaths = parsePagePaths(process.env.WWW_E2E_PAGE_PATHS) + +test.describe('WWW content pages', () => { + test('resolved page list must not be empty', () => { + expect( + pagePaths.length, + 'No pages to test. `pnpm e2e:www` resolves pages from git changes by default, ' + + 'or set WWW_E2E_PAGE_PATHS explicitly.' + ).toBeGreaterThan(0) + }) + + for (const pagePath of pagePaths) { + test(`${pagePath} loads and passes enforced a11y rules @a11y`, async ({ page }) => { + const response = await page.goto(pagePath) + expect( + response?.ok(), + `${pagePath} should return a successful status, got ${response?.status()}` + ).toBeTruthy() + + const violations = await scan(page, { rules: ENFORCED_RULES }) + + expect( + violationIds(violations), + `${pagePath} violates enforced a11y rules (${ENFORCED_RULES.join(', ')}):\n` + + formatViolations(violations) + ).toEqual([]) + }) + } +}) diff --git a/e2e/www/package.json b/e2e/www/package.json new file mode 100644 index 00000000000..30184e12f42 --- /dev/null +++ b/e2e/www/package.json @@ -0,0 +1,18 @@ +{ + "name": "e2e-www", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "e2e:www": "node --experimental-strip-types scripts/run-e2e-www.ts", + "e2e:www:all": "node --experimental-strip-types scripts/run-e2e-www.ts --all" + }, + "dependencies": { + "@playwright/test": "^1.59.1" + }, + "devDependencies": { + "@axe-core/playwright": "^4.12.1", + "@types/node": "catalog:", + "axe-core": "^4.12.1" + } +} diff --git a/e2e/www/playwright.config.ts b/e2e/www/playwright.config.ts new file mode 100644 index 00000000000..21044632f8b --- /dev/null +++ b/e2e/www/playwright.config.ts @@ -0,0 +1,31 @@ +import { defineConfig } from '@playwright/test' + +const IS_CI = !!process.env.CI + +export default defineConfig({ + testDir: './features', + testMatch: /.*\.spec\.ts/, + timeout: 60_000, + forbidOnly: IS_CI, + retries: IS_CI ? 2 : 0, + maxFailures: 3, + expect: { timeout: 15_000 }, + fullyParallel: true, + workers: 1, + use: { + baseURL: process.env.PLAYWRIGHT_BASE_URL || 'http://localhost:3000', + browserName: 'chromium', + headless: true, + navigationTimeout: 30_000, + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + extraHTTPHeaders: process.env.VERCEL_AUTOMATION_BYPASS_SECRET + ? { + 'x-vercel-protection-bypass': process.env.VERCEL_AUTOMATION_BYPASS_SECRET, + 'x-vercel-set-bypass-cookie': 'true', + } + : undefined, + }, + reporter: [['list'], ['html', { open: 'never', outputFolder: './playwright-report' }]], + outputDir: './test-results', +}) diff --git a/e2e/www/scripts/resolve-www-scope.ts b/e2e/www/scripts/resolve-www-scope.ts new file mode 100644 index 00000000000..6d1109e6030 --- /dev/null +++ b/e2e/www/scripts/resolve-www-scope.ts @@ -0,0 +1,15 @@ +#!/usr/bin/env node +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { runResolveScopeCli } from '../../shared/resolve-scope-cli.ts' +import { resolveWwwScope } from '../utils/resolve-www-scope.ts' + +runResolveScopeCli({ + label: 'www', + repoRoot: join(dirname(fileURLToPath(import.meta.url)), '../../..'), + resolveScope: resolveWwwScope, +}).catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exit(1) +}) diff --git a/e2e/www/scripts/run-e2e-www.ts b/e2e/www/scripts/run-e2e-www.ts new file mode 100644 index 00000000000..cc7d518f867 --- /dev/null +++ b/e2e/www/scripts/run-e2e-www.ts @@ -0,0 +1,20 @@ +#!/usr/bin/env node +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { runSuite } from '../../shared/run-suite.ts' +import { resolveAllWwwPages, resolveWwwScope } from '../utils/resolve-www-scope.ts' + +runSuite({ + root: join(dirname(fileURLToPath(import.meta.url)), '..'), + label: 'www', + devCommand: 'pnpm dev:www', + pagePathsEnv: 'WWW_E2E_PAGE_PATHS', + baseRefEnv: 'WWW_E2E_BASE_REF', + defaultBaseUrl: 'http://localhost:3000', + resolveScope: resolveWwwScope, + resolveAllPages: resolveAllWwwPages, +}).catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exit(1) +}) diff --git a/e2e/www/tsconfig.json b/e2e/www/tsconfig.json new file mode 100644 index 00000000000..cecfba3008e --- /dev/null +++ b/e2e/www/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "skipLibCheck": true, + "esModuleInterop": true, + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "types": ["node"] + } +} diff --git a/e2e/www/utils/resolve-www-scope.ts b/e2e/www/utils/resolve-www-scope.ts new file mode 100644 index 00000000000..fa966c820f8 --- /dev/null +++ b/e2e/www/utils/resolve-www-scope.ts @@ -0,0 +1,91 @@ +import { readdir, readFile } from 'node:fs/promises' +import { basename, join } from 'node:path' + +import { normalizeRepoPath } from '../../shared/paths.ts' + +const DATE_PREFIX_LENGTH = 11 + +export const MAX_SCOPED_PAGES = 20 + +const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---/ +const DISABLE_PAGE_BUILD_RE = /^\s*disable_page_build:\s*true\s*$/m + +const CONTENT_SOURCES = [ + { dir: 'apps/www/_blog', urlPrefix: '/blog/', datePrefixed: true }, + { dir: 'apps/www/_events', urlPrefix: '/events/', datePrefixed: true }, + { dir: 'apps/www/_customers', urlPrefix: '/customers/', datePrefixed: false }, + { dir: 'apps/www/_alternatives', urlPrefix: '/alternatives/', datePrefixed: false }, +] as const + +export type ResolveWwwScopeOptions = { + changedFiles: string[] + repoRoot: string + maxPages?: number +} + +export function changedFileToPagePath(filePath: string): string | null { + const normalized = normalizeRepoPath(filePath) + if (!normalized.endsWith('.mdx')) return null + + for (const { dir, urlPrefix, datePrefixed } of CONTENT_SOURCES) { + if (!normalized.startsWith(`${dir}/`)) continue + + const filename = normalized.slice(dir.length + 1) + if (filename.includes('/')) return null + + const name = basename(filename, '.mdx') + const slug = datePrefixed ? name.substring(DATE_PREFIX_LENGTH) : name + return slug ? `${urlPrefix}${slug}` : null + } + + return null +} + +async function isPageBuilt(absolutePath: string): Promise { + let source: string + try { + source = await readFile(absolutePath, 'utf8') + } catch (error) { + // A deleted page has no route to test; anything else is a real failure. + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false + throw error + } + + const frontmatter = FRONTMATTER_RE.exec(source)?.[1] + return !frontmatter || !DISABLE_PAGE_BUILD_RE.test(frontmatter) +} + +export async function resolveWwwScope(options: ResolveWwwScopeOptions) { + const pages = new Set() + + for (const file of options.changedFiles) { + const page = changedFileToPagePath(file) + if (page && (await isPageBuilt(join(options.repoRoot, file)))) { + pages.add(page) + } + } + + const sorted = [...pages].sort().slice(0, options.maxPages ?? MAX_SCOPED_PAGES) + + return { pages: sorted, skip: sorted.length === 0 } +} + +export async function resolveAllWwwPages(repoRoot: string): Promise { + const pages = new Set() + + for (const { dir, urlPrefix, datePrefixed } of CONTENT_SOURCES) { + const absoluteDir = join(repoRoot, dir) + const entries = await readdir(absoluteDir, { withFileTypes: true }).catch(() => []) + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.mdx')) continue + if (!(await isPageBuilt(join(absoluteDir, entry.name)))) continue + + const name = basename(entry.name, '.mdx') + const slug = datePrefixed ? name.substring(DATE_PREFIX_LENGTH) : name + if (slug) pages.add(`${urlPrefix}${slug}`) + } + } + + return [...pages].sort() +} diff --git a/package.json b/package.json index 831395fa112..35774f9083a 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,8 @@ "e2e:docs:a11y": "pnpm --prefix e2e/docs run e2e:docs:a11y", "e2e:docs:ui": "pnpm --prefix e2e/docs run e2e:ui", "e2e:docs:local-smoke": "pnpm --prefix e2e/docs run e2e:docs:local-smoke", + "e2e:www": "pnpm --prefix e2e/www run e2e:www", + "e2e:www:all": "pnpm --prefix e2e/www run e2e:www:all", "perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/", "perf:meta": "ab -t 5 -c 20 -T application/json http://localhost:5555/tables", "setup:cli": "supabase start -x studio && supabase status --output json > keys.json && node scripts/generateLocalEnv.js", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 368781bcaea..b98f9d83fbc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2034,6 +2034,22 @@ importers: specifier: ^4.12.1 version: 4.12.1 + e2e/shared: + dependencies: + '@axe-core/playwright': + specifier: ^4.12.1 + version: 4.12.1(playwright-core@1.59.1) + '@playwright/test': + specifier: ^1.59.1 + version: 1.59.1 + axe-core: + specifier: ^4.12.1 + version: 4.12.1 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.13.14 + e2e/studio: dependencies: '@playwright/test': @@ -2062,6 +2078,22 @@ importers: specifier: 'catalog:' version: 4.22.4 + e2e/www: + dependencies: + '@playwright/test': + specifier: ^1.59.1 + version: 1.59.1 + devDependencies: + '@axe-core/playwright': + specifier: ^4.12.1 + version: 4.12.1(playwright-core@1.59.1) + '@types/node': + specifier: 'catalog:' + version: 22.13.14 + axe-core: + specifier: ^4.12.1 + version: 4.12.1 + packages/ai-commands: dependencies: '@serafin/schema-builder': diff --git a/scripts/waitForVercelDocsPreview.js b/scripts/waitForVercelPreview.js similarity index 77% rename from scripts/waitForVercelDocsPreview.js rename to scripts/waitForVercelPreview.js index 781d62370ba..37dc2d34616 100644 --- a/scripts/waitForVercelDocsPreview.js +++ b/scripts/waitForVercelPreview.js @@ -1,15 +1,17 @@ // Vercel's GitHub App has stopped writing GitHub Deployment objects since // 2026-02-17 (broken app auth), so polling the Deployments API (as -// vercel/wait-for-deployment-action does) times out even though the docs -// preview builds fine. Poll the "Vercel – docs" commit status instead, then -// resolve the actual preview URL via Vercel's own deployments API. +// vercel/wait-for-deployment-action does) times out even though the preview +// builds fine. Poll the project's Vercel commit status instead, then resolve +// the actual preview URL via Vercel's own deployments API. +// +// Set VERCEL_STATUS_CONTEXT to the project's commit status name, e.g. +// "Vercel – docs" or "Vercel – zone-www-dot-com". const { appendFileSync } = require('fs') -const STATUS_CONTEXT = 'Vercel – docs' const TIMEOUT_MS = 900_000 const POLL_INTERVAL_MS = 15_000 -async function fetchLatestStatus(repository, sha, githubToken) { +async function fetchLatestStatus(repository, sha, githubToken, statusContext) { const url = `https://api.github.com/repos/${repository}/commits/${sha}/statuses` const response = await fetch(url, { headers: { @@ -25,7 +27,7 @@ async function fetchLatestStatus(repository, sha, githubToken) { const statuses = await response.json() return statuses - .filter((status) => status.context === STATUS_CONTEXT) + .filter((status) => status.context === statusContext) .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())[0] } @@ -68,21 +70,23 @@ async function main() { const githubToken = process.env.GITHUB_TOKEN const vercelToken = process.env.VERCEL_TOKEN const teamId = process.env.VERCEL_TEAM_ID + const statusContext = process.env.VERCEL_STATUS_CONTEXT if (!repository) throw new Error('GITHUB_REPOSITORY environment variable is required') if (!sha) throw new Error('HEAD_SHA environment variable is required') if (!githubToken) throw new Error('GITHUB_TOKEN environment variable is required') if (!vercelToken) throw new Error('VERCEL_TOKEN environment variable is required') + if (!statusContext) throw new Error('VERCEL_STATUS_CONTEXT environment variable is required') const start = Date.now() for (;;) { - const latest = await fetchLatestStatus(repository, sha, githubToken) + const latest = await fetchLatestStatus(repository, sha, githubToken, statusContext) if (latest?.state === 'success') { if (!latest.target_url) { throw new Error( - 'Vercel docs commit status succeeded but had no target_url to resolve a deployment from' + `"${statusContext}" commit status succeeded but had no target_url to resolve a deployment from` ) } const deploymentUrl = await resolveDeploymentUrl(latest.target_url, vercelToken, teamId) @@ -91,11 +95,11 @@ async function main() { } if (latest?.state === 'failure' || latest?.state === 'error') { - throw new Error(`Vercel docs deployment failed (commit status: ${latest.state})`) + throw new Error(`"${statusContext}" deployment failed (commit status: ${latest.state})`) } if (Date.now() - start > TIMEOUT_MS) { - throw new Error('Timed out after 900s waiting for the Vercel docs preview deployment') + throw new Error(`Timed out after 900s waiting for the "${statusContext}" preview deployment`) } await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))