diff --git a/apps/docs/DEVELOPERS.md b/apps/docs/DEVELOPERS.md index 10ea54cdb32..37f48953ea1 100644 --- a/apps/docs/DEVELOPERS.md +++ b/apps/docs/DEVELOPERS.md @@ -35,6 +35,26 @@ This creates Markdown files for all routes under the `public/markdown/guides` di For production this setup runs as a `prebuild` task to allow Vercel to bundle these files with middleware and functions. +## Accessibility checks + +Docs pages are scanned for WCAG 2.1 A/AA issues with axe-core, as part of the +Playwright suite in `e2e/docs`. Pull requests scan the pages your change affects, +limited to the main article. + +To scan the pages your current branch changes: + +```bash +PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:docs:a11y +``` + +That resolves which pages to scan from your branch, but reads them from +production, so it won't see your edits and will 404 on a page you just added. +Point `PLAYWRIGHT_BASE_URL` at your pull request's preview to scan your own +content. + +See [`e2e/docs/README.md`](https://github.com/supabase/supabase/blob/master/e2e/docs/README.md) +for coverage and skipped rules. + ## Contributing For repo organization and style guide, see the [contributing guide](https://github.com/supabase/supabase/blob/master/apps/docs/CONTRIBUTING.md). diff --git a/e2e/docs/README.md b/e2e/docs/README.md index 357f022892e..ea80e37ad60 100644 --- a/e2e/docs/README.md +++ b/e2e/docs/README.md @@ -15,6 +15,7 @@ This page covers: - [Override which pages run](#override-which-pages-run) — when the default git scope is wrong - [What the suite covers](#what-the-suite-covers) — in-scope paths and limits +- [Accessibility scans](#accessibility-scans) — WCAG coverage and skipped rules - [Debug failures](#debug-failures) — reports and traces - [How CI uses this suite](#how-ci-uses-this-suite) — pull request behavior @@ -168,6 +169,30 @@ staged and unstaged working-tree changes. } | pnpm -C e2e/docs resolve-docs-scope ``` +## Accessibility scans + +The `@a11y`-tagged test scans each in-scope page for WCAG 2.1 A/AA violations using +`@axe-core/playwright`, limited to the main article. + +```bash +PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:docs:a11y +``` + +Which pages get scanned comes from your branch, but the content comes from +whatever you point `PLAYWRIGHT_BASE_URL` at. Production won't have your edits and +will 404 on a page you just added, so use your pull request's preview to scan your +own content. + +`EXCLUDED_RULES` in `utils/axe-helpers.ts` lists the rules the scan skips. +`color-contrast` is most of the scan time and finds nothing inside an article, since +docs contrast comes from shared tokens and chrome. The rest target ``, ``, +and ``, which an article-scoped scan can't reach. + +Cross-origin frames are skipped, so a third-party embed isn't reported as ours. + +Not covered: `/docs/reference/*`, shared chrome, and most of WCAG. Keyboard +navigation, focus management, and screen reader behavior need manual testing. + ## Debug failures 1. Open the HTML report after a run: @@ -186,7 +211,8 @@ owned docs content, partials, or `e2e/docs`. 1. Diff the pull request against its base branch and resolve in-scope page paths. 2. Skip Playwright when nothing in scope changed. 3. When `apps/docs` changed, wait for the Vercel docs preview and set - `PLAYWRIGHT_BASE_URL` to that preview. Otherwise use production. + `PLAYWRIGHT_BASE_URL` to that preview. When no preview resolves, skip rather than + test against production. 4. Run the suite with `DOCS_E2E_PAGE_PATHS` set to the resolved list. Draft pull requests stay skipped until you mark them ready for review. Manual diff --git a/e2e/docs/features/docs-pages.spec.ts b/e2e/docs/features/docs-pages.spec.ts index 3881a0b54d6..3a58f155021 100644 --- a/e2e/docs/features/docs-pages.spec.ts +++ b/e2e/docs/features/docs-pages.spec.ts @@ -1,6 +1,17 @@ -import { AxeBuilder } from '@axe-core/playwright' import { expect, test } from '@playwright/test' +import type { TestInfo } from '@playwright/test' +import { + attachScanReport, + blockingViolations, + ENFORCED_RULES, + formatViolations, + scanArticle, + scanLooksEmpty, + settleForAxe, + shouldEnforceAll, + unloadedResult, +} from '../utils/axe-helpers.js' import { articleSelectorForPagePath, browserLikeUserAgent, @@ -10,6 +21,11 @@ import { const pagePaths = parseDocsE2EPagePaths(process.env.DOCS_E2E_PAGE_PATHS) +function annotate(testInfo: TestInfo, description: string) { + testInfo.annotations.push({ type: 'warning', description }) + console.warn(`::warning title=Accessibility::${description}`) +} + test.describe('Docs owned pages', () => { // playwright.config.ts sets fullyParallel: false, and Playwright shards // work by file rather than by test in that mode — without this, every test @@ -63,19 +79,69 @@ test.describe('Docs owned pages', () => { } for (const pagePath of pagePaths) { - test(`${pagePath} has a valid heading hierarchy @a11y`, async ({ page }) => { - const articleSelector = articleSelectorForPagePath(pagePath) - const response = await page.goto(pagePath) - expect(response?.ok(), `Expected a successful response for ${pagePath}`).toBeTruthy() + test(`${pagePath} has no blocking accessibility violations @a11y`, async ({ + page, + }, testInfo) => { + test.setTimeout(120_000) - const axeResults = await new AxeBuilder({ page }) - .include(articleSelector) - .withRules(['heading-order', 'page-has-heading-one']) - .analyze() + const include = articleSelectorForPagePath(pagePath) + + let response + try { + response = await page.goto(pagePath) + } catch (error) { + await attachScanReport(testInfo, unloadedResult(pagePath, pagePath, null, include)) + throw error + } + + const status = response?.status() ?? null + + if (!response?.ok()) { + await attachScanReport(testInfo, unloadedResult(pagePath, page.url(), status, include)) + expect( + response?.ok(), + `Expected a successful response for ${pagePath}, got ${status}` + ).toBeTruthy() + return + } + + await settleForAxe(page) + + await expect( + page.locator(include), + `No article matching "${include}" on ${pagePath}. This suite covers guides and ` + + 'troubleshooting entries; other routes have no article element to scan.' + ).toBeVisible() + + const result = await scanArticle(page, pagePath, include) + result.status = status + + await attachScanReport(testInfo, result) + + if (scanLooksEmpty(result)) { + annotate( + testInfo, + `${pagePath} scanned only ${result.elementCount} element(s) in ${include}, so a clean ` + + 'result here proves nothing. Most likely the page had not finished rendering.' + ) + } + + const blocking = blockingViolations(result) + + const reported = result.violations.filter((violation) => !blocking.includes(violation)) + if (reported.length) { + annotate( + testInfo, + `${pagePath} has ${reported.length} non-blocking accessibility finding(s): ` + + reported.map((v) => `${v.id} (${v.nodes.length})`).join(', ') + ) + } + + const enforced = shouldEnforceAll() ? 'all WCAG 2.1 A/AA rules' : ENFORCED_RULES.join(', ') expect( - axeResults.violations, - `Heading hierarchy issues in ${articleSelector}:\n${JSON.stringify(axeResults.violations, null, 2)}` + blocking, + `${pagePath} has blocking a11y violations (${enforced}):\n${formatViolations(blocking)}` ).toEqual([]) }) } diff --git a/e2e/docs/package.json b/e2e/docs/package.json index b1e0371f232..c1eb67b9036 100644 --- a/e2e/docs/package.json +++ b/e2e/docs/package.json @@ -16,6 +16,7 @@ }, "devDependencies": { "@axe-core/playwright": "^4.12.1", - "@types/node": "catalog:" + "@types/node": "catalog:", + "axe-core": "^4.12.1" } } diff --git a/e2e/docs/scripts/run-e2e-docs.ts b/e2e/docs/scripts/run-e2e-docs.ts index f13ddb76494..c8883aa90bb 100644 --- a/e2e/docs/scripts/run-e2e-docs.ts +++ b/e2e/docs/scripts/run-e2e-docs.ts @@ -122,10 +122,7 @@ async function resolvePagePaths(): Promise { async function main() { const rawArgs = process.argv.slice(2) const runAll = rawArgs.includes('--all') - const withoutAll = rawArgs.filter((arg) => arg !== '--all') - // pnpm's `--` separator (from `pnpm run ... -- --list`) can land at index 0 - // or, once `--all` is stripped, wherever `--all` used to precede it. - const playwrightArgs = withoutAll[0] === '--' ? withoutAll.slice(1) : withoutAll + const playwrightArgs = rawArgs.filter((arg) => arg !== '--all' && arg !== '--') const pages = runAll ? await resolveAllPagePaths() : await resolvePagePaths() if (pages === null) { diff --git a/e2e/docs/utils/axe-helpers.ts b/e2e/docs/utils/axe-helpers.ts new file mode 100644 index 00000000000..ba82e3d0a96 --- /dev/null +++ b/e2e/docs/utils/axe-helpers.ts @@ -0,0 +1,148 @@ +import { AxeBuilder } from '@axe-core/playwright' +import type { Page, TestInfo } from '@playwright/test' +import type { Result } from 'axe-core' + +export const WCAG_TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'] + +export const ENFORCED_RULES = ['heading-order', 'page-has-heading-one'] + +export const EXCLUDED_RULES = [ + 'color-contrast', + 'html-has-lang', + 'html-lang-valid', + 'html-xml-lang-mismatch', + 'document-title', + 'aria-hidden-body', + 'meta-viewport', + 'meta-refresh', + 'css-orientation-lock', +] + +export interface A11yScanResult { + surface: string + url: string + include: string + excludedRules: string[] + loaded: boolean + status: number | null + elementCount: number + violations: Result[] +} + +export function shouldEnforceAll(): boolean { + return !!process.env.A11Y_ENFORCE_ALL +} + +export async function settleForAxe(page: Page): Promise { + await page.waitForLoadState('domcontentloaded') + await page.waitForLoadState('networkidle', { timeout: 15_000 }).catch(() => {}) + + await page + .evaluate( + ({ quietMs, capMs }) => + new Promise((resolve) => { + let timer: ReturnType + const observer = new MutationObserver(() => { + clearTimeout(timer) + timer = setTimeout(finish, quietMs) + }) + + function finish() { + clearTimeout(cap) + clearTimeout(timer) + observer.disconnect() + resolve() + } + + const cap = setTimeout(finish, capMs) + timer = setTimeout(finish, quietMs) + observer.observe(document.body, { subtree: true, childList: true, attributes: true }) + }), + { quietMs: 500, capMs: 5_000 } + ) + .catch(() => {}) +} + +export async function scanArticle( + page: Page, + surface: string, + include: string +): Promise { + const scan = () => new AxeBuilder({ page }).setLegacyMode(true).include(include) + + const reported = await scan().withTags(WCAG_TAGS).disableRules(EXCLUDED_RULES).analyze() + + const enforced = await scan().withRules(ENFORCED_RULES).analyze() + + const byRule = new Map( + [...reported.violations, ...enforced.violations].map((violation) => [violation.id, violation]) + ) + + const elementCount = await page.evaluate( + (selector) => document.querySelector(selector)?.querySelectorAll('*').length ?? 0, + include + ) + + return { + surface, + url: page.url(), + include, + excludedRules: EXCLUDED_RULES, + loaded: true, + status: null, + elementCount, + violations: [...byRule.values()], + } +} + +export function unloadedResult( + surface: string, + url: string, + status: number | null, + include: string +): A11yScanResult { + return { + surface, + url, + include, + excludedRules: EXCLUDED_RULES, + loaded: false, + status, + elementCount: 0, + violations: [], + } +} + +export const MIN_MEANINGFUL_ELEMENTS = 20 + +export function scanLooksEmpty( + result: A11yScanResult, + minElements: number = MIN_MEANINGFUL_ELEMENTS +): boolean { + return result.elementCount < minElements +} + +export async function attachScanReport(testInfo: TestInfo, result: A11yScanResult): Promise { + await testInfo.attach('axe-results.json', { + body: JSON.stringify(result, null, 2), + contentType: 'application/json', + }) +} + +export function blockingViolations(result: A11yScanResult): Result[] { + if (shouldEnforceAll()) return result.violations + return result.violations.filter((violation) => ENFORCED_RULES.includes(violation.id)) +} + +export function formatViolations(violations: Result[]): string { + return violations + .map( + (violation) => + `${violation.id} (${violation.impact}, ${violation.nodes.length} node(s)): ${violation.help}\n` + + violation.nodes + .slice(0, 5) + .map((node) => ` ${node.target.join(' ')}\n ${node.html.slice(0, 200)}`) + .join('\n') + ) + .join('\n') +} diff --git a/package.json b/package.json index a495efbcbb6..831395fa112 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "e2e:ui": "pnpm --prefix e2e/studio run e2e:ui", "e2e:docs": "pnpm --prefix e2e/docs run e2e:docs", "e2e:docs:all": "pnpm --prefix e2e/docs run e2e:docs:all", + "e2e:docs:a11y": "pnpm --prefix e2e/docs run e2e:docs:a11y", "e2e:docs:ui": "pnpm --prefix e2e/docs run e2e:ui", "e2e:docs:local-smoke": "pnpm --prefix e2e/docs run e2e:docs:local-smoke", "perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dad4f818cfb..79bce50d667 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2027,6 +2027,9 @@ importers: '@types/node': specifier: 'catalog:' version: 22.13.14 + axe-core: + specifier: ^4.12.1 + version: 4.12.1 e2e/studio: dependencies: @@ -10253,6 +10256,7 @@ packages: cron-parser@4.9.0: resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} engines: {node: '>=12.0.0'} + deprecated: v4 is no longer maintained, upgrade to v5 croner@10.0.1: resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==}