From b5cae478bc7a3b093217ba6ea035098a674957b4 Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Fri, 24 Jul 2026 10:29:12 -0700 Subject: [PATCH] fix(docs) Add smoke test for local development without credentials (#48218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes DOCS-1210 Closes DOCS-1209 #48226 needs to merge first for CI failure ## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. YES ## What kind of change does this PR introduce? Test coverage and several small bug fixes discovered during implementation. ## What is the current behavior? Nothing verified that `pnpm run dev:docs` keeps working without private credentials. We value this command working, especially for community contributors. However, this issue can go undetected by employees at Supabase since many of us have credentials in place. We do not want this to go a week before finding and fixing like in the previous instance. ## What is the new behavior? - **New Playwright test suite**: `e2e/docs/local-smoke/no-credentials.spec.ts` boots the docs dev server with zero GitHub App/Supabase secrets and checks 5 routes covering each known failure point. - **CI**: a new `local-dev-smoke` job in `docs-tests.yml` runs this suite with no credentials configured. ## Additional bugs resolved Setting up this test exposed other issues that are fixed in this PR: - **Troubleshooting.utils.ts crash** — Unguarded Supabase call pattern, crashing every troubleshooting article. Added the same guard as previous fixes. - **Missing manifest.json** — middleware.ts statically imports public/markdown/manifest.json, which is gitignored and only generated by a build step that's skipped in local dev. On a fresh checkout it doesn't exist, so middleware fails to compile and takes down every page. Fixed by committing a placeholder [] (real builds still regenerate the full file). - **Phantom @code-hike/mdx import** — apps/docs/app/layout.tsx imported @code-hike/mdx/styles.css, but only apps/www actually declares that dependency. Worked by accident whenever both apps were installed together; broke in CI's docs-only install. Turned out to be dead code (nothing in docs actually uses code-hike), so fixed by deleting the unused imports rather than adding the dependency. ## Summary by CodeRabbit ## Summary by CodeRabbit * **New Features** * Added a credential-free “local smoke” end-to-end test suite for key documentation routes. * **Bug Fixes** * Improved troubleshooting behavior when required external service credentials are missing. * Updated federated “wrappers” documentation pages to gracefully show a fallback message when external content can’t be fetched. * **Tests** * Added a dedicated local-smoke Playwright runner and enhanced CI path-based triggering and reporting (failure-focused artifacts). * **Chores** * Refined docs workflow path filters and adjusted docs markdown manifest/ignore rules for generated content. --- .github/workflows/docs-e2e.yml | 6 +- .github/workflows/docs-tests.yml | 54 ++++++++++++++ apps/docs/.gitignore | 7 +- apps/docs/app/layout.tsx | 2 - .../docs/components/DatabaseAdvisorsIndex.tsx | 5 +- .../features/docs/Troubleshooting.utils.ts | 10 ++- apps/docs/public/markdown/manifest.json | 1 + apps/docs/types/code-hike.d.ts | 6 -- e2e/docs/.gitignore | 1 + e2e/docs/local-smoke/no-credentials.spec.ts | 72 +++++++++++++++++++ e2e/docs/package.json | 3 +- e2e/docs/playwright.local-smoke.config.ts | 56 +++++++++++++++ package.json | 1 + 13 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 apps/docs/public/markdown/manifest.json delete mode 100644 apps/docs/types/code-hike.d.ts create mode 100644 e2e/docs/local-smoke/no-credentials.spec.ts create mode 100644 e2e/docs/playwright.local-smoke.config.ts diff --git a/.github/workflows/docs-e2e.yml b/.github/workflows/docs-e2e.yml index f27fa3a407d..2db833df070 100644 --- a/.github/workflows/docs-e2e.yml +++ b/.github/workflows/docs-e2e.yml @@ -8,7 +8,11 @@ on: - 'apps/docs/content/guides/getting-started/quickstarts/nextjs.mdx' - 'apps/docs/content/_partials/quickstart_db_setup.mdx' - 'apps/docs/content/_partials/api_settings.mdx' - - 'e2e/docs/**' + - 'e2e/docs/features/**' + - 'e2e/docs/utils/**' + - 'e2e/docs/playwright.config.ts' + - 'e2e/docs/package.json' + - 'e2e/docs/tsconfig.json' - 'pnpm-lock.yaml' - '.github/workflows/docs-e2e.yml' workflow_dispatch: diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.yml index b92cdead161..51c6b3a277a 100644 --- a/.github/workflows/docs-tests.yml +++ b/.github/workflows/docs-tests.yml @@ -6,6 +6,11 @@ on: paths: - 'apps/docs/**/*.ts*' - 'apps/docs/spec/**/*.json' + - 'apps/docs/.env.development' + - 'apps/docs/package.json' + - 'e2e/docs/local-smoke/**' + - 'e2e/docs/playwright.local-smoke.config.ts' + - 'e2e/docs/package.json' # Cancel old builds on new commit for same workflow + branch/PR concurrency: @@ -70,3 +75,52 @@ jobs: echo "GITHUB_CLIENT_ID=dummy-id" >> .env echo "GITHUB_SECRET=dummy-secret" >> .env pnpm run test:docs + + local-dev-smoke: + name: Local dev smoke (no credentials) + runs-on: blacksmith-4vcpu-ubuntu-2404 + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + sparse-checkout: | + apps/docs + examples + packages + supabase + patches + e2e/docs + + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + name: Install pnpm + with: + run_install: false + + - name: Use Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + + - name: Install deps + run: pnpm install --frozen-lockfile + + - name: Install Playwright Chromium + run: pnpm -C e2e/docs exec playwright install chromium --with-deps --only-shell + + # Deliberately does not set DOCS_GITHUB_APP_*, SUPABASE_SECRET_KEY, + # OPENAI_API_KEY, or DOCS_REVALIDATION_KEYS — their absence here is what + # verifies `pnpm run dev:docs` still works without private credentials. + - name: Run local dev smoke tests + run: pnpm run e2e:docs:local-smoke + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: docs-local-smoke-playwright-report + path: | + e2e/docs/playwright-report-local-smoke/ + e2e/docs/test-results/ + retention-days: 7 diff --git a/apps/docs/.gitignore b/apps/docs/.gitignore index 89db757aa41..abe7aa2494f 100644 --- a/apps/docs/.gitignore +++ b/apps/docs/.gitignore @@ -29,8 +29,11 @@ yarn-error.log* public/sitemap.xml # Per-source llms files (generated by build:llms, served by www) public/llms/ -# Generated guide and reference markdown files -public/markdown/ +# Generated guide and reference markdown files. manifest.json is committed +# with a placeholder empty array so middleware.ts's import always resolves, +# even when build:markdown hasn't run (e.g. in local dev, which skips it). +public/markdown/* +!public/markdown/manifest.json public/docs.tar.gz public/docs/ diff --git a/apps/docs/app/layout.tsx b/apps/docs/app/layout.tsx index e0a43504b95..6fcd8c1c856 100644 --- a/apps/docs/app/layout.tsx +++ b/apps/docs/app/layout.tsx @@ -1,5 +1,3 @@ -import '@code-hike/mdx/styles.css' -import 'config/code-hike.css' import 'ui-patterns/ShimmeringLoader/index.css' import '../styles/globals.css' import '../styles/prism-okaidia.css' diff --git a/apps/docs/components/DatabaseAdvisorsIndex.tsx b/apps/docs/components/DatabaseAdvisorsIndex.tsx index 9e5cda92899..32c93df2aa6 100644 --- a/apps/docs/components/DatabaseAdvisorsIndex.tsx +++ b/apps/docs/components/DatabaseAdvisorsIndex.tsx @@ -17,7 +17,10 @@ export async function DatabaseAdvisorsIndex() { const raw = await readFile(join(GENERATED_DIRECTORY, 'database-advisors.json'), 'utf-8') lints = JSON.parse(raw) } catch (error) { - throw error('[database-advisors] Failed to read generated advisor docs', error) + console.warn( + '[database-advisors] Failed to read generated advisor docs; rendering without them', + error + ) } return ( diff --git a/apps/docs/features/docs/Troubleshooting.utils.ts b/apps/docs/features/docs/Troubleshooting.utils.ts index d8cf6d7d5c0..045e3e16c56 100644 --- a/apps/docs/features/docs/Troubleshooting.utils.ts +++ b/apps/docs/features/docs/Troubleshooting.utils.ts @@ -1,13 +1,13 @@ +import { cache_fullProcess_withDevCacheBust } from '~/features/helpers.fs' +import { supabase } from '~/lib/supabase' import { cache } from 'react' import { z } from 'zod' -import { cache_fullProcess_withDevCacheBust } from '~/features/helpers.fs' -import { supabase } from '~/lib/supabase' import { getAllTroubleshootingEntriesInternal, getArticleSlug as getArticleSlugInternal, - TroubleshootingSchema, TROUBLESHOOTING_DIRECTORY, + TroubleshootingSchema, } from './Troubleshooting.utils.common.mjs' import { formatError } from './Troubleshooting.utils.shared' @@ -93,6 +93,10 @@ async function getTroubleshootingUpdatedDatesInternal() { .map((entry) => entry.data.database_id) .filter((id) => !id.startsWith('pseudo-')) + if (!process.env.NEXT_PUBLIC_SUPABASE_URL || !process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY) { + return new Map() + } + const { data, error } = await supabase() .from('troubleshooting_entries') .select('id, date_updated') diff --git a/apps/docs/public/markdown/manifest.json b/apps/docs/public/markdown/manifest.json new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/apps/docs/public/markdown/manifest.json @@ -0,0 +1 @@ +[] diff --git a/apps/docs/types/code-hike.d.ts b/apps/docs/types/code-hike.d.ts deleted file mode 100644 index 5d2ca4d54d0..00000000000 --- a/apps/docs/types/code-hike.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { remarkCodeHike } from '@code-hike/mdx' - -declare module '@code-hike/mdx' { - export type CodeHikeRemarkPlugin = typeof remarkCodeHike - export type CodeHikeConfig = Parameters[0] -} diff --git a/e2e/docs/.gitignore b/e2e/docs/.gitignore index 68c5d18f00d..a79a1524a53 100644 --- a/e2e/docs/.gitignore +++ b/e2e/docs/.gitignore @@ -1,5 +1,6 @@ node_modules/ /test-results/ /playwright-report/ +/playwright-report-local-smoke/ /blob-report/ /playwright/.cache/ diff --git a/e2e/docs/local-smoke/no-credentials.spec.ts b/e2e/docs/local-smoke/no-credentials.spec.ts new file mode 100644 index 00000000000..835b37d6138 --- /dev/null +++ b/e2e/docs/local-smoke/no-credentials.spec.ts @@ -0,0 +1,72 @@ +import { expect, test } from '@playwright/test' + +const SUPABASE_URL_ERROR = /supabaseUrl is required/i + +function collectPageErrors(page: import('@playwright/test').Page) { + const errors: string[] = [] + page.on('pageerror', (err) => errors.push(err.message)) + page.on('console', (msg) => { + if (msg.type() === 'error') errors.push(msg.text()) + }) + return errors +} + +test.describe('docs dev runs locally without credentials', () => { + test('a normal guide page renders and the feedback widget does not crash', async ({ page }) => { + const errors = collectPageErrors(page) + + const response = await page.goto('/docs/guides/database/postgres/row-level-security') + expect(response?.ok(), `expected 200, got ${response?.status()}`).toBeTruthy() + + const yesButton = page.getByRole('button', { name: 'Yes', exact: true }) + await expect(yesButton).toBeVisible() + await yesButton.click() + + expect(errors.some((message) => SUPABASE_URL_ERROR.test(message))).toBeFalsy() + }) + + test('database-advisors page renders (full content or graceful fallback, never a crash)', async ({ + page, + }) => { + const response = await page.goto('/docs/guides/database/database-advisors') + expect(response?.ok(), `expected 200, got ${response?.status()}`).toBeTruthy() + + await expect( + page.getByRole('heading', { name: 'Performance and Security Advisors' }) + ).toBeVisible() + }) + + test('a troubleshooting article page renders without Supabase credentials', async ({ page }) => { + const errors = collectPageErrors(page) + + const response = await page.goto('/docs/guides/troubleshooting/rls-simplified-BJTcS8') + expect(response?.status(), `expected 200, got ${response?.status()}`).toBe(200) + + await expect(page.getByRole('heading', { name: 'RLS Simplified' })).toBeVisible() + + expect(errors.some((message) => SUPABASE_URL_ERROR.test(message))).toBeFalsy() + }) + + test('a federated guide missing locally returns a clean 404, not a crash', async ({ page }) => { + const response = await page.goto('/docs/guides/graphql') + expect(response?.status()).toBe(404) + }) + + test('the wrappers overview page renders without GitHub credentials', async ({ page }) => { + const errors = collectPageErrors(page) + + const response = await page.goto('/docs/guides/database/extensions/wrappers/overview') + expect(response?.status(), `expected 200, got ${response?.status()}`).toBe(200) + + await expect(page.getByRole('heading', { name: 'Foreign Data Wrappers' })).toBeVisible() + + expect(errors.some((message) => SUPABASE_URL_ERROR.test(message))).toBeFalsy() + }) + + test('a federated wrappers page missing locally returns a clean 404, not a crash', async ({ + page, + }) => { + const response = await page.goto('/docs/guides/database/extensions/wrappers/stripe') + expect(response?.status()).toBe(404) + }) +}) diff --git a/e2e/docs/package.json b/e2e/docs/package.json index 3a53ee4976e..aa0812855d2 100644 --- a/e2e/docs/package.json +++ b/e2e/docs/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "e2e:docs": "playwright test", - "e2e:ui": "playwright test --ui" + "e2e:ui": "playwright test --ui", + "e2e:docs:local-smoke": "playwright test --config=playwright.local-smoke.config.ts" }, "dependencies": { "@playwright/test": "^1.59.1" diff --git a/e2e/docs/playwright.local-smoke.config.ts b/e2e/docs/playwright.local-smoke.config.ts new file mode 100644 index 00000000000..9d28020de51 --- /dev/null +++ b/e2e/docs/playwright.local-smoke.config.ts @@ -0,0 +1,56 @@ +import { defineConfig } from '@playwright/test' + +const IS_CI = !!process.env.CI + +const WEB_SERVER_PORT = Number(process.env.WEB_SERVER_PORT) || 3001 + +export default defineConfig({ + testDir: './local-smoke', + testMatch: /.*\.spec\.ts/, + timeout: 90_000, + forbidOnly: IS_CI, + retries: IS_CI ? 2 : 0, + maxFailures: 3, + expect: { + timeout: 15_000, + }, + fullyParallel: false, + workers: 1, + use: { + baseURL: `http://localhost:${WEB_SERVER_PORT}`, + browserName: 'chromium', + headless: true, + // Higher than usual: each run boots a fresh dev server, so the first + // request to a route pays Next.js's on-demand compile cost. + navigationTimeout: 60_000, + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + video: 'off', + }, + reporter: IS_CI + ? [['list'], ['html', { open: 'never', outputFolder: './playwright-report-local-smoke' }]] + : [ + ['list'], + ['html', { open: 'never', outputFolder: './playwright-report-local-smoke' }], + ['json', { outputFile: './test-results/local-smoke-results.json' }], + ], + outputDir: './test-results', + // Explicitly cleared so this suite is credential-free even if the local + // .env.local has real secrets, and never reused so a server already + // running on this port (possibly with real credentials) can't be reused. + webServer: { + command: 'pnpm --workspace-root run dev:docs', + port: WEB_SERVER_PORT, + timeout: 4 * 60 * 1000, + reuseExistingServer: false, + env: { + DOCS_GITHUB_APP_PRIVATE_KEY: '', + DOCS_GITHUB_APP_ID: '', + DOCS_GITHUB_APP_INSTALLATION_ID: '', + SUPABASE_SECRET_KEY: '', + OPENAI_API_KEY: '', + DOCS_REVALIDATION_KEYS: '', + DOCS_REVALIDATION_OVERRIDE_KEYS: '', + }, + }, +}) diff --git a/package.json b/package.json index 69710876339..1040329e8f9 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "e2e:ui": "pnpm --prefix e2e/studio run e2e:ui", "e2e:docs": "pnpm --prefix e2e/docs run e2e:docs", "e2e:docs:ui": "pnpm --prefix e2e/docs run e2e:ui", + "e2e:docs:local-smoke": "pnpm --prefix e2e/docs run e2e:docs:local-smoke", "perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/", "perf:meta": "ab -t 5 -c 20 -T application/json http://localhost:5555/tables", "setup:cli": "supabase start -x studio && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",