diff --git a/.github/workflows/studio-e2e-test.yml b/.github/workflows/studio-e2e-test.yml index 1667a9c5a0f..68fa270e72c 100644 --- a/.github/workflows/studio-e2e-test.yml +++ b/.github/workflows/studio-e2e-test.yml @@ -27,28 +27,12 @@ permissions: jobs: test: timeout-minutes: 60 - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 # Require approval only for pull requests from forks environment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && 'Studio E2E Tests' || '' }} - env: - EMAIL: ${{ secrets.CI_EMAIL }} - PASSWORD: ${{ secrets.CI_PASSWORD }} - NEXT_PUBLIC_API_URL: https://api.supabase.green - VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} - # Studio Self-Hosted project ID - VERCEL_PROJECT_ID: prj_CnatEuo7L6bUZAgmujMrm5P1rxtv - NEXT_PUBLIC_HCAPTCHA_SITE_KEY: 10000000-ffff-ffff-ffff-000000000001 - VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }} - steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - name: Verify Vercel bypass secret exists - run: | - if [ -z "${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }}" ]; then - echo "Required secret VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO is not set" >&2 - exit 1 - fi - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 name: Install pnpm with: @@ -62,36 +46,11 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # Deploy a preview to Vercel (CLI mode) and capture the URL - - name: Install Vercel CLI - run: pnpm add --global vercel@latest - - - name: Pull Vercel Environment Information (Preview) - run: vercel pull --yes --environment=preview --token=${{ secrets.VERCEL_TOKEN }} - - - name: Build Project Artifacts for Vercel (is_platform=false) - env: - NEXT_PUBLIC_IS_PLATFORM: false - run: vercel build --token=${{ secrets.VERCEL_TOKEN }} - - - name: Deploy Project to Vercel and Get URL - id: deploy_vercel - run: | - DEPLOY_URL=$(vercel deploy --prebuilt --token=${{ secrets.VERCEL_TOKEN }}) - echo "Vercel Preview URL: $DEPLOY_URL" - echo "DEPLOY_URL=$DEPLOY_URL" >> $GITHUB_OUTPUT - - name: Install Playwright Browsers run: pnpm -C e2e/studio exec playwright install chromium --with-deps --only-shell - name: 🚀 Run Playwright tests against Vercel Preview id: playwright - continue-on-error: true - env: - AUTHENTICATION: false - STUDIO_URL: ${{ steps.deploy_vercel.outputs.DEPLOY_URL }} - API_URL: ${{ steps.deploy_vercel.outputs.DEPLOY_URL }} - VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }} run: pnpm e2e - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -103,89 +62,12 @@ jobs: e2e/studio/test-results/ retention-days: 7 - - name: Prepare summary (outputs) - if: always() - id: summarize - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + - name: Comment Playwright test results on PR + uses: daun/playwright-report-comment@be9e270edd5ad86038604d3caa84a819a6ff6fed # v3.10.0 + if: always() && github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork with: - script: | - const fs = require('fs') - const p = 'e2e/studio/test-results/test-results.json' - // Initialize a summary object to hold test statistics. - let s={total:0,passed:0,failed:0,skipped:0,timedOut:0,interrupted:0,flaky:0,durationMs:0,note:''} - try { - const data = JSON.parse(fs.readFileSync(p,'utf8')) - // Recursively walk through the test suites to process each test. - const walk=suite=>{ - if(!suite)return; - suite.specs?.forEach(spec=>{ - spec.tests?.forEach(test=>{ - s.total++; - // Get the last result of the test, as tests can be retried. - const lastResult = test.results[test.results.length - 1]; - s.durationMs += lastResult.duration || 0; - // A test is considered flaky if it has more than one run and the final status is 'passed'. - if (test.results.length > 1 && lastResult.status === 'passed') { - s.flaky++ - } - const status = lastResult.status === 'passed' && s.flaky > 0 ? 'flaky' : lastResult.status - s[status] = (s[status]||0)+1; - }) - }) - suite.suites?.forEach(walk) - } - walk(data.suites?.[0]) - } catch { s.note='No JSON report found or parse error.' } - // Generate the markdown for the summary comment. - const md = s.note ? `Note: ${s.note}` : `- Total: ${s.total}\n- Passed: ${s.passed||0}\n- Failed: ${s.failed||0}\n- Skipped: ${s.skipped||0}\n- Timed out: ${s.timedOut||0}\n- Interrupted: ${s.interrupted||0}\n- Flaky: ${s.flaky||0}\n- Duration: ${(s.durationMs/1000).toFixed(1)}s` - // Set the summary and flaky_count as outputs for subsequent steps. - core.setOutput('summary', md) - core.setOutput('flaky_count', s.flaky) - - - name: Comment summary on PR - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 - with: - script: | - const owner = context.repo.owner - const repo = context.repo.repo - const issue_number = context.issue.number - const summary = `${{ steps.summarize.outputs.summary }}`.replace(/^"|"$/g,'') - const runUrl = `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}` - const marker = '' - - const now = new Date() - const weekday = now.toLocaleString('en-US', { weekday: 'long', timeZone: 'UTC' }) - const day = now.toLocaleString('en-US', { day: 'numeric', timeZone: 'UTC' }) - const month = now.toLocaleString('en-US', { month: 'long', timeZone: 'UTC' }) - const year = now.toLocaleString('en-US', { year: 'numeric', timeZone: 'UTC' }) - const time = now.toLocaleTimeString('en-US', { - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - hour12: false, - timeZone: 'UTC', - }) - const date = `${weekday} ${day}, ${month}, ${year} ${time} (UTC)` - - const body = [ - marker, - `**Studio E2E Results**`, - '', - summary, - '', - `Artifacts: ${runUrl}`, - '', - `Last updated: ${date}` - ].join('\n') - - const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number, per_page: 100 }) - const existing = comments.find(c => c.body && c.body.includes(marker)) - if (existing) { - await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }) - } else { - await github.rest.issues.createComment({ owner, repo, issue_number, body }) - } + report-file: e2e/studio/test-results/test-results.json + comment-title: '🎭 Playwright Test Results' - name: Fail job if tests failed if: steps.playwright.outcome != 'success' || steps.summarize.outputs.flaky_count > 0 diff --git a/apps/studio/package.json b/apps/studio/package.json index eb32f9e207e..2e7f4c4096c 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -5,7 +5,7 @@ "scripts": { "preinstall": "npx only-allow pnpm", "dev": "next dev --turbopack -p 8082", - "build": "next build && ./../../scripts/upload-static-assets.sh", + "build": "next build && if [ \"$SKIP_ASSET_UPLOAD\" != \"1\" ]; then ./../../scripts/upload-static-assets.sh; fi", "start": "next start", "lint": "eslint .", "lint:ratchet": "tsx scripts/ratchet-eslint-rules.ts --rule react-hooks/exhaustive-deps --rule import/no-anonymous-default-export --rule @tanstack/query/exhaustive-deps --rule @tanstack/query/no-deprecated-options", diff --git a/e2e/studio/features/_global.setup.ts b/e2e/studio/features/_global.setup.ts index acf4bfa6c0b..187e72fb8fb 100644 --- a/e2e/studio/features/_global.setup.ts +++ b/e2e/studio/features/_global.setup.ts @@ -25,18 +25,6 @@ setup('Global Setup', async ({ page }) => { - Is Platform: ${IS_PLATFORM} `) - /* - * Check if we're in CI, if so, check VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO - * is set to true. - */ - const VERCEL_BYPASS = process.env.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO - - if (process.env.CI === 'true') { - if (!VERCEL_BYPASS || VERCEL_BYPASS.length === 0) { - throw new Error('VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO is not set') - } - } - /** * Studio Check */ diff --git a/e2e/studio/features/database.spec.ts b/e2e/studio/features/database.spec.ts index b3d328fa8a6..d90b1100dae 100644 --- a/e2e/studio/features/database.spec.ts +++ b/e2e/studio/features/database.spec.ts @@ -185,7 +185,11 @@ test.describe.serial('Database', () => { // drop database tables if exists if ((await page.getByText(databaseTableNameNew, { exact: true }).count()) > 0) { - await page.getByRole('row', { name: databaseTableNameNew }).getByRole('button').click() + await page + .getByRole('row', { name: databaseTableNameNew }) + .getByRole('button') + .last() + .click() await page.getByRole('menuitem', { name: 'Delete table' }).click() await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check() await page.getByRole('button', { name: 'Delete' }).click() @@ -193,7 +197,11 @@ test.describe.serial('Database', () => { } if ((await page.getByText(databaseTableNameUpdated, { exact: true }).count()) > 0) { - await page.getByRole('row', { name: databaseTableNameUpdated }).getByRole('button').click() + await page + .getByRole('row', { name: databaseTableNameUpdated }) + .getByRole('button') + .last() + .click() await page.getByRole('menuitem', { name: 'Delete table' }).click() await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check() await page.getByRole('button', { name: 'Delete' }).click() @@ -204,6 +212,7 @@ test.describe.serial('Database', () => { await page .getByRole('row', { name: databaseTableNameDuplicate }) .getByRole('button') + .last() .click() await page.getByRole('menuitem', { name: 'Delete table' }).click() await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check() @@ -222,7 +231,7 @@ test.describe.serial('Database', () => { await expect(page.getByText(databaseTableNameNew, { exact: true })).toBeVisible() // edit a new table - await page.getByRole('row', { name: databaseTableNameNew }).getByRole('button').click() + await page.getByRole('row', { name: databaseTableNameNew }).getByRole('button').last().click() await page.getByRole('menuitem', { name: 'Edit table' }).click() await page.getByTestId('table-name-input').fill(databaseTableNameUpdated) await page.getByRole('button', { name: 'Save' }).click() @@ -233,7 +242,11 @@ test.describe.serial('Database', () => { await expect(page.getByText(databaseTableNameUpdated, { exact: true })).toBeVisible() // duplicate table - await page.getByRole('row', { name: databaseTableNameUpdated }).getByRole('button').click() + await page + .getByRole('row', { name: databaseTableNameUpdated }) + .getByRole('button') + .last() + .click() await page.getByRole('menuitem', { name: 'Duplicate Table' }).click() await page.getByTestId('table-name-input').fill(databaseTableNameDuplicate) await page.getByRole('textbox', { name: 'Optional' }).fill('') @@ -248,6 +261,7 @@ test.describe.serial('Database', () => { await page .getByRole('row', { name: `${databaseTableNameDuplicate}` }) .getByRole('button') + .last() .click() await page.getByRole('menuitem', { name: 'Delete table' }).click() await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check() @@ -257,6 +271,7 @@ test.describe.serial('Database', () => { await page .getByRole('row', { name: `${databaseTableNameUpdated}` }) .getByRole('button') + .last() .click() await page.getByRole('menuitem', { name: 'Delete table' }).click() await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check() @@ -264,7 +279,7 @@ test.describe.serial('Database', () => { await waitForApiResponse(page, 'pg-meta', ref, 'query?key=table-delete') // validate navigating to table editor from database table page - await page.getByRole('row', { name: databaseTableName }).getByRole('button').click() + await page.getByRole('row', { name: databaseTableName }).getByRole('button').last().click() await page.getByRole('menuitem', { name: 'View in Table Editor' }).click() await page.waitForTimeout(1000) // wait for the table editor to be loaded expect(page.url().includes('editor')).toBe(true) @@ -468,7 +483,11 @@ test.describe.serial('Database', () => { expect(page.getByText('confirmation_token_idx')).toBeVisible() // check index definition - await page.getByRole('row', { name: 'confirmation_token_idx' }).getByRole('button').click() + await page + .getByRole('row', { name: 'confirmation_token_idx' }) + .getByRole('button') + .last() + .click() await page.getByText('Index:confirmation_token_idx') await page.waitForTimeout(500) // wait for text content to be visible expect(await page.getByRole('presentation').textContent()).toBe( diff --git a/e2e/studio/features/table-editor.spec.ts b/e2e/studio/features/table-editor.spec.ts index aa5d6a1aec1..dfb5116bfb3 100644 --- a/e2e/studio/features/table-editor.spec.ts +++ b/e2e/studio/features/table-editor.spec.ts @@ -196,14 +196,21 @@ test.describe.serial('table editor', () => { // change schema from public to auth await page.getByTestId('schema-selector').click() await page.getByPlaceholder('Find schema...').fill('auth') + + // Set up the waiter BEFORE clicking to avoid race condition + const tableLoadPromise = waitForTableToLoad(page, ref, 'auth') await page.getByRole('option', { name: 'auth' }).click() - await waitForTableToLoad(page, ref, 'auth') // load auth tables + await tableLoadPromise // wait for auth tables to load + await expect(page.getByLabel(`View ${authTableSso}`)).toBeVisible() await expect(page.getByLabel(`View ${authTableMfa}`)).toBeVisible() - // can find auth tables + // Search is client-side filtering - no API call needed await page.getByRole('textbox', { name: 'Search tables...' }).fill('mfa') - await waitForTableToLoad(page, ref, 'auth') // load tables + + // Wait for the UI to update after search (allow debounce to complete) + await page.waitForTimeout(300) + await expect(page.getByLabel(`View ${authTableSso}`)).not.toBeVisible() await expect(page.getByLabel(`View ${authTableMfa}`)).toBeVisible() }) diff --git a/e2e/studio/playwright.config.ts b/e2e/studio/playwright.config.ts index dfb75c6ef29..d64ba62916a 100644 --- a/e2e/studio/playwright.config.ts +++ b/e2e/studio/playwright.config.ts @@ -9,20 +9,15 @@ dotenv.config({ const IS_CI = !!process.env.CI -const webServerConfig = IS_CI - ? undefined - : { - command: 'pnpm -w run e2e:setup', - port: 8082, - timeout: 5 * 60 * 1000, - } +const WEB_SERVER_TIMEOUT = Number(process.env.WEB_SERVER_TIMEOUT) || 10 * 60 * 1000 +const WEB_SERVER_PORT = Number(process.env.WEB_SERVER_PORT) || 8082 export default defineConfig({ - timeout: 90 * 1000, + timeout: 120 * 1000, testDir: './features', testMatch: /.*\.spec\.ts/, forbidOnly: IS_CI, - retries: IS_CI ? 3 : 0, + retries: IS_CI ? 5 : 0, maxFailures: 3, fullyParallel: true, use: { @@ -61,5 +56,9 @@ export default defineConfig({ ['html', { open: 'never' }], ['json', { outputFile: 'test-results/test-results.json' }], ], - webServer: webServerConfig, + webServer: { + command: 'pnpm --workspace-root run e2e:setup', + port: WEB_SERVER_PORT, + timeout: WEB_SERVER_TIMEOUT, + }, }) diff --git a/package.json b/package.json index 1ad7ff51243..ce03f99c6ae 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,10 @@ "test:ui-patterns": "turbo run test --filter=ui-patterns", "test:studio": "turbo run test --filter=studio", "test:studio:watch": "turbo run test --filter=studio -- watch", - "e2e:setup": "pnpm setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082", + "e2e:setup:cli": "supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js", + "e2e:setup": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082", "e2e": "pnpm --prefix e2e/studio run e2e", + "e2e:ui": "pnpm --prefix e2e/studio run e2e:ui", "perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/", "perf:meta": "ab -t 5 -c 20 -T application/json http://localhost:5555/tables", "setup:cli": "supabase start -x studio && supabase status --output json > keys.json && node scripts/generateLocalEnv.js", diff --git a/scripts/generateLocalEnv.js b/scripts/generateLocalEnv.js index 1b6b541b081..5aa2cb88408 100644 --- a/scripts/generateLocalEnv.js +++ b/scripts/generateLocalEnv.js @@ -40,3 +40,21 @@ fs.writeFileSync( .map((key) => `${key}=${environment[key]}`) .join('\n') ) + +const STUDIO_URL = environment.NEXT_PUBLIC_SITE_URL +const WEB_SERVER_PORT = new URL(STUDIO_URL).port ?? undefined +const API_URL = environment.API_URL + +const e2eTestEnv = { + STUDIO_URL, + API_URL, + WEB_SERVER_PORT, + IS_PLATFORM: 'false', +} + +fs.writeFileSync( + './e2e/studio/.env.local', + Object.keys(e2eTestEnv) + .map((key) => `${key}=${e2eTestEnv[key]}`) + .join('\n') +)