Fix: Make self hosted tests more reliable locally and in CI (#40479)

* added ui studio command * added SKIP ASSET UPLOAD option * updated failing tests * updated e2e scripts to include CI instructions * removed unused envs and slack messages * updated timeouts and max retries on CI * updated generated local env * updated env * updated env generation for e2e tests * remove vercel preview * updated tests and trying junit report on PR * update test comment * updated commit hash for reporter * updated file path * updated runner to blacksmith * update test file for reporting * Update .github/workflows/studio-e2e-test.yml Co-authored-by: Charis <26616127+charislam@users.noreply.github.com> * Update package.json Co-authored-by: Charis <26616127+charislam@users.noreply.github.com> * Update e2e/studio/playwright.config.ts Co-authored-by: Charis <26616127+charislam@users.noreply.github.com> * updated to skip comment on forks --------- Co-authored-by: Charis <26616127+charislam@users.noreply.github.com>
2026-10-06 01:45:10 +03:00 · 2025-11-17 15:00:31 -07:00
1 parent 482c225f29
commit 7898abe06c
@@ -27,28 +27,12 @@ permissions:
 jobs:
  test:
    timeout-minutes: 60
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-4vcpu-ubuntu-2404
    # Require approval only for pull requests from forks
    environment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && 'Studio E2E Tests' || '' }}

-    env:
-      EMAIL: ${{ secrets.CI_EMAIL }}
-      PASSWORD: ${{ secrets.CI_PASSWORD }}
-      NEXT_PUBLIC_API_URL: https://api.supabase.green
-      VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
-      # Studio Self-Hosted project ID
-      VERCEL_PROJECT_ID: prj_CnatEuo7L6bUZAgmujMrm5P1rxtv
-      NEXT_PUBLIC_HCAPTCHA_SITE_KEY: 10000000-ffff-ffff-ffff-000000000001
-      VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }}
-
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-      - name: Verify Vercel bypass secret exists
-        run: |
-          if [ -z "${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }}" ]; then
-            echo "Required secret VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO is not set" >&2
-            exit 1
-          fi
      - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
        name: Install pnpm
        with:
@@ -62,36 +46,11 @@ jobs:
      - name: Install dependencies
        run: pnpm install --frozen-lockfile

-      # Deploy a preview to Vercel (CLI mode) and capture the URL
-      - name: Install Vercel CLI
-        run: pnpm add --global vercel@latest
-
-      - name: Pull Vercel Environment Information (Preview)
-        run: vercel pull --yes --environment=preview --token=${{ secrets.VERCEL_TOKEN }}
-
-      - name: Build Project Artifacts for Vercel (is_platform=false)
-        env:
-          NEXT_PUBLIC_IS_PLATFORM: false
-        run: vercel build --token=${{ secrets.VERCEL_TOKEN }}
-
-      - name: Deploy Project to Vercel and Get URL
-        id: deploy_vercel
-        run: |
-          DEPLOY_URL=$(vercel deploy --prebuilt --token=${{ secrets.VERCEL_TOKEN }})
-          echo "Vercel Preview URL: $DEPLOY_URL"
-          echo "DEPLOY_URL=$DEPLOY_URL" >> $GITHUB_OUTPUT
-
      - name: Install Playwright Browsers
        run: pnpm -C e2e/studio exec playwright install chromium --with-deps --only-shell

      - name: 🚀 Run Playwright tests against Vercel Preview
        id: playwright
-        continue-on-error: true
-        env:
-          AUTHENTICATION: false
-          STUDIO_URL: ${{ steps.deploy_vercel.outputs.DEPLOY_URL }}
-          API_URL: ${{ steps.deploy_vercel.outputs.DEPLOY_URL }}
-          VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }}
        run: pnpm e2e

      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -103,89 +62,12 @@ jobs:
            e2e/studio/test-results/
          retention-days: 7

-      - name: Prepare summary (outputs)
-        if: always()
-        id: summarize
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+      - name: Comment Playwright test results on PR
+        uses: daun/playwright-report-comment@be9e270edd5ad86038604d3caa84a819a6ff6fed # v3.10.0
+        if: always() && github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork
        with:
-          script: |
-            const fs = require('fs')
-            const p = 'e2e/studio/test-results/test-results.json'
-            // Initialize a summary object to hold test statistics.
-            let s={total:0,passed:0,failed:0,skipped:0,timedOut:0,interrupted:0,flaky:0,durationMs:0,note:''}
-            try {
-              const data = JSON.parse(fs.readFileSync(p,'utf8'))
-              // Recursively walk through the test suites to process each test.
-              const walk=suite=>{
-                if(!suite)return;
-                suite.specs?.forEach(spec=>{
-                  spec.tests?.forEach(test=>{
-                    s.total++;
-                    // Get the last result of the test, as tests can be retried.
-                    const lastResult = test.results[test.results.length - 1];
-                    s.durationMs += lastResult.duration || 0;
-                    // A test is considered flaky if it has more than one run and the final status is 'passed'.
-                    if (test.results.length > 1 && lastResult.status === 'passed') {
-                      s.flaky++
-                    }
-                    const status = lastResult.status === 'passed' && s.flaky > 0 ? 'flaky' : lastResult.status
-                    s[status] = (s[status]||0)+1;
-                  })
-                })
-                suite.suites?.forEach(walk)
-              }
-              walk(data.suites?.[0])
-            } catch { s.note='No JSON report found or parse error.' }
-            // Generate the markdown for the summary comment.
-            const md = s.note ? `Note: ${s.note}` : `- Total: ${s.total}\n- Passed: ${s.passed||0}\n- Failed: ${s.failed||0}\n- Skipped: ${s.skipped||0}\n- Timed out: ${s.timedOut||0}\n- Interrupted: ${s.interrupted||0}\n- Flaky: ${s.flaky||0}\n- Duration: ${(s.durationMs/1000).toFixed(1)}s`
-            // Set the summary and flaky_count as outputs for subsequent steps.
-            core.setOutput('summary', md)
-            core.setOutput('flaky_count', s.flaky)
-
-      - name: Comment summary on PR
-        if: always() && github.event_name == 'pull_request'
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
-        with:
-          script: |
-            const owner = context.repo.owner
-            const repo = context.repo.repo
-            const issue_number = context.issue.number
-            const summary = `${{ steps.summarize.outputs.summary }}`.replace(/^"|"$/g,'')
-            const runUrl = `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`
-            const marker = '<!-- studio-e2e-summary -->'
-
-            const now = new Date()
-            const weekday = now.toLocaleString('en-US', { weekday: 'long', timeZone: 'UTC' })
-            const day = now.toLocaleString('en-US', { day: 'numeric', timeZone: 'UTC' })
-            const month = now.toLocaleString('en-US', { month: 'long', timeZone: 'UTC' })
-            const year = now.toLocaleString('en-US', { year: 'numeric', timeZone: 'UTC' })
-            const time = now.toLocaleTimeString('en-US', {
-              hour: '2-digit',
-              minute: '2-digit',
-              second: '2-digit',
-              hour12: false,
-              timeZone: 'UTC',
-            })
-            const date = `${weekday} ${day}, ${month}, ${year} ${time} (UTC)`
-
-            const body = [
-              marker,
-              `**Studio E2E Results**`,
-              '',
-              summary,
-              '',
-              `Artifacts: ${runUrl}`,
-              '',
-              `Last updated: ${date}`
-            ].join('\n')
-
-            const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number, per_page: 100 })
-            const existing = comments.find(c => c.body && c.body.includes(marker))
-            if (existing) {
-              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })
-            } else {
-              await github.rest.issues.createComment({ owner, repo, issue_number, body })
-            }
+          report-file: e2e/studio/test-results/test-results.json
+          comment-title: '🎭 Playwright Test Results'

      - name: Fail job if tests failed
        if: steps.playwright.outcome != 'success' || steps.summarize.outputs.flaky_count > 0
@@ -5,7 +5,7 @@
  "scripts": {
    "preinstall": "npx only-allow pnpm",
    "dev": "next dev --turbopack -p 8082",
-    "build": "next build && ./../../scripts/upload-static-assets.sh",
+    "build": "next build && if [ \"$SKIP_ASSET_UPLOAD\" != \"1\" ]; then ./../../scripts/upload-static-assets.sh; fi",
    "start": "next start",
    "lint": "eslint .",
    "lint:ratchet": "tsx scripts/ratchet-eslint-rules.ts --rule react-hooks/exhaustive-deps --rule import/no-anonymous-default-export --rule @tanstack/query/exhaustive-deps --rule @tanstack/query/no-deprecated-options",
@@ -25,18 +25,6 @@ setup('Global Setup', async ({ page }) => {
    - Is Platform: ${IS_PLATFORM}
    `)

-  /*
-   * Check if we're in CI, if so, check VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO
-   * is set to true.
-   */
-  const VERCEL_BYPASS = process.env.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO
-
-  if (process.env.CI === 'true') {
-    if (!VERCEL_BYPASS || VERCEL_BYPASS.length === 0) {
-      throw new Error('VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO is not set')
-    }
-  }
-
  /**
   * Studio Check
   */
@@ -185,7 +185,11 @@ test.describe.serial('Database', () => {

      // drop database tables if exists
      if ((await page.getByText(databaseTableNameNew, { exact: true }).count()) > 0) {
-        await page.getByRole('row', { name: databaseTableNameNew }).getByRole('button').click()
+        await page
+          .getByRole('row', { name: databaseTableNameNew })
+          .getByRole('button')
+          .last()
+          .click()
        await page.getByRole('menuitem', { name: 'Delete table' }).click()
        await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check()
        await page.getByRole('button', { name: 'Delete' }).click()
@@ -193,7 +197,11 @@ test.describe.serial('Database', () => {
      }

      if ((await page.getByText(databaseTableNameUpdated, { exact: true }).count()) > 0) {
-        await page.getByRole('row', { name: databaseTableNameUpdated }).getByRole('button').click()
+        await page
+          .getByRole('row', { name: databaseTableNameUpdated })
+          .getByRole('button')
+          .last()
+          .click()
        await page.getByRole('menuitem', { name: 'Delete table' }).click()
        await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check()
        await page.getByRole('button', { name: 'Delete' }).click()
@@ -204,6 +212,7 @@ test.describe.serial('Database', () => {
        await page
          .getByRole('row', { name: databaseTableNameDuplicate })
          .getByRole('button')
+          .last()
          .click()
        await page.getByRole('menuitem', { name: 'Delete table' }).click()
        await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check()
@@ -222,7 +231,7 @@ test.describe.serial('Database', () => {
      await expect(page.getByText(databaseTableNameNew, { exact: true })).toBeVisible()

      // edit a new table
-      await page.getByRole('row', { name: databaseTableNameNew }).getByRole('button').click()
+      await page.getByRole('row', { name: databaseTableNameNew }).getByRole('button').last().click()
      await page.getByRole('menuitem', { name: 'Edit table' }).click()
      await page.getByTestId('table-name-input').fill(databaseTableNameUpdated)
      await page.getByRole('button', { name: 'Save' }).click()
@@ -233,7 +242,11 @@ test.describe.serial('Database', () => {
      await expect(page.getByText(databaseTableNameUpdated, { exact: true })).toBeVisible()

      // duplicate table
-      await page.getByRole('row', { name: databaseTableNameUpdated }).getByRole('button').click()
+      await page
+        .getByRole('row', { name: databaseTableNameUpdated })
+        .getByRole('button')
+        .last()
+        .click()
      await page.getByRole('menuitem', { name: 'Duplicate Table' }).click()
      await page.getByTestId('table-name-input').fill(databaseTableNameDuplicate)
      await page.getByRole('textbox', { name: 'Optional' }).fill('')
@@ -248,6 +261,7 @@ test.describe.serial('Database', () => {
      await page
        .getByRole('row', { name: `${databaseTableNameDuplicate}` })
        .getByRole('button')
+        .last()
        .click()
      await page.getByRole('menuitem', { name: 'Delete table' }).click()
      await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check()
@@ -257,6 +271,7 @@ test.describe.serial('Database', () => {
      await page
        .getByRole('row', { name: `${databaseTableNameUpdated}` })
        .getByRole('button')
+        .last()
        .click()
      await page.getByRole('menuitem', { name: 'Delete table' }).click()
      await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).check()
@@ -264,7 +279,7 @@ test.describe.serial('Database', () => {
      await waitForApiResponse(page, 'pg-meta', ref, 'query?key=table-delete')

      // validate navigating to table editor from database table page
-      await page.getByRole('row', { name: databaseTableName }).getByRole('button').click()
+      await page.getByRole('row', { name: databaseTableName }).getByRole('button').last().click()
      await page.getByRole('menuitem', { name: 'View in Table Editor' }).click()
      await page.waitForTimeout(1000) // wait for the table editor to be loaded
      expect(page.url().includes('editor')).toBe(true)
@@ -468,7 +483,11 @@ test.describe.serial('Database', () => {
      expect(page.getByText('confirmation_token_idx')).toBeVisible()

      // check index definition
-      await page.getByRole('row', { name: 'confirmation_token_idx' }).getByRole('button').click()
+      await page
+        .getByRole('row', { name: 'confirmation_token_idx' })
+        .getByRole('button')
+        .last()
+        .click()
      await page.getByText('Index:confirmation_token_idx')
      await page.waitForTimeout(500) // wait for text content to be visible
      expect(await page.getByRole('presentation').textContent()).toBe(
@@ -196,14 +196,21 @@ test.describe.serial('table editor', () => {
    // change schema from public to auth
    await page.getByTestId('schema-selector').click()
    await page.getByPlaceholder('Find schema...').fill('auth')
+
+    // Set up the waiter BEFORE clicking to avoid race condition
+    const tableLoadPromise = waitForTableToLoad(page, ref, 'auth')
    await page.getByRole('option', { name: 'auth' }).click()
-    await waitForTableToLoad(page, ref, 'auth') // load auth tables
+    await tableLoadPromise // wait for auth tables to load
+
    await expect(page.getByLabel(`View ${authTableSso}`)).toBeVisible()
    await expect(page.getByLabel(`View ${authTableMfa}`)).toBeVisible()

-    // can find auth tables
+    // Search is client-side filtering - no API call needed
    await page.getByRole('textbox', { name: 'Search tables...' }).fill('mfa')
-    await waitForTableToLoad(page, ref, 'auth') // load tables
+
+    // Wait for the UI to update after search (allow debounce to complete)
+    await page.waitForTimeout(300)
+
    await expect(page.getByLabel(`View ${authTableSso}`)).not.toBeVisible()
    await expect(page.getByLabel(`View ${authTableMfa}`)).toBeVisible()
  })
@@ -9,20 +9,15 @@ dotenv.config({

 const IS_CI = !!process.env.CI

-const webServerConfig = IS_CI
-  ? undefined
-  : {
-      command: 'pnpm -w run e2e:setup',
-      port: 8082,
-      timeout: 5 * 60 * 1000,
-    }
+const WEB_SERVER_TIMEOUT = Number(process.env.WEB_SERVER_TIMEOUT) || 10 * 60 * 1000
+const WEB_SERVER_PORT = Number(process.env.WEB_SERVER_PORT) || 8082

 export default defineConfig({
-  timeout: 90 * 1000,
+  timeout: 120 * 1000,
  testDir: './features',
  testMatch: /.*\.spec\.ts/,
  forbidOnly: IS_CI,
-  retries: IS_CI ? 3 : 0,
+  retries: IS_CI ? 5 : 0,
  maxFailures: 3,
  fullyParallel: true,
  use: {
@@ -61,5 +56,9 @@ export default defineConfig({
    ['html', { open: 'never' }],
    ['json', { outputFile: 'test-results/test-results.json' }],
  ],
-  webServer: webServerConfig,
+  webServer: {
+    command: 'pnpm --workspace-root run e2e:setup',
+    port: WEB_SERVER_PORT,
+    timeout: WEB_SERVER_TIMEOUT,
+  },
 })
@@ -29,8 +29,10 @@
    "test:ui-patterns": "turbo run test --filter=ui-patterns",
    "test:studio": "turbo run test --filter=studio",
    "test:studio:watch": "turbo run test --filter=studio -- watch",
-    "e2e:setup": "pnpm setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082",
+    "e2e:setup:cli": "supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",
+    "e2e:setup": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082",
    "e2e": "pnpm --prefix e2e/studio run e2e",
+    "e2e:ui": "pnpm --prefix e2e/studio run e2e:ui",
    "perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/",
    "perf:meta": "ab -t 5 -c 20 -T application/json http://localhost:5555/tables",
    "setup:cli": "supabase start -x studio && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",
@@ -40,3 +40,21 @@ fs.writeFileSync(
    .map((key) => `${key}=${environment[key]}`)
    .join('\n')
 )
+
+const STUDIO_URL = environment.NEXT_PUBLIC_SITE_URL
+const WEB_SERVER_PORT = new URL(STUDIO_URL).port ?? undefined
+const API_URL = environment.API_URL
+
+const e2eTestEnv = {
+  STUDIO_URL,
+  API_URL,
+  WEB_SERVER_PORT,
+  IS_PLATFORM: 'false',
+}
+
+fs.writeFileSync(
+  './e2e/studio/.env.local',
+  Object.keys(e2eTestEnv)
+    .map((key) => `${key}=${e2eTestEnv[key]}`)
+    .join('\n')
+)