a new hope (#38893)

* a new hope

* run tests in ci against cli mode

* summary

* try vercel action to run e2e against studio self hosted preview

* believe

* debug

* gh pages artifact

* test

* rm pages step

* fix automation bypass missing

* continue on error

* only install necessary deps for CI

* fix bypass

* remove

* fail job if test fails

* disable customer query if is_platform false

* vercel check

* fix var name, make comment update instead

* check bypass on runtime

* add env var

* fix tests going to project ref instead of default

* fix

* better dates in comment

* Update E2E test workflow to include flaky test detection and improve summary output

* fix

* fix dumb mistake

.github/workflows/studio-e2e-test.yml

@@ -1,4 +1,4 @@
-name: Studio E2E Tests
+name: Selfhosted Studio E2E Tests
 on:
   push:
     branches: [master]
@@ -22,28 +22,33 @@ concurrency:
 
 permissions:
   contents: write
+  pull-requests: write
 
 jobs:
   test:
     timeout-minutes: 60
     runs-on: ubuntu-latest
-    # Make the job non-blocking
-    continue-on-error: true
     # Require approval only for external contributors
     environment: ${{ github.event.pull_request.author_association != 'MEMBER' && 'Studio E2E Tests' || '' }}
 
     env:
       EMAIL: ${{ secrets.CI_EMAIL }}
       PASSWORD: ${{ secrets.CI_PASSWORD }}
-      PROJECT_REF: ${{ secrets.CI_PROJECT_REF }}
-      NEXT_PUBLIC_IS_PLATFORM: true
       NEXT_PUBLIC_API_URL: https://api.supabase.green
       VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
-      VERCEL_PROJECT_ID: ${{ secrets.VERCEL_STUDIO_HOSTED_PROJECT_ID }}
+      # Studio Self-Hosted project ID
+      VERCEL_PROJECT_ID: prj_CnatEuo7L6bUZAgmujMrm5P1rxtv
       NEXT_PUBLIC_HCAPTCHA_SITE_KEY: 10000000-ffff-ffff-ffff-000000000001
+      VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }}
 
     steps:
       - uses: actions/checkout@v4
+      - name: Verify Vercel bypass secret exists
+        run: |
+          if [ -z "${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }}" ]; then
+            echo "Required secret VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO is not set" >&2
+            exit 1
+          fi
       - uses: pnpm/action-setup@v4
         name: Install pnpm
         with:
@@ -57,13 +62,16 @@ jobs:
       - name: Install dependencies
         run: pnpm i
 
+      # Deploy a preview to Vercel (CLI mode) and capture the URL
       - name: Install Vercel CLI
         run: pnpm add --global vercel@latest
 
       - name: Pull Vercel Environment Information (Preview)
         run: vercel pull --yes --environment=preview --token=${{ secrets.VERCEL_TOKEN }}
 
-      - name: Build Project Artifacts for Vercel
+      - name: Build Project Artifacts for Vercel (is_platform=false)
+        env:
+          NEXT_PUBLIC_IS_PLATFORM: false
         run: vercel build --token=${{ secrets.VERCEL_TOKEN }}
 
       - name: Deploy Project to Vercel and Get URL
@@ -74,13 +82,16 @@ jobs:
           echo "DEPLOY_URL=$DEPLOY_URL" >> $GITHUB_OUTPUT
 
       - name: Install Playwright Browsers
-        run: pnpm -C e2e/studio exec playwright install --with-deps
+        run: pnpm -C e2e/studio exec playwright install chromium --with-deps --only-shell
 
-      - name: Run Playwright tests
+      - name: 🚀 Run Playwright tests against Vercel Preview
         id: playwright
+        continue-on-error: true
         env:
-          AUTHENTICATION: true
-          STUDIO_URL: ${{ steps.deploy_vercel.outputs.DEPLOY_URL }}/dashboard
+          AUTHENTICATION: false
+          STUDIO_URL: ${{ steps.deploy_vercel.outputs.DEPLOY_URL }}
+          API_URL: ${{ steps.deploy_vercel.outputs.DEPLOY_URL }}
+          VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO }}
         run: pnpm e2e
 
       - uses: actions/upload-artifact@v4
@@ -91,3 +102,93 @@ jobs:
             e2e/studio/playwright-report/
             e2e/studio/test-results/
           retention-days: 7
+
+      - name: Prepare summary (outputs)
+        if: always()
+        id: summarize
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs')
+            const p = 'e2e/studio/test-results/test-results.json'
+            // Initialize a summary object to hold test statistics.
+            let s={total:0,passed:0,failed:0,skipped:0,timedOut:0,interrupted:0,flaky:0,durationMs:0,note:''}
+            try {
+              const data = JSON.parse(fs.readFileSync(p,'utf8'))
+              // Recursively walk through the test suites to process each test.
+              const walk=suite=>{
+                if(!suite)return;
+                suite.specs?.forEach(spec=>{
+                  spec.tests?.forEach(test=>{
+                    s.total++;
+                    // Get the last result of the test, as tests can be retried.
+                    const lastResult = test.results[test.results.length - 1];
+                    s.durationMs += lastResult.duration || 0;
+                    // A test is considered flaky if it has more than one run and the final status is 'passed'.
+                    if (test.results.length > 1 && lastResult.status === 'passed') {
+                      s.flaky++
+                    }
+                    const status = lastResult.status === 'passed' && s.flaky > 0 ? 'flaky' : lastResult.status
+                    s[status] = (s[status]||0)+1;
+                  })
+                })
+                suite.suites?.forEach(walk)
+              }
+              walk(data.suites?.[0])
+            } catch { s.note='No JSON report found or parse error.' }
+            // Generate the markdown for the summary comment.
+            const md = s.note ? `Note: ${s.note}` : `- Total: ${s.total}\n- Passed: ${s.passed||0}\n- Failed: ${s.failed||0}\n- Skipped: ${s.skipped||0}\n- Timed out: ${s.timedOut||0}\n- Interrupted: ${s.interrupted||0}\n- Flaky: ${s.flaky||0}\n- Duration: ${(s.durationMs/1000).toFixed(1)}s`
+            // Set the summary and flaky_count as outputs for subsequent steps.
+            core.setOutput('summary', md)
+            core.setOutput('flaky_count', s.flaky)
+
+      - name: Comment summary on PR
+        if: always() && github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const owner = context.repo.owner
+            const repo = context.repo.repo
+            const issue_number = context.issue.number
+            const summary = `${{ steps.summarize.outputs.summary }}`.replace(/^"|"$/g,'')
+            const runUrl = `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`
+            const marker = '<!-- studio-e2e-summary -->'
+
+            const now = new Date()
+            const weekday = now.toLocaleString('en-US', { weekday: 'long', timeZone: 'UTC' })
+            const day = now.toLocaleString('en-US', { day: 'numeric', timeZone: 'UTC' })
+            const month = now.toLocaleString('en-US', { month: 'long', timeZone: 'UTC' })
+            const year = now.toLocaleString('en-US', { year: 'numeric', timeZone: 'UTC' })
+            const time = now.toLocaleTimeString('en-US', {
+              hour: '2-digit',
+              minute: '2-digit',
+              second: '2-digit',
+              hour12: false,
+              timeZone: 'UTC',
+            })
+            const date = `${weekday} ${day}, ${month}, ${year} ${time} (UTC)`
+
+            const body = [
+              marker,
+              `**Studio E2E Results**`,
+              '',
+              summary,
+              '',
+              `Artifacts: ${runUrl}`,
+              '',
+              `Last updated: ${date}`
+            ].join('\n')
+
+            const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number, per_page: 100 })
+            const existing = comments.find(c => c.body && c.body.includes(marker))
+            if (existing) {
+              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })
+            } else {
+              await github.rest.issues.createComment({ owner, repo, issue_number, body })
+            }
+
+      - name: Fail job if tests failed
+        if: steps.playwright.outcome != 'success' || steps.summarize.outputs.flaky_count > 0
+        run: |
+          echo "E2E tests failed" >&2
+          exit 1