From ca37f16f585fd7c7eb09bd845736e08b9218398d Mon Sep 17 00:00:00 2001 From: Joshen Lim Date: Thu, 20 Feb 2025 12:02:41 +0800 Subject: [PATCH] Chore/refactor e2e tests (#33647) * Set up E2E tests for staging * update * Update * Update .env.staging.example * Fix node modules * Fix * Attempt to swap local-studio-tests to use the new studio-tests directory * Omit storage state if running tests for local self host * Fix test:local command * Deprecate local-studio-test folder * Rename local studio tests GH action to E2E test --- ...-studio-tests.yml => studio-e2e-tests.yml} | 12 +- ...studio-tests.yml => studio-unit-tests.yml} | 0 package.json | 2 +- pnpm-lock.yaml | 64 ++++++----- tests/local-studio-tests/package.json | 23 ---- tests/local-studio-tests/playwright.config.ts | 92 --------------- .../tests/snapshot/spec/logs.spec.ts | 81 ------------- tests/studio-tests/.env.staging.example | 3 + .../.gitignore | 4 +- .../README.md | 4 +- tests/studio-tests/base.ts | 25 ++++ .../generate-local-env.js} | 2 +- tests/studio-tests/package.json | 23 ++++ tests/studio-tests/playwright.config.ts | 75 ++++++++++++ .../supabase/.gitignore | 0 .../supabase/config.toml | 0 .../supabase/seed.sql | 0 tests/studio-tests/tests/auth.setup.ts | 21 ++++ .../tests/common-functionality/home.spec.ts | 13 +++ .../tests/common-functionality/logs.spec.ts | 107 ++++++++++++++++++ .../table-editor.spec.ts | 53 +++++---- .../production-functionality/example.spec.ts | 11 ++ 22 files changed, 359 insertions(+), 256 deletions(-) rename .github/workflows/{local-studio-tests.yml => studio-e2e-tests.yml} (81%) rename .github/workflows/{studio-tests.yml => studio-unit-tests.yml} (100%) delete mode 100644 tests/local-studio-tests/package.json delete mode 100644 tests/local-studio-tests/playwright.config.ts delete mode 100644 tests/local-studio-tests/tests/snapshot/spec/logs.spec.ts create mode 100644 tests/studio-tests/.env.staging.example rename tests/{local-studio-tests => studio-tests}/.gitignore (66%) rename tests/{local-studio-tests => studio-tests}/README.md (91%) create mode 100644 tests/studio-tests/base.ts rename tests/{local-studio-tests/scripts/generate-env.js => studio-tests/generate-local-env.js} (96%) create mode 100644 tests/studio-tests/package.json create mode 100644 tests/studio-tests/playwright.config.ts rename tests/{local-studio-tests => studio-tests}/supabase/.gitignore (100%) rename tests/{local-studio-tests => studio-tests}/supabase/config.toml (100%) rename tests/{local-studio-tests => studio-tests}/supabase/seed.sql (100%) create mode 100644 tests/studio-tests/tests/auth.setup.ts create mode 100644 tests/studio-tests/tests/common-functionality/home.spec.ts create mode 100644 tests/studio-tests/tests/common-functionality/logs.spec.ts rename tests/{local-studio-tests/tests/snapshot/spec => studio-tests/tests/common-functionality}/table-editor.spec.ts (83%) create mode 100644 tests/studio-tests/tests/production-functionality/example.spec.ts diff --git a/.github/workflows/local-studio-tests.yml b/.github/workflows/studio-e2e-tests.yml similarity index 81% rename from .github/workflows/local-studio-tests.yml rename to .github/workflows/studio-e2e-tests.yml index 24ccc70cce5..ec79edae963 100644 --- a/.github/workflows/local-studio-tests.yml +++ b/.github/workflows/studio-e2e-tests.yml @@ -1,16 +1,16 @@ -name: Local Studio Tests +name: Studio E2E Tests on: push: branches: [master] paths: - 'apps/studio/**' - - 'tests/local-studio-tests/**' + - 'tests/studio-tests/**' - 'pnpm-lock.yaml' pull_request: branches: [master] paths: - 'apps/studio/**' - - 'tests/local-studio-tests/**' + - 'tests/studio-tests/**' - 'pnpm-lock.yaml' # Cancel old builds on new commit for same workflow + branch/PR @@ -38,14 +38,14 @@ jobs: - name: Install dependencies run: pnpm i - name: Install Playwright Browsers - run: pnpm -C tests/local-studio-tests exec playwright install --with-deps + run: pnpm -C tests/studio-tests exec playwright install --with-deps - name: Run Playwright tests - run: pnpm test:local-studio + run: pnpm test:e2e:studio-local # mark the action as succeeded even if the tests failed. This is temporarily until we make the tests more stable. # continue-on-error: true - uses: actions/upload-artifact@v4 if: always() with: name: playwright-report - path: tests/local-studio-tests/playwright-report/ + path: tests/studio-tests/playwright-report/ retention-days: 7 diff --git a/.github/workflows/studio-tests.yml b/.github/workflows/studio-unit-tests.yml similarity index 100% rename from .github/workflows/studio-tests.yml rename to .github/workflows/studio-unit-tests.yml diff --git a/package.json b/package.json index f719b75a8a3..483cc362ea7 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "test:ui-patterns": "turbo run test --filter=ui-patterns", "test:studio": "turbo run test --filter=studio", "test:studio:watch": "turbo run test --filter=studio -- watch", - "test:local-studio": "pnpm --prefix tests/local-studio-tests run test", + "test:e2e:studio-local": "pnpm --prefix tests/studio-tests run test:local", "perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/", "perf:meta": "ab -t 5 -c 20 -T application/json http://localhost:5555/tables", "generate:types": "supabase gen types typescript --local > ./supabase/functions/common/database-types.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ccbc6da9d6e..dacb128ea6e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1956,6 +1956,18 @@ importers: specifier: ^4.17.21 version: 4.17.21 + tests/studio-tests: + devDependencies: + '@playwright/test': + specifier: ^1.49.0 + version: 1.49.1 + dotenv: + specifier: ^16.4.7 + version: 16.4.7 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + packages: '@aashutoshrathi/word-wrap@1.2.6': @@ -13626,7 +13638,7 @@ snapshots: '@babel/traverse': 7.24.7(supports-color@8.1.1) '@babel/types': 7.24.7 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13922,7 +13934,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.24.7 '@babel/parser': 7.24.7 '@babel/types': 7.24.7 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -14243,7 +14255,7 @@ snapshots: '@esbuild-plugins/node-resolve@0.2.2(esbuild@0.21.5)(supports-color@8.1.1)': dependencies: '@types/resolve': 1.20.6 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) esbuild: 0.21.5 escape-string-regexp: 4.0.0 resolve: 1.22.10 @@ -14467,7 +14479,7 @@ snapshots: '@eslint/eslintrc@2.1.4(supports-color@8.1.1)': dependencies: ajv: 6.12.6 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) espree: 9.6.1 globals: 13.24.0 ignore: 5.2.4 @@ -14655,7 +14667,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14(supports-color@8.1.1)': dependencies: '@humanwhocodes/object-schema': 2.0.2 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -14953,7 +14965,7 @@ snapshots: '@kwsites/file-exists@1.1.1(supports-color@8.1.1)': dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -18146,7 +18158,7 @@ snapshots: '@typescript-eslint/types': 7.2.0 '@typescript-eslint/typescript-estree': 7.2.0(supports-color@8.1.1)(typescript@5.5.2) '@typescript-eslint/visitor-keys': 7.2.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) eslint: 8.57.0(supports-color@8.1.1) optionalDependencies: typescript: 5.5.2 @@ -18164,7 +18176,7 @@ snapshots: dependencies: '@typescript-eslint/types': 7.2.0 '@typescript-eslint/visitor-keys': 7.2.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -18500,19 +18512,19 @@ snapshots: agent-base@6.0.2(supports-color@8.1.1): dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color agent-base@7.1.1(supports-color@8.1.1): dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color agent-base@7.1.1(supports-color@9.4.0): dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -19610,18 +19622,18 @@ snapshots: optionalDependencies: supports-color: 8.1.1 - debug@4.3.7(supports-color@9.4.0): - dependencies: - ms: 2.1.3 - optionalDependencies: - supports-color: 9.4.0 - debug@4.4.0(supports-color@8.1.1): dependencies: ms: 2.1.3 optionalDependencies: supports-color: 8.1.1 + debug@4.4.0(supports-color@9.4.0): + dependencies: + ms: 2.1.3 + optionalDependencies: + supports-color: 9.4.0 + decimal.js-light@2.5.1: {} decimal.js@10.5.0: @@ -20152,7 +20164,7 @@ snapshots: eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9(supports-color@8.1.1))(eslint-plugin-import@2.29.1)(eslint@8.57.0(supports-color@8.1.1))(supports-color@8.1.1): dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) enhanced-resolve: 5.17.1 eslint: 8.57.0(supports-color@8.1.1) eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9(supports-color@8.1.1))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0(supports-color@8.1.1))(supports-color@8.1.1) @@ -21249,21 +21261,21 @@ snapshots: https-proxy-agent@5.0.1(supports-color@8.1.1): dependencies: agent-base: 6.0.2(supports-color@8.1.1) - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4(supports-color@8.1.1): dependencies: agent-base: 7.1.1(supports-color@8.1.1) - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4(supports-color@9.4.0): dependencies: agent-base: 7.1.1(supports-color@9.4.0) - debug: 4.3.7(supports-color@9.4.0) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -22117,7 +22129,7 @@ snapshots: json-schema-resolver@2.0.0(supports-color@8.1.1): dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) rfdc: 1.4.1 uri-js: 4.4.1 transitivePeerDependencies: @@ -23409,7 +23421,7 @@ snapshots: micromark@3.2.0(supports-color@8.1.1): dependencies: '@types/debug': 4.1.9 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) decode-named-character-reference: 1.0.2 micromark-core-commonmark: 1.1.0 micromark-factory-space: 1.1.0 @@ -23431,7 +23443,7 @@ snapshots: micromark@4.0.0(supports-color@8.1.1): dependencies: '@types/debug': 4.1.9 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) decode-named-character-reference: 1.0.2 devlop: 1.1.0 micromark-core-commonmark: 2.0.1 @@ -25743,7 +25755,7 @@ snapshots: simple-websocket@9.1.0(supports-color@8.1.1): dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) queue-microtask: 1.2.3 randombytes: 2.1.0 readable-stream: 3.6.2 @@ -26995,7 +27007,7 @@ snapshots: vite-node@2.1.8(@types/node@22.12.0)(sass@1.72.0)(supports-color@8.1.1)(terser@5.37.0): dependencies: cac: 6.7.14 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.4.0(supports-color@8.1.1) es-module-lexer: 1.6.0 pathe: 1.1.2 vite: 5.4.12(@types/node@22.12.0)(sass@1.72.0)(terser@5.37.0) diff --git a/tests/local-studio-tests/package.json b/tests/local-studio-tests/package.json deleted file mode 100644 index 5121ff429db..00000000000 --- a/tests/local-studio-tests/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "local-studio-tests", - "version": "1.0.0", - "description": "", - "main": "index.js", - "scripts": { - "preinstall": "npx only-allow pnpm", - "env:setup": "supabase start -x studio && supabase status --output json > keys.json && node scripts/generate-env.js", - "pretest": "pnpm env:setup", - "test": "playwright test", - "posttest": "supabase stop --no-backup", - "codegen:setup": "pnpm env:setup && NODE_ENV=test pnpm --prefix ../../apps/studio dev", - "codegen": "playwright codegen http://localhost:8082/project/default", - "clean": "rimraf node_modules" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "@playwright/test": "^1.49.0", - "lodash": "^4.17.21" - } -} diff --git a/tests/local-studio-tests/playwright.config.ts b/tests/local-studio-tests/playwright.config.ts deleted file mode 100644 index b3a260763ef..00000000000 --- a/tests/local-studio-tests/playwright.config.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { defineConfig, devices } from '@playwright/test' - -/** - * Read environment variables from file. - * https://github.com/motdotla/dotenv - */ -// require('dotenv').config(); - -/** - * See https://playwright.dev/docs/test-configuration. - */ -export default defineConfig({ - testDir: './tests', - /* Run tests in files in parallel */ - fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: 'http://localhost:8082', - - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', - // record a video for failed tests, but only for local testing. We can't store videos on CI atm. - video: 'retain-on-failure', - launchOptions: { - env: { - NODE_ENV: 'test', - }, - }, - }, - - /* Configure projects for major browsers */ - projects: [ - { - name: 'chromium', - use: { ...devices['Desktop Chrome'], viewport: { width: 1366, height: 768 } }, - }, - - // { - // name: 'firefox', - // use: { ...devices['Desktop Firefox'] }, - // }, - - // { - // name: 'webkit', - // use: { ...devices['Desktop Safari'] }, - // }, - - /* Test against mobile viewports. */ - // { - // name: 'Mobile Chrome', - // use: { ...devices['Pixel 5'] }, - // }, - // { - // name: 'Mobile Safari', - // use: { ...devices['iPhone 12'] }, - // }, - - /* Test against branded browsers. */ - // { - // name: 'Microsoft Edge', - // use: { ...devices['Desktop Edge'], channel: 'msedge' }, - // }, - // { - // name: 'Google Chrome', - // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, - // }, - ], - - /* Run your local dev server before starting the tests */ - webServer: { - // using npm run dev instead of turbo because turbo doesn't stop the server after a test (doesn't handle SIGTERM). - command: 'pnpm dev', - cwd: '../../apps/studio', - url: 'http://localhost:8082', - reuseExistingServer: !process.env.CI, - env: { - NODE_ENV: 'test', - }, - // Show the nextjs logs in the terminal. Useful for debugging. - // stdout: 'pipe', - // stderr: 'pipe', - }, -}) diff --git a/tests/local-studio-tests/tests/snapshot/spec/logs.spec.ts b/tests/local-studio-tests/tests/snapshot/spec/logs.spec.ts deleted file mode 100644 index ab7c8f5b37b..00000000000 --- a/tests/local-studio-tests/tests/snapshot/spec/logs.spec.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { test, expect } from '@playwright/test' - -const LOGS_PAGES = [ - 'API Gateway', - 'Postgres', - 'PostgREST', - // 'Auth', Wont have logs on first load - // 'Storage', Wont have logs on first load - //'Realtime' Wont have logs on first load -] - -test.describe('Logs', async () => { - for (const logPage of LOGS_PAGES) { - test.describe(`${logPage} logs page`, () => { - test('can navigate to logs page', async ({ page }) => { - await page.goto('http://localhost:8082/project/default') - await page.locator('a', { hasText: 'Logs' }).click({ timeout: 4000 }) - await expect(page.getByText('Logs & Analytics')).toBeVisible() - - // Click anywhere on the screen to close the sidebar - await page.click('body') - - await page - .getByRole('link', { name: logPage, exact: true }) - .click() - .catch((e) => { - console.log('🔴 Error clicking', logPage, e) - throw e - }) - - // Wait for and verify the logs table is present - const logsTable = page.getByRole('table') - await expect(logsTable).toBeVisible() - }) - - test('shows logs data without errors', async ({ page }) => { - // Navigate to page first - await page.goto('http://localhost:8082/project/default') - await page.locator('a', { hasText: 'Logs' }).click({ timeout: 4000 }) - await page.click('body') - await page.getByRole('link', { name: logPage, exact: true }).click() - - // Wait a bit and check for errors with a longer timeout - const error = page.getByText('Error fetching logs') - await expect(error).not.toBeVisible({ timeout: 10000 }) - - // Check if the logs table has any rows - const gridcells = page.getByRole('gridcell') - await expect(gridcells.first()).toBeVisible() - }) - - test('can select and view log details', async ({ page }) => { - // Navigate to page first - await page.goto('http://localhost:8082/project/default') - await page.locator('a', { hasText: 'Logs' }).click({ timeout: 4000 }) - await page.click('body') - await page.getByRole('link', { name: logPage, exact: true }).click() - - const gridcells = page.getByRole('gridcell') - - // Click first row and verify details - await gridcells.first().click() - const tabPanel = page.getByTestId('log-selection') - await expect(tabPanel).toBeVisible({ timeout: 2000 }) - - const selectionPanelTimestamp = tabPanel.getByTestId('log-selection-timestamp') - await expect(selectionPanelTimestamp).toBeVisible() - - const rawTimestamp = await selectionPanelTimestamp.textContent() - const timestamp = rawTimestamp?.replace('timestamp', '') - const rowText = await gridcells.first().textContent() - expect(rowText).toContain(timestamp) - - // Click second row and verify different content - await gridcells.nth(1).click() - const tabPanelText2 = await tabPanel.textContent() - expect(tabPanelText2).not.toBe(rowText) - }) - }) - } -}) diff --git a/tests/studio-tests/.env.staging.example b/tests/studio-tests/.env.staging.example new file mode 100644 index 00000000000..9595e3fa1f8 --- /dev/null +++ b/tests/studio-tests/.env.staging.example @@ -0,0 +1,3 @@ +EMAIL= +PASSWORD= +PROJECT_REF= \ No newline at end of file diff --git a/tests/local-studio-tests/.gitignore b/tests/studio-tests/.gitignore similarity index 66% rename from tests/local-studio-tests/.gitignore rename to tests/studio-tests/.gitignore index 224648d25a2..1988c7fd355 100644 --- a/tests/local-studio-tests/.gitignore +++ b/tests/studio-tests/.gitignore @@ -3,4 +3,6 @@ node_modules/ /playwright-report/ /blob-report/ /playwright/.cache/ -keys.json +/playwright/.auth/ +.env.staging +keys.json \ No newline at end of file diff --git a/tests/local-studio-tests/README.md b/tests/studio-tests/README.md similarity index 91% rename from tests/local-studio-tests/README.md rename to tests/studio-tests/README.md index 25cb04d2096..e877f2fc672 100644 --- a/tests/local-studio-tests/README.md +++ b/tests/studio-tests/README.md @@ -1,6 +1,6 @@ -# Local studio tests +# Studio E2E Tests -In an effort to make the local studio more stable, we've added tests which test features which are commonly used in local +In an effort to make the local + hosted studio more stable, we've added tests which test features which are commonly used in local development. Built with [Playwright](https://playwright.dev/docs/intro). ## How to run tests diff --git a/tests/studio-tests/base.ts b/tests/studio-tests/base.ts new file mode 100644 index 00000000000..5484171153e --- /dev/null +++ b/tests/studio-tests/base.ts @@ -0,0 +1,25 @@ +import { test as base } from '@playwright/test' +import dotenv from 'dotenv' +import path from 'path' + +dotenv.config({ + path: path.resolve(__dirname, process.env.ENV === 'staging' ? '../.env.staging' : ''), + override: true, +}) + +export interface TestOptions { + env: string + ref: string + apiUrl: string +} + +export const test = base.extend({ + env: process.env.ENV, + ref: process.env.PROJECT_REF, + apiUrl: + process.env.ENV === 'local' + ? 'http://localhost:8082/api' + : process.env.ENV === 'staging' + ? 'https://api.supabase.green' + : '', +}) diff --git a/tests/local-studio-tests/scripts/generate-env.js b/tests/studio-tests/generate-local-env.js similarity index 96% rename from tests/local-studio-tests/scripts/generate-env.js rename to tests/studio-tests/generate-local-env.js index f1df4987dcd..d51ffb3775f 100644 --- a/tests/local-studio-tests/scripts/generate-env.js +++ b/tests/studio-tests/generate-local-env.js @@ -1,5 +1,5 @@ const fs = require('fs') -const generatedEnv = require('../keys.json') +const generatedEnv = require('./keys.json') /** * This script takes the API keys from the local environment, merges them with some predefined variables and saves them diff --git a/tests/studio-tests/package.json b/tests/studio-tests/package.json new file mode 100644 index 00000000000..e15ee220a07 --- /dev/null +++ b/tests/studio-tests/package.json @@ -0,0 +1,23 @@ +{ + "name": "studio-tests", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "preinstall": "npx only-allow pnpm", + "env:setup": "supabase start -x studio && supabase status --output json > keys.json && node generate-local-env.js", + "pretest:local": "pnpm env:setup", + "test:local": "pnpm env:setup && export ENV=local && export PROJECT_REF=default && playwright test", + "posttest:local": "supabase stop --no-backup", + "test:staging": "export ENV=staging && playwright test", + "clean": "rimraf node_modules" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.49.0", + "lodash": "^4.17.21", + "dotenv": "^16.4.7" + } +} diff --git a/tests/studio-tests/playwright.config.ts b/tests/studio-tests/playwright.config.ts new file mode 100644 index 00000000000..8f0b6484ad0 --- /dev/null +++ b/tests/studio-tests/playwright.config.ts @@ -0,0 +1,75 @@ +import { defineConfig, devices } from '@playwright/test' + +// See https://playwright.dev/docs/test-configuration. + +export default defineConfig({ + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: 'html', + use: { + baseURL: + process.env.ENV === 'local' + ? 'http://localhost:8082' + : process.env.ENV === 'staging' + ? 'https://supabase.green/dashboard/' + : '', + trace: 'on-first-retry', + video: 'retain-on-failure', + launchOptions: { + env: { + NODE_ENV: + process.env.ENV === 'local' ? 'test' : process.env.ENV === 'staging' ? 'staging' : '', + }, + }, + }, + projects: [ + ...(process.env.ENV !== 'local' + ? [ + { + name: 'Authentication setup', + testDir: './tests', + testMatch: /.*\.setup\.ts/, + use: { headless: false }, + }, + ] + : []), + { + name: 'Common Functionality', + use: { + ...devices['Desktop Chrome'], + ...(process.env.ENV !== 'local' ? { storageState: 'playwright/.auth/user.json' } : {}), + viewport: { width: 1366, height: 768 }, + }, + testDir: './tests/common-functionality', + dependencies: process.env.ENV !== 'local' ? ['Authentication setup'] : undefined, + }, + ...(process.env.ENV !== 'local' + ? [ + { + name: 'Production Functionality', + use: { + ...devices['Desktop Chrome'], + storageState: 'playwright/.auth/user.json', + viewport: { width: 1366, height: 768 }, + }, + testDir: './tests/production-functionality', + dependencies: ['Authentication setup'], + }, + ] + : []), + ], + /* Run your local dev server before starting the tests */ + webServer: + process.env.ENV === 'local' + ? { + // using npm run dev instead of turbo because turbo doesn't stop the server after a test (doesn't handle SIGTERM). + command: 'pnpm dev', + cwd: '../../apps/studio', + url: 'http://localhost:8082', + reuseExistingServer: !process.env.CI, + env: { NODE_ENV: 'test' }, + } + : undefined, +}) diff --git a/tests/local-studio-tests/supabase/.gitignore b/tests/studio-tests/supabase/.gitignore similarity index 100% rename from tests/local-studio-tests/supabase/.gitignore rename to tests/studio-tests/supabase/.gitignore diff --git a/tests/local-studio-tests/supabase/config.toml b/tests/studio-tests/supabase/config.toml similarity index 100% rename from tests/local-studio-tests/supabase/config.toml rename to tests/studio-tests/supabase/config.toml diff --git a/tests/local-studio-tests/supabase/seed.sql b/tests/studio-tests/supabase/seed.sql similarity index 100% rename from tests/local-studio-tests/supabase/seed.sql rename to tests/studio-tests/supabase/seed.sql diff --git a/tests/studio-tests/tests/auth.setup.ts b/tests/studio-tests/tests/auth.setup.ts new file mode 100644 index 00000000000..4bcd2715257 --- /dev/null +++ b/tests/studio-tests/tests/auth.setup.ts @@ -0,0 +1,21 @@ +import { test as setup } from '@playwright/test' +import dotenv from 'dotenv' +import path from 'path' + +dotenv.config({ + path: path.resolve(__dirname, process.env.ENV === 'staging' ? '../.env.staging' : ''), + override: true, +}) +const authFile = path.join(__dirname, '../playwright/.auth/user.json') + +setup('Authenticate', async ({ page }) => { + await page.goto('./sign-in') + await page.getByLabel('Email').fill(process.env.EMAIL ?? '') + await page.getByLabel('Password').fill(process.env.PASSWORD ?? '') + await page.getByRole('button', { name: 'Sign In' }).click() + + await page.pause() + + await page.waitForURL('./projects') + await page.context().storageState({ path: authFile }) +}) diff --git a/tests/studio-tests/tests/common-functionality/home.spec.ts b/tests/studio-tests/tests/common-functionality/home.spec.ts new file mode 100644 index 00000000000..9a590ce7a4d --- /dev/null +++ b/tests/studio-tests/tests/common-functionality/home.spec.ts @@ -0,0 +1,13 @@ +import { expect } from '@playwright/test' +import { test } from '../../base' + +test.describe('Project', async () => { + test('Can navigate to project home page', async ({ page, env, ref }) => { + await page.goto(`./project/${ref}`) + await expect( + page.getByRole('heading', { + name: env === 'local' ? 'Welcome to your project' : 'Playwright Test', + }) + ).toBeVisible({ timeout: 10000 }) + }) +}) diff --git a/tests/studio-tests/tests/common-functionality/logs.spec.ts b/tests/studio-tests/tests/common-functionality/logs.spec.ts new file mode 100644 index 00000000000..4ee103d8e9c --- /dev/null +++ b/tests/studio-tests/tests/common-functionality/logs.spec.ts @@ -0,0 +1,107 @@ +import { expect } from '@playwright/test' +import { test } from '../../base' + +const LOGS_PAGES = [ + { label: 'API Gateway', route: 'edge-logs' }, + { label: 'Postgres', route: 'postgres-logs' }, + { label: 'PostgREST', route: 'postgrest-logs' }, +] + +test.describe('Logs', async () => { + for (const logPage of LOGS_PAGES) { + test.describe(`${logPage.label} logs page`, () => { + test('can navigate to logs page', async ({ page, ref }) => { + await page.goto(`./project/${ref}`) + await page.locator('a', { hasText: 'Logs' }).click({ timeout: 10000 }) + await expect(page.getByRole('heading', { name: 'Logs & Analytics' })).toBeVisible() + + // Click anywhere on the screen to close the sidebar + await page.click('body') + + await page + .getByRole('link', { name: logPage.label, exact: true }) + .click() + .catch((e) => { + console.log('🔴 Error clicking', logPage, e) + throw e + }) + + // Wait for and verify the logs table is present + const logsTable = page.getByRole('table') + await expect(logsTable).toBeVisible() + }) + + test('shows logs data without errors', async ({ page, ref, apiUrl }) => { + // Navigate to page first + await page.goto(`./project/${ref}/logs/${logPage.route}`) + + await page.waitForResponse((response) => + response + .url() + .includes( + `${apiUrl}/platform/projects/${ref}/analytics/endpoints/logs.all?project=${ref}` + ) + ) + + // Wait a bit and check for errors with a longer timeout + const error = page.getByText('Error fetching logs') + await expect(error).not.toBeVisible({ timeout: 10000 }) + + const emptyState = page.getByText('No results found') + if (await emptyState.isVisible()) { + // Empty state, no need to check further + } else { + // Check if the logs table has any rows + const gridcells = page.getByRole('gridcell') + await expect(gridcells.first()).toBeVisible() + } + }) + + test('can select and view log details', async ({ page, ref, apiUrl }) => { + // Navigate to page first + await page.goto(`./project/${ref}/logs/${logPage.route}`) + + await page.waitForResponse((response) => + response + .url() + .includes( + `${apiUrl}/platform/projects/${ref}/analytics/endpoints/logs.all?project=${ref}` + ) + ) + + const emptyState = page.getByText('No results found') + if (await emptyState.isVisible()) { + // Empty state, no need to check further + } else { + // Check if the logs table has any rows + const gridcells = page.getByRole('gridcell') + + // Click first row and verify details + await gridcells.first().click() + const tabPanel = page.getByTestId('log-selection') + await page.waitForResponse((response) => + response + .url() + .includes( + `${apiUrl}/platform/projects/${ref}/analytics/endpoints/logs.all?project=${ref}` + ) + ) + await expect(tabPanel).toBeVisible() + + const selectionPanelTimestamp = tabPanel.getByTestId('log-selection-timestamp') + await expect(selectionPanelTimestamp).toBeVisible() + + const rawTimestamp = await selectionPanelTimestamp.textContent() + const timestamp = rawTimestamp?.replace('timestamp', '') + const rowText = await gridcells.first().textContent() + expect(rowText).toContain(timestamp) + + // Click second row and verify different content + await gridcells.nth(1).click() + const tabPanelText2 = await tabPanel.textContent() + expect(tabPanelText2).not.toBe(rowText) + } + }) + }) + } +}) diff --git a/tests/local-studio-tests/tests/snapshot/spec/table-editor.spec.ts b/tests/studio-tests/tests/common-functionality/table-editor.spec.ts similarity index 83% rename from tests/local-studio-tests/tests/snapshot/spec/table-editor.spec.ts rename to tests/studio-tests/tests/common-functionality/table-editor.spec.ts index a689cdcbf90..7d4322545c3 100644 --- a/tests/local-studio-tests/tests/snapshot/spec/table-editor.spec.ts +++ b/tests/studio-tests/tests/common-functionality/table-editor.spec.ts @@ -1,23 +1,30 @@ -import { Page, expect, test } from '@playwright/test' +import { Page, expect } from '@playwright/test' import { kebabCase } from 'lodash' +import { test } from '../../base' const dismissToast = async (page: Page) => { - await page.locator('li.toast').getByRole('button').waitFor({ state: 'visible' }) - await page.locator('li.toast').getByRole('button').click() + await page + .locator('li.toast') + .getByRole('button', { name: 'Opt out' }) + .waitFor({ state: 'visible' }) + await page.locator('li.toast').getByRole('button', { name: 'Opt out' }).click() } -test.describe('Table Editor page', () => { - test.beforeEach(async ({ page }) => { +test.describe('Table Editor', () => { + test.beforeEach(async ({ page, env, ref, apiUrl }) => { const tableResponsePromise = page.waitForResponse( - 'http://localhost:8082/api/platform/pg-meta/default/query?key=entity-types-public-0', + `${apiUrl}/platform/pg-meta/${ref}/query?key=entity-types-public-0`, { timeout: 0 } ) - await page.goto('/project/default/editor') + await page.goto(`./project/${ref}/editor`) + if (env !== 'local') await dismissToast(page) await tableResponsePromise }) test('should create a new table, view its definition, add new rows, sort and filter', async ({ page, + ref, + apiUrl, }, testInfo) => { const tableName = `${kebabCase(testInfo.title).slice(0, 24)}-${testInfo.retry}-${Math.floor(Math.random() * 10000)}` @@ -44,7 +51,6 @@ test.describe('Table Editor page', () => { await page.getByRole('button', { name: 'Save' }).waitFor({ state: 'visible' }) await page.getByRole('button', { name: 'Save' }).click() - await dismissToast(page) // hide React Query DevTools if present await page.evaluate(() => { @@ -67,7 +73,6 @@ test.describe('Table Editor page', () => { await page.getByTestId('defaultValueColumn-input').click() await page.getByTestId('defaultValueColumn-input').fill('100') await page.getByTestId('action-bar-save-row').click() - await dismissToast(page) // add a second row await page.getByRole('button', { name: tableName }).click() @@ -75,11 +80,10 @@ test.describe('Table Editor page', () => { await page.getByText('Insert a new row into').click() // the default value should be '100' for defaultValueColumn await page.getByTestId('action-bar-save-row').click() - await dismissToast(page) // Wait for both rows to be visible in the grid await page.waitForResponse((response) => - response.url().includes('/api/platform/pg-meta/default/query') + response.url().includes(`${apiUrl}/platform/pg-meta/${ref}/query`) ) await expect(page.getByRole('grid')).toContainText('2') await expect(page.getByRole('grid')).toContainText('100') @@ -94,8 +98,8 @@ test.describe('Table Editor page', () => { await page.getByLabel('Pick a column to sort by').getByText('defaultValueColumn').click() await page.getByRole('button', { name: 'Apply sorting' }).click() - // click away to close the sorting dialog - await page.locator('#spec-click-target').click() + // Close the sorting dialog + await page.keyboard.down('Escape') // expect the row to be sorted by defaultValueColumn. They're inserted in the order 100, 2 await expect(rows.nth(1)).toContainText('2') @@ -112,24 +116,27 @@ test.describe('Table Editor page', () => { await page.getByPlaceholder('Enter a value').click() await page.getByPlaceholder('Enter a value').fill('2') await page.getByRole('button', { name: 'Apply filter' }).click() - // click away to close the filter dialog - await page.locator('#spec-click-target').click() + + // Close the filter dialog + await page.keyboard.down('Escape') + await expect(page.getByRole('grid')).toContainText('2') await expect(page.getByRole('grid')).not.toContainText('100') + + // Delete the table as clean up + await page.getByLabel(`View ${tableName}`).click() + await page.getByLabel(`View ${tableName}`).getByRole('button').nth(1).click() + await page.getByText('Delete table').click() + await page.getByRole('button', { name: 'Delete' }).click() + await page.waitForResponse((response) => + response.url().includes(`${apiUrl}/platform/pg-meta/${ref}/tables`) + ) }) test('should check the auth schema', async ({ page }) => { - const tableResponsePromise = page.waitForResponse( - 'http://localhost:8082/api/platform/pg-meta/default/query?key=entity-types-public-0', - { timeout: 0 } - ) - await page.getByTestId('schema-selector').click() await page.getByRole('option', { name: 'auth' }).click() - // wait for the table data to load for the auth schema - await tableResponsePromise - // extract the tables names from the sidebar const tables = await page .getByTestId('tables-list') diff --git a/tests/studio-tests/tests/production-functionality/example.spec.ts b/tests/studio-tests/tests/production-functionality/example.spec.ts new file mode 100644 index 00000000000..a550cd44818 --- /dev/null +++ b/tests/studio-tests/tests/production-functionality/example.spec.ts @@ -0,0 +1,11 @@ +import { expect } from '@playwright/test' +import { test } from '../../base' + +// [Joshen] This file is redundant, so once we start putting in production specific spec tests, can remove this + +test.describe('Example', async () => { + test('Can navigate to project home page', async ({ page, ref }) => { + await page.goto(`./project/${ref}`) + await expect(page.getByRole('heading', { name: 'Playwright Test' })).toBeVisible() + }) +})