supabase/.github/workflows/braintrust-evals.yml

name: Run Braintrust evals

on:
  push:
    branches: [master]
  pull_request:
    types: [opened, synchronize, labeled]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  # Cheap, always-on gate: verifies the eval harness can reach the MCP server
  # (its one real tool, search_docs). Runs on every push/PR — no OpenAI, no full
  # eval suite — so a broken MCP connection is caught early with an actionable
  # message. The eval job depends on this, so evals never run against a broken
  # MCP connection.
  preflight:
    name: Eval MCP preflight
    runs-on: ubuntu-latest
    timeout-minutes: 10

    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Install pnpm
        uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
        with:
          run_install: false

      - name: Use Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version-file: '.nvmrc'
          cache: 'pnpm'

      - name: Install Dependencies
        run: pnpm install --frozen-lockfile

      - name: Preflight — MCP connectivity
        run: cd apps/studio && pnpm evals:preflight

  eval:
    name: Run evals
    needs: preflight
    permissions:
      pull-requests: write
    if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-evals') && github.event.pull_request.head.repo.full_name == github.repository)
    runs-on: ubuntu-latest
    timeout-minutes: 20

    env:
      BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          fetch-depth: 0
          persist-credentials: false
          # For PR events, checkout the actual branch so Braintrust can report the correct branch name instead of detached HEAD.
          # github.head_ref is the PR source branch, github.ref_name is the fallback for push events (e.g., master).
          ref: ${{ github.head_ref || github.ref_name }}

      - name: Install pnpm
        uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
        with:
          run_install: false

      - name: Use Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version-file: '.nvmrc'
          cache: 'pnpm'

      - name: Install Dependencies
        run: pnpm install --frozen-lockfile

      - name: Setup Evals
        run: cd apps/studio && pnpm evals:setup

      - name: Run Evals
        uses: braintrustdata/eval-action@c0dd75b29984a0cc63a827d6e8da2f23f2752be4 # v1.0.16
        with:
          api_key: ${{ secrets.BRAINTRUST_API_KEY }}
          runtime: node
          package_manager: pnpm
          root: apps/studio