name: Run Braintrust evals

on:
  push:
    branches: [master]
  pull_request:
    types: [opened, synchronize, labeled]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  pull-requests: write
  contents: read

jobs:
  eval:
    name: Run evals
    if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-evals') && github.event.pull_request.head.repo.full_name == github.repository)
    runs-on: ubuntu-latest
    timeout-minutes: 20

    env:
      BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

    steps:
      - name: Checkout
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          fetch-depth: 0
          # For PR events, checkout the actual branch so Braintrust can report the correct branch name instead of detached HEAD.
          # github.head_ref is the PR source branch, github.ref_name is the fallback for push events (e.g., master).
          ref: ${{ github.head_ref || github.ref_name }}

      - name: Install pnpm
        uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
        with:
          run_install: false

      - name: Use Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version-file: ".nvmrc"
          cache: "pnpm"

      - name: Install Dependencies
        run: pnpm install --frozen-lockfile

      - name: Setup Evals
        run: cd apps/studio && pnpm evals:setup

      - name: Run Evals
        uses: braintrustdata/eval-action@c0dd75b29984a0cc63a827d6e8da2f23f2752be4 # v1.0.16
        with:
          api_key: ${{ secrets.BRAINTRUST_API_KEY }}
          runtime: node
          package_manager: pnpm
          root: apps/studio