mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 01:15:03 +03:00
Charis
da724eb8c6
## Summary - Add a zizmor config and CI job that lints `.github/workflows` on every PR touching it, downloading and attestation-verifying the pinned v1.26.1 release binary (cached across runs) - Fix the mutable-tag and excess-permission findings zizmor surfaces in `braintrust-evals.yml`, `publish_image.yml`, and `self-host-tests-smoke.yml`: pin `actions/checkout`/`actions/setup-node` to commit SHAs, and scope `pull-requests`/`packages`/`id-token` permissions down to the specific jobs that need them ## Test plan - [x] Confirm the `zizmor` job runs and passes on this PR <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added automated security scanning for workflow changes on pull requests. * Added configuration to allow specific workflow trigger exceptions. * **Security** * Tightened GitHub Actions permissions at the workflow level and re-granted only where required per job. * Pinned common build action versions to specific commits for more consistent execution. * **Maintenance** * Updated workflow caching and action step annotations without changing linting or fixing behavior. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
97 lines
3.1 KiB
YAML
97 lines
3.1 KiB
YAML
name: Run Braintrust evals
|
|
|
|
on:
|
|
push:
|
|
branches: [master]
|
|
pull_request:
|
|
types: [opened, synchronize, labeled]
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
# Cheap, always-on gate: verifies the eval harness can reach the MCP server
|
|
# (its one real tool, search_docs). Runs on every push/PR — no OpenAI, no full
|
|
# eval suite — so a broken MCP connection is caught early with an actionable
|
|
# message. The eval job depends on this, so evals never run against a broken
|
|
# MCP connection.
|
|
preflight:
|
|
name: Eval MCP preflight
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
|
|
with:
|
|
run_install: false
|
|
|
|
- name: Use Node.js
|
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version-file: '.nvmrc'
|
|
cache: 'pnpm'
|
|
|
|
- name: Install Dependencies
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: Preflight — MCP connectivity
|
|
run: cd apps/studio && pnpm evals:preflight
|
|
|
|
eval:
|
|
name: Run evals
|
|
needs: preflight
|
|
permissions:
|
|
pull-requests: write
|
|
if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-evals') && github.event.pull_request.head.repo.full_name == github.repository)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
|
|
env:
|
|
BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
# For PR events, checkout the actual branch so Braintrust can report the correct branch name instead of detached HEAD.
|
|
# github.head_ref is the PR source branch, github.ref_name is the fallback for push events (e.g., master).
|
|
ref: ${{ github.head_ref || github.ref_name }}
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
|
|
with:
|
|
run_install: false
|
|
|
|
- name: Use Node.js
|
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version-file: '.nvmrc'
|
|
cache: 'pnpm'
|
|
|
|
- name: Install Dependencies
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: Setup Evals
|
|
run: cd apps/studio && pnpm evals:setup
|
|
|
|
- name: Run Evals
|
|
uses: braintrustdata/eval-action@c0dd75b29984a0cc63a827d6e8da2f23f2752be4 # v1.0.16
|
|
with:
|
|
api_key: ${{ secrets.BRAINTRUST_API_KEY }}
|
|
runtime: node
|
|
package_manager: pnpm
|
|
root: apps/studio
|