mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 01:15:03 +03:00
Matt Rossman
517171b246
Lays groundwork for online evals on Assistant chat logs. https://www.braintrust.dev/docs/observe/score-online ### Changes - New workflows: - `braintrust-scorers-deploy.yml` keeps prod scorers in sync on push to `master` - `braintrust-preview-scorers-deploy.yml` deploys preview scorers to the staging project for PRs labeled `preview-scorers`, posting a comment with scorer links ([example](https://github.com/supabase/supabase/pull/43194#issuecomment-4000097222)) - `braintrust-preview-scorers-cleanup.yml` deletes preview scorers when the PR is closed ([example](https://github.com/supabase/supabase/pull/43194#issuecomment-4000749847)) - Adds `evals/scorer-online.ts` entry point invoked with `pnpm scorers:deploy`, registering scorers for online evals in the Braintrust "Assistant" project - Refactors scorer code to separate online-compatible scorers (`scorer-online.ts`) from WASM-dependent ones (`scorer-wasm.ts`) - "URL Validity" scorer now only checks Supabase domains to prevent requests to untrusted origins - Span `input` is now shaped `{ prompt: string }` instead of plain `string` for compatibility with offline eval scorers - Env vars `BRAINTRUST_STAGING_PROJECT_ID` and `BRAINTRUST_PROJECT_ID` configured in GitHub repo settings - `generateAssistantResponse` now uses `startSpan` + `withCurrent` instead of `traced()` to manually manage the root span lifecycle — this ensures `onFinish` logs output to the span _before_ `span.end()` is called, which is when Braintrust triggers scoring automations ### Online Scorers We share scoring logic across offline and online evals, but some of our scorers aren't transferrable to an "online" setting due to runtime challenges or ground truth requirements. **Supported** - Goal Completion - Conciseness - Completeness - Docs Faithfulness - URL Validity **Unsupported** - Correctness (requires ground truth output) - Tool Usage (requires ground truth requiredTools) - SQL Syntax (uses libpg-query WASM) - SQL Identifier Quoting (uses libpg-query WASM) ### How to use these scorers Going forward if you want to add/edit online eval scorers, add the `preview-scorers` label to a PR. This deploys scorers to the [Assistant (Staging Scorers)](https://www.braintrust.dev/app/supabase.io/p/Assistant%20(Staging%20Scorers)?v=Overview) project in Braintrust with branch-specific slugs, and comments on the PR ([example](https://github.com/supabase/supabase/pull/43194#issuecomment-4000097222)). From the Braintrust dashboard you can "Test" the scorer with traces from any project. <img width="1866" height="528" alt="CleanShot 2026-03-05 at 15 15 00@2x" src="https://github.com/user-attachments/assets/4f15cebc-3f2d-4e8a-9ee2-fe8ef7bf4199" /> Once merged, scorers are deployed to the primary [Assistant](https://www.braintrust.dev/app/supabase.io/p/Assistant) project, and preview scorers are deleted from the staging project. Down the road, scorers on the Assistant project will run automatically on a sample of production traces. Closes AI-437
243 lines
7.6 KiB
JSON
243 lines
7.6 KiB
JSON
{
|
|
"$schema": "https://turbo.build/schema.json",
|
|
"ui": "stream",
|
|
"tasks": {
|
|
"build": {
|
|
"dependsOn": ["^build"],
|
|
"outputs": ["dist/**", ".next/**"]
|
|
},
|
|
"design-system#build": {
|
|
"dependsOn": ["^build"],
|
|
"env": [
|
|
"NEXT_PUBLIC_BASE_PATH",
|
|
"NEXT_PUBLIC_APP_URL",
|
|
// These envs are used in the packages
|
|
"NEXT_PUBLIC_STORAGE_KEY",
|
|
"NEXT_PUBLIC_AUTH_DEBUG_KEY",
|
|
"NEXT_PUBLIC_AUTH_PERSISTED_KEY",
|
|
"NEXT_PUBLIC_AUTH_NAVIGATOR_LOCK_KEY",
|
|
"NEXT_PUBLIC_AUTH_DETECT_SESSION_IN_URL",
|
|
"NEXT_PUBLIC_GOTRUE_URL",
|
|
"NEXT_PUBLIC_SUPABASE_ANON_KEY"
|
|
],
|
|
"outputs": [".next/**", "!.next/cache/**", ".contentlayer/**"]
|
|
},
|
|
"studio#build": {
|
|
"dependsOn": ["^build"],
|
|
"env": [
|
|
"ANALYZE",
|
|
"CI",
|
|
"NEXT_PUBLIC_SUPPORT_API_URL",
|
|
"NEXT_PUBLIC_CONTENT_API_URL",
|
|
"NEXT_PUBLIC_BASE_PATH",
|
|
"NEXT_PUBLIC_STRIPE_PUBLIC_KEY",
|
|
"NEXT_PUBLIC_SUPPORT_ANON_KEY",
|
|
"NEXT_PUBLIC_ENVIRONMENT",
|
|
"NEXT_PUBLIC_IS_PLATFORM",
|
|
"NEXT_PUBLIC_SITE_URL",
|
|
"NEXT_PUBLIC_API_URL",
|
|
"NEXT_PUBLIC_DOCS_URL",
|
|
"NEXT_PUBLIC_CONFIGCAT_SDK_KEY",
|
|
"NEXT_PUBLIC_CONFIGCAT_PROXY_URL",
|
|
"NEXT_PUBLIC_HCAPTCHA_SITE_KEY",
|
|
"NEXT_PUBLIC_SUPABASE_URL",
|
|
"NEXT_PUBLIC_SUPABASE_ANON_KEY",
|
|
"NEXT_PUBLIC_NODE_ENV",
|
|
"NEXT_PUBLIC_GOTRUE_URL",
|
|
"NEXT_PUBLIC_VERCEL_BRANCH_URL",
|
|
"NEXT_PUBLIC_GOOGLE_MAPS_KEY",
|
|
"NEXT_RUNTIME",
|
|
"NIMBUS_PROD_PROJECTS_URL",
|
|
"NIMBUS_PROD_PROJECTS_URL_WS",
|
|
"NODE_ENV",
|
|
"SUPABASE_URL",
|
|
"VERCEL",
|
|
"VERCEL_ENV",
|
|
"MAINTENANCE_MODE",
|
|
// These envs are used in the packages
|
|
"NEXT_PUBLIC_STORAGE_KEY",
|
|
"NEXT_PUBLIC_AUTH_DEBUG_KEY",
|
|
"NEXT_PUBLIC_AUTH_PERSISTED_KEY",
|
|
"NEXT_PUBLIC_AUTH_NAVIGATOR_LOCK_KEY",
|
|
"NEXT_PUBLIC_AUTH_DETECT_SESSION_IN_URL",
|
|
"NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID",
|
|
"NEXT_PUBLIC_VERCEL_ENV",
|
|
"NEXT_PUBLIC_USERCENTRICS_RULESET_ID",
|
|
"NEXT_PUBLIC_MCP_URL",
|
|
"NEXT_PUBLIC_IS_NIMBUS",
|
|
"NEXT_PUBLIC_ONGOING_INCIDENT",
|
|
// These envs are technically passthrough env vars because they're only used on the server side of Nextjs
|
|
"PLATFORM_PG_META_URL",
|
|
"STUDIO_PG_META_URL",
|
|
"PG_META_CRYPTO_KEY",
|
|
"POSTGRES_PASSWORD",
|
|
"POSTGRES_HOST",
|
|
"POSTGRES_USER_READ_WRITE",
|
|
"POSTGRES_USER_READ_ONLY",
|
|
"POSTGRES_DB",
|
|
"POSTGRES_PORT",
|
|
"READ_ONLY_URL",
|
|
"READ_ONLY_API_KEY",
|
|
"SUPABASE_SERVICE_KEY",
|
|
"SUPABASE_ANON_KEY",
|
|
"SUPABASE_PUBLIC_URL",
|
|
"DEFAULT_PROJECT_NAME",
|
|
"DEFAULT_ORGANIZATION_NAME",
|
|
"OPENAI_API_KEY",
|
|
"BRAINTRUST_API_KEY",
|
|
"BRAINTRUST_PROJECT_ID",
|
|
"AUTH_JWT_SECRET",
|
|
"LOGFLARE_API_KEY",
|
|
"LOGFLARE_PUBLIC_ACCESS_TOKEN",
|
|
"LOGFLARE_PRIVATE_ACCESS_TOKEN",
|
|
"LOGFLARE_URL",
|
|
"SENTRY_ORG",
|
|
"SENTRY_PROJECT",
|
|
"SENTRY_AUTH_TOKEN",
|
|
"NEXT_PUBLIC_SENTRY_DSN",
|
|
"AWS_BEDROCK_PROFILE",
|
|
"AWS_BEDROCK_ROLE_ARN",
|
|
"AWS_ACCESS_KEY_ID",
|
|
"AWS_SECRET_ACCESS_KEY",
|
|
"FORCE_ASSET_CDN",
|
|
"ASSET_CDN_S3_ENDPOINT",
|
|
"SITE_NAME",
|
|
"VERCEL_URL",
|
|
"IS_BRAINTRUST_PUSH",
|
|
"GITHUB_HEAD_REF",
|
|
"GITHUB_REF_NAME",
|
|
"GITHUB_PR_NUMBER",
|
|
"IS_THROTTLED",
|
|
"AI_PRO_MODEL",
|
|
"AI_NORMAL_MODEL",
|
|
"SUPPORT_SUPABASE_SECRET_KEY",
|
|
"STATUSPAGE_API_KEY",
|
|
"STATUSPAGE_PAGE_ID",
|
|
"LIVE_SUPABASE_SECRET_KEY"
|
|
],
|
|
"passThroughEnv": [
|
|
"CURRENT_CLI_VERSION",
|
|
"VERCEL_GIT_COMMIT_REF",
|
|
"VERCEL_GIT_COMMIT_SHA",
|
|
"SNIPPETS_MANAGEMENT_FOLDER",
|
|
"EDGE_FUNCTIONS_MANAGEMENT_FOLDER"
|
|
],
|
|
"outputs": [".next/**", "!.next/cache/**"]
|
|
},
|
|
"www#build": {
|
|
"dependsOn": ["^build"],
|
|
"env": [
|
|
"ANALYZE",
|
|
"CI",
|
|
"CF_ACCESS_CLIENT_ID",
|
|
"CF_ACCESS_CLIENT_SECRET",
|
|
"NEXT_PUBLIC_MISC_USE_URL",
|
|
"NEXT_PUBLIC_MISC_USE_ANON_KEY",
|
|
"NEXT_PUBLIC_STUDIO_URL",
|
|
"NEXT_PUBLIC_DOCS_URL",
|
|
"NEXT_PUBLIC_REFERENCE_DOCS_URL",
|
|
"NEXT_PUBLIC_UI_LIBRARY_URL",
|
|
"NEXT_PUBLIC_SUPABASE_URL",
|
|
"NEXT_PUBLIC_SUPABASE_ANON_KEY",
|
|
"NEXT_PUBLIC_VERCEL_ENV",
|
|
"NEXT_PUBLIC_VERCEL_BRANCH_URL",
|
|
"NEXT_PUBLIC_API_URL",
|
|
"NEXT_PUBLIC_HCAPTCHA_SITE_KEY",
|
|
"HCAPTCHA_SECRET_KEY",
|
|
"NODE_ENV",
|
|
"NEXT_PUBLIC_SENTRY_DSN",
|
|
"NEXT_PUBLIC_SURVEY_SUPABASE_URL",
|
|
"NEXT_PUBLIC_SURVEY_SUPABASE_ANON_KEY",
|
|
"VERCEL",
|
|
"VERCEL_ENV",
|
|
"VERCEL_GIT_COMMIT_SHA",
|
|
// These envs are used in the packages
|
|
"NEXT_PUBLIC_STORAGE_KEY",
|
|
"NEXT_PUBLIC_AUTH_DEBUG_KEY",
|
|
"NEXT_PUBLIC_AUTH_PERSISTED_KEY",
|
|
"NEXT_PUBLIC_AUTH_NAVIGATOR_LOCK_KEY",
|
|
"NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID",
|
|
"NEXT_PUBLIC_IS_PLATFORM",
|
|
"NEXT_PUBLIC_AUTH_DETECT_SESSION_IN_URL",
|
|
"NEXT_PUBLIC_GOTRUE_URL",
|
|
"NEXT_PUBLIC_BASE_PATH",
|
|
"NEXT_PUBLIC_EXAMPLES_SUPABASE_ANON_KEY",
|
|
"NEXT_PUBLIC_EXAMPLES_SUPABASE_URL",
|
|
"NEXT_PUBLIC_USERCENTRICS_RULESET_ID",
|
|
// These envs are technically passthrough env vars because they're only used on the server side of Nextjs
|
|
"LIVE_SUPABASE_COM_SERVICE_ROLE_KEY",
|
|
"GITHUB_CHANGELOG_APP_ID",
|
|
"GITHUB_CHANGELOG_APP_INSTALLATION_ID",
|
|
"GITHUB_CHANGELOG_APP_REST_KEY",
|
|
"GITHUB_CHANGELOG_APP_PRIVATE_KEY",
|
|
"NEXT_PUBLIC_EMAIL_ABUSE_URL",
|
|
"EMAIL_ABUSE_SERVICE_KEY",
|
|
"HUBSPOT_PORTAL_ID",
|
|
"HUBSPOT_ENTERPRISE_FORM_GUID",
|
|
"HUBSPOT_PARTNERSHIP_FORM_GUID",
|
|
"OPENAI_API_KEY",
|
|
"EMAIL_REPORT_SLACK_WEBHOOK",
|
|
"npm_lifecycle_event",
|
|
"AWS_ACCESS_KEY_ID",
|
|
"AWS_SECRET_ACCESS_KEY",
|
|
"FORCE_ASSET_CDN",
|
|
"ASSET_CDN_S3_ENDPOINT",
|
|
"SITE_NAME",
|
|
"LUMA_API_KEY",
|
|
"NEXT_RUNTIME",
|
|
// These env vars are used by some community and marketing integrations
|
|
"SENTRY_DSN_COMMUNITY",
|
|
"NOTION_SUPASQUAD_API_KEY",
|
|
"NOTION_SUPASQUAD_APPLICATIONS_DB_ID",
|
|
"NEXT_PUBLIC_SUPASQUAD_APPLICATION_FORM_URL",
|
|
"CUSTOMERIO_SITE_ID",
|
|
"CUSTOMERIO_API_KEY",
|
|
"CUSTOMERIO_APP_API_KEY"
|
|
],
|
|
"outputs": [".next/**", "!.next/cache/**", ".contentlayer/**"]
|
|
},
|
|
"ui-library#build": {
|
|
"dependsOn": ["^build"],
|
|
"env": [
|
|
"NEXT_PUBLIC_BASE_PATH",
|
|
"NEXT_PUBLIC_APP_URL",
|
|
"NEXT_PUBLIC_VERCEL_TARGET_ENV",
|
|
"NEXT_PUBLIC_VERCEL_BRANCH_URL",
|
|
"NEXT_PUBLIC_API_URL",
|
|
"NODE_ENV",
|
|
"HIDE_TAILWIND_INDICATOR",
|
|
// These envs are used in the packages
|
|
"NEXT_PUBLIC_STORAGE_KEY",
|
|
"NEXT_PUBLIC_AUTH_DEBUG_KEY",
|
|
"NEXT_PUBLIC_AUTH_PERSISTED_KEY",
|
|
"NEXT_PUBLIC_AUTH_NAVIGATOR_LOCK_KEY",
|
|
"NEXT_PUBLIC_AUTH_DETECT_SESSION_IN_URL",
|
|
"NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID",
|
|
"NEXT_PUBLIC_GOTRUE_URL",
|
|
"NEXT_PUBLIC_SUPABASE_ANON_KEY",
|
|
"NEXT_PUBLIC_USERCENTRICS_RULESET_ID",
|
|
"SUPABASE_MANAGEMENT_API_TOKEN",
|
|
"OPENAI_API_KEY"
|
|
],
|
|
"outputs": [".next/**", "!.next/cache/**", ".contentlayer/**"]
|
|
},
|
|
"lint": {
|
|
"outputs": []
|
|
},
|
|
"clean": {
|
|
"outputs": [],
|
|
"cache": false
|
|
},
|
|
"dev": {
|
|
"cache": false
|
|
},
|
|
"test": {
|
|
"cache": false
|
|
},
|
|
"typecheck": {
|
|
"dependsOn": ["^typecheck"],
|
|
"outputs": []
|
|
}
|
|
}
|
|
}
|