diff --git a/apps/studio/evals/assistant.eval.ts b/apps/studio/evals/assistant.eval.ts index b21387a6bbf..a5fedcc9b67 100644 --- a/apps/studio/evals/assistant.eval.ts +++ b/apps/studio/evals/assistant.eval.ts @@ -1,51 +1,22 @@ import { openai } from '@ai-sdk/openai' import { Eval } from 'braintrust' -import { stripIndent } from 'common-tags' import { - AssistantEvalCase, criteriaMetScorer, sqlSimilarityScorer, textIncludesScorer, toolUsageScorer, -} from 'lib/ai/evals/scorer' +} from './scorer' import { generateAssistantResponse } from 'lib/ai/generate-assistant-response' import { getMockTools } from 'lib/ai/tools/mock-tools' import assert from 'node:assert' +import { dataset } from './dataset' assert(process.env.BRAINTRUST_PROJECT_ID, 'BRAINTRUST_PROJECT_ID is not set') assert(process.env.OPENAI_API_KEY, 'OPENAI_API_KEY is not set') Eval('Assistant', { projectId: process.env.BRAINTRUST_PROJECT_ID, - data: () => { - return [ - { - input: 'Hello!', - expected: { textIncludes: 'Hi' }, - metadata: { category: ['other'] }, - }, - { - input: 'How do I implement IP address rate limiting?', - expected: { requiredTools: ['search_docs'] }, - metadata: { category: ['general_help'] }, - }, - { - input: 'Check if my project is having issues right now and tell me what to fix first.', - expected: { - requiredTools: ['get_advisors', 'get_logs'], - criteria: 'Response reflects there are RLS issues to fix.', - }, - metadata: { category: ['debugging', 'rls_policies'] }, - }, - { - input: 'Create a new table "foods" with columns for "name" and "color"', - expected: { - sqlQuery: stripIndent`CREATE TABLE IF NOT EXISTS public.foods ( id bigserial PRIMARY KEY, name text NOT NULL, color text );`, - }, - metadata: { category: ['sql_generation', 'schema_design'] }, - }, - ] satisfies AssistantEvalCase[] - }, + data: () => dataset, task: async (input) => { const result = await generateAssistantResponse({ model: openai('gpt-5-mini'), diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.ts new file mode 100644 index 00000000000..f7c5aa7a3bf --- /dev/null +++ b/apps/studio/evals/dataset.ts @@ -0,0 +1,30 @@ +import { stripIndent } from 'common-tags' +import { AssistantEvalCase } from './scorer' + +export const dataset: AssistantEvalCase[] = [ + { + input: 'Hello!', + expected: { textIncludes: 'Hi' }, + metadata: { category: ['other'] }, + }, + { + input: 'How do I implement IP address rate limiting?', + expected: { requiredTools: ['search_docs'] }, + metadata: { category: ['general_help'] }, + }, + { + input: 'Check if my project is having issues right now and tell me what to fix first.', + expected: { + requiredTools: ['get_advisors', 'get_logs'], + criteria: 'Response reflects there are RLS issues to fix.', + }, + metadata: { category: ['debugging', 'rls_policies'] }, + }, + { + input: 'Create a new table "foods" with columns for "name" and "color"', + expected: { + sqlQuery: stripIndent`CREATE TABLE IF NOT EXISTS public.foods ( id bigserial PRIMARY KEY, name text NOT NULL, color text );`, + }, + metadata: { category: ['sql_generation', 'schema_design'] }, + }, +] diff --git a/apps/studio/lib/ai/evals/scorer.ts b/apps/studio/evals/scorer.ts similarity index 99% rename from apps/studio/lib/ai/evals/scorer.ts rename to apps/studio/evals/scorer.ts index ab7b7b073ab..a59b637861e 100644 --- a/apps/studio/lib/ai/evals/scorer.ts +++ b/apps/studio/evals/scorer.ts @@ -110,3 +110,4 @@ export const textIncludesScorer: EvalScorer = async ({ score: includes ? 1 : 0, } } +