feat: category in test case metadata

apps/studio/evals/assistant.eval.ts

@@ -22,10 +22,12 @@ Eval('Assistant', {
       {
         input: 'Hello!',
         expected: { textIncludes: 'Hi' },
+        metadata: { category: ['other'] },
       },
       {
         input: 'How do I implement IP address rate limiting?',
         expected: { requiredTools: ['search_docs'] },
+        metadata: { category: ['general_help'] },
       },
       {
         input: 'Check if my project is having issues right now and tell me what to fix first.',
@@ -33,12 +35,14 @@ Eval('Assistant', {
           requiredTools: ['get_advisors', 'get_logs'],
           criteria: 'Response reflects there are RLS issues to fix.',
         },
+        metadata: { category: ['debugging', 'rls_policies'] },
       },
       {
         input: 'Create a new table "foods" with columns for "name" and "color"',
         expected: {
           sqlQuery: stripIndent`CREATE TABLE IF NOT EXISTS public.foods ( id bigserial PRIMARY KEY, name text NOT NULL, color text );`,
         },
+        metadata: { category: ['sql_generation', 'schema_design'] },
       },
     ] satisfies AssistantEvalCase[]
   },

apps/studio/lib/ai/evals/scorer.ts

@@ -16,7 +16,22 @@ export type Expected = {
   textIncludes?: string
 }
 
-export type AssistantEvalCase = EvalCase<Input, Expected, void>
+// Based on categories in the AssistantMessageRatingSubmittedEvent
+export type AssistantEvalCaseCategory =
+  | 'sql_generation'
+  | 'schema_design'
+  | 'rls_policies'
+  | 'edge_functions'
+  | 'database_optimization'
+  | 'debugging'
+  | 'general_help'
+  | 'other'
+
+export type AssistantEvalCaseMetadata = {
+  category?: AssistantEvalCaseCategory[]
+}
+
+export type AssistantEvalCase = EvalCase<Input, Expected, AssistantEvalCaseMetadata>
 
 export const toolUsageScorer: EvalScorer<Input, Output, Expected> = async ({
   output,