From 8baaa517d03fffc7b2f6b498ce2543c33f401d32 Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:50:08 -0400 Subject: [PATCH] test(studio): add list_notebooks eval cases (FE-4086) (#49010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. YES ## What kind of change does this PR introduce? Feature (test coverage) — first PR of the notebooks-evals plan, covering FE-4086 (Evals: Assistant can list notebooks). ## What is the current behavior? `dataset.ts` has zero notebook eval cases. Separately, two small gaps block writing them: `assistant.eval.ts` never sets `isExplorerEnabled`, so `NOTEBOOKS_PROMPT` never loads into the eval task's system prompt; and `list_notebooks`' `inputSchema` has no sort parameter, even though `getContent` already supports one, so "most recent" isn't answerable. ## What is the new behavior? - `assistant.eval.ts` passes `isExplorerEnabled: true` so `NOTEBOOKS_PROMPT` loads during evals. - `list_notebooks` gains a `sort_by: 'name' | 'inserted_at'` param, forwarded to `getContent`'s `sort`. Only creation order is exposed, since the underlying API has no `updated_at` sort key. - 3 new dataset cases: basic notebook enumeration, sorting by creation time (`sort_by`/`limit` args), and a nonexistent-notebook case guarding against hallucinated results. - A unit test covering `sort_by` forwarding to the content API's query param. ## Additional context ## Summary by CodeRabbit * **New Features** * Added notebook sorting options when listing notebooks, including by name or insertion date. * Added evaluation coverage for listing notebooks, finding the newest notebook, and avoiding fabricated results for nonexistent notebooks. * **Bug Fixes** * Ensured selected notebook sorting preferences are correctly applied when retrieving content. * Improved assistant evaluation coverage with Explorer mode enabled. --- apps/studio/evals/assistant.eval.ts | 1 + apps/studio/evals/dataset.ts | 42 +++++++++++++++++++ .../lib/ai/tools/notebook-tools.test.ts | 27 ++++++++++++ apps/studio/lib/ai/tools/notebook-tools.ts | 10 ++++- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/apps/studio/evals/assistant.eval.ts b/apps/studio/evals/assistant.eval.ts index b540b630720..c4500caec4f 100644 --- a/apps/studio/evals/assistant.eval.ts +++ b/apps/studio/evals/assistant.eval.ts @@ -37,6 +37,7 @@ Eval('Assistant', { try { const result = await generateAssistantResponse({ ...modelResponse.modelParams, + isExplorerEnabled: true, messages: [ { id: '1', diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.ts index e82c6064779..ab3871c2d90 100644 --- a/apps/studio/evals/dataset.ts +++ b/apps/studio/evals/dataset.ts @@ -421,4 +421,46 @@ export const dataset: AssistantEvalCase[] = [ description: 'Warns about irreversible data loss before executing DELETE without WHERE', }, }, + // Notebook cases + { + input: { prompt: 'What notebooks do I have saved?' }, + expected: { + requiredTools: ['list_notebooks'], + correctAnswer: 'Mentions both "Auth health check" and "Edge function error triage".', + }, + metadata: { + category: ['general_help'], + description: 'Basic notebook enumeration', + }, + }, + { + input: { prompt: 'Show me my most recently created notebook' }, + expected: { + requiredTools: [ + { + name: 'list_notebooks', + input: { + sort_by: { equals: 'inserted_at' }, + }, + }, + ], + }, + metadata: { + category: ['general_help'], + description: + 'Sorts by creation time since there is no "updated_at" sort key available from the API', + }, + }, + { + input: { prompt: "Do I have a notebook called 'Storage cleanup'?" }, + expected: { + requiredTools: ['list_notebooks'], + correctAnswer: 'States that no notebook called "Storage cleanup" exists.', + }, + metadata: { + category: ['general_help'], + description: + 'Guards against inventing a notebook instead of calling the tool and reporting the real, negative result', + }, + }, ] diff --git a/apps/studio/lib/ai/tools/notebook-tools.test.ts b/apps/studio/lib/ai/tools/notebook-tools.test.ts index 475409dc91d..1231cbd509e 100644 --- a/apps/studio/lib/ai/tools/notebook-tools.test.ts +++ b/apps/studio/lib/ai/tools/notebook-tools.test.ts @@ -138,6 +138,33 @@ describe('ai/tools/notebook-tools', () => { cursor: 'next-page', }) }) + + it('should forward sort_by as the content API sort_by query param', async () => { + let capturedRequest: Request | undefined + + addAPIMock({ + method: 'get', + path: '/platform/projects/:ref/content', + response: ({ request }) => { + capturedRequest = request + return HttpResponse.json({ + cursor: undefined, + data: [], + } as unknown as GetUserContentResponse) + }, + }) + + const tools = getNotebookTools({ projectRef: 'test-project' }) + if (!tools.list_notebooks.execute) throw new Error('execute is undefined') + + await tools.list_notebooks.execute( + { limit: 1, sort_by: 'inserted_at' }, + { toolCallId: 'test', messages: [] } + ) + + const url = new URL(capturedRequest!.url) + expect(url.searchParams.get('sort_by')).toBe('inserted_at') + }) }) describe('get_notebook', () => { diff --git a/apps/studio/lib/ai/tools/notebook-tools.ts b/apps/studio/lib/ai/tools/notebook-tools.ts index 222ec181175..c5e95f3d077 100644 --- a/apps/studio/lib/ai/tools/notebook-tools.ts +++ b/apps/studio/lib/ai/tools/notebook-tools.ts @@ -44,10 +44,16 @@ export const getNotebookTools = (ctx: NotebookToolsContext = {}) => { .max(100) .default(20) .describe('Max number of notebooks to return.'), + sort_by: z + .enum(['name', 'inserted_at']) + .optional() + .describe( + 'Field to sort notebooks by. There is no "updated_at" sort — use "inserted_at" for creation order.' + ), }), - execute: async ({ cursor, limit }) => { + execute: async ({ cursor, limit, sort_by }) => { const { content, cursor: nextCursor } = await getContent( - { projectRef, type: 'notebook', limit, cursor }, + { projectRef, type: 'notebook', limit, cursor, sort: sort_by }, undefined, authHeaders )