test(studio): add list_notebooks eval cases (FE-4086) (#49010)

## I have read the
[CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md)
file.

YES

## What kind of change does this PR introduce?

Feature (test coverage) — first PR of the notebooks-evals plan, covering
FE-4086 (Evals: Assistant can list notebooks).

## What is the current behavior?

`dataset.ts` has zero notebook eval cases. Separately, two small gaps
block writing them: `assistant.eval.ts` never sets `isExplorerEnabled`,
so `NOTEBOOKS_PROMPT` never loads into the eval task's system prompt;
and `list_notebooks`' `inputSchema` has no sort parameter, even though
`getContent` already supports one, so "most recent" isn't answerable.

## What is the new behavior?

- `assistant.eval.ts` passes `isExplorerEnabled: true` so
`NOTEBOOKS_PROMPT` loads during evals.
- `list_notebooks` gains a `sort_by: 'name' | 'inserted_at'` param,
forwarded to `getContent`'s `sort`. Only creation order is exposed,
since the underlying API has no `updated_at` sort key.
- 3 new dataset cases: basic notebook enumeration, sorting by creation
time (`sort_by`/`limit` args), and a nonexistent-notebook case guarding
against hallucinated results.
- A unit test covering `sort_by` forwarding to the content API's query
param.

## Additional context

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added notebook sorting options when listing notebooks, including by
name or insertion date.
* Added evaluation coverage for listing notebooks, finding the newest
notebook, and avoiding fabricated results for nonexistent notebooks.

* **Bug Fixes**
* Ensured selected notebook sorting preferences are correctly applied
when retrieving content.
  * Improved assistant evaluation coverage with Explorer mode enabled.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Charis authored and GitHub committed 2026-08-12 16:50:08 -04:00
1 parent de39dda387
commit 8baaa517d0
4 files changed
+78 -2

No files matched your search

+1
View File
@@ -37,6 +37,7 @@ Eval('Assistant', {
try {
const result = await generateAssistantResponse({
...modelResponse.modelParams,
isExplorerEnabled: true,
messages: [
{
id: '1',
+42
View File
@@ -421,4 +421,46 @@ export const dataset: AssistantEvalCase[] = [
description: 'Warns about irreversible data loss before executing DELETE without WHERE',
},
},
// Notebook cases
{
input: { prompt: 'What notebooks do I have saved?' },
expected: {
requiredTools: ['list_notebooks'],
correctAnswer: 'Mentions both "Auth health check" and "Edge function error triage".',
},
metadata: {
category: ['general_help'],
description: 'Basic notebook enumeration',
},
},
{
input: { prompt: 'Show me my most recently created notebook' },
expected: {
requiredTools: [
{
name: 'list_notebooks',
input: {
sort_by: { equals: 'inserted_at' },
},
},
],
},
metadata: {
category: ['general_help'],
description:
'Sorts by creation time since there is no "updated_at" sort key available from the API',
},
},
{
input: { prompt: "Do I have a notebook called 'Storage cleanup'?" },
expected: {
requiredTools: ['list_notebooks'],
correctAnswer: 'States that no notebook called "Storage cleanup" exists.',
},
metadata: {
category: ['general_help'],
description:
'Guards against inventing a notebook instead of calling the tool and reporting the real, negative result',
},
},
]
@@ -138,6 +138,33 @@ describe('ai/tools/notebook-tools', () => {
cursor: 'next-page',
})
})
it('should forward sort_by as the content API sort_by query param', async () => {
let capturedRequest: Request | undefined
addAPIMock({
method: 'get',
path: '/platform/projects/:ref/content',
response: ({ request }) => {
capturedRequest = request
return HttpResponse.json<GetUserContentResponse>({
cursor: undefined,
data: [],
} as unknown as GetUserContentResponse)
},
})
const tools = getNotebookTools({ projectRef: 'test-project' })
if (!tools.list_notebooks.execute) throw new Error('execute is undefined')
await tools.list_notebooks.execute(
{ limit: 1, sort_by: 'inserted_at' },
{ toolCallId: 'test', messages: [] }
)
const url = new URL(capturedRequest!.url)
expect(url.searchParams.get('sort_by')).toBe('inserted_at')
})
})
describe('get_notebook', () => {
+8 -2
View File
@@ -44,10 +44,16 @@ export const getNotebookTools = (ctx: NotebookToolsContext = {}) => {
.max(100)
.default(20)
.describe('Max number of notebooks to return.'),
sort_by: z
.enum(['name', 'inserted_at'])
.optional()
.describe(
'Field to sort notebooks by. There is no "updated_at" sort — use "inserted_at" for creation order.'
),
}),
execute: async ({ cursor, limit }) => {
execute: async ({ cursor, limit, sort_by }) => {
const { content, cursor: nextCursor } = await getContent(
{ projectRef, type: 'notebook', limit, cursor },
{ projectRef, type: 'notebook', limit, cursor, sort: sort_by },
undefined,
authHeaders
)