mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 09:25:06 +03:00
test(studio): add list_notebooks eval cases (FE-4086) (#49010)
## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. YES ## What kind of change does this PR introduce? Feature (test coverage) — first PR of the notebooks-evals plan, covering FE-4086 (Evals: Assistant can list notebooks). ## What is the current behavior? `dataset.ts` has zero notebook eval cases. Separately, two small gaps block writing them: `assistant.eval.ts` never sets `isExplorerEnabled`, so `NOTEBOOKS_PROMPT` never loads into the eval task's system prompt; and `list_notebooks`' `inputSchema` has no sort parameter, even though `getContent` already supports one, so "most recent" isn't answerable. ## What is the new behavior? - `assistant.eval.ts` passes `isExplorerEnabled: true` so `NOTEBOOKS_PROMPT` loads during evals. - `list_notebooks` gains a `sort_by: 'name' | 'inserted_at'` param, forwarded to `getContent`'s `sort`. Only creation order is exposed, since the underlying API has no `updated_at` sort key. - 3 new dataset cases: basic notebook enumeration, sorting by creation time (`sort_by`/`limit` args), and a nonexistent-notebook case guarding against hallucinated results. - A unit test covering `sort_by` forwarding to the content API's query param. ## Additional context <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added notebook sorting options when listing notebooks, including by name or insertion date. * Added evaluation coverage for listing notebooks, finding the newest notebook, and avoiding fabricated results for nonexistent notebooks. * **Bug Fixes** * Ensured selected notebook sorting preferences are correctly applied when retrieving content. * Improved assistant evaluation coverage with Explorer mode enabled. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
1 parent
de39dda387
commit
8baaa517d0
4 files changed
+78
-2
No files matched your search
@@ -37,6 +37,7 @@ Eval('Assistant', {
|
||||
try {
|
||||
const result = await generateAssistantResponse({
|
||||
...modelResponse.modelParams,
|
||||
isExplorerEnabled: true,
|
||||
messages: [
|
||||
{
|
||||
id: '1',
|
||||
|
||||
@@ -421,4 +421,46 @@ export const dataset: AssistantEvalCase[] = [
|
||||
description: 'Warns about irreversible data loss before executing DELETE without WHERE',
|
||||
},
|
||||
},
|
||||
// Notebook cases
|
||||
{
|
||||
input: { prompt: 'What notebooks do I have saved?' },
|
||||
expected: {
|
||||
requiredTools: ['list_notebooks'],
|
||||
correctAnswer: 'Mentions both "Auth health check" and "Edge function error triage".',
|
||||
},
|
||||
metadata: {
|
||||
category: ['general_help'],
|
||||
description: 'Basic notebook enumeration',
|
||||
},
|
||||
},
|
||||
{
|
||||
input: { prompt: 'Show me my most recently created notebook' },
|
||||
expected: {
|
||||
requiredTools: [
|
||||
{
|
||||
name: 'list_notebooks',
|
||||
input: {
|
||||
sort_by: { equals: 'inserted_at' },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
metadata: {
|
||||
category: ['general_help'],
|
||||
description:
|
||||
'Sorts by creation time since there is no "updated_at" sort key available from the API',
|
||||
},
|
||||
},
|
||||
{
|
||||
input: { prompt: "Do I have a notebook called 'Storage cleanup'?" },
|
||||
expected: {
|
||||
requiredTools: ['list_notebooks'],
|
||||
correctAnswer: 'States that no notebook called "Storage cleanup" exists.',
|
||||
},
|
||||
metadata: {
|
||||
category: ['general_help'],
|
||||
description:
|
||||
'Guards against inventing a notebook instead of calling the tool and reporting the real, negative result',
|
||||
},
|
||||
},
|
||||
]
|
||||
@@ -138,6 +138,33 @@ describe('ai/tools/notebook-tools', () => {
|
||||
cursor: 'next-page',
|
||||
})
|
||||
})
|
||||
|
||||
it('should forward sort_by as the content API sort_by query param', async () => {
|
||||
let capturedRequest: Request | undefined
|
||||
|
||||
addAPIMock({
|
||||
method: 'get',
|
||||
path: '/platform/projects/:ref/content',
|
||||
response: ({ request }) => {
|
||||
capturedRequest = request
|
||||
return HttpResponse.json<GetUserContentResponse>({
|
||||
cursor: undefined,
|
||||
data: [],
|
||||
} as unknown as GetUserContentResponse)
|
||||
},
|
||||
})
|
||||
|
||||
const tools = getNotebookTools({ projectRef: 'test-project' })
|
||||
if (!tools.list_notebooks.execute) throw new Error('execute is undefined')
|
||||
|
||||
await tools.list_notebooks.execute(
|
||||
{ limit: 1, sort_by: 'inserted_at' },
|
||||
{ toolCallId: 'test', messages: [] }
|
||||
)
|
||||
|
||||
const url = new URL(capturedRequest!.url)
|
||||
expect(url.searchParams.get('sort_by')).toBe('inserted_at')
|
||||
})
|
||||
})
|
||||
|
||||
describe('get_notebook', () => {
|
||||
|
||||
@@ -44,10 +44,16 @@ export const getNotebookTools = (ctx: NotebookToolsContext = {}) => {
|
||||
.max(100)
|
||||
.default(20)
|
||||
.describe('Max number of notebooks to return.'),
|
||||
sort_by: z
|
||||
.enum(['name', 'inserted_at'])
|
||||
.optional()
|
||||
.describe(
|
||||
'Field to sort notebooks by. There is no "updated_at" sort — use "inserted_at" for creation order.'
|
||||
),
|
||||
}),
|
||||
execute: async ({ cursor, limit }) => {
|
||||
execute: async ({ cursor, limit, sort_by }) => {
|
||||
const { content, cursor: nextCursor } = await getContent(
|
||||
{ projectRef, type: 'notebook', limit, cursor },
|
||||
{ projectRef, type: 'notebook', limit, cursor, sort: sort_by },
|
||||
undefined,
|
||||
authHeaders
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user