From e3a937edb027b52199c801d594a953b02102e7ca Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:17:14 -0400 Subject: [PATCH] chore(studio): pin online scorer threads to the built-in preprocessor (#50981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins the online scorers' `trace.getThread()` to the built-in `thread` preprocessor, so we can set the Assistant project's default preprocessor to a [custom one for Topics](https://linear.app/supabase/issue/AI-1258/add-a-topics-preprocessor-that-caps-tool-results-in-assistant-traces) without changing scorer input. `getThread()` otherwise [uses the project default](https://github.com/braintrustdata/braintrust-sdk-javascript/blob/cc165a4843805b531645ddb1d27969204aab9ade/js/src/trace.ts#L807). Ref AI-1258 ## Summary by CodeRabbit * **Bug Fixes** * Corrected thread retrieval to use Braintrust’s thread preprocessor, ensuring evaluation traces are processed consistently. --- apps/studio/evals/scorer.test.ts | 2 +- apps/studio/evals/trace-utils.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/studio/evals/scorer.test.ts b/apps/studio/evals/scorer.test.ts index e6571893c21..55ca83411ac 100644 --- a/apps/studio/evals/scorer.test.ts +++ b/apps/studio/evals/scorer.test.ts @@ -70,7 +70,7 @@ describe('scorers with online (null) output', () => { const { trace, getThread } = mockTrace(THREAD_WITH_DOCS_URL) const result = await runUrlValidityScorer(null, trace) - expect(getThread).toHaveBeenCalled() + expect(getThread).toHaveBeenCalledWith({ preprocessor: 'thread' }) expect(result).toMatchObject({ name: 'URL Validity', score: 1, metadata: { urls: [DOCS_URL] } }) vi.unstubAllGlobals() diff --git a/apps/studio/evals/trace-utils.ts b/apps/studio/evals/trace-utils.ts index 3a7aef8ea7f..e3d68b51879 100644 --- a/apps/studio/evals/trace-utils.ts +++ b/apps/studio/evals/trace-utils.ts @@ -223,5 +223,6 @@ export function getThreadPartsFromThread(thread: unknown[]): Transcript { * in a way the offline path isn't). */ export async function getThreadParts(trace: Trace): Promise { - return getThreadPartsFromThread(await trace.getThread()) + // Pinned to not rely on project's default preprocessor, which may be custom for Topics + return getThreadPartsFromThread(await trace.getThread({ preprocessor: 'thread' })) }