mirror of
https://github.com/supabase/supabase.git
synced 2026-10-05 09:25:06 +03:00
Matt Rossman
adf8b0c67c
We're exploring support for newer models like [gpt-5.4-nano](https://openai.com/index/introducing-gpt-5-4-mini-and-nano/) in Assistant. This model doesn't support the `'minimal'` reasoning effort level we use for gpt-5-mini which leads to vague errors. <img width="595" height="263" alt="CleanShot 2026-03-18 at 17 13 05@2x" src="https://github.com/user-attachments/assets/cf7c2370-322d-4a8a-be55-23e680db0aa0" /> Also, we've [previously discussed](https://supabase.slack.com/archives/C0161K73J1J/p1771544464850199?thread_ts=1771493920.775699&cid=C0161K73J1J) that reasoning adds unnecessary latency to otherwise simple AI completion endpoints like `title-v2`. We want more control of reasoning level independent of model/endpoint. This PR aims to solve both problems by: - making reasoning effort configurable on a per-request basis - adding compile-time guardrails to prevent selecting an incompatible reasoning level for models - adding a `DEFAULT_COMPLETION_MODEL` with minimal reasoning that we can update with newer models that support disabling reasoning (independent of Assistant chat model reasoning) Other improvements to our model config logic: - Fixes bug in `onboarding/design.ts` and `assistant.eval.ts` where `providerOptions` was being dropped - `getModel()` now returns a bundled `modelParams` object (spread into AI SDK calls) so `providerOptions` can't be accidentally omitted (this [has happened before](https://supabase.slack.com/archives/C0161K73J1J/p1771518443534309?thread_ts=1771493920.775699&cid=C0161K73J1J)) - Introduces an `ASSISTANT_MODELS` registry as a single source of truth for assistant model config, eliminating hardcoded model IDs across the codebase - Aligns free/pro model conditional logic with `assistant.advance_model` entitlement naming conventions instead of the `isLimited` pattern - Adds `console.error` logging of Assistant stream errors so we can interpret reasoning effort compatibility errors in the future (instead of just opaque "Sorry, I'm having trouble responding right now" card) - Removes unnecessary type casts and generally making the model config logic stricter - Removes pre-existing dead code: `anthropic` provider variant in `GetModelParams` / `PROVIDERS` registry that was never implemented in `getModel()` Now if you try to select an unsupported reasoning level you get a type error: <img width="1306" height="320" alt="CleanShot 2026-03-20 at 14 37 24@2x" src="https://github.com/user-attachments/assets/a6ac234b-5ea5-4d81-8e01-ac4be34a0800" /> And if for some reason an invalid reasoning level slips through, you now get a server-side error surfacing the issue: <img width="1268" height="204" alt="CleanShot 2026-03-20 at 14 58 14@2x" src="https://github.com/user-attachments/assets/aadc1b7a-9495-475f-9741-39979bd27cd7" /> I've tested gpt-5 and gpt-5-mini are still working on the staging preview and verified the models were selected properly in Braintrust logs. Both models are available on my Pro test account, and my Free test account shows the Pro upgrade CTA. Closes AI-446 Closes AI-551
100 lines
3.2 KiB
TypeScript
100 lines
3.2 KiB
TypeScript
import { openai } from '@ai-sdk/openai'
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
|
import * as bedrockModule from './bedrock'
|
|
import { getModel } from './model'
|
|
import { DEFAULT_COMPLETION_MODEL, openaiModelEntry } from './model.utils'
|
|
|
|
vi.mock('@ai-sdk/openai', () => ({
|
|
openai: vi.fn(() => 'openai-model'),
|
|
}))
|
|
|
|
vi.mock('./bedrock', async () => ({
|
|
...(await vi.importActual('./bedrock')),
|
|
createRoutedBedrock: vi.fn(() => async (_modelId: string) => 'bedrock-model'),
|
|
checkAwsCredentials: vi.fn(),
|
|
}))
|
|
|
|
describe('getModel', () => {
|
|
const originalEnv = { ...process.env }
|
|
|
|
beforeEach(() => {
|
|
vi.resetAllMocks()
|
|
})
|
|
|
|
afterEach(() => {
|
|
process.env = { ...originalEnv }
|
|
})
|
|
|
|
it('returns bedrock model without promptProviderOptions', async () => {
|
|
vi.mocked(bedrockModule.checkAwsCredentials).mockResolvedValue(true)
|
|
vi.stubEnv('AWS_BEDROCK_ROLE_ARN', 'test')
|
|
|
|
const { modelParams, error, promptProviderOptions } = await getModel({
|
|
provider: 'bedrock',
|
|
routingKey: 'test',
|
|
})
|
|
|
|
expect(modelParams?.model).toEqual('bedrock-model')
|
|
expect(promptProviderOptions).toBeUndefined()
|
|
expect(error).toBeUndefined()
|
|
})
|
|
|
|
it('returns error when bedrock credentials are not available', async () => {
|
|
vi.mocked(bedrockModule.checkAwsCredentials).mockResolvedValue(false)
|
|
|
|
const { error } = await getModel({ provider: 'bedrock', routingKey: 'test' })
|
|
expect(error).toBeDefined()
|
|
})
|
|
|
|
it('returns openai model with default model', async () => {
|
|
vi.stubEnv('OPENAI_API_KEY', 'test-key')
|
|
|
|
const { modelParams, promptProviderOptions } = await getModel({
|
|
provider: 'openai',
|
|
modelEntry: openaiModelEntry({ id: 'gpt-5-mini' }),
|
|
})
|
|
|
|
expect(modelParams?.model).toEqual('openai-model')
|
|
expect(openai).toHaveBeenCalledWith('gpt-5-mini')
|
|
expect(promptProviderOptions).toBeUndefined()
|
|
})
|
|
|
|
it('returns error when OPENAI_API_KEY is not available', async () => {
|
|
vi.stubEnv('OPENAI_API_KEY', '')
|
|
|
|
const { error } = await getModel({
|
|
provider: 'openai',
|
|
modelEntry: openaiModelEntry({ id: 'gpt-5-mini' }),
|
|
})
|
|
expect(error).toEqual(new Error('OPENAI_API_KEY not available'))
|
|
})
|
|
|
|
it('returns openai gpt-5 when hasAccessToAdvanceModel and not throttled', async () => {
|
|
vi.stubEnv('OPENAI_API_KEY', 'test-key')
|
|
vi.stubEnv('IS_THROTTLED', 'false')
|
|
|
|
const { modelParams, error } = await getModel({
|
|
provider: 'openai',
|
|
modelEntry: openaiModelEntry({ id: 'gpt-5', reasoningEffort: 'minimal' }),
|
|
})
|
|
|
|
expect(error).toBeUndefined()
|
|
expect(modelParams?.model).toEqual('openai-model')
|
|
expect(openai).toHaveBeenCalledWith('gpt-5')
|
|
expect(modelParams?.providerOptions?.openai?.reasoningEffort).toBe('minimal')
|
|
})
|
|
|
|
it('applies reasoningEffort from DEFAULT_COMPLETION_MODEL', async () => {
|
|
vi.stubEnv('OPENAI_API_KEY', 'test-key')
|
|
|
|
const { modelParams, error } = await getModel({
|
|
provider: 'openai',
|
|
modelEntry: DEFAULT_COMPLETION_MODEL,
|
|
})
|
|
|
|
expect(error).toBeUndefined()
|
|
expect(openai).toHaveBeenCalledWith('gpt-5-mini')
|
|
expect(modelParams?.providerOptions?.openai?.reasoningEffort).toBe('minimal')
|
|
})
|
|
})
|