diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4ffe181c..539d38b2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -356,7 +356,7 @@ flowchart TD Q -- No --> R[Return all unavailable] ``` -Fallback decisions are driven by `open-sse/services/accountFallback.ts` using status codes and error-message heuristics. +Fallback decisions are driven by `open-sse/services/accountFallback.ts` using status codes and error-message heuristics. Combo routing adds one extra guard: provider-scoped 400s such as upstream content-block and role-validation failures are treated as model-local failures so later combo targets can still run. ## OAuth Onboarding and Token Refresh Lifecycle diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 7074b3bd..044679c6 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -20,6 +20,15 @@ import { supportsToolCalling } from "./modelCapabilities.ts"; // Status codes that should mark semaphore + record circuit breaker failures const TRANSIENT_FOR_BREAKER = [429, 502, 503, 504]; +const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [ + /\bprohibited_content\b/i, + /request blocked by .*api/i, + /provided message roles? is not valid/i, + /unsupported .*message role/i, + /no such tool available/i, + /unsupported content part type/i, + /tool(?:_call|_use)? .* not (?:available|found)/i, +]; const MAX_COMBO_DEPTH = 3; @@ -258,6 +267,12 @@ function extractPromptForIntent(body) { return ""; } +export function shouldFallbackComboBadRequest(status, errorText) { + if (status !== 400 || !errorText) return false; + const message = String(errorText); + return COMBO_BAD_REQUEST_FALLBACK_PATTERNS.some((pattern) => pattern.test(message)); +} + function mapIntentToTaskType(intent) { switch (intent) { case "code": @@ -890,17 +905,25 @@ export async function handleComboChat({ provider, result.headers ); + const comboBadRequestFallback = shouldFallbackComboBadRequest(result.status, errorText); // Record failure in circuit breaker for transient errors if (TRANSIENT_FOR_BREAKER.includes(result.status)) { breaker._onFailure(); } - if (!shouldFallback) { + if (!shouldFallback && !comboBadRequestFallback) { log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status }); return result; } + if (comboBadRequestFallback) { + log.info( + "COMBO", + `Treating provider-scoped 400 from ${modelStr} as model-local failure; trying next combo target` + ); + } + // Check if this is a transient error worth retrying on same model const isTransient = [408, 429, 500, 502, 503, 504].includes(result.status); if (retry < maxRetries && isTransient) { @@ -1146,6 +1169,7 @@ async function handleRoundRobinCombo({ provider, result.headers ); + const comboBadRequestFallback = shouldFallbackComboBadRequest(result.status, errorText); // Transient errors → mark in semaphore AND record circuit breaker failure if (TRANSIENT_FOR_BREAKER.includes(result.status) && cooldownMs > 0) { @@ -1157,11 +1181,18 @@ async function handleRoundRobinCombo({ ); } - if (!shouldFallback) { + if (!shouldFallback && !comboBadRequestFallback) { log.warn("COMBO-RR", `${modelStr} failed (no fallback)`, { status: result.status }); return result; } + if (comboBadRequestFallback) { + log.info( + "COMBO-RR", + `Treating provider-scoped 400 from ${modelStr} as model-local failure; trying next model` + ); + } + // Transient error → retry same model const isTransient = [408, 429, 500, 502, 503, 504].includes(result.status); if (retry < maxRetries && isTransient) { diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index f1291523..f5e4cd21 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -1122,18 +1122,27 @@ function TestResultsView({ results }) { {results.results?.map((r, i) => (
- {r.status === "ok" ? "check_circle" : r.status === "skipped" ? "skip_next" : "error"} + {r.status === "ok" + ? "check_circle" + : r.status === "reachable" + ? "network_check" + : r.status === "skipped" + ? "skip_next" + : "error"} {r.model} {r.latencyMs !== undefined && {r.latencyMs}ms} @@ -1141,9 +1150,11 @@ function TestResultsView({ results }) { className={`text-[10px] uppercase font-medium ${ r.status === "ok" ? "text-emerald-500" - : r.status === "skipped" - ? "text-text-muted" - : "text-red-500" + : r.status === "reachable" + ? "text-amber-500" + : r.status === "skipped" + ? "text-text-muted" + : "text-red-500" }`} > {r.status} diff --git a/src/app/api/combos/test/route.ts b/src/app/api/combos/test/route.ts index 3506acb4..726f8fa6 100644 --- a/src/app/api/combos/test/route.ts +++ b/src/app/api/combos/test/route.ts @@ -1,4 +1,9 @@ import { NextResponse } from "next/server"; +import { + buildComboTestRequestBody, + probeComboModelReachability, + shouldProbeComboTestReachability, +} from "@/lib/combos/testHealth"; import { getComboByName } from "@/lib/localDb"; import { testComboSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; @@ -49,13 +54,9 @@ export async function POST(request) { const startTime = Date.now(); try { // Send a minimal chat request to the internal SSE handler - // Use OpenAI-compatible format — universally accepted by all providers via the translator - const testBody = { - model: modelStr, - messages: [{ role: "user", content: "Hi" }], - max_tokens: 5, - stream: false, - }; + // Use a tiny but realistic request body so gateway-routed models do not + // get flagged as dead just because the probe payload is too synthetic. + const testBody = buildComboTestRequestBody(modelStr); const internalUrl = `${getBaseUrl(request)}/v1/chat/completions`; const controller = new AbortController(); @@ -88,6 +89,29 @@ export async function POST(request) { } catch { errorMsg = res.statusText; } + + let reachability = null; + if (shouldProbeComboTestReachability(res.status)) { + try { + reachability = await probeComboModelReachability(modelStr); + } catch { + reachability = null; + } + } + + if (reachability?.reachable) { + results.push({ + model: modelStr, + status: "reachable", + statusCode: res.status, + error: errorMsg, + latencyMs, + provider: reachability.provider, + probeMethod: reachability.method, + }); + continue; + } + results.push({ model: modelStr, status: "error", diff --git a/src/lib/combos/testHealth.ts b/src/lib/combos/testHealth.ts new file mode 100644 index 00000000..997de666 --- /dev/null +++ b/src/lib/combos/testHealth.ts @@ -0,0 +1,74 @@ +import { validateProviderApiKey } from "@/lib/providers/validation"; +import { getProviderCredentials } from "@/sse/services/auth"; +import { getModelInfo } from "@/sse/services/model"; + +const SOFT_REACHABILITY_STATUSES = new Set([400, 405, 406, 409, 422]); + +export function buildComboTestRequestBody(modelStr: string) { + return { + model: modelStr, + messages: [{ role: "user", content: "Reply with OK only." }], + // Some gateway-routed models reject ultra-tiny budgets during smoke tests. + max_tokens: 16, + stream: false, + }; +} + +export function shouldProbeComboTestReachability(statusCode: number) { + return SOFT_REACHABILITY_STATUSES.has(Number(statusCode)); +} + +type ProbeDeps = { + getModelInfo?: typeof getModelInfo; + getProviderCredentials?: typeof getProviderCredentials; + validateProviderApiKey?: typeof validateProviderApiKey; +}; + +export async function probeComboModelReachability(modelStr: string, deps: ProbeDeps = {}) { + const resolveModel = deps.getModelInfo || getModelInfo; + const loadCredentials = deps.getProviderCredentials || getProviderCredentials; + const validateKey = deps.validateProviderApiKey || validateProviderApiKey; + + const modelInfo = await resolveModel(modelStr); + if (!modelInfo?.provider) { + return { reachable: false, reason: "unresolved_model" }; + } + + const credentials = await loadCredentials( + modelInfo.provider, + null, + null, + modelInfo.model || modelStr + ); + if (!credentials || credentials.allRateLimited) { + return { reachable: false, reason: "credentials_unavailable" }; + } + + const apiKey = credentials.apiKey || credentials.accessToken; + if (typeof apiKey !== "string" || apiKey.trim().length === 0) { + return { reachable: false, reason: "missing_auth_material" }; + } + + const providerSpecificData = + credentials.providerSpecificData && typeof credentials.providerSpecificData === "object" + ? { ...credentials.providerSpecificData } + : {}; + + if (!providerSpecificData.validationModelId && modelInfo.model) { + providerSpecificData.validationModelId = modelInfo.model; + } + + const validation = await validateKey({ + provider: modelInfo.provider, + apiKey, + providerSpecificData, + }); + + return { + reachable: Boolean(validation?.valid), + provider: modelInfo.provider, + model: modelInfo.model || null, + method: validation?.method || null, + warning: validation?.warning || null, + }; +} diff --git a/tests/unit/combo-test-health.test.mjs b/tests/unit/combo-test-health.test.mjs new file mode 100644 index 00000000..80bed2fd --- /dev/null +++ b/tests/unit/combo-test-health.test.mjs @@ -0,0 +1,51 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { buildComboTestRequestBody, shouldProbeComboTestReachability, probeComboModelReachability } = + await import("../../src/lib/combos/testHealth.ts"); + +test("combo test helper builds a realistic smoke payload", () => { + const body = buildComboTestRequestBody("openrouter/openai/gpt-5.4"); + + assert.equal(body.model, "openrouter/openai/gpt-5.4"); + assert.equal(body.messages[0].content, "Reply with OK only."); + assert.equal(body.max_tokens, 16); + assert.equal(body.stream, false); +}); + +test("combo test helper probes only soft 4xx responses", () => { + assert.equal(shouldProbeComboTestReachability(400), true); + assert.equal(shouldProbeComboTestReachability(422), true); + assert.equal(shouldProbeComboTestReachability(401), false); + assert.equal(shouldProbeComboTestReachability(404), false); + assert.equal(shouldProbeComboTestReachability(429), false); +}); + +test("combo reachability probe reuses resolved provider credentials and model id", async () => { + let validationInput = null; + + const result = await probeComboModelReachability("openrouter/openai/gpt-5.4", { + getModelInfo: async () => ({ provider: "openrouter", model: "openai/gpt-5.4" }), + getProviderCredentials: async () => ({ + apiKey: "test-key", + providerSpecificData: { baseUrl: "https://openrouter.ai/api/v1" }, + }), + validateProviderApiKey: async (input) => { + validationInput = input; + return { valid: true, method: "models_endpoint" }; + }, + }); + + assert.equal(result.reachable, true); + assert.equal(result.provider, "openrouter"); + assert.equal(result.model, "openai/gpt-5.4"); + assert.equal(result.method, "models_endpoint"); + assert.deepEqual(validationInput, { + provider: "openrouter", + apiKey: "test-key", + providerSpecificData: { + baseUrl: "https://openrouter.ai/api/v1", + validationModelId: "openai/gpt-5.4", + }, + }); +}); diff --git a/tests/unit/t23-t24-fallback-resilience.test.mjs b/tests/unit/t23-t24-fallback-resilience.test.mjs index 55b6e22d..db64ac42 100644 --- a/tests/unit/t23-t24-fallback-resilience.test.mjs +++ b/tests/unit/t23-t24-fallback-resilience.test.mjs @@ -2,7 +2,8 @@ import test from "node:test"; import assert from "node:assert/strict"; const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts"); -const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { handleComboChat, shouldFallbackComboBadRequest } = + await import("../../open-sse/services/combo.ts"); const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); test.beforeEach(() => { @@ -139,3 +140,43 @@ test("T24: all inactive accounts return 503 service_unavailable (not 406)", asyn const body = await result.json(); assert.equal(body.error?.code, "ALL_ACCOUNTS_INACTIVE"); }); + +test("combo falls through provider-scoped 400s and reaches the next model", async () => { + const log = createLog(); + + const result = await handleComboChat({ + body: {}, + combo: { + name: "t24-provider-scoped-400", + strategy: "priority", + models: [ + { model: "free/gemini-3.1-pro-preview", weight: 0 }, + { model: "aio/gemini-3.1-pro-preview-thinking-high", weight: 0 }, + { model: "openrouter/google/gemini-3.1-pro-preview", weight: 0 }, + ], + }, + handleSingleModel: createStatusSequenceHandler([ + { status: 429, message: "No capacity available for model gemini-3.1-pro-preview" }, + { status: 400, message: "request blocked by Gemini API: PROHIBITED_CONTENT" }, + { status: 200 }, + ]), + isModelAvailable: () => true, + log, + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + const badRequestLog = log.entries.find((entry) => entry.msg.includes("provider-scoped 400")); + assert.ok(badRequestLog); +}); + +test("combo bad-request fallback helper keeps generic 400s terminal", () => { + assert.equal(shouldFallbackComboBadRequest(400, "request blocked by Gemini API"), true); + assert.equal( + shouldFallbackComboBadRequest(400, "One or more of the provided message roles is not valid"), + true + ); + assert.equal(shouldFallbackComboBadRequest(400, "bad request"), false); + assert.equal(shouldFallbackComboBadRequest(422, "request blocked by Gemini API"), false); +});