fix: harden combo fallback and health checks (#704)

docs/ARCHITECTURE.md

@@ -356,7 +356,7 @@ flowchart TD
     Q -- No --> R[Return all unavailable]
 ```
 
-Fallback decisions are driven by `open-sse/services/accountFallback.ts` using status codes and error-message heuristics.
+Fallback decisions are driven by `open-sse/services/accountFallback.ts` using status codes and error-message heuristics. Combo routing adds one extra guard: provider-scoped 400s such as upstream content-block and role-validation failures are treated as model-local failures so later combo targets can still run.
 
 ## OAuth Onboarding and Token Refresh Lifecycle
 

open-sse/services/combo.ts

@@ -20,6 +20,15 @@ import { supportsToolCalling } from "./modelCapabilities.ts";
 
 // Status codes that should mark semaphore + record circuit breaker failures
 const TRANSIENT_FOR_BREAKER = [429, 502, 503, 504];
+const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [
+  /\bprohibited_content\b/i,
+  /request blocked by .*api/i,
+  /provided message roles? is not valid/i,
+  /unsupported .*message role/i,
+  /no such tool available/i,
+  /unsupported content part type/i,
+  /tool(?:_call|_use)? .* not (?:available|found)/i,
+];
 
 const MAX_COMBO_DEPTH = 3;
 
@@ -258,6 +267,12 @@ function extractPromptForIntent(body) {
   return "";
 }
 
+export function shouldFallbackComboBadRequest(status, errorText) {
+  if (status !== 400 || !errorText) return false;
+  const message = String(errorText);
+  return COMBO_BAD_REQUEST_FALLBACK_PATTERNS.some((pattern) => pattern.test(message));
+}
+
 function mapIntentToTaskType(intent) {
   switch (intent) {
     case "code":
@@ -890,17 +905,25 @@ export async function handleComboChat({
         provider,
         result.headers
       );
+      const comboBadRequestFallback = shouldFallbackComboBadRequest(result.status, errorText);
 
       // Record failure in circuit breaker for transient errors
       if (TRANSIENT_FOR_BREAKER.includes(result.status)) {
         breaker._onFailure();
       }
 
-      if (!shouldFallback) {
+      if (!shouldFallback && !comboBadRequestFallback) {
         log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status });
         return result;
       }
 
+      if (comboBadRequestFallback) {
+        log.info(
+          "COMBO",
+          `Treating provider-scoped 400 from ${modelStr} as model-local failure; trying next combo target`
+        );
+      }
+
       // Check if this is a transient error worth retrying on same model
       const isTransient = [408, 429, 500, 502, 503, 504].includes(result.status);
       if (retry < maxRetries && isTransient) {
@@ -1146,6 +1169,7 @@ async function handleRoundRobinCombo({
           provider,
           result.headers
         );
+        const comboBadRequestFallback = shouldFallbackComboBadRequest(result.status, errorText);
 
         // Transient errors → mark in semaphore AND record circuit breaker failure
         if (TRANSIENT_FOR_BREAKER.includes(result.status) && cooldownMs > 0) {
@@ -1157,11 +1181,18 @@ async function handleRoundRobinCombo({
           );
         }
 
-        if (!shouldFallback) {
+        if (!shouldFallback && !comboBadRequestFallback) {
           log.warn("COMBO-RR", `${modelStr} failed (no fallback)`, { status: result.status });
           return result;
         }
 
+        if (comboBadRequestFallback) {
+          log.info(
+            "COMBO-RR",
+            `Treating provider-scoped 400 from ${modelStr} as model-local failure; trying next model`
+          );
+        }
+
         // Transient error → retry same model
         const isTransient = [408, 429, 500, 502, 503, 504].includes(result.status);
         if (retry < maxRetries && isTransient) {

src/app/(dashboard)/dashboard/combos/page.tsx

@@ -1122,18 +1122,27 @@ function TestResultsView({ results }) {
       {results.results?.map((r, i) => (
         <div
           key={i}
+          title={r.error || undefined}
           className="flex items-center gap-2 text-xs px-2 py-1.5 rounded bg-black/[0.02] dark:bg-white/[0.02]"
         >
           <span
             className={`material-symbols-outlined text-[14px] ${
               r.status === "ok"
                 ? "text-emerald-500"
-                : r.status === "skipped"
-                  ? "text-text-muted"
-                  : "text-red-500"
+                : r.status === "reachable"
+                  ? "text-amber-500"
+                  : r.status === "skipped"
+                    ? "text-text-muted"
+                    : "text-red-500"
             }`}
           >
-            {r.status === "ok" ? "check_circle" : r.status === "skipped" ? "skip_next" : "error"}
+            {r.status === "ok"
+              ? "check_circle"
+              : r.status === "reachable"
+                ? "network_check"
+                : r.status === "skipped"
+                  ? "skip_next"
+                  : "error"}
           </span>
           <code className="font-mono flex-1">{r.model}</code>
           {r.latencyMs !== undefined && <span className="text-text-muted">{r.latencyMs}ms</span>}
@@ -1141,9 +1150,11 @@ function TestResultsView({ results }) {
             className={`text-[10px] uppercase font-medium ${
               r.status === "ok"
                 ? "text-emerald-500"
-                : r.status === "skipped"
-                  ? "text-text-muted"
-                  : "text-red-500"
+                : r.status === "reachable"
+                  ? "text-amber-500"
+                  : r.status === "skipped"
+                    ? "text-text-muted"
+                    : "text-red-500"
             }`}
           >
             {r.status}

src/app/api/combos/test/route.ts

@@ -1,4 +1,9 @@
 import { NextResponse } from "next/server";
+import {
+  buildComboTestRequestBody,
+  probeComboModelReachability,
+  shouldProbeComboTestReachability,
+} from "@/lib/combos/testHealth";
 import { getComboByName } from "@/lib/localDb";
 import { testComboSchema } from "@/shared/validation/schemas";
 import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
@@ -49,13 +54,9 @@ export async function POST(request) {
       const startTime = Date.now();
       try {
         // Send a minimal chat request to the internal SSE handler
-        // Use OpenAI-compatible format — universally accepted by all providers via the translator
-        const testBody = {
-          model: modelStr,
-          messages: [{ role: "user", content: "Hi" }],
-          max_tokens: 5,
-          stream: false,
-        };
+        // Use a tiny but realistic request body so gateway-routed models do not
+        // get flagged as dead just because the probe payload is too synthetic.
+        const testBody = buildComboTestRequestBody(modelStr);
 
         const internalUrl = `${getBaseUrl(request)}/v1/chat/completions`;
         const controller = new AbortController();
@@ -88,6 +89,29 @@ export async function POST(request) {
           } catch {
             errorMsg = res.statusText;
           }
+
+          let reachability = null;
+          if (shouldProbeComboTestReachability(res.status)) {
+            try {
+              reachability = await probeComboModelReachability(modelStr);
+            } catch {
+              reachability = null;
+            }
+          }
+
+          if (reachability?.reachable) {
+            results.push({
+              model: modelStr,
+              status: "reachable",
+              statusCode: res.status,
+              error: errorMsg,
+              latencyMs,
+              provider: reachability.provider,
+              probeMethod: reachability.method,
+            });
+            continue;
+          }
+
           results.push({
             model: modelStr,
             status: "error",

src/lib/combos/testHealth.ts

@@ -0,0 +1,74 @@
+import { validateProviderApiKey } from "@/lib/providers/validation";
+import { getProviderCredentials } from "@/sse/services/auth";
+import { getModelInfo } from "@/sse/services/model";
+
+const SOFT_REACHABILITY_STATUSES = new Set([400, 405, 406, 409, 422]);
+
+export function buildComboTestRequestBody(modelStr: string) {
+  return {
+    model: modelStr,
+    messages: [{ role: "user", content: "Reply with OK only." }],
+    // Some gateway-routed models reject ultra-tiny budgets during smoke tests.
+    max_tokens: 16,
+    stream: false,
+  };
+}
+
+export function shouldProbeComboTestReachability(statusCode: number) {
+  return SOFT_REACHABILITY_STATUSES.has(Number(statusCode));
+}
+
+type ProbeDeps = {
+  getModelInfo?: typeof getModelInfo;
+  getProviderCredentials?: typeof getProviderCredentials;
+  validateProviderApiKey?: typeof validateProviderApiKey;
+};
+
+export async function probeComboModelReachability(modelStr: string, deps: ProbeDeps = {}) {
+  const resolveModel = deps.getModelInfo || getModelInfo;
+  const loadCredentials = deps.getProviderCredentials || getProviderCredentials;
+  const validateKey = deps.validateProviderApiKey || validateProviderApiKey;
+
+  const modelInfo = await resolveModel(modelStr);
+  if (!modelInfo?.provider) {
+    return { reachable: false, reason: "unresolved_model" };
+  }
+
+  const credentials = await loadCredentials(
+    modelInfo.provider,
+    null,
+    null,
+    modelInfo.model || modelStr
+  );
+  if (!credentials || credentials.allRateLimited) {
+    return { reachable: false, reason: "credentials_unavailable" };
+  }
+
+  const apiKey = credentials.apiKey || credentials.accessToken;
+  if (typeof apiKey !== "string" || apiKey.trim().length === 0) {
+    return { reachable: false, reason: "missing_auth_material" };
+  }
+
+  const providerSpecificData =
+    credentials.providerSpecificData && typeof credentials.providerSpecificData === "object"
+      ? { ...credentials.providerSpecificData }
+      : {};
+
+  if (!providerSpecificData.validationModelId && modelInfo.model) {
+    providerSpecificData.validationModelId = modelInfo.model;
+  }
+
+  const validation = await validateKey({
+    provider: modelInfo.provider,
+    apiKey,
+    providerSpecificData,
+  });
+
+  return {
+    reachable: Boolean(validation?.valid),
+    provider: modelInfo.provider,
+    model: modelInfo.model || null,
+    method: validation?.method || null,
+    warning: validation?.warning || null,
+  };
+}

tests/unit/combo-test-health.test.mjs

@@ -0,0 +1,51 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { buildComboTestRequestBody, shouldProbeComboTestReachability, probeComboModelReachability } =
+  await import("../../src/lib/combos/testHealth.ts");
+
+test("combo test helper builds a realistic smoke payload", () => {
+  const body = buildComboTestRequestBody("openrouter/openai/gpt-5.4");
+
+  assert.equal(body.model, "openrouter/openai/gpt-5.4");
+  assert.equal(body.messages[0].content, "Reply with OK only.");
+  assert.equal(body.max_tokens, 16);
+  assert.equal(body.stream, false);
+});
+
+test("combo test helper probes only soft 4xx responses", () => {
+  assert.equal(shouldProbeComboTestReachability(400), true);
+  assert.equal(shouldProbeComboTestReachability(422), true);
+  assert.equal(shouldProbeComboTestReachability(401), false);
+  assert.equal(shouldProbeComboTestReachability(404), false);
+  assert.equal(shouldProbeComboTestReachability(429), false);
+});
+
+test("combo reachability probe reuses resolved provider credentials and model id", async () => {
+  let validationInput = null;
+
+  const result = await probeComboModelReachability("openrouter/openai/gpt-5.4", {
+    getModelInfo: async () => ({ provider: "openrouter", model: "openai/gpt-5.4" }),
+    getProviderCredentials: async () => ({
+      apiKey: "test-key",
+      providerSpecificData: { baseUrl: "https://openrouter.ai/api/v1" },
+    }),
+    validateProviderApiKey: async (input) => {
+      validationInput = input;
+      return { valid: true, method: "models_endpoint" };
+    },
+  });
+
+  assert.equal(result.reachable, true);
+  assert.equal(result.provider, "openrouter");
+  assert.equal(result.model, "openai/gpt-5.4");
+  assert.equal(result.method, "models_endpoint");
+  assert.deepEqual(validationInput, {
+    provider: "openrouter",
+    apiKey: "test-key",
+    providerSpecificData: {
+      baseUrl: "https://openrouter.ai/api/v1",
+      validationModelId: "openai/gpt-5.4",
+    },
+  });
+});

tests/unit/t23-t24-fallback-resilience.test.mjs

@@ -2,7 +2,8 @@ import test from "node:test";
 import assert from "node:assert/strict";
 
 const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts");
-const { handleComboChat } = await import("../../open-sse/services/combo.ts");
+const { handleComboChat, shouldFallbackComboBadRequest } =
+  await import("../../open-sse/services/combo.ts");
 const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
 
 test.beforeEach(() => {
@@ -139,3 +140,43 @@ test("T24: all inactive accounts return 503 service_unavailable (not 406)", asyn
   const body = await result.json();
   assert.equal(body.error?.code, "ALL_ACCOUNTS_INACTIVE");
 });
+
+test("combo falls through provider-scoped 400s and reaches the next model", async () => {
+  const log = createLog();
+
+  const result = await handleComboChat({
+    body: {},
+    combo: {
+      name: "t24-provider-scoped-400",
+      strategy: "priority",
+      models: [
+        { model: "free/gemini-3.1-pro-preview", weight: 0 },
+        { model: "aio/gemini-3.1-pro-preview-thinking-high", weight: 0 },
+        { model: "openrouter/google/gemini-3.1-pro-preview", weight: 0 },
+      ],
+    },
+    handleSingleModel: createStatusSequenceHandler([
+      { status: 429, message: "No capacity available for model gemini-3.1-pro-preview" },
+      { status: 400, message: "request blocked by Gemini API: PROHIBITED_CONTENT" },
+      { status: 200 },
+    ]),
+    isModelAvailable: () => true,
+    log,
+    settings: null,
+    allCombos: null,
+  });
+
+  assert.equal(result.ok, true);
+  const badRequestLog = log.entries.find((entry) => entry.msg.includes("provider-scoped 400"));
+  assert.ok(badRequestLog);
+});
+
+test("combo bad-request fallback helper keeps generic 400s terminal", () => {
+  assert.equal(shouldFallbackComboBadRequest(400, "request blocked by Gemini API"), true);
+  assert.equal(
+    shouldFallbackComboBadRequest(400, "One or more of the provided message roles is not valid"),
+    true
+  );
+  assert.equal(shouldFallbackComboBadRequest(400, "bad request"), false);
+  assert.equal(shouldFallbackComboBadRequest(422, "request blocked by Gemini API"), false);
+});