fix: resolve Gemini CLI 403 project-routing errors and content accumulation
- Remove x-goog-user-project header and executor-level project override
that caused 403 "Cloud Code Private API has not been used in project X"
- Add PROJECT_ROUTE_ERROR classifier type so project-routing 403s don't
permanently ban accounts (keeps accounts active, tracks the error)
- Fix Cloud Code envelope unwrapping for content accumulation in stream.ts
(Cloud Code wraps responses in { response: { candidates: [...] } })
- Extract unwrapGeminiChunk() into streamHelpers.ts with format guard
- Remove _currentModel singleton race condition from GeminiCLIExecutor
- Add handler for PROJECT_ROUTE_ERROR in chatCore.ts
- Add TODO in antigravity.ts about same stale-project risk
- Add 7 unit tests for error classifier and stream unwrap paths
This commit is contained in:
@@ -37,6 +37,9 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
}
|
||||
|
||||
transformRequest(model, body, stream, credentials) {
|
||||
// TODO: Consider removing project override like gemini-cli.ts — stored projectId
|
||||
// can become stale for Cloud Code accounts, causing 403 "has not been used in project X".
|
||||
// Antigravity accounts may have more stable project IDs, but the risk exists.
|
||||
const bodyProjectId = body?.project;
|
||||
const credentialsProjectId = credentials?.projectId;
|
||||
const allowBodyProjectOverride = process.env.OMNIROUTE_ALLOW_BODY_PROJECT_OVERRIDE === "1";
|
||||
|
||||
@@ -2,8 +2,6 @@ import { BaseExecutor } from "./base.ts";
|
||||
import { PROVIDERS, OAUTH_ENDPOINTS } from "../config/constants.ts";
|
||||
|
||||
export class GeminiCLIExecutor extends BaseExecutor {
|
||||
private _currentModel: string = "";
|
||||
|
||||
constructor() {
|
||||
super("gemini-cli", PROVIDERS["gemini-cli"]);
|
||||
}
|
||||
@@ -18,29 +16,19 @@ export class GeminiCLIExecutor extends BaseExecutor {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${credentials.accessToken}`,
|
||||
// Fingerprint headers matching native GeminiCLI client (prevents upstream rejection)
|
||||
"User-Agent": `GeminiCLI/0.31.0/${this._currentModel || "unknown"} (linux; x64)`,
|
||||
"User-Agent": "GeminiCLI/0.31.0/unknown (linux; x64)",
|
||||
"X-Goog-Api-Client": "google-genai-sdk/1.41.0 gl-node/v22.19.0",
|
||||
...(stream && { Accept: "text/event-stream" }),
|
||||
...(credentials?.projectId && { "x-goog-user-project": credentials.projectId }),
|
||||
// NOTE: x-goog-user-project removed — the stored projectId can become stale for
|
||||
// free-tier accounts, causing 403 "Cloud Code Private API has not been used in
|
||||
// project X". The API resolves the correct project from the OAuth token alone.
|
||||
};
|
||||
}
|
||||
|
||||
transformRequest(model, body, stream, credentials) {
|
||||
// Capture model so buildHeaders (called after transformRequest) can include it in User-Agent
|
||||
this._currentModel = model || "";
|
||||
|
||||
const allowBodyProjectOverride = process.env.OMNIROUTE_ALLOW_BODY_PROJECT_OVERRIDE === "1";
|
||||
|
||||
// Default: prefer OAuth-stored projectId. Incoming body.project can be stale
|
||||
// when clients cache older Cloud Code project values.
|
||||
// Opt-in escape hatch: set OMNIROUTE_ALLOW_BODY_PROJECT_OVERRIDE=1.
|
||||
if (allowBodyProjectOverride && body?.project) {
|
||||
return body;
|
||||
}
|
||||
|
||||
if (credentials?.projectId) {
|
||||
body.project = credentials.projectId;
|
||||
}
|
||||
// NOTE: project override removed — the stored projectId can become stale for free-tier
|
||||
// accounts, causing 403 errors. The translator (wrapInCloudCodeEnvelope) handles
|
||||
// project injection; the executor should not re-override with potentially stale data.
|
||||
return body;
|
||||
}
|
||||
|
||||
|
||||
@@ -1250,6 +1250,16 @@ export async function handleChatCore({
|
||||
lastError: message,
|
||||
errorCode: statusCode,
|
||||
});
|
||||
} else if (errorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR) {
|
||||
// Cloud Code 403 with stale project: not a ban, keep account active.
|
||||
await updateProviderConnection(connectionId, {
|
||||
lastErrorType: errorType,
|
||||
lastError: message,
|
||||
errorCode: statusCode,
|
||||
});
|
||||
console.warn(
|
||||
`[provider] Node ${connectionId} project routing error (${statusCode}) — not banning`
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort state update; request flow should continue with fallback handling.
|
||||
|
||||
@@ -7,6 +7,7 @@ export const PROVIDER_ERROR_TYPES = {
|
||||
FORBIDDEN: "forbidden", // 403 — account banned/revoked, disable node
|
||||
SERVER_ERROR: "server_error", // 500/502/503 — retry limited
|
||||
QUOTA_EXHAUSTED: "quota_exhausted", // 402/429/400 + billing signals
|
||||
PROJECT_ROUTE_ERROR: "project_route_error", // 403 + stale project — transient, not a ban
|
||||
};
|
||||
|
||||
function responseBodyToString(responseBody: unknown): string {
|
||||
@@ -49,7 +50,15 @@ export function classifyProviderError(statusCode: number, responseBody: unknown)
|
||||
if (statusCode === 403 && accountDeactivated) {
|
||||
return PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED;
|
||||
}
|
||||
if (statusCode === 403) return PROVIDER_ERROR_TYPES.FORBIDDEN;
|
||||
if (statusCode === 403) {
|
||||
// Cloud Code API returns 403 with "has not been used in project X" when the project
|
||||
// field is wrong or stale. This is a routing/config error, not an account ban.
|
||||
// Classify as project_route_error so the account stays active but the error is tracked.
|
||||
if (bodyStr.includes("has not been used in project")) {
|
||||
return PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR;
|
||||
}
|
||||
return PROVIDER_ERROR_TYPES.FORBIDDEN;
|
||||
}
|
||||
if (statusCode >= 500) return PROVIDER_ERROR_TYPES.SERVER_ERROR;
|
||||
|
||||
return null;
|
||||
|
||||
@@ -339,12 +339,11 @@ export function openaiToGeminiCLIRequest(model, body, stream) {
|
||||
|
||||
// Wrap Gemini CLI format in Cloud Code wrapper
|
||||
function wrapInCloudCodeEnvelope(model, geminiCLI, credentials = null, isAntigravity = false) {
|
||||
// Both Antigravity and Gemini CLI need the project field for the Cloud Code API.
|
||||
// For Gemini CLI, the stored project comes from loadCodeAssist during OAuth.
|
||||
let projectId = credentials?.projectId;
|
||||
|
||||
if (!projectId) {
|
||||
// Graceful fallback: warn instead of hard-throw so the request reaches
|
||||
// the provider and fails with a meaningful provider-side error (#338).
|
||||
// Users who reconnect OAuth will get their real projectId loaded.
|
||||
console.warn(
|
||||
`[OmniRoute] ${isAntigravity ? "Antigravity" : "GeminiCLI"} account is missing projectId. ` +
|
||||
`Attempting request with empty project — reconnect OAuth to resolve.`
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
filterUsageForFormat,
|
||||
COLORS,
|
||||
} from "./usageTracking.ts";
|
||||
import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE } from "./streamHelpers.ts";
|
||||
import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE, unwrapGeminiChunk } from "./streamHelpers.ts";
|
||||
import {
|
||||
createStructuredSSECollector,
|
||||
buildStreamSummaryFromEvents,
|
||||
@@ -531,9 +531,16 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
// Gemini format - may have multiple parts
|
||||
if (parsed.candidates?.[0]?.content?.parts) {
|
||||
for (const part of parsed.candidates[0].content.parts) {
|
||||
// Gemini / Cloud Code format - may have multiple parts
|
||||
// Cloud Code API wraps in { response: { candidates: [...] } }, so unwrap.
|
||||
// Only applies to Gemini-family formats — skip for OpenAI, Claude, etc.
|
||||
const isGeminiFormat =
|
||||
targetFormat === FORMATS.GEMINI ||
|
||||
targetFormat === FORMATS.GEMINI_CLI ||
|
||||
targetFormat === FORMATS.ANTIGRAVITY;
|
||||
const geminiChunk = isGeminiFormat ? unwrapGeminiChunk(parsed) : parsed;
|
||||
if (geminiChunk.candidates?.[0]?.content?.parts) {
|
||||
for (const part of geminiChunk.candidates[0].content.parts) {
|
||||
if (part.text && typeof part.text === "string") {
|
||||
totalContentLength += part.text.length;
|
||||
if (state?.accumulatedContent !== undefined) state.accumulatedContent += part.text;
|
||||
|
||||
@@ -83,6 +83,15 @@ export function hasValuableContent(chunk, format) {
|
||||
return true; // Other formats: keep all chunks
|
||||
}
|
||||
|
||||
/**
|
||||
* Unwrap Cloud Code API envelope from a Gemini response chunk.
|
||||
* The Cloud Code API wraps responses in { response: { candidates: [...] } }
|
||||
* while standard Gemini returns { candidates: [...] } directly.
|
||||
*/
|
||||
export function unwrapGeminiChunk(parsed) {
|
||||
return parsed.candidates ? parsed : parsed.response || parsed;
|
||||
}
|
||||
|
||||
// Fix invalid id (generic or too short)
|
||||
export function fixInvalidId(parsed) {
|
||||
if (parsed.id && (parsed.id === "chat" || parsed.id === "completion" || parsed.id.length < 8)) {
|
||||
|
||||
@@ -33,3 +33,27 @@ test("classifyProviderError: 429 without billing signal => RATE_LIMITED", () =>
|
||||
const result = classifyProviderError(429, { error: { message: "too many requests" } });
|
||||
assert.equal(result, PROVIDER_ERROR_TYPES.RATE_LIMITED);
|
||||
});
|
||||
|
||||
test("classifyProviderError: 403 with 'has not been used in project' => PROJECT_ROUTE_ERROR (transient)", () => {
|
||||
const result = classifyProviderError(403, {
|
||||
error: {
|
||||
message: "Cloud Code Private API has not been used in project 12345 before or it is disabled.",
|
||||
},
|
||||
});
|
||||
assert.equal(result, PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR);
|
||||
});
|
||||
|
||||
test("classifyProviderError: 403 plain => FORBIDDEN (terminal)", () => {
|
||||
const result = classifyProviderError(403, {
|
||||
error: { message: "The caller does not have permission" },
|
||||
});
|
||||
assert.equal(result, PROVIDER_ERROR_TYPES.FORBIDDEN);
|
||||
});
|
||||
|
||||
test("classifyProviderError: 403 with project string as plain string body => PROJECT_ROUTE_ERROR", () => {
|
||||
const body = JSON.stringify({
|
||||
error: { message: "API has not been used in project abc-xyz before" },
|
||||
});
|
||||
const result = classifyProviderError(403, body);
|
||||
assert.equal(result, PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert";
|
||||
import { hasValuableContent } from "../../open-sse/utils/streamHelpers.ts";
|
||||
import { hasValuableContent, unwrapGeminiChunk } from "../../open-sse/utils/streamHelpers.ts";
|
||||
import { FORMATS } from "../../open-sse/translator/formats.ts";
|
||||
|
||||
describe("hasValuableContent", () => {
|
||||
@@ -70,3 +70,42 @@ describe("hasValuableContent", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("unwrapGeminiChunk", () => {
|
||||
it("returns chunk directly when candidates is at top level (standard Gemini)", () => {
|
||||
const chunk = { candidates: [{ content: { parts: [{ text: "Hi" }] } }], usageMetadata: {} };
|
||||
const result = unwrapGeminiChunk(chunk);
|
||||
assert.strictEqual(result, chunk);
|
||||
});
|
||||
|
||||
it("unwraps Cloud Code envelope { response: { candidates: [...] } }", () => {
|
||||
const inner = { candidates: [{ content: { parts: [{ text: "Hello" }] } }] };
|
||||
const chunk = { response: inner, modelVersion: "gemini-2.5-flash" };
|
||||
const result = unwrapGeminiChunk(chunk);
|
||||
assert.strictEqual(result, inner);
|
||||
assert.deepEqual(result.candidates[0].content.parts[0].text, "Hello");
|
||||
});
|
||||
|
||||
it("returns parsed directly when no candidates and no response", () => {
|
||||
const chunk = { someOther: "data" };
|
||||
const result = unwrapGeminiChunk(chunk);
|
||||
assert.strictEqual(result, chunk);
|
||||
});
|
||||
|
||||
it("returns parsed when response exists but is null/undefined", () => {
|
||||
const chunk = { response: null, other: "data" };
|
||||
const result = unwrapGeminiChunk(chunk);
|
||||
assert.strictEqual(result, chunk);
|
||||
});
|
||||
|
||||
it("prefers top-level candidates over response when both exist", () => {
|
||||
const inner = { candidates: [{ content: { parts: [{ text: "inner" }] } }] };
|
||||
const chunk = {
|
||||
candidates: [{ content: { parts: [{ text: "outer" }] } }],
|
||||
response: inner,
|
||||
};
|
||||
const result = unwrapGeminiChunk(chunk);
|
||||
assert.strictEqual(result, chunk);
|
||||
assert.equal(result.candidates[0].content.parts[0].text, "outer");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user