diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index c60b379a..5c3312ab 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -950,11 +950,24 @@ export async function handleChatCore({ const executeProviderRequest = async (modelToCall = effectiveModel, allowDedup = false) => { const execute = async () => { - const bodyToSend = + let bodyToSend = translatedBody.model === modelToCall ? translatedBody : { ...translatedBody, model: modelToCall }; + // Inject prompt_cache_key for OpenAI providers if not already set + if ( + targetFormat === FORMATS.OPENAI && + !bodyToSend.prompt_cache_key && + Array.isArray(bodyToSend.messages) + ) { + const { generatePromptCacheKey } = await import("@/lib/promptCache"); + const cacheKey = generatePromptCacheKey(bodyToSend.messages); + if (cacheKey) { + bodyToSend = { ...bodyToSend, prompt_cache_key: cacheKey }; + } + } + const rawResult = await withRateLimit(provider, connectionId, modelToCall, () => executor.execute({ model: modelToCall, @@ -1444,11 +1457,19 @@ export async function handleChatCore({ const cachedTokens = toPositiveNumber( usage.cache_read_input_tokens ?? usage.cached_tokens ?? - ((usage as Record).prompt_tokens_details as Record | undefined)?.cached_tokens + ( + (usage as Record).prompt_tokens_details as + | Record + | undefined + )?.cached_tokens ); const cacheCreationTokens = toPositiveNumber( usage.cache_creation_input_tokens ?? - ((usage as Record).prompt_tokens_details as Record | undefined)?.cache_creation_tokens + ( + (usage as Record).prompt_tokens_details as + | Record + | undefined + )?.cache_creation_tokens ); saveRequestUsage({ @@ -1604,11 +1625,19 @@ export async function handleChatCore({ const cachedTokens = toPositiveNumber( streamUsage.cache_read_input_tokens ?? streamUsage.cached_tokens ?? - ((streamUsage as Record).prompt_tokens_details as Record | undefined)?.cached_tokens + ( + (streamUsage as Record).prompt_tokens_details as + | Record + | undefined + )?.cached_tokens ); const cacheCreationTokens = toPositiveNumber( streamUsage.cache_creation_input_tokens ?? - ((streamUsage as Record).prompt_tokens_details as Record | undefined)?.cache_creation_tokens + ( + (streamUsage as Record).prompt_tokens_details as + | Record + | undefined + )?.cache_creation_tokens ); saveRequestUsage({ diff --git a/open-sse/translator/request/openai-to-gemini.ts b/open-sse/translator/request/openai-to-gemini.ts index 4aa3bba9..8d4d2b40 100644 --- a/open-sse/translator/request/openai-to-gemini.ts +++ b/open-sse/translator/request/openai-to-gemini.ts @@ -52,6 +52,7 @@ type GeminiRequest = { safetySettings: unknown; systemInstruction?: GeminiContent; tools?: Array<{ functionDeclarations: GeminiFunctionDeclaration[] }>; + cachedContent?: string; }; type CloudCodeEnvelope = { @@ -82,6 +83,11 @@ function openaiToGeminiBase(model, body, stream) { safetySettings: DEFAULT_SAFETY_SETTINGS, }; + // Preserve cachedContent if provided by client (for explicit Gemini caching) + if (body.cachedContent) { + result.cachedContent = body.cachedContent; + } + // Generation config if (body.temperature !== undefined) { result.generationConfig.temperature = body.temperature; diff --git a/src/lib/promptCache/index.ts b/src/lib/promptCache/index.ts index f4ee1fd5..2d1446f8 100644 --- a/src/lib/promptCache/index.ts +++ b/src/lib/promptCache/index.ts @@ -1 +1 @@ -export { analyzePrefix, shouldInjectCacheControl } from "./prefixAnalyzer"; +export { analyzePrefix, shouldInjectCacheControl, generatePromptCacheKey } from "./prefixAnalyzer"; diff --git a/src/lib/promptCache/prefixAnalyzer.ts b/src/lib/promptCache/prefixAnalyzer.ts index 43f622c8..6f8e86e1 100644 --- a/src/lib/promptCache/prefixAnalyzer.ts +++ b/src/lib/promptCache/prefixAnalyzer.ts @@ -75,3 +75,11 @@ export function analyzePrefix(messages: Message[]): PrefixAnalysis { export function shouldInjectCacheControl(analysis: PrefixAnalysis, minTokens = 1024): boolean { return analysis.prefixTokens >= minTokens && analysis.confidence >= 0.7; } + +export function generatePromptCacheKey(messages: Message[]): string { + const analysis = analyzePrefix(messages); + if (analysis.prefixHash) { + return `omni-${analysis.prefixHash.slice(0, 32)}`; + } + return ""; +}