From 94eb7b155e53287d4c243fbcc0a34f85b116de94 Mon Sep 17 00:00:00 2001 From: "R.D." Date: Thu, 2 Apr 2026 03:12:09 -0400 Subject: [PATCH] refactor: add shared request timeout baseline --- .env.example | 4 ++- README.md | 38 +++++++++++++++++----------- src/shared/utils/runtimeTimeouts.ts | 38 +++++++++++++++++++++++----- tests/unit/runtime-timeouts.test.mjs | 17 +++++++++++++ 4 files changed, 74 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index 757984d8..5ff8b6ac 100644 --- a/.env.example +++ b/.env.example @@ -189,8 +189,10 @@ GEMINI_CLI_USER_AGENT=google-api-nodejs-client/9.15.1 # Provider keys above (openai, mistral, together, fireworks, nvidia) also work for embeddings # Timeout settings -# FETCH_TIMEOUT_MS=600000 +# REQUEST_TIMEOUT_MS=600000 # STREAM_IDLE_TIMEOUT_MS=600000 +# Advanced timeout overrides (optional) +# FETCH_TIMEOUT_MS=600000 # FETCH_HEADERS_TIMEOUT_MS=600000 # FETCH_BODY_TIMEOUT_MS=600000 # FETCH_CONNECT_TIMEOUT_MS=30000 diff --git a/README.md b/README.md index cc31bef3..e8c7d5f4 100644 --- a/README.md +++ b/README.md @@ -805,22 +805,30 @@ PORT=20128 DASHBOARD_PORT=20129 omniroute ### Long-Running Streaming Timeouts -For long reasoning models or slow upstream streams, tune these env vars: +For most deployments, you only need: -| Variable | Default | Purpose | -| ---------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------- | -| `FETCH_TIMEOUT_MS` | `600000` | Total upstream request timeout used by the main fetch abort signal | -| `STREAM_IDLE_TIMEOUT_MS` | `600000` | Maximum gap between streaming chunks before OmniRoute aborts the SSE stream | -| `FETCH_HEADERS_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit for receiving upstream response headers | -| `FETCH_BODY_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit between upstream body chunks (`0` disables it) | -| `FETCH_CONNECT_TIMEOUT_MS` | `30000` | Undici TCP connect timeout | -| `FETCH_KEEPALIVE_TIMEOUT_MS` | `4000` | Undici idle keep-alive socket timeout | -| `TLS_CLIENT_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Timeout for TLS fingerprint requests made through `wreq-js` | -| `API_BRIDGE_PROXY_TIMEOUT_MS` | `30000` | Timeout for `/v1` proxy forwarding from API port to dashboard port | -| `API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS` | `max(API_BRIDGE_PROXY_TIMEOUT_MS, 300000)` | Incoming request timeout on the API bridge server | -| `API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS` | `60000` | Incoming header timeout on the API bridge server | -| `API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS` | `5000` | Keep-alive timeout on the API bridge server | -| `API_BRIDGE_SERVER_SOCKET_TIMEOUT_MS` | `0` | Socket inactivity timeout on the API bridge server (`0` disables it) | +| Variable | Default | Purpose | +| ------------------------ | ----------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `REQUEST_TIMEOUT_MS` | `600000` | Shared baseline for upstream fetch, hidden Undici timeouts, TLS fingerprint requests, and API bridge request/proxy timeouts | +| `STREAM_IDLE_TIMEOUT_MS` | inherits `REQUEST_TIMEOUT_MS` | Maximum gap between streaming chunks before OmniRoute aborts the SSE stream | + +Backward compatibility is preserved: existing `FETCH_TIMEOUT_MS`, `API_BRIDGE_PROXY_TIMEOUT_MS`, and other per-layer timeout vars still work and override the shared baseline. + +Advanced overrides are available if you need finer control: + +| Variable | Default | Purpose | +| ---------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------- | +| `FETCH_TIMEOUT_MS` | inherits `REQUEST_TIMEOUT_MS` | Total upstream request timeout used by the main fetch abort signal | +| `FETCH_HEADERS_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit for receiving upstream response headers | +| `FETCH_BODY_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit between upstream body chunks (`0` disables it) | +| `FETCH_CONNECT_TIMEOUT_MS` | `30000` | Undici TCP connect timeout | +| `FETCH_KEEPALIVE_TIMEOUT_MS` | `4000` | Undici idle keep-alive socket timeout | +| `TLS_CLIENT_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Timeout for TLS fingerprint requests made through `wreq-js` | +| `API_BRIDGE_PROXY_TIMEOUT_MS` | inherits `REQUEST_TIMEOUT_MS` or `30000` | Timeout for `/v1` proxy forwarding from API port to dashboard port | +| `API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS` | `max(API_BRIDGE_PROXY_TIMEOUT_MS, 300000)` | Incoming request timeout on the API bridge server | +| `API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS` | `60000` | Incoming header timeout on the API bridge server | +| `API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS` | `5000` | Keep-alive timeout on the API bridge server | +| `API_BRIDGE_SERVER_SOCKET_TIMEOUT_MS` | `0` | Socket inactivity timeout on the API bridge server (`0` disables it) | If you run OmniRoute behind Nginx, Caddy, Cloudflare, or another reverse proxy, make sure the proxy timeouts are also higher than your OmniRoute stream/fetch timeouts. diff --git a/src/shared/utils/runtimeTimeouts.ts b/src/shared/utils/runtimeTimeouts.ts index 585b5eaa..9d5e3979 100644 --- a/src/shared/utils/runtimeTimeouts.ts +++ b/src/shared/utils/runtimeTimeouts.ts @@ -16,6 +16,11 @@ export const DEFAULT_API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS = 60_000; export const DEFAULT_API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS = 5_000; export const DEFAULT_API_BRIDGE_SERVER_SOCKET_TIMEOUT_MS = 0; +function hasEnvValue(env: EnvSource, name: string): boolean { + const raw = env[name]; + return raw != null && raw.trim() !== ""; +} + export type UpstreamTimeoutConfig = { fetchTimeoutMs: number; streamIdleTimeoutMs: number; @@ -60,14 +65,25 @@ export function getUpstreamTimeoutConfig( env: EnvSource = process.env, logger?: TimeoutLogger ): UpstreamTimeoutConfig { - const fetchTimeoutMs = readTimeoutMs(env, "FETCH_TIMEOUT_MS", DEFAULT_FETCH_TIMEOUT_MS, { - allowZero: true, - logger, - }); + const sharedRequestTimeoutMs = hasEnvValue(env, "REQUEST_TIMEOUT_MS") + ? readTimeoutMs(env, "REQUEST_TIMEOUT_MS", DEFAULT_FETCH_TIMEOUT_MS, { + allowZero: true, + logger, + }) + : undefined; + const fetchTimeoutMs = readTimeoutMs( + env, + "FETCH_TIMEOUT_MS", + sharedRequestTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS, + { + allowZero: true, + logger, + } + ); const streamIdleTimeoutMs = readTimeoutMs( env, "STREAM_IDLE_TIMEOUT_MS", - DEFAULT_STREAM_IDLE_TIMEOUT_MS, + sharedRequestTimeoutMs ?? DEFAULT_STREAM_IDLE_TIMEOUT_MS, { allowZero: true, logger, @@ -123,10 +139,16 @@ export function getApiBridgeTimeoutConfig( env: EnvSource = process.env, logger?: TimeoutLogger ): ApiBridgeTimeoutConfig { + const sharedRequestTimeoutMs = hasEnvValue(env, "REQUEST_TIMEOUT_MS") + ? readTimeoutMs(env, "REQUEST_TIMEOUT_MS", DEFAULT_FETCH_TIMEOUT_MS, { + allowZero: true, + logger, + }) + : undefined; const proxyTimeoutMs = readTimeoutMs( env, "API_BRIDGE_PROXY_TIMEOUT_MS", - DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS, + sharedRequestTimeoutMs ?? DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS, { allowZero: true, logger, @@ -160,7 +182,9 @@ export function getApiBridgeTimeoutConfig( serverRequestTimeoutMs: readTimeoutMs( env, "API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS", - derivedRequestTimeoutMs, + sharedRequestTimeoutMs + ? Math.max(sharedRequestTimeoutMs, derivedRequestTimeoutMs) + : derivedRequestTimeoutMs, { allowZero: true, logger, diff --git a/tests/unit/runtime-timeouts.test.mjs b/tests/unit/runtime-timeouts.test.mjs index 4a39c795..8a1c5e6c 100644 --- a/tests/unit/runtime-timeouts.test.mjs +++ b/tests/unit/runtime-timeouts.test.mjs @@ -19,8 +19,25 @@ test("upstream timeout config derives hidden fetch timeouts from FETCH_TIMEOUT_M }); }); +test("REQUEST_TIMEOUT_MS becomes the common timeout baseline when specific overrides are unset", () => { + const upstreamConfig = runtimeTimeouts.getUpstreamTimeoutConfig({ + REQUEST_TIMEOUT_MS: "600000", + }); + const apiBridgeConfig = runtimeTimeouts.getApiBridgeTimeoutConfig({ + REQUEST_TIMEOUT_MS: "600000", + }); + + assert.equal(upstreamConfig.fetchTimeoutMs, 600000); + assert.equal(upstreamConfig.streamIdleTimeoutMs, 600000); + assert.equal(upstreamConfig.fetchHeadersTimeoutMs, 600000); + assert.equal(upstreamConfig.fetchBodyTimeoutMs, 600000); + assert.equal(apiBridgeConfig.proxyTimeoutMs, 600000); + assert.equal(apiBridgeConfig.serverRequestTimeoutMs, 600000); +}); + test("upstream timeout config honors explicit overrides and falls back on invalid values", () => { const config = runtimeTimeouts.getUpstreamTimeoutConfig({ + REQUEST_TIMEOUT_MS: "550000", FETCH_TIMEOUT_MS: "600000", STREAM_IDLE_TIMEOUT_MS: "600000", FETCH_HEADERS_TIMEOUT_MS: "610000",