Dima/OmniRoute
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: add shared request timeout baseline

Browse Source
This commit is contained in:
R.D.
2026-04-02 03:12:09 -04:00
parent 50f12869bf
commit 94eb7b155e
4 changed files with 74 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+3 -1
View File
@@ -189,8 +189,10 @@ GEMINI_CLI_USER_AGENT=google-api-nodejs-client/9.15.1
# Provider keys above (openai, mistral, together, fireworks, nvidia) also work for embeddings
# Timeout settings
# FETCH_TIMEOUT_MS=600000
# REQUEST_TIMEOUT_MS=600000
# STREAM_IDLE_TIMEOUT_MS=600000
# Advanced timeout overrides (optional)
# FETCH_TIMEOUT_MS=600000
# FETCH_HEADERS_TIMEOUT_MS=600000
# FETCH_BODY_TIMEOUT_MS=600000
# FETCH_CONNECT_TIMEOUT_MS=30000
README.md
+23 -15
View File
@@ -805,22 +805,30 @@ PORT=20128 DASHBOARD_PORT=20129 omniroute
### Long-Running Streaming Timeouts
For long reasoning models or slow upstream streams, tune these env vars:
For most deployments, you only need:
| Variable | Default | Purpose |
| ---------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------- |
| `FETCH_TIMEOUT_MS` | `600000` | Total upstream request timeout used by the main fetch abort signal |
| `STREAM_IDLE_TIMEOUT_MS` | `600000` | Maximum gap between streaming chunks before OmniRoute aborts the SSE stream |
| `FETCH_HEADERS_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit for receiving upstream response headers |
| `FETCH_BODY_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit between upstream body chunks (`0` disables it) |
| `FETCH_CONNECT_TIMEOUT_MS` | `30000` | Undici TCP connect timeout |
| `FETCH_KEEPALIVE_TIMEOUT_MS` | `4000` | Undici idle keep-alive socket timeout |
| `TLS_CLIENT_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Timeout for TLS fingerprint requests made through `wreq-js` |
| `API_BRIDGE_PROXY_TIMEOUT_MS` | `30000` | Timeout for `/v1` proxy forwarding from API port to dashboard port |
| `API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS` | `max(API_BRIDGE_PROXY_TIMEOUT_MS, 300000)` | Incoming request timeout on the API bridge server |
| `API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS` | `60000` | Incoming header timeout on the API bridge server |
| `API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS` | `5000` | Keep-alive timeout on the API bridge server |
| `API_BRIDGE_SERVER_SOCKET_TIMEOUT_MS` | `0` | Socket inactivity timeout on the API bridge server (`0` disables it) |
| Variable | Default | Purpose |
| ------------------------ | ----------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
| `REQUEST_TIMEOUT_MS` | `600000` | Shared baseline for upstream fetch, hidden Undici timeouts, TLS fingerprint requests, and API bridge request/proxy timeouts |
| `STREAM_IDLE_TIMEOUT_MS` | inherits `REQUEST_TIMEOUT_MS` | Maximum gap between streaming chunks before OmniRoute aborts the SSE stream |
Backward compatibility is preserved: existing `FETCH_TIMEOUT_MS`, `API_BRIDGE_PROXY_TIMEOUT_MS`, and other per-layer timeout vars still work and override the shared baseline.
Advanced overrides are available if you need finer control:
| Variable | Default | Purpose |
| ---------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------- |
| `FETCH_TIMEOUT_MS` | inherits `REQUEST_TIMEOUT_MS` | Total upstream request timeout used by the main fetch abort signal |
| `FETCH_HEADERS_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit for receiving upstream response headers |
| `FETCH_BODY_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Undici time limit between upstream body chunks (`0` disables it) |
| `FETCH_CONNECT_TIMEOUT_MS` | `30000` | Undici TCP connect timeout |
| `FETCH_KEEPALIVE_TIMEOUT_MS` | `4000` | Undici idle keep-alive socket timeout |
| `TLS_CLIENT_TIMEOUT_MS` | inherits `FETCH_TIMEOUT_MS` | Timeout for TLS fingerprint requests made through `wreq-js` |
| `API_BRIDGE_PROXY_TIMEOUT_MS` | inherits `REQUEST_TIMEOUT_MS` or `30000` | Timeout for `/v1` proxy forwarding from API port to dashboard port |
| `API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS` | `max(API_BRIDGE_PROXY_TIMEOUT_MS, 300000)` | Incoming request timeout on the API bridge server |
| `API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS` | `60000` | Incoming header timeout on the API bridge server |
| `API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS` | `5000` | Keep-alive timeout on the API bridge server |
| `API_BRIDGE_SERVER_SOCKET_TIMEOUT_MS` | `0` | Socket inactivity timeout on the API bridge server (`0` disables it) |
If you run OmniRoute behind Nginx, Caddy, Cloudflare, or another reverse proxy, make sure the proxy
timeouts are also higher than your OmniRoute stream/fetch timeouts.
src/shared/utils/runtimeTimeouts.ts
+31 -7
View File
@@ -16,6 +16,11 @@ export const DEFAULT_API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS = 60_000;
export const DEFAULT_API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS = 5_000;
export const DEFAULT_API_BRIDGE_SERVER_SOCKET_TIMEOUT_MS = 0;
function hasEnvValue(env: EnvSource, name: string): boolean {
const raw = env[name];
return raw != null && raw.trim() !== "";
}
export type UpstreamTimeoutConfig = {
fetchTimeoutMs: number;
streamIdleTimeoutMs: number;
@@ -60,14 +65,25 @@ export function getUpstreamTimeoutConfig(
env: EnvSource = process.env,
logger?: TimeoutLogger
): UpstreamTimeoutConfig {
const fetchTimeoutMs = readTimeoutMs(env, "FETCH_TIMEOUT_MS", DEFAULT_FETCH_TIMEOUT_MS, {
allowZero: true,
logger,
});
const sharedRequestTimeoutMs = hasEnvValue(env, "REQUEST_TIMEOUT_MS")
? readTimeoutMs(env, "REQUEST_TIMEOUT_MS", DEFAULT_FETCH_TIMEOUT_MS, {
allowZero: true,
logger,
})
: undefined;
const fetchTimeoutMs = readTimeoutMs(
env,
"FETCH_TIMEOUT_MS",
sharedRequestTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS,
{
allowZero: true,
logger,
}
);
const streamIdleTimeoutMs = readTimeoutMs(
env,
"STREAM_IDLE_TIMEOUT_MS",
DEFAULT_STREAM_IDLE_TIMEOUT_MS,
sharedRequestTimeoutMs ?? DEFAULT_STREAM_IDLE_TIMEOUT_MS,
{
allowZero: true,
logger,
@@ -123,10 +139,16 @@ export function getApiBridgeTimeoutConfig(
env: EnvSource = process.env,
logger?: TimeoutLogger
): ApiBridgeTimeoutConfig {
const sharedRequestTimeoutMs = hasEnvValue(env, "REQUEST_TIMEOUT_MS")
? readTimeoutMs(env, "REQUEST_TIMEOUT_MS", DEFAULT_FETCH_TIMEOUT_MS, {
allowZero: true,
logger,
})
: undefined;
const proxyTimeoutMs = readTimeoutMs(
env,
"API_BRIDGE_PROXY_TIMEOUT_MS",
DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS,
sharedRequestTimeoutMs ?? DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS,
{
allowZero: true,
logger,
@@ -160,7 +182,9 @@ export function getApiBridgeTimeoutConfig(
serverRequestTimeoutMs: readTimeoutMs(
env,
"API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS",
derivedRequestTimeoutMs,
sharedRequestTimeoutMs
? Math.max(sharedRequestTimeoutMs, derivedRequestTimeoutMs)
: derivedRequestTimeoutMs,
{
allowZero: true,
logger,
tests/unit/runtime-timeouts.test.mjs
+17
View File
@@ -19,8 +19,25 @@ test("upstream timeout config derives hidden fetch timeouts from FETCH_TIMEOUT_M
});
});
test("REQUEST_TIMEOUT_MS becomes the common timeout baseline when specific overrides are unset", () => {
const upstreamConfig = runtimeTimeouts.getUpstreamTimeoutConfig({
REQUEST_TIMEOUT_MS: "600000",
});
const apiBridgeConfig = runtimeTimeouts.getApiBridgeTimeoutConfig({
REQUEST_TIMEOUT_MS: "600000",
});
assert.equal(upstreamConfig.fetchTimeoutMs, 600000);
assert.equal(upstreamConfig.streamIdleTimeoutMs, 600000);
assert.equal(upstreamConfig.fetchHeadersTimeoutMs, 600000);
assert.equal(upstreamConfig.fetchBodyTimeoutMs, 600000);
assert.equal(apiBridgeConfig.proxyTimeoutMs, 600000);
assert.equal(apiBridgeConfig.serverRequestTimeoutMs, 600000);
});
test("upstream timeout config honors explicit overrides and falls back on invalid values", () => {
const config = runtimeTimeouts.getUpstreamTimeoutConfig({
REQUEST_TIMEOUT_MS: "550000",
FETCH_TIMEOUT_MS: "600000",
STREAM_IDLE_TIMEOUT_MS: "600000",
FETCH_HEADERS_TIMEOUT_MS: "610000",