diff --git a/docker/volumes/functions/main/index.ts b/docker/volumes/functions/main/index.ts index ee55cf0f887..7a2b16f8482 100644 --- a/docker/volumes/functions/main/index.ts +++ b/docker/volumes/functions/main/index.ts @@ -2,6 +2,8 @@ import * as jose from 'jsr:@panva/jose@6' console.log('main function started') +const MAX_WORKER_RETRIES = 3 + const JWT_SECRET = Deno.env.get('JWT_SECRET') const SUPABASE_JWKS = parseJwks(Deno.env.get('SUPABASE_JWKS')) const LOCAL_JWKS = SUPABASE_JWKS ? jose.createLocalJWKSet(SUPABASE_JWKS) : null @@ -354,23 +356,41 @@ Deno.serve(async (req: Request) => { const envVarsObj = { ...Deno.env.toObject(), SUPABASE_FUNCTION_SLUG: service_name } const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]) - try { - const worker = await EdgeRuntime.userWorkers.create({ - servicePath, - memoryLimitMb, - workerTimeoutMs, - context: { supervisor: { requestAbsentTimeoutMs } }, - noModuleCache, - importMapPath, - envVars, - }) - // Gateway-minted internal JWT is for this router only; never expose it to user functions. - const userReq = new Request(req) - userReq.headers.delete('sb-api-key') - EdgeRuntime.applySupabaseTag(req, userReq) - return handleWorkerResponse(await worker.fetch(userReq)) - } catch (e) { - console.error(e) - return getFunctionErrorResponse(resolveRuntimeError(e)) + const callWorker = async (req: Request, retriesLeft = MAX_WORKER_RETRIES): Promise => { + // Preserve the body before fetch() can consume it, even on a failed attempt. + // Must run before `new Request(req)` below, which takes over the body. + // The unread retry branch can buffer the entire body in main-worker memory, + // even when the first attempt succeeds. + const retryReq = retriesLeft > 0 ? req.clone() : null + + try { + const worker = await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + context: { supervisor: { requestAbsentTimeoutMs } }, + noModuleCache, + importMapPath, + envVars, + }) + // Gateway-minted internal JWT is for this router only; never expose it to user functions. + const userReq = new Request(req) + userReq.headers.delete('sb-api-key') + EdgeRuntime.applySupabaseTag(req, userReq) + return handleWorkerResponse(await worker.fetch(userReq)) + } catch (e) { + // Retirement rejects before dispatch, so user code has not run yet. + if (e instanceof Deno.errors.WorkerAlreadyRetired && retryReq) { + console.warn(`${service_name}: worker retired before dispatch; retrying (${retriesLeft} left)`) + // Request.clone() does not copy the tag that connects streaming to the client. + EdgeRuntime.applySupabaseTag(req, retryReq) + return await callWorker(retryReq, retriesLeft - 1) + } + + console.error(e) + return getFunctionErrorResponse(resolveRuntimeError(e)) + } } + + return await callWorker(req) }) diff --git a/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl b/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl index f7dd77b5dad..91419f269f6 100644 --- a/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl +++ b/docker/volumes/proxy/nginx/supabase-nginx.conf.tpl @@ -83,6 +83,9 @@ server { location /functions { proxy_pass http://api_gw_upstream; + + # Outlast the runtime's 150s request idle timeout, matching the API gateway. + proxy_read_timeout 160s; } location /mcp {