From 3fc8af387ec4dfb449510828a938e1f6e57a9575 Mon Sep 17 00:00:00 2001 From: Luiz Felipe Machado <56140722+luizfelmach@users.noreply.github.com> Date: Wed, 30 Sep 2026 07:57:27 -0300 Subject: [PATCH] feat(self-hosted): add function runtime errors (#50589) --- docker/docker-compose.yml | 2 + docker/tests/test-self-hosted.sh | 8 ++ docker/volumes/api/envoy/lds.template.yaml | 5 +- docker/volumes/api/kong.yml | 3 +- docker/volumes/functions/main/index.ts | 143 +++++++++++++++++++-- 5 files changed, 148 insertions(+), 13 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a37d39dc7c7..49af446e2ae 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -467,6 +467,8 @@ services: command: [ "start", + "--user-worker-request-idle-timeout", + "150000", "--main-service", "/home/deno/functions/main" ] diff --git a/docker/tests/test-self-hosted.sh b/docker/tests/test-self-hosted.sh index 775a853cdf4..086dcea8594 100644 --- a/docker/tests/test-self-hosted.sh +++ b/docker/tests/test-self-hosted.sh @@ -457,6 +457,14 @@ fn_resp=$(http_body "$BASE_URL/functions/v1/hello" \ -d '{}') check "Call hello function" '{"message":"Hello from Edge Functions!"}' "$fn_resp" +fn_headers=$(mktemp); cleanup_files="$cleanup_files $fn_headers" +fn_missing_status=$(http_status "$BASE_URL/functions/v1/smoke-test-missing-$$" \ + -H "apikey: $SUPABASE_PUBLISHABLE_KEY" \ + -D "$fn_headers") +check "Unknown function returns 404" "404" "$fn_missing_status" +fn_error_code=$(awk 'tolower($1) == "sb-error-code:" { sub(/^[^:]*:[[:space:]]*/, ""); sub(/[[:space:]]*$/, ""); print }' "$fn_headers") +check "Unknown function returns sb-error-code header" "NOT_FOUND" "$fn_error_code" + # A non-sb_ value (typo / legacy / third-party JWT) is not rejected at the # gateway - it passes to the function, where the Supabase Server SDK rejects it. # (Detailed sb_-key translation/rejection is covered in test-auth-keys.sh.) diff --git a/docker/volumes/api/envoy/lds.template.yaml b/docker/volumes/api/envoy/lds.template.yaml index b13b7891bd6..a414496cee2 100644 --- a/docker/volumes/api/envoy/lds.template.yaml +++ b/docker/volumes/api/envoy/lds.template.yaml @@ -264,7 +264,10 @@ resources: route: cluster: functions prefix_rewrite: / - timeout: 150s + # Allow the runtime's 400s wall clock to expire first. + timeout: 410s + # Limit inactivity, matching Kong's read_timeout. + idle_timeout: 160s request_headers_to_add: - header: key: X-Forwarded-Prefix diff --git a/docker/volumes/api/kong.yml b/docker/volumes/api/kong.yml index d9608f04b8f..74a04c0ecb1 100644 --- a/docker/volumes/api/kong.yml +++ b/docker/volumes/api/kong.yml @@ -359,7 +359,8 @@ services: - name: functions-v1 _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*' url: http://functions:9000/ - read_timeout: 150000 + # Limit inactivity between reads, leaving 10s for the runtime's 150s idle timeout. + read_timeout: 160000 routes: - name: functions-v1-all strip_path: true diff --git a/docker/volumes/functions/main/index.ts b/docker/volumes/functions/main/index.ts index 3eeb28375db..ee55cf0f887 100644 --- a/docker/volumes/functions/main/index.ts +++ b/docker/volumes/functions/main/index.ts @@ -12,12 +12,108 @@ type AuthFailure = { message?: string } +type FunctionFailure = { + code: RequestErrors + message: string + status: number +} + export enum RequestErrors { InvalidLegacyJWT = 'UNAUTHORIZED_LEGACY_JWT', InvalidAsymmetricJWT = 'UNAUTHORIZED_ASYMMETRIC_JWT', InvalidTokenFormat = 'UNAUTHORIZED_INVALID_JWT_FORMAT', UnsupportedTokenAlgorithm = 'UNAUTHORIZED_UNSUPPORTED_TOKEN_ALGORITHM', MissingAuthHeader = 'UNAUTHORIZED_NO_AUTH_HEADER', + NotFound = 'NOT_FOUND', + BootError = 'BOOT_ERROR', + EdgeFunctionError = 'EDGE_FUNCTION_ERROR', + IdleTimeout = 'IDLE_TIMEOUT', + WorkerResourceLimit = 'WORKER_RESOURCE_LIMIT', + WorkerError = 'WORKER_ERROR', + InvalidResponseStatusCode = 'INVALID_RESPONSE_STATUS_CODE', +} + +function getFunctionErrorResponse({ code, message, status }: FunctionFailure): Response { + return Response.json( + { code, message }, + { + status, + headers: { + 'sb-error-code': code, + 'Access-Control-Expose-Headers': 'sb-error-code', + }, + } + ) +} + +function handleWorkerResponse(response: Response): Response { + if (response.status < 500) return response + + const headers = new Headers(response.headers) + headers.set('sb-error-code', RequestErrors.EdgeFunctionError) + + const exposedHeaders = (headers.get('Access-Control-Expose-Headers') ?? '') + .split(',') + .map((name) => name.trim()) + .filter(Boolean) + if (!exposedHeaders.some((name) => name.toLowerCase() === 'sb-error-code')) { + exposedHeaders.push('sb-error-code') + } + headers.set('Access-Control-Expose-Headers', exposedHeaders.join(', ')) + + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }) +} + +function resolveRuntimeError(e: unknown): FunctionFailure { + // These error classes are supplied by Edge Runtime, rather than stock Deno. + if (e instanceof Deno.errors.InvalidWorkerCreation) { + return { + code: RequestErrors.BootError, + message: 'Function failed to start (please check logs)', + status: 503, + } + } + if (e instanceof Deno.errors.WorkerRequestCancelled) { + return { + code: RequestErrors.WorkerResourceLimit, + message: 'Function failed due to not having enough compute resources (please check logs)', + status: 546, + } + } + if (e instanceof Deno.errors.WorkerRequestIdleTimeout) { + return { + code: RequestErrors.IdleTimeout, + message: 'Request idle timeout limit (150s) reached', + status: 504, + } + } + // No dedicated runtime error class exists for invalid response statuses. + // The Response constructor throws directly here or inside the user worker. + if ( + (e instanceof RangeError || e instanceof Deno.errors.InvalidWorkerResponse) && + e.message.includes('is not equal to 101 and outside the range [200, 599]') + ) { + return { + code: RequestErrors.InvalidResponseStatusCode, + message: 'Function returned an invalid HTTP status code (please check logs)', + status: 500, + } + } + if ( + e instanceof Deno.errors.WorkerAlreadyRetired || + e instanceof Deno.errors.InvalidWorkerResponse + ) { + return { + code: RequestErrors.WorkerError, + message: 'Function exited due to an error (please check logs)', + status: 500, + } + } + return { code: RequestErrors.EdgeFunctionError, message: 'Internal Server Error', status: 500 } } // NOTE:(kallebysantos) We don't check for valid keys but just the bare array parsing, @@ -209,18 +305,45 @@ Deno.serve(async (req: Request) => { const service_name = path_parts[1] if (!service_name || service_name === '') { - const error = { msg: 'missing function name in request' } - return new Response(JSON.stringify(error), { - status: 400, - headers: { 'Content-Type': 'application/json' }, + return getFunctionErrorResponse({ + code: RequestErrors.NotFound, + message: 'Requested function was not found', + status: 404, }) } const servicePath = `/home/deno/functions/${service_name}` console.error(`serving the request with ${servicePath}`) + try { + const serviceInfo = await Deno.stat(servicePath) + if (!serviceInfo.isDirectory) { + return getFunctionErrorResponse({ + code: RequestErrors.NotFound, + message: 'Requested function was not found', + status: 404, + }) + } + } catch (e) { + if (e instanceof Deno.errors.NotFound) { + return getFunctionErrorResponse({ + code: RequestErrors.NotFound, + message: 'Requested function was not found', + status: 404, + }) + } + console.error(e) + return getFunctionErrorResponse({ + code: RequestErrors.BootError, + message: 'Function failed to start (please check logs)', + status: 503, + }) + } + const memoryLimitMb = 150 - const workerTimeoutMs = 1 * 60 * 1000 + // Keep the wall clock above the 150s request idle timeout configured in Compose. + const workerTimeoutMs = 400_000 + const requestAbsentTimeoutMs = 60_000 const noModuleCache = false // Using a common Import Map for all functions // to use a scope 'deno.json' it must be dinamically resolved base on the 'service_name' @@ -236,6 +359,7 @@ Deno.serve(async (req: Request) => { servicePath, memoryLimitMb, workerTimeoutMs, + context: { supervisor: { requestAbsentTimeoutMs } }, noModuleCache, importMapPath, envVars, @@ -244,12 +368,9 @@ Deno.serve(async (req: Request) => { const userReq = new Request(req) userReq.headers.delete('sb-api-key') EdgeRuntime.applySupabaseTag(req, userReq) - return await worker.fetch(userReq) + return handleWorkerResponse(await worker.fetch(userReq)) } catch (e) { - const error = { msg: e.toString() } - return new Response(JSON.stringify(error), { - status: 500, - headers: { 'Content-Type': 'application/json' }, - }) + console.error(e) + return getFunctionErrorResponse(resolveRuntimeError(e)) } })