feat(self-hosted): add function runtime errors (#50589)

docker/docker-compose.yml

@@ -467,6 +467,8 @@ services:
     command:
       [
         "start",
+        "--user-worker-request-idle-timeout",
+        "150000",
         "--main-service",
         "/home/deno/functions/main"
       ]

docker/tests/test-self-hosted.sh

@@ -457,6 +457,14 @@ fn_resp=$(http_body "$BASE_URL/functions/v1/hello" \
     -d '{}')
 check "Call hello function" '{"message":"Hello from Edge Functions!"}' "$fn_resp"
 
+fn_headers=$(mktemp); cleanup_files="$cleanup_files $fn_headers"
+fn_missing_status=$(http_status "$BASE_URL/functions/v1/smoke-test-missing-$$" \
+    -H "apikey: $SUPABASE_PUBLISHABLE_KEY" \
+    -D "$fn_headers")
+check "Unknown function returns 404" "404" "$fn_missing_status"
+fn_error_code=$(awk 'tolower($1) == "sb-error-code:" { sub(/^[^:]*:[[:space:]]*/, ""); sub(/[[:space:]]*$/, ""); print }' "$fn_headers")
+check "Unknown function returns sb-error-code header" "NOT_FOUND" "$fn_error_code"
+
 # A non-sb_ value (typo / legacy / third-party JWT) is not rejected at the
 # gateway - it passes to the function, where the Supabase Server SDK rejects it.
 # (Detailed sb_-key translation/rejection is covered in test-auth-keys.sh.)

docker/volumes/api/envoy/lds.template.yaml

@@ -264,7 +264,10 @@ resources:
                         route:
                           cluster: functions
                           prefix_rewrite: /
-                          timeout: 150s
+                          # Allow the runtime's 400s wall clock to expire first.
+                          timeout: 410s
+                          # Limit inactivity, matching Kong's read_timeout.
+                          idle_timeout: 160s
                         request_headers_to_add:
                           - header:
                               key: X-Forwarded-Prefix

docker/volumes/api/kong.yml

@@ -359,7 +359,8 @@ services:
   - name: functions-v1
     _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*'
     url: http://functions:9000/
-    read_timeout: 150000
+    # Limit inactivity between reads, leaving 10s for the runtime's 150s idle timeout.
+    read_timeout: 160000
     routes:
       - name: functions-v1-all
         strip_path: true

docker/volumes/functions/main/index.ts

@@ -12,12 +12,108 @@ type AuthFailure = {
   message?: string
 }
 
+type FunctionFailure = {
+  code: RequestErrors
+  message: string
+  status: number
+}
+
 export enum RequestErrors {
   InvalidLegacyJWT = 'UNAUTHORIZED_LEGACY_JWT',
   InvalidAsymmetricJWT = 'UNAUTHORIZED_ASYMMETRIC_JWT',
   InvalidTokenFormat = 'UNAUTHORIZED_INVALID_JWT_FORMAT',
   UnsupportedTokenAlgorithm = 'UNAUTHORIZED_UNSUPPORTED_TOKEN_ALGORITHM',
   MissingAuthHeader = 'UNAUTHORIZED_NO_AUTH_HEADER',
+  NotFound = 'NOT_FOUND',
+  BootError = 'BOOT_ERROR',
+  EdgeFunctionError = 'EDGE_FUNCTION_ERROR',
+  IdleTimeout = 'IDLE_TIMEOUT',
+  WorkerResourceLimit = 'WORKER_RESOURCE_LIMIT',
+  WorkerError = 'WORKER_ERROR',
+  InvalidResponseStatusCode = 'INVALID_RESPONSE_STATUS_CODE',
+}
+
+function getFunctionErrorResponse({ code, message, status }: FunctionFailure): Response {
+  return Response.json(
+    { code, message },
+    {
+      status,
+      headers: {
+        'sb-error-code': code,
+        'Access-Control-Expose-Headers': 'sb-error-code',
+      },
+    }
+  )
+}
+
+function handleWorkerResponse(response: Response): Response {
+  if (response.status < 500) return response
+
+  const headers = new Headers(response.headers)
+  headers.set('sb-error-code', RequestErrors.EdgeFunctionError)
+
+  const exposedHeaders = (headers.get('Access-Control-Expose-Headers') ?? '')
+    .split(',')
+    .map((name) => name.trim())
+    .filter(Boolean)
+  if (!exposedHeaders.some((name) => name.toLowerCase() === 'sb-error-code')) {
+    exposedHeaders.push('sb-error-code')
+  }
+  headers.set('Access-Control-Expose-Headers', exposedHeaders.join(', '))
+
+  return new Response(response.body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+  })
+}
+
+function resolveRuntimeError(e: unknown): FunctionFailure {
+  // These error classes are supplied by Edge Runtime, rather than stock Deno.
+  if (e instanceof Deno.errors.InvalidWorkerCreation) {
+    return {
+      code: RequestErrors.BootError,
+      message: 'Function failed to start (please check logs)',
+      status: 503,
+    }
+  }
+  if (e instanceof Deno.errors.WorkerRequestCancelled) {
+    return {
+      code: RequestErrors.WorkerResourceLimit,
+      message: 'Function failed due to not having enough compute resources (please check logs)',
+      status: 546,
+    }
+  }
+  if (e instanceof Deno.errors.WorkerRequestIdleTimeout) {
+    return {
+      code: RequestErrors.IdleTimeout,
+      message: 'Request idle timeout limit (150s) reached',
+      status: 504,
+    }
+  }
+  // No dedicated runtime error class exists for invalid response statuses.
+  // The Response constructor throws directly here or inside the user worker.
+  if (
+    (e instanceof RangeError || e instanceof Deno.errors.InvalidWorkerResponse) &&
+    e.message.includes('is not equal to 101 and outside the range [200, 599]')
+  ) {
+    return {
+      code: RequestErrors.InvalidResponseStatusCode,
+      message: 'Function returned an invalid HTTP status code (please check logs)',
+      status: 500,
+    }
+  }
+  if (
+    e instanceof Deno.errors.WorkerAlreadyRetired ||
+    e instanceof Deno.errors.InvalidWorkerResponse
+  ) {
+    return {
+      code: RequestErrors.WorkerError,
+      message: 'Function exited due to an error (please check logs)',
+      status: 500,
+    }
+  }
+  return { code: RequestErrors.EdgeFunctionError, message: 'Internal Server Error', status: 500 }
 }
 
 // NOTE:(kallebysantos) We don't check for valid keys but just the bare array parsing,
@@ -209,18 +305,45 @@ Deno.serve(async (req: Request) => {
   const service_name = path_parts[1]
 
   if (!service_name || service_name === '') {
-    const error = { msg: 'missing function name in request' }
-    return new Response(JSON.stringify(error), {
-      status: 400,
-      headers: { 'Content-Type': 'application/json' },
+    return getFunctionErrorResponse({
+      code: RequestErrors.NotFound,
+      message: 'Requested function was not found',
+      status: 404,
     })
   }
 
   const servicePath = `/home/deno/functions/${service_name}`
   console.error(`serving the request with ${servicePath}`)
 
+  try {
+    const serviceInfo = await Deno.stat(servicePath)
+    if (!serviceInfo.isDirectory) {
+      return getFunctionErrorResponse({
+        code: RequestErrors.NotFound,
+        message: 'Requested function was not found',
+        status: 404,
+      })
+    }
+  } catch (e) {
+    if (e instanceof Deno.errors.NotFound) {
+      return getFunctionErrorResponse({
+        code: RequestErrors.NotFound,
+        message: 'Requested function was not found',
+        status: 404,
+      })
+    }
+    console.error(e)
+    return getFunctionErrorResponse({
+      code: RequestErrors.BootError,
+      message: 'Function failed to start (please check logs)',
+      status: 503,
+    })
+  }
+
   const memoryLimitMb = 150
-  const workerTimeoutMs = 1 * 60 * 1000
+  // Keep the wall clock above the 150s request idle timeout configured in Compose.
+  const workerTimeoutMs = 400_000
+  const requestAbsentTimeoutMs = 60_000
   const noModuleCache = false
   // Using a common Import Map for all functions 
   // to use a scope 'deno.json' it must be dinamically resolved base on the 'service_name'
@@ -236,6 +359,7 @@ Deno.serve(async (req: Request) => {
       servicePath,
       memoryLimitMb,
       workerTimeoutMs,
+      context: { supervisor: { requestAbsentTimeoutMs } },
       noModuleCache,
       importMapPath,
       envVars,
@@ -244,12 +368,9 @@ Deno.serve(async (req: Request) => {
     const userReq = new Request(req)
     userReq.headers.delete('sb-api-key')
     EdgeRuntime.applySupabaseTag(req, userReq)
-    return await worker.fetch(userReq)
+    return handleWorkerResponse(await worker.fetch(userReq))
   } catch (e) {
-    const error = { msg: e.toString() }
-    return new Response(JSON.stringify(error), {
-      status: 500,
-      headers: { 'Content-Type': 'application/json' },
-    })
+    console.error(e)
+    return getFunctionErrorResponse(resolveRuntimeError(e))
   }
 })