Merge branch 'feat/unified-logs' into feat/unified-logs-openstatus-view

apps/studio/components/interfaces/Settings/Logs/LogColumnRenderers/UnifiedLogsColumnRender.tsx

@@ -1,4 +1,5 @@
 import { LogData } from 'components/interfaces/Settings/Logs/Logs.types'
+import { Badge } from 'ui'
 import { TimestampInfo } from 'ui-patterns/TimestampInfo'
 
 interface UnifiedLogsColumnRenderProps {
@@ -9,6 +10,7 @@ interface UnifiedLogsColumnRenderProps {
     method?: string
     code?: string
     level?: string
+    api_role?: string
   }
 }
 
@@ -45,6 +47,9 @@ export const UnifiedLogsColumnRender = ({ log }: UnifiedLogsColumnRenderProps) =
   return (
     <div className="flex flex-row gap-3 items-center" key={log.id}>
       <div className="flex items-center gap-2">
+        {log.method && <Badge>{log.method}</Badge>}
+        {log.auth_user && <Badge>User: {log.auth_user}</Badge>}
+        {log.api_role && <Badge>{log.api_role}</Badge>}
         <TimestampInfo utcTimestamp={log.timestamp} />
         {logLevel && (
           <span
@@ -59,7 +64,8 @@ export const UnifiedLogsColumnRender = ({ log }: UnifiedLogsColumnRenderProps) =
             {logLevel}
           </span>
         )}
-        <span className="text-xs text-foreground-light">{log_type}</span>
+        <Badge>{log_type}</Badge>
+        {/* <span className="text-xs text-foreground-light">{log_type}</span> */}
         {logCode && <span className="text-xs text-foreground-light">{logCode}</span>}
       </div>
       {logPath && <div className="text-xs text-foreground-light">{logPath}</div>}

apps/studio/components/interfaces/Settings/Logs/Logs.utils.ts

@@ -206,39 +206,9 @@ limit ${limit}
       return `select id, ${table}.timestamp, event_message from ${table} ${joins} ${where} ${orderBy} limit ${limit}`
 
     case 'unified_logs':
-      if (IS_PLATFORM === false) {
-        return `
-select 
-  id,
-  el.timestamp as timestamp,
-  'edge' as log_type,
-  edge_logs_response.status_code as code,
-  'undefined' as level,
-  edge_logs_request.path as path,
-  'undefined' as event_message
-from edge_logs as el
-cross join unnest(el.metadata) as edge_logs_metadata
-cross join unnest(edge_logs_metadata.request) as edge_logs_request
-cross join unnest(edge_logs_metadata.response) as edge_logs_response
-
-union all
-
-select 
-  id,
-  pgl.timestamp as timestamp,
-  'postgres' as log_type,
-  'undefined' as code,
-  'undefined' as level,
-  'undefined' as path,
-  'undefined' as event_message
-from postgres_logs as pgl
-
-${where}
-${orderBy}
-limit ${limit}
-`
-      }
       return `
+
+-- edge logs
 select 
   id,
   el.timestamp as timestamp,
@@ -246,14 +216,22 @@ select
   CAST(edge_logs_response.status_code AS STRING) as code,
   'undefined' as level,
   edge_logs_request.path as path,
-  'undefined' as event_message
+  'undefined' as event_message,
+  edge_logs_request.method as method,
+  authorization_payload.role as api_role,
+  COALESCE(sb.auth_user, null) as auth_user,
 from edge_logs as el
 cross join unnest(metadata) as edge_logs_metadata
 cross join unnest(edge_logs_metadata.request) as edge_logs_request
 cross join unnest(edge_logs_metadata.response) as edge_logs_response
+left join unnest(edge_logs_request.sb) as sb
+left join unnest(sb.jwt) as jwt
+left join unnest(jwt.authorization) as auth
+left join unnest(auth.payload) as authorization_payload
 
 union all
 
+-- postgres logs
 select 
   id,
   pgl.timestamp as timestamp,
@@ -261,11 +239,143 @@ select
   pgl_parsed.sql_state_code as code,
   'undefined' as level,
   null as path,
-  'undefined' as event_message
+  'undefined' as event_message,
+  'undefined' as method,
+  'api_role' as api_role,
+  null as auth_user,
 from postgres_logs as pgl
 cross join unnest(pgl.metadata) as pgl_metadata
 cross join unnest(pgl_metadata.parsed) as pgl_parsed
 
+union all 
+
+-- function event logs
+select 
+  id, 
+  fl.timestamp as timestamp,
+  'function logs' as log_type,
+  'undefined' as code,
+  fl_metadata.level as level,
+  null as path,
+  fl.event_message as event_message, 
+  'undefined' as method,
+  'api_role' as api_role,
+  null as auth_user,
+  -- fl_metadata.function_id as function_id, 
+  -- fl_metadata.event_type as event_type, 
+from function_logs as fl
+cross join unnest(metadata) as fl_metadata
+
+union all
+
+-- function edge logs
+select 
+  id, 
+  fel.timestamp as timestamp,
+  'edge function' as log_type,
+  CAST(fel_response.status_code as STRING) as code,
+  'undefined' as level,
+  fel_request.url as path,
+  'undefined' as event_message,
+  'undefined' as method,
+  'api_role' as api_role,
+  null as auth_user,
+  -- fel.event_message as event_message,
+  -- fel_request.path as path,
+  -- fel_metadata.function_id as function_id,
+  -- fel_metadata.execution_time_ms as execution_time_ms, 
+  -- fel_metadata.deployment_id as deployment_id, 
+  -- fel_metadata.version as version 
+from function_edge_logs as fel
+cross join unnest(metadata) as fel_metadata
+cross join unnest(fel_metadata.response) as fel_response
+cross join unnest(fel_metadata.request) as fel_request
+
+union all
+
+-- OLDER QUERY FOR AUTH LOGS
+--
+-- auth logs
+-- select 
+--  id, 
+--  al.timestamp as timestamp, 
+--  'auth' as log_type,
+--  REGEXP_EXTRACT(al_metadata.msg, r'^(\d+):') as code, 
+--  -- 'undefined' as code, 
+--  al_metadata.level as level, 
+--  al_metadata.path as path, 
+--  'undefined' as event_message,
+  -- al_metadata.status as status, 
+  -- 'undefined' as method,
+-- from auth_logs as al
+-- cross join unnest(metadata) as al_metadata
+
+-- NEW QUERY FOR AUTH LOGS
+-- THIS USES EDGE LOGS AS WELL TO GET STATUS AND PATH
+--
+select
+  al.id as id, 
+  el_in_al.timestamp as timestamp, 
+  el_in_al_request.path as path, 
+  'auth' as log_type,
+  CAST(el_in_al_response.status_code as STRING) as code,
+  'undefined' as level,
+  null as path,
+  el_in_al_request.method as method,
+  authorization_payload.role as api_role,
+  COALESCE(sb.auth_user, null) as auth_user,
+from auth_logs as al
+cross join unnest(metadata) as al_metadata left join (
+  edge_logs as el_in_al
+    cross join unnest (metadata) as el_in_al_metadata 
+    cross join unnest (el_in_al_metadata. response) as el_in_al_response 
+    cross join unnest (el_in_al_response.headers) as el_in_al_response_headers 
+    cross join unnest (el_in_al_metadata. request) as el_in_al_request
+    left join unnest(el_in_al_request.sb) as sb
+    left join unnest(sb.jwt) as jwt
+    left join unnest(jwt.authorization) as auth
+    left join unnest(auth.payload) as authorization_payload
+)
+on al_metadata. request_id = el_in_al_response_headers.cf_ray
+-- filter out auth logs without a request_id
+-- it looks like most auth logs are logged twice, one of them has a request_id
+where al_metadata.request_id is not null
+
+union all
+
+-- supavisor logs
+select 
+  id, 
+  svl.timestamp as timestamp, 
+  'supavisor' as log_type,
+  'undefined' as code,
+  svl_metadata.level as level,
+  null as path,
+  'undefined' as event_message,
+  'undefined' as method,
+  'api_role' as api_role,
+  null as auth_user,
+from supavisor_logs as svl
+cross join unnest(metadata) as svl_metadata
+
+union all
+
+-- pg_upgrade logs
+select 
+  id, 
+  pgul.timestamp as timestamp,
+  'postgres upgrade' as log_type,
+  'undefined' as code,
+  'undefined' as level,
+  -- pgul.level as level,
+  null as path,  
+  'undefined' as event_message,
+  'undefined' as method,
+  'api_role' as api_role,
+  null as auth_user,
+from pg_upgrade_logs as pgul
+
+
 ${where}
 ${orderBy}
 limit ${limit}
@@ -405,6 +515,109 @@ export const genChartQuery = (
 
   let joins = genCrossJoinUnnests(table)
 
+  // For unified logs, we need a special query to combine data from multiple sources
+  if (table === LogsTableName.UNIFIED) {
+    return `
+WITH unified_data AS (
+  -- edge logs
+  SELECT
+    el.timestamp as timestamp,
+    CAST(edge_logs_response.status_code AS INT64) as status_code,
+    'undefined' as level
+  FROM edge_logs as el
+  CROSS JOIN UNNEST(metadata) as edge_logs_metadata
+  CROSS JOIN UNNEST(edge_logs_metadata.request) as edge_logs_request
+  CROSS JOIN UNNEST(edge_logs_metadata.response) as edge_logs_response
+  WHERE el.timestamp > '${startOffset.toISOString()}'
+  
+  UNION ALL
+  
+  -- postgres logs
+  SELECT
+    pgl.timestamp as timestamp,
+    CASE 
+      WHEN pgl_parsed.error_severity IN ('ERROR', 'FATAL', 'PANIC') THEN 500
+      WHEN pgl_parsed.error_severity IN ('WARNING') THEN 400
+      ELSE 200
+    END as status_code,
+    pgl_parsed.error_severity as level
+  FROM postgres_logs as pgl
+  CROSS JOIN UNNEST(pgl.metadata) as pgl_metadata
+  CROSS JOIN UNNEST(pgl_metadata.parsed) as pgl_parsed
+  WHERE pgl.timestamp > '${startOffset.toISOString()}'
+  
+  UNION ALL
+  
+  -- function logs
+  SELECT
+    fl.timestamp as timestamp,
+    CASE 
+      WHEN fl_metadata.level IN ('error', 'fatal') THEN 500
+      WHEN fl_metadata.level IN ('warning') THEN 400
+      ELSE 200
+    END as status_code,
+    fl_metadata.level as level
+  FROM function_logs as fl
+  CROSS JOIN UNNEST(metadata) as fl_metadata
+  WHERE fl.timestamp > '${startOffset.toISOString()}'
+  
+  UNION ALL
+  
+  -- function edge logs
+  SELECT
+    fel.timestamp as timestamp,
+    CAST(fel_response.status_code AS INT64) as status_code,
+    'undefined' as level
+  FROM function_edge_logs as fel
+  CROSS JOIN UNNEST(metadata) as fel_metadata
+  CROSS JOIN UNNEST(fel_metadata.response) as fel_response
+  WHERE fel.timestamp > '${startOffset.toISOString()}'
+  
+  UNION ALL
+  
+  -- auth logs
+  SELECT
+    al.timestamp as timestamp,
+    CASE 
+      WHEN al_metadata.level = 'error' THEN 500
+      WHEN al_metadata.level = 'warning' THEN 400
+      ELSE 200
+    END as status_code,
+    al_metadata.level as level
+  FROM auth_logs as al
+  CROSS JOIN UNNEST(metadata) as al_metadata
+  WHERE al.timestamp > '${startOffset.toISOString()}'
+  
+  UNION ALL
+  
+  -- supavisor logs
+  SELECT
+    svl.timestamp as timestamp,
+    CASE 
+      WHEN svl_metadata.level = 'error' THEN 500
+      WHEN svl_metadata.level = 'warning' THEN 400
+      ELSE 200
+    END as status_code,
+    svl_metadata.level as level
+  FROM supavisor_logs as svl
+  CROSS JOIN UNNEST(metadata) as svl_metadata
+  WHERE svl.timestamp > '${startOffset.toISOString()}'
+)
+
+SELECT
+  timestamp_trunc(timestamp, ${trunc}) as timestamp,
+  count(CASE WHEN NOT (status_code >= 500 OR level IN ('error', 'fatal')) AND NOT (status_code >= 400 AND status_code < 500 OR level = 'warning') THEN 1 END) as ok_count,
+  count(CASE WHEN status_code >= 500 OR level IN ('error', 'fatal') THEN 1 END) as error_count,
+  count(CASE WHEN status_code >= 400 AND status_code < 500 OR level = 'warning' THEN 1 END) as warning_count
+FROM
+  unified_data
+GROUP BY
+  timestamp
+ORDER BY
+  timestamp ASC
+`
+  }
+
   const q = `
 SELECT
 -- log-event-chart
@@ -678,6 +891,8 @@ function getErrorCondition(table: LogsTableName): string {
       return "metadata.level IN ('error', 'fatal')"
     case 'pg_cron_logs':
       return "parsed.error_severity IN ('ERROR', 'FATAL', 'PANIC')"
+    case 'unified_logs':
+      return "(code >= '500' OR level = 'error' OR level = 'fatal')"
     default:
       return 'false'
   }
@@ -695,6 +910,8 @@ function getWarningCondition(table: LogsTableName): string {
       return 'response.status_code >= 400 AND response.status_code < 500'
     case 'function_logs':
       return "metadata.level IN ('warning')"
+    case 'unified_logs':
+      return "(code >= '400' AND code < '500') OR level = 'warning'"
     default:
       return 'false'
   }