From 5e38366b37d9e57eba4f6a93c6957e2fadadefcf Mon Sep 17 00:00:00 2001 From: Jonathan Summers-Muir Date: Mon, 12 May 2025 17:01:24 +0800 Subject: [PATCH] level param now being removed correctly. --- .../DataTableDemo/lib/delimiters.ts | 10 +- .../components/interfaces/logs-new/client.tsx | 38 +++++++- .../logs-new/constants/query-params.ts | 35 +++++++ .../interfaces/logs-new/logs.queries.ts | 94 ++++++++++--------- .../interfaces/logs-new/query-options-new.ts | 16 +++- 5 files changed, 136 insertions(+), 57 deletions(-) create mode 100644 apps/studio/components/interfaces/logs-new/constants/query-params.ts diff --git a/apps/studio/components/interfaces/DataTableDemo/lib/delimiters.ts b/apps/studio/components/interfaces/DataTableDemo/lib/delimiters.ts index 5368d8884f9..edb49a5f478 100644 --- a/apps/studio/components/interfaces/DataTableDemo/lib/delimiters.ts +++ b/apps/studio/components/interfaces/DataTableDemo/lib/delimiters.ts @@ -1,5 +1,5 @@ -export const ARRAY_DELIMITER = ","; -export const SLIDER_DELIMITER = "-"; -export const SPACE_DELIMITER = "_"; -export const RANGE_DELIMITER = "-"; -export const SORT_DELIMITER = "."; +export const ARRAY_DELIMITER = ',' +export const SLIDER_DELIMITER = '-' +export const SPACE_DELIMITER = '_' +export const RANGE_DELIMITER = '-' +export const SORT_DELIMITER = '.' diff --git a/apps/studio/components/interfaces/logs-new/client.tsx b/apps/studio/components/interfaces/logs-new/client.tsx index ea9b37aa932..ab4689a3b70 100644 --- a/apps/studio/components/interfaces/logs-new/client.tsx +++ b/apps/studio/components/interfaces/logs-new/client.tsx @@ -53,6 +53,9 @@ import { dataOptions, useChartData } from './query-options-new' import { filterFields as defaultFilterFields, sheetFields } from './constants' import { useParams } from 'common' +// Debug mode flag - set to true to enable detailed logs +const DEBUG_FILTER_PROCESSING = false + function TooltipLabel({ level }: { level: keyof Omit }) { return (
@@ -226,26 +229,53 @@ export function Client() { }, [rowSelection, table, isLoading, isFetching, flatData]) React.useEffect(() => { + if (DEBUG_FILTER_PROCESSING) console.log('========== FILTER CHANGE DETECTED ==========') + if (DEBUG_FILTER_PROCESSING) console.log('Raw columnFilters:', JSON.stringify(columnFilters)) + + // Check for level filters specifically + const levelColumnFilter = columnFilters.find((filter) => filter.id === 'level') + if (DEBUG_FILTER_PROCESSING) console.log('Level column filter:', levelColumnFilter) + const columnFiltersWithNullable = filterFields.map((field) => { const filterValue = columnFilters.find((filter) => filter.id === field.value) + if (DEBUG_FILTER_PROCESSING) console.log(`Processing field ${field.value}:`, filterValue) if (!filterValue) return { id: field.value, value: null } return { id: field.value, value: filterValue.value } }) + // Debug level filter specifically + const levelFilter = columnFiltersWithNullable.find((f) => f.id === 'level') + if (DEBUG_FILTER_PROCESSING) console.log('Level filter after mapping:', levelFilter) + + if (DEBUG_FILTER_PROCESSING) + console.log('All column filters after mapping:', columnFiltersWithNullable) + const search = columnFiltersWithNullable.reduce( (prev, curr) => { + if (DEBUG_FILTER_PROCESSING) + console.log(`Processing filter for URL: ${curr.id}`, { + value: curr.value, + type: Array.isArray(curr.value) ? 'array' : typeof curr.value, + isEmpty: Array.isArray(curr.value) && curr.value.length === 0, + isNull: curr.value === null, + }) + + // Add to search parameters prev[curr.id as string] = curr.value return prev }, {} as Record ) - // Debug column filters - if (columnFilters.length > 0) { - console.log('Setting search from column filters:', columnFilters, '->', search) - } + if (DEBUG_FILTER_PROCESSING) console.log('Final search object to be set in URL:', search) + if (DEBUG_FILTER_PROCESSING) console.log('Level value in final search:', search.level) + if (DEBUG_FILTER_PROCESSING) console.log('Is level in search object:', 'level' in search) + // Set the search state without any console logs + if (DEBUG_FILTER_PROCESSING) console.log('CALLING setSearch with:', JSON.stringify(search)) setSearch(search) + if (DEBUG_FILTER_PROCESSING) console.log('========== END FILTER PROCESSING ==========') + // eslint-disable-next-line react-hooks/exhaustive-deps }, [columnFilters]) diff --git a/apps/studio/components/interfaces/logs-new/constants/query-params.ts b/apps/studio/components/interfaces/logs-new/constants/query-params.ts new file mode 100644 index 00000000000..eb7956088fb --- /dev/null +++ b/apps/studio/components/interfaces/logs-new/constants/query-params.ts @@ -0,0 +1,35 @@ +/** + * Parameters that should be excluded from standard SQL query condition building + */ + +// Pagination and control parameters +export const PAGINATION_PARAMS = [ + 'sort', + 'start', + 'size', + 'uuid', + 'cursor', + 'direction', + 'live', +] as const + +export type PaginationParamType = (typeof PAGINATION_PARAMS)[number] + +// Special filter parameters that need custom handling +export const SPECIAL_FILTER_PARAMS = ['date'] as const + +export type SpecialFilterParamType = (typeof SPECIAL_FILTER_PARAMS)[number] + +// Combined list of all parameters to exclude from standard filtering +export const EXCLUDED_QUERY_PARAMS = [...PAGINATION_PARAMS, ...SPECIAL_FILTER_PARAMS] as const + +export type ExcludedQueryParamType = (typeof EXCLUDED_QUERY_PARAMS)[number] + +/** + * Parameters that need special handling in buildBaseConditions and raw SQL queries + * This is different from the EXCLUDED_QUERY_PARAMS used in getUnifiedLogsQuery. + * + * In raw SQL queries against log type tables, 'level' needs to be processed differently + * because it maps to different columns/conditions depending on the log type. + */ +export const BASE_CONDITIONS_EXCLUDED_PARAMS = [...PAGINATION_PARAMS, 'date', 'level'] as const diff --git a/apps/studio/components/interfaces/logs-new/logs.queries.ts b/apps/studio/components/interfaces/logs-new/logs.queries.ts index 3bef3b27bae..bef3d14e471 100644 --- a/apps/studio/components/interfaces/logs-new/logs.queries.ts +++ b/apps/studio/components/interfaces/logs-new/logs.queries.ts @@ -3,6 +3,40 @@ */ import { SearchParamsType } from './search-params' +import { BASE_CONDITIONS_EXCLUDED_PARAMS, EXCLUDED_QUERY_PARAMS } from './constants/query-params' + +/** + * Builds query conditions from search parameters and returns WHERE clause + * @param search SearchParamsType object containing query parameters + * @returns Object with whereConditions array and formatted WHERE clause + */ +export const buildQueryConditions = (search: SearchParamsType) => { + const whereConditions: string[] = [] + + // Process all search parameters for filtering + Object.entries(search).forEach(([key, value]) => { + // Skip pagination/control parameters + if (EXCLUDED_QUERY_PARAMS.includes(key as any)) { + return + } + + // Handle array filters (IN clause) + if (Array.isArray(value) && value.length > 0) { + whereConditions.push(`${key} IN (${value.map((v) => `'${v}'`).join(',')})`) + return + } + + // Handle scalar values + if (value !== null && value !== undefined) { + whereConditions.push(`${key} = '${value}'`) + } + }) + + // Create final WHERE clause + const finalWhere = whereConditions.length > 0 ? `WHERE ${whereConditions.join(' AND ')}` : '' + + return { whereConditions, finalWhere } +} /** * Builds level-specific condition for different log types @@ -99,22 +133,16 @@ export const buildBaseConditions = (search: SearchParamsType): string[] => { Object.entries(search).forEach(([key, value]) => { // Skip pagination/control parameters, date and level (handled separately) - if ( - ['sort', 'start', 'size', 'uuid', 'cursor', 'direction', 'live', 'date', 'level'].includes( - key - ) - ) { + if (BASE_CONDITIONS_EXCLUDED_PARAMS.includes(key as any)) { return } // Handle array filters (IN clause) if (Array.isArray(value) && value.length > 0) { - baseConditions.push(`${key} IN (${value.map((v) => `'${v}'`).join(',')})`) - return + baseConditions.push(`${key} IN (${value.map((v) => `'${v}'`).join(', ')})`) } - // Handle scalar values - if (value !== null && value !== undefined) { + else if (value !== null && value !== undefined) { baseConditions.push(`${key} = '${value}'`) } }) @@ -126,15 +154,8 @@ export const buildBaseConditions = (search: SearchParamsType): string[] => { * Unified logs SQL query */ export const getUnifiedLogsQuery = (search: SearchParamsType): string => { - const baseConditions = buildBaseConditions(search) - const levelFilter = search.level || [] - - // Create source-specific where clauses with level filters - const edgeWhere = createFilterWhereClause('edge', levelFilter, baseConditions) - const postgresWhere = createFilterWhereClause('postgres', levelFilter, baseConditions) - const functionWhere = createFilterWhereClause('edge function', levelFilter, baseConditions) - const authWhere = createFilterWhereClause('auth', levelFilter, baseConditions) - const supavisorWhere = createFilterWhereClause('supavisor', levelFilter, baseConditions) + // Use the buildQueryConditions helper + const { finalWhere } = buildQueryConditions(search) // The unified SQL query with UNION ALL statements const sql = ` @@ -166,7 +187,6 @@ WITH unified_logs AS ( left join unnest(sb.jwt) as jwt left join unnest(jwt.authorization) as auth left join unnest(auth.payload) as authorization_payload - ${edgeWhere} union all @@ -191,7 +211,6 @@ WITH unified_logs AS ( from postgres_logs as pgl cross join unnest(pgl.metadata) as pgl_metadata cross join unnest(pgl_metadata.parsed) as pgl_parsed - ${postgresWhere} union all @@ -231,7 +250,6 @@ WITH unified_logs AS ( WHERE fl_metadata.execution_id IS NOT NULL GROUP BY fl_metadata.execution_id ) as function_logs_agg on fel_metadata.execution_id = function_logs_agg.execution_id - ${functionWhere} union all @@ -267,7 +285,7 @@ WITH unified_logs AS ( left join unnest(auth.payload) as authorization_payload ) on al_metadata.request_id = el_in_al_response_headers.cf_ray - ${authWhere} + WHERE al_metadata.request_id is not null union all @@ -290,7 +308,6 @@ WITH unified_logs AS ( null as log_count from supavisor_logs as svl cross join unnest(metadata) as svl_metadata - ${supavisorWhere} ) SELECT @@ -307,24 +324,15 @@ SELECT auth_user, log_count FROM unified_logs - -WHERE - level = "error" +${finalWhere} ` return sql } export const getLogsChartQuery2 = (search: SearchParamsType): string => { - const baseConditions = buildBaseConditions(search) - const levelFilter = search.level || [] - - // Create source-specific where clauses with level filters - const edgeWhere = createFilterWhereClause('edge', levelFilter, baseConditions) - const postgresWhere = createFilterWhereClause('postgres', levelFilter, baseConditions) - const functionWhere = createFilterWhereClause('edge function', levelFilter, baseConditions) - const authWhere = createFilterWhereClause('auth', levelFilter, baseConditions) - const supavisorWhere = createFilterWhereClause('supavisor', levelFilter, baseConditions) + // Use the buildQueryConditions helper + const { finalWhere } = buildQueryConditions(search) return ` WITH unified_logs AS ( @@ -346,7 +354,7 @@ WITH unified_logs AS ( edge_logs_request.method as method, authorization_payload.role as api_role, COALESCE(sb.auth_user, null) as auth_user, - null as log_count, + null as log_count from edge_logs as el cross join unnest(metadata) as edge_logs_metadata cross join unnest(edge_logs_metadata.request) as edge_logs_request @@ -355,7 +363,6 @@ WITH unified_logs AS ( left join unnest(sb.jwt) as jwt left join unnest(jwt.authorization) as auth left join unnest(auth.payload) as authorization_payload - ${edgeWhere} union all @@ -376,11 +383,10 @@ WITH unified_logs AS ( null as method, 'api_role' as api_role, null as auth_user, - null as log_count, + null as log_count from postgres_logs as pgl cross join unnest(pgl.metadata) as pgl_metadata cross join unnest(pgl_metadata.parsed) as pgl_parsed - ${postgresWhere} union all @@ -420,7 +426,6 @@ WITH unified_logs AS ( WHERE fl_metadata.execution_id IS NOT NULL GROUP BY fl_metadata.execution_id ) as function_logs_agg on fel_metadata.execution_id = function_logs_agg.execution_id - ${functionWhere} union all @@ -441,7 +446,7 @@ WITH unified_logs AS ( el_in_al_request.method as method, authorization_payload.role as api_role, COALESCE(sb.auth_user, null) as auth_user, - null as log_count, + null as log_count from auth_logs as al cross join unnest(metadata) as al_metadata left join ( @@ -456,7 +461,7 @@ WITH unified_logs AS ( left join unnest(auth.payload) as authorization_payload ) on al_metadata.request_id = el_in_al_response_headers.cf_ray - ${authWhere} + WHERE al_metadata.request_id is not null union all @@ -476,10 +481,9 @@ WITH unified_logs AS ( null as method, 'api_role' as api_role, null as auth_user, - null as log_count, + null as log_count from supavisor_logs as svl cross join unnest(metadata) as svl_metadata - ${supavisorWhere} ) SELECT @@ -488,10 +492,12 @@ SELECT COUNTIF(level = 'warning') as warning, COUNTIF(level = 'error') as error FROM unified_logs +${finalWhere} GROUP BY minute ORDER BY minute ASC ` } + /** * Chart data query for logs time series */ diff --git a/apps/studio/components/interfaces/logs-new/query-options-new.ts b/apps/studio/components/interfaces/logs-new/query-options-new.ts index 65ad4d88254..87676150c63 100644 --- a/apps/studio/components/interfaces/logs-new/query-options-new.ts +++ b/apps/studio/components/interfaces/logs-new/query-options-new.ts @@ -235,10 +235,18 @@ export const useChartData = (search: SearchParamsType, projectRef: string) => { } // Only include the levels that were requested in the filter (or all if no filter) - const levelFilter = search.level || ['success', 'warning', 'error'] - if (levelFilter.includes('success')) dataPoint.success = minuteData.success - if (levelFilter.includes('warning')) dataPoint.warning = minuteData.warning - if (levelFilter.includes('error')) dataPoint.error = minuteData.error + const levelFilter = search.level + // If no level filter is specified or it's empty, include all levels + if (!levelFilter || levelFilter.length === 0) { + dataPoint.success = minuteData.success + dataPoint.warning = minuteData.warning + dataPoint.error = minuteData.error + } else { + // Otherwise only include the selected levels + if (levelFilter.includes('success')) dataPoint.success = minuteData.success + if (levelFilter.includes('warning')) dataPoint.warning = minuteData.warning + if (levelFilter.includes('error')) dataPoint.error = minuteData.error + } // Always push a data point for this minute chartData.push(dataPoint)