From e12e884d2df403243e8d25f714624a591d79773e Mon Sep 17 00:00:00 2001 From: TzeYiing Date: Thu, 16 Feb 2023 19:50:09 +0800 Subject: [PATCH] feat: added headers filtering --- .../interfaces/Reports/ReportFilterBar.tsx | 28 +- .../interfaces/Reports/Reports.constants.ts | 496 +++++++++--------- 2 files changed, 269 insertions(+), 255 deletions(-) diff --git a/studio/components/interfaces/Reports/ReportFilterBar.tsx b/studio/components/interfaces/Reports/ReportFilterBar.tsx index 5fbd3485d14..01a92844281 100644 --- a/studio/components/interfaces/Reports/ReportFilterBar.tsx +++ b/studio/components/interfaces/Reports/ReportFilterBar.tsx @@ -73,16 +73,23 @@ const ReportFilterBar: React.FC = ({ onDatepickerChange, datepickerTo = '', datepickerFrom = '', - onRemoveFilters + onRemoveFilters, }) => { - const filterKeys = ['request.path', 'request.host', 'response.status_code'] + const filterKeys = [ + 'request.path', + 'request.method', + 'request.search', + 'request.headers.x_client_info', + 'request.headers.user_agent', + 'response.status_code', + ] const [showAdder, setShowAdder] = useState(false) const [currentProductFilter, setCurrentProductFilter] = useState< null | typeof PRODUCT_FILTERS[number] >(null) const [addFilterValues, setAddFilterValues] = useState({ key: filterKeys[0], - compare: 'is', + compare: 'matches', value: '', }) @@ -97,11 +104,14 @@ const ReportFilterBar: React.FC = ({ const handleProductFilterChange = async ( nextProductFilter: null | typeof PRODUCT_FILTERS[number] ) => { - const toRemove = PRODUCT_FILTERS.map(productFilter=>({ - key: productFilter.filterKey, - compare: 'matches', - value: productFilter.filterValue, - }) as ReportFilterItem) + const toRemove = PRODUCT_FILTERS.map( + (productFilter) => + ({ + key: productFilter.filterKey, + compare: 'matches', + value: productFilter.filterValue, + } as ReportFilterItem) + ) onRemoveFilters(toRemove) if (nextProductFilter) { onAddFilter({ @@ -220,7 +230,7 @@ const ReportFilterBar: React.FC = ({ setAddFilterValues((prev) => ({ ...prev, key: e.target.value })) }} label="Attribute Filter" - defaultValue={'request.host'} + defaultValue={'request.path'} > {filterKeys.map((key) => ( diff --git a/studio/components/interfaces/Reports/Reports.constants.ts b/studio/components/interfaces/Reports/Reports.constants.ts index 18808f3a730..86a2c7e2f7c 100644 --- a/studio/components/interfaces/Reports/Reports.constants.ts +++ b/studio/components/interfaces/Reports/Reports.constants.ts @@ -33,20 +33,21 @@ export const DEFAULT_QUERY_PARAMS = { iso_timestamp_end: REPORTS_DATEPICKER_HELPERS[0].calcTo(), } -const generateRexepWhere = (filters: ReportFilterItem[], prepend=true) => { +const generateRexepWhere = (filters: ReportFilterItem[], prepend = true) => { if (filters.length === 0) return '' const conditions = filters .map((filter) => { - if (filter.compare === "matches"){ - return `REGEXP_CONTAINS(${filter.key}, '${filter.value}')` - } else if (filter.compare === "is") { - return `${filter.key} = ${filter.value}` + const splitKey = filter.key.split('.') + const normalizedKey = [splitKey[splitKey.length - 2], splitKey[splitKey.length-1]].join('.') + if (filter.compare === 'matches') { + return `REGEXP_CONTAINS(${normalizedKey}, '${filter.value}')` + } else if (filter.compare === 'is') { + return `${normalizedKey} = ${filter.value}` } }) .join(' AND ') - if (prepend){ + if (prepend) { return 'WHERE ' + conditions - } else { return conditions } @@ -66,7 +67,8 @@ export const PRESET_CONFIG: Record = { cross join unnest(metadata) as m cross join unnest(m.response) as response cross join unnest(m.request) as request - ${generateRexepWhere(filters)} + cross join unnest(request.headers) as headers + ${generateRexepWhere(filters)} GROUP BY timestamp ORDER BY @@ -82,6 +84,7 @@ export const PRESET_CONFIG: Record = { cross join unnest(metadata) as m cross join unnest(m.response) as response cross join unnest(m.request) as request + cross join unnest(request.headers) as headers WHERE response.status_code >= 400 ${generateRexepWhere(filters, false)} @@ -103,18 +106,19 @@ export const PRESET_CONFIG: Record = { cross join unnest(metadata) as m cross join unnest(m.response) as response cross join unnest(m.request) as request - ${generateRexepWhere(filters)} + cross join unnest(request.headers) as headers + ${generateRexepWhere(filters)} GROUP BY timestamp ORDER BY timestamp ASC `, }, - } + }, }, [Presets.AUTH]: { - title: "", - queries: {} + title: '', + queries: {}, }, [Presets.QUERY_PERFORMANCE]: { title: 'Query performance', @@ -201,241 +205,241 @@ select }, }, } - // userAgents: { - // queryType: 'logs', - // sql: ` - // select - // h.user_agent as user_agent, - // cf.asOrganization as request_source, - // count(f.id) as count - // from - // edge_logs as f - // cross join unnest(f.metadata) as m - // cross join unnest(m.request) as r - // cross join unnest(r.cf) as cf - // cross join unnest(r.headers) as h - // group by - // user_agent, - // request_source - // order by - // count desc - // limit 12 - // `, - // }, - // botScores: { - // queryType: 'logs', - // sql: ` - // select - // h.cf_connecting_ip as ip, - // h.cf_ipcountry as country, - // h.user_agent as user_agent, - // r.path as path, - // b.score as bot_score, - // b.verifiedBot as bot_verified, - // count(f.id) as count - // from - // edge_logs as f - // cross join unnest(f.metadata) as m - // cross join unnest(m.request) as r - // cross join unnest(r.cf) as cf - // cross join unnest(r.headers) as h - // cross join unnest(cf.botManagement) as b - // group by - // ip, - // country, - // path, - // user_agent, - // bot_score, - // bot_verified - // order by - // bot_score desc - // limit 20 - // `, - // }, +// userAgents: { +// queryType: 'logs', +// sql: ` +// select +// h.user_agent as user_agent, +// cf.asOrganization as request_source, +// count(f.id) as count +// from +// edge_logs as f +// cross join unnest(f.metadata) as m +// cross join unnest(m.request) as r +// cross join unnest(r.cf) as cf +// cross join unnest(r.headers) as h +// group by +// user_agent, +// request_source +// order by +// count desc +// limit 12 +// `, +// }, +// botScores: { +// queryType: 'logs', +// sql: ` +// select +// h.cf_connecting_ip as ip, +// h.cf_ipcountry as country, +// h.user_agent as user_agent, +// r.path as path, +// b.score as bot_score, +// b.verifiedBot as bot_verified, +// count(f.id) as count +// from +// edge_logs as f +// cross join unnest(f.metadata) as m +// cross join unnest(m.request) as r +// cross join unnest(r.cf) as cf +// cross join unnest(r.headers) as h +// cross join unnest(cf.botManagement) as b +// group by +// ip, +// country, +// path, +// user_agent, +// bot_score, +// bot_verified +// order by +// bot_score desc +// limit 20 +// `, +// }, // }, - // [Presets.AUTH]: { - // title: 'Auth', - // queries: { - // bannedUsers: { - // queryType: 'db', - // sql: (_params) => ` - // select - // count(distinct u.id) as count - // from auth.users u - // where u.banned_until is not null and u.banned_until > now() - // limit 1 - // `, - // }, - // unverifiedUsers: { - // queryType: 'db', - // sql: (params) => ` - // select - // date_trunc('day', u.created_at) as timestamp, - // count(distinct u.id) as count - // from auth.users u - // where u.confirmed_at is null and '${params.iso_timestamp_start}' < u.created_at - // group by timestamp - // order by timestamp desc - // `, - // }, - // signUpProviders: { - // queryType: 'db', - // sql: (params) => ` - // select - // sum(count(u.id)) over (order by date_trunc('day', u.created_at)) as count, - // date_trunc('day', u.created_at) as timestamp, - // i.provider as provider - // from auth.users u - // join auth.identities as i on u.id = i.user_id - // where date_trunc('day', u.created_at) > '${params.iso_timestamp_start}' - // group by timestamp, provider - // order by timestamp desc - // `, - // }, - // failedMigrations: { - // queryType: 'logs', - // sql: ` - // select count(f.id) - // from auth_logs - // where json_value(f.event_message, "$.level") = "fatal" - // and regexp_contains(f.event_message, "error executing migrations") - // limit 1 - // `, - // }, - // dailyActiveUsers: { - // queryType: 'logs', - // sql: ` - // select - // timestamp_trunc(timestamp, day) as timestamp, - // count(distinct json_value(f.event_message, "$.user_id")) as count - // from auth_logs f - // where json_value(f.event_message, "$.action") = "login" - // group by - // timestamp - // order by - // timestamp desc - // `, - // }, - // cumulativeUsers: { - // queryType: 'db', - // sql: (params) => ` +// [Presets.AUTH]: { +// title: 'Auth', +// queries: { +// bannedUsers: { +// queryType: 'db', +// sql: (_params) => ` +// select +// count(distinct u.id) as count +// from auth.users u +// where u.banned_until is not null and u.banned_until > now() +// limit 1 +// `, +// }, +// unverifiedUsers: { +// queryType: 'db', +// sql: (params) => ` +// select +// date_trunc('day', u.created_at) as timestamp, +// count(distinct u.id) as count +// from auth.users u +// where u.confirmed_at is null and '${params.iso_timestamp_start}' < u.created_at +// group by timestamp +// order by timestamp desc +// `, +// }, +// signUpProviders: { +// queryType: 'db', +// sql: (params) => ` +// select +// sum(count(u.id)) over (order by date_trunc('day', u.created_at)) as count, +// date_trunc('day', u.created_at) as timestamp, +// i.provider as provider +// from auth.users u +// join auth.identities as i on u.id = i.user_id +// where date_trunc('day', u.created_at) > '${params.iso_timestamp_start}' +// group by timestamp, provider +// order by timestamp desc +// `, +// }, +// failedMigrations: { +// queryType: 'logs', +// sql: ` +// select count(f.id) +// from auth_logs +// where json_value(f.event_message, "$.level") = "fatal" +// and regexp_contains(f.event_message, "error executing migrations") +// limit 1 +// `, +// }, +// dailyActiveUsers: { +// queryType: 'logs', +// sql: ` +// select +// timestamp_trunc(timestamp, day) as timestamp, +// count(distinct json_value(f.event_message, "$.user_id")) as count +// from auth_logs f +// where json_value(f.event_message, "$.action") = "login" +// group by +// timestamp +// order by +// timestamp desc +// `, +// }, +// cumulativeUsers: { +// queryType: 'db', +// sql: (params) => ` - // select - // date_trunc('day', u.created_at) as timestamp, - // sum(count(u.id)) over (order by date_trunc('day', u.created_at)) as count - // from auth.users as u - // where u.confirmed_at is not null and date_trunc('day', u.created_at) > '${params.iso_timestamp_start}' - // group by timestamp - // order by timestamp desc - // `, - // }, - // newUsers: { - // queryType: 'db', - // sql: (params) => ` - // select - // date_trunc('day', u.created_at) as timestamp, - // count(u.id) as count - // from auth.users as u - // where u.confirmed_at is not null and date_trunc('day', u.created_at) > '${params.iso_timestamp_start}' - // group by - // timestamp - // order by - // timestamp desc - // `, - // }, - // }, - // }, - // [Presets.STORAGE]: { - // title: 'Storage', - // queries: { - // largestObjectsPerBucket: { - // queryType: 'db', - // sql: (_params) => ` - // select * - // from ( - // SELECT - // name, - // metadata->>'mimetype' as mimetype, - // bucket_id, - // (metadata->>'size')::bigint as size, - // row_number() over (partition by bucket_id ORDER BY (metadata->>'size')::bigint DESC ) AS row - // FROM storage.objects - // ) as f - // where f.row <= 10 - // order by size desc`, - // }, - // topDownloaded: { - // queryType: 'logs', - // sql: ` - // select - // count(f.id) as count, - // r.path as path - // from edge_logs f - // cross join unnest(f.metadata) as m - // cross join unnest(m.request) as r - // where starts_with(r.path, '/storage/v1/object') and r.method = 'GET' - // group by path - // order by count desc - // `, - // }, - // mostFiles: { - // queryType: 'db', - // sql: () => ` - // select owner, - // count(id) as file_count, - // sum((metadata->>'size')::bigint)::bigint as total_size - // from storage.objects - // where owner is not null - // group by owner - // order by total_size desc - // `, - // }, - // staleFiles: { - // queryType: 'db', - // sql: () => ` - // select - // count(id) filter (where last_accessed_at < (NOW() - interval '1 month') ) as one_month, - // count(id) filter (where last_accessed_at < (NOW() - interval '3 month') ) as three_month, - // count(id) filter (where last_accessed_at < (NOW() - interval '6 month') ) as six_month, - // count(id) filter (where last_accessed_at < (NOW() - interval '1 year') ) as twelve_month - // from storage.objects - // `, - // }, - // topSizes: { - // queryType: 'logs', - // sql: ` - // SELECT - // count(f.id) as count, - // r.search as search - // from edge_logs f - // cross join unnest(f.metadata) as m - // cross join unnest(m.request) as r - // where starts_with(r.path, '/storage/v1/object') - // and r.method = 'GET' - // and regexp_contains(r.search, 'width|height') - // group by search - // order by count desc - // `, - // }, - // cacheHitRate: { - // queryType: 'logs', - // sql: ` - // SELECT - // timestamp_trunc(timestamp, hour) as timestamp, - // countif( h.cf_cache_status in ('HIT', 'STALE', 'REVALIDATED', 'UPDATING') ) as hit_count, - // countif( h.cf_cache_status in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC') ) as miss_count - // from edge_logs f - // cross join unnest(f.metadata) as m - // cross join unnest(m.request) as r - // cross join unnest(m.response) as res - // cross join unnest(res.headers) as h - // where starts_with(r.path, '/storage/v1/object') - // and r.method = 'GET' - // group by timestamp - // order by timestamp desc - // `, - // }, - // }, - // }, +// select +// date_trunc('day', u.created_at) as timestamp, +// sum(count(u.id)) over (order by date_trunc('day', u.created_at)) as count +// from auth.users as u +// where u.confirmed_at is not null and date_trunc('day', u.created_at) > '${params.iso_timestamp_start}' +// group by timestamp +// order by timestamp desc +// `, +// }, +// newUsers: { +// queryType: 'db', +// sql: (params) => ` +// select +// date_trunc('day', u.created_at) as timestamp, +// count(u.id) as count +// from auth.users as u +// where u.confirmed_at is not null and date_trunc('day', u.created_at) > '${params.iso_timestamp_start}' +// group by +// timestamp +// order by +// timestamp desc +// `, +// }, +// }, +// }, +// [Presets.STORAGE]: { +// title: 'Storage', +// queries: { +// largestObjectsPerBucket: { +// queryType: 'db', +// sql: (_params) => ` +// select * +// from ( +// SELECT +// name, +// metadata->>'mimetype' as mimetype, +// bucket_id, +// (metadata->>'size')::bigint as size, +// row_number() over (partition by bucket_id ORDER BY (metadata->>'size')::bigint DESC ) AS row +// FROM storage.objects +// ) as f +// where f.row <= 10 +// order by size desc`, +// }, +// topDownloaded: { +// queryType: 'logs', +// sql: ` +// select +// count(f.id) as count, +// r.path as path +// from edge_logs f +// cross join unnest(f.metadata) as m +// cross join unnest(m.request) as r +// where starts_with(r.path, '/storage/v1/object') and r.method = 'GET' +// group by path +// order by count desc +// `, +// }, +// mostFiles: { +// queryType: 'db', +// sql: () => ` +// select owner, +// count(id) as file_count, +// sum((metadata->>'size')::bigint)::bigint as total_size +// from storage.objects +// where owner is not null +// group by owner +// order by total_size desc +// `, +// }, +// staleFiles: { +// queryType: 'db', +// sql: () => ` +// select +// count(id) filter (where last_accessed_at < (NOW() - interval '1 month') ) as one_month, +// count(id) filter (where last_accessed_at < (NOW() - interval '3 month') ) as three_month, +// count(id) filter (where last_accessed_at < (NOW() - interval '6 month') ) as six_month, +// count(id) filter (where last_accessed_at < (NOW() - interval '1 year') ) as twelve_month +// from storage.objects +// `, +// }, +// topSizes: { +// queryType: 'logs', +// sql: ` +// SELECT +// count(f.id) as count, +// r.search as search +// from edge_logs f +// cross join unnest(f.metadata) as m +// cross join unnest(m.request) as r +// where starts_with(r.path, '/storage/v1/object') +// and r.method = 'GET' +// and regexp_contains(r.search, 'width|height') +// group by search +// order by count desc +// `, +// }, +// cacheHitRate: { +// queryType: 'logs', +// sql: ` +// SELECT +// timestamp_trunc(timestamp, hour) as timestamp, +// countif( h.cf_cache_status in ('HIT', 'STALE', 'REVALIDATED', 'UPDATING') ) as hit_count, +// countif( h.cf_cache_status in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC') ) as miss_count +// from edge_logs f +// cross join unnest(f.metadata) as m +// cross join unnest(m.request) as r +// cross join unnest(m.response) as res +// cross join unnest(res.headers) as h +// where starts_with(r.path, '/storage/v1/object') +// and r.method = 'GET' +// group by timestamp +// order by timestamp desc +// `, +// }, +// }, +// }, export const DATETIME_FORMAT = 'MMM D, ha'