diff --git a/studio/components/interfaces/Settings/Logs/LogEventChart.tsx b/studio/components/interfaces/Settings/Logs/LogEventChart.tsx index 844b460c67e..2b9864fb0a0 100644 --- a/studio/components/interfaces/Settings/Logs/LogEventChart.tsx +++ b/studio/components/interfaces/Settings/Logs/LogEventChart.tsx @@ -1,111 +1,29 @@ -import BarChart from 'components/ui/Charts/BarChart' +import BarChart, { BarChartProps } from 'components/ui/Charts/BarChart' import { EventChartData, isUnixMicro, LogData, unixMicroToIsoTimestamp } from '.' -import { useMemo } from 'react' interface Props { - data?: EventChartData[] + data: EventChartData[] onBarClick: (isoTimestamp: string) => void } -const LogEventChart: React.FC = ({ data, onBarClick }) => { - // TODO: remove once endpoint returns iso timestamp directly - const transformedData = useMemo(() => { - return data?.map((d) => { - const iso = isUnixMicro(d.timestamp) ? unixMicroToIsoTimestamp(d.timestamp) : d.timestamp - - return { - ...d, - timestamp: iso, - } - }) - }, [JSON.stringify(data)]) - - if (!transformedData) return null - - return ( - { - if (!v || !v?.activePayload?.[0]?.payload) return - const unixOrIsoTimestamp = v.activePayload[0].payload.timestamp - const isoTimestamp = isUnixMicro(unixOrIsoTimestamp) - ? unixMicroToIsoTimestamp(unixOrIsoTimestamp) - : unixOrIsoTimestamp - // 60s before - onBarClick(isoTimestamp) - }} - customDateFormat="MMM D, HH:mm:s" - /> - ) -} - -// const useAggregated = (data: LogData[]) => { -// const truncateToMinute = (micro: number) => Math.floor(micro / 1000 / 1000 / 60) * 60 -// const truncateToHour = (micro: number) => Math.floor(micro / 1000 / 1000 / 60 / 60) * 60 * 60 -// const getDiffMinute = (currentTimestamp: number, olderTimestamp: number) => -// Math.round((currentTimestamp - olderTimestamp) / 1000 / 60) -// const getDiffHour = (currentTimestamp: number, olderTimestamp: number) => -// Math.round((currentTimestamp - olderTimestamp) / 1000 / 60 / 60) -// const diffMultiplierMinute = (v: number) => v * 60 -// const diffMultiplierHour = (v: number) => v * 60 * 60 -// return useMemo(() => { -// const oldest = data[data.length - 1] -// if (!oldest) return -// const latest = data[0] -// const oldestDayjs = dayjs(oldest.timestamp / 1000) -// const latestDayjs = dayjs(latest.timestamp / 1000) -// let truncFunc = truncateToMinute -// let getDiff = getDiffMinute -// let diffMultiplier = diffMultiplierMinute -// if (Math.abs(oldestDayjs.diff(latestDayjs, 'day', true)) > 0.25) { -// truncFunc = truncateToHour -// getDiff = getDiffHour -// diffMultiplier = diffMultiplierHour -// } - -// const countMap = data -// .map((d) => { -// // truncate to per-minute -// return { ...d, timestamp: truncateToMinute(d.timestamp) } -// }) -// .reduce((acc, d) => { -// if (acc[d.timestamp]) { -// acc[d.timestamp] = acc[d.timestamp] + 1 -// } else { -// acc[d.timestamp] = 1 -// } -// return acc -// }, {} as TimestampMap) - -// // Add in additional data points for empty minutes -// const oldestEvent = data[data.length - 1] -// if (!oldestEvent) return [] -// const currentTimestamp = new Date().getTime() -// const oldestTimestampMicro = oldestEvent.timestamp -// const latestTimestamp = truncFunc(data[0]['timestamp']) -// const diff = getDiff(currentTimestamp, oldestTimestampMicro / 1000) -// for (const toAdd of Array.from(Array(diff).keys())) { -// const tsToCheck = truncFunc(oldestTimestampMicro) + diffMultiplier(toAdd) -// if (!(tsToCheck in countMap) && tsToCheck <= latestTimestamp) { -// countMap[tsToCheck] = 0 -// } -// } - -// let aggregated = [] -// for (const [key, value] of Object.entries(countMap)) { -// const v: number = Number(key) -// aggregated.push({ -// timestamp: key, -// timestampMicro: v * 1000 * 1000, -// count: value, -// }) -// } -// return aggregated.sort((a, b) => a.timestampMicro - b.timestampMicro) -// }, [JSON.stringify(data)]) -// } +const LogEventChart: React.FC = ({ data, onBarClick }) => ( + { + if (!v || !v?.activePayload?.[0]?.payload) return + const unixOrIsoTimestamp = v.activePayload[0].payload.timestamp + const isoTimestamp = isUnixMicro(unixOrIsoTimestamp) + ? unixMicroToIsoTimestamp(unixOrIsoTimestamp) + : unixOrIsoTimestamp + // 60s before + onBarClick(isoTimestamp) + }} + customDateFormat="MMM D, HH:mm:s" + /> +) export default LogEventChart diff --git a/studio/components/interfaces/Settings/Logs/Logs.utils.ts b/studio/components/interfaces/Settings/Logs/Logs.utils.ts index 5d12a05e676..c2c7a4a85a6 100644 --- a/studio/components/interfaces/Settings/Logs/Logs.utils.ts +++ b/studio/components/interfaces/Settings/Logs/Logs.utils.ts @@ -1,12 +1,14 @@ import { Filters, LogData, LogsEndpointParams, LogsTableName, SQL_FILTER_TEMPLATES } from '.' import dayjs, { Dayjs } from 'dayjs' -import { get } from 'lodash' +import { get, isEqual } from 'lodash' import { StripeSubscription } from 'components/interfaces/Billing' import { useMonaco } from '@monaco-editor/react' import logConstants from 'shared-data/logConstants' import BackwardIterator from 'components/ui/CodeEditor/Providers/BackwardIterator' -import { uniqBy } from 'lodash' +import uniqBy from 'lodash/uniqBy' import { useEffect } from 'react' +import utc from 'dayjs/plugin/utc' +dayjs.extend(utc) /** * Convert a micro timestamp from number/string to iso timestamp @@ -228,13 +230,22 @@ export const genChartQuery = ( ) => { const [startOffset, trunc] = calcChartStart(params) const where = _genWhereStatement(table, filters) + + let joins = 'cross join unnest(t.metadata) as metadata' + if (table === LogsTableName.EDGE) { + joins += ' \n cross join unnest(metadata.request) as request' + joins += ' \n cross join unnest(metadata.response) as response' + } else if (table === LogsTableName.POSTGRES) { + joins += ' \n cross join unnest(metadata.parsed) as parsed' + } + return ` SELECT timestamp_trunc(t.timestamp, ${trunc}) as timestamp, count(t.timestamp) as count FROM ${table} t - cross join unnest(t.metadata) as metadata + ${joins} ${ where ? where + ` and t.timestamp > '${startOffset.toISOString()}'` @@ -349,3 +360,64 @@ export const useEditorHints = () => { } }, [monaco]) } + +/** + * Assumes that all timestamps are in ISO-8601 UTC timezone. + * + * min/max are the datetime strings that extend beyond the given timeseries data. + */ +export const fillTimeseries = ( + timeseriesData: any[], + timestampKey: string, + valueKey: string, + defaultValue: number, + min?: string, + max?: string +) => { + if (timeseriesData.length <= 1 && !(min || max)) return timeseriesData + const dates: unknown[] = timeseriesData.map((datum) => dayjs.utc(datum[timestampKey])) + + const maxDate = max ? dayjs.utc(max) : dayjs.utc(Math.max.apply(null, dates as number[])) + const minDate = min ? dayjs.utc(min) : dayjs.utc(Math.min.apply(null, dates as number[])) + + const truncationSample = timeseriesData.length > 0 ? timeseriesData[0][timestampKey] : min || max + const truncation = getTimestampTruncation(truncationSample) + + let newData = timeseriesData.map((datum) => { + const iso = dayjs.utc(datum[timestampKey]).toISOString() + datum[timestampKey] = iso + return datum + }) + + const diff = maxDate.diff(minDate, truncation as dayjs.UnitType) + for (let i = 0; i <= diff; i++) { + const dateToMaybeAdd = minDate.add(i, truncation as dayjs.ManipulateType) + if (!dates.find((d) => isEqual(d, dateToMaybeAdd))) { + newData.push({ + [timestampKey]: dateToMaybeAdd.toISOString(), + [valueKey]: defaultValue, + }) + } + } + + return newData +} + +export const getTimestampTruncation = (datetime: string): 'second' | 'minute' | 'hour' | 'day' => { + const values = ['second', 'minute', 'hour', 'day'].map((key) => + dayjs(datetime).get(key as dayjs.UnitType) + ) + const zeroCount = values.reduce((acc, value) => { + if (value === 0) { + acc += 1 + } + return acc + }, 0) + let truncation = { + 0: 'second' as const, + 1: 'minute' as const, + 2: 'hour' as const, + 3: 'day' as const, + }[zeroCount]! + return truncation +} diff --git a/studio/components/interfaces/Settings/Logs/LogsPreviewer.tsx b/studio/components/interfaces/Settings/Logs/LogsPreviewer.tsx index 1d718bcfb33..0653913f627 100644 --- a/studio/components/interfaces/Settings/Logs/LogsPreviewer.tsx +++ b/studio/components/interfaces/Settings/Logs/LogsPreviewer.tsx @@ -191,9 +191,9 @@ export const LogsPreviewer: React.FC = ({ } >
- {showChart && ( + {!isLoading && showChart && ( { handleSearch('event-chart-bar-click', { query: filters.search_query as string, diff --git a/studio/hooks/analytics/useFillTimeseriesSorted.ts b/studio/hooks/analytics/useFillTimeseriesSorted.ts new file mode 100644 index 00000000000..5c266ee67ee --- /dev/null +++ b/studio/hooks/analytics/useFillTimeseriesSorted.ts @@ -0,0 +1,15 @@ +import { fillTimeseries } from 'components/interfaces/Settings/Logs' +import { useMemo } from 'react' + +/** + * Convenience hook for memoized filling of timeseries data. + */ +const useFillTimeseriesSorted = (...args: Parameters) => { + return useMemo(() => { + const filled = fillTimeseries(...args) + return filled.sort((a, b) => { + return (new Date(a[args[1]]) as any) - (new Date(b[args[1]]) as any) + }) + }, [JSON.stringify(args[0])]) +} +export default useFillTimeseriesSorted diff --git a/studio/hooks/analytics/useLogsPreview.tsx b/studio/hooks/analytics/useLogsPreview.tsx index 15e5be9a7ab..d8aec5cb351 100644 --- a/studio/hooks/analytics/useLogsPreview.tsx +++ b/studio/hooks/analytics/useLogsPreview.tsx @@ -15,10 +15,12 @@ import { LogsTableName, PREVIEWER_DATEPICKER_HELPERS, } from 'components/interfaces/Settings/Logs' -import { Dispatch, SetStateAction, useEffect, useRef, useState } from 'react' +import { Dispatch, SetStateAction, useEffect, useMemo, useRef, useState } from 'react' import { API_URL } from 'lib/constants' import { get } from 'lib/common/fetch' import dayjs from 'dayjs' +import useFillTimeseriesSorted from './useFillTimeseriesSorted' +import useTimeseriesUnixToIso from './useTimeseriesUnixToIso' interface Data { logData: LogData[] @@ -29,7 +31,7 @@ interface Data { filters: Filters params: LogsEndpointParams oldestTimestamp?: string - eventChartData: EventChartData[] | null + eventChartData: EventChartData[] } interface Handlers { loadOlder: () => void @@ -176,6 +178,21 @@ function useLogsPreview( setFilters({ ...newFilters, ...filterOverride }) } } + + const normalizedEventChartData = useTimeseriesUnixToIso( + eventChartResponse?.result || [], + 'timestamp' + ) + + const eventChartData = useFillTimeseriesSorted( + normalizedEventChartData, + 'timestamp', + 'count', + 0, + params.iso_timestamp_start, + params.iso_timestamp_end + ) + return [ { newCount, @@ -186,7 +203,7 @@ function useLogsPreview( filters, params, oldestTimestamp: oldestTimestamp ? String(oldestTimestamp) : undefined, - eventChartData: eventChartResponse?.result || null, + eventChartData, }, { setFilters: handleSetFilters, diff --git a/studio/hooks/analytics/useTimeseriesUnixToIso.ts b/studio/hooks/analytics/useTimeseriesUnixToIso.ts new file mode 100644 index 00000000000..1172dd29518 --- /dev/null +++ b/studio/hooks/analytics/useTimeseriesUnixToIso.ts @@ -0,0 +1,21 @@ +import { isUnixMicro, unixMicroToIsoTimestamp } from 'components/interfaces/Settings/Logs' +import { useMemo } from 'react' + +/** + * Convenience hook for converting timeseries timestamp from unix microsecond to iso + * + * memoized + */ +const useTimeseriesUnixToIso = (data: any[], timestampKey: string) => { + return useMemo(() => { + // check if need to convert or not + if (data.length === 0) return data + if (!isUnixMicro(data[0][timestampKey])) return data + + return data?.map((d) => { + d[timestampKey] = unixMicroToIsoTimestamp(d[timestampKey]) + return d + }) + }, [JSON.stringify(data)]) +} +export default useTimeseriesUnixToIso diff --git a/studio/tests/pages/projects/Logs.utils.test.js b/studio/tests/pages/projects/Logs.utils.test.js index 1b4f66eee48..b705fa4c729 100644 --- a/studio/tests/pages/projects/Logs.utils.test.js +++ b/studio/tests/pages/projects/Logs.utils.test.js @@ -1,11 +1,13 @@ import { ensureNoTimestampConflict, + fillTimeseries, genChartQuery, genDefaultQuery, LogsTableName, SQL_FILTER_TEMPLATES, } from 'components/interfaces/Settings/Logs' import dayjs from 'dayjs' +import { isEqual } from 'lodash' describe.each(Object.values(LogsTableName))('%s', (table) => { const templates = SQL_FILTER_TEMPLATES[table] @@ -98,3 +100,57 @@ test.each([ expect(result[0]).toEqual(expected[0]) expect(result[1]).toEqual(expected[1]) }) + +// test for log trunc filling +test.each([ + { + // truncate timestamp string from bigquery + case: 'bq utc timestamp, truncated minutely', + data: [ + { timestamp: '2023-04-26T17:18:00', count: 123 }, + { timestamp: '2023-04-26T17:20:00', count: 123 }, + ], + // expected + len: 3, + includes: [{ timestamp: '2023-04-26T17:19:00.000Z', count: 0 }], + }, + // hourly truncation + { + case: 'bq timestamp truncated hourly, different options', + data: [ + { period: '2023-04-26T17:00:00.000Z', value: 123 }, + { period: '2023-04-26T20:00:00.000Z', value: 123 }, + ], + // expected + len: 4, + defaultVal: 1, + valKey: 'value', + tsKey: 'period', + includes: [ + { period: '2023-04-26T18:00:00.000Z', value: 1 }, + { period: '2023-04-26T19:00:00.000Z', value: 1 }, + ], + }, + // fill beyong data min/max + { + case: 'fill beyond min/max', + data: [], + len: 3, + min: '2023-04-26T18:00:00.000Z', + max: '2023-04-26T20:00:00.000Z', + includes: [ + { timestamp: '2023-04-26T18:00:00.000Z', count: 0 }, + { timestamp: '2023-04-26T19:00:00.000Z', count: 0 }, + { timestamp: '2023-04-26T20:00:00.000Z', count: 0 }, + ], + }, +])( + 'fillTimeseries : $case', + ({ data, len, includes, min, max, tsKey = 'timestamp', valKey = 'count', defaultVal = 0 }) => { + const result = fillTimeseries(data, tsKey, valKey, defaultVal, min, max) + expect(result.length).toEqual(len) + for (const inc of includes) { + expect(result.find((d) => isEqual(d, inc))).toBeTruthy() + } + } +)