feat: LogEventChart timestamp filling, chart query bugfix for filters

studio/components/interfaces/Settings/Logs/LogEventChart.tsx

@@ -1,111 +1,29 @@
-import BarChart from 'components/ui/Charts/BarChart'
+import BarChart, { BarChartProps } from 'components/ui/Charts/BarChart'
 import { EventChartData, isUnixMicro, LogData, unixMicroToIsoTimestamp } from '.'
-import { useMemo } from 'react'
 
 interface Props {
-  data?: EventChartData[]
+  data: EventChartData[]
   onBarClick: (isoTimestamp: string) => void
 }
 
-const LogEventChart: React.FC<Props> = ({ data, onBarClick }) => {
-  // TODO: remove once endpoint returns iso timestamp directly
-  const transformedData = useMemo(() => {
-    return data?.map((d) => {
-      const iso = isUnixMicro(d.timestamp) ? unixMicroToIsoTimestamp(d.timestamp) : d.timestamp
-
-      return {
-        ...d,
-        timestamp: iso,
-      }
-    })
-  }, [JSON.stringify(data)])
-
-  if (!transformedData) return null
-
-  return (
-    <BarChart
-      minimalHeader
-      size="tiny"
-      yAxisKey="count"
-      xAxisKey="timestamp"
-      data={transformedData}
-      title="Logs / Time"
-      onBarClick={(v?: { activePayload?: { payload: any }[] }) => {
-        if (!v || !v?.activePayload?.[0]?.payload) return
-        const unixOrIsoTimestamp = v.activePayload[0].payload.timestamp
-        const isoTimestamp = isUnixMicro(unixOrIsoTimestamp)
-          ? unixMicroToIsoTimestamp(unixOrIsoTimestamp)
-          : unixOrIsoTimestamp
-        // 60s before
-        onBarClick(isoTimestamp)
-      }}
-      customDateFormat="MMM D, HH:mm:s"
-    />
-  )
-}
-
-// const useAggregated = (data: LogData[]) => {
-//   const truncateToMinute = (micro: number) => Math.floor(micro / 1000 / 1000 / 60) * 60
-//   const truncateToHour = (micro: number) => Math.floor(micro / 1000 / 1000 / 60 / 60) * 60 * 60
-//   const getDiffMinute = (currentTimestamp: number, olderTimestamp: number) =>
-//     Math.round((currentTimestamp - olderTimestamp) / 1000 / 60)
-//   const getDiffHour = (currentTimestamp: number, olderTimestamp: number) =>
-//     Math.round((currentTimestamp - olderTimestamp) / 1000 / 60 / 60)
-//   const diffMultiplierMinute = (v: number) => v * 60
-//   const diffMultiplierHour = (v: number) => v * 60 * 60
-//   return useMemo(() => {
-//     const oldest = data[data.length - 1]
-//     if (!oldest) return
-//     const latest = data[0]
-//     const oldestDayjs = dayjs(oldest.timestamp / 1000)
-//     const latestDayjs = dayjs(latest.timestamp / 1000)
-//     let truncFunc = truncateToMinute
-//     let getDiff = getDiffMinute
-//     let diffMultiplier = diffMultiplierMinute
-//     if (Math.abs(oldestDayjs.diff(latestDayjs, 'day', true)) > 0.25) {
-//       truncFunc = truncateToHour
-//       getDiff = getDiffHour
-//       diffMultiplier = diffMultiplierHour
-//     }
-
-//     const countMap = data
-//       .map((d) => {
-//         //   truncate to per-minute
-//         return { ...d, timestamp: truncateToMinute(d.timestamp) }
-//       })
-//       .reduce((acc, d) => {
-//         if (acc[d.timestamp]) {
-//           acc[d.timestamp] = acc[d.timestamp] + 1
-//         } else {
-//           acc[d.timestamp] = 1
-//         }
-//         return acc
-//       }, {} as TimestampMap)
-
-//     // Add in additional data points for empty minutes
-//     const oldestEvent = data[data.length - 1]
-//     if (!oldestEvent) return []
-//     const currentTimestamp = new Date().getTime()
-//     const oldestTimestampMicro = oldestEvent.timestamp
-//     const latestTimestamp = truncFunc(data[0]['timestamp'])
-//     const diff = getDiff(currentTimestamp, oldestTimestampMicro / 1000)
-//     for (const toAdd of Array.from(Array(diff).keys())) {
-//       const tsToCheck = truncFunc(oldestTimestampMicro) + diffMultiplier(toAdd)
-//       if (!(tsToCheck in countMap) && tsToCheck <= latestTimestamp) {
-//         countMap[tsToCheck] = 0
-//       }
-//     }
-
-//     let aggregated = []
-//     for (const [key, value] of Object.entries(countMap)) {
-//       const v: number = Number(key)
-//       aggregated.push({
-//         timestamp: key,
-//         timestampMicro: v * 1000 * 1000,
-//         count: value,
-//       })
-//     }
-//     return aggregated.sort((a, b) => a.timestampMicro - b.timestampMicro)
-//   }, [JSON.stringify(data)])
-// }
+const LogEventChart: React.FC<Props> = ({ data, onBarClick }) => (
+  <BarChart
+    minimalHeader
+    size="tiny"
+    yAxisKey="count"
+    xAxisKey="timestamp"
+    data={data as unknown as BarChartProps['data']}
+    title="Logs / Time"
+    onBarClick={(v?: { activePayload?: { payload: any }[] }) => {
+      if (!v || !v?.activePayload?.[0]?.payload) return
+      const unixOrIsoTimestamp = v.activePayload[0].payload.timestamp
+      const isoTimestamp = isUnixMicro(unixOrIsoTimestamp)
+        ? unixMicroToIsoTimestamp(unixOrIsoTimestamp)
+        : unixOrIsoTimestamp
+      // 60s before
+      onBarClick(isoTimestamp)
+    }}
+    customDateFormat="MMM D, HH:mm:s"
+  />
+)
 export default LogEventChart

studio/components/interfaces/Settings/Logs/Logs.utils.ts

@@ -1,12 +1,14 @@
 import { Filters, LogData, LogsEndpointParams, LogsTableName, SQL_FILTER_TEMPLATES } from '.'
 import dayjs, { Dayjs } from 'dayjs'
-import { get } from 'lodash'
+import { get, isEqual } from 'lodash'
 import { StripeSubscription } from 'components/interfaces/Billing'
 import { useMonaco } from '@monaco-editor/react'
 import logConstants from 'shared-data/logConstants'
 import BackwardIterator from 'components/ui/CodeEditor/Providers/BackwardIterator'
-import { uniqBy } from 'lodash'
+import uniqBy from 'lodash/uniqBy'
 import { useEffect } from 'react'
+import utc from 'dayjs/plugin/utc'
+dayjs.extend(utc)
 
 /**
  * Convert a micro timestamp from number/string to iso timestamp
@@ -228,13 +230,22 @@ export const genChartQuery = (
 ) => {
   const [startOffset, trunc] = calcChartStart(params)
   const where = _genWhereStatement(table, filters)
+
+  let joins = 'cross join unnest(t.metadata) as metadata'
+  if (table === LogsTableName.EDGE) {
+    joins += ' \n  cross join unnest(metadata.request) as request'
+    joins += ' \n  cross join unnest(metadata.response) as response'
+  } else if (table === LogsTableName.POSTGRES) {
+    joins += ' \n  cross join unnest(metadata.parsed) as parsed'
+  }
+
   return `
 SELECT
   timestamp_trunc(t.timestamp, ${trunc}) as timestamp,
   count(t.timestamp) as count
 FROM
   ${table} t
-  cross join unnest(t.metadata) as metadata
+  ${joins}
   ${
     where
       ? where + ` and t.timestamp > '${startOffset.toISOString()}'`
@@ -349,3 +360,64 @@ export const useEditorHints = () => {
     }
   }, [monaco])
 }
+
+/**
+ * Assumes that all timestamps are in ISO-8601 UTC timezone.
+ *
+ * min/max are the datetime strings that extend beyond the given timeseries data.
+ */
+export const fillTimeseries = (
+  timeseriesData: any[],
+  timestampKey: string,
+  valueKey: string,
+  defaultValue: number,
+  min?: string,
+  max?: string
+) => {
+  if (timeseriesData.length <= 1 && !(min || max)) return timeseriesData
+  const dates: unknown[] = timeseriesData.map((datum) => dayjs.utc(datum[timestampKey]))
+
+  const maxDate = max ? dayjs.utc(max) : dayjs.utc(Math.max.apply(null, dates as number[]))
+  const minDate = min ? dayjs.utc(min) : dayjs.utc(Math.min.apply(null, dates as number[]))
+
+  const truncationSample = timeseriesData.length > 0 ? timeseriesData[0][timestampKey] : min || max
+  const truncation = getTimestampTruncation(truncationSample)
+
+  let newData = timeseriesData.map((datum) => {
+    const iso = dayjs.utc(datum[timestampKey]).toISOString()
+    datum[timestampKey] = iso
+    return datum
+  })
+
+  const diff = maxDate.diff(minDate, truncation as dayjs.UnitType)
+  for (let i = 0; i <= diff; i++) {
+    const dateToMaybeAdd = minDate.add(i, truncation as dayjs.ManipulateType)
+    if (!dates.find((d) => isEqual(d, dateToMaybeAdd))) {
+      newData.push({
+        [timestampKey]: dateToMaybeAdd.toISOString(),
+        [valueKey]: defaultValue,
+      })
+    }
+  }
+
+  return newData
+}
+
+export const getTimestampTruncation = (datetime: string): 'second' | 'minute' | 'hour' | 'day' => {
+  const values = ['second', 'minute', 'hour', 'day'].map((key) =>
+    dayjs(datetime).get(key as dayjs.UnitType)
+  )
+  const zeroCount = values.reduce((acc, value) => {
+    if (value === 0) {
+      acc += 1
+    }
+    return acc
+  }, 0)
+  let truncation = {
+    0: 'second' as const,
+    1: 'minute' as const,
+    2: 'hour' as const,
+    3: 'day' as const,
+  }[zeroCount]!
+  return truncation
+}

studio/components/interfaces/Settings/Logs/LogsPreviewer.tsx

@@ -191,9 +191,9 @@ export const LogsPreviewer: React.FC<Props> = ({
         }
       >
         <div className={condensedLayout ? 'px-4' : ''}>
-          {showChart && (
+          {!isLoading && showChart && (
             <LogEventChart
-              data={!isLoading && eventChartData ? eventChartData : undefined}
+              data={ eventChartData }
               onBarClick={(isoTimestamp) => {
                 handleSearch('event-chart-bar-click', {
                   query: filters.search_query as string,

studio/hooks/analytics/useFillTimeseriesSorted.ts

@@ -0,0 +1,15 @@
+import { fillTimeseries } from 'components/interfaces/Settings/Logs'
+import { useMemo } from 'react'
+
+/**
+ * Convenience hook for memoized filling of timeseries data.
+ */
+const useFillTimeseriesSorted = (...args: Parameters<typeof fillTimeseries>) => {
+  return useMemo(() => {
+    const filled = fillTimeseries(...args)
+    return filled.sort((a, b) => {
+      return (new Date(a[args[1]]) as any) - (new Date(b[args[1]]) as any)
+    })
+  }, [JSON.stringify(args[0])])
+}
+export default useFillTimeseriesSorted

studio/hooks/analytics/useLogsPreview.tsx

@@ -15,10 +15,12 @@ import {
   LogsTableName,
   PREVIEWER_DATEPICKER_HELPERS,
 } from 'components/interfaces/Settings/Logs'
-import { Dispatch, SetStateAction, useEffect, useRef, useState } from 'react'
+import { Dispatch, SetStateAction, useEffect, useMemo, useRef, useState } from 'react'
 import { API_URL } from 'lib/constants'
 import { get } from 'lib/common/fetch'
 import dayjs from 'dayjs'
+import useFillTimeseriesSorted from './useFillTimeseriesSorted'
+import useTimeseriesUnixToIso from './useTimeseriesUnixToIso'
 
 interface Data {
   logData: LogData[]
@@ -29,7 +31,7 @@ interface Data {
   filters: Filters
   params: LogsEndpointParams
   oldestTimestamp?: string
-  eventChartData: EventChartData[] | null
+  eventChartData: EventChartData[]
 }
 interface Handlers {
   loadOlder: () => void
@@ -176,6 +178,21 @@ function useLogsPreview(
       setFilters({ ...newFilters, ...filterOverride })
     }
   }
+
+  const normalizedEventChartData = useTimeseriesUnixToIso(
+    eventChartResponse?.result || [],
+    'timestamp'
+  )
+
+  const eventChartData = useFillTimeseriesSorted(
+    normalizedEventChartData,
+    'timestamp',
+    'count',
+    0,
+    params.iso_timestamp_start,
+    params.iso_timestamp_end
+  )
+
   return [
     {
       newCount,
@@ -186,7 +203,7 @@ function useLogsPreview(
       filters,
       params,
       oldestTimestamp: oldestTimestamp ? String(oldestTimestamp) : undefined,
-      eventChartData: eventChartResponse?.result || null,
+      eventChartData,
     },
     {
       setFilters: handleSetFilters,

studio/hooks/analytics/useTimeseriesUnixToIso.ts

@@ -0,0 +1,21 @@
+import { isUnixMicro, unixMicroToIsoTimestamp } from 'components/interfaces/Settings/Logs'
+import { useMemo } from 'react'
+
+/**
+ * Convenience hook for converting timeseries timestamp from unix microsecond to iso
+ *
+ * memoized
+ */
+const useTimeseriesUnixToIso = (data: any[], timestampKey: string) => {
+  return useMemo(() => {
+    // check if need to convert or not
+    if (data.length === 0) return data
+    if (!isUnixMicro(data[0][timestampKey])) return data
+
+    return data?.map((d) => {
+      d[timestampKey] = unixMicroToIsoTimestamp(d[timestampKey])
+      return d
+    })
+  }, [JSON.stringify(data)])
+}
+export default useTimeseriesUnixToIso

studio/tests/pages/projects/Logs.utils.test.js

@@ -1,11 +1,13 @@
 import {
   ensureNoTimestampConflict,
+  fillTimeseries,
   genChartQuery,
   genDefaultQuery,
   LogsTableName,
   SQL_FILTER_TEMPLATES,
 } from 'components/interfaces/Settings/Logs'
 import dayjs from 'dayjs'
+import { isEqual } from 'lodash'
 
 describe.each(Object.values(LogsTableName))('%s', (table) => {
   const templates = SQL_FILTER_TEMPLATES[table]
@@ -98,3 +100,57 @@ test.each([
   expect(result[0]).toEqual(expected[0])
   expect(result[1]).toEqual(expected[1])
 })
+
+// test for log trunc filling
+test.each([
+  {
+    // truncate timestamp string from bigquery
+    case: 'bq utc timestamp, truncated minutely',
+    data: [
+      { timestamp: '2023-04-26T17:18:00', count: 123 },
+      { timestamp: '2023-04-26T17:20:00', count: 123 },
+    ],
+    // expected
+    len: 3,
+    includes: [{ timestamp: '2023-04-26T17:19:00.000Z', count: 0 }],
+  },
+  // hourly truncation
+  {
+    case: 'bq timestamp truncated hourly, different options',
+    data: [
+      { period: '2023-04-26T17:00:00.000Z', value: 123 },
+      { period: '2023-04-26T20:00:00.000Z', value: 123 },
+    ],
+    // expected
+    len: 4,
+    defaultVal: 1,
+    valKey: 'value',
+    tsKey: 'period',
+    includes: [
+      { period: '2023-04-26T18:00:00.000Z', value: 1 },
+      { period: '2023-04-26T19:00:00.000Z', value: 1 },
+    ],
+  },
+  // fill beyong data min/max
+  {
+    case: 'fill beyond min/max',
+    data: [],
+    len: 3,
+    min: '2023-04-26T18:00:00.000Z',
+    max: '2023-04-26T20:00:00.000Z',
+    includes: [
+      { timestamp: '2023-04-26T18:00:00.000Z', count: 0 },
+      { timestamp: '2023-04-26T19:00:00.000Z', count: 0 },
+      { timestamp: '2023-04-26T20:00:00.000Z', count: 0 },
+    ],
+  },
+])(
+  'fillTimeseries : $case',
+  ({ data, len, includes, min, max, tsKey = 'timestamp', valKey = 'count', defaultVal = 0 }) => {
+    const result = fillTimeseries(data, tsKey, valKey, defaultVal, min, max)
+    expect(result.length).toEqual(len)
+    for (const inc of includes) {
+      expect(result.find((d) => isEqual(d, inc))).toBeTruthy()
+    }
+  }
+)