Merge branch 'feat/unified-logs-simple' into feat/unified-logs-openstatus-view

apps/studio/components/interfaces/UnifiedLogs/UnifiedLogsPreviewer.tsx

@@ -0,0 +1,251 @@
+import dayjs from 'dayjs'
+import { Rewind } from 'lucide-react'
+import { useRouter } from 'next/router'
+import { PropsWithChildren, useEffect, useState } from 'react'
+
+import { useParams } from 'common'
+import PreviewFilterPanel from 'components/interfaces/Settings/Logs/PreviewFilterPanel'
+import LoadingOpacity from 'components/ui/LoadingOpacity'
+import ShimmerLine from 'components/ui/ShimmerLine'
+import useUnifiedLogs from 'hooks/analytics/useUnifiedLogs'
+import { useLogsUrlState } from 'hooks/analytics/useLogsUrlState'
+import { useSelectedLog } from 'hooks/analytics/useSelectedLog'
+import useSingleLog from 'hooks/analytics/useSingleLog'
+import { useFlag } from 'hooks/ui/useFlag'
+import { Button } from 'ui'
+import { LogsBarChart } from 'ui-patterns/LogsBarChart'
+import LogTable from '../Settings/Logs/LogTable'
+import { DatePickerValue } from '../Settings/Logs/Logs.DatePickers'
+import { LogsTableName, PREVIEWER_DATEPICKER_HELPERS } from '../Settings/Logs/Logs.constants'
+import type {
+  Filters,
+  LogSearchCallback,
+  LogTemplate,
+  QueryType,
+} from '../Settings/Logs/Logs.types'
+import { PreviewFilterPanelWithUniversal } from '../Settings/Logs/PreviewFilterPanelWithUniversal'
+
+interface UnifiedLogsPreviewerProps {
+  projectRef: string
+  filterOverride?: Filters
+  condensedLayout?: boolean
+  EmptyState?: React.ReactNode
+  filterPanelClassName?: string
+}
+
+export const UnifiedLogsPreviewer = ({
+  projectRef,
+  filterOverride,
+  condensedLayout = false,
+  children,
+  EmptyState,
+  filterPanelClassName,
+}: PropsWithChildren<UnifiedLogsPreviewerProps>) => {
+  const useUniversalFilterBar = useFlag('universalFilterBar')
+  const router = useRouter()
+  const { db } = useParams()
+
+  const [showChart, setShowChart] = useState(true)
+  const [selectedDatePickerValue, setSelectedDatePickerValue] = useState<DatePickerValue>(
+    getDefaultDatePickerValue()
+  )
+
+  const { search, setSearch, timestampStart, timestampEnd, setTimeRange, filters, setFilters } =
+    useLogsUrlState()
+  const [selectedLogId, setSelectedLogId] = useSelectedLog()
+
+  // Use the custom useUnifiedLogs hook
+  const {
+    error,
+    logData,
+    params,
+    newCount,
+    isLoading,
+    eventChartData,
+    isLoadingOlder,
+    loadOlder,
+    refresh,
+  } = useUnifiedLogs({ projectRef, filterOverride })
+
+  const {
+    data: selectedLog,
+    isLoading: isSelectedLogLoading,
+    error: selectedLogError,
+  } = useSingleLog({
+    projectRef,
+    id: selectedLogId ?? undefined,
+    queryType: 'unified',
+    paramsToMerge: params,
+  })
+
+  // TODO: Move this to useLogsUrlState to simplify LogsPreviewer
+  function getDefaultDatePickerValue() {
+    const iso_timestamp_start = router.query.iso_timestamp_start as string
+    const iso_timestamp_end = router.query.iso_timestamp_end as string
+
+    if (iso_timestamp_start && iso_timestamp_end) {
+      return {
+        to: iso_timestamp_end,
+        from: iso_timestamp_start,
+        text: `${dayjs(iso_timestamp_start).format('DD MMM, HH:mm')} - ${dayjs(iso_timestamp_end).format('DD MMM, HH:mm')}`,
+        isHelper: false,
+      }
+    }
+
+    const defaultDatePickerValue = PREVIEWER_DATEPICKER_HELPERS.find((x) => x.default)
+    return {
+      to: defaultDatePickerValue!.calcTo(),
+      from: defaultDatePickerValue!.calcFrom(),
+      text: defaultDatePickerValue!.text,
+      isHelper: true,
+    }
+  }
+
+  const onSelectTemplate = (template: LogTemplate) => {
+    setFilters({ ...filters, search_query: template.searchString })
+  }
+
+  // For helper date picker values, reset the timestamp start to prevent data caching
+  const handleRefresh = () => {
+    if (selectedDatePickerValue.isHelper) {
+      const helper = PREVIEWER_DATEPICKER_HELPERS.find(
+        (x) => x.text === selectedDatePickerValue.text
+      )
+      if (helper) {
+        const newTimestampStart = helper.calcFrom()
+        setTimeRange(newTimestampStart, timestampEnd)
+      }
+    }
+    refresh()
+  }
+
+  const handleSearch: LogSearchCallback = async (event, { query, to, from }) => {
+    if (event === 'search-input-change') {
+      setSearch(query || '')
+      setSelectedLogId(null)
+    } else if (event === 'event-chart-bar-click') {
+      setTimeRange(from || '', to || '')
+    } else if (event === 'datepicker-change') {
+      setTimeRange(from || '', to || '')
+    }
+  }
+
+  // Common props shared between both filter panel components to avoid duplication
+  const filterPanelProps = {
+    className: filterPanelClassName,
+    csvData: logData,
+    isLoading,
+    newCount,
+    onRefresh: handleRefresh,
+    onSearch: handleSearch,
+    defaultSearchValue: search,
+    defaultToValue: timestampEnd,
+    defaultFromValue: timestampStart,
+    queryUrl: `/project/${projectRef}/logs/explorer?q=${encodeURIComponent(
+      params.sql || ''
+    )}&its=${encodeURIComponent(timestampStart)}&ite=${encodeURIComponent(timestampEnd)}`,
+    onSelectTemplate,
+    filters,
+    onFiltersChange: setFilters,
+    table: LogsTableName.UNIFIED,
+    condensedLayout,
+    isShowingEventChart: showChart,
+    onToggleEventChart: () => setShowChart(!showChart),
+    onSelectedDatabaseChange: (id: string) => {
+      setFilters({ ...filters, database: id !== projectRef ? id : undefined })
+      const { db, ...params } = router.query
+      router.push({
+        pathname: router.pathname,
+        query: id !== projectRef ? { ...router.query, db: id } : params,
+      })
+    },
+    selectedDatePickerValue,
+    setSelectedDatePickerValue,
+  }
+
+  return (
+    <div className="flex-1 flex flex-col h-full">
+      {useUniversalFilterBar ? (
+        // Experimental Universal Filter Bar
+        <PreviewFilterPanelWithUniversal {...filterPanelProps} />
+      ) : (
+        // Legacy Filter Panel
+        <PreviewFilterPanel {...filterPanelProps} />
+      )}
+      {children}
+      <div
+        className={
+          'transition-all duration-500 ' +
+          (showChart && logData.length > 0 ? 'mb-2 mt-1 opacity-100' : 'h-0 opacity-0')
+        }
+      >
+        <div className={condensedLayout ? 'px-3' : ''}>
+          {showChart && (
+            <LogsBarChart
+              data={eventChartData}
+              onBarClick={(datum) => {
+                if (!datum?.timestamp) return
+
+                const datumTimestamp = dayjs(datum.timestamp).toISOString()
+
+                const start = dayjs(datumTimestamp).subtract(1, 'minute').toISOString()
+                const end = dayjs(datumTimestamp).add(1, 'minute').toISOString()
+
+                handleSearch('event-chart-bar-click', {
+                  query: filters.search_query?.toString(),
+                  to: end,
+                  from: start,
+                })
+              }}
+              EmptyState={
+                <div className="flex flex-col items-center justify-center h-[67px]">
+                  <h2 className="text-foreground-light text-xs">No data</h2>
+                  <p className="text-foreground-lighter text-xs">
+                    It may take up to 24 hours for data to refresh
+                  </p>
+                </div>
+              }
+            />
+          )}
+        </div>
+      </div>
+      <div className="relative flex flex-col flex-grow flex-1 overflow-auto">
+        <ShimmerLine active={isLoading} />
+        <LoadingOpacity active={isLoading}>
+          <LogTable
+            projectRef={projectRef}
+            isLoading={isLoading}
+            data={logData}
+            queryType="unified"
+            isHistogramShowing={showChart}
+            onHistogramToggle={() => setShowChart(!showChart)}
+            error={error}
+            EmptyState={EmptyState}
+            onSelectedLogChange={(log) => setSelectedLogId(log?.id ?? null)}
+            selectedLog={selectedLog}
+            isSelectedLogLoading={isSelectedLogLoading}
+            selectedLogError={selectedLogError ?? undefined}
+          />
+        </LoadingOpacity>
+      </div>
+      {!error && logData.length > 0 && (
+        <div className="border-t flex flex-row items-center gap-3 p-2">
+          <Button
+            onClick={loadOlder}
+            icon={<Rewind />}
+            type="default"
+            loading={isLoadingOlder}
+            disabled={isLoadingOlder}
+          >
+            Load older
+          </Button>
+          <div className="text-sm text-foreground-lighter">
+            Showing <span className="font-mono">{logData.length}</span> results
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
+export default UnifiedLogsPreviewer

apps/studio/components/layouts/ProjectLayout/NavigationBar/NavigationBar.utils.tsx

@@ -1,4 +1,4 @@
-import { Blocks, FileText, Lightbulb, List, Settings } from 'lucide-react'
+import { Blocks, FileText, Lightbulb, List, Logs, Settings } from 'lucide-react'
 
 import { ICON_SIZE, ICON_STROKE_WIDTH } from 'components/interfaces/Sidebar'
 import { generateAuthMenu } from 'components/layouts/AuthLayout/AuthLayout.utils'
@@ -149,6 +149,12 @@ export const generateOtherRoutes = (ref?: string, project?: Project): Route[] =>
       icon: <List size={ICON_SIZE} strokeWidth={ICON_STROKE_WIDTH} />,
       link: ref && (isProjectBuilding ? buildingUrl : `/project/${ref}/logs`),
     },
+    {
+      key: 'unified-logs',
+      label: 'Unified Logs',
+      icon: <Logs size={ICON_SIZE} strokeWidth={ICON_STROKE_WIDTH} />,
+      link: ref && (isProjectBuilding ? buildingUrl : `/project/${ref}/unified-logs`),
+    },
     {
       key: 'api',
       label: 'API Docs',

apps/studio/hooks/analytics/useUnifiedLogs.tsx

@@ -0,0 +1,291 @@
+import { useInfiniteQuery, useQuery } from '@tanstack/react-query'
+import dayjs from 'dayjs'
+import { useMemo, useState, useCallback } from 'react'
+
+import {
+  LogsTableName,
+  PREVIEWER_DATEPICKER_HELPERS,
+  getDefaultHelper,
+} from 'components/interfaces/Settings/Logs/Logs.constants'
+import type {
+  Count,
+  EventChart,
+  EventChartData,
+  Filters,
+  LogData,
+  Logs,
+  LogsEndpointParams,
+} from 'components/interfaces/Settings/Logs/Logs.types'
+import {
+  genChartQuery,
+  genCountQuery,
+  genDefaultQuery,
+} from 'components/interfaces/Settings/Logs/Logs.utils'
+import { get } from 'data/fetchers'
+import { useLogsUrlState } from './useLogsUrlState'
+import { useFillTimeseriesSorted } from './useFillTimeseriesSorted'
+import useTimeseriesUnixToIso from './useTimeseriesUnixToIso'
+
+interface UnifiedLogsHook {
+  logData: LogData[]
+  error: string | Object | null
+  newCount: number
+  isSuccess: boolean
+  isLoading: boolean
+  isLoadingOlder: boolean
+  filters: Filters
+  params: LogsEndpointParams
+  oldestTimestamp?: string
+  eventChartData: EventChartData[]
+  loadOlder: () => void
+  refresh: () => void
+}
+
+function useUnifiedLogs({
+  projectRef,
+  filterOverride,
+  limit,
+}: {
+  projectRef: string
+  filterOverride?: Filters
+  limit?: number
+}): UnifiedLogsHook {
+  const defaultHelper = getDefaultHelper(PREVIEWER_DATEPICKER_HELPERS)
+  const [latestRefresh, setLatestRefresh] = useState(new Date().toISOString())
+
+  const {
+    timestampStart: urlTimestampStart,
+    timestampEnd: urlTimestampEnd,
+    filters: urlFilters,
+    search,
+  } = useLogsUrlState()
+
+  const timestampStart = useMemo(
+    () => urlTimestampStart || defaultHelper.calcFrom(),
+    [urlTimestampStart, defaultHelper]
+  )
+  const timestampEnd = useMemo(
+    () => urlTimestampEnd || defaultHelper.calcTo(),
+    [urlTimestampEnd, defaultHelper]
+  )
+
+  const mergedFilters = useMemo(
+    () => ({
+      ...urlFilters,
+      ...filterOverride,
+      ...(search ? { search_query: search } : {}),
+    }),
+    [JSON.stringify(urlFilters), JSON.stringify(filterOverride), search]
+  )
+
+  const params: LogsEndpointParams = useMemo(() => {
+    const currentSql = genDefaultQuery(LogsTableName.UNIFIED, mergedFilters, limit)
+    return {
+      project: projectRef,
+      iso_timestamp_start: timestampStart,
+      iso_timestamp_end: timestampEnd,
+      sql: currentSql,
+    }
+  }, [projectRef, timestampStart, timestampEnd, mergedFilters, limit])
+
+  const queryKey = useMemo(
+    () => ['projects', projectRef, 'unified-logs', params],
+    [projectRef, params]
+  )
+
+  console.log('useUnifiedLogs SQL:', params.sql)
+
+  const {
+    data,
+    isSuccess,
+    isLoading,
+    isRefetching,
+    error: rqError,
+    fetchNextPage,
+    isFetchingNextPage,
+    refetch,
+  } = useInfiniteQuery(
+    queryKey,
+    async ({ signal, pageParam }) => {
+      const { data, error } = await get(`/platform/projects/{ref}/analytics/endpoints/logs.all`, {
+        params: {
+          path: { ref: projectRef },
+          query: {
+            ...params,
+            iso_timestamp_end: pageParam || params.iso_timestamp_end,
+          },
+        },
+        signal,
+      })
+      if (error) {
+        throw error
+      }
+
+      return data as unknown as Logs
+    },
+    {
+      refetchOnWindowFocus: false,
+      getNextPageParam(lastPage) {
+        if ((lastPage.result?.length ?? 0) === 0) {
+          return undefined
+        }
+        const len = lastPage.result.length
+        const { timestamp: tsLimit }: LogData = lastPage.result[len - 1]
+        const isoTsLimit = dayjs.utc(Number(tsLimit / 1000)).toISOString()
+        return isoTsLimit
+      },
+    }
+  )
+
+  const { logData, error, oldestTimestamp } = useMemo(() => {
+    let logData: LogData[] = []
+    let error: null | string | object = rqError ? (rqError as any).message : null
+
+    data?.pages?.forEach((response) => {
+      if (response?.result) {
+        logData = [...logData, ...response.result]
+      }
+      if (!error && response?.error) {
+        error = response.error
+      }
+    })
+
+    // Log unified logs data for debugging
+    if (logData.length > 0) {
+      console.log('Unified Logs Data Sample:', logData.slice(0, 1))
+    }
+
+    const oldestTimestamp = logData.length > 0 ? logData[logData.length - 1]?.timestamp : undefined
+
+    return { logData, error, oldestTimestamp }
+  }, [data?.pages])
+
+  const countQuerySql = useMemo(
+    () => genCountQuery(LogsTableName.UNIFIED, mergedFilters),
+    [mergedFilters]
+  )
+  const countQueryKey = useMemo(
+    () => [
+      'projects',
+      projectRef,
+      'unified-logs-count',
+      {
+        projectRef,
+        sql: countQuerySql,
+        iso_timestamp_start: latestRefresh,
+        iso_timestamp_end: timestampEnd,
+        mergedFilters,
+      },
+    ],
+    [projectRef, countQuerySql, latestRefresh, timestampEnd, mergedFilters]
+  )
+
+  const { data: countData } = useQuery(
+    countQueryKey,
+    async ({ signal }) => {
+      const { data, error } = await get(`/platform/projects/{ref}/analytics/endpoints/logs.all`, {
+        params: {
+          path: { ref: projectRef },
+          query: {
+            project: projectRef,
+            sql: countQuerySql,
+            iso_timestamp_start: latestRefresh,
+            iso_timestamp_end: timestampEnd,
+          },
+        },
+        signal,
+      })
+      if (error) {
+        throw error
+      }
+
+      return data as unknown as Count
+    },
+    {
+      refetchOnWindowFocus: false,
+      refetchInterval: 60000,
+      enabled: !error && data && data?.pages?.length > 0 ? true : false,
+    }
+  )
+
+  const newCount = countData?.result?.[0]?.count ?? 0
+
+  const chartQuery = useMemo(
+    () => genChartQuery(LogsTableName.UNIFIED, params, mergedFilters),
+    [params, mergedFilters]
+  )
+  const chartQueryKey = useMemo(
+    () => [
+      'projects',
+      projectRef,
+      'unified-logs-chart',
+      {
+        projectRef,
+        sql: chartQuery,
+        iso_timestamp_start: timestampStart,
+        iso_timestamp_end: timestampEnd,
+      },
+    ],
+    [projectRef, chartQuery, timestampStart, timestampEnd]
+  )
+
+  const { data: eventChartResponse, refetch: refreshEventChart } = useQuery(
+    chartQueryKey,
+    async ({ signal }) => {
+      const { data, error } = await get(`/platform/projects/{ref}/analytics/endpoints/logs.all`, {
+        params: {
+          path: { ref: projectRef },
+          query: {
+            iso_timestamp_start: timestampStart,
+            iso_timestamp_end: timestampEnd,
+            project: projectRef,
+            sql: chartQuery,
+          },
+        },
+        signal,
+      })
+      if (error) {
+        throw error
+      }
+
+      return data as unknown as EventChart
+    },
+    { refetchOnWindowFocus: false }
+  )
+
+  const refresh = useCallback(async () => {
+    setLatestRefresh(new Date().toISOString())
+    refreshEventChart()
+    refetch()
+  }, [refetch, refreshEventChart])
+
+  const normalizedEventChartData = useTimeseriesUnixToIso(
+    eventChartResponse?.result ?? [],
+    'timestamp'
+  )
+
+  const { data: eventChartData, error: eventChartError } = useFillTimeseriesSorted(
+    normalizedEventChartData,
+    'timestamp',
+    'count',
+    0,
+    timestampStart,
+    timestampEnd || new Date().toISOString()
+  )
+
+  return {
+    newCount,
+    logData,
+    isSuccess,
+    isLoading: isLoading || isRefetching,
+    isLoadingOlder: isFetchingNextPage,
+    error: error || eventChartError,
+    filters: mergedFilters,
+    params,
+    oldestTimestamp: oldestTimestamp ? String(oldestTimestamp) : undefined,
+    eventChartData,
+    refresh,
+    loadOlder: () => fetchNextPage(),
+  }
+}
+export default useUnifiedLogs

apps/studio/pages/project/[ref]/unified-logs.tsx

@@ -1,22 +1,14 @@
 import { useRouter } from 'next/router'
-import { LogsTableName } from 'components/interfaces/Settings/Logs/Logs.constants'
-import LogsPreviewer from 'components/interfaces/Settings/Logs/LogsPreviewer'
 import LogsLayout from 'components/layouts/LogsLayout/LogsLayout'
 import type { NextPageWithLayout } from 'types'
 import DefaultLayout from 'components/layouts/DefaultLayout'
+import UnifiedLogsPreviewer from 'components/interfaces/UnifiedLogs/UnifiedLogsPreviewer'
 
 export const LogPage: NextPageWithLayout = () => {
   const router = useRouter()
   const { ref } = router.query
 
-  return (
-    <LogsPreviewer
-      condensedLayout
-      queryType="unified"
-      projectRef={ref as string}
-      tableName={LogsTableName.UNIFIED}
-    />
-  )
+  return <UnifiedLogsPreviewer condensedLayout projectRef={ref as string} />
 }
 
 LogPage.getLayout = (page) => (