From 64681e52856a9e74746257ad5cf6bdea99d2c80a Mon Sep 17 00:00:00 2001 From: Jonathan Summers-Muir Date: Tue, 29 Apr 2025 23:09:17 +0800 Subject: [PATCH 1/2] move unified logs --- .../ProjectLayout/NavigationBar/NavigationBar.utils.tsx | 8 +++++++- .../pages/project/[ref]/{logs => }/unified-logs.tsx | 0 2 files changed, 7 insertions(+), 1 deletion(-) rename apps/studio/pages/project/[ref]/{logs => }/unified-logs.tsx (100%) diff --git a/apps/studio/components/layouts/ProjectLayout/NavigationBar/NavigationBar.utils.tsx b/apps/studio/components/layouts/ProjectLayout/NavigationBar/NavigationBar.utils.tsx index 50449c9e005..83f7788a2ac 100644 --- a/apps/studio/components/layouts/ProjectLayout/NavigationBar/NavigationBar.utils.tsx +++ b/apps/studio/components/layouts/ProjectLayout/NavigationBar/NavigationBar.utils.tsx @@ -1,4 +1,4 @@ -import { Blocks, FileText, Lightbulb, List, Settings } from 'lucide-react' +import { Blocks, FileText, Lightbulb, List, Logs, Settings } from 'lucide-react' import { ICON_SIZE, ICON_STROKE_WIDTH } from 'components/interfaces/Sidebar' import { generateAuthMenu } from 'components/layouts/AuthLayout/AuthLayout.utils' @@ -149,6 +149,12 @@ export const generateOtherRoutes = (ref?: string, project?: Project): Route[] => icon: , link: ref && (isProjectBuilding ? buildingUrl : `/project/${ref}/logs`), }, + { + key: 'unified-logs', + label: 'Unified Logs', + icon: , + link: ref && (isProjectBuilding ? buildingUrl : `/project/${ref}/unified-logs`), + }, { key: 'api', label: 'API Docs', diff --git a/apps/studio/pages/project/[ref]/logs/unified-logs.tsx b/apps/studio/pages/project/[ref]/unified-logs.tsx similarity index 100% rename from apps/studio/pages/project/[ref]/logs/unified-logs.tsx rename to apps/studio/pages/project/[ref]/unified-logs.tsx From d18c60bcd40b923f0d0d464f125281f1d480df73 Mon Sep 17 00:00:00 2001 From: Jonathan Summers-Muir Date: Wed, 30 Apr 2025 00:07:01 +0800 Subject: [PATCH 2/2] init --- .../UnifiedLogs/UnifiedLogsPreviewer.tsx | 251 +++++++++++++++ .../studio/hooks/analytics/useUnifiedLogs.tsx | 291 ++++++++++++++++++ .../pages/project/[ref]/unified-logs.tsx | 12 +- 3 files changed, 544 insertions(+), 10 deletions(-) create mode 100644 apps/studio/components/interfaces/UnifiedLogs/UnifiedLogsPreviewer.tsx create mode 100644 apps/studio/hooks/analytics/useUnifiedLogs.tsx diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogsPreviewer.tsx b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogsPreviewer.tsx new file mode 100644 index 00000000000..4d98010eebe --- /dev/null +++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogsPreviewer.tsx @@ -0,0 +1,251 @@ +import dayjs from 'dayjs' +import { Rewind } from 'lucide-react' +import { useRouter } from 'next/router' +import { PropsWithChildren, useEffect, useState } from 'react' + +import { useParams } from 'common' +import PreviewFilterPanel from 'components/interfaces/Settings/Logs/PreviewFilterPanel' +import LoadingOpacity from 'components/ui/LoadingOpacity' +import ShimmerLine from 'components/ui/ShimmerLine' +import useUnifiedLogs from 'hooks/analytics/useUnifiedLogs' +import { useLogsUrlState } from 'hooks/analytics/useLogsUrlState' +import { useSelectedLog } from 'hooks/analytics/useSelectedLog' +import useSingleLog from 'hooks/analytics/useSingleLog' +import { useFlag } from 'hooks/ui/useFlag' +import { Button } from 'ui' +import { LogsBarChart } from 'ui-patterns/LogsBarChart' +import LogTable from '../Settings/Logs/LogTable' +import { DatePickerValue } from '../Settings/Logs/Logs.DatePickers' +import { LogsTableName, PREVIEWER_DATEPICKER_HELPERS } from '../Settings/Logs/Logs.constants' +import type { + Filters, + LogSearchCallback, + LogTemplate, + QueryType, +} from '../Settings/Logs/Logs.types' +import { PreviewFilterPanelWithUniversal } from '../Settings/Logs/PreviewFilterPanelWithUniversal' + +interface UnifiedLogsPreviewerProps { + projectRef: string + filterOverride?: Filters + condensedLayout?: boolean + EmptyState?: React.ReactNode + filterPanelClassName?: string +} + +export const UnifiedLogsPreviewer = ({ + projectRef, + filterOverride, + condensedLayout = false, + children, + EmptyState, + filterPanelClassName, +}: PropsWithChildren) => { + const useUniversalFilterBar = useFlag('universalFilterBar') + const router = useRouter() + const { db } = useParams() + + const [showChart, setShowChart] = useState(true) + const [selectedDatePickerValue, setSelectedDatePickerValue] = useState( + getDefaultDatePickerValue() + ) + + const { search, setSearch, timestampStart, timestampEnd, setTimeRange, filters, setFilters } = + useLogsUrlState() + const [selectedLogId, setSelectedLogId] = useSelectedLog() + + // Use the custom useUnifiedLogs hook + const { + error, + logData, + params, + newCount, + isLoading, + eventChartData, + isLoadingOlder, + loadOlder, + refresh, + } = useUnifiedLogs({ projectRef, filterOverride }) + + const { + data: selectedLog, + isLoading: isSelectedLogLoading, + error: selectedLogError, + } = useSingleLog({ + projectRef, + id: selectedLogId ?? undefined, + queryType: 'unified', + paramsToMerge: params, + }) + + // TODO: Move this to useLogsUrlState to simplify LogsPreviewer + function getDefaultDatePickerValue() { + const iso_timestamp_start = router.query.iso_timestamp_start as string + const iso_timestamp_end = router.query.iso_timestamp_end as string + + if (iso_timestamp_start && iso_timestamp_end) { + return { + to: iso_timestamp_end, + from: iso_timestamp_start, + text: `${dayjs(iso_timestamp_start).format('DD MMM, HH:mm')} - ${dayjs(iso_timestamp_end).format('DD MMM, HH:mm')}`, + isHelper: false, + } + } + + const defaultDatePickerValue = PREVIEWER_DATEPICKER_HELPERS.find((x) => x.default) + return { + to: defaultDatePickerValue!.calcTo(), + from: defaultDatePickerValue!.calcFrom(), + text: defaultDatePickerValue!.text, + isHelper: true, + } + } + + const onSelectTemplate = (template: LogTemplate) => { + setFilters({ ...filters, search_query: template.searchString }) + } + + // For helper date picker values, reset the timestamp start to prevent data caching + const handleRefresh = () => { + if (selectedDatePickerValue.isHelper) { + const helper = PREVIEWER_DATEPICKER_HELPERS.find( + (x) => x.text === selectedDatePickerValue.text + ) + if (helper) { + const newTimestampStart = helper.calcFrom() + setTimeRange(newTimestampStart, timestampEnd) + } + } + refresh() + } + + const handleSearch: LogSearchCallback = async (event, { query, to, from }) => { + if (event === 'search-input-change') { + setSearch(query || '') + setSelectedLogId(null) + } else if (event === 'event-chart-bar-click') { + setTimeRange(from || '', to || '') + } else if (event === 'datepicker-change') { + setTimeRange(from || '', to || '') + } + } + + // Common props shared between both filter panel components to avoid duplication + const filterPanelProps = { + className: filterPanelClassName, + csvData: logData, + isLoading, + newCount, + onRefresh: handleRefresh, + onSearch: handleSearch, + defaultSearchValue: search, + defaultToValue: timestampEnd, + defaultFromValue: timestampStart, + queryUrl: `/project/${projectRef}/logs/explorer?q=${encodeURIComponent( + params.sql || '' + )}&its=${encodeURIComponent(timestampStart)}&ite=${encodeURIComponent(timestampEnd)}`, + onSelectTemplate, + filters, + onFiltersChange: setFilters, + table: LogsTableName.UNIFIED, + condensedLayout, + isShowingEventChart: showChart, + onToggleEventChart: () => setShowChart(!showChart), + onSelectedDatabaseChange: (id: string) => { + setFilters({ ...filters, database: id !== projectRef ? id : undefined }) + const { db, ...params } = router.query + router.push({ + pathname: router.pathname, + query: id !== projectRef ? { ...router.query, db: id } : params, + }) + }, + selectedDatePickerValue, + setSelectedDatePickerValue, + } + + return ( +
+ {useUniversalFilterBar ? ( + // Experimental Universal Filter Bar + + ) : ( + // Legacy Filter Panel + + )} + {children} +
0 ? 'mb-2 mt-1 opacity-100' : 'h-0 opacity-0') + } + > +
+ {showChart && ( + { + if (!datum?.timestamp) return + + const datumTimestamp = dayjs(datum.timestamp).toISOString() + + const start = dayjs(datumTimestamp).subtract(1, 'minute').toISOString() + const end = dayjs(datumTimestamp).add(1, 'minute').toISOString() + + handleSearch('event-chart-bar-click', { + query: filters.search_query?.toString(), + to: end, + from: start, + }) + }} + EmptyState={ +
+

No data

+

+ It may take up to 24 hours for data to refresh +

+
+ } + /> + )} +
+
+
+ + + setShowChart(!showChart)} + error={error} + EmptyState={EmptyState} + onSelectedLogChange={(log) => setSelectedLogId(log?.id ?? null)} + selectedLog={selectedLog} + isSelectedLogLoading={isSelectedLogLoading} + selectedLogError={selectedLogError ?? undefined} + /> + +
+ {!error && logData.length > 0 && ( +
+ +
+ Showing {logData.length} results +
+
+ )} +
+ ) +} + +export default UnifiedLogsPreviewer diff --git a/apps/studio/hooks/analytics/useUnifiedLogs.tsx b/apps/studio/hooks/analytics/useUnifiedLogs.tsx new file mode 100644 index 00000000000..91c8c3af811 --- /dev/null +++ b/apps/studio/hooks/analytics/useUnifiedLogs.tsx @@ -0,0 +1,291 @@ +import { useInfiniteQuery, useQuery } from '@tanstack/react-query' +import dayjs from 'dayjs' +import { useMemo, useState, useCallback } from 'react' + +import { + LogsTableName, + PREVIEWER_DATEPICKER_HELPERS, + getDefaultHelper, +} from 'components/interfaces/Settings/Logs/Logs.constants' +import type { + Count, + EventChart, + EventChartData, + Filters, + LogData, + Logs, + LogsEndpointParams, +} from 'components/interfaces/Settings/Logs/Logs.types' +import { + genChartQuery, + genCountQuery, + genDefaultQuery, +} from 'components/interfaces/Settings/Logs/Logs.utils' +import { get } from 'data/fetchers' +import { useLogsUrlState } from './useLogsUrlState' +import { useFillTimeseriesSorted } from './useFillTimeseriesSorted' +import useTimeseriesUnixToIso from './useTimeseriesUnixToIso' + +interface UnifiedLogsHook { + logData: LogData[] + error: string | Object | null + newCount: number + isSuccess: boolean + isLoading: boolean + isLoadingOlder: boolean + filters: Filters + params: LogsEndpointParams + oldestTimestamp?: string + eventChartData: EventChartData[] + loadOlder: () => void + refresh: () => void +} + +function useUnifiedLogs({ + projectRef, + filterOverride, + limit, +}: { + projectRef: string + filterOverride?: Filters + limit?: number +}): UnifiedLogsHook { + const defaultHelper = getDefaultHelper(PREVIEWER_DATEPICKER_HELPERS) + const [latestRefresh, setLatestRefresh] = useState(new Date().toISOString()) + + const { + timestampStart: urlTimestampStart, + timestampEnd: urlTimestampEnd, + filters: urlFilters, + search, + } = useLogsUrlState() + + const timestampStart = useMemo( + () => urlTimestampStart || defaultHelper.calcFrom(), + [urlTimestampStart, defaultHelper] + ) + const timestampEnd = useMemo( + () => urlTimestampEnd || defaultHelper.calcTo(), + [urlTimestampEnd, defaultHelper] + ) + + const mergedFilters = useMemo( + () => ({ + ...urlFilters, + ...filterOverride, + ...(search ? { search_query: search } : {}), + }), + [JSON.stringify(urlFilters), JSON.stringify(filterOverride), search] + ) + + const params: LogsEndpointParams = useMemo(() => { + const currentSql = genDefaultQuery(LogsTableName.UNIFIED, mergedFilters, limit) + return { + project: projectRef, + iso_timestamp_start: timestampStart, + iso_timestamp_end: timestampEnd, + sql: currentSql, + } + }, [projectRef, timestampStart, timestampEnd, mergedFilters, limit]) + + const queryKey = useMemo( + () => ['projects', projectRef, 'unified-logs', params], + [projectRef, params] + ) + + console.log('useUnifiedLogs SQL:', params.sql) + + const { + data, + isSuccess, + isLoading, + isRefetching, + error: rqError, + fetchNextPage, + isFetchingNextPage, + refetch, + } = useInfiniteQuery( + queryKey, + async ({ signal, pageParam }) => { + const { data, error } = await get(`/platform/projects/{ref}/analytics/endpoints/logs.all`, { + params: { + path: { ref: projectRef }, + query: { + ...params, + iso_timestamp_end: pageParam || params.iso_timestamp_end, + }, + }, + signal, + }) + if (error) { + throw error + } + + return data as unknown as Logs + }, + { + refetchOnWindowFocus: false, + getNextPageParam(lastPage) { + if ((lastPage.result?.length ?? 0) === 0) { + return undefined + } + const len = lastPage.result.length + const { timestamp: tsLimit }: LogData = lastPage.result[len - 1] + const isoTsLimit = dayjs.utc(Number(tsLimit / 1000)).toISOString() + return isoTsLimit + }, + } + ) + + const { logData, error, oldestTimestamp } = useMemo(() => { + let logData: LogData[] = [] + let error: null | string | object = rqError ? (rqError as any).message : null + + data?.pages?.forEach((response) => { + if (response?.result) { + logData = [...logData, ...response.result] + } + if (!error && response?.error) { + error = response.error + } + }) + + // Log unified logs data for debugging + if (logData.length > 0) { + console.log('Unified Logs Data Sample:', logData.slice(0, 1)) + } + + const oldestTimestamp = logData.length > 0 ? logData[logData.length - 1]?.timestamp : undefined + + return { logData, error, oldestTimestamp } + }, [data?.pages]) + + const countQuerySql = useMemo( + () => genCountQuery(LogsTableName.UNIFIED, mergedFilters), + [mergedFilters] + ) + const countQueryKey = useMemo( + () => [ + 'projects', + projectRef, + 'unified-logs-count', + { + projectRef, + sql: countQuerySql, + iso_timestamp_start: latestRefresh, + iso_timestamp_end: timestampEnd, + mergedFilters, + }, + ], + [projectRef, countQuerySql, latestRefresh, timestampEnd, mergedFilters] + ) + + const { data: countData } = useQuery( + countQueryKey, + async ({ signal }) => { + const { data, error } = await get(`/platform/projects/{ref}/analytics/endpoints/logs.all`, { + params: { + path: { ref: projectRef }, + query: { + project: projectRef, + sql: countQuerySql, + iso_timestamp_start: latestRefresh, + iso_timestamp_end: timestampEnd, + }, + }, + signal, + }) + if (error) { + throw error + } + + return data as unknown as Count + }, + { + refetchOnWindowFocus: false, + refetchInterval: 60000, + enabled: !error && data && data?.pages?.length > 0 ? true : false, + } + ) + + const newCount = countData?.result?.[0]?.count ?? 0 + + const chartQuery = useMemo( + () => genChartQuery(LogsTableName.UNIFIED, params, mergedFilters), + [params, mergedFilters] + ) + const chartQueryKey = useMemo( + () => [ + 'projects', + projectRef, + 'unified-logs-chart', + { + projectRef, + sql: chartQuery, + iso_timestamp_start: timestampStart, + iso_timestamp_end: timestampEnd, + }, + ], + [projectRef, chartQuery, timestampStart, timestampEnd] + ) + + const { data: eventChartResponse, refetch: refreshEventChart } = useQuery( + chartQueryKey, + async ({ signal }) => { + const { data, error } = await get(`/platform/projects/{ref}/analytics/endpoints/logs.all`, { + params: { + path: { ref: projectRef }, + query: { + iso_timestamp_start: timestampStart, + iso_timestamp_end: timestampEnd, + project: projectRef, + sql: chartQuery, + }, + }, + signal, + }) + if (error) { + throw error + } + + return data as unknown as EventChart + }, + { refetchOnWindowFocus: false } + ) + + const refresh = useCallback(async () => { + setLatestRefresh(new Date().toISOString()) + refreshEventChart() + refetch() + }, [refetch, refreshEventChart]) + + const normalizedEventChartData = useTimeseriesUnixToIso( + eventChartResponse?.result ?? [], + 'timestamp' + ) + + const { data: eventChartData, error: eventChartError } = useFillTimeseriesSorted( + normalizedEventChartData, + 'timestamp', + 'count', + 0, + timestampStart, + timestampEnd || new Date().toISOString() + ) + + return { + newCount, + logData, + isSuccess, + isLoading: isLoading || isRefetching, + isLoadingOlder: isFetchingNextPage, + error: error || eventChartError, + filters: mergedFilters, + params, + oldestTimestamp: oldestTimestamp ? String(oldestTimestamp) : undefined, + eventChartData, + refresh, + loadOlder: () => fetchNextPage(), + } +} +export default useUnifiedLogs diff --git a/apps/studio/pages/project/[ref]/unified-logs.tsx b/apps/studio/pages/project/[ref]/unified-logs.tsx index f550245217d..490f831a836 100644 --- a/apps/studio/pages/project/[ref]/unified-logs.tsx +++ b/apps/studio/pages/project/[ref]/unified-logs.tsx @@ -1,22 +1,14 @@ import { useRouter } from 'next/router' -import { LogsTableName } from 'components/interfaces/Settings/Logs/Logs.constants' -import LogsPreviewer from 'components/interfaces/Settings/Logs/LogsPreviewer' import LogsLayout from 'components/layouts/LogsLayout/LogsLayout' import type { NextPageWithLayout } from 'types' import DefaultLayout from 'components/layouts/DefaultLayout' +import UnifiedLogsPreviewer from 'components/interfaces/UnifiedLogs/UnifiedLogsPreviewer' export const LogPage: NextPageWithLayout = () => { const router = useRouter() const { ref } = router.query - return ( - - ) + return } LogPage.getLayout = (page) => (