Merge pull request #4536 from supabase/feat/studio-load-older-logs

feat/studio/logs: Load older logs

studio/components/interfaces/Settings/Logs/LogPanel.tsx

@@ -1,4 +1,4 @@
-import { FC } from 'react'
+import { FC, useEffect, useState } from 'react'
 import {
   Button,
   Input,
@@ -13,13 +13,11 @@ import {
 import { LogTemplate } from '.'
 
 interface Props {
-  searchValue?: string
+  defaultSearchValue?: string
   templates?: any
   isLoading: boolean
   isCustomQuery: boolean
   newCount: number
-  showReset: boolean
-  onReset: () => void
   onRefresh?: () => void
   onSearch?: (query: string) => void
   onCustomClick?: () => void
@@ -30,18 +28,25 @@ interface Props {
  * Logs control panel header + wrapper
  */
 const LogPanel: FC<Props> = ({
-  searchValue,
   templates = [],
   isLoading,
   isCustomQuery,
   newCount,
-  showReset,
-  onReset,
   onRefresh,
   onSearch = () => {},
+  defaultSearchValue = '',
   onCustomClick,
   onSelectTemplate,
 }) => {
+  const [search, setSearch] = useState('')
+
+  // sync local state with provided default value
+  useEffect(() => {
+    if (search !== defaultSearchValue) {
+      setSearch(defaultSearchValue)
+    }
+  }, [defaultSearchValue])
+
   return (
     <div className="bg-panel-header-light dark:bg-panel-header-dark">
       <div className="px-2 py-1 flex items-center justify-between w-full">
@@ -94,16 +99,43 @@ const LogPanel: FC<Props> = ({
           </div>
         </div>
         <div className="flex items-center gap-x-4">
-          <Input
-            size="tiny"
-            icon={<IconSearch size={16} />}
-            placeholder="Search event messages"
-            onChange={(e) => onSearch(e.target.value)}
-            value={searchValue}
-            actions={
-              showReset && <IconX size="tiny" className="cursor-pointer mx-1" title="Clear search" onClick={onReset} />
-            }
-          />
+          {/* wrap with form so that if user presses enter, the search value will submit automatically */}
+          <form
+            id="log-panel-search"
+            onSubmit={(e) => {
+              // prevent redirection
+              e.preventDefault()
+              onSearch(search)
+            }}
+          >
+            <Input
+              size="tiny"
+              icon={<IconSearch size={16} />}
+              placeholder="Search events"
+              onChange={(e) => setSearch(e.target.value)}
+              value={search}
+              actions={[
+                search && (
+                  <IconX
+                    key="clear-search"
+                    size="tiny"
+                    className="cursor-pointer mx-1"
+                    title="Clear search"
+                    onClick={() => setSearch('')}
+                  />
+                ),
+                <Button
+                  key="go"
+                  size="tiny"
+                  title="Go"
+                  type="secondary"
+                  onClick={() => onSearch(search)}
+                >
+                  Go
+                </Button>,
+              ]}
+            />
+          </form>
         </div>
       </div>
     </div>

studio/components/interfaces/Settings/Logs/LogTable.tsx

@@ -1,5 +1,5 @@
 import dayjs from 'dayjs'
-import { useEffect, useState } from 'react'
+import { useEffect, useState, useMemo } from 'react'
 import { Typography } from '@supabase/ui'
 import DataGrid from '@supabase/react-data-grid'
 
@@ -43,10 +43,12 @@ const LogTable = ({ isCustomQuery, data }: Props) => {
     },
   }))
 
-  const logMap = (data || []).reduce((acc: any, d) => {
-    acc[d.id] = d
-    return acc
-  }, {})
+  const logMap: { [id: string]: LogData } = useMemo(() => {
+    return (data || []).reduce((acc: any, d) => {
+      acc[d.id] = d
+      return acc
+    }, {})
+  }, [JSON.stringify(data)])
 
   useEffect(() => {
     if (!data) return
@@ -60,6 +62,9 @@ const LogTable = ({ isCustomQuery, data }: Props) => {
   // [Joshen] Hmm quite hacky now, but will do
   const maxHeight = isCustomQuery ? 'calc(100vh - 42px - 10rem)' : 'calc(100vh - 42px - 3rem)'
 
+  const logDataRows = useMemo(() => {
+    return Object.values(logMap).sort((a, b) => a.timestamp - b.timestamp)
+  }, [JSON.stringify(Object.keys(logMap))])
   return (
     <section className="flex flex-1 flex-row" style={{ maxHeight }}>
       <DataGrid
@@ -75,7 +80,7 @@ const LogTable = ({ isCustomQuery, data }: Props) => {
         }
         columns={columns as any}
         rowClass={(r) => `${r.id === focusedLog?.id ? 'bg-green-800' : 'cursor-pointer'}`}
-        rows={data}
+        rows={logDataRows}
         rowKeyGetter={(r) => r.id}
         onRowClick={(r) => setFocusedLog(logMap[r.id])}
       />

studio/pages/project/[ref]/settings/logs/[type].tsx

@@ -1,10 +1,9 @@
-import useSWR from 'swr'
-import debounce from 'lodash/debounce'
-import { useEffect, useRef, useState } from 'react'
+import useSWR, { KeyLoader } from 'swr'
+import React, { useEffect, useRef, useState } from 'react'
 import { NextPage } from 'next'
 import { useRouter } from 'next/router'
 import { observer } from 'mobx-react-lite'
-import { Typography, IconLoader, IconAlertCircle } from '@supabase/ui'
+import { Typography, IconLoader, IconAlertCircle, IconRewind, Button } from '@supabase/ui'
 
 import { withAuth } from 'hooks'
 import { get } from 'lib/common/fetch'
@@ -18,54 +17,99 @@ import {
   Logs,
   LogTemplate,
   TEMPLATES,
+  LogData,
 } from 'components/interfaces/Settings/Logs'
 import { uuidv4 } from 'lib/helpers'
+import useSWRInfinite from 'swr/infinite'
+import { isUndefined } from 'lodash'
+import Flag from 'components/ui/Flag/Flag'
 
 /**
  * Acts as a container component for the entire log display
+ *
+ *
  */
 export const LogPage: NextPage = () => {
   const router = useRouter()
   const { ref, type } = router.query
 
   const [editorId, setEditorId] = useState<string>(uuidv4())
-  const [search, setSearch] = useState<string>('')
-  const [queryParams, setQueryParams] = useState<string>('')
-  const [where, setWhere] = useState<string>('')
+  const [editorValue, setEditorValue] = useState('')
   const [mode, setMode] = useState<'simple' | 'custom'>('simple')
   const [latestRefresh, setLatestRefresh] = useState<string>(new Date().toISOString())
-
+  const [params, setParams] = useState({
+    type: '',
+    search_query: '',
+    where: '',
+    timestamp_start: '',
+    timestamp_end: '',
+  })
   const title = `Logs - ${LOG_TYPE_LABEL_MAPPING[type as string]}`
-  const debouncedQueryParams = useRef(debounce(setQueryParams, 600)).current
 
   useEffect(() => {
-    const params = {
-      type: type as string,
-      search_query: search || '',
-      where: mode === 'custom' ? where || '' : '',
+    setParams({ ...params, type: type as string })
+  }, [type])
+
+  const genQueryParams = (params: { [k: string]: string }) => {
+    // remove keys which are empty strings, null, or undefined
+    for (const k in params) {
+      const v = params[k]
+      if (v === null || v === '' || isUndefined(v)) {
+        delete params[k]
+      }
     }
     const qs = new URLSearchParams(params).toString()
-    debouncedQueryParams(qs)
-    return () => debouncedQueryParams.cancel()
-  }, [mode, search, where, type])
-
+    return qs
+  }
   // handle log fetching
-  const logUrl = `${API_URL}/projects/${ref}/logs?${queryParams}`
-  const { data, isValidating, mutate } = useSWR<Logs>(logUrl, get, { revalidateOnFocus: false })
-  const { data: logData, error } = data || {}
+  const getKeyLogs: KeyLoader<Logs> = (_pageIndex: number, prevPageData) => {
+    let queryParams
+    // if prev page data is 100 items, could possibly have more records that are not yet fetched within this interval
+    if (prevPageData === null) {
+      // reduce interval window limit by using the timestamp of the last log
+      queryParams = genQueryParams(params)
+    } else {
+      const len = prevPageData.data.length
+      const { timestamp: tsLimit }: LogData = prevPageData.data[len - 1]
+      // create new key from params
+      queryParams = genQueryParams({ ...params, timestamp_end: String(tsLimit) })
+    }
 
-  const countUrl = `${API_URL}/projects/${ref}/logs?${queryParams}&count=true&period_start=${latestRefresh}`
+    const logUrl = `${API_URL}/projects/${ref}/logs?${queryParams}`
+    return logUrl
+  }
+  const {
+    data = [],
+    isValidating,
+    mutate,
+    size,
+    setSize,
+  } = useSWRInfinite<Logs>(getKeyLogs, get, { revalidateOnFocus: false })
+
+  // const { data, isValidating, mutate } = useSWR<Logs>(logUrl, get, { revalidateOnFocus: false })
+  let logData: LogData[] = []
+  let error: null | string = null
+
+  data.forEach((response: Logs) => {
+    if (response && response.data) {
+      logData = [...logData, ...response.data]
+    }
+    if (!error && response && response.error) {
+      error = response.error
+    }
+  })
+
+  const countUrl = `${API_URL}/projects/${ref}/logs?${genQueryParams({
+    ...params,
+    count: String(true),
+    period_start: String(latestRefresh),
+  })}`
   const { data: countData } = useSWR<Count>(countUrl, get, { refreshInterval: 5000 })
   const newCount = countData?.data?.[0]?.count ?? 0
 
-  const handleReset = () => {
-    setWhere('')
-    setSearch('')
-    setEditorId(uuidv4())
-  }
-
   const handleRefresh = () => {
     setLatestRefresh(new Date().toISOString())
+    setSize(1)
     mutate()
   }
 
@@ -81,14 +125,19 @@ export const LogPage: NextPage = () => {
   const onSelectTemplate = (template: LogTemplate) => {
     setMode(template.mode)
     if (template.mode === 'simple') {
-      setSearch(template.searchString)
-      setWhere('')
+      setParams((prev) => ({ ...prev, search_query: template.searchString, where: '' }))
     } else {
-      setWhere(template.searchString)
-      setSearch('')
+      setEditorValue(template.searchString)
+      setParams((prev) => ({ ...prev, where: template.searchString, search_query: '' }))
       setEditorId(uuidv4())
     }
   }
+  const handleEditorSubmit = () => {
+    setParams((prev) => ({ ...prev, where: editorValue }))
+  }
+  const handleSearch = (v: string) => {
+    setParams((prev) => ({ ...prev, search_query: v || '' }))
+  }
 
   return (
     <SettingsLayout title={title}>
@@ -97,25 +146,41 @@ export const LogPage: NextPage = () => {
           isCustomQuery={mode === 'custom'}
           isLoading={isValidating}
           newCount={newCount}
-          showReset={search.length > 0}
-          searchValue={search}
           templates={TEMPLATES}
-          onReset={handleReset}
           onRefresh={handleRefresh}
-          onSearch={setSearch}
+          onSearch={handleSearch}
+          defaultSearchValue={params.search_query}
           onCustomClick={handleModeToggle}
           onSelectTemplate={onSelectTemplate}
         />
         {mode === 'custom' && (
-          <div className="min-h-[7rem] h-28">
-            <CodeEditor
-              id={editorId}
-              language="pgsql"
-              defaultValue={where}
-              onInputChange={(v) => setWhere(v || '')}
-              onInputRun={handleRefresh}
-            />
-          </div>
+          <React.Fragment>
+            <div className="min-h-[7rem] h-28">
+              <CodeEditor
+                id={editorId}
+                language="pgsql"
+                defaultValue={editorValue}
+                onInputChange={(v) => setEditorValue(v || '')}
+                onInputRun={handleRefresh}
+              />
+            </div>
+            <div className="flex flex-row gap-x-2 justify-end p-2">
+              {editorValue && (
+                <Button
+                  type="text"
+                  onClick={() => {
+                    setEditorValue('')
+                    setEditorId(uuidv4())
+                  }}
+                >
+                  Clear
+                </Button>
+              )}
+              <Button type={editorValue ? 'secondary' : 'text'} onClick={handleEditorSubmit}>
+                Run
+              </Button>
+            </div>
+          </React.Fragment>
         )}
         <div className="flex flex-col flex-grow relative">
           {isValidating && (
@@ -135,6 +200,19 @@ export const LogPage: NextPage = () => {
             </div>
           )}
           <LogTable data={logData} isCustomQuery={mode === 'custom'} />
+          {/* Footer section of log ui, appears below table */}
+          <div className="p-2">
+            <Flag name="logsLoadOlder">
+              <Button
+                // trigger page increase
+                onClick={() => setSize(size + 1)}
+                icon={<IconRewind />}
+                type="secondary"
+              >
+                Load older
+              </Button>
+            </Flag>
+          </div>
         </div>
       </div>
     </SettingsLayout>

studio/tests/pages/projects/LogPanel.test.js

@@ -4,12 +4,9 @@ import userEvent from '@testing-library/user-event'
 
 test('templates', async () => {
   const mockFn = jest.fn()
-  render(<LogPanel templates={[
-    { label: 'Some option', onClick: mockFn }
-  ]} />)
+  render(<LogPanel templates={[{ label: 'Some option', onClick: mockFn }]} />)
   const search = screen.getByPlaceholderText(/Search/)
-  userEvent.type(search, "12345")
-
+  userEvent.type(search, '12345')
 
   // TODO templates dropdown interaction currently cannot be tested
   // https://github.com/supabase/ui/issues/299
@@ -21,21 +18,22 @@ test('templates', async () => {
   // expect(mockFn).toBeCalled()
 })
 
-test("filter input change", async () => {
+test('filter input change and submit', async () => {
   const mockFn = jest.fn()
   render(<LogPanel onSearch={mockFn} />)
   const search = screen.getByPlaceholderText(/Search/)
-  userEvent.type(search, "12345")
+  userEvent.type(search, '12345')
+  expect(mockFn).not.toBeCalled()
+  userEvent.click(screen.getByText('Go'))
   expect(mockFn).toBeCalled()
 })
 
-test("filter input value", async () => {
-  render(<LogPanel searchValue={"1234"} />)
-  screen.getByDisplayValue("1234")
+test('filter input value', async () => {
+  render(<LogPanel defaultSearchValue={'1234'} />)
+  screen.getByDisplayValue('1234')
 })
 
-
-test("Manual refresh", async () => {
+test('Manual refresh', async () => {
   const mockFn = jest.fn()
   render(<LogPanel onRefresh={mockFn} />)
   let btn
@@ -46,12 +44,13 @@ test("Manual refresh", async () => {
   expect(mockFn).toBeCalled()
 })
 
-test("reset filters", () => {
-  const mockFn = jest.fn()
-  const { rerender } = render(<LogPanel showReset={false} />)
-  expect(() => screen.getByText(/Clear search/)).toThrow()
+test('reset search filter', async () => {
+  const { rerender } = render(<LogPanel />)
+  expect(() => screen.getByTitle(/Clear search/)).toThrow()
 
-  rerender(<LogPanel showReset={true} onReset={mockFn} />)
+  rerender(<LogPanel defaultSearchValue="something123" />)
+  await waitFor(() => screen.getByDisplayValue(/something123/))
   userEvent.click(screen.getByTitle(/Clear search/))
-  expect(mockFn).toBeCalled()
-})
+  expect(() => screen.getByTitle(/Clear search/)).toThrow()
+  expect(() => screen.getByDisplayValue(/something123/)).toThrow()
+})

studio/tests/pages/projects/LogTable.test.js

@@ -16,3 +16,18 @@ test('can display log data', async () => {
   await waitFor(() => screen.getByText(/my_key/))
   await waitFor(() => screen.getByText(/something_value/))
 })
+
+test ("dedupes log lines with exact id", async ()=>{
+  render(<LogTable data={[{
+    id: "seome-uuid",
+    timestamp: 1621323232312,
+    event_message: "some event happened",
+  },{
+    id: "seome-uuid",
+    timestamp: 1621323232312,
+    event_message: "some event happened",
+  }]} />)
+
+  // should only have one element, this line will fail if there are >1 element
+  await waitFor(() => screen.getByText(/happened/))
+})

studio/tests/pages/projects/logs.test.js

@@ -28,12 +28,29 @@ Editor.mockImplementation((props) => {
 })
 useMonaco.mockImplementation((v) => v)
 
+jest.mock('components/ui/Flag/Flag')
+import Flag from 'components/ui/Flag/Flag'
+Flag.mockImplementation(({ children }) => <>{children}</>)
+
+import { SWRConfig } from 'swr'
+jest.mock('pages/project/[ref]/settings/logs/[type]')
 import { LogPage } from 'pages/project/[ref]/settings/logs/[type]'
+LogPage.mockImplementation((props) => {
+  const Page = jest.requireActual('pages/project/[ref]/settings/logs/[type]').LogPage
+  // wrap with SWR to reset the cache each time
+  return (
+    <SWRConfig value={{ provider: () => new Map() }}>
+      <Page {...props} />
+    </SWRConfig>
+  )
+})
+
 import { render, fireEvent, waitFor, screen } from '@testing-library/react'
 import userEvent from '@testing-library/user-event'
-import {getToggleByText} from "../../helpers"
+import { getToggleByText } from '../../helpers'
 
 beforeEach(() => {
+  // reset mocks between tests
   get.mockReset()
 })
 test('can display log data and metadata', async () => {
@@ -57,7 +74,7 @@ test('can display log data and metadata', async () => {
   await waitFor(() => screen.getByText(/something_value/))
 })
 
-test('Refresh', async () => {
+test('Refreshpage', async () => {
   const data = [
     {
       id: 'some-uuid',
@@ -68,9 +85,13 @@ test('Refresh', async () => {
       },
     },
   ]
-  get.mockResolvedValueOnce({ data }).mockResolvedValueOnce({ data: [] })
+  get.mockImplementation((url) => {
+    if (url.includes('count')) return { count: 0 }
+    return { data }
+  })
   render(<LogPage />)
-
+  await waitFor(() => screen.getByText(/happened/))
+  get.mockResolvedValueOnce({ data: [] })
   const row = screen.getByText(/happened/)
   fireEvent.click(row)
   await waitFor(() => screen.getByText(/my_key/))
@@ -101,6 +122,8 @@ test('Search will trigger a log refresh', async () => {
   render(<LogPage />)
 
   userEvent.type(screen.getByPlaceholderText(/Search/), 'something')
+  userEvent.click(screen.getByText('Go'))
+
   await waitFor(
     () => {
       expect(get).toHaveBeenCalledWith(expect.stringContaining('search_query'))
@@ -167,6 +190,7 @@ test('where clause will trigger a log refresh', async () => {
   })
   editor = container.querySelector('.monaco-editor')
   userEvent.type(editor, 'metadata.field = something')
+  userEvent.click(screen.getByText('Run'))
   await waitFor(
     () => {
       expect(get).toHaveBeenCalledWith(expect.stringContaining('where'))
@@ -177,3 +201,43 @@ test('where clause will trigger a log refresh', async () => {
 
   await waitFor(() => screen.getByText(/happened/))
 })
+
+test('load older btn will fetch older logs', async () => {
+  get.mockImplementation((url) => {
+    if (url.includes('count')) {
+      return {}
+    }
+    return {
+      data: [
+        {
+          id: 'some-uuid',
+          timestamp: 1621323232312,
+          event_message: 'first event',
+          metadata: {},
+        },
+      ],
+    }
+  })
+  render(<LogPage />)
+  // should display first log but not second
+  await waitFor(() => screen.getByText('first event'))
+  expect(() => screen.getByText('second event')).toThrow()
+
+  get.mockResolvedValueOnce({
+    data: [
+      {
+        id: 'some-uuid2',
+        timestamp: 1621323232310,
+        event_message: 'second event',
+        metadata: {},
+      },
+    ],
+  })
+  // should display first and second log
+  userEvent.click(screen.getByText('Load older'))
+  await waitFor(() => screen.getByText('first event'))
+  await waitFor(() => {
+    expect(get).toHaveBeenCalledWith(expect.stringContaining('timestamp_end=1'))
+  })
+  await waitFor(() => screen.getByText('second event'))
+})