Merge pull request #14063 from supabase/feat/report-widget-expandable-api-requests

feat: add route information to API report

studio/components/interfaces/Reports/ReportFilterBar.tsx

@@ -12,7 +12,7 @@ import {
   IconBox,
   IconCode,
   IconDatabase,
-  IconZapOff,
+  IconZap,
 } from 'ui'
 import DatePickers from '../Settings/Logs/Logs.DatePickers'
 import { REPORTS_DATEPICKER_HELPERS } from './Reports.constants'
@@ -59,7 +59,7 @@ const PRODUCT_FILTERS = [
     filterValue: '/realtime',
     label: 'Realtime',
     description: 'Realtime connection requests',
-    icon: IconZapOff,
+    icon: IconZap,
   },
   // TODO: support functions once union parsing is fixed
   // {

studio/components/interfaces/Reports/ReportPadding.tsx

@@ -2,7 +2,7 @@
  * Standardized padding and width layout for non-custom reports
  */
 const ReportPadding: React.FC = ({ children }) => (
-  <div className="flex flex-col gap-4 px-5 py-6 h-full mx-auto 1xl:px-28 lg:px-16 xl:px-24 2xl:px-32">
+  <div className="flex flex-col gap-4 px-5 py-6 mx-auto 1xl:px-28 lg:px-16 xl:px-24 2xl:px-32">
     {children}
   </div>
 )

studio/components/interfaces/Reports/ReportWidget.tsx

@@ -1,7 +1,6 @@
 import { NextRouter, useRouter } from 'next/router'
 import * as Tooltip from '@radix-ui/react-tooltip'
 import { Button, IconExternalLink, IconHelpCircle } from 'ui'
-
 import { BaseReportParams } from './Reports.types'
 import { LogsEndpointParams } from '../Settings/Logs'
 import Panel from 'components/ui/Panel'
@@ -14,12 +13,15 @@ export interface ReportWidgetProps<T = any> {
   tooltip?: string
   className?: string
   renderer: (props: ReportWidgetRendererProps) => React.ReactNode
+  append?: (props: ReportWidgetRendererProps) => React.ReactNode
+  // for overriding props, such as data
+  appendProps?: Partial<ReportWidgetRendererProps>
   // omitting params will hide the "View in logs explorer" button
   params?: BaseReportParams | LogsEndpointParams
   isLoading: boolean
 }
 
-export interface ReportWidgetRendererProps extends ReportWidgetProps {
+export interface ReportWidgetRendererProps<T = any> extends ReportWidgetProps<T> {
   router: NextRouter
   projectRef: string
 }
@@ -28,7 +30,6 @@ const ReportWidget: React.FC<ReportWidgetProps> = (props) => {
   const router = useRouter()
   const { ref } = router.query
   const projectRef = ref as string
-
   return (
     <Panel
       noMargin
@@ -104,6 +105,9 @@ const ReportWidget: React.FC<ReportWidgetProps> = (props) => {
         <LoadingOpacity className="w-full" active={props.isLoading}>
           {props.data === undefined ? null : props.renderer({ ...props, router, projectRef })}
         </LoadingOpacity>
+
+        {props.append &&
+          props.append({ ...props, ...(props.appendProps || {}), router, projectRef })}
       </Panel.Content>
     </Panel>
   )

studio/components/interfaces/Reports/Reports.constants.ts

@@ -74,6 +74,29 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {
         ORDER BY
           timestamp ASC`,
       },
+      topRoutes: {
+        queryType: "logs",
+        sql: (filters)=> `
+        select
+          request.path as path,
+          request.method as method,
+          request.search as search,
+          response.status_code as status_code,
+          count(t.id) as count
+        from edge_logs t
+          cross join unnest(metadata) as m
+          cross join unnest(m.response) as response
+          cross join unnest(m.request) as request
+          cross join unnest(request.headers) as headers
+          ${generateRexepWhere(filters)}
+        group by
+          request.path, request.method, request.search, response.status_code
+        order by
+          count desc
+        limit
+        3
+        `
+      },
       errorCounts: {
         queryType: 'logs',
         sql: (filters) => `
@@ -94,13 +117,37 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {
           timestamp ASC
         `,
       },
+      topErrorRoutes: {
+        queryType: "logs",
+        sql: (filters)=> `
+        select
+          request.path as path,
+          request.method as method,
+          request.search as search,
+          response.status_code as status_code,
+          count(t.id) as count
+        from edge_logs t
+          cross join unnest(metadata) as m
+          cross join unnest(m.response) as response
+          cross join unnest(m.request) as request
+          cross join unnest(request.headers) as headers
+        where
+          response.status_code >= 400
+        ${generateRexepWhere(filters, false)}
+        group by
+          request.path, request.method, request.search, response.status_code
+        order by
+          count desc
+        limit
+        3
+        `
+      },
       responseSpeed: {
         queryType: 'logs',
         sql: (filters) => `
         select
           cast(timestamp_trunc(t.timestamp, hour) as datetime) as timestamp,
-          avg(response.origin_time) as avg,
-          APPROX_QUANTILES(response.origin_time, 100) as quantiles
+          avg(response.origin_time) as avg
         FROM
           edge_logs t
           cross join unnest(metadata) as m
@@ -114,6 +161,30 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {
           timestamp ASC
       `,
       },
+      topSlowRoutes: {
+        queryType: "logs",
+        sql: (filters)=> `
+        select
+          request.path as path,
+          request.method as method,
+          request.search as search,
+          response.status_code as status_code,
+          count(t.id) as count,
+          avg(response.origin_time) as avg
+        from edge_logs t
+          cross join unnest(metadata) as m
+          cross join unnest(m.response) as response
+          cross join unnest(m.request) as request
+          cross join unnest(request.headers) as headers
+        ${generateRexepWhere(filters)}
+        group by
+          request.path, request.method, request.search, response.status_code
+        order by
+          avg desc
+        limit
+        3
+        `
+      },
     },
   },
   [Presets.AUTH]: {

studio/components/interfaces/Reports/renderers/ApiRenderers.tsx

@@ -1,7 +1,16 @@
-import { ReportWidgetProps } from '../ReportWidget'
+import { ReportWidgetProps, ReportWidgetRendererProps } from '../ReportWidget'
 import BarChart from 'components/ui/Charts/BarChart'
+import Table from 'components/to-be-cleaned/Table'
+import {
+  jsonSyntaxHighlight,
+  TextFormatter,
+} from 'components/interfaces/Settings/Logs/LogsFormatters'
+import { Button, Collapsible, IconChevronRight } from 'ui'
+import { queryParamsToObject } from '../Reports.utils'
+import { Fragment } from 'react'
+import useFillTimeseriesSorted from 'hooks/analytics/useFillTimeseriesSorted'
 
-export const renderTotalRequests = (
+export const TotalRequestsChartRenderer = (
   props: ReportWidgetProps<{
     timestamp: string
     count: number
@@ -10,13 +19,21 @@ export const renderTotalRequests = (
   const total = props.data.reduce((acc, datum) => {
     return acc + datum.count
   }, 0)
+  const data = useFillTimeseriesSorted(
+    props.data,
+    'timestamp',
+    'count',
+    0,
+    props.params?.iso_timestamp_start,
+    props.params?.iso_timestamp_end
+  )
   return (
     <BarChart
       size="small"
       minimalHeader
       highlightedValue={total}
       className="w-full"
-      data={props.data}
+      data={data}
       yAxisKey="count"
       xAxisKey="timestamp"
       displayDateInUtc
@@ -24,7 +41,58 @@ export const renderTotalRequests = (
   )
 }
 
-export const renderErrorCounts = (
+export const TopApiRoutesRenderer = (
+  props: ReportWidgetRendererProps<{
+    method: string
+    // shown for error table but not all requests table
+    status_code?: number
+    path: string
+    search: string
+    count: number
+    // used for response speed table only
+    avg?: number
+  }>
+) => {
+  if (props.data.length === 0) return null
+  const headerClasses = '!text-xs !py-2 p-0 font-bold !bg-scale-400'
+  const cellClasses = '!text-xs !py-2'
+  return (
+    <Table
+      head={
+        <>
+          <Table.th className={headerClasses}>Request</Table.th>
+          <Table.th className={headerClasses + ' text-right'}>Count</Table.th>
+          {props.data[0].avg !== undefined && (
+            <Table.th className={headerClasses + ' text-right'}>Avg</Table.th>
+          )}
+        </>
+      }
+      body={
+        <>
+          {props.data.map((datum) => (
+            <Fragment key={datum.path + (datum.search || '')}>
+              <Table.tr className="p-0">
+                <Table.td className={[cellClasses].join(' ')}>
+                  <RouteTdContent {...datum} />
+                </Table.td>
+                <Table.td className={[cellClasses, 'text-right align-top'].join(' ')}>
+                  {datum.count}
+                </Table.td>
+                {props.data[0].avg !== undefined && (
+                  <Table.td className={[cellClasses, 'text-right align-top'].join(' ')}>
+                    {Number(datum.avg).toFixed(2)}ms
+                  </Table.td>
+                )}
+              </Table.tr>
+            </Fragment>
+          ))}
+        </>
+      }
+    />
+  )
+}
+
+export const ErrorCountsChartRenderer = (
   props: ReportWidgetProps<{
     timestamp: string
     count: number
@@ -33,13 +101,23 @@ export const renderErrorCounts = (
   const total = props.data.reduce((acc, datum) => {
     return acc + datum.count
   }, 0)
+
+  const data = useFillTimeseriesSorted(
+    props.data,
+    'timestamp',
+    'count',
+    0,
+    props.params?.iso_timestamp_start,
+    props.params?.iso_timestamp_end
+  )
+
   return (
     <BarChart
       size="small"
       minimalHeader
       className="w-full"
       highlightedValue={total}
-      data={props.data}
+      data={data}
       yAxisKey="count"
       xAxisKey="timestamp"
       displayDateInUtc
@@ -47,18 +125,26 @@ export const renderErrorCounts = (
   )
 }
 
-export const renderResponseSpeed = (
+export const ResponseSpeedChartRenderer = (
   props: ReportWidgetProps<{
     timestamp: string
     avg: number
-    quantiles: number[]
   }>
 ) => {
   const transformedData = props.data.map((datum) => ({
     timestamp: datum.timestamp,
     avg: datum.avg,
-    median: datum.quantiles[49],
   }))
+
+  const data = useFillTimeseriesSorted(
+    transformedData,
+    'timestamp',
+    'avg',
+    0,
+    props.params?.iso_timestamp_start,
+    props.params?.iso_timestamp_end
+  )
+
   const lastAvg = props.data[props.data.length - 1]?.avg
   return (
     <BarChart
@@ -67,10 +153,65 @@ export const renderResponseSpeed = (
       format="ms"
       minimalHeader
       className="w-full"
-      data={transformedData}
+      data={data}
       yAxisKey="avg"
       xAxisKey="timestamp"
       displayDateInUtc
     />
   )
 }
+
+interface RouteTdContentProps {
+  method: string
+  status_code?: number
+  path: string
+  search: string
+}
+const RouteTdContent = (datum: RouteTdContentProps) => (
+  <Collapsible>
+    <Collapsible.Trigger asChild>
+      <div className="flex gap-2">
+        <Button as="span" type="text" className=" !py-0 !p-1" title="Show more route details">
+          <IconChevronRight
+            size={14}
+            className="transition data-open-parent:rotate-90 data-closed-parent:rotate-0"
+          />
+        </Button>
+        <TextFormatter className="w-10 h-4 text-center rounded bg-scale-500" value={datum.method} />
+        {datum.status_code && (
+          <TextFormatter
+            className={`w-10 h-4 text-center rounded ${
+              datum.status_code >= 400
+                ? 'bg-orange-500'
+                : datum.status_code >= 300
+                ? 'bg-yellow-500'
+                : 'bg-green-500'
+            }`}
+            value={String(datum.status_code)}
+          />
+        )}
+        <div className=" truncate max-w-sm lg:max-w-lg">
+          <TextFormatter className="text-scale-1100" value={datum.path} />
+          <TextFormatter
+            className="max-w-sm text-scale-900 truncate "
+            value={decodeURIComponent(datum.search || '')}
+          />
+        </div>
+      </div>
+    </Collapsible.Trigger>
+    <Collapsible.Content className="pt-2">
+      {datum.search ? (
+        <pre className={`syntax-highlight overflow-auto rounded bg-scale-300 p-2 !text-xs`}>
+          <div
+            className="text-wrap"
+            dangerouslySetInnerHTML={{
+              __html: jsonSyntaxHighlight(queryParamsToObject(datum.search)),
+            }}
+          />
+        </pre>
+      ) : (
+        <p className="text-xs text-scale-900">No query parameters in this request</p>
+      )}
+    </Collapsible.Content>
+  </Collapsible>
+)

studio/components/ui/Charts/BarChart.tsx

@@ -43,12 +43,12 @@ const BarChart: React.FC<BarChartProps> = ({
   const resolvedHighlightedLabel =
     (focusDataIndex !== null &&
       data &&
-      data[focusDataIndex] &&
+      data[focusDataIndex] !== undefined &&
       day(data[focusDataIndex][xAxisKey]).format(customDateFormat)) ||
     highlightedLabel
 
   const resolvedHighlightedValue =
-    (focusDataIndex !== null ? data[focusDataIndex]?.[yAxisKey] : null) || highlightedValue
+    (focusDataIndex !== null ? data[focusDataIndex]?.[yAxisKey] : highlightedValue) 
 
   return (
     <div className={['flex flex-col gap-3', className].join(' ')}>

studio/pages/project/[ref]/reports/api-overview.tsx

@@ -9,9 +9,10 @@ import {
 import ReportWidget from 'components/interfaces/Reports/ReportWidget'
 import { queriesFactory } from 'components/interfaces/Reports/Reports.utils'
 import {
-  renderTotalRequests,
-  renderErrorCounts,
-  renderResponseSpeed,
+  TotalRequestsChartRenderer,
+  ErrorCountsChartRenderer,
+  ResponseSpeedChartRenderer,
+  TopApiRoutesRenderer,
 } from 'components/interfaces/Reports/renderers/ApiRenderers'
 import { useState, useEffect } from 'react'
 import ReportHeader from 'components/interfaces/Reports/ReportHeader'
@@ -63,7 +64,9 @@ export const ApiReport: NextPageWithLayout = () => {
         params={report.params.totalRequests}
         title="Total Requests"
         data={report.data.totalRequests || []}
-        renderer={renderTotalRequests}
+        renderer={TotalRequestsChartRenderer}
+        append={TopApiRoutesRenderer}
+        appendProps={{ data: report.data.topRoutes || [] }}
       />
       <ReportWidget
         isLoading={report.isLoading}
@@ -71,7 +74,9 @@ export const ApiReport: NextPageWithLayout = () => {
         title="Response Errors"
         tooltip="Error responses with 4XX or 5XX status codes"
         data={report.data.errorCounts || []}
-        renderer={renderErrorCounts}
+        renderer={ErrorCountsChartRenderer}
+        appendProps={{ data: report.data.topErrorRoutes || [] }}
+        append={TopApiRoutesRenderer}
       />
       <ReportWidget
         isLoading={report.isLoading}
@@ -79,7 +84,9 @@ export const ApiReport: NextPageWithLayout = () => {
         title="Response Speed"
         tooltip="Average response speed (in miliseconds) of a request"
         data={report.data.responseSpeed || []}
-        renderer={renderResponseSpeed}
+        renderer={ResponseSpeedChartRenderer}
+        appendProps={{ data: report.data.topSlowRoutes || [] }}
+        append={TopApiRoutesRenderer}
       />
     </ReportPadding>
   )
@@ -94,9 +101,19 @@ const useApiReport = () => {
     projectRef ?? 'default'
   )
   const totalRequests = queryHooks.totalRequests()
+  const topRoutes = queryHooks.topRoutes()
   const errorCounts = queryHooks.errorCounts()
+  const topErrorRoutes = queryHooks.topErrorRoutes()
   const responseSpeed = queryHooks.responseSpeed()
-  const activeHooks = [totalRequests, errorCounts, responseSpeed]
+  const topSlowRoutes = queryHooks.topSlowRoutes()
+  const activeHooks = [
+    totalRequests,
+    topRoutes,
+    errorCounts,
+    topErrorRoutes,
+    responseSpeed,
+    topSlowRoutes,
+  ]
   const [filters, setFilters] = useState<ReportFilterItem[]>([])
   const addFilter = (filter: ReportFilterItem) => {
     // use a deep equal when comparing objects.
@@ -127,12 +144,23 @@ const useApiReport = () => {
     if (totalRequests[1].changeQuery) {
       totalRequests[1].changeQuery(PRESET_CONFIG.api.queries.totalRequests.sql(filters))
     }
+    if (topRoutes[1].changeQuery) {
+      topRoutes[1].changeQuery(PRESET_CONFIG.api.queries.topRoutes.sql(filters))
+    }
     if (errorCounts[1].changeQuery) {
       errorCounts[1].changeQuery(PRESET_CONFIG.api.queries.errorCounts.sql(filters))
     }
+
+    if (topErrorRoutes[1].changeQuery) {
+      topErrorRoutes[1].changeQuery(PRESET_CONFIG.api.queries.topErrorRoutes.sql(filters))
+    }
     if (responseSpeed[1].changeQuery) {
       responseSpeed[1].changeQuery(PRESET_CONFIG.api.queries.responseSpeed.sql(filters))
     }
+
+    if (topSlowRoutes[1].changeQuery) {
+      topSlowRoutes[1].changeQuery(PRESET_CONFIG.api.queries.topSlowRoutes.sql(filters))
+    }
   }, [JSON.stringify(filters)])
 
   const handleRefresh = async () => {
@@ -151,11 +179,17 @@ const useApiReport = () => {
       totalRequests: totalRequests[0].logData,
       errorCounts: errorCounts[0].logData,
       responseSpeed: responseSpeed[0].logData,
+      topRoutes: topRoutes[0].logData,
+      topErrorRoutes: topErrorRoutes[0].logData,
+      topSlowRoutes: topSlowRoutes[0].logData,
     },
     params: {
       totalRequests: totalRequests[0].params,
       errorCounts: errorCounts[0].params,
       responseSpeed: responseSpeed[0].params,
+      topRoutes: topRoutes[0].params,
+      topErrorRoutes: topErrorRoutes[0].params,
+      topSlowRoutes: topSlowRoutes[0].params,
     },
     mergeParams: handleSetParams,
     filters,

studio/tests/components/Reports/ReportWidget.test.js

@@ -1,46 +1,12 @@
-// mock the fetch function
-jest.mock('lib/common/fetch')
 import { get } from 'lib/common/fetch'
-
-// mock mobx
-jest.mock('mobx-react-lite')
-import { observer } from 'mobx-react-lite'
-observer.mockImplementation((v) => v)
-
-// mock the router
-jest.mock('next/router')
 import { useRouter } from 'next/router'
-const defaultRouterMock = () => {
-  const router = jest.fn()
-  router.query = {}
-  router.push = jest.fn()
-  router.pathname = 'logs/path'
-  return router
-}
-useRouter.mockReturnValue(defaultRouterMock())
-
-// mock usage flags
-jest.mock('components/ui/Flag/Flag')
-import Flag from 'components/ui/Flag/Flag'
-Flag.mockImplementation(({ children }) => <>{children}</>)
-jest.mock('hooks')
-import { useFlag } from 'hooks'
-useFlag.mockReturnValue(true)
-
-import { fireEvent, waitFor, screen, act } from '@testing-library/react'
+import { screen } from '@testing-library/react'
 import { render } from '../../helpers'
-import userEvent from '@testing-library/user-event'
-import { wait } from '@testing-library/user-event/dist/utils'
-import { logDataFixture } from '../../fixtures'
-import { LogsTableName } from 'components/interfaces/Settings/Logs'
-import { Presets } from 'components/interfaces/Reports/Reports.types'
 import ReportWidget from 'components/interfaces/Reports/ReportWidget'
-import { clickDropdown } from 'tests/helpers'
+import userEvent from '@testing-library/user-event'
 beforeEach(() => {
   // reset mocks between tests
   get.mockReset()
-  useRouter.mockReset()
-  useRouter.mockReturnValue(defaultRouterMock())
 })
 
 test('static elements', async () => {
@@ -48,3 +14,9 @@ test('static elements', async () => {
   await screen.findByText(/Some chart/)
   await screen.findByText(/something/)
 })
+
+test("append", async ()=>{
+  const appendable = ()=> "some text"
+  render(<ReportWidget data={[]} renderer={() => null} append={appendable} />)
+  await screen.findByText(/some text/)
+})

studio/tests/pages/projects/reports/api-report.test.js

@@ -7,7 +7,7 @@ import userEvent from '@testing-library/user-event'
 beforeEach(() => {
   // reset mocks between tests
   get.mockReset()
-  get.mockImplementation(async (_url) => [{ data: [] }])
+  get.mockImplementation(async (_url) => [{ result: [] }])
 })
 
 test(`static elements`, async () => {
@@ -26,3 +26,86 @@ test('refresh button', async () => {
   userEvent.click(await screen.findByText(/Refresh/))
   await waitFor(() => expect(get).toBeCalled())
 })
+
+test('append - api request routes', async () => {
+  get.mockImplementation(async (url) => {
+    if (decodeURIComponent(url).includes("request.path")) {
+      return { result: [{ path: 'mypath', method: 'GET', status_code: 200, search: 'some-query', count: 22 }] }
+    }
+    return { result: [{ timestamp: new Date().toISOString(), count: 123 }] }
+  })
+  render(<ApiReport />)
+  await waitFor(() => expect(get).toBeCalled())
+  await screen.findAllByText(/mypath/)
+  await screen.findAllByText(/GET/)
+  await screen.findAllByText(/200/)
+  await screen.findAllByText(/some\-query/)
+  await screen.findAllByText(/22/)
+})
+
+
+test('append - error routes', async () => {
+  get.mockImplementation(async (url) => {
+    const uri = decodeURIComponent(url)
+    if (uri.includes("400")) {
+      return { result: [{ path: 'mypath', method: 'GET', status_code: 200, search: 'some-query', count: 22 }] }
+    }
+    return { result: [{ timestamp: new Date().toISOString(), count: 123 }] }
+  })
+  render(<ApiReport />)
+  await waitFor(() => expect(get).toBeCalled())
+  await screen.findAllByText(/mypath/)
+  await screen.findAllByText(/GET/)
+  await screen.findAllByText(/200/)
+  await screen.findAllByText(/some\-query/)
+  await screen.findAllByText(/22/)
+})
+
+
+test('append - error routes', async () => {
+  get.mockImplementation(async (url) => {
+    const uri = decodeURIComponent(url)
+    if (uri.includes("avg") && uri.includes("request.path")) {
+      return { result: [{ path: 'mypath', method: 'GET', status_code: 200, avg: 534, search: 'some-query', count: 22 }] }
+    }
+    return { result: [{ timestamp: new Date().toISOString(), count: 123 }] }
+  })
+  render(<ApiReport />)
+  await waitFor(() => expect(get).toBeCalled())
+  await screen.findAllByText(/mypath/)
+  await screen.findAllByText(/GET/)
+  await screen.findAllByText(/200/)
+  await screen.findAllByText(/some\-query/)
+  await screen.findAllByText(/22/)
+  await screen.findAllByText(/534\.00ms/)
+})
+
+
+
+// test('expandable error routes', async () => {
+//   get.mockImplementation(async (url) => {
+//     if (url.includes('path')) {
+//       return [{ data: [{ path: '/my-path', method: 'GET', search: '?=123', count: 22 }] }]
+//     }
+//     return [{ data: [{ timestamp: new Date().toISOString(), count: 123 }] }]
+//   })
+
+//   render(<ApiReport />)
+//   await screen.findByText(/\/my\-path/)
+//   await screen.findByText(/\?\=123/)
+//   await screen.findByText(/22/)
+// })
+
+// test('expandable high latency routes', async () => {
+//   get.mockImplementation(async (url) => {
+//     if (url.includes('path')) {
+//       return [{ data: [{ path: '/my-path', method: 'GET', search: '?=123', avg_ms: 55, count: 22 }] }]
+//     }
+//     return [{ data: [{ timestamp: new Date().toISOString(), avg_ms: 123  }] }]
+//   })
+//   render(<ApiReport />)
+//   await screen.findByText(/\/my\-path/)
+//   await screen.findByText(/\?\=123/)
+//   await screen.findByText(/22/)
+//   await screen.findByText(/55/)
+// })