fix filltimeseries for reports (supabase#37160)

* fix filltimeseries

* fix type err

* fix seconds in tooltip changing

* fix filter default values

Author: jordienr

apps/studio/components/interfaces/Reports/ReportChart.tsx

@@ -7,6 +7,7 @@
  * This component acts as a bridge between the data-fetching logic and the
  * presentational chart component.
  */
+import { useFillTimeseriesSorted } from 'hooks/analytics/useFillTimeseriesSorted'
 import LogChartHandler from 'components/ui/Charts/LogChartHandler'
 import { useChartData } from 'hooks/useChartData'
 import type { UpdateDateRange } from 'pages/project/[ref]/reports/database'
@@ -60,6 +61,28 @@ const ReportChart = ({
         : chart.showMaxValue,
   })
 
+  const isTopListChart = chart.id === 'top-api-routes' || chart.id === 'top-rpc-functions'
+
+  const chartDataArray = Array.isArray(data) ? data : []
+
+  const { data: filledData, isError: isFillError } = useFillTimeseriesSorted(
+    chartDataArray,
+    'period_start',
+    (chartAttributes.length > 0 ? chartAttributes : chart.attributes).map(
+      (attr: any) => attr.attribute
+    ),
+    0,
+    startDate,
+    endDate,
+    undefined,
+    interval
+  )
+
+  const finalData =
+    chartDataArray.length > 0 && chartDataArray.length < 20 && !isFillError && !isTopListChart
+      ? filledData
+      : chartDataArray
+
   const getExpDemoChartData = () =>
     new Array(20).fill(0).map((_, index) => ({
       period_start: new Date(startDate).getTime() + index * 1000,
@@ -129,7 +152,7 @@ const ReportChart = ({
       attributes={
         (chartAttributes.length > 0 ? chartAttributes : chart.attributes) as MultiAttribute[]
       }
-      data={data}
+      data={finalData}
       isLoading={isLoadingChart || isLoading}
       highlightedValue={highlightedValue as any}
       updateDateRange={updateDateRange}

apps/studio/components/interfaces/Reports/Reports.constants.ts

@@ -36,38 +36,38 @@ export const REPORTS_DATEPICKER_HELPERS: ReportsDatetimeHelper[] = [
   },
   {
     text: REPORT_DATERANGE_HELPER_LABELS.LAST_60_MINUTES,
-    calcFrom: () => dayjs().subtract(1, 'hour').startOf('day').toISOString(),
+    calcFrom: () => dayjs().subtract(1, 'hour').toISOString(),
     calcTo: () => dayjs().toISOString(),
     default: true,
     availableIn: ['free', 'pro', 'team', 'enterprise'],
   },
   {
     text: REPORT_DATERANGE_HELPER_LABELS.LAST_3_HOURS,
-    calcFrom: () => dayjs().subtract(3, 'hour').startOf('day').toISOString(),
+    calcFrom: () => dayjs().subtract(3, 'hour').toISOString(),
     calcTo: () => dayjs().toISOString(),
     availableIn: ['free', 'pro', 'team', 'enterprise'],
   },
   {
     text: REPORT_DATERANGE_HELPER_LABELS.LAST_24_HOURS,
-    calcFrom: () => dayjs().subtract(1, 'day').startOf('day').toISOString(),
+    calcFrom: () => dayjs().subtract(1, 'day').toISOString(),
     calcTo: () => dayjs().toISOString(),
     availableIn: ['free', 'pro', 'team', 'enterprise'],
   },
   {
     text: REPORT_DATERANGE_HELPER_LABELS.LAST_7_DAYS,
-    calcFrom: () => dayjs().subtract(7, 'day').startOf('day').toISOString(),
+    calcFrom: () => dayjs().subtract(7, 'day').toISOString(),
     calcTo: () => dayjs().toISOString(),
     availableIn: ['pro', 'team', 'enterprise'],
   },
   {
     text: REPORT_DATERANGE_HELPER_LABELS.LAST_14_DAYS,
-    calcFrom: () => dayjs().subtract(14, 'day').startOf('day').toISOString(),
+    calcFrom: () => dayjs().subtract(14, 'day').toISOString(),
     calcTo: () => dayjs().toISOString(),
     availableIn: ['team', 'enterprise'],
   },
   {
     text: REPORT_DATERANGE_HELPER_LABELS.LAST_28_DAYS,
-    calcFrom: () => dayjs().subtract(28, 'day').startOf('day').toISOString(),
+    calcFrom: () => dayjs().subtract(28, 'day').toISOString(),
     calcTo: () => dayjs().toISOString(),
     availableIn: ['team', 'enterprise'],
   },

apps/studio/components/interfaces/Settings/Logs/Logs.utils.ts

@@ -474,8 +474,12 @@ export const fillTimeseries = (
   defaultValue: number,
   min?: string,
   max?: string,
-  minPointsToFill: number = 20
+  minPointsToFill: number = 20,
+  interval?: string
 ) => {
+  if (timeseriesData.length === 0 && !(min && max)) {
+    return []
+  }
   // If we have more points than minPointsToFill, just normalize timestamps and return
   if (timeseriesData.length > minPointsToFill) {
     return timeseriesData.map((datum) => {
@@ -493,41 +497,61 @@ export const fillTimeseries = (
 
   // const truncationSample = timeseriesData.length > 0 ? timeseriesData[0][timestampKey] : min || max
   const truncationSamples = timeseriesData.length > 0 ? dates : [minDate, maxDate]
-  const truncation = getTimestampTruncation(truncationSamples as Dayjs[])
+  let truncation: 'second' | 'minute' | 'hour' | 'day'
+  let step = 1
+
+  if (interval) {
+    const match = interval.match(/^(\d+)(m|h|d|s)$/)
+    if (match) {
+      step = parseInt(match[1], 10)
+      const unitChar = match[2] as 'm' | 'h' | 'd' | 's'
+      const unitMap = { s: 'second', m: 'minute', h: 'hour', d: 'day' } as const
+      truncation = unitMap[unitChar]
+    } else {
+      // Fallback for invalid format
+      truncation = getTimestampTruncation(truncationSamples as Dayjs[])
+    }
+  } else {
+    truncation = getTimestampTruncation(truncationSamples as Dayjs[])
+  }
 
   const newData = timeseriesData.map((datum) => {
-    const iso = dayjs.utc(datum[timestampKey]).toISOString()
+    const timestamp = datum[timestampKey]
+    const iso = isUnixMicro(timestamp)
+      ? unixMicroToIsoTimestamp(timestamp)
+      : dayjs.utc(timestamp).toISOString()
     datum[timestampKey] = iso
     return datum
   })
 
-  const diff = maxDate.diff(minDate, truncation as dayjs.UnitType)
-  // Intentional throwing of error here to be caught by Sentry, as this would indicate a bug since charts shouldn't be rendering more than 10k data points
-  if (diff > 10000) {
-    throw new Error(
-      'The selected date range will render more than 10,000 data points within the charts, which will degrade browser performance. Please select a smaller date range.'
-    )
-  }
-
-  for (let i = 0; i <= diff; i++) {
-    const dateToMaybeAdd = minDate.add(i, truncation as dayjs.ManipulateType)
-
-    const keys = typeof valueKey === 'string' ? [valueKey] : valueKey
-
-    const toMerge = keys.reduce(
-      (acc, key) => ({
-        ...acc,
-        [key]: defaultValue,
-      }),
-      {}
-    )
-
-    if (!dates.find((d) => isEqual(d, dateToMaybeAdd))) {
+  let currentDate = minDate
+  while (currentDate.isBefore(maxDate) || currentDate.isSame(maxDate)) {
+    const found = dates.find((d) => {
+      const d_date = d as Dayjs
+      return (
+        d_date.year() === currentDate.year() &&
+        d_date.month() === currentDate.month() &&
+        d_date.date() === currentDate.date() &&
+        d_date.hour() === currentDate.hour() &&
+        d_date.minute() === currentDate.minute()
+      )
+    })
+    if (!found) {
+      const keys = typeof valueKey === 'string' ? [valueKey] : valueKey
+
+      const toMerge = keys.reduce(
+        (acc, key) => ({
+          ...acc,
+          [key]: defaultValue,
+        }),
+        {}
+      )
       newData.push({
-        [timestampKey]: dateToMaybeAdd.toISOString(),
+        [timestampKey]: currentDate.toISOString(),
         ...toMerge,
       })
     }
+    currentDate = currentDate.add(step, truncation)
   }
 
   return newData

apps/studio/components/ui/Charts/ComposedChart.utils.tsx

@@ -131,6 +131,7 @@ export const calculateTotalChartAggregate = (
 const CustomTooltip = ({
   active,
   payload,
+  label,
   attributes,
   isPercentage,
   format,
@@ -139,7 +140,6 @@ const CustomTooltip = ({
   isActiveHoveredChart,
 }: TooltipProps) => {
   if (active && payload && payload.length) {
-    const timestamp = payload[0].payload.timestamp
     const maxValueAttribute = isMaxAttribute(attributes)
     const maxValueData =
       maxValueAttribute && payload?.find((p: any) => p.dataKey === maxValueAttribute.attribute)
@@ -203,7 +203,7 @@ const CustomTooltip = ({
           !isActiveHoveredChart && 'opacity-0'
         )}
       >
-        <p className="font-medium">{dayjs(timestamp).format(DateTimeFormats.FULL_SECONDS)}</p>
+        <p className="font-medium">{label}</p>
         <div className="grid gap-0">
           {payload
             .reverse()

apps/studio/hooks/misc/useReportDateRange.ts

@@ -175,17 +175,21 @@ export const useReportDateRange = (
 
   const handleIntervalGranularity = useCallback((from: string, to: string) => {
     const diffInDays = dayjs(to).diff(from, 'day', true)
+    const diffInHours = dayjs(to).diff(from, 'hour', true)
     const conditions = {
-      '1m': dayjs(to).diff(from, 'hour') < 3, // less than 3 hours
-      '10m': dayjs(to).diff(from, 'hour') < 6, // less than 6 hours
-      '30m': dayjs(to).diff(from, 'hour') < 18, // less than 18 hours
+      '1m': diffInHours < 1.1, // less than 1.1 hours
+      '5m': diffInHours < 3.1, // less than 3.1 hours
+      '10m': diffInHours < 6.1, // less than 6.1 hours
+      '30m': diffInHours < 25, // less than 25 hours
       '1h': diffInDays < 10, // less than 10 days
       '1d': diffInDays >= 10, // more than 10 days
     }
 
     switch (true) {
       case conditions['1m']:
         return '1m'
+      case conditions['5m']:
+        return '5m'
       case conditions['10m']:
         return '10m'
       case conditions['30m']: