feat: Optimize and fix filtering on toStartOfX primary key expressions

Author: pulpdrew

.changeset/fluffy-mails-sparkle.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/app": patch
+---
+
+feat: Optimize and fix filtering on toStartOfX primary key expressions

packages/app/src/DBDashboardPage.tsx

@@ -66,6 +66,7 @@ import OnboardingModal from './components/OnboardingModal';
 import { Tags } from './components/Tags';
 import useDashboardFilters from './hooks/useDashboardFilters';
 import { useDashboardRefresh } from './hooks/useDashboardRefresh';
+import { useOptimizedTimestampValueExpression } from './hooks/useOptimizedTimestampValueExpression';
 import { parseAsStringWithNewLines } from './utils/queryParsers';
 import api from './api';
 import { DEFAULT_CHART_CONFIG } from './ChartUtils';
@@ -158,6 +159,10 @@ const Tile = forwardRef(
     const { data: source } = useSource({
       id: chart.config.source,
     });
+    const {
+      timestampValueExpression: optimizedTimestampValueExpression,
+      isLoading: isOptimizedTimestampExpressionLoading,
+    } = useOptimizedTimestampValueExpression(chart.config.source);
 
     // const prevSource = usePrevious(source);
     // const prevChart = usePrevious(chart);
@@ -166,7 +171,7 @@ const Tile = forwardRef(
     // const prevFilters = usePrevious(filters);
 
     useEffect(() => {
-      if (source != null) {
+      if (source != null && !isOptimizedTimestampExpressionLoading) {
         // TODO: will need to update this when we allow for multiple metrics per chart
         const firstSelect = chart.config.select[0];
         const metricType =
@@ -178,7 +183,9 @@ const Tile = forwardRef(
             connection: source.connection,
             dateRange,
             granularity,
-            timestampValueExpression: source.timestampValueExpression,
+            timestampValueExpression:
+              optimizedTimestampValueExpression ||
+              source.timestampValueExpression,
             from: {
               databaseName: source.from?.databaseName || 'default',
               tableName: tableName || '',
@@ -189,7 +196,15 @@ const Tile = forwardRef(
           });
         }
       }
-    }, [source, chart, dateRange, granularity, filters]);
+    }, [
+      source,
+      chart,
+      dateRange,
+      granularity,
+      filters,
+      optimizedTimestampValueExpression,
+      isOptimizedTimestampExpressionLoading,
+    ]);
 
     const [hovered, setHovered] = useState(false);
 

packages/app/src/DBSearchPage.tsx

@@ -100,6 +100,7 @@ import PatternTable from './components/PatternTable';
 import { DBSearchHeatmapChart } from './components/Search/DBSearchHeatmapChart';
 import SourceSchemaPreview from './components/SourceSchemaPreview';
 import { useTableMetadata } from './hooks/useMetadata';
+import { useOptimizedTimestampValueExpression } from './hooks/useOptimizedTimestampValueExpression';
 import { useSqlSuggestions } from './hooks/useSqlSuggestions';
 import {
   parseAsSortingStateString,
@@ -484,13 +485,19 @@ function useSearchedConfigToChartConfig({
   filters,
   orderBy,
 }: SearchConfig) {
-  const { data: sourceObj, isLoading } = useSource({
+  const { data: sourceObj, isLoading: isSourceLoading } = useSource({
     id: source,
   });
   const defaultOrderBy = useDefaultOrderBy(source);
+  const {
+    timestampValueExpression: optimizedTimestampValueExpression,
+    isLoading: isTimestampValueExpressionLoading,
+  } = useOptimizedTimestampValueExpression(source);
+
+  const isLoading = isSourceLoading || isTimestampValueExpressionLoading;
 
   return useMemo(() => {
-    if (sourceObj != null) {
+    if (sourceObj != null && !isLoading) {
       return {
         data: {
           select: select || (sourceObj.defaultTableSelectExpression ?? ''),
@@ -509,7 +516,9 @@ function useSearchedConfigToChartConfig({
           ...(filters != null ? { filters } : {}),
           where: where ?? '',
           whereLanguage: whereLanguage ?? 'sql',
-          timestampValueExpression: sourceObj.timestampValueExpression,
+          timestampValueExpression:
+            optimizedTimestampValueExpression ||
+            sourceObj.timestampValueExpression,
           implicitColumnExpression: sourceObj.implicitColumnExpression,
           connection: sourceObj.connection,
           displayType: DisplayType.Search,
@@ -527,6 +536,7 @@ function useSearchedConfigToChartConfig({
     where,
     whereLanguage,
     defaultOrderBy,
+    optimizedTimestampValueExpression,
   ]);
 }
 

packages/app/src/components/DBEditTimeChartForm.tsx

@@ -57,6 +57,7 @@ import { TimePicker } from '@/components/TimePicker';
 import { IS_LOCAL_MODE } from '@/config';
 import { GranularityPickerControlled } from '@/GranularityPicker';
 import { useFetchMetricResourceAttrs } from '@/hooks/useFetchMetricResourceAttrs';
+import { useOptimizedTimestampValueExpression } from '@/hooks/useOptimizedTimestampValueExpression';
 import SearchInputV2 from '@/SearchInputV2';
 import { getFirstTimestampValueExpression, useSource } from '@/source';
 import { parseTimeQuery } from '@/timeQuery';
@@ -490,15 +491,22 @@ export default function EditTimeChartForm({
     ChartConfigWithDateRange | undefined
   >(undefined);
 
+  const {
+    timestampValueExpression: optimizedTimestampValueExpression,
+    isLoading: isLoadingOptimizedTimestampExpression,
+  } = useOptimizedTimestampValueExpression(tableSource?.id);
+
   const onSubmit = useCallback(() => {
     handleSubmit(form => {
       setChartConfig(form);
-      if (tableSource != null) {
+      if (tableSource != null && !isLoadingOptimizedTimestampExpression) {
         const isSelectEmpty = !form.select || form.select.length === 0; // select is string or array
         const newConfig = {
           ...form,
           from: tableSource.from,
-          timestampValueExpression: tableSource.timestampValueExpression,
+          timestampValueExpression:
+            optimizedTimestampValueExpression ||
+            tableSource.timestampValueExpression,
           dateRange,
           connection: tableSource.connection,
           implicitColumnExpression: tableSource.implicitColumnExpression,
@@ -516,7 +524,15 @@ export default function EditTimeChartForm({
         );
       }
     })();
-  }, [handleSubmit, setChartConfig, setQueriedConfig, tableSource, dateRange]);
+  }, [
+    handleSubmit,
+    setChartConfig,
+    setQueriedConfig,
+    tableSource,
+    dateRange,
+    optimizedTimestampValueExpression,
+    isLoadingOptimizedTimestampExpression,
+  ]);
 
   useEffect(() => {
     if (submitRef) {
@@ -589,7 +605,9 @@ export default function EditTimeChartForm({
               },
             ],
             dateRange,
-            timestampValueExpression: tableSource.timestampValueExpression,
+            timestampValueExpression:
+              optimizedTimestampValueExpression ||
+              tableSource.timestampValueExpression,
             connection: tableSource.connection,
             from: tableSource.from,
             limit: { limit: 200 },
@@ -600,7 +618,13 @@ export default function EditTimeChartForm({
             granularity: undefined,
           }
         : null,
-    [queriedConfig, tableSource, dateRange, queryReady],
+    [
+      queriedConfig,
+      tableSource,
+      dateRange,
+      queryReady,
+      optimizedTimestampValueExpression,
+    ],
   );
 
   // Need to force a rerender on change as the modal will not be mounted when initially rendered