feat: Add percentages to filter values (#1250)

# Summary

Closes HDX-1960

This PR adds a button to our search filters component which can be used to show the _approximate_ percentage of rows which have each filter value.

https://github.com/user-attachments/assets/2dba1b28-d2b9-4414-986c-0c515d252c89

Notes:
- The percentages are based on a sample of 100k rows. The sampling is done similarly to how EE version samples logs for patterns.
- We only fetch the most common 100 values in the sample. All other values are assumed to represent <1% of the data.
- The percentages represent the distribution within the dataset after it has been filtered by the selected filters and the where clause.
- This is a potentially expensive query, even with sampling, so the percentages are only queried if they're toggled on for a particular filter, and do not refresh in live mode. They do refresh if the search or date ranges changes (outside of live mode).

Author: pulpdrew

.changeset/tricky-brooms-thank.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/app": patch
+---
+
+feat: Add percentages to filter values

packages/app/src/DBSearchPage.tsx

@@ -55,7 +55,7 @@ import {
   useDocumentVisibility,
 } from '@mantine/hooks';
 import { notifications } from '@mantine/notifications';
-import { useIsFetching } from '@tanstack/react-query';
+import { keepPreviousData, useIsFetching } from '@tanstack/react-query';
 import { SortingState } from '@tanstack/react-table';
 import CodeMirror from '@uiw/react-codemirror';
 
@@ -1099,7 +1099,10 @@ function DBSearchPage() {
     }
   }, [isReady, queryReady, isChartConfigLoading, onSearch]);
 
-  const { data: aliasMap } = useAliasMapFromChartConfig(dbSqlRowTableConfig);
+  const { data: aliasMap } = useAliasMapFromChartConfig(dbSqlRowTableConfig, {
+    placeholderData: keepPreviousData,
+    queryKey: ['aliasMap', dbSqlRowTableConfig, 'withPlaceholder'],
+  });
 
   const aliasWith = useMemo(
     () =>

packages/app/src/components/DBSearchPageFilters.tsx

@@ -40,6 +40,7 @@ import { useExplainQuery } from '@/hooks/useExplainQuery';
 import {
   useAllFields,
   useGetKeyValues,
+  useGetValuesDistribution,
   useJsonColumns,
   useTableMetadata,
 } from '@/hooks/useMetadata';
@@ -76,6 +77,8 @@ type FilterCheckboxProps = {
   onClickExclude?: VoidFunction;
   onClickPin: VoidFunction;
   className?: string;
+  percentage?: number;
+  isPercentageLoading?: boolean;
 };
 
 export const TextButton = ({
@@ -105,6 +108,26 @@ export const TextButton = ({
   );
 };
 
+type FilterPercentageProps = {
+  percentage: number;
+  isLoading?: boolean;
+};
+
+const FilterPercentage = ({ percentage, isLoading }: FilterPercentageProps) => {
+  const formattedPercentage =
+    percentage < 1
+      ? `<1%`
+      : percentage >= 99.5
+        ? `>99%`
+        : `~${Math.round(percentage)}%`;
+
+  return (
+    <Text size="xs" c="gray.3" className={isLoading ? 'effect-pulse' : ''}>
+      {formattedPercentage}
+    </Text>
+  );
+};
+
 const emptyFn = () => {};
 export const FilterCheckbox = ({
   value,
@@ -115,6 +138,8 @@ export const FilterCheckbox = ({
   onClickExclude,
   onClickPin,
   className,
+  percentage,
+  isPercentageLoading,
 }: FilterCheckboxProps) => {
   return (
     <div
@@ -146,15 +171,30 @@ export const FilterCheckbox = ({
           fz="xxs"
           color="gray"
         >
-          <Text
-            size="xs"
-            c={value === 'excluded' ? 'red.4' : 'gray.3'}
-            truncate="end"
+          <Group
             w="100%"
-            title={label}
+            gap="xs"
+            wrap="nowrap"
+            justify="space-between"
+            pe={'11px'}
+            miw={0}
           >
-            {label}
-          </Text>
+            <Text
+              size="xs"
+              c={value === 'excluded' ? 'red.4' : 'gray.3'}
+              truncate="end"
+              flex={1}
+              title={label}
+            >
+              {label}
+            </Text>
+            {percentage != null && (
+              <FilterPercentage
+                percentage={percentage}
+                isLoading={isPercentageLoading}
+              />
+            )}
+          </Group>
         </Tooltip>
       </Group>
       <div className={classes.filterActions}>
@@ -208,6 +248,8 @@ export type FilterGroupProps = {
   hasLoadedMore: boolean;
   isDefaultExpanded?: boolean;
   'data-testid'?: string;
+  chartConfig: ChartConfigWithDateRange;
+  isLive?: boolean;
 };
 
 const MAX_FILTER_GROUP_ITEMS = 10;
@@ -230,6 +272,8 @@ export const FilterGroup = ({
   hasLoadedMore,
   isDefaultExpanded,
   'data-testid': dataTestId,
+  chartConfig,
+  isLive,
 }: FilterGroupProps) => {
   const [search, setSearch] = useState('');
   // "Show More" button when there's lots of options
@@ -238,13 +282,60 @@ export const FilterGroup = ({
   const [isExpanded, setExpanded] = useState(isDefaultExpanded ?? false);
   // Track recently moved items for highlight animation
   const [recentlyMoved, setRecentlyMoved] = useState<Set<string>>(new Set());
+  // Show what percentage of the data has each value
+  const [showDistributions, setShowDistributions] = useState(false);
+  // For live searches, don't refresh percentages when date range changes
+  const [dateRange, setDateRange] = useState<[Date, Date]>(
+    chartConfig.dateRange,
+  );
+
+  const toggleShowDistributions = () => {
+    if (!showDistributions) {
+      setExpanded(true);
+      setDateRange(chartConfig.dateRange);
+    }
+    setShowDistributions(prev => !prev);
+  };
+
+  useEffect(() => {
+    if (!isLive) {
+      setDateRange(chartConfig.dateRange);
+    }
+  }, [chartConfig.dateRange, isLive]);
 
   useEffect(() => {
     if (isDefaultExpanded) {
       setExpanded(true);
     }
   }, [isDefaultExpanded]);
 
+  const {
+    data: distributionData,
+    isFetching: isFetchingDistribution,
+    error: distributionError,
+  } = useGetValuesDistribution(
+    {
+      chartConfig: { ...chartConfig, dateRange },
+      key: name,
+      limit: 100, // The 100 most common values are enough to find any values that are present in at least 1% of rows
+    },
+    {
+      enabled: showDistributions,
+    },
+  );
+
+  useEffect(() => {
+    if (distributionError) {
+      notifications.show({
+        color: 'red',
+        title: 'Error loading filter distribution',
+        message: distributionError?.message,
+        autoClose: 5000,
+      });
+      setShowDistributions(false);
+    }
+  }, [distributionError]);
+
   const totalFiltersSize =
     selectedValues.included.size + selectedValues.excluded.size;
 
@@ -292,6 +383,13 @@ export const FilterGroup = ({
       if (aExcluded && !bExcluded) return -1;
       if (!aExcluded && bExcluded) return 1;
 
+      // Then sort by estimated percentage of rows with this value, if available
+      const aPercentage = distributionData?.get(a.value) ?? 0;
+      const bPercentage = distributionData?.get(b.value) ?? 0;
+      if (aPercentage !== bPercentage) {
+        return bPercentage - aPercentage;
+      }
+
       // Finally sort alphabetically/numerically
       return a.value.localeCompare(b.value, undefined, { numeric: true });
     });
@@ -310,6 +408,7 @@ export const FilterGroup = ({
     augmentedOptions,
     selectedValues,
     totalFiltersSize,
+    distributionData,
   ]);
 
   // Simple highlight animation when checkbox is checked
@@ -402,13 +501,30 @@ export const FilterGroup = ({
               </Tooltip>
             </Accordion.Control>
             <Group gap="xxxs" wrap="nowrap">
+              <ActionIcon
+                size="xs"
+                variant="subtle"
+                color="gray"
+                onClick={toggleShowDistributions}
+                title={
+                  showDistributions ? 'Hide distribution' : 'Show distribution'
+                }
+                data-testid={`toggle-distribution-button-${name}`}
+                aria-checked={showDistributions}
+                role="checkbox"
+              >
+                <i
+                  className={`bi ${isFetchingDistribution ? 'spinner-border spinner-border-sm' : showDistributions ? 'bi-bar-chart-line-fill' : 'bi-bar-chart-line'}`}
+                />
+              </ActionIcon>
               {onFieldPinClick && (
                 <ActionIcon
                   size="xs"
                   variant="subtle"
                   color="gray"
                   onClick={onFieldPinClick}
                   title={isFieldPinned ? 'Unpin field' : 'Pin field'}
+                  me={'4px'}
                 >
                   <i
                     className={`bi bi-pin-angle${isFieldPinned ? '-fill' : ''}`}
@@ -452,6 +568,12 @@ export const FilterGroup = ({
                   onClickOnly={() => onOnlyClick(option.value)}
                   onClickExclude={() => onExcludeClick(option.value)}
                   onClickPin={() => onPinClick(option.value)}
+                  isPercentageLoading={isFetchingDistribution}
+                  percentage={
+                    showDistributions && distributionData
+                      ? (distributionData.get(option.value) ?? 0)
+                      : undefined
+                  }
                 />
               ))}
               {optionsLoading ? (
@@ -900,6 +1022,8 @@ const DBSearchPageFiltersComponent = ({
                   (filterState[facet.key].included.size > 0 ||
                     filterState[facet.key].excluded.size > 0))
               }
+              chartConfig={chartConfig}
+              isLive={isLive}
             />
           ))}