Chore/async filters for unified logs (supabase#37200)

* Refactor retrieval of log counts

* Async filters

* Clean up

* Clean up

* Fix

Author: joshenlim

apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.fields.tsx

@@ -3,7 +3,7 @@ import { User } from 'lucide-react'
 
 import { LEVELS } from 'components/ui/DataTable/DataTable.constants'
 import { DataTableFilterField, Option } from 'components/ui/DataTable/DataTable.types'
-import { getLevelColor, getStatusColor } from 'components/ui/DataTable/DataTable.utils'
+import { getLevelColor } from 'components/ui/DataTable/DataTable.utils'
 import { cn } from 'ui'
 import { LOG_TYPES, METHODS, STATUS_CODE_LABELS } from './UnifiedLogs.constants'
 import { ColumnSchema } from './UnifiedLogs.schema'
@@ -41,22 +41,15 @@ export const filterFields = [
     value: 'status',
     type: 'checkbox',
     defaultOpen: true,
-    options: [
-      { label: '2xx', value: 200 },
-      { label: '4xx', value: 400 },
-      { label: '4xx', value: 500 },
-    ], // REMINDER: this is a placeholder to set the type in the client.tsx
+    options: [],
+    hasDynamicOptions: true,
     component: (props: Option) => {
       if (typeof props.value === 'boolean') return null
       if (typeof props.value === 'undefined') return null
 
       const statusValue = String(props.value)
       const statusLabel = STATUS_CODE_LABELS[statusValue as keyof typeof STATUS_CODE_LABELS]
 
-      // Convert string status codes to numbers for HTTP status styling
-      const statusCode = typeof props.value === 'string' ? parseInt(props.value, 10) : props.value
-      const isHttpStatus = !isNaN(statusCode) && statusCode >= 100 && statusCode < 600
-
       return (
         <div className="flex items-center gap-2 w-full min-w-0">
           <span className="flex-shrink-0 text-foreground">{statusValue}</span>
@@ -110,7 +103,9 @@ export const filterFields = [
     value: 'host',
     type: 'checkbox',
     defaultOpen: false,
-    options: [], // Will be populated dynamically from facets
+    options: [],
+    hasDynamicOptions: true,
+    hasAsyncSearch: true,
     component: (props: Option) => {
       return (
         <span className="truncate block text-[0.75rem]" title={props.value as string}>
@@ -123,8 +118,10 @@ export const filterFields = [
     label: 'Pathname',
     value: 'pathname',
     type: 'checkbox',
-    defaultOpen: true,
-    options: [], // Will be populated dynamically from facets
+    defaultOpen: false,
+    options: [],
+    hasDynamicOptions: true,
+    hasAsyncSearch: true,
     component: (props: Option) => {
       return (
         <span className="truncate block w-full text-[0.75rem]" title={props.value as string}>
@@ -138,7 +135,9 @@ export const filterFields = [
     value: 'auth_user',
     type: 'checkbox',
     defaultOpen: false,
-    options: [], // Will be populated dynamically from facets
+    options: [],
+    hasDynamicOptions: true,
+    hasAsyncSearch: true,
     component: (props: Option) => {
       return (
         <div className="flex items-center gap-2 min-w-0">

apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts

@@ -459,7 +459,7 @@ const getSupabaseStorageLogsQuery = () => {
 /**
  * Combine all log sources to create the unified logs CTE
  */
-const getUnifiedLogsCTE = () => {
+export const getUnifiedLogsCTE = () => {
   return `
 WITH unified_logs AS (
     ${getPostgrestLogsQuery()}
@@ -510,39 +510,72 @@ ${finalWhere}
  * Get a count query for the total logs within the timeframe
  * Uses proper faceted search behavior where facets show "what would I get if I selected ONLY this option"
  */
-export const getLogsCountQuery = (search: QuerySearchParamsType): string => {
-  const { finalWhere } = buildQueryConditions(search)
 
-  // Helper function to build WHERE clause excluding a specific field
-  const buildFacetWhere = (excludeField: string): string => {
-    const conditions: string[] = []
+// Helper function to build WHERE clause excluding a specific field
+const buildFacetWhere = (search: QuerySearchParamsType, excludeField: string): string => {
+  const conditions: string[] = []
 
-    Object.entries(search).forEach(([key, value]) => {
-      if (key === excludeField) return // Skip the field we're getting facets for
-      if (EXCLUDED_QUERY_PARAMS.includes(key as any)) return // Skip pagination and special params
+  Object.entries(search).forEach(([key, value]) => {
+    if (key === excludeField) return // Skip the field we're getting facets for
+    if (EXCLUDED_QUERY_PARAMS.includes(key as any)) return // Skip pagination and special params
 
-      // Handle array filters (IN clause)
-      if (Array.isArray(value) && value.length > 0) {
-        conditions.push(`${key} IN (${value.map((v) => `'${v}'`).join(',')})`)
-        return
-      }
+    // Handle array filters (IN clause)
+    if (Array.isArray(value) && value.length > 0) {
+      conditions.push(`${key} IN (${value.map((v) => `'${v}'`).join(',')})`)
+      return
+    }
 
-      // Handle scalar values
-      if (value !== null && value !== undefined) {
-        if (['host', 'pathname'].includes(key)) {
-          conditions.push(`${key} LIKE '%${value}%'`)
-        } else {
-          conditions.push(`${key} = '${value}'`)
-        }
+    // Handle scalar values
+    if (value !== null && value !== undefined) {
+      if (['host', 'pathname'].includes(key)) {
+        conditions.push(`${key} LIKE '%${value}%'`)
+      } else {
+        conditions.push(`${key} = '${value}'`)
       }
-    })
+    }
+  })
 
-    return conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''
-  }
+  return conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''
+}
+
+export const getFacetCountCTE = ({
+  search,
+  facet,
+  facetSearch,
+}: {
+  search: QuerySearchParamsType
+  facet: string
+  facetSearch?: string
+}) => {
+  const MAX_FACETS_QUANTITY = 20
+
+  return `
+${facet}_count AS (
+  SELECT '${facet}' as dimension, ${facet} as value, COUNT(*) as count
+  FROM unified_logs
+  ${buildFacetWhere(search, `${facet}`) || `WHERE ${facet} IS NOT NULL`}
+  ${buildFacetWhere(search, `${facet}`) ? ` AND ${facet} IS NOT NULL` : ''}
+  ${!!facetSearch ? `AND ${facet} LIKE '%${facetSearch}%'` : ''}
+  GROUP BY ${facet}
+  LIMIT ${MAX_FACETS_QUANTITY}
+)
+`.trim()
+}
+
+export const getLogsCountQuery = (search: QuerySearchParamsType): string => {
+  const { finalWhere } = buildQueryConditions(search)
 
   // Create a count query using the same unified logs CTE
   const sql = `
-${getUnifiedLogsCTE()}
+${getUnifiedLogsCTE()},
+${getFacetCountCTE({ search, facet: 'log_type' })},
+${getFacetCountCTE({ search, facet: 'method' })},
+${getFacetCountCTE({ search, facet: 'level' })},
+${getFacetCountCTE({ search, facet: 'status' })},
+${getFacetCountCTE({ search, facet: 'host' })},
+${getFacetCountCTE({ search, facet: 'pathname' })},
+${getFacetCountCTE({ search, facet: 'auth_user' })}
+
 -- Get total count
 SELECT 'total' as dimension, 'all' as value, COUNT(*) as count
 FROM unified_logs
@@ -551,65 +584,37 @@ ${finalWhere}
 UNION ALL
 
 -- Get counts by log_type (exclude log_type filter to avoid self-filtering)
-SELECT 'log_type' as dimension, log_type as value, COUNT(*) as count
-FROM unified_logs
-${buildFacetWhere('log_type') || 'WHERE log_type IS NOT NULL'}
-${buildFacetWhere('log_type') ? ' AND log_type IS NOT NULL' : ''}
-GROUP BY log_type
+SELECT dimension, value, count from log_type_count
 
 UNION ALL
 
 -- Get counts by method (exclude method filter to avoid self-filtering)  
-SELECT 'method' as dimension, method as value, COUNT(*) as count
-FROM unified_logs
-${buildFacetWhere('method') || 'WHERE method IS NOT NULL'}
-${buildFacetWhere('method') ? ' AND method IS NOT NULL' : ''}
-GROUP BY method
+SELECT dimension, value, count from method_count
 
 UNION ALL
 
 -- Get counts by level (exclude level filter to avoid self-filtering)
-SELECT 'level' as dimension, level as value, COUNT(*) as count
-FROM unified_logs
-${buildFacetWhere('level') || 'WHERE level IS NOT NULL'}
-${buildFacetWhere('level') ? ' AND level IS NOT NULL' : ''}
-GROUP BY level
+SELECT dimension, value, count from level_count
 
 UNION ALL
 
 -- Get counts by status (exclude status filter to avoid self-filtering)
-SELECT 'status' as dimension, status as value, COUNT(*) as count
-FROM unified_logs
-${buildFacetWhere('status') || 'WHERE status IS NOT NULL'}
-${buildFacetWhere('status') ? ' AND status IS NOT NULL' : ''}
-GROUP BY status
+SELECT dimension, value, count from status_count
 
 UNION ALL
 
 -- Get counts by host (exclude host filter to avoid self-filtering)
-SELECT 'host' as dimension, host as value, COUNT(*) as count
-FROM unified_logs
-${buildFacetWhere('host') || 'WHERE host IS NOT NULL'}
-${buildFacetWhere('host') ? ' AND host IS NOT NULL' : ''}
-GROUP BY host
+SELECT dimension, value, count from host_count
 
 UNION ALL
 
 -- Get counts by pathname (exclude pathname filter to avoid self-filtering)
-SELECT 'pathname' as dimension, pathname as value, COUNT(*) as count
-FROM unified_logs
-${buildFacetWhere('pathname') || 'WHERE pathname IS NOT NULL'}
-${buildFacetWhere('pathname') ? ' AND pathname IS NOT NULL' : ''}
-GROUP BY pathname
+SELECT dimension, value, count from pathname_count
 
 UNION ALL
 
 -- Get counts by auth_user (exclude auth_user filter to avoid self-filtering)
-SELECT 'auth_user' as dimension, auth_user as value, COUNT(*) as count
-FROM unified_logs
-${buildFacetWhere('auth_user') || 'WHERE auth_user IS NOT NULL'}
-${buildFacetWhere('auth_user') ? ' AND auth_user IS NOT NULL' : ''}
-GROUP BY auth_user
+SELECT dimension, value, count from auth_user_count
 `
 
   return sql

apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.tsx

@@ -71,8 +71,6 @@ export const UnifiedLogs = () => {
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>(defaultColumnFilters)
   const [rowSelection, setRowSelection] = useState<RowSelectionState>(defaultRowSelection)
 
-  const [showBottomLogsPanel, setShowBottomLogsPanelState] = useState(false)
-
   const [columnVisibility, setColumnVisibility] = useLocalStorageQuery<VisibilityState>(
     'data-table-visibility',
     defaultColumnVisibility
@@ -203,7 +201,11 @@ export const UnifiedLogs = () => {
   }, [isLoading, isFetching, flatData.length, table, selectedRowKey])
 
   // REMINDER: this is currently needed for the cmdk search
+  // [Joshen] This is where facets are getting dynamically loaded
   // TODO: auto search via API when the user changes the filter instead of hardcoded
+
+  // Will need to refactor this bit
+  // - Each facet just handles its own state, rather than getting passed down like this
   const filterFields = useMemo(() => {
     return defaultFilterFields.map((field) => {
       const facetsField = facets?.[field.value]
@@ -289,6 +291,7 @@ export const UnifiedLogs = () => {
       rowSelection={rowSelection}
       columnOrder={columnOrder}
       columnVisibility={columnVisibility}
+      searchParameters={searchParameters}
       enableColumnOrdering={true}
       isFetching={isFetching}
       isLoading={isLoading}

apps/studio/components/ui/DataTable/DataTable.types.ts

@@ -50,6 +50,8 @@ export type Base<TData> = {
    * Defines if the command input is disabled for this field
    */
   commandDisabled?: boolean
+  hasDynamicOptions?: boolean
+  hasAsyncSearch?: boolean
 }
 
 export type DataTableCheckboxFilterField<TData> = Base<TData> & Checkbox

apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterCheckbox.tsx

@@ -6,6 +6,7 @@ import type { DataTableCheckboxFilterField } from '../DataTable.types'
 import { formatCompactNumber } from '../DataTable.utils'
 import { InputWithAddons } from '../primitives/InputWithAddons'
 import { useDataTable } from '../providers/DataTableProvider'
+import { DataTableFilterCheckboxLoader } from './DataTableFilterCheckboxLoader'
 
 export function DataTableFilterCheckbox<TData>({
   value: _value,
@@ -34,31 +35,13 @@ export function DataTableFilterCheckbox<TData>({
   const filters = filterValue ? (Array.isArray(filterValue) ? filterValue : [filterValue]) : []
 
   // REMINDER: if no options are defined, while fetching data, we should show a skeleton
-  if (isLoading && !filterOptions?.length)
-    return (
-      <div className="grid divide-y rounded border border-border">
-        {Array.from({ length: 3 }).map((_, index) => (
-          <div key={index} className="flex items-center justify-between gap-2 px-2 py-2.5">
-            <Skeleton className="h-4 w-4 rounded-sm" />
-            <Skeleton className="h-4 w-full rounded-sm" />
-          </div>
-        ))}
-      </div>
-    )
+  if (isLoading && !filterOptions?.length) return <DataTableFilterCheckboxLoader />
 
   // Show empty state when no original options are available (not due to search filtering)
   if (!options?.length)
     return (
-      <div className="flex items-center justify-center px-2 py-6 text-center">
-        <div className="flex flex-col items-center gap-2">
-          <div className="h-8 w-8 rounded-full bg-muted flex items-center justify-center">
-            <Search className="h-4 w-4 text-muted-foreground" />
-          </div>
-          <div className="space-y-1">
-            <p className="text-sm text-muted-foreground">No options available</p>
-            <p className="text-xs text-muted-foreground/70">Try adjusting your filters</p>
-          </div>
-        </div>
+      <div className="flex items-center justify-center px-2 py-4 text-center border border-border rounded">
+        <p className="text-xs text-foreground-light">No options available</p>
       </div>
     )
 
@@ -77,9 +60,9 @@ export function DataTableFilterCheckbox<TData>({
       <div className="max-h-[200px] overflow-y-auto rounded border border-border empty:border-none">
         {filterOptions.length === 0 && inputValue !== '' ? (
           <div className="flex items-center justify-center px-2 py-4 text-center">
-            <div className="space-y-1">
-              <p className="text-sm text-muted-foreground">No results found</p>
-              <p className="text-xs text-muted-foreground/70">Try a different search term</p>
+            <div className="space-y-0.5">
+              <p className="text-xs text-foreground">No results found</p>
+              <p className="text-xs text-foreground-lighter">Try a different search term</p>
             </div>
           </div>
         ) : (