Merge pull request #358 from ethpandaops/refactor/exec-perf-improvements

refactor(engine-timings): derive data from hourly & el-client

Author: mattevans

src/pages/ethereum/execution/timings/components/GetBlobsTab/GetBlobsTab.tsx

@@ -66,7 +66,7 @@ export function GetBlobsTab({ data, isLoading }: GetBlobsTabProps): JSX.Element
     return <GetBlobsTabSkeleton />;
   }
 
-  const { getBlobsBySlot, getBlobsByElClient, getBlobsDurationHistogram, getBlobsHourly, getBlobsDaily } = data;
+  const { getBlobsByElClient, getBlobsDurationHistogram, getBlobsHourly, getBlobsDaily } = data;
 
   // Filter to SUCCESS status only for duration-based charts
   const successBlobsByElClient = getBlobsByElClient.filter(r => r.status?.toUpperCase() === 'SUCCESS');
@@ -112,21 +112,27 @@ export function GetBlobsTab({ data, isLoading }: GetBlobsTabProps): JSX.Element
 
   const { totalObservations, successRate, avgDuration } = aggregatedStats;
 
-  // Prepare slot data for charts
-  const slotData = [...getBlobsBySlot].sort((a, b) => (a.slot ?? 0) - (b.slot ?? 0));
-  // Filter to SUCCESS only for metrics that should match the table
-  const successSlotData = slotData.filter(s => s.status?.toUpperCase() === 'SUCCESS');
+  // Calculate slot bounds from per-EL-client data (which has per-slot granularity)
+  const slotBounds = (() => {
+    let min = Infinity;
+    let max = -Infinity;
+    successBlobsByElClient.forEach(item => {
+      const slot = item.slot ?? 0;
+      if (slot < min) min = slot;
+      if (slot > max) max = slot;
+    });
+    return { minSlot: min === Infinity ? 0 : min, maxSlot: max === -Infinity ? 0 : max };
+  })();
 
-  const minSlot = slotData.length > 0 ? (slotData[0].slot ?? 0) : 0;
-  const maxSlot = slotData.length > 0 ? (slotData[slotData.length - 1].slot ?? 0) : 0;
+  const { minSlot, maxSlot } = slotBounds;
 
-  // Calculate weighted average blobs per request (SUCCESS only to match table)
-  const successSlotTotalObs = successSlotData.reduce((sum, s) => sum + (s.observation_count ?? 0), 0);
-  const totalWeightedBlobs = successSlotData.reduce(
+  // Calculate weighted average blobs per request from per-EL-client data (SUCCESS only)
+  const successTotalObs = successBlobsByElClient.reduce((sum, s) => sum + (s.observation_count ?? 0), 0);
+  const totalWeightedBlobs = successBlobsByElClient.reduce(
     (sum, s) => sum + (s.avg_returned_count ?? 0) * (s.observation_count ?? 0),
     0
   );
-  const avgBlobsPerRequest = successSlotTotalObs > 0 ? (totalWeightedBlobs / successSlotTotalObs).toFixed(1) : '0';
+  const avgBlobsPerRequest = successTotalObs > 0 ? (totalWeightedBlobs / successTotalObs).toFixed(1) : '0';
 
   // Prepare duration histogram data (SUCCESS status only)
   const histogramMap = new Map<number, number>();
@@ -140,30 +146,15 @@ export function GetBlobsTab({ data, isLoading }: GetBlobsTabProps): JSX.Element
   const histogramLabels = histogramBuckets.map(bucket => `${bucket}ms`);
   const histogramData = histogramBuckets.map(bucket => histogramMap.get(bucket) ?? 0);
 
-  // Calculate status breakdown from slot data for the status distribution chart
-  const statusTotals = slotData.reduce(
-    (acc, item) => {
-      const status = item.status?.toUpperCase() ?? 'UNKNOWN';
-      const count = item.observation_count ?? 0;
-      switch (status) {
-        case 'SUCCESS':
-          acc.success += count;
-          break;
-        case 'PARTIAL':
-          acc.partial += count;
-          break;
-        case 'EMPTY':
-          acc.empty += count;
-          break;
-        case 'ERROR':
-          acc.error += count;
-          break;
-        case 'UNSUPPORTED':
-          acc.unsupported += count;
-          break;
-      }
-      return acc;
-    },
+  // Calculate status breakdown from pre-aggregated hourly data
+  const statusTotals = getBlobsHourly.reduce(
+    (acc, item) => ({
+      success: acc.success + (item.success_count ?? 0),
+      partial: acc.partial + (item.partial_count ?? 0),
+      empty: acc.empty + (item.empty_count ?? 0),
+      error: acc.error + (item.error_count ?? 0),
+      unsupported: acc.unsupported + (item.unsupported_count ?? 0),
+    }),
     { success: 0, partial: 0, empty: 0, error: 0, unsupported: 0 }
   );
 
@@ -177,9 +168,9 @@ export function GetBlobsTab({ data, isLoading }: GetBlobsTabProps): JSX.Element
     { value: statusTotals.unsupported, color: themeColors.muted },
   ];
 
-  // Prepare blob count distribution (SUCCESS only to match table)
+  // Prepare blob count distribution from per-EL-client data (SUCCESS only to match table)
   const blobCountMap = new Map<number, number>();
-  successSlotData.forEach(item => {
+  successBlobsByElClient.forEach(item => {
     const blobCount = item.avg_returned_count ?? 0;
     const roundedCount = Math.round(blobCount);
     const obsCount = item.observation_count ?? 0;

src/pages/ethereum/execution/timings/components/NewPayloadTab/NewPayloadTab.tsx

@@ -20,7 +20,7 @@ export interface NewPayloadTabProps {
  */
 export function NewPayloadTab({ data }: NewPayloadTabProps): JSX.Element {
   const themeColors = useThemeColors();
-  const { newPayloadBySlot, newPayloadDurationHistogram, newPayloadByElClient } = data;
+  const { newPayloadDurationHistogram, newPayloadByElClient } = data;
 
   // Filter to VALID status only for duration-based charts
   const validPayloadByElClient = newPayloadByElClient.filter(r => r.status?.toUpperCase() === 'VALID');
@@ -74,11 +74,20 @@ export function NewPayloadTab({ data }: NewPayloadTabProps): JSX.Element {
 
   const { totalObservations, validRate, avgDuration, avgP95Duration } = aggregatedStats;
 
-  // Prepare slot data for charts (not stats)
-  const slotData = [...newPayloadBySlot].sort((a, b) => (a.slot ?? 0) - (b.slot ?? 0));
+  // Calculate slot bounds from per-EL-client data (which has per-slot granularity)
+  // This avoids needing the slow newPayloadBySlot endpoint
+  const slotBounds = (() => {
+    let min = Infinity;
+    let max = -Infinity;
+    validPayloadByElClient.forEach(item => {
+      const slot = item.slot ?? 0;
+      if (slot < min) min = slot;
+      if (slot > max) max = slot;
+    });
+    return { minSlot: min === Infinity ? 0 : min, maxSlot: max === -Infinity ? 0 : max };
+  })();
 
-  const minSlot = slotData.length > 0 ? (slotData[0].slot ?? 0) : 0;
-  const maxSlot = slotData.length > 0 ? (slotData[slotData.length - 1].slot ?? 0) : 0;
+  const { minSlot, maxSlot } = slotBounds;
 
   // Prepare duration histogram data (VALID status only)
   // Group histogram data by bucket and sum VALID counts
@@ -93,33 +102,16 @@ export function NewPayloadTab({ data }: NewPayloadTabProps): JSX.Element {
   const histogramLabels = histogramBuckets.map(bucket => `${bucket}ms`);
   const histogramData = histogramBuckets.map(bucket => histogramMap.get(bucket) ?? 0);
 
-  // Calculate status breakdown from slot data for the status distribution chart
-  const statusTotals = slotData.reduce(
-    (acc, item) => {
-      const status = item.status?.toUpperCase() ?? 'UNKNOWN';
-      const count = item.observation_count ?? 0;
-      switch (status) {
-        case 'VALID':
-          acc.valid += count;
-          break;
-        case 'INVALID':
-          acc.invalid += count;
-          break;
-        case 'SYNCING':
-          acc.syncing += count;
-          break;
-        case 'ACCEPTED':
-          acc.accepted += count;
-          break;
-        case 'INVALID_BLOCK_HASH':
-          acc.invalidBlockHash += count;
-          break;
-        case 'ERROR':
-          acc.error += count;
-          break;
-      }
-      return acc;
-    },
+  // Calculate status breakdown from pre-aggregated hourly data
+  const statusTotals = newPayloadHourly.reduce(
+    (acc, item) => ({
+      valid: acc.valid + (item.valid_count ?? 0),
+      invalid: acc.invalid + (item.invalid_count ?? 0),
+      syncing: acc.syncing + (item.syncing_count ?? 0),
+      accepted: acc.accepted + (item.accepted_count ?? 0),
+      invalidBlockHash: acc.invalidBlockHash + (item.invalid_block_hash_count ?? 0),
+      error: 0, // Hourly data doesn't track errors separately (they're rare)
+    }),
     { valid: 0, invalid: 0, syncing: 0, accepted: 0, invalidBlockHash: 0, error: 0 }
   );
 

src/pages/ethereum/execution/timings/hooks/useEngineTimingsData.ts

@@ -3,24 +3,20 @@ import { useQueries } from '@tanstack/react-query';
 import {
   fctEngineNewPayloadStatusHourlyServiceList,
   fctEngineNewPayloadStatusDailyServiceList,
-  fctEngineNewPayloadBySlotServiceList,
   fctEngineNewPayloadByElClientServiceList,
   fctEngineNewPayloadDurationChunked50MsServiceList,
   fctEngineGetBlobsStatusHourlyServiceList,
   fctEngineGetBlobsStatusDailyServiceList,
-  fctEngineGetBlobsBySlotServiceList,
   fctEngineGetBlobsByElClientServiceList,
   fctEngineGetBlobsDurationChunked50MsServiceList,
 } from '@/api/sdk.gen';
 import type {
   FctEngineNewPayloadStatusHourly,
   FctEngineNewPayloadStatusDaily,
-  FctEngineNewPayloadBySlot,
   FctEngineNewPayloadByElClient,
   FctEngineNewPayloadDurationChunked50Ms,
   FctEngineGetBlobsStatusHourly,
   FctEngineGetBlobsStatusDaily,
-  FctEngineGetBlobsBySlot,
   FctEngineGetBlobsByElClient,
   FctEngineGetBlobsDurationChunked50Ms,
 } from '@/api/types.gen';
@@ -33,10 +29,7 @@ export interface EngineTimingsData {
   newPayloadHourly: FctEngineNewPayloadStatusHourly[];
   newPayloadDaily: FctEngineNewPayloadStatusDaily[];
 
-  // newPayload per-slot data
-  newPayloadBySlot: FctEngineNewPayloadBySlot[];
-
-  // newPayload per-EL-client aggregations
+  // newPayload per-EL-client aggregations (includes per-slot granularity)
   newPayloadByElClient: FctEngineNewPayloadByElClient[];
 
   // newPayload duration histogram
@@ -46,10 +39,7 @@ export interface EngineTimingsData {
   getBlobsHourly: FctEngineGetBlobsStatusHourly[];
   getBlobsDaily: FctEngineGetBlobsStatusDaily[];
 
-  // getBlobs per-slot data
-  getBlobsBySlot: FctEngineGetBlobsBySlot[];
-
-  // getBlobs per-EL-client aggregations
+  // getBlobs per-EL-client aggregations (includes per-slot granularity)
   getBlobsByElClient: FctEngineGetBlobsByElClient[];
 
   // getBlobs duration histogram
@@ -192,27 +182,7 @@ export function useEngineTimingsData({
           ),
         enabled: !!currentNetwork && !useHourlyData,
       },
-      // newPayload per-slot data
-      {
-        queryKey: ['engine-timings', 'newPayload-by-slot', start, end, timeRange, referenceNodesOnly],
-        queryFn: ({ signal }) =>
-          fetchAllPages<FctEngineNewPayloadBySlot>(
-            fctEngineNewPayloadBySlotServiceList,
-            {
-              query: {
-                slot_start_date_time_gte: start,
-                slot_start_date_time_lte: end,
-                order_by: 'slot DESC',
-                page_size: 10000,
-                ...refNodeFilter,
-              },
-            },
-            'fct_engine_new_payload_by_slot',
-            signal
-          ),
-        enabled: !!currentNetwork,
-      },
-      // newPayload per-EL-client aggregations
+      // newPayload per-EL-client aggregations (includes per-slot data)
       {
         queryKey: ['engine-timings', 'newPayload-by-el-client', start, end, referenceNodesOnly],
         queryFn: ({ signal }) =>
@@ -291,27 +261,7 @@ export function useEngineTimingsData({
           ),
         enabled: !!currentNetwork && !useHourlyData && fetchBlobs,
       },
-      // getBlobs per-slot data
-      {
-        queryKey: ['engine-timings', 'getBlobs-by-slot', start, end, timeRange, referenceNodesOnly],
-        queryFn: ({ signal }) =>
-          fetchAllPages<FctEngineGetBlobsBySlot>(
-            fctEngineGetBlobsBySlotServiceList,
-            {
-              query: {
-                slot_start_date_time_gte: start,
-                slot_start_date_time_lte: end,
-                order_by: 'slot DESC',
-                page_size: 10000,
-                ...refNodeFilter,
-              },
-            },
-            'fct_engine_get_blobs_by_slot',
-            signal
-          ),
-        enabled: !!currentNetwork && fetchBlobs,
-      },
-      // getBlobs per-EL-client aggregations
+      // getBlobs per-EL-client aggregations (includes per-slot data)
       {
         queryKey: ['engine-timings', 'getBlobs-by-el-client', start, end, referenceNodesOnly],
         queryFn: ({ signal }) =>
@@ -358,19 +308,17 @@ export function useEngineTimingsData({
   const [
     newPayloadHourlyQuery,
     newPayloadDailyQuery,
-    newPayloadBySlotQuery,
     newPayloadByElClientQuery,
     newPayloadDurationHistogramQuery,
     getBlobsHourlyQuery,
     getBlobsDailyQuery,
-    getBlobsBySlotQuery,
     getBlobsByElClientQuery,
     getBlobsDurationHistogramQuery,
   ] = queries;
 
-  // Check loading state for newPayload queries only (first 5)
-  const newPayloadQueries = queries.slice(0, 5);
-  const blobQueries = queries.slice(5);
+  // Check loading state for newPayload queries only (first 4)
+  const newPayloadQueries = queries.slice(0, 4);
+  const blobQueries = queries.slice(4);
 
   const isLoading = newPayloadQueries.some(q => q.isLoading);
   const isLoadingBlobs = blobQueries.some(q => q.isLoading);
@@ -384,12 +332,10 @@ export function useEngineTimingsData({
       ? {
           newPayloadHourly: newPayloadHourlyQuery.data ?? [],
           newPayloadDaily: newPayloadDailyQuery.data ?? [],
-          newPayloadBySlot: newPayloadBySlotQuery.data ?? [],
           newPayloadByElClient: newPayloadByElClientQuery.data ?? [],
           newPayloadDurationHistogram: newPayloadDurationHistogramQuery.data ?? [],
           getBlobsHourly: getBlobsHourlyQuery.data ?? [],
           getBlobsDaily: getBlobsDailyQuery.data ?? [],
-          getBlobsBySlot: getBlobsBySlotQuery.data ?? [],
           getBlobsByElClient: getBlobsByElClientQuery.data ?? [],
           getBlobsDurationHistogram: getBlobsDurationHistogramQuery.data ?? [],
         }