chore: Store sentinel proven performance for a longer time (#16955)

To support `slashInactivityConsecutiveEpochThreshold` greater than 1.

To avoid also storing a lot of raw stats, this PR also decouples the
history length and proven performance length stored.

Author: spalladino

yarn-project/aztec-node/src/sentinel/config.ts

@@ -2,6 +2,7 @@ import { type ConfigMappingsType, booleanConfigHelper, numberConfigHelper } from
 
 export type SentinelConfig = {
   sentinelHistoryLengthInEpochs: number;
+  sentinelHistoricProvenPerformanceLengthInEpochs: number;
   sentinelEnabled: boolean;
 };
 
@@ -11,6 +12,23 @@ export const sentinelConfigMappings: ConfigMappingsType<SentinelConfig> = {
     env: 'SENTINEL_HISTORY_LENGTH_IN_EPOCHS',
     ...numberConfigHelper(24),
   },
+  /**
+   * The number of L2 epochs kept of proven performance history for each validator.
+   * This value must be large enough so that we have proven performance for every validator
+   * for at least slashInactivityConsecutiveEpochThreshold. Assuming this value is 3,
+   * and the committee size is 48, and we have 10k validators, then we pick 48 out of 10k each draw.
+   * For any fixed element, per-draw prob = 48/10000 = 0.0048.
+   * After n draws, count ~ Binomial(n, 0.0048). We want P(X >= 3).
+   * Results (exact binomial):
+   * - 90% chance: n = 1108
+   * - 95% chance: n = 1310
+   * - 99% chance: n = 1749
+   */
+  sentinelHistoricProvenPerformanceLengthInEpochs: {
+    description: 'The number of L2 epochs kept of proven performance history for each validator.',
+    env: 'SENTINEL_HISTORIC_PROVEN_PERFORMANCE_LENGTH_IN_EPOCHS',
+    ...numberConfigHelper(2000),
+  },
   sentinelEnabled: {
     description: 'Whether the sentinel is enabled or not.',
     env: 'SENTINEL_ENABLED',

yarn-project/aztec-node/src/sentinel/factory.ts

@@ -27,6 +27,10 @@ export async function createSentinel(
     createLogger('node:sentinel:lmdb'),
   );
   const storeHistoryLength = config.sentinelHistoryLengthInEpochs * epochCache.getL1Constants().epochDuration;
-  const sentinelStore = new SentinelStore(kvStore, { historyLength: storeHistoryLength });
+  const storeHistoricProvenPerformanceLength = config.sentinelHistoricProvenPerformanceLengthInEpochs;
+  const sentinelStore = new SentinelStore(kvStore, {
+    historyLength: storeHistoryLength,
+    historicProvenPerformanceLength: storeHistoricProvenPerformanceLength,
+  });
   return new Sentinel(epochCache, archiver, p2p, sentinelStore, config, logger);
 }

yarn-project/aztec-node/src/sentinel/sentinel.test.ts

@@ -60,7 +60,7 @@ describe('sentinel', () => {
     blockStream = mock<L2BlockStream>();
 
     kvStore = await openTmpStore('sentinel-test');
-    store = new SentinelStore(kvStore, { historyLength: 10 });
+    store = new SentinelStore(kvStore, { historyLength: 10, historicProvenPerformanceLength: 5 });
 
     slot = 10n;
     epoch = 0n;

yarn-project/aztec-node/src/sentinel/sentinel.ts

@@ -217,7 +217,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
     }));
 
     if (criminals.length > 0) {
-      this.logger.info(
+      this.logger.verbose(
         `Identified ${criminals.length} validators to slash due to inactivity in at least ${epochThreshold} consecutive epochs`,
         { ...args, epochThreshold },
       );

yarn-project/aztec-node/src/sentinel/store.test.ts

@@ -1,5 +1,6 @@
 import { times } from '@aztec/foundation/collection';
 import { EthAddress } from '@aztec/foundation/eth-address';
+import { type Logger, createLogger } from '@aztec/foundation/log';
 import { AztecLMDBStoreV2, openTmpStore } from '@aztec/kv-store/lmdb-v2';
 import type { ValidatorStatusInSlot } from '@aztec/stdlib/validators';
 
@@ -8,11 +9,15 @@ import { SentinelStore } from './store.js';
 describe('sentinel-store', () => {
   let kvStore: AztecLMDBStoreV2;
   let store: SentinelStore;
+  let log: Logger;
+
   const historyLength = 4;
+  const historicProvenPerformanceLength = 3;
 
   beforeEach(async () => {
+    log = createLogger('sentinel:store:test');
     kvStore = await openTmpStore('sentinel-store-test');
-    store = new SentinelStore(kvStore, { historyLength });
+    store = new SentinelStore(kvStore, { historyLength, historicProvenPerformanceLength });
   });
 
   afterEach(async () => {
@@ -114,6 +119,85 @@ describe('sentinel-store', () => {
     ]);
   });
 
+  it('trims proven performance to the specified historicProvenPerformanceLength', async () => {
+    const validator = EthAddress.random();
+
+    // Add 5 epochs worth of proven performance data (more than historicProvenPerformanceLength = 3)
+    for (let i = 1; i <= 5; i++) {
+      await store.updateProvenPerformance(BigInt(i), { [validator.toString()]: { missed: i, total: 10 } });
+    }
+
+    const provenPerformance = await store.getProvenPerformance(validator);
+
+    // Should only keep the most recent 3 entries (epochs 3, 4, 5)
+    expect(provenPerformance).toHaveLength(historicProvenPerformanceLength);
+    expect(provenPerformance).toEqual([
+      { epoch: 3n, missed: 3, total: 10 },
+      { epoch: 4n, missed: 4, total: 10 },
+      { epoch: 5n, missed: 5, total: 10 },
+    ]);
+  });
+
+  it('getHistoricProvenPerformanceLength returns the correct value', () => {
+    expect(store.getHistoricProvenPerformanceLength()).toBe(historicProvenPerformanceLength);
+  });
+
+  it('proven performance with 2k entries', async () => {
+    const validator = EthAddress.random();
+    const totalEntries = 2000;
+
+    log.info(`Starting stress test with ${totalEntries} proven performance entries`);
+
+    // Track timing for additions
+    const addTimes: number[] = [];
+    const startTime = Date.now();
+
+    // Add 2k entries
+    for (let i = 1; i <= totalEntries; i++) {
+      const addStart = Date.now();
+      await store.updateProvenPerformance(BigInt(i), {
+        [validator.toString()]: { missed: i % 10, total: 10 },
+      });
+      const addEnd = Date.now();
+      addTimes.push(addEnd - addStart);
+
+      // Log progress every 500 entries
+      if (i % 500 === 0) {
+        const avgAddTime = addTimes.slice(-500).reduce((a, b) => a + b, 0) / 500;
+        log.info(`Added ${i}/${totalEntries} entries, avg time per entry: ${avgAddTime.toFixed(2)}ms`);
+      }
+    }
+
+    const totalAddTime = Date.now() - startTime;
+    const avgAddTime = addTimes.reduce((a, b) => a + b, 0) / addTimes.length;
+
+    log.info(`Added ${totalEntries} entries in ${totalAddTime}ms with avg ${avgAddTime.toFixed(2)}ms per entry`);
+
+    // Track timing for retrievals
+    const retrievalTimes: number[] = [];
+    const numRetrievals = 10;
+
+    log.info(`Starting ${numRetrievals} retrieval tests`);
+
+    for (let i = 0; i < numRetrievals; i++) {
+      const retrievalStart = Date.now();
+      const performance = await store.getProvenPerformance(validator);
+      const retrievalEnd = Date.now();
+      retrievalTimes.push(retrievalEnd - retrievalStart);
+
+      // Verify we only keep the configured number of entries
+      expect(performance).toHaveLength(historicProvenPerformanceLength);
+
+      // Verify we kept the most recent entries
+      const expectedStartEpoch = totalEntries - historicProvenPerformanceLength + 1;
+      expect(performance[0].epoch).toBe(BigInt(expectedStartEpoch));
+      expect(performance[performance.length - 1].epoch).toBe(BigInt(totalEntries));
+    }
+
+    const avgRetrievalTime = retrievalTimes.reduce((a, b) => a + b, 0) / retrievalTimes.length;
+    log.info(`Completed ${numRetrievals} retrievals with avg time of ${avgRetrievalTime.toFixed(2)}ms per retrieval`);
+  });
+
   it('does not allow insertion of invalid validator addresses', async () => {
     const validator = '0x123';
     await expect(store.updateProvenPerformance(1n, { [validator]: { missed: 2, total: 10 } })).rejects.toThrow();

yarn-project/aztec-node/src/sentinel/store.ts

@@ -19,7 +19,7 @@ export class SentinelStore {
 
   constructor(
     private store: AztecAsyncKVStore,
-    private config: { historyLength: number },
+    private config: { historyLength: number; historicProvenPerformanceLength: number },
   ) {
     this.historyMap = store.openMap('sentinel-validator-status');
     this.provenMap = store.openMap('sentinel-validator-proven');
@@ -29,6 +29,10 @@ export class SentinelStore {
     return this.config.historyLength;
   }
 
+  public getHistoricProvenPerformanceLength() {
+    return this.config.historicProvenPerformanceLength;
+  }
+
   public async updateProvenPerformance(epoch: bigint, performance: ValidatorsEpochPerformance) {
     await this.store.transactionAsync(async () => {
       for (const [who, { missed, total }] of Object.entries(performance)) {
@@ -65,8 +69,8 @@ export class SentinelStore {
     // Since we keep the size small, this is not a big deal.
     currentPerformance.sort((a, b) => Number(a.epoch - b.epoch));
 
-    // keep the most recent `historyLength` entries.
-    const performanceToKeep = currentPerformance.slice(-this.config.historyLength);
+    // keep the most recent `historicProvenPerformanceLength` entries.
+    const performanceToKeep = currentPerformance.slice(-this.config.historicProvenPerformanceLength);
 
     await this.provenMap.set(who.toString(), this.serializePerformance(performanceToKeep));
   }

yarn-project/foundation/src/config/env_var.ts

@@ -172,6 +172,7 @@ export type EnvVar =
   | 'RPC_SIMULATE_PUBLIC_MAX_GAS_LIMIT'
   | 'SENTINEL_ENABLED'
   | 'SENTINEL_HISTORY_LENGTH_IN_EPOCHS'
+  | 'SENTINEL_HISTORIC_PROVEN_PERFORMANCE_LENGTH_IN_EPOCHS'
   | 'SEQ_MAX_BLOCK_SIZE_IN_BYTES'
   | 'SEQ_MAX_TX_PER_BLOCK'
   | 'SEQ_MIN_TX_PER_BLOCK'

yarn-project/slasher/src/slash_offenses_collector.ts

@@ -95,10 +95,9 @@ export class SlashOffensesCollector {
    * Clears expired offenses from stores.
    */
   public async handleNewRound(round: bigint) {
-    this.log.verbose(`Clearing expired offenses for new slashing round ${round}`);
     const cleared = await this.offensesStore.clearExpiredOffenses(round);
     if (cleared && cleared > 0) {
-      this.log.verbose(`Cleared ${cleared} expired offenses`);
+      this.log.debug(`Cleared ${cleared} expired offenses for round ${round}`);
     }
   }