feat: ValidatorKeystore implementation for high-availability signer

yarn-project/aztec-node/package.json

@@ -87,6 +87,7 @@
     "@aztec/stdlib": "workspace:^",
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/validator-client": "workspace:^",
+    "@aztec/validator-ha-signer": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "koa": "^2.16.1",
     "koa-router": "^13.1.1",

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -332,7 +332,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     const watchers: Watcher[] = [];
 
     // Create validator client if required
-    const validatorClient = createValidatorClient(config, {
+    const validatorClient = await createValidatorClient(config, {
       checkpointsBuilder: validatorCheckpointsBuilder,
       worldState: worldStateSynchronizer,
       p2pClient,

yarn-project/aztec-node/tsconfig.json

@@ -72,6 +72,9 @@
     {
       "path": "../validator-client"
     },
+    {
+      "path": "../validator-ha-signer"
+    },
     {
       "path": "../world-state"
     }

yarn-project/end-to-end/package.json

@@ -61,6 +61,7 @@
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/test-wallet": "workspace:^",
     "@aztec/validator-client": "workspace:^",
+    "@aztec/validator-ha-signer": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "@iarna/toml": "^2.2.5",
     "@jest/globals": "^30.0.0",

yarn-project/end-to-end/src/e2e_p2p/reex.test.ts

@@ -11,6 +11,7 @@ import type { CppPublicTxSimulator, PublicTxResult } from '@aztec/simulator/serv
 import { BlockProposal } from '@aztec/stdlib/p2p';
 import { ReExFailedTxsError, ReExStateMismatchError, ReExTimeoutError } from '@aztec/stdlib/validators';
 import type { ValidatorKeyStore } from '@aztec/validator-client';
+import { DutyType } from '@aztec/validator-ha-signer/types';
 
 import { describe, it, jest } from '@jest/globals';
 import fs from 'fs';
@@ -148,7 +149,13 @@ describe('e2e_p2p_reex', () => {
           proposal.archiveRoot,
           proposal.txHashes,
           undefined,
-          payload => signer.signMessageWithAddress(proposerAddress!, payload),
+          payload =>
+            signer.signMessageWithAddress(proposerAddress!, payload, {
+              slot: proposal.blockHeader.globalVariables.slotNumber,
+              blockNumber: proposal.blockHeader.globalVariables.blockNumber,
+              blockIndexWithinCheckpoint: proposal.indexWithinCheckpoint,
+              dutyType: DutyType.BLOCK_PROPOSAL,
+            }),
         );
 
         const p2pService = (p2pClient as any).p2pService as LibP2PService;

yarn-project/end-to-end/tsconfig.json

@@ -109,6 +109,9 @@
     {
       "path": "../validator-client"
     },
+    {
+      "path": "../validator-ha-signer"
+    },
     {
       "path": "../world-state"
     }

yarn-project/foundation/src/config/env_var.ts

@@ -314,11 +314,11 @@ export type EnvVar =
   | 'MAX_ALLOWED_ETH_CLIENT_DRIFT_SECONDS'
   | 'LEGACY_BLS_CLI'
   | 'DEBUG_FORCE_TX_PROOF_VERIFICATION'
-  | 'SLASHING_PROTECTION_NODE_ID'
-  | 'SLASHING_PROTECTION_POLLING_INTERVAL_MS'
-  | 'SLASHING_PROTECTION_SIGNING_TIMEOUT_MS'
-  | 'SLASHING_PROTECTION_ENABLED'
-  | 'SLASHING_PROTECTION_MAX_STUCK_DUTIES_AGE_MS'
+  | 'VALIDATOR_HA_SIGNING_ENABLED'
+  | 'VALIDATOR_HA_NODE_ID'
+  | 'VALIDATOR_HA_POLLING_INTERVAL_MS'
+  | 'VALIDATOR_HA_SIGNING_TIMEOUT_MS'
+  | 'VALIDATOR_HA_MAX_STUCK_DUTIES_AGE_MS'
   | 'VALIDATOR_HA_DATABASE_URL'
   | 'VALIDATOR_HA_RUN_MIGRATIONS'
   | 'VALIDATOR_HA_POOL_MAX'

yarn-project/sequencer-client/package.json

@@ -49,6 +49,7 @@
     "@aztec/stdlib": "workspace:^",
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/validator-client": "workspace:^",
+    "@aztec/validator-ha-signer": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "lodash.chunk": "^4.2.0",
     "tslib": "^2.4.0",
@@ -57,6 +58,7 @@
   "devDependencies": {
     "@aztec/archiver": "workspace:^",
     "@aztec/kv-store": "workspace:^",
+    "@electric-sql/pglite": "^0.3.14",
     "@jest/globals": "^30.0.0",
     "@types/jest": "^30.0.0",
     "@types/lodash.chunk": "^4.2.7",

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts

@@ -33,6 +33,8 @@ import { GlobalVariables, type Tx } from '@aztec/stdlib/tx';
 import { AttestationTimeoutError } from '@aztec/stdlib/validators';
 import { getTelemetryClient } from '@aztec/telemetry-client';
 import type { FullNodeCheckpointsBuilder, ValidatorClient } from '@aztec/validator-client';
+import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validator-ha-signer/errors';
+import { DutyType } from '@aztec/validator-ha-signer/types';
 
 import { expect, jest } from '@jest/globals';
 import EventEmitter from 'events';
@@ -420,6 +422,72 @@ describe('CheckpointProposalJob', () => {
     });
   });
 
+  /**
+   * Helper to set up multiple blocks for testing.
+   * Creates the specified number of blocks with proper global variables and seeds the checkpoint builder.
+   * @param numBlocks - Number of blocks to create
+   * @param txsPerBlock - Number of transactions per block (or array for different counts per block)
+   * @param startBlockNumber - Starting block number (defaults to newBlockNumber)
+   * @returns Object containing the created blocks, txs, and the last block for attestations
+   */
+  async function setupMultipleBlocks(
+    numBlocks: number,
+    txsPerBlock: number | number[] = 1,
+    startBlockNumber: BlockNumber = newBlockNumber,
+  ): Promise<{
+    blocks: Awaited<ReturnType<typeof makeBlock>>[];
+    txs: Awaited<ReturnType<typeof makeTx>>[];
+    lastBlock: Awaited<ReturnType<typeof makeBlock>>;
+  }> {
+    // Create txs - determine total needed
+    const txsPerBlockArray = Array.isArray(txsPerBlock) ? txsPerBlock : Array(numBlocks).fill(txsPerBlock);
+    const totalTxs = txsPerBlockArray.reduce((sum, count) => sum + count, 0);
+    const txs = await Promise.all(Array.from({ length: totalTxs }, (_, i) => makeTx(i + 1, chainId)));
+
+    // Set up p2p mocks
+    p2p.getPendingTxCount.mockResolvedValue(10);
+    p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs)));
+
+    // Create blocks with incrementing block numbers
+    const blocks: Awaited<ReturnType<typeof makeBlock>>[] = [];
+    const blockTxs: Awaited<ReturnType<typeof makeTx>>[][] = [];
+    let txIndex = 0;
+
+    for (let i = 0; i < numBlocks; i++) {
+      const blockNum = BlockNumber(startBlockNumber + i);
+      const blockGlobalVariables =
+        i === 0
+          ? globalVariables
+          : new GlobalVariables(
+              chainId,
+              version,
+              blockNum,
+              SlotNumber(newSlotNumber),
+              0n,
+              coinbase,
+              feeRecipient,
+              gasFees,
+            );
+
+      const blockTxCount = txsPerBlockArray[i];
+      const blockTxsSlice = txs.slice(txIndex, txIndex + blockTxCount);
+      txIndex += blockTxCount;
+
+      const block = await makeBlock(blockTxsSlice, blockGlobalVariables);
+      blocks.push(block);
+      blockTxs.push(blockTxsSlice);
+    }
+
+    // Seed checkpoint builder with all blocks
+    checkpointBuilder.seedBlocks(blocks, blockTxs);
+
+    return {
+      blocks,
+      txs,
+      lastBlock: blocks[blocks.length - 1],
+    };
+  }
+
   /**
    * Helper to create a TestCheckpointProposalJob instance with current mocks.
    * Uses TestCheckpointProposalJob which has waitUntilTimeInSlot as a no-op.
@@ -483,31 +551,9 @@ describe('CheckpointProposalJob', () => {
         .mockReturnValueOnce({ canStart: true, deadline: 18, isLastBlock: true })
         .mockReturnValue({ canStart: false, deadline: undefined, isLastBlock: false });
 
-      // Create enough txs for 2 blocks
-      const txs = await Promise.all([makeTx(1, chainId), makeTx(2, chainId), makeTx(3, chainId)]);
-
-      // Always have txs available
-      p2p.getPendingTxCount.mockResolvedValue(10);
-      p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs)));
-
-      // Create 2 blocks
-      const block1 = await makeBlock(txs.slice(0, 2), globalVariables);
-      const globalVariables2 = new GlobalVariables(
-        chainId,
-        version,
-        BlockNumber(newBlockNumber + 1),
-        SlotNumber(newSlotNumber),
-        0n,
-        coinbase,
-        feeRecipient,
-        gasFees,
-      );
-      const block2 = await makeBlock([txs[2]], globalVariables2);
-
-      // Seed MockCheckpointBuilder with blocks to return sequentially
-      checkpointBuilder.seedBlocks([block1, block2], [txs.slice(0, 2), [txs[2]]]);
-
-      validatorClient.collectAttestations.mockResolvedValue(getAttestations(block2));
+      // Set up test data for 2 blocks
+      const { lastBlock } = await setupMultipleBlocks(2, [2, 1]);
+      validatorClient.collectAttestations.mockResolvedValue(getAttestations(lastBlock));
 
       // Install spy on waitUntilTimeInSlot to verify it's called with expected deadlines
       const waitSpy = jest.spyOn(job, 'waitUntilTimeInSlot');
@@ -569,16 +615,9 @@ describe('CheckpointProposalJob', () => {
         .mockReturnValueOnce({ canStart: true, deadline: 2 + 3 * blockDurationSeconds, isLastBlock: true })
         .mockReturnValue({ canStart: false, deadline: undefined, isLastBlock: false });
 
-      const txs = await Promise.all([makeTx(1, chainId), makeTx(2, chainId), makeTx(3, chainId)]);
-      const block = await makeBlock(txs, globalVariables);
-
-      p2p.getPendingTxCount.mockResolvedValue(10);
-      p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs)));
-
-      // Seed with 3 identical blocks (each with 1 tx)
-      checkpointBuilder.seedBlocks([block, block, block], [[txs[0]], [txs[1]], [txs[2]]]);
-
-      validatorClient.collectAttestations.mockResolvedValue(getAttestations(block));
+      // Set up test data for 3 blocks
+      const { lastBlock } = await setupMultipleBlocks(3, 1);
+      validatorClient.collectAttestations.mockResolvedValue(getAttestations(lastBlock));
 
       const waitSpy = jest.spyOn(job, 'waitUntilTimeInSlot');
 
@@ -801,6 +840,90 @@ describe('CheckpointProposalJob', () => {
       expect(validatorClient.collectAttestations).toHaveBeenCalled();
     });
   });
+
+  describe('HA error handling during block building', () => {
+    it('should stop checkpoint building when block proposal throws DutyAlreadySignedError on first block', async () => {
+      // Set up test data for 3 blocks (to verify it stops even with multiple blocks configured)
+      const { lastBlock } = await setupMultipleBlocks(3, 1);
+      validatorClient.collectAttestations.mockResolvedValue(getAttestations(lastBlock));
+
+      // Create job first
+      job.setTimetable(
+        new SequencerTimetable({
+          ethereumSlotDuration,
+          aztecSlotDuration: slotDuration,
+          l1PublishingTime: ethereumSlotDuration,
+          blockDurationMs: 8000,
+          enforce: true,
+        }),
+      );
+
+      // Mock timetable to allow multiple blocks
+      jest
+        .spyOn(job.getTimetable(), 'canStartNextBlock')
+        .mockReturnValueOnce({ canStart: true, deadline: 4, isLastBlock: false })
+        .mockReturnValueOnce({ canStart: true, deadline: 8, isLastBlock: false })
+        .mockReturnValueOnce({ canStart: true, deadline: 12, isLastBlock: false })
+        .mockReturnValue({ canStart: false, deadline: undefined, isLastBlock: false });
+
+      // Mock to throw on first block proposal
+      validatorClient.createBlockProposal.mockImplementation(() => {
+        throw new DutyAlreadySignedError(SlotNumber(1), DutyType.BLOCK_PROPOSAL, 0, 'node-2');
+      });
+
+      const result = await job.execute();
+
+      // Should return undefined and stop building
+      expect(result).toBeUndefined();
+      // Should have attempted only 1 block proposal (first one threw)
+      expect(validatorClient.createBlockProposal).toHaveBeenCalledTimes(1);
+      // Should not have attempted checkpoint proposal
+      expect(validatorClient.createCheckpointProposal).not.toHaveBeenCalled();
+      // Should not publish anything
+      expect(publisher.enqueueProposeCheckpoint).not.toHaveBeenCalled();
+    });
+
+    it('should stop checkpoint building when block proposal throws SlashingProtectionError on first block', async () => {
+      // Set up test data for 3 blocks (to verify it stops even with multiple blocks configured)
+      const { lastBlock } = await setupMultipleBlocks(3, 1);
+      validatorClient.collectAttestations.mockResolvedValue(getAttestations(lastBlock));
+
+      // Create job first
+      job.setTimetable(
+        new SequencerTimetable({
+          ethereumSlotDuration,
+          aztecSlotDuration: slotDuration,
+          l1PublishingTime: ethereumSlotDuration,
+          blockDurationMs: 8000,
+          enforce: true,
+        }),
+      );
+
+      // Mock timetable to allow multiple blocks
+      jest
+        .spyOn(job.getTimetable(), 'canStartNextBlock')
+        .mockReturnValueOnce({ canStart: true, deadline: 4, isLastBlock: false })
+        .mockReturnValueOnce({ canStart: true, deadline: 8, isLastBlock: false })
+        .mockReturnValueOnce({ canStart: true, deadline: 12, isLastBlock: false })
+        .mockReturnValue({ canStart: false, deadline: undefined, isLastBlock: false });
+
+      // Mock to throw on first block proposal
+      validatorClient.createBlockProposal.mockImplementation(() => {
+        throw new SlashingProtectionError(SlotNumber(1), DutyType.BLOCK_PROPOSAL, 0, 'hash1', 'hash2', 'node-1');
+      });
+
+      const result = await job.execute();
+
+      // Should return undefined and stop building
+      expect(result).toBeUndefined();
+      // Should have attempted only 1 block proposal (first one threw)
+      expect(validatorClient.createBlockProposal).toHaveBeenCalledTimes(1);
+      // Should not have attempted checkpoint proposal
+      expect(validatorClient.createCheckpointProposal).not.toHaveBeenCalled();
+      // Should not publish anything
+      expect(publisher.enqueueProposeCheckpoint).not.toHaveBeenCalled();
+    });
+  });
 });
 
 class TestCheckpointProposalJob extends CheckpointProposalJob {