feat: ValidatorKeystore implementation for high-availability signer

Fixes [A-354](https://linear.app/aztec-labs/issue/A-354/integrate-slash-protection-in-signing-path)

Implements `ExtendedValidatorKeyStore` utilizing `ValidatorHASigner`.
Updated interfaces to return `null` when the signature isn't acquired & updated handling of signing for `null` results.

Also some cleanup on previous HA work like consolidating config

Author: spypsy

yarn-project/aztec-node/package.json

@@ -88,6 +88,7 @@
     "@aztec/stdlib": "workspace:^",
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/validator-client": "workspace:^",
+    "@aztec/validator-ha-signer": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "koa": "^2.16.1",
     "koa-router": "^13.1.1",

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -345,7 +345,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     const watchers: Watcher[] = [];
 
     // Create validator client if required
-    const validatorClient = createValidatorClient(config, {
+    const validatorClient = await createValidatorClient(config, {
       p2pClient,
       telemetry,
       dateProvider,

yarn-project/aztec-node/tsconfig.json

@@ -72,6 +72,9 @@
     {
       "path": "../validator-client"
     },
+    {
+      "path": "../validator-ha-signer"
+    },
     {
       "path": "../world-state"
     }

yarn-project/end-to-end/src/e2e_p2p/reex.test.ts

@@ -143,10 +143,10 @@ describe('e2e_p2p_reex', () => {
           .keyStore as ValidatorKeyStore;
         const newProposal = new BlockProposal(
           proposal.payload,
-          await signer.signMessageWithAddress(
+          (await signer.signMessageWithAddress(
             proposerAddress!,
             getHashedSignaturePayload(proposal.payload, SignatureDomainSeparator.blockProposal),
-          ),
+          ))!,
           proposal.txHashes,
         );
 

yarn-project/foundation/src/config/env_var.ts

@@ -313,4 +313,14 @@ export type EnvVar =
   | 'FISHERMAN_MODE'
   | 'MAX_ALLOWED_ETH_CLIENT_DRIFT_SECONDS'
   | 'LEGACY_BLS_CLI'
-  | 'DEBUG_FORCE_TX_PROOF_VERIFICATION';
+  | 'DEBUG_FORCE_TX_PROOF_VERIFICATION'
+  | 'VALIDATOR_HA_ENABLED'
+  | 'VALIDATOR_HA_NODE_ID'
+  | 'VALIDATOR_HA_POLLING_INTERVAL_MS'
+  | 'VALIDATOR_HA_SIGNING_TIMEOUT_MS'
+  | 'VALIDATOR_HA_DB_URL'
+  | 'VALIDATOR_HA_RUN_MIGRATIONS'
+  | 'VALIDATOR_HA_POOL_MAX'
+  | 'VALIDATOR_HA_POOL_MIN'
+  | 'VALIDATOR_HA_POOL_IDLE_TIMEOUT_MS'
+  | 'VALIDATOR_HA_POOL_CONNECTION_TIMEOUT_MS';

yarn-project/package.json

@@ -61,6 +61,7 @@
     "txe",
     "test-wallet",
     "validator-client",
+    "validator-ha-signer",
     "wallet-sdk",
     "world-state"
   ],

yarn-project/sequencer-client/package.json

@@ -49,6 +49,7 @@
     "@aztec/stdlib": "workspace:^",
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/validator-client": "workspace:^",
+    "@aztec/validator-ha-signer": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "lodash.chunk": "^4.2.0",
     "tslib": "^2.4.0",

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts

@@ -33,6 +33,7 @@ import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
 import { type FailedTx, Tx } from '@aztec/stdlib/tx';
 import { AttestationTimeoutError } from '@aztec/stdlib/validators';
 import type { ValidatorClient } from '@aztec/validator-client';
+import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validator-ha-signer/errors';
 
 import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
 import type { InvalidateBlockRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
@@ -207,21 +208,62 @@ export class CheckpointProposalJob {
       }
 
       // TODO(palla/mbps): Wire this to the new p2p API once available, including the pendingBroadcast.block
-      const proposal = await this.validatorClient.createCheckpointProposal(
-        checkpoint.header,
-        checkpoint.archive.root,
-        pendingBroadcast?.txs ?? [],
-        this.proposer,
-        blockProposalOptions,
-      );
+      let proposal: BlockProposal;
+      try {
+        proposal = await this.validatorClient.createCheckpointProposal(
+          checkpoint.header,
+          checkpoint.archive.root,
+          pendingBroadcast?.txs ?? [],
+          this.proposer,
+          blockProposalOptions,
+        );
+      } catch (err) {
+        if (err instanceof DutyAlreadySignedError) {
+          this.log.info(`Checkpoint proposal for slot ${this.slot} already signed by another HA node, yielding`, {
+            slot: this.slot,
+            signedByNode: err.signedByNode,
+          });
+          return undefined;
+        }
+        if (err instanceof SlashingProtectionError) {
+          this.log.warn(`Checkpoint proposal for slot ${this.slot} blocked by slashing protection`, {
+            slot: this.slot,
+            existingSigningRoot: err.existingSigningRoot,
+            attemptedSigningRoot: err.attemptedSigningRoot,
+          });
+          return undefined;
+        }
+        throw err;
+      }
+
       await this.p2pClient.broadcastProposal(proposal);
 
       this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.slot);
       const attestations = await this.waitForAttestations(proposal);
 
       // Proposer must sign over the attestations before pushing them to L1
       const signer = this.proposer ?? this.publisher.getSenderAddress();
-      const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer);
+      let attestationsSignature: Signature;
+      try {
+        attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer);
+      } catch (err) {
+        if (err instanceof DutyAlreadySignedError) {
+          this.log.info(`Attestations signature for slot ${this.slot} already signed by another HA node, yielding`, {
+            slot: this.slot,
+            signedByNode: err.signedByNode,
+          });
+          return undefined;
+        }
+        if (err instanceof SlashingProtectionError) {
+          this.log.warn(`Attestations signature for slot ${this.slot} blocked by slashing protection`, {
+            slot: this.slot,
+            existingSigningRoot: err.existingSigningRoot,
+            attemptedSigningRoot: err.attemptedSigningRoot,
+          });
+          return undefined;
+        }
+        throw err;
+      }
 
       // Enqueue publishing the checkpoint to L1
       this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.slot);
@@ -331,15 +373,38 @@ export class CheckpointProposalJob {
       // If the block is the last one, we'll broadcast it along with the checkpoint at the end of the loop
       if (!this.config.fishermanMode) {
         // TODO(palla/mbps): Wire this to the new p2p API once available
-        const proposal = await this.validatorClient.createBlockProposal(
-          block.header.globalVariables.blockNumber,
-          (await checkpointBuilder.getCheckpoint()).header,
-          block.archive.root,
-          usedTxs,
-          this.proposer,
-          blockProposalOptions,
-        );
-        await this.p2pClient.broadcastProposal(proposal);
+        try {
+          const proposal = await this.validatorClient.createBlockProposal(
+            block.header.globalVariables.blockNumber,
+            (await checkpointBuilder.getCheckpoint()).header,
+            block.archive.root,
+            usedTxs,
+            this.proposer,
+            blockProposalOptions,
+          );
+          await this.p2pClient.broadcastProposal(proposal);
+        } catch (err) {
+          if (err instanceof DutyAlreadySignedError) {
+            this.log.info(`Block proposal for slot ${this.slot} already signed by another HA node, yielding`, {
+              slot: this.slot,
+              blockNumber,
+              signedByNode: err.signedByNode,
+            });
+            // Another HA node is handling this slot, stop building
+            break;
+          }
+          if (err instanceof SlashingProtectionError) {
+            this.log.warn(`Block proposal for slot ${this.slot} blocked by slashing protection`, {
+              slot: this.slot,
+              blockNumber,
+              existingSigningRoot: err.existingSigningRoot,
+              attemptedSigningRoot: err.attemptedSigningRoot,
+            });
+            // Stop building to avoid further slashing issues
+            break;
+          }
+          throw err;
+        }
       }
 
       // Wait until the next block's start time

yarn-project/sequencer-client/tsconfig.json

@@ -69,6 +69,9 @@
     {
       "path": "../validator-client"
     },
+    {
+      "path": "../validator-ha-signer"
+    },
     {
       "path": "../world-state"
     },

yarn-project/stdlib/package.json

@@ -84,6 +84,7 @@
     "@aztec/foundation": "workspace:^",
     "@aztec/l1-artifacts": "workspace:^",
     "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi",
+    "@aztec/validator-ha-signer": "workspace:^",
     "@google-cloud/storage": "^7.15.0",
     "axios": "^1.12.0",
     "json-stringify-deterministic": "1.0.12",