feat: high-availability validator signer

aztec-up/bin/aztec

@@ -562,6 +562,12 @@ EOF
     exec $(dirname $0)/.aztec-run aztec \
       node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js "$@"
     ;;
+  migrate-ha-db)
+    # DB migration command, inject the database URL
+    export ENV_VARS_TO_INJECT="DATABASE_URL"
+    exec $(dirname $0)/.aztec-run aztec \
+      node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js "$@"
+    ;;
   *)
     export ENV_VARS_TO_INJECT="SECRET_KEY"
     exec $(dirname $0)/.aztec-run aztec \

yarn-project/aztec/package.json

@@ -59,6 +59,7 @@
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/test-wallet": "workspace:^",
     "@aztec/txe": "workspace:^",
+    "@aztec/validator-ha-signer": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "@types/chalk": "^2.2.0",
     "abitype": "^0.8.11",

yarn-project/aztec/src/bin/index.ts

@@ -14,6 +14,7 @@ import { createConsoleLogger, createLogger } from '@aztec/foundation/log';
 
 import { Command } from 'commander';
 
+import { injectMigrateCommand } from '../cli/cmds/migrate_ha_db.js';
 import { injectAztecCommands } from '../cli/index.js';
 import { getCliVersion } from '../cli/release_version.js';
 
@@ -55,6 +56,7 @@ async function main() {
   program = injectAztecNodeCommands(program, userLog, debugLogger);
   program = injectMiscCommands(program, userLog);
   program = injectValidatorKeysCommands(program, userLog);
+  program = injectMigrateCommand(program, userLog);
 
   await program.parseAsync(process.argv);
 }

yarn-project/aztec/src/cli/cmds/migrate_ha_db.ts

@@ -0,0 +1,43 @@
+import { runMigrations } from '@aztec/validator-ha-signer/migrations';
+
+import type { Command } from 'commander';
+
+export function injectMigrateCommand(program: Command, log: (msg: string) => void): Command {
+  const migrateCommand = program.command('migrate-ha-db').description('Run validator-ha-signer database migrations');
+
+  migrateCommand
+    .command('up')
+    .description('Apply pending migrations')
+    .requiredOption('--database-url <string>', 'PostgreSQL connection string', process.env.DATABASE_URL)
+    .option('--verbose', 'Enable verbose output', false)
+    .action(async options => {
+      const migrations = await runMigrations(options.databaseUrl, {
+        direction: 'up',
+        verbose: options.verbose,
+      });
+      if (migrations.length > 0) {
+        log(`Applied migrations: ${migrations.join(', ')}`);
+      } else {
+        log('No migrations to apply - schema is up to date');
+      }
+    });
+
+  migrateCommand
+    .command('down')
+    .description('Rollback the last migration')
+    .requiredOption('--database-url <string>', 'PostgreSQL connection string', process.env.DATABASE_URL)
+    .option('--verbose', 'Enable verbose output', false)
+    .action(async options => {
+      const migrations = await runMigrations(options.databaseUrl, {
+        direction: 'down',
+        verbose: options.verbose,
+      });
+      if (migrations.length > 0) {
+        log(`Rolled back migrations: ${migrations.join(', ')}`);
+      } else {
+        log('No migrations to rollback');
+      }
+    });
+
+  return program;
+}

yarn-project/aztec/tsconfig.json

@@ -96,6 +96,9 @@
     {
       "path": "../txe"
     },
+    {
+      "path": "../validator-ha-signer"
+    },
     {
       "path": "../world-state"
     }

yarn-project/foundation/src/config/env_var.ts

@@ -314,4 +314,15 @@ export type EnvVar =
   | 'FISHERMAN_MODE'
   | 'MAX_ALLOWED_ETH_CLIENT_DRIFT_SECONDS'
   | 'LEGACY_BLS_CLI'
-  | 'DEBUG_FORCE_TX_PROOF_VERIFICATION';
+  | 'DEBUG_FORCE_TX_PROOF_VERIFICATION'
+  | 'SLASHING_PROTECTION_NODE_ID'
+  | 'SLASHING_PROTECTION_POLLING_INTERVAL_MS'
+  | 'SLASHING_PROTECTION_SIGNING_TIMEOUT_MS'
+  | 'SLASHING_PROTECTION_ENABLED'
+  | 'SLASHING_PROTECTION_MAX_STUCK_DUTIES_AGE_MS'
+  | 'VALIDATOR_HA_DATABASE_URL'
+  | 'VALIDATOR_HA_RUN_MIGRATIONS'
+  | 'VALIDATOR_HA_POOL_MAX'
+  | 'VALIDATOR_HA_POOL_MIN'
+  | 'VALIDATOR_HA_POOL_IDLE_TIMEOUT_MS'
+  | 'VALIDATOR_HA_POOL_CONNECTION_TIMEOUT_MS';

yarn-project/package.json

@@ -61,6 +61,7 @@
     "txe",
     "test-wallet",
     "validator-client",
+    "validator-ha-signer",
     "wallet-sdk",
     "world-state"
   ],

yarn-project/tsconfig.json

@@ -29,6 +29,7 @@
     { "path": "aztec.js/tsconfig.json" },
     { "path": "aztec-node/tsconfig.json" },
     { "path": "validator-client/tsconfig.json" },
+    { "path": "validator-ha-signer/tsconfig.json" },
     { "path": "bb-prover/tsconfig.json" },
     { "path": "bot/tsconfig.json" },
     { "path": "constants/tsconfig.json" },

yarn-project/validator-ha-signer/MIGRATIONS.md

@@ -0,0 +1,186 @@
+# Database Migrations Guide
+
+This package uses [node-pg-migrate](https://github.com/salsita/node-pg-migrate) for managing database schema changes.
+
+## Quick Reference
+
+```bash
+# Run pending migrations
+aztec migrate-ha-db up --database-url postgresql://...
+
+# Rollback last migration
+aztec migrate-ha-db down --database-url postgresql://...
+```
+
+## Migration Files
+
+Migrations are located in the `migrations/` directory and are named with timestamps:
+
+```
+migrations/
+  └── 1_initial-schema.ts
+```
+
+## Creating New Migrations
+
+When you need to modify the database schema:
+
+```bash
+# Generate a new migration file
+npx node-pg-migrate create add-new-field
+
+# This creates: migrations/[timestamp]_add-new-field.ts
+```
+
+Edit the generated file:
+
+```typescript
+import type { MigrationBuilder } from 'node-pg-migrate';
+
+export async function up(pgm: MigrationBuilder): Promise<void> {
+  // Add your schema changes here
+  pgm.addColumn('validator_duties', {
+    new_field: { type: 'text', notNull: false },
+  });
+}
+
+export async function down(pgm: MigrationBuilder): Promise<void> {
+  // Reverse the changes
+  pgm.dropColumn('validator_duties', 'new_field');
+}
+```
+
+## Production Deployment
+
+### Option 1: Kubernetes Init Container
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: validator
+spec:
+  template:
+    spec:
+      initContainers:
+        - name: db-migrate
+          image: aztecprotocol/aztec:<image_tag>
+          command: ['node', '--no-warnings', '/usr/src/yarn-project/aztec/dest/bin/index.js', 'migrate-ha-db', 'up']
+          env:
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: db-credentials
+                  key: connection-string
+      containers:
+        - name: validator
+          image: aztecprotocol/aztec:<image_tag>
+          # ... validator config
+```
+
+### Option 2: Separate Migration Job
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: validator-migrate-v1
+spec:
+  template:
+    spec:
+      containers:
+        - name: migrate
+          image: aztecprotocol/aztec:<image_tag>
+          command: ['node', '--no-warnings', '/usr/src/yarn-project/aztec/dest/bin/index.js', 'migrate-ha-db', 'up']
+          env:
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: db-credentials
+                  key: connection-string
+      restartPolicy: Never
+```
+
+### Option 3: CI/CD Pipeline
+
+```yaml
+# GitHub Actions example
+- name: Run Database Migrations
+  run: |
+    docker run --rm \
+      -e DATABASE_URL=${{ secrets.DATABASE_URL }} \
+      aztecprotocol/aztec:<image_tag> \
+      migrate-ha-db up
+```
+
+## High Availability Considerations
+
+The migrations use idempotent SQL operations (`IF NOT EXISTS`, `ON CONFLICT`, etc.), making them safe to run concurrently from multiple nodes. However, for cleaner logs and faster deployments, we recommend:
+
+1. **Run migrations once** from an init container or migration job
+2. **Then start** multiple validator nodes
+
+If multiple nodes run migrations simultaneously, they will all succeed, but you'll see redundant log output.
+
+## Development Workflow
+
+```bash
+# 1. Create migration
+npx node-pg-migrate create my-feature
+
+# 2. Edit migrations/[timestamp]_my-feature.ts
+
+# 3. Test migration locally
+aztec migrate-ha-db up --database-url postgresql://localhost:5432/validator_dev
+
+# 4. Test rollback
+aztec migrate-ha-db down --database-url postgresql://localhost:5432/validator_dev
+
+# 5. Re-apply
+aztec migrate-ha-db up --database-url postgresql://localhost:5432/validator_dev
+
+# 6. Run tests
+yarn test
+```
+
+## Troubleshooting
+
+### Migration Failed Midway
+
+If a migration fails partway through:
+
+```bash
+# The failed migration will be marked as running
+# Fix the issue and re-run
+aztec migrate-ha-db up --database-url postgresql://...
+```
+
+### Reset Development Database
+
+```bash
+# Drop all migrations
+while aztec migrate-ha-db down --database-url postgresql://localhost:5432/validator_dev; do :; done
+
+# Or drop the database entirely
+psql -c "DROP DATABASE validator_dev;"
+psql -c "CREATE DATABASE validator_dev;"
+
+# Re-run migrations
+aztec migrate-ha-db up --database-url postgresql://localhost:5432/validator_dev
+```
+
+### Check Applied Migrations
+
+```bash
+# Query the migrations table
+psql $DATABASE_URL -c "SELECT * FROM pgmigrations ORDER BY id;"
+```
+
+## Migration Best Practices
+
+1. **Always provide `down()` migrations** for rollback capability
+2. **Test migrations on a copy of production data** before deploying
+3. **Make migrations backward compatible** when possible
+4. **Avoid data migrations in schema migrations** - use separate data migration scripts
+5. **Keep migrations small and focused** - one logical change per migration
+6. **Never modify committed migrations** - create a new migration instead