chore: (A-475) add load balancer flag (#19888)

- load balancers for eth-execution and rpc to red uce port flakiness
- add new environment variable to enable load balancers for
eth-execution and rpc
- introduce getServiceEndpoint boilerplate wrapper to get ip (answers:
is it external or port-forwarded?) and update tests to use new function

Author: PhilWindle

spartan/environments/next-scenario.env

@@ -11,6 +11,7 @@ FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf
 REDEPLOY_ROLLUP_CONTRACTS=true
 DESTROY_AZTEC_INFRA=true
 VERIFY_CONTRACTS=false
+USE_LOAD_BALANCERS=true
 
 AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=2
 AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=2

spartan/scripts/deploy_network.sh

@@ -275,6 +275,7 @@ BLOCK_TIME = ${ETHEREUM_BLOCK_TIME}
 GAS_LIMIT = ${ETHEREUM_GAS_LIMIT}
 PREFUNDED_MNEMONIC_INDICES = "${LABS_INFRA_INDICES}"
 RESOURCE_PROFILE = "${RESOURCE_PROFILE}"
+USE_LOAD_BALANCERS = ${USE_LOAD_BALANCERS:-false}
 EOF
 
   "${SCRIPT_DIR}/override_terraform_backend.sh" "${DEPLOY_ETH_DEVNET_DIR}" "${CLUSTER}" "${BASE_STATE_PATH}/deploy-eth-devnet"

spartan/terraform/deploy-eth-devnet/main.tf

@@ -71,6 +71,12 @@ resource "null_resource" "generate_genesis" {
   }
 }
 
+# Determine if this is a kind cluster (local) based on context name
+locals {
+  is_kind = strcontains(var.K8S_CLUSTER_CONTEXT, "kind")
+  use_lb = var.USE_LOAD_BALANCERS && !local.is_kind
+}
+
 # Deploy eth-devnet helm chart
 resource "helm_release" "eth_devnet" {
   depends_on       = [null_resource.generate_genesis]
@@ -97,6 +103,16 @@ resource "helm_release" "eth_devnet" {
     value = var.RELEASE_PREFIX
   }
 
+  set {
+    name  = "ethereum.execution.service.type"
+    value = local.use_lb ? "LoadBalancer" : "ClusterIP"
+  }
+
+  set {
+    name  = "ethereum.beacon.service.type"
+    value = local.use_lb ? "LoadBalancer" : "ClusterIP"
+  }
+
   timeout       = 1200
   wait          = true
   wait_for_jobs = false

spartan/terraform/deploy-eth-devnet/variables.tf

@@ -76,3 +76,9 @@ variable "RESOURCE_PROFILE" {
   }
 }
 
+variable "USE_LOAD_BALANCERS" {
+  description = "If true, use LoadBalancer service type for external access"
+  type        = bool
+  default     = false
+}
+

yarn-project/end-to-end/src/spartan/1tps.test.ts

@@ -8,15 +8,14 @@ import type { ProvenTx, TestWallet } from '@aztec/test-wallet/server';
 import { proveInteraction } from '@aztec/test-wallet/server';
 
 import { jest } from '@jest/globals';
-import type { ChildProcess } from 'child_process';
 
 import { getSponsoredFPCAddress } from '../fixtures/utils.js';
 import {
   type TestAccounts,
   createWalletAndAztecNodeClient,
   deploySponsoredTestAccountsWithTokens,
 } from './setup_test_wallets.js';
-import { setupEnvironment, startPortForwardForRPC } from './utils.js';
+import { type ServiceEndpoint, getRPCEndpoint, setupEnvironment } from './utils.js';
 
 const config = { ...setupEnvironment(process.env) };
 describe('token transfer test', () => {
@@ -29,19 +28,19 @@ describe('token transfer test', () => {
 
   let testAccounts: TestAccounts;
   let wallet: TestWallet;
-  const forwardProcesses: ChildProcess[] = [];
+  const endpoints: ServiceEndpoint[] = [];
   let cleanup: undefined | (() => Promise<void>);
 
   afterAll(async () => {
     await cleanup?.();
-    forwardProcesses.forEach(p => p.kill());
+    endpoints.forEach(e => e.process?.kill());
   });
 
   beforeAll(async () => {
-    logger.info('Starting port forward for PXE');
-    const { process: aztecRpcProcess, port: aztecRpcPort } = await startPortForwardForRPC(config.NAMESPACE);
-    forwardProcesses.push(aztecRpcProcess);
-    const rpcUrl = `http://127.0.0.1:${aztecRpcPort}`;
+    logger.info('Connecting to RPC node');
+    const rpcEndpoint = await getRPCEndpoint(config.NAMESPACE);
+    endpoints.push(rpcEndpoint);
+    const rpcUrl = rpcEndpoint.url;
 
     const {
       wallet: _wallet,

yarn-project/end-to-end/src/spartan/4epochs.test.ts

@@ -9,15 +9,14 @@ import { DateProvider } from '@aztec/foundation/timer';
 import { TestWallet, proveInteraction } from '@aztec/test-wallet/server';
 
 import { jest } from '@jest/globals';
-import type { ChildProcess } from 'child_process';
 
 import { getSponsoredFPCAddress } from '../fixtures/utils.js';
 import {
   type TestAccounts,
   createWalletAndAztecNodeClient,
   deploySponsoredTestAccountsWithTokens,
 } from './setup_test_wallets.js';
-import { ChainHealth, setupEnvironment, startPortForwardForEthereum, startPortForwardForRPC } from './utils.js';
+import { ChainHealth, type ServiceEndpoint, getEthereumEndpoint, getRPCEndpoint, setupEnvironment } from './utils.js';
 
 const config = { ...setupEnvironment(process.env) };
 
@@ -35,7 +34,7 @@ describe('token transfer test', () => {
 
   let testAccounts: TestAccounts;
   let ETHEREUM_HOSTS: string[];
-  const forwardProcesses: ChildProcess[] = [];
+  const endpoints: ServiceEndpoint[] = [];
   let wallet: TestWallet;
   let aztecNode: AztecNode;
   let cleanup: undefined | (() => Promise<void>);
@@ -44,19 +43,18 @@ describe('token transfer test', () => {
   afterAll(async () => {
     await health.teardown();
     await cleanup?.();
-    forwardProcesses.forEach(p => p.kill());
+    endpoints.forEach(e => e.process?.kill());
   });
 
   beforeAll(async () => {
     await health.setup();
-    logger.info('Starting port forward for PXE and Ethereum');
-    const { process: aztecRpcProcess, port: aztecRpcPort } = await startPortForwardForRPC(config.NAMESPACE);
-    const { process: ethereumProcess, port: ethereumPort } = await startPortForwardForEthereum(config.NAMESPACE);
-    forwardProcesses.push(aztecRpcProcess);
-    forwardProcesses.push(ethereumProcess);
-
-    const rpcUrl = `http://127.0.0.1:${aztecRpcPort}`;
-    ETHEREUM_HOSTS = [`http://127.0.0.1:${ethereumPort}`];
+    logger.info('Connecting to RPC and Ethereum nodes');
+    const rpcEndpoint = await getRPCEndpoint(config.NAMESPACE);
+    const ethEndpoint = await getEthereumEndpoint(config.NAMESPACE);
+    endpoints.push(rpcEndpoint, ethEndpoint);
+
+    const rpcUrl = rpcEndpoint.url;
+    ETHEREUM_HOSTS = [ethEndpoint.url];
 
     ({ wallet, aztecNode, cleanup } = await createWalletAndAztecNodeClient(rpcUrl, config.REAL_VERIFIER, logger));
 

yarn-project/end-to-end/src/spartan/gating-passive.test.ts

@@ -6,23 +6,23 @@ import { createLogger } from '@aztec/foundation/log';
 import { DateProvider } from '@aztec/foundation/timer';
 
 import { expect, jest } from '@jest/globals';
-import type { ChildProcess } from 'child_process';
 
 import { type AlertConfig, GrafanaClient } from '../quality_of_service/grafana_client.js';
 import {
   ChainHealth,
+  type ServiceEndpoint,
   applyBootNodeFailure,
   applyNetworkShaping,
   applyValidatorKill,
   awaitCheckpointNumber,
   deleteResourceByLabel,
+  getEthereumEndpoint,
   getGitProjectRoot,
+  getRPCEndpoint,
   installTransferBot,
   restartBot,
   setupEnvironment,
   startPortForward,
-  startPortForwardForEthereum,
-  startPortForwardForRPC,
   uninstallTransferBot,
 } from './utils.js';
 
@@ -54,7 +54,7 @@ describe('a test that passively observes the network in the presence of network
   let ETHEREUM_HOST: string;
   let alertChecker: GrafanaClient;
   let spartanDir: string;
-  const forwardProcesses: ChildProcess[] = [];
+  const endpoints: ServiceEndpoint[] = [];
   const podChaosInstances: string[] = [];
   const networkShapingInstance = `${NAMESPACE}-network-shaping`;
   const health = new ChainHealth(NAMESPACE, debugLogger);
@@ -63,7 +63,7 @@ describe('a test that passively observes the network in the presence of network
     await health.setup();
     // Try Prometheus in a dedicated metrics namespace first; if not present, fall back to the network namespace
     let promPort = 0;
-    let promProc: ChildProcess | undefined;
+    let promProc: ServiceEndpoint['process'];
     {
       const { process: p, port } = await startPortForward({
         resource: `svc/metrics-prometheus-server`,
@@ -89,7 +89,7 @@ describe('a test that passively observes the network in the presence of network
     }
 
     if (promProc && promPort !== 0) {
-      forwardProcesses.push(promProc);
+      endpoints.push({ url: `http://127.0.0.1:${promPort}`, process: promProc });
       const grafanaEndpoint = `http://127.0.0.1:${promPort}/api/v1`;
       const grafanaCredentials = '';
       alertChecker = new GrafanaClient(debugLogger, { grafanaEndpoint, grafanaCredentials });
@@ -126,17 +126,17 @@ describe('a test that passively observes the network in the presence of network
 
     // Teardown transfer bot installed for this test
     await uninstallTransferBot(NAMESPACE, debugLogger);
-    forwardProcesses.forEach(p => p.kill());
+    endpoints.forEach(e => e.process?.kill());
   });
 
   it('survives network chaos', async () => {
-    const { process: aztecRpcProcess, port: aztecRpcPort } = await startPortForwardForRPC(NAMESPACE);
-    forwardProcesses.push(aztecRpcProcess);
-    const nodeUrl = `http://127.0.0.1:${aztecRpcPort}`;
+    const rpcEndpoint = await getRPCEndpoint(NAMESPACE);
+    endpoints.push(rpcEndpoint);
+    const nodeUrl = rpcEndpoint.url;
 
-    const { process: ethProcess, port: ethPort } = await startPortForwardForEthereum(NAMESPACE);
-    forwardProcesses.push(ethProcess);
-    ETHEREUM_HOST = `http://127.0.0.1:${ethPort}`;
+    const ethEndpoint = await getEthereumEndpoint(NAMESPACE);
+    endpoints.push(ethEndpoint);
+    ETHEREUM_HOST = ethEndpoint.url;
 
     const node = createAztecNodeClient(nodeUrl);
     const ethCheatCodes = new EthCheatCodesWithState([ETHEREUM_HOST], new DateProvider());

yarn-project/end-to-end/src/spartan/invalidate_blocks.test.ts

@@ -10,10 +10,10 @@ import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import type { AztecNode, AztecNodeAdminConfig } from '@aztec/stdlib/interfaces/client';
 
 import { jest } from '@jest/globals';
-import type { ChildProcess } from 'child_process';
 
 import {
   ChainHealth,
+  type ServiceEndpoint,
   getL1DeploymentAddresses,
   getNodeClient,
   getPublicViemClient,
@@ -34,7 +34,7 @@ describe('invalidate blocks test', () => {
 
   const logger = createLogger(`e2e:invalidate-blocks`);
 
-  const forwardProcesses: ChildProcess[] = [];
+  const endpoints: ServiceEndpoint[] = [];
 
   let client: ViemPublicClient;
   let rollup: RollupContract;
@@ -59,15 +59,17 @@ describe('invalidate blocks test', () => {
   beforeAll(async () => {
     await health.setup();
     const deployAddresses = await getL1DeploymentAddresses(config);
-    ({ client } = await getPublicViemClient(config, forwardProcesses));
+    const viemResult = await getPublicViemClient(config);
+    client = viemResult.client;
+    endpoints.push({ url: viemResult.url, process: viemResult.process });
     rollup = new RollupContract(client, deployAddresses.rollupAddress);
     monitor = new ChainMonitor(rollup, undefined, logger.createChild('chain-monitor'), 500).start();
     const c = await rollup.getRollupConstants();
     constants = { ...c, ethereumSlotDuration: ETHEREUM_SLOT_DURATION } as L1RollupConstants;
 
-    const { node: nodeClient, process } = await getNodeClient(config);
+    const { node: nodeClient, process: nodeProcess, url: nodeUrl } = await getNodeClient(config);
     node = nodeClient;
-    forwardProcesses.push(process);
+    endpoints.push({ url: nodeUrl, process: nodeProcess });
   });
 
   afterAll(async () => {
@@ -83,7 +85,7 @@ describe('invalidate blocks test', () => {
     await waitForSequencersToApplyConfig(restoreConfig, 'restore after invalidate-blocks');
     monitor.removeAllListeners();
     await monitor.stop();
-    forwardProcesses.forEach(p => p.kill());
+    endpoints.forEach(e => e.process?.kill());
   });
 
   /** Waits for a CheckpointProposed event */

yarn-project/end-to-end/src/spartan/mempool_limit.test.ts

@@ -28,7 +28,6 @@ import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import { proveInteraction } from '@aztec/test-wallet/server';
 
 import { jest } from '@jest/globals';
-import type { ChildProcess } from 'child_process';
 
 import { getSponsoredFPCAddress } from '../fixtures/utils.js';
 import {
@@ -37,7 +36,14 @@ import {
   deploySponsoredTestAccounts,
   deploySponsoredTestAccountsWithTokens,
 } from './setup_test_wallets.js';
-import { ChainHealth, getExternalIP, getSequencersConfig, setupEnvironment, updateSequencersConfig } from './utils.js';
+import {
+  ChainHealth,
+  type ServiceEndpoint,
+  getRPCEndpoint,
+  getSequencersConfig,
+  setupEnvironment,
+  updateSequencersConfig,
+} from './utils.js';
 
 const config = setupEnvironment(process.env);
 
@@ -62,13 +68,14 @@ describe('mempool limiter test', () => {
     from: AztecAddress;
   }[] = [];
 
-  const forwardProcesses: ChildProcess[] = [];
+  const endpoints: ServiceEndpoint[] = [];
   const health = new ChainHealth(config.NAMESPACE, debugLogger);
 
   beforeAll(async () => {
     await health.setup();
-    const rpcIP = await getExternalIP(config.NAMESPACE, 'rpc-aztec-node');
-    rpcUrl = `http://${rpcIP}:8080`;
+    const rpcEndpoint = await getRPCEndpoint(config.NAMESPACE);
+    rpcUrl = rpcEndpoint.url;
+    endpoints.push(rpcEndpoint);
     node = createAztecNodeClient(rpcUrl);
     const initialBlock = await node.getBlockNumber().catch(() => 0n);
     debugLogger.info(`Connected to RPC at ${rpcUrl}; initial L2 block: ${initialBlock}`);
@@ -188,7 +195,7 @@ describe('mempool limiter test', () => {
     for (const cleanup of cleanups) {
       await cleanup();
     }
-    forwardProcesses.forEach(p => p.kill());
+    endpoints.forEach(e => e.process?.kill());
   });
 
   it('evicts txs to keep mempool under specified limit', async () => {