Reload improvements (#1135)

Various Reload endpoint enhancements. /reload now serves GET requests to provide status of current and last reload operation. The POST request only service one reload operation at a time.

Fixed potential memory leaks by correctly utilizing ExecutorService. Moved SignerLoader processing of "config files to keystore files to ArtifactSigners" to virtual threads with batch handling that ease up CPU and memory requirements.

Introduced volatile immutable maps with write-on-copy semantics for read-heavy ArtifactSigners in DefaultArtifactSigner and SignerLoader.

Author: usmansaleem

CHANGELOG.md

@@ -2,12 +2,39 @@
 
 ## Upcoming Releases
 ### Breaking Changes
-- `--swagger-ui-enabled` cli option has been removed. The Specs UI can be accessed at https://consensys.github.io/web3signer/.
+#### `/reload` Endpoint Response Format Changed
+- Now returns HTTP `202 Accepted` (previously `200 OK`) with JSON response body
+- Returns `409 Conflict` with error message if reload already in progress
+- **Migration:** Update automation to expect `202` status code instead of `200`
+
+#### Removed Swagger UI CLI Option
+- `--swagger-ui-enabled` option removed
+- Access OpenAPI specs at https://consensys.github.io/web3signer/
+
+### Features Added
+- **Enhanced `/reload` endpoint with status monitoring:**
+  - New `GET /reload` endpoint to check reload operation status
+  - Reports detailed status: `idle`, `running`, `completed`, `completed_with_errors`, `failed`
+  - Exposes error counts when individual signer configurations fail to load
+  - Distinguishes between complete success and partial success (some signers failed)
+  - Provides timestamps and error messages for last reload operation
+- Virtual thread-based signer loading for improved performance
+- New file system signer loading configuration:
+  - `--signer-load-timeout` (default: 60 sec) - Timeout per file during parallel processing
+  - `--signer-load-batch-size` (default: 500) - Files processed per batch in parallel mode
+  - `--signer-load-sequential-threshold` (default: 100) - Minimum files to trigger parallel processing
+  - `--signer-load-parallel` (default: true) - Enable/disable parallel processing
+- New `/reload` endpoint configuration:
+  - `--reload-timeout` (default: 30 min) - Maximum time for entire reload operation
+- Improved reload concurrency control prevents multiple simultaneous reloads
 
-### Features Added
+### Bugs Fixed
+- Fix memory leak during reload API endpoint. Issue [#1073][issue_1073] via PR [#1135][PR_1135].
+- Fix race condition in reload flag management when executor initialization fails synchronously
 
+[issue_1073]: https://github.com/Consensys/web3signer/issues/1073
+[PR_1135]: https://github.com/Consensys/web3signer/pull/1135
 
-### Bugs Fixed
 
 ---
 ## 25.11.0

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/Signer.java

@@ -264,6 +264,10 @@ public Response callReload() {
     return given().baseUri(getUrl()).post(RELOAD_ENDPOINT);
   }
 
+  public Response getReloadStatus() {
+    return given().baseUri(getUrl()).get(RELOAD_ENDPOINT);
+  }
+
   public Response healthcheck() {
     return given().baseUri(getUrl()).get(HEALTHCHECK_ENDPOINT);
   }

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/commitboost/CommitBoostGetPubKeysAcceptanceTest.java

@@ -13,7 +13,10 @@
 package tech.pegasys.web3signer.tests.commitboost;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
 
 import tech.pegasys.teku.bls.BLSKeyPair;
 import tech.pegasys.web3signer.KeystoreUtil;
@@ -27,6 +30,8 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -35,6 +40,7 @@
 import io.restassured.http.ContentType;
 import io.restassured.response.Response;
 import org.apache.commons.lang3.tuple.Pair;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@@ -126,6 +132,41 @@ void listCommitBoostPublicKeys() {
     }
   }
 
+  @Test
+  void reloadReturnsEmptyProxyKeysAfterPhysicalKeystoreDeletion() throws Exception {
+    // call commit boost get pub keys, the proxy_bls and proxy_ecdsa should not be empty
+    final Response initResponse = signer.callCommitBoostGetPubKeys();
+    initResponse
+        .then()
+        .log()
+        .body()
+        .statusCode(200)
+        .contentType(ContentType.JSON)
+        .body("keys", hasSize(2))
+        .body("keys.proxy_bls", everyItem(not(empty())))
+        .body("keys.proxy_ecdsa", everyItem(not(empty())));
+
+    // delete everything under commitBoostKeystoresPath and /reload
+    deleteDirectoryContents(commitBoostKeystoresPath);
+    signer.callReload().then().statusCode(202);
+
+    // Wait until proxy keys are empty (reload completed)
+    Awaitility.await()
+        .atMost(Duration.ofSeconds(5))
+        .pollInterval(Duration.ofMillis(100)) // Check every 100ms
+        .untilAsserted(
+            () -> {
+              final Response response = signer.callCommitBoostGetPubKeys();
+              response
+                  .then()
+                  .statusCode(200)
+                  .contentType(ContentType.JSON)
+                  .body("keys", hasSize(2))
+                  .body("keys.proxy_bls", everyItem(empty()))
+                  .body("keys.proxy_ecdsa", everyItem(empty()));
+            });
+  }
+
   private List<String> getProxyECPubKeys(final String consensusKeyHex) {
     // return compressed secp256k1 public keys in hex format
     return proxySECPKeysMap.get(consensusKeyHex).stream()
@@ -239,4 +280,22 @@ static List<ECKeyPair> randomECKeyPairs(final int count) {
   static List<BLSKeyPair> randomBLSKeyPairs(final int count) {
     return Stream.generate(() -> BLSKeyPair.random(SECURE_RANDOM)).limit(count).toList();
   }
+
+  private void deleteDirectoryContents(Path directory) throws IOException {
+    if (Files.exists(directory)) {
+      try (Stream<Path> paths = Files.walk(directory)) {
+        paths
+            .filter(path -> !path.equals(directory)) // Keep the directory itself
+            .sorted(Comparator.reverseOrder()) // Delete children before parents
+            .forEach(
+                path -> {
+                  try {
+                    Files.delete(path);
+                  } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                  }
+                });
+      }
+    }
+  }
 }

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/eth1rpc/Eth1RpcReloadKeysTest.java

@@ -71,7 +71,7 @@ public void additionalPublicKeyAreReportedAfterReloadUsingEth1RPC() throws IOExc
     assertThat(accounts).isNotEmpty();
 
     final String[] additionalKeys = createSecpKeys(prvKeys[1]);
-    signer.callReload().then().statusCode(200);
+    signer.callReload().then().statusCode(202);
 
     // reload is async ...
     Awaitility.await()

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/keymanager/ListKeysAcceptanceTest.java

@@ -57,7 +57,7 @@ public void additionalPublicKeyAreReportedAfterReload() throws URISyntaxExceptio
     validateApiResponse(callListKeys(), "data.validating_pubkey", hasItem(firstPubKey));
 
     final String secondPubKey = createKeystoreYamlFile(BLS_PRIVATE_KEY_2);
-    signer.callReload().then().statusCode(200);
+    signer.callReload().then().statusCode(202);
 
     // reload is async
     Awaitility.await()

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/publickeys/KeyIdentifiersAcceptanceTest.java

@@ -107,7 +107,7 @@ public void additionalPublicKeyAreReportedAfterReload(final KeyType keyType) {
     validateApiResponse(response, contains(keys));
 
     final String[] additionalKeys = createKeys(keyType, true, prvKeys[1]);
-    signer.callReload().then().statusCode(200);
+    signer.callReload().then().statusCode(202);
 
     // reload is async ...
     Awaitility.await()
@@ -131,7 +131,7 @@ public void healthCheckReportsKeysLoadedAfterReloadInEth2Mode() {
     assertThat(getHealthcheckStatusValue(jsonBody)).isEqualTo("UP");
 
     final String[] additionalKeys = createKeys(keyType, true, prvKeys[1]);
-    signer.callReload().then().statusCode(200);
+    signer.callReload().then().statusCode(202);
 
     // reload is async ...
     Awaitility.await()
@@ -160,7 +160,7 @@ public void publicKeysAreRemovedAfterReloadDefault(final KeyType keyType) {
     assertThat(testDirectory.resolve(keys[1] + ".yaml").toFile().delete()).isTrue();
 
     // reload API call
-    signer.callReload().then().statusCode(200);
+    signer.callReload().then().statusCode(202);
 
     // reload is async ... assert that the key is removed
     Awaitility.await()