Finetune Azure Key Vault acceptance tests (#1130)

* chore: Update Azure Acceptance test
* chore: Update AzureKeyVaultTest to not rely on known key/secret names
 -- The AT assumes that at least two Secrets and Keys are pre-created in Azure Vault
* chore: Simplify AzureKeyVaultSignerTest by not relying on known key names
* chore: Delete Azure multivalue test as existing test already covers it
* chore: Fix Azure AT to fetch Secrets and Keys at runtime
* chore: Readme for Azure acceptance test key vault setup
* try/resource executor service
* Update acceptance-tests/README.md

---------

Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>

Author: usmansaleem

acceptance-tests/README.md

@@ -0,0 +1,116 @@
+## Acceptance Tests
+
+Following instructions are required to setup environment for running acceptance tests for external vault providers:
+
+### Azure Key Vault
+In order to run Azure Key Vault acceptance tests, you need to set the following environment variables:
+```
+export AZURE_CLIENT_ID=<YOUR AZURE CLIENT ID>
+export AZURE_CLIENT_SECRET=<YOUR AZURE CLIENT SECRET>
+export AZURE_KEY_VAULT_NAME=<YOUR AZURE KEY VAULT NAME>
+export AZURE_TENANT_ID=<YOUR AZURE TENANT ID>
+```
+
+### Azure Key Vault BLS Test Keys
+The "Secret" object contains at least two entries for BLS private keys. The first entry is a multi-line secret and the 
+second one is a single line secret. For example, Web3Signer `BLSTestUtil.java` can generate following keys for 
+non-production/test environment which can be imported into Azure Key Vault as secrets:
+
+```
+cat << EOF > bls-test-keys.txt
+0x60b420bb3851d9d47acb933dbe70399bf6c92da33af01d4fb770e98c0325f41d
+0x73d51abbd89cb8196f0efb6892f94d68fccc2c35f0b84609e5f12c55dd85aba8
+0x39722cbbf8b91a4b9045c5e6175f1001eac32f7fcd5eccda5c6e62fc4e638508
+0x4c9326bb9805fa8f85882c12eae724cef0c62e118427f5948aefa5c428c43c93
+0x384a62688ee1d9a01c9d58e303f2b3c9bc1885e8131565386f75f7ae6ca8d147
+0x4b6b5c682f2db7e510e0c00ed67ac896c21b847acadd8df29cf63a77470989d2
+0x13086d684f4b1a1632178a8c5be08a2fb01287c4a78313c41373701eb8e66232
+0x25296867ee96fa5b275af1b72f699efcb61586565d4c3c7e41f4b3e692471abd
+0x10e1a313e573d96abe701d8848742cf88166dd2ded38ac22267a05d1d62baf71
+0x0bdeebbad8f9b240192635c42f40f2d02ee524c5a3fe8cda53fb4897b08c66fe
+EOF
+```
+
+Then import the above keys into Azure Key Vault as a secret (assuming Azure CLI is installed and logged in):
+
+```
+az keyvault secret set --vault-name "YOUR VAULT NAME" --name "BLS-TEST-KEYS" --file "./bls-test-keys.txt"
+```
+
+Finally, one more with single-line secret with a tag value ENV=TEST:
+
+```
+az keyvault secret set --vault-name "YOUR VAULT NAME" --name "BLS-TEST-TAGGED-KEY" --tags ENV=TEST --value 0x5e8d5667ce78982a07242739ab03dc63c91e830c80a5b6adca777e3f216a405d
+```
+
+### Azure Key Vault SECP Test Keys
+
+The tests perform SECP remote signing operations using Key Vault "Key" objects. The tests assume that you have imported
+following SECP256K1 private keys into Azure Key Vault (test keys are adapted from [ethpandaops](https://github.com/ethpandaops/ethereum-package/blob/main/src/prelaunch_data_generator/genesis_constants/genesis_constants.star)
+where at least 1 key is tagged with `ENV=TEST`.
+```
+# m/44'/60'/0'/0/18
+    new_prefunded_account(
+        "0xD9211042f35968820A3407ac3d80C725f8F75c14",
+        "a492823c3e193d6c595f37a18e3c06650cf4c74558cc818b16130b293716106f",
+    ),
+    # m/44'/60'/0'/0/19
+    new_prefunded_account(
+        "0xD8F3183DEF51A987222D845be228e0Bbb932C222",
+        "c5114526e042343c6d1899cad05e1c00ba588314de9b96929914ee0df18d46b2",
+    ),
+    # m/44'/60'/0'/0/20
+    new_prefunded_account(
+        "0xafF0CA253b97e54440965855cec0A8a2E2399896",
+        "04b9f63ecf84210c5366c66d68fa1f5da1fa4f634fad6dfc86178e4d79ff9e59",
+    ),
+```
+
+In order to import the above keys, you need to use Azure CLI commands like below:
+- Key 18
+```sh
+PRIVATE_KEY_HEX="a492823c3e193d6c595f37a18e3c06650cf4c74558cc818b16130b293716106f"
+
+# Create DER format then convert to PEM
+(echo "302e0201010420${PRIVATE_KEY_HEX}a00706052b8104000a" | xxd -r -p) | \
+openssl ec -inform DER -outform PEM -out private-key.pem
+
+# Verify it
+openssl ec -in private-key.pem -text -noout
+
+# Import into Azure (using your own values for vault-name)
+az keyvault key import --curve P-256K --kty EC --vault-name "YOUR VAULT NAME" --name "SECP-18" --pem-file ./private-key.pem 
+```
+
+- Key 19
+
+```sh
+PRIVATE_KEY_HEX="c5114526e042343c6d1899cad05e1c00ba588314de9b96929914ee0df18d46b2"
+
+# Create DER format then convert to PEM
+(echo "302e0201010420${PRIVATE_KEY_HEX}a00706052b8104000a" | xxd -r -p) | \
+openssl ec -inform DER -outform PEM -out private-key.pem
+
+# Verify it
+openssl ec -in private-key.pem -text -noout
+
+# Import into Azure (using your own values for vault-name)
+az keyvault key import --curve P-256K --kty EC --vault-name "YOUR VAULT NAME" --name "SECP-19" --pem-file ./private-key.pem 
+```
+
+- Key 20 (tagged with ENV=TEST)
+
+```sh
+PRIVATE_KEY_HEX="04b9f63ecf84210c5366c66d68fa1f5da1fa4f634fad6dfc86178e4d79ff9e59"
+
+# Create DER format then convert to PEM
+(echo "302e0201010420${PRIVATE_KEY_HEX}a00706052b8104000a" | xxd -r -p) | \
+openssl ec -inform DER -outform PEM -out private-key.pem
+
+# Verify it
+openssl ec -in private-key.pem -text -noout
+
+# Import into Azure (using your own values for vault-name)
+az keyvault key import --curve P-256K --kty EC --vault-name "YOUR VAULT NAME" --name "SECP-20-TAGGED" --pem-file ./private-key.pem --tags ENV=TEST 
+
+```

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/utils/MetadataFileHelpers.java

@@ -173,12 +173,13 @@ public void createAzureKeyYamlFileAt(
       final String clientId,
       final String clientSecret,
       final String keyVaultName,
-      final String tenantId) {
+      final String tenantId,
+      final String keyName) {
     try {
       final Map<String, String> signingMetadata = new HashMap<>();
       signingMetadata.put("type", "azure-key");
       signingMetadata.put("vaultName", keyVaultName);
-      signingMetadata.put("keyName", "TestKey2");
+      signingMetadata.put("keyName", keyName);
       signingMetadata.put("clientId", clientId);
       signingMetadata.put("clientSecret", clientSecret);
       signingMetadata.put("tenantId", tenantId);

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AzureKeyVaultAcceptanceTest.java

@@ -13,47 +13,43 @@
 package tech.pegasys.web3signer.tests.bulkloading;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.hasSize;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_AZURE_BULK_LOADING;
 import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealtcheckKeysLoaded;
 import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealthcheckErrorCount;
+import static tech.pegasys.web3signer.keystorage.azure.AzureKeyVault.createUsingClientSecretCredentials;
 
+import tech.pegasys.teku.bls.BLSSecretKey;
 import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder;
+import tech.pegasys.web3signer.keystorage.azure.AzureKeyVault;
 import tech.pegasys.web3signer.signing.KeyType;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
 import tech.pegasys.web3signer.signing.config.DefaultAzureKeyVaultParameters;
 import tech.pegasys.web3signer.tests.AcceptanceTestBase;
 
+import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.annotations.VisibleForTesting;
 import io.restassured.http.ContentType;
 import io.restassured.response.Response;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.tuweni.bytes.Bytes32;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.EnumSource;
-import org.junit.jupiter.params.provider.MethodSource;
 
 public class AzureKeyVaultAcceptanceTest extends AcceptanceTestBase {
 
   private static final String CLIENT_ID = System.getenv("AZURE_CLIENT_ID");
   private static final String CLIENT_SECRET = System.getenv("AZURE_CLIENT_SECRET");
   private static final String TENANT_ID = System.getenv("AZURE_TENANT_ID");
   private static final String VAULT_NAME = System.getenv("AZURE_KEY_VAULT_NAME");
-  private static final String BLS_KEY =
-      "0x989d34725a2bfc3f15105f3f5fc8741f436c25ee1ee4f948e425d6bcb8c56bce6e06c269635b7e985a7ffa639e2409bf";
-  private static final String BLS_TAGGED_KEY =
-      "0xb3b6fb8dab2a4c9d00247c18c4b7e91c62da3f7ad31c822c00097f93ac8ff2c4a526611f7d0a9c85946e93f371852c69";
-  private static final String SECP_KEY =
-      "0xa95663509e608da3c2af5a48eb4315321f8430cbed5518a44590cc9d367f01dc72ebbc583fc7d94f9fdc20eb6e162c9f8cb35be8a91a3b1d32a63ecc10be4e08";
-  private static final String SECP_TAGGED_KEY =
-      "0x234053dbe014ebe573e5e8f6eab5e0417bf705466009f7c15b8d23593abd1bda426593d92b32efb240afe6efa46d5679fad0dc427e0aa0fc61c2464ce93c7c5e";
 
   @BeforeAll
   public static void setup() {
@@ -63,10 +59,84 @@ public static void setup() {
     assumeTrue(!StringUtils.isEmpty(VAULT_NAME), "Set AZURE_KEY_VAULT_NAME environment variable");
   }
 
+  /**
+   * These keys are expected to be pre-created in Azure keystore. The first secret is multivalue
+   * with 10 keys. The second secret is created with single value/key and tagged with ENV:TEST.
+   *
+   * @return list of expected BLS public keys in hex format
+   */
+  static List<String> expectedBLSPubKeys() {
+    return getBLSSecretsFromAzureVault().stream()
+        .flatMap(azureSecret -> azureSecret.values().stream())
+        .map(
+            secret ->
+                BLSSecretKey.fromBytes(Bytes32.fromHexString(secret)).toPublicKey().toHexString())
+        .toList();
+  }
+
+  static List<String> expectedBLSPubKeyWithTag(final String tagKey, final String tagValue) {
+    return getBLSSecretsFromAzureVault().stream()
+        .filter(
+            azureSecret ->
+                azureSecret.tags() != null
+                    && azureSecret.tags().containsKey(tagKey)
+                    && azureSecret.tags().get(tagKey).equals(tagValue))
+        .flatMap(azureSecret -> azureSecret.values().stream())
+        .map(
+            secret ->
+                BLSSecretKey.fromBytes(Bytes32.fromHexString(secret)).toPublicKey().toHexString())
+        .toList();
+  }
+
+  /**
+   * Expected SECP256K1 public keys pre-created in Azure keystore.
+   *
+   * @return list of expected SECP256K1 public keys in hex format
+   */
+  static List<String> expectedSECPPubKeys() {
+    return getSECPKeysFromAzureVault().stream().map(AzureKeyVault.AzureKey::publicKeyHex).toList();
+  }
+
+  static List<String> expectedSECPPubKeyWithTag(final String tagKey, final String tagValue) {
+    return getSECPKeysFromAzureVault().stream()
+        .filter(
+            azureSecret ->
+                azureSecret.tags() != null
+                    && azureSecret.tags().containsKey(tagKey)
+                    && azureSecret.tags().get(tagKey).equals(tagValue))
+        .map(AzureKeyVault.AzureKey::publicKeyHex)
+        .toList();
+  }
+
+  @VisibleForTesting
+  public static List<AzureKeyVault.AzureKey> getSECPKeysFromAzureVault() {
+    try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
+      final AzureKeyVault azureKeyVault =
+          createUsingClientSecretCredentials(
+              CLIENT_ID, CLIENT_SECRET, TENANT_ID, VAULT_NAME, executor, 60);
+
+      final var azureKeys = azureKeyVault.getAzureKeys();
+      assertThat(azureKeys).isNotEmpty();
+      return azureKeys;
+    }
+  }
+
+  public static List<AzureKeyVault.AzureSecret> getBLSSecretsFromAzureVault() {
+    try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
+      final AzureKeyVault azureKeyVault =
+          createUsingClientSecretCredentials(
+              CLIENT_ID, CLIENT_SECRET, TENANT_ID, VAULT_NAME, executor, 60);
+
+      return azureKeyVault.getAzureSecrets();
+    }
+  }
+
   @ParameterizedTest
   @EnumSource(KeyType.class)
-  void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi(final KeyType keyType)
-      throws JsonProcessingException {
+  void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi(final KeyType keyType) {
+    final List<String> expectedPubKeys =
+        keyType == KeyType.BLS ? expectedBLSPubKeys() : expectedSECPPubKeys();
+
     final AzureKeyVaultParameters azureParams =
         new DefaultAzureKeyVaultParameters(VAULT_NAME, CLIENT_ID, TENANT_ID, CLIENT_SECRET);
 
@@ -83,7 +153,7 @@ void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi(final KeyType k
         .then()
         .statusCode(200)
         .contentType(ContentType.JSON)
-        .body("", hasItems(expectedKey(keyType, false)));
+        .body("", containsInAnyOrder(expectedPubKeys.toArray()));
 
     final Response healthcheckResponse = signer.healthcheck();
     healthcheckResponse
@@ -92,36 +162,36 @@ void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi(final KeyType k
         .contentType(ContentType.JSON)
         .body("status", equalTo("UP"));
 
-    // BLS keys include additional multi-line key with 200 keys
-    final int expectedKeyLoaded = keyType == KeyType.BLS ? 202 : 2;
-
     final String jsonBody = healthcheckResponse.body().asString();
     final int keysLoaded = getHealtcheckKeysLoaded(jsonBody, KEYS_CHECK_AZURE_BULK_LOADING);
-    assertThat(keysLoaded).isEqualTo(expectedKeyLoaded);
+    assertThat(keysLoaded).isEqualTo(expectedPubKeys.size());
   }
 
-  @ParameterizedTest(name = "{index} - KeyType: {0}, using config file: {1}")
-  @MethodSource("azureSecretsViaTag")
-  void azureSecretsViaTag(final KeyType keyType, boolean useConfigFile) {
+  @ParameterizedTest(name = "{index} - KeyType: {0}")
+  @EnumSource(KeyType.class)
+  void azureSecretsViaTag(final KeyType keyType) {
     final AzureKeyVaultParameters azureParams =
         new DefaultAzureKeyVaultParameters(
             VAULT_NAME, CLIENT_ID, TENANT_ID, CLIENT_SECRET, Map.of("ENV", "TEST"));
 
     final SignerConfigurationBuilder configBuilder =
         new SignerConfigurationBuilder()
             .withMode(calculateMode(keyType))
-            .withUseConfigFile(useConfigFile)
             .withAzureKeyVaultParameters(azureParams)
             .withUseConfigFile(true);
 
     startSigner(configBuilder.build());
 
+    final List<String> expectedPubKey =
+        keyType == KeyType.BLS
+            ? expectedBLSPubKeyWithTag("ENV", "TEST")
+            : expectedSECPPubKeyWithTag("ENV", "TEST");
     final Response response = signer.callApiPublicKeys(keyType);
     response
         .then()
         .statusCode(200)
         .contentType(ContentType.JSON)
-        .body("", hasItems(expectedKey(keyType, true)));
+        .body("", containsInAnyOrder(expectedPubKey.toArray()));
 
     // the tag filter will return only valid keys. The healthcheck should be UP
     final Response healthcheckResponse = signer.healthcheck();
@@ -131,22 +201,14 @@ void azureSecretsViaTag(final KeyType keyType, boolean useConfigFile) {
         .contentType(ContentType.JSON)
         .body("status", equalTo("UP"));
 
-    // keys loaded should be 1 as well.
+    // keys loaded should be >= 1 and error count should be 0
     final String jsonBody = healthcheckResponse.body().asString();
     final int keysLoaded = getHealtcheckKeysLoaded(jsonBody, KEYS_CHECK_AZURE_BULK_LOADING);
     final int errorCount = getHealthcheckErrorCount(jsonBody, KEYS_CHECK_AZURE_BULK_LOADING);
-    assertThat(keysLoaded).isOne();
+    assertThat(keysLoaded).isNotZero();
     assertThat(errorCount).isZero();
   }
 
-  private static Stream<Arguments> azureSecretsViaTag() {
-    return Stream.of(
-        Arguments.arguments(KeyType.BLS, false),
-        Arguments.arguments(KeyType.BLS, true),
-        Arguments.arguments(KeyType.SECP256K1, false),
-        Arguments.arguments(KeyType.SECP256K1, true));
-  }
-
   @ParameterizedTest
   @EnumSource(KeyType.class)
   void invalidVaultParametersFailsToLoadKeys(final KeyType keyType) {
@@ -194,16 +256,12 @@ void envVarsAreUsedToDefaultAzureParams(final KeyType keyType) {
     startSigner(configBuilder.build());
 
     final Response response = signer.callApiPublicKeys(keyType);
+    final List<String> expectedPubKeys =
+        keyType == KeyType.BLS ? expectedBLSPubKeys() : expectedSECPPubKeys();
     response
         .then()
         .statusCode(200)
         .contentType(ContentType.JSON)
-        .body("", hasItems(expectedKey(keyType, false)));
-  }
-
-  private String expectedKey(final KeyType keyType, final boolean tagged) {
-    return keyType == KeyType.BLS
-        ? tagged ? BLS_TAGGED_KEY : BLS_KEY
-        : tagged ? SECP_TAGGED_KEY : SECP_KEY;
+        .body("", containsInAnyOrder(expectedPubKeys.toArray()));
   }
 }