More concurrency in repo analysis integ tests (#138414)

Since #133633 these testclusters-based tests set `node.processors: 2`,
and since they also set `-Xmx512m` the resulting nodes only have a
single snapshot thread so the repo analysis tests do not meaningfully
exercise concurrent operations any more (at least for repositories that
have blocking implementations of their CAS operations). This commit adds
more snapshot threads to these tests to reinstate the lost coverage.

Author: DaveCTurner

x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AzureRepositoryAnalysisRestIT.java

@@ -74,6 +74,7 @@ private static Predicate<String> decideAuthHeaderPredicate() {
     private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-azure")
         .module("snapshot-repo-test-kit")
+        .setting("thread_pool.snapshot.max", "10")
         .keystore("azure.client.repository_test_kit.account", AZURE_TEST_ACCOUNT)
         .keystore("azure.client.repository_test_kit.key", () -> AZURE_TEST_KEY, s -> Strings.hasText(AZURE_TEST_KEY))
         .keystore("azure.client.repository_test_kit.sas_token", () -> AZURE_TEST_SASTOKEN, s -> Strings.hasText(AZURE_TEST_SASTOKEN))

x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java

@@ -31,6 +31,7 @@ public class GCSRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestT
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-gcs")
         .module("snapshot-repo-test-kit")
+        .setting("thread_pool.snapshot.max", "10")
         .setting("gcs.client.repository_test_kit.endpoint", () -> fixture.getAddress(), s -> USE_FIXTURE)
         .setting("gcs.client.repository_test_kit.token_uri", () -> fixture.getAddress() + "/o/oauth2/token", s -> USE_FIXTURE)
         .apply(c -> {

x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/HdfsRepositoryAnalysisRestIT.java

@@ -24,6 +24,7 @@ public class HdfsRepositoryAnalysisRestIT extends AbstractHdfsRepositoryAnalysis
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
         .plugin("repository-hdfs")
+        .setting("thread_pool.snapshot.max", "10")
         .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "false")
         .build();

x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/SecureHdfsRepositoryAnalysisRestIT.java

@@ -31,6 +31,7 @@ public class SecureHdfsRepositoryAnalysisRestIT extends AbstractHdfsRepositoryAn
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
         .plugin("repository-hdfs")
+        .setting("thread_pool.snapshot.max", "10")
         .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "false")
         .systemProperty("java.security.krb5.conf", () -> krb5Fixture.getConfPath().toString())

x-pack/plugin/snapshot-repo-test-kit/qa/minio/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/MinioRepositoryAnalysisRestIT.java

@@ -31,6 +31,7 @@ public class MinioRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRes
         .distribution(DistributionType.DEFAULT)
         .keystore("s3.client.repository_test_kit.access_key", "s3_test_access_key")
         .keystore("s3.client.repository_test_kit.secret_key", "s3_test_secret_key")
+        .setting("thread_pool.snapshot.max", "10")
         .setting("s3.client.repository_test_kit.endpoint", minioFixture::getAddress)
         .setting("xpack.security.enabled", "false")
         .setting("xpack.ml.enabled", "false")

x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java

@@ -82,6 +82,7 @@ private static void ensurePurposeParameterPresent(S3HttpHandler.S3Request reques
         .setting("s3.client.repo_test_kit.region", regionSupplier, (n) -> USE_FIXTURE)
         .setting("s3.client.repo-test_kit.add_purpose_custom_query_parameter", () -> randomFrom("true", "false"), n -> randomBoolean())
         .setting("xpack.security.enabled", "false")
+        .setting("thread_pool.snapshot.max", "10")
         .build();
 
     @ClassRule