[9.0] Unmute docker tests, add more logging and increase startup timeout (#131203) (#131293)

* Unmute docker tests, add more logging and increase startup timeout (#131203)

Unmuting 33 DockerTests with added additional logging on failures.
Also, bumping docker startup timeout to account for slow security
auto-configuration, and additionally moving CA certificates inside
the retry while loop.

Resolves #129808
Resolves #123437
Resolves #120911
Resolves #120914
Resolves #120910
Resolves #120918
Resolves #120925
Resolves #124990
Resolves #116636
Resolves #125680
Resolves #116147
Resolves #126232
Resolves #126145
Resolves #116866
Resolves #126936
Resolves #127081
Resolves #127158
Resolves #127138
Resolves #128018
Resolves #128031
Resolves #128110
Resolves #128006
Resolves #128174
Resolves #128144
Resolves #128165
Resolves #128075
Resolves #128117
Resolves #128121
Resolves #128115
Resolves #128230
Resolves #128120
Resolves #128996
Resolves #128867

(cherry picked from commit 4806156)

# Conflicts:
#	muted-tests.yml

* call dumpDebug

Author: slobodanadamovic

muted-tests.yml

@@ -161,24 +161,9 @@ tests:
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
   method: testAuthenticateShouldNotFallThroughInCaseOfFailure
   issue: https://github.com/elastic/elasticsearch/issues/120902
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test050BasicApiTests
-  issue: https://github.com/elastic/elasticsearch/issues/120911
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test140CgroupOsStatsAreAvailable
-  issue: https://github.com/elastic/elasticsearch/issues/120914
 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
   method: testReservedStatePersistsOnRestart
   issue: https://github.com/elastic/elasticsearch/issues/120923
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test070BindMountCustomPathConfAndJvmOptions
-  issue: https://github.com/elastic/elasticsearch/issues/120910
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test071BindMountCustomPathWithDifferentUID
-  issue: https://github.com/elastic/elasticsearch/issues/120918
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test171AdditionalCliOptionsAreForwarded
-  issue: https://github.com/elastic/elasticsearch/issues/120925
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=nodes.stats/11_indices_metrics/indices mappings exact count test for indices level}
   issue: https://github.com/elastic/elasticsearch/issues/120950
@@ -208,9 +193,6 @@ tests:
 - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   method: test {yaml=analysis-common/40_token_filters/stemmer_override file access}
   issue: https://github.com/elastic/elasticsearch/issues/121625
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test151MachineDependentHeapWithSizeOverride
-  issue: https://github.com/elastic/elasticsearch/issues/123437
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/cat/nodes/line_361}
   issue: https://github.com/elastic/elasticsearch/issues/124103
@@ -244,39 +226,15 @@ tests:
 - class: org.elasticsearch.packaging.test.EnrollmentProcessTests
   method: test20DockerAutoFormCluster
   issue: https://github.com/elastic/elasticsearch/issues/128113
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test011SecurityEnabledStatus
-  issue: https://github.com/elastic/elasticsearch/issues/124990
 - class: org.elasticsearch.packaging.test.TemporaryDirectoryConfigTests
   method: test21AcceptsCustomPathInDocker
   issue: https://github.com/elastic/elasticsearch/issues/128114
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test130JavaHasCorrectOwnership
-  issue: https://github.com/elastic/elasticsearch/issues/128174
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test024InstallPluginFromArchiveUsingConfigFile
-  issue: https://github.com/elastic/elasticsearch/issues/126936
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test120DockerLogsIncludeElasticsearchLogs
-  issue: https://github.com/elastic/elasticsearch/issues/128117
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test072RunEsAsDifferentUserAndGroup
-  issue: https://github.com/elastic/elasticsearch/issues/128031
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test010Install
-  issue: https://github.com/elastic/elasticsearch/issues/125680
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test600Interrupt
-  issue: https://github.com/elastic/elasticsearch/issues/128144
 - class: org.elasticsearch.action.admin.cluster.node.tasks.CancellableTasksIT
   method: testChildrenTasksCancelledOnTimeout
   issue: https://github.com/elastic/elasticsearch/issues/123568
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceDiskSpaceTests
   method: testUnavailableBudgetBlocksNewMergeTasksFromStartingExecution
   issue: https://github.com/elastic/elasticsearch/issues/130205
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test080ConfigurePasswordThroughEnvironmentVariableFile
-  issue: https://github.com/elastic/elasticsearch/issues/128075
 
 # Examples:
 #
@@ -315,4 +273,4 @@ tests:
 #    issue: "https://github.com/elastic/elasticsearch/..."
 #  - class: "org.elasticsearch.xpack.esql.**"
 #    method: "test {union_types.MultiIndexIpStringStatsInline *}"
-#    issue: "https://github.com/elastic/elasticsearch/..."
+#    issue: "https://github.com/elastic/elasticsearch/..."

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -145,6 +145,10 @@ public abstract class PackagingTestCase extends Assert {
         @Override
         protected void failed(Throwable e, Description description) {
             failed = true;
+            if (installation != null && installation.distribution.isDocker()) {
+                logger.warn("Test {} failed. Printing logs for failed test...", description.getMethodName());
+                dumpDebug();
+            }
         }
     };
 

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -150,7 +150,19 @@ private static HttpResponse execute(Request request, String username, String pas
             executor.auth(username, password);
             executor.authPreemptive(new HttpHost("localhost", 9200));
         }
-        return executor.execute(request).returnResponse();
+        try {
+            return executor.execute(request).returnResponse();
+        } catch (Exception e) {
+            logger.warn(
+                "Failed to execute request [{}] with username/password [{}/{}] and caCert [{}]",
+                request.toString(),
+                username,
+                password,
+                caCert,
+                e
+            );
+            throw e;
+        }
     }
 
     // polls every two seconds for Elasticsearch to be running on 9200
@@ -238,14 +250,13 @@ public static void waitForElasticsearch(
         long timeElapsed = 0;
         boolean started = false;
         Throwable thrownException = null;
-        if (caCert == null) {
-            caCert = getCaCert(installation);
-        }
 
         while (started == false && timeElapsed < waitTime) {
             if (System.currentTimeMillis() - lastRequest > requestInterval) {
+                if (caCert == null) {
+                    caCert = getCaCert(installation);
+                }
                 try {
-
                     final HttpResponse response = execute(
                         Request.Get((caCert != null ? "https" : "http") + "://localhost:9200/_cluster/health")
                             .connectTimeout((int) timeoutLength)
@@ -276,7 +287,7 @@ public static void waitForElasticsearch(
                     }
                     started = true;
 
-                } catch (IOException e) {
+                } catch (Exception e) {
                     if (thrownException == null) {
                         thrownException = e;
                     } else {

qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java

@@ -73,7 +73,7 @@ public class Docker {
     public static final Shell sh = new Shell();
     public static final DockerShell dockerShell = new DockerShell();
     public static final int STARTUP_SLEEP_INTERVAL_MILLISECONDS = 1000;
-    public static final int STARTUP_ATTEMPTS_MAX = 30;
+    public static final int STARTUP_ATTEMPTS_MAX = 45;
 
     /**
      * The length of the command exceeds what we can use for COLUMNS so we use