Fix for intermittent failure for HPACustomMetrics test (#4126)

* Fix for intermittent failure for HPACustomMetrics test

Author: marinakog

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java

@@ -28,6 +28,7 @@
 import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams;
 import oracle.weblogic.kubernetes.annotations.IntegrationTest;
 import oracle.weblogic.kubernetes.annotations.Namespaces;
+import oracle.weblogic.kubernetes.assertions.impl.Kubernetes;
 import oracle.weblogic.kubernetes.logging.LoggingFacade;
 import oracle.weblogic.kubernetes.utils.ExecCommand;
 import oracle.weblogic.kubernetes.utils.ExecResult;
@@ -305,11 +306,27 @@ void testHPAWithCustomMetrics() {
         () -> verifyHPA(domainNamespace, "0/5"),
         logger,
         "Checking if total_open_session metric is 0");
+    testUntil(
+        withLongRetryPolicy,
+        () -> verifyHPA(domainNamespace, "2         3         2"),
+        logger,
+        "Checking if replica switched to 2");
+
     try {
       checkPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace);
     } catch (Exception ex) {
-      //retry again
-      checkPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace);
+      //check if different server was scaled down
+      try {
+        if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 1)) {
+          logger.info("HPA scaled down managed server 1");
+        } else if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 2)) {
+          logger.info("HPA scaled down managed server 2");
+        } else {
+          checkPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace);
+        }
+      } catch (Exception ex1) {
+        throw ex;
+      }
     }
   }