fix intermittent test issues in olcne (#3908)

Author: xiancao

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicClusterMiiDomain.java

@@ -269,7 +269,7 @@ private void createDomainCrAndVerify(
                         .domainType("JRF")
                         .runtimeEncryptionSecret(encryptionSecretName))
                     .addSecretsItem(rcuAccessSecretName)
-                    .introspectorJobActiveDeadlineSeconds(600L)));
+                    .introspectorJobActiveDeadlineSeconds(900L)));
 
     createDomainAndVerify(domain, domainNamespace);
   }

integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Kubernetes.java

@@ -624,10 +624,12 @@ public static OffsetDateTime getPodCreationTimestamp(String namespace, String la
     V1Pod pod = getPod(namespace, labelSelector, podName);
     if (pod != null && pod.getMetadata() != null) {
       return pod.getMetadata().getCreationTimestamp();
+    } else if (pod == null) {
+      getLogger().info("Pod {0} does not exist in namespace {1}", podName, namespace);
     } else {
-      getLogger().info("Pod doesn't exist or pod metadata is null");
-      return null;
+      getLogger().info("The metadata of Pod {0} in namespace {1} is null", podName, namespace);
     }
+    return null;
   }
 
   /**

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/DbUtils.java

@@ -284,6 +284,7 @@ public static synchronized void startOracleDB(String dbBaseImageName, int dbPort
 
     // wait for the Oracle DB pod to be ready
     testUntil(
+        withLongRetryPolicy,
         assertDoesNotThrow(() -> checkDBPodNameReady(dbNamespace, dbPodNamePrefix),
             String.format("DB pod %s is not ready yet in namespace %s", dbPodNamePrefix, dbNamespace)),
         logger,
@@ -294,6 +295,7 @@ public static synchronized void startOracleDB(String dbBaseImageName, int dbPort
             dbNamespace));
     logger.info("Wait for the oracle Db pod: {0} ready in namespace {1}", dbPodName, dbNamespace);
     testUntil(
+        withLongRetryPolicy,
         assertDoesNotThrow(() -> podIsReady(dbNamespace, "app=database", dbPodName),
           "oracleDBService podReady failed with ApiException"),
         logger,
@@ -527,6 +529,7 @@ private static Callable<Boolean> podLogContains(String matchStr, String podName,
   public static void checkDbReady(String matchStr, String podName, String dbNamespace) {
     LoggingFacade logger = getLogger();
     testUntil(
+        withLongRetryPolicy,
         assertDoesNotThrow(() -> podLogContains(matchStr, podName, dbNamespace),
           String.format("podLogContains failed with ApiException for pod %s in namespace %s", podName, dbNamespace)),
         logger,

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/OperatorUtils.java

@@ -34,6 +34,7 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorRestServiceRunning;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorWebhookIsReady;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
 import static oracle.weblogic.kubernetes.utils.OKDUtils.createRouteForOKD;
 import static oracle.weblogic.kubernetes.utils.OKDUtils.setTlsTerminationForRoute;
@@ -492,6 +493,7 @@ public static OperatorParams installAndVerifyOperator(String opNamespace,
     if (webhookOnly) {
       logger.info("Wait for the operator webhook pod is ready in namespace {0}", opNamespace);
       testUntil(
+          withLongRetryPolicy,
           assertDoesNotThrow(() -> operatorWebhookIsReady(opNamespace),
               "operatorWebhookIsReady failed with ApiException"),
           logger,
@@ -500,6 +502,7 @@ public static OperatorParams installAndVerifyOperator(String opNamespace,
     } else {
       logger.info("Wait for the operator pod is ready in namespace {0}", opNamespace);
       testUntil(
+          withLongRetryPolicy,
           assertDoesNotThrow(() -> operatorIsReady(opNamespace),
               "operatorIsReady failed with ApiException"),
           logger,
@@ -510,6 +513,7 @@ public static OperatorParams installAndVerifyOperator(String opNamespace,
     if (withRestAPI) {
       logger.info("Wait for the operator external service in namespace {0}", opNamespace);
       testUntil(
+          withLongRetryPolicy,
           assertDoesNotThrow(() -> operatorRestServiceRunning(opNamespace),
             "operator external service is not running"),
           logger,