Merge branch 'retry-on-early-missing' into 'main'

Requeue on missing resource validation failure for new domain

See merge request weblogic-cloud/weblogic-kubernetes-operator!4696

Author: rjeberhard

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainValidationSteps.java

@@ -139,11 +139,14 @@ static class DomainValidationStep extends Step {
       DomainPresenceInfo info = (DomainPresenceInfo) packet.get(ProcessingConstants.DOMAIN_PRESENCE_INFO);
       DomainResource domain = info.getDomain();
       List<String> fatalValidationFailures = domain.getFatalValidationFailures();
-      List<String> validationFailures = domain.getValidationFailures(new KubernetesResourceLookupImpl(packet));
-      String errorMsg = getErrorMessage(fatalValidationFailures, validationFailures);
-      if (validationFailures.isEmpty()) {
+      DomainResource.ValidationResult validationResult
+          = domain.getValidationFailures(new KubernetesResourceLookupImpl(packet));
+      if (validationResult.isDelay()) {
+        return doRequeue(packet);
+      } else if (validationResult.failures().isEmpty()) {
         return doNext(createRemoveSelectedFailuresStep(getNext(), DOMAIN_INVALID), packet);
       } else {
+        String errorMsg = getErrorMessage(fatalValidationFailures, validationResult.failures());
         LOGGER.severe(DOMAIN_VALIDATION_FAILED, domain.getDomainUid(), errorMsg);
         return doNext(DomainStatusUpdater.createDomainInvalidFailureSteps(errorMsg), packet);
       }

operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainResource.java

@@ -49,6 +49,7 @@
 import oracle.kubernetes.operator.processing.EffectiveClusterSpec;
 import oracle.kubernetes.operator.processing.EffectiveIntrospectorJobPodSpec;
 import oracle.kubernetes.operator.processing.EffectiveServerSpec;
+import oracle.kubernetes.utils.SystemClock;
 import oracle.kubernetes.weblogic.domain.EffectiveConfigurationFactory;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
@@ -964,7 +965,7 @@ public List<String> getFatalValidationFailures() {
   }
 
   // used by the operator
-  public List<String> getValidationFailures(KubernetesResourceLookup kubernetesResources) {
+  public ValidationResult getValidationFailures(KubernetesResourceLookup kubernetesResources) {
     return new DomainValidator().getValidationFailures(kubernetesResources);
   }
 
@@ -1038,14 +1039,18 @@ public int getMaxConcurrentShutdown(ClusterSpec clusterSpec) {
     }
   }
 
+  public static record ValidationResult(List<String> failures, boolean isDelay) {
+  }
+
   class DomainValidator extends Validator {
     private final Set<String> clusterNames = new HashSet<>();
     private final Set<String> serverNames = new HashSet<>();
+    private boolean isDelay;
 
-    private List<String> getValidationFailures(KubernetesResourceLookup kubernetesResources) {
+    private ValidationResult getValidationFailures(KubernetesResourceLookup kubernetesResources) {
       addFatalValidationFailures();
       addCrossReferenceValidationFailures(kubernetesResources);
-      return failures;
+      return new ValidationResult(failures, isDelay);
     }
 
     private void addCrossReferenceValidationFailures(KubernetesResourceLookup kubernetesResources) {
@@ -1542,16 +1547,27 @@ private void addMissingClusterResource(KubernetesResourceLookup resourceLookup)
           .forEach(cluster -> verifyClusterExists(resourceLookup, cluster));
     }
 
+    private boolean isRecentlyCreatedDomain() {
+      return Optional.ofNullable(getMetadata()).map(V1ObjectMeta::getCreationTimestamp)
+              .map(timestamp -> timestamp.isAfter(SystemClock.now().minusSeconds(10))).orElse(false);
+    }
+
     private void verifyClusterExists(KubernetesResourceLookup resourceLookup, V1LocalObjectReference cluster) {
       if (resourceLookup.findClusterInNamespace(cluster, getNamespace()) == null) {
         failures.add(missingClusterResource(cluster.getName(), getNamespace()));
+        if (isRecentlyCreatedDomain()) {
+          isDelay = true;
+        }
       }
     }
 
     @SuppressWarnings("SameParameterValue")
     private void verifySecretExists(KubernetesResourceLookup resources, String secretName, SecretType type) {
       if (secretName != null && !isSecretExists(resources.getSecrets(), secretName, getNamespace())) {
         failures.add(DomainValidationMessages.noSuchSecret(secretName, getNamespace(), type));
+        if (isRecentlyCreatedDomain()) {
+          isDelay = true;
+        }
       }
     }
 
@@ -1611,6 +1627,9 @@ private void verifyModelConfigMapExists(KubernetesResourceLookup resources, Stri
           && modelConfigMapName != null && !resources.isConfigMapExists(modelConfigMapName, getNamespace())) {
         failures.add(DomainValidationMessages.noSuchModelConfigMap(modelConfigMapName,
             "spec.configuration.model.configMap", getNamespace()));
+        if (isRecentlyCreatedDomain()) {
+          isDelay = true;
+        }
       }
     }
 
@@ -1621,6 +1640,9 @@ && getDomainCreationConfigMap() != null
           && !resources.isConfigMapExists(configMapName, getNamespace())) {
         failures.add(DomainValidationMessages.noSuchModelConfigMap(configMapName,
             "spec.configuration.initializeDomainOnPV.domain.domainCreationConfigMap", getNamespace()));
+        if (isRecentlyCreatedDomain()) {
+          isDelay = true;
+        }
       }
     }
 
@@ -1629,9 +1651,15 @@ private void verifyVolumeWithPVCWhenInitDomainOnPVDefined() {
         if (isPVCConfigured()) {
           if (noMatchVolumeWithPVC(getInitPvDomainPVCName())) {
             failures.add(DomainValidationMessages.noMatchVolumeWithPVC(getInitPvDomainPVCName()));
+            if (isRecentlyCreatedDomain()) {
+              isDelay = true;
+            }
           }
         } else if (noVolumeWithPVC()) {
           failures.add(DomainValidationMessages.noVolumeWithPVC());
+          if (isRecentlyCreatedDomain()) {
+            isDelay = true;
+          }
         }
       }
     }

operator/src/test/java/oracle/kubernetes/operator/helpers/DomainValidationStepTest.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2019, 2023, Oracle and/or its affiliates.
+// Copyright (c) 2019, 2024, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator.helpers;
@@ -25,6 +25,7 @@
 import oracle.kubernetes.operator.DomainProcessorTestSetup;
 import oracle.kubernetes.operator.work.Step;
 import oracle.kubernetes.operator.work.TerminalStep;
+import oracle.kubernetes.utils.SystemClock;
 import oracle.kubernetes.utils.TestUtils;
 import oracle.kubernetes.weblogic.domain.model.ClusterResource;
 import oracle.kubernetes.weblogic.domain.model.Configuration;
@@ -231,6 +232,7 @@ private boolean isFailedCondition(DomainCondition condition) {
   @Test
   void whenDomainRefersToUnknownSecret_updateStatus() {
     domain.getSpec().withWebLogicCredentialsSecret(new V1LocalObjectReference().name("name"));
+    domain.getMetadata().setCreationTimestamp(SystemClock.now().minusSeconds(20));
 
     testSupport.runSteps(domainValidationSteps);
 

operator/src/test/java/oracle/kubernetes/operator/makeright/FailureReportingTest.java

@@ -191,6 +191,11 @@ private void executeMakeRight() {
     executeMakeRight((String) null);
   }
 
+  private void updateDomain() {
+    removeCredentialsSecret();
+    domain.getMetadata().setCreationTimestamp(SystemClock.now().minusSeconds(20));
+  }
+
   private void removeCredentialsSecret() {
     DomainCommonConfigurator commonConfigurator = new DomainCommonConfigurator(domain);
     commonConfigurator.withWebLogicCredentialsSecret("no-such-secret");
@@ -292,7 +297,7 @@ enum TestCase {
     DOMAIN_VALIDATION_FAILURE {
       @Override
       Consumer<FailureReportingTest> getMutator() {
-        return FailureReportingTest::removeCredentialsSecret;
+        return FailureReportingTest::updateDomain;
       }
 
       @Override