address review comments- update changes to runDomainPlan, set initial activeDeadlineSeconds for introspector job to 120s, and increment 1 minute for each subsequent retries

Author: alai8

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -813,10 +813,7 @@ void runDomainPlan(
         new CompletionCallback() {
           @Override
           public void onCompletion(Packet packet) {
-            DomainPresenceInfo existing = getExistingDomainPresenceInfo(ns, domainUID);
-            if (existing != null) {
-              existing.resetFailureCount();
-            }
+            // no-op
           }
 
           @Override
@@ -840,29 +837,33 @@ public void onThrowable(Packet packet, Throwable throwable) {
                   }
                 });
 
-            DomainPresenceInfo existing = getExistingDomainPresenceInfo(ns, domainUID);
-            if (existing != null) {
-              int failureCount = existing.incrementAndGetFailureCount();
-              LOGGER.fine(
-                  "Failure count for DomainPresenceInfo: " + existing + " is now: " + failureCount);
-              if (failureCount > DomainPresence.getDomainPresenceFailureRetryMaxCount()) {
-                LOGGER.warning(
-                    MessageKeys.CANNOT_START_DOMAIN_AFTER_MAX_RETRIES,
-                    domainUID,
-                    ns,
-                    DomainPresence.getDomainPresenceFailureRetryMaxCount(),
-                    throwable);
-              } else {
-                gate.getExecutor()
-                    .schedule(
-                        () -> {
-                          existing.setPopulated(false);
+            gate.getExecutor()
+                .schedule(
+                    () -> {
+                      DomainPresenceInfo existing = getExistingDomainPresenceInfo(ns, domainUID);
+                      if (existing != null) {
+                        existing.setPopulated(false);
+                        // proceed only if we have not already retried max number of times
+                        int retryCount = existing.incrementAndGetFailureCount();
+                        LOGGER.fine(
+                            "Failure count for DomainPresenceInfo: "
+                                + existing
+                                + " is now: "
+                                + retryCount);
+                        if (retryCount <= DomainPresence.getDomainPresenceFailureRetryMaxCount()) {
                           makeRightDomainPresence(existing, true, isDeleting, false);
-                        },
-                        DomainPresence.getDomainPresenceFailureRetrySeconds(),
-                        TimeUnit.SECONDS);
-              }
-            }
+                        } else {
+                          LOGGER.severe(
+                              MessageKeys.CANNOT_START_DOMAIN_AFTER_MAX_RETRIES,
+                              domainUID,
+                              ns,
+                              DomainPresence.getDomainPresenceFailureRetryMaxCount(),
+                              throwable);
+                        }
+                      }
+                    },
+                    DomainPresence.getDomainPresenceFailureRetrySeconds(),
+                    TimeUnit.SECONDS);
           }
         };
 

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

@@ -29,7 +29,7 @@ public class DomainPresenceInfo {
   private final AtomicReference<Domain> domain;
   private final AtomicBoolean isDeleting = new AtomicBoolean(false);
   private final AtomicBoolean isPopulated = new AtomicBoolean(false);
-  private final AtomicInteger failureCount = new AtomicInteger(0);
+  private final AtomicInteger retryCount = new AtomicInteger(0);
   private final AtomicReference<Collection<ServerStartupInfo>> serverStartupInfo;
 
   private final ConcurrentMap<String, ServerKubernetesObjects> servers = new ConcurrentHashMap<>();
@@ -78,11 +78,15 @@ public void setPopulated(boolean populated) {
   }
 
   public void resetFailureCount() {
-    failureCount.set(0);
+    retryCount.set(0);
   }
 
   public int incrementAndGetFailureCount() {
-    return failureCount.incrementAndGet();
+    return retryCount.incrementAndGet();
+  }
+
+  public int getRetryCount() {
+    return retryCount.get();
   }
 
   /**
@@ -97,6 +101,7 @@ public DateTime getLastCompletionTime() {
   /** Sets the last completion time to now */
   public void complete() {
     this.lastCompletionTime = new DateTime();
+    resetFailureCount();
   }
 
   /**

operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java

@@ -21,6 +21,8 @@ public abstract class JobStepContext implements StepContextConstants {
 
   private final DomainPresenceInfo info;
   private V1Job jobModel;
+  final long DEFAULT_ACTIVE_DEADLINE_SECONDS = 120L;
+  final long DEFAULT_ACTIVE_DEADLINE_INCREMENT_SECONDS = 60L;
 
   JobStepContext(Packet packet) {
     info = packet.getSPI(DomainPresenceInfo.class);
@@ -165,9 +167,22 @@ V1ObjectMeta createMetadata() {
     return metadata;
   }
 
+  private long getActiveDeadlineSeconds() {
+    return DEFAULT_ACTIVE_DEADLINE_SECONDS
+        + (DEFAULT_ACTIVE_DEADLINE_INCREMENT_SECONDS * info.getRetryCount());
+  }
+
   protected V1JobSpec createJobSpec(TuningParameters tuningParameters) {
+    LOGGER.fine(
+        "Creating job "
+            + getJobName()
+            + " with activeDeadlineSeconds = "
+            + getActiveDeadlineSeconds());
     V1JobSpec jobSpec =
-        new V1JobSpec().backoffLimit(0).template(createPodTemplateSpec(tuningParameters));
+        new V1JobSpec()
+            .backoffLimit(0)
+            .activeDeadlineSeconds(getActiveDeadlineSeconds())
+            .template(createPodTemplateSpec(tuningParameters));
 
     return jobSpec;
   }