Merge pull request #625 from oracle/owls-70659

rjeberhard · web-flow · commit c5a975d27d3e · 2018-12-15T07:01:50.000-05:00
OWLS-70659 Provide ability to retry introspection due to DeadlineExceeded status
diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainPresence.java b/operator/src/main/java/oracle/kubernetes/operator/DomainPresence.java
@@ -8,10 +8,17 @@
 
 public class DomainPresence {
   private static final int DEFAULT_TIMEOUT_SECONDS = 5;
+  private static final int DEFAULT_RETRY_MAX_COUNT = 5;
 
   static int getDomainPresenceFailureRetrySeconds() {
     return Optional.ofNullable(TuningParameters.getInstance())
         .map(parameters -> parameters.getMainTuning().domainPresenceFailureRetrySeconds)
         .orElse(DEFAULT_TIMEOUT_SECONDS);
   }
+
+  static int getDomainPresenceFailureRetryMaxCount() {
+    return Optional.ofNullable(TuningParameters.getInstance())
+        .map(parameters -> parameters.getMainTuning().domainPresenceFailureRetryMaxCount)
+        .orElse(DEFAULT_RETRY_MAX_COUNT);
+  }
 }
diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java b/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java
@@ -843,7 +843,23 @@ public void onThrowable(Packet packet, Throwable throwable) {
                       DomainPresenceInfo existing = getExistingDomainPresenceInfo(ns, domainUID);
                       if (existing != null) {
                         existing.setPopulated(false);
-                        makeRightDomainPresence(existing, true, isDeleting, false);
+                        // proceed only if we have not already retried max number of times
+                        int retryCount = existing.incrementAndGetFailureCount();
+                        LOGGER.fine(
+                            "Failure count for DomainPresenceInfo: "
+                                + existing
+                                + " is now: "
+                                + retryCount);
+                        if (retryCount <= DomainPresence.getDomainPresenceFailureRetryMaxCount()) {
+                          makeRightDomainPresence(existing, true, isDeleting, false);
+                        } else {
+                          LOGGER.severe(
+                              MessageKeys.CANNOT_START_DOMAIN_AFTER_MAX_RETRIES,
+                              domainUID,
+                              ns,
+                              DomainPresence.getDomainPresenceFailureRetryMaxCount(),
+                              throwable);
+                        }
                       }
                     },
                     DomainPresence.getDomainPresenceFailureRetrySeconds(),
diff --git a/operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java b/operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java
@@ -83,10 +83,11 @@ public void receivedResponse(Watch.Response<V1Job> item) {
       case "ADDED":
       case "MODIFIED":
         V1Job job = item.object;
-        Boolean isComplete = isComplete(job); // isReady(job);
+        Boolean isComplete = isComplete(job);
+        Boolean isFailed = isFailed(job);
         String jobName = job.getMetadata().getName();
-        if (isComplete) {
-          Complete complete = completeCallbackRegistrations.remove(jobName);
+        if (isComplete || isFailed) {
+          Complete complete = completeCallbackRegistrations.get(jobName);
           if (complete != null) {
             complete.isComplete(job);
           }
@@ -150,6 +151,11 @@ private WaitForJobReadyStep(V1Job job, Step next) {
       this.job = job;
     }
 
+    boolean shouldProcessJob(V1Job job) {
+      return (this.job.getMetadata().getCreationTimestamp().getMillis()
+          == job.getMetadata().getCreationTimestamp().getMillis());
+    }
+
     @Override
     public NextAction apply(Packet packet) {
       if (isComplete(job)) {
@@ -165,6 +171,10 @@ public NextAction apply(Packet packet) {
           (fiber) -> {
             Complete complete =
                 (V1Job job) -> {
+                  if (!shouldProcessJob(job)) {
+                    return;
+                  }
+                  completeCallbackRegistrations.remove(job.getMetadata().getName());
                   if (didResume.compareAndSet(false, true)) {
                     LOGGER.fine("Job status: " + job.getStatus());
                     packet.put(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB, job);
diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java
@@ -21,6 +21,7 @@ public static TuningParameters getInstance() {
 
   public static class MainTuning {
     public final int domainPresenceFailureRetrySeconds;
+    public final int domainPresenceFailureRetryMaxCount;
     public final int domainPresenceRecheckIntervalSeconds;
     public final int targetNamespaceRecheckIntervalSeconds;
     public final int statusUpdateTimeoutSeconds;
@@ -30,13 +31,15 @@ public static class MainTuning {
 
     public MainTuning(
         int domainPresenceFailureRetrySeconds,
+        int domainPresenceFailureRetryMaxCount,
         int domainPresenceRecheckIntervalSeconds,
         int targetNamespaceRecheckIntervalSeconds,
         int statusUpdateTimeoutSeconds,
         int unchangedCountToDelayStatusRecheck,
         long initialShortDelay,
         long eventualLongDelay) {
       this.domainPresenceFailureRetrySeconds = domainPresenceFailureRetrySeconds;
+      this.domainPresenceFailureRetryMaxCount = domainPresenceFailureRetryMaxCount;
       this.domainPresenceRecheckIntervalSeconds = domainPresenceRecheckIntervalSeconds;
       this.targetNamespaceRecheckIntervalSeconds = targetNamespaceRecheckIntervalSeconds;
       this.statusUpdateTimeoutSeconds = statusUpdateTimeoutSeconds;
diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java
@@ -52,6 +52,7 @@ private void update() {
     MainTuning main =
         new MainTuning(
             (int) readTuningParameter("domainPresenceFailureRetrySeconds", 10),
+            (int) readTuningParameter("domainPresenceFailureRetryMaxCount", 5),
             (int) readTuningParameter("domainPresenceRecheckIntervalSeconds", 120),
             (int) readTuningParameter("targetNamespaceRecheckIntervalSeconds", 3),
             (int) readTuningParameter("statusUpdateTimeoutSeconds", 10),
diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java
@@ -12,6 +12,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import oracle.kubernetes.operator.wlsconfig.WlsServerConfig;
 import oracle.kubernetes.weblogic.domain.v2.Domain;
@@ -28,6 +29,7 @@ public class DomainPresenceInfo {
   private final AtomicReference<Domain> domain;
   private final AtomicBoolean isDeleting = new AtomicBoolean(false);
   private final AtomicBoolean isPopulated = new AtomicBoolean(false);
+  private final AtomicInteger retryCount = new AtomicInteger(0);
   private final AtomicReference<Collection<ServerStartupInfo>> serverStartupInfo;
 
   private final ConcurrentMap<String, ServerKubernetesObjects> servers = new ConcurrentHashMap<>();
@@ -75,6 +77,18 @@ public void setPopulated(boolean populated) {
     isPopulated.set(populated);
   }
 
+  public void resetFailureCount() {
+    retryCount.set(0);
+  }
+
+  public int incrementAndGetFailureCount() {
+    return retryCount.incrementAndGet();
+  }
+
+  public int getRetryCount() {
+    return retryCount.get();
+  }
+
   /**
    * Last completion time
    *
@@ -87,6 +101,7 @@ public DateTime getLastCompletionTime() {
   /** Sets the last completion time to now */
   public void complete() {
     this.lastCompletionTime = new DateTime();
+    resetFailureCount();
   }
 
   /**
diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java
@@ -21,6 +21,8 @@ public abstract class JobStepContext implements StepContextConstants {
 
   private final DomainPresenceInfo info;
   private V1Job jobModel;
+  final long DEFAULT_ACTIVE_DEADLINE_SECONDS = 120L;
+  final long DEFAULT_ACTIVE_DEADLINE_INCREMENT_SECONDS = 60L;
 
   JobStepContext(Packet packet) {
     info = packet.getSPI(DomainPresenceInfo.class);
@@ -165,9 +167,22 @@ V1ObjectMeta createMetadata() {
     return metadata;
   }
 
+  private long getActiveDeadlineSeconds() {
+    return DEFAULT_ACTIVE_DEADLINE_SECONDS
+        + (DEFAULT_ACTIVE_DEADLINE_INCREMENT_SECONDS * info.getRetryCount());
+  }
+
   protected V1JobSpec createJobSpec(TuningParameters tuningParameters) {
+    LOGGER.fine(
+        "Creating job "
+            + getJobName()
+            + " with activeDeadlineSeconds = "
+            + getActiveDeadlineSeconds());
     V1JobSpec jobSpec =
-        new V1JobSpec().backoffLimit(0).template(createPodTemplateSpec(tuningParameters));
+        new V1JobSpec()
+            .backoffLimit(0)
+            .activeDeadlineSeconds(getActiveDeadlineSeconds())
+            .template(createPodTemplateSpec(tuningParameters));
 
     return jobSpec;
   }
diff --git a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java
@@ -144,4 +144,5 @@ private MessageKeys() {}
   public static final String CANNOT_PARSE_TOPOLOGY = "WLSKO-0141";
   public static final String CANNOT_PARSE_INTROSPECTOR_RESULT = "WLSKO-0142";
   public static final String CANNOT_PARSE_INTROSPECTOR_FILE = "WLSKO-0143";
+  public static final String CANNOT_START_DOMAIN_AFTER_MAX_RETRIES = "WLSKO-0144";
 }
diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties
@@ -142,3 +142,4 @@ WLSKO-0140=Job {0} is completed with status: {1}
 WLSKO-0141=Failed to parse WebLogic Domain topology due to exception: {0}
 WLSKO-0142=Failed to parse results from domain introspector for domain {0} due to exception: {1}
 WLSKO-0143=Failed to parse file {0} from domain introspector for domain {1} due to exception: {2}
+WLSKO-0144=Unable to start domain with domainUID {0} in namespace {1} after {2} attempts due to exception: {3}
diff --git a/operator/src/main/resources/scripts/startNodeManager.sh b/operator/src/main/resources/scripts/startNodeManager.sh
@@ -113,9 +113,12 @@ createFolder ${NODEMGR_LOG_HOME}
 
 nodemgr_log_file=${NODEMGR_LOG_HOME}/${SERVER_NAME}_nodemanager.log
 nodemgr_out_file=${NODEMGR_LOG_HOME}/${SERVER_NAME}_nodemanager.out
+nodemgr_lck_file=${NODEMGR_LOG_HOME}/${SERVER_NAME}_nodemanager.log.lck
 
-checkEnv NODEMGR_LOG_HOME nodemgr_log_file nodemgr_out_file
+checkEnv NODEMGR_LOG_HOME nodemgr_log_file nodemgr_out_file nodemgr_lck_file
 
+trace "remove nodemanager .lck file"
+rm -f ${nodemgr_lck_file}
 
 
 ###############################################################################
@@ -209,6 +212,7 @@ if [ ! "${SERVER_NAME}" = "introspector" ]; then
     rm -f ${wl_state_file} || exit 1
   fi
 
+
 cat <<EOF > ${wl_props_file}
 # Server startup properties
 AutoRestart=true

Original file line number	Diff line number	Diff line change
`@@ -144,4 +144,5 @@ private MessageKeys() {}`
`144`	`144`	`public static final String CANNOT_PARSE_TOPOLOGY = "WLSKO-0141";`
`145`	`145`	`public static final String CANNOT_PARSE_INTROSPECTOR_RESULT = "WLSKO-0142";`
`146`	`146`	`public static final String CANNOT_PARSE_INTROSPECTOR_FILE = "WLSKO-0143";`
	`147`	`+ public static final String CANNOT_START_DOMAIN_AFTER_MAX_RETRIES = "WLSKO-0144";`
`147`	`148`	`}`