Merge pull request #556 from oracle/fix-take9

Continuing to squash possible timing issues and add diagnosability

Author: rjeberhard

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -49,6 +49,7 @@
 import oracle.kubernetes.operator.work.Fiber;
 import oracle.kubernetes.operator.work.Fiber.CompletionCallback;
 import oracle.kubernetes.operator.work.FiberGate;
+import oracle.kubernetes.operator.work.FiberGateFactory;
 import oracle.kubernetes.operator.work.NextAction;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
@@ -62,7 +63,15 @@ public class DomainProcessorImpl implements DomainProcessor {
 
   private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator");
 
-  private static final FiberGate FIBER_GATE = new FiberGate(Main.engine);
+  private static final FiberGateFactory FACTORY =
+      () -> {
+        return new FiberGate(Main.engine);
+      };
+  private static final ConcurrentMap<String, FiberGate> fiberGates = new ConcurrentHashMap<>();
+
+  private static FiberGate getFiberGate(String ns) {
+    return fiberGates.computeIfAbsent(ns, k -> FACTORY.get());
+  }
 
   // Map from namespace to map of domainUID to Domain
   private static final ConcurrentMap<String, ConcurrentMap<String, DomainPresenceInfo>> domains =
@@ -754,6 +763,7 @@ void runDomainPlan(
       Step.StepAndPacket plan,
       boolean isDeleting,
       boolean isWillInterrupt) {
+    FiberGate gate = getFiberGate(ns);
     CompletionCallback cc =
         new CompletionCallback() {
           @Override
@@ -765,7 +775,7 @@ public void onCompletion(Packet packet) {
           public void onThrowable(Packet packet, Throwable throwable) {
             LOGGER.severe(MessageKeys.EXCEPTION, throwable);
 
-            FIBER_GATE.startFiberIfLastFiberMatches(
+            gate.startFiberIfLastFiberMatches(
                 domainUID,
                 Fiber.getCurrentIfSet(),
                 DomainStatusUpdater.createFailedStep(throwable, null),
@@ -782,8 +792,7 @@ public void onThrowable(Packet packet, Throwable throwable) {
                   }
                 });
 
-            FIBER_GATE
-                .getExecutor()
+            gate.getExecutor()
                 .schedule(
                     () -> {
                       DomainPresenceInfo existing = getExisting(ns, domainUID);
@@ -798,9 +807,9 @@ public void onThrowable(Packet packet, Throwable throwable) {
         };
 
     if (isWillInterrupt) {
-      FIBER_GATE.startFiber(domainUID, plan.step, plan.packet, cc);
+      gate.startFiber(domainUID, plan.step, plan.packet, cc);
     } else {
-      FIBER_GATE.startFiberIfNoCurrentFiber(domainUID, plan.step, plan.packet, cc);
+      gate.startFiberIfNoCurrentFiber(domainUID, plan.step, plan.packet, cc);
     }
   }
 

operator/src/main/java/oracle/kubernetes/operator/work/Fiber.java

@@ -808,7 +808,7 @@ private synchronized void addBreadCrumb(BreadCrumb bc) {
 
   private synchronized void recordBreadCrumb() {
     if (breadCrumbs != null) {
-      if (parent == null || parent.isDone()) {
+      if (parent == null) {
         StringBuilder sb = new StringBuilder();
         writeBreadCrumb(sb);
 
@@ -822,22 +822,20 @@ private synchronized void recordBreadCrumb() {
 
   private synchronized void writeBreadCrumb(StringBuilder sb) {
     if (breadCrumbs != null) {
-      if (parent == null || parent.isDone()) {
-        sb.append('[');
-        Iterator<BreadCrumb> it = breadCrumbs.iterator();
-        BreadCrumb previous = null;
-        while (it.hasNext()) {
-          BreadCrumb bc = it.next();
-          if (!bc.isMarker()) {
-            if (previous != null) {
-              sb.append(previous.isMarker() ? "][" : ",");
-            }
-            bc.writeTo(sb);
+      sb.append('[');
+      Iterator<BreadCrumb> it = breadCrumbs.iterator();
+      BreadCrumb previous = null;
+      while (it.hasNext()) {
+        BreadCrumb bc = it.next();
+        if (!bc.isMarker()) {
+          if (previous != null) {
+            sb.append(previous.isMarker() ? "][" : ",");
           }
-          previous = bc;
+          bc.writeTo(sb);
         }
-        sb.append(']');
+        previous = bc;
       }
+      sb.append(']');
     }
   }
 
@@ -887,7 +885,15 @@ public ChildFiberBreadCrumb(Fiber child) {
 
     @Override
     public void writeTo(StringBuilder sb) {
-      child.writeBreadCrumb(sb);
+      sb.append("{child-");
+      sb.append(child.id);
+      sb.append(": ");
+      if (child.status.get() == NOT_COMPLETE) {
+        sb.append("not-complete");
+      } else {
+        child.writeBreadCrumb(sb);
+      }
+      sb.append("}");
     }
   }
 

operator/src/main/java/oracle/kubernetes/operator/work/FiberGateFactory.java

@@ -0,0 +1,9 @@
+// Copyright 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Licensed under the Universal Permissive License v 1.0 as shown at
+// http://oss.oracle.com/licenses/upl.
+
+package oracle.kubernetes.operator.work;
+
+import java.util.function.Supplier;
+
+public interface FiberGateFactory extends Supplier<FiberGate> {}

operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java

@@ -49,6 +49,7 @@
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.LegalNames;
 import oracle.kubernetes.operator.helpers.ServerKubernetesObjects;
+import oracle.kubernetes.operator.work.FiberGateFactoryStub;
 import oracle.kubernetes.operator.work.ThreadFactorySingleton;
 import oracle.kubernetes.weblogic.domain.v2.Domain;
 import oracle.kubernetes.weblogic.domain.v2.DomainList;
@@ -83,8 +84,7 @@ public void setUp() throws Exception {
     mementos.add(ClientFactoryStub.install());
     mementos.add(StubWatchFactory.install());
     mementos.add(installStub(ThreadFactorySingleton.class, "INSTANCE", this));
-    mementos.add(
-        installStub(DomainProcessorImpl.class, "FIBER_GATE", testSupport.createFiberGateStub()));
+    mementos.add(FiberGateFactoryStub.install(testSupport));
     testSupport.addContainerComponent("TF", ThreadFactory.class, this);
 
     getStartedVariable();

operator/src/test/java/oracle/kubernetes/operator/work/FiberGateFactoryStub.java

@@ -0,0 +1,27 @@
+// Copyright  2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Licensed under the Universal Permissive License v 1.0 as shown at
+// http://oss.oracle.com/licenses/upl.
+
+package oracle.kubernetes.operator.work;
+
+import com.meterware.simplestub.Memento;
+import com.meterware.simplestub.StaticStubSupport;
+import oracle.kubernetes.operator.DomainProcessorImpl;
+
+public class FiberGateFactoryStub implements FiberGateFactory {
+  public static Memento install(FiberTestSupport testSupport) throws NoSuchFieldException {
+    return StaticStubSupport.install(
+        DomainProcessorImpl.class, "FACTORY", new FiberGateFactoryStub(testSupport));
+  }
+
+  private final FiberTestSupport testSupport;
+
+  public FiberGateFactoryStub(FiberTestSupport testSupport) {
+    this.testSupport = testSupport;
+  }
+
+  @Override
+  public FiberGate get() {
+    return testSupport.createFiberGateStub();
+  }
+}