avoid repeated logging of errors when ReadHealthStep for only some servers fail

Author: alai8

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -21,6 +21,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.Nullable;
 import oracle.kubernetes.operator.TuningParameters.MainTuning;
 import oracle.kubernetes.operator.calls.CallResponse;
@@ -314,7 +315,9 @@ public void run() {
                   new CompletionCallback() {
                     @Override
                     public void onCompletion(Packet packet) {
-                      if (Boolean.TRUE.equals(packet.get(ProcessingConstants.SERVER_HEALTH_READ))) {
+                      AtomicInteger serverHealthRead =
+                          packet.getValue(ProcessingConstants.REMAINING_SERVERS_HEALTH_READ);
+                      if (serverHealthRead == null || serverHealthRead.get() == 0) {
                         loggingFilter.setFiltering(false).resetLogHistory();
                       } else {
                         loggingFilter.setFiltering(true);

operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java

@@ -32,5 +32,5 @@ public interface ProcessingConstants {
   public static final String DOMAIN_INTROSPECTOR_LOG_RESULT = "domainIntrospectorLogResult";
   public static final String SIT_CONFIG_MAP = "sitConfigMap";
 
-  public static final String SERVER_HEALTH_READ = "serverHealthRead";
+  public static final String REMAINING_SERVERS_HEALTH_READ = "serverHealthRead";
 }

operator/src/main/java/oracle/kubernetes/operator/ServerStatusReader.java

@@ -21,6 +21,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import oracle.kubernetes.operator.helpers.ClientPool;
@@ -68,6 +69,9 @@ public NextAction apply(Packet packet) {
       packet.put(SERVER_STATE_MAP, new ConcurrentHashMap<String, String>());
       packet.put(SERVER_HEALTH_MAP, new ConcurrentHashMap<String, ServerHealth>());
 
+      AtomicInteger serverHealthRead = new AtomicInteger();
+      packet.put(ProcessingConstants.REMAINING_SERVERS_HEALTH_READ, serverHealthRead);
+
       Collection<StepAndPacket> startDetails =
           info.getServerPods()
               .map(pod -> createStatusReaderStep(packet, pod))
@@ -76,6 +80,7 @@ public NextAction apply(Packet packet) {
       if (startDetails.isEmpty()) {
         return doNext(packet);
       } else {
+        serverHealthRead.set(startDetails.size());
         return doForkJoin(getNext(), packet, startDetails);
       }
     }

operator/src/main/java/oracle/kubernetes/operator/steps/ReadHealthStep.java

@@ -16,6 +16,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import oracle.kubernetes.operator.ProcessingConstants;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.http.HttpClient;
@@ -153,7 +154,9 @@ public NextAction apply(Packet packet) {
                 (ConcurrentMap<String, ServerHealth>)
                     packet.get(ProcessingConstants.SERVER_HEALTH_MAP);
             serverHealthMap.put((String) packet.get(ProcessingConstants.SERVER_NAME), health);
-            packet.put(ProcessingConstants.SERVER_HEALTH_READ, Boolean.TRUE);
+            AtomicInteger serverHealthRead =
+                packet.getValue(ProcessingConstants.REMAINING_SERVERS_HEALTH_READ);
+            serverHealthRead.getAndDecrement();
           }
         }
         return doNext(packet);

operator/src/test/java/oracle/kubernetes/operator/steps/ReadHealthStepTest.java

@@ -9,7 +9,6 @@
 import static oracle.kubernetes.operator.logging.MessageKeys.WLS_HEALTH_READ_FAILED_NO_HTTPCLIENT;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.core.IsNull.nullValue;
 
 import com.meterware.simplestub.Memento;
 import com.meterware.simplestub.Stub;
@@ -19,6 +18,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
 import oracle.kubernetes.TestUtils;
@@ -91,17 +91,20 @@ public void withHttpClientStep_logIfMissingHTTPClient() {
     final String SERVER_NAME = ADMIN_NAME;
     Packet packet =
         Stub.createStub(PacketStub.class).withServerName(SERVER_NAME).withGetKeyReturnValue(null);
+    packet.put(ProcessingConstants.REMAINING_SERVERS_HEALTH_READ, new AtomicInteger(1));
 
     ReadHealthWithHttpClientStep withHttpClientStep =
         new ReadHealthWithHttpClientStep(service, null, next);
     withHttpClientStep.apply(packet);
 
     assertThat(logRecords, containsInfo(WLS_HEALTH_READ_FAILED_NO_HTTPCLIENT, SERVER_NAME));
-    assertThat(packet.get(ProcessingConstants.SERVER_HEALTH_READ), nullValue());
+    assertThat(
+        ((AtomicInteger) packet.get(ProcessingConstants.REMAINING_SERVERS_HEALTH_READ)).get(),
+        is(1));
   }
 
   @Test
-  public void withHttpClientStep_putServerHealthReadToPacketIfSucceeded() {
+  public void withHttpClientStep_decrmentServerHealthReadInPacketIfSucceeded() {
     V1Service service = Stub.createStub(V1ServiceStub.class);
     Step next = new MockStep(null);
     final String SERVER_NAME = ADMIN_NAME;
@@ -114,12 +117,15 @@ public void withHttpClientStep_putServerHealthReadToPacketIfSucceeded() {
             .withGetKeyReturnValue(httpClientStub);
     packet.put(
         ProcessingConstants.SERVER_HEALTH_MAP, new ConcurrentHashMap<String, ServerHealth>());
+    packet.put(ProcessingConstants.REMAINING_SERVERS_HEALTH_READ, new AtomicInteger(1));
 
     ReadHealthWithHttpClientStep withHttpClientStep =
         new ReadHealthWithHttpClientStep(service, null, next);
     withHttpClientStep.apply(packet);
 
-    assertThat(packet.get(ProcessingConstants.SERVER_HEALTH_READ), is(Boolean.TRUE));
+    assertThat(
+        ((AtomicInteger) packet.get(ProcessingConstants.REMAINING_SERVERS_HEALTH_READ)).get(),
+        is(0));
   }
 
   abstract static class PacketStub extends Packet {