Owls 118470 topology spread constraints error report

jshum2479 · rjeberhard · commit 4e57b3af61fb · 2024-06-05T20:01:41.000Z
diff --git a/common/src/main/java/oracle/kubernetes/common/logging/MessageKeys.java b/common/src/main/java/oracle/kubernetes/common/logging/MessageKeys.java
@@ -222,6 +222,8 @@ public class MessageKeys {
   public static final String NO_MATCH_VOLUME_WITH_PVC = "WLSDO-0063";
   public static final String NO_VOLUME_WITH_PVC = "WLSDO-0064";
   public static final String WALLET_KEY_NOT_FOUND = "WLSDO-0065";
+  public static final String POD_UNSCHEDULABLE = "WLSDO-0066";
+  public static final String POD_UNSCHEDULABLE_MESSAGE = "WLSDO-0067";
 
   // domain event messages
   public static final String DOMAIN_AVAILABLE_EVENT_PATTERN = "WLSEO-0001";
diff --git a/common/src/main/resources/Operator.properties b/common/src/main/resources/Operator.properties
@@ -262,6 +262,8 @@ WLSDO-0064=When ''spec.configuration.initializeDomainOnPV'' is specified to auto
   but not the PersistentVolumeClaim (PVC), at least one of the volumes in ''spec.serverPod.volumes'' should contain a PVC.
 WLSDO-0065=The OPSS wallet password secret ''{0}'' is specified but the required ''walletPassword'' key is missing \
   in the specified secret ''{1}''.
+WLSDO-0066=Pod ''{0}'' is Unschedulable, reason: ''{1}''
+WLSDO-0067=One or more pods in the domain cannot be scheduled. Please check individual pod status for details.
 
 oneEnvVar=variable
 multipleEnvVars=variables
diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java b/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java
@@ -24,7 +24,9 @@
 import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
 import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimStatus;
 import io.kubernetes.client.openapi.models.V1Pod;
+import io.kubernetes.client.openapi.models.V1PodCondition;
 import io.kubernetes.client.openapi.models.V1PodDisruptionBudget;
+import io.kubernetes.client.openapi.models.V1PodStatus;
 import io.kubernetes.client.openapi.models.V1Service;
 import io.kubernetes.client.util.Watch;
 import oracle.kubernetes.common.logging.LoggingFilter;
@@ -64,9 +66,12 @@
 import oracle.kubernetes.weblogic.domain.model.ServerStatus;
 import org.jetbrains.annotations.NotNull;
 
+import static oracle.kubernetes.common.logging.MessageKeys.POD_UNSCHEDULABLE;
 import static oracle.kubernetes.common.logging.MessageKeys.PVC_NOT_BOUND_ERROR;
 import static oracle.kubernetes.operator.DomainStatusUpdater.createInternalFailureSteps;
 import static oracle.kubernetes.operator.DomainStatusUpdater.createIntrospectionFailureSteps;
+import static oracle.kubernetes.operator.KubernetesConstants.POD_SCHEDULED;
+import static oracle.kubernetes.operator.KubernetesConstants.UNSCHEDULABLE_REASON;
 import static oracle.kubernetes.operator.ProcessingConstants.SERVER_HEALTH_MAP;
 import static oracle.kubernetes.operator.ProcessingConstants.SERVER_STATE_MAP;
 import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.CLUSTER_CHANGED;
@@ -582,6 +587,10 @@ private void processServerPodWatch(V1Pod pod, String watchType) {
         if ((isEvicted || isReady != isLabedlForShutdown || PodHelper.isFailed(pod)) && !PodHelper.isDeleting(pod)) {
           createMakeRightOperation(info).interrupt().withExplicitRecheck().execute();
         }
+        boolean isUnschedulable = PodHelper.hasUnSchedulableCondition(pod);
+        if (isUnschedulable) {
+          LOGGER.info(POD_UNSCHEDULABLE,  getPodName(pod), getUnSchedulableConditionMessage(pod));
+        }
         break;
       case DELETED:
         boolean removed = info.deleteServerPodFromEvent(serverName, pod);
@@ -596,6 +605,25 @@ private void processServerPodWatch(V1Pod pod, String watchType) {
     }
   }
 
+  /**
+   * If a pod is unschedulable, return the condition's message.
+   * @param pod Kubernetes V1Pod
+   * @return message for the unschedulable pod condition
+   */
+  public static String getUnSchedulableConditionMessage(V1Pod pod) {
+    return Optional.ofNullable(pod)
+            .filter(PodHelper::isPending)
+            .map(V1Pod::getStatus)
+            .map(V1PodStatus::getConditions)
+            .orElse(Collections.emptyList())
+            .stream()
+            .filter(condition -> POD_SCHEDULED.equals(condition.getType())
+                    && UNSCHEDULABLE_REASON.equals(condition.getReason()))
+            .map(V1PodCondition::getMessage)
+            .findFirst().orElse(null);
+  }
+
+
   private String getPodLabel(V1Pod pod, String labelName) {
     return Optional.ofNullable(pod)
         .map(V1Pod::getMetadata)
diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java b/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java
@@ -74,6 +74,7 @@
 import static oracle.kubernetes.common.logging.MessageKeys.PODS_FAILED;
 import static oracle.kubernetes.common.logging.MessageKeys.PODS_NOT_READY;
 import static oracle.kubernetes.common.logging.MessageKeys.PODS_NOT_RUNNING;
+import static oracle.kubernetes.common.logging.MessageKeys.POD_UNSCHEDULABLE_MESSAGE;
 import static oracle.kubernetes.operator.ClusterResourceStatusUpdater.createClusterResourceStatusUpdaterStep;
 import static oracle.kubernetes.operator.KubernetesConstants.HTTP_NOT_FOUND;
 import static oracle.kubernetes.operator.KubernetesConstants.MINIMUM_CLUSTER_COUNT;
@@ -744,6 +745,11 @@ private void setStatusConditions(DomainStatus status) {
         if (isHasFailedPod()) {
           addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD)
               .withFailureInfo(getDomain().getSpec()).withMessage(getPodFailedMessage()));
+        } else if (isPodUnSchedulable()) {
+          if (!alreadyReportedPodUnSchedulableCondition(status)) {
+            addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD)
+                    .withFailureInfo(getDomain().getSpec()).withMessage(getPodUnSchedulableMessage()));
+          }
         } else if (hasPodNotRunningInTime()) {
           addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD)
               .withFailureInfo(getDomain().getSpec()).withMessage(getPodNotRunningMessage()));
@@ -770,6 +776,10 @@ private String getPodNotReadyMessage() {
         return LOGGER.formatMessage(PODS_NOT_READY);
       }
 
+      private String getPodUnSchedulableMessage() {
+        return LOGGER.formatMessage(POD_UNSCHEDULABLE_MESSAGE);
+      }
+
       private String getPodFailedMessage() {
         return LOGGER.formatMessage(PODS_FAILED);
       }
@@ -1251,6 +1261,11 @@ private boolean stillHasPodPendingRestart(DomainStatus status) {
             .anyMatch(m -> m.containsKey(LabelConstants.MII_UPDATED_RESTART_REQUIRED_LABEL));
       }
 
+      private boolean alreadyReportedPodUnSchedulableCondition(DomainStatus status) {
+        return status.getConditions().stream().anyMatch(
+                condition -> getPodUnSchedulableMessage().equals(condition.getMessage()));
+      }
+
       private V1Pod getServerPod(ServerStatus serverStatus) {
         return getInfo().getServerPod(serverStatus.getServerName());
       }
@@ -1318,6 +1333,14 @@ private boolean hasPodNotRunningInTime() {
         return getInfo().getServerPodsNotBeingDeleted().anyMatch(this::isNotRunningInTime);
       }
 
+      private boolean isPodUnSchedulable() {
+        return getInfo().getServerPodsNotBeingDeleted().anyMatch(this::isPodUnSchedulable);
+      }
+
+      private boolean isPodUnSchedulable(V1Pod pod) {
+        return PodHelper.isPending(pod) && PodHelper.hasUnSchedulableCondition(pod);
+      }
+
       private boolean isNotRunningInTime(V1Pod pod) {
         return PodHelper.isPending(pod) && hasBeenPendingExceededWaitTime(pod);
       }
diff --git a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java
@@ -56,7 +56,8 @@ public interface KubernetesConstants {
   String NAMESPACE = "Namespace";
   String POD = "Pod";
   String EVICTED_REASON = "Evicted";
-
+  String UNSCHEDULABLE_REASON = "Unschedulable";
+  String POD_SCHEDULED = "PodScheduled";
   int DEFAULT_EXPORTER_SIDECAR_PORT = 8080;
 
   //---------- HTTP statuses returned from Kubernetes ----------
diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java
@@ -51,6 +51,8 @@
 
 import static oracle.kubernetes.operator.KubernetesConstants.EVICTED_REASON;
 import static oracle.kubernetes.operator.KubernetesConstants.HTTP_NOT_FOUND;
+import static oracle.kubernetes.operator.KubernetesConstants.POD_SCHEDULED;
+import static oracle.kubernetes.operator.KubernetesConstants.UNSCHEDULABLE_REASON;
 import static oracle.kubernetes.operator.LabelConstants.CLUSTERNAME_LABEL;
 import static oracle.kubernetes.operator.LabelConstants.SERVERNAME_LABEL;
 import static oracle.kubernetes.operator.ProcessingConstants.SERVERS_TO_ROLL;
@@ -214,6 +216,11 @@ private static boolean isReadyCondition(V1PodCondition condition) {
     return "Ready".equals(condition.getType()) && "True".equals(condition.getStatus());
   }
 
+  private static boolean isUnSchedulableTheReason(V1PodCondition condition) {
+    return POD_SCHEDULED.equals(condition.getType()) && "False".equals(condition.getStatus())
+            && UNSCHEDULABLE_REASON.equals(condition.getReason());
+  }
+
   private static boolean isReadyNotTrueCondition(V1PodCondition condition) {
     return "Ready".equals(condition.getType()) && !"True".equals(condition.getStatus());
   }
@@ -299,6 +306,21 @@ public static boolean shouldRestartEvictedPod(V1Pod pod) {
     return isEvicted(pod) && TuningParameters.getInstance().isRestartEvictedPods();
   }
 
+  /**
+   * Return true if the pod has unschedulable condition.
+   * @param pod  Kubernetes Pod
+   * @return true if the pod is unschedulable
+   */
+  public static boolean hasUnSchedulableCondition(V1Pod pod) {
+    return Optional.ofNullable(pod)
+            .filter(PodHelper::isPending)
+            .map(V1Pod::getStatus)
+            .map(V1PodStatus::getConditions)
+            .orElse(Collections.emptyList())
+            .stream()
+            .anyMatch(PodHelper::isUnSchedulableTheReason);
+  }
+
   /**
    * Returns the domain UID associated with the specified pod.
    * @param pod the pod
diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainStatusUpdateTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/DomainStatusUpdateTestBase.java
@@ -238,6 +238,22 @@ private V1Pod getPod(String serverName) {
     return info.getServerPod(serverName);
   }
 
+  private void setPodUnschedulable(String serverName) {
+    V1Pod pod = getPod(serverName);
+    V1PodStatus status = pod.getStatus();
+    if (status == null) {
+      status = new V1PodStatus();
+    }
+    pod.setStatus(status
+            .startTime(SystemClock.now())
+            .phase("Pending")
+            .addConditionsItem(new V1PodCondition().type("PodScheduled").status("False").reason("Unschedulable")
+                    .message("0/1 nodes are available: 1 node(s) didn't match pod topology"
+                            + " spread constraints (missing required label). preemption: 0/1 nodes are available: "
+                            + "1 Preemption is not helpful for scheduling.."))
+    );
+  }
+
   @Test
   void statusStep_usesServerFromWlsConfig() {
     defineScenario()
@@ -1032,6 +1048,20 @@ void whenPodPendingWithinTimeLimit_removePreviousServerPodFailures() {
     assertThat(getRecordedDomain(), not(hasCondition(FAILED)));
   }
 
+  @Test
+  void whenPodIsUnschedulable_reportServerPodFailure() {
+    defineScenario()
+            .withServerState("server1", new V1ContainerStateWaiting().reason(null))
+            .build();
+    setPodUnschedulable("server1");
+
+    SystemClockTestSupport.increment(21);
+    updateDomainStatus();
+    assertThat(getRecordedDomain(), hasCondition(FAILED).withReason(SERVER_POD)
+            .withMessageContaining("One or more pods in the domain cannot be scheduled."));
+  }
+
+
   // todo remove server pod failures when OK
 
   @Test
