Limit introspection to once per activity to make rolling and other use cases faster

rjeberhard · rjeberhard · commit 5205d8b3d7a8 · 2024-05-30T19:41:49.000Z
diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainResourcesValidation.java b/operator/src/main/java/oracle/kubernetes/operator/DomainResourcesValidation.java
@@ -289,7 +289,6 @@ private boolean isActive(DomainPresenceInfo dpi) {
   }
 
   private void activateDomain(DomainProcessor dp, DomainPresenceInfo info) {
-    info.setPopulated(true);
     EventItem eventItem = getEventItem(info);
     MakeRightDomainOperation makeRight = dp.createMakeRightOperation(info).withExplicitRecheck();
     if (eventItem != null) {
diff --git a/operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java b/operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java
@@ -24,6 +24,8 @@ public interface LabelConstants {
   String INTROSPECTION_STATE_LABEL = "weblogic.introspectVersion";
   String MII_UPDATED_RESTART_REQUIRED_LABEL = "weblogic.configChangesPendingRestart";
   String INTROSPECTION_DOMAIN_SPEC_GENERATION = "weblogic.domainSpecGeneration";
+  String INTROSPECTION_CLUSTER_SPEC_GENERATION = "weblogic.clusterSpecGeneration";
+  String INTROSPECTION_TIME = "weblogic.introspectionTime";
   String TO_BE_ROLLED_LABEL = "weblogic.awaitingPodRoll";
   String TO_BE_SHUTDOWN_LABEL = "weblogic.awaitingShutdown";
   String DOMAIN_OBSERVED_GENERATION_LABEL = "weblogic.domainObservedGeneration";
diff --git a/operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java b/operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java
@@ -3,22 +3,29 @@
 
 package oracle.kubernetes.operator;
 
+import java.time.OffsetDateTime;
+import java.util.Collections;
 import java.util.Optional;
 import javax.annotation.Nonnull;
 
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.EventHelper.EventData;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
+import oracle.kubernetes.weblogic.domain.model.DomainCondition;
 import oracle.kubernetes.weblogic.domain.model.DomainResource;
+import oracle.kubernetes.weblogic.domain.model.DomainStatus;
 
+import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_TIME;
+import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_PRESENCE_INFO;
 import static oracle.kubernetes.operator.ProcessingConstants.MAKE_RIGHT_DOMAIN_OPERATION;
+import static oracle.kubernetes.weblogic.domain.model.DomainCondition.FALSE;
+import static oracle.kubernetes.weblogic.domain.model.DomainConditionType.COMPLETED;
 
 /**
  * Defines the operation to bring a running domain into compliance with its domain resource and introspection result.
  */
 public interface MakeRightDomainOperation extends MakeRightOperation<DomainPresenceInfo> {
-
   /**
    * Defines the operation as pertaining to the deletion of a domain.
    * @return The make right domain operation for deletion
@@ -64,6 +71,23 @@ public interface MakeRightDomainOperation extends MakeRightOperation<DomainPrese
   boolean wasInspectionRun();
 
   private static boolean wasInspectionRun(Packet packet) {
+    // check if introspection has run since Completed=False set
+    OffsetDateTime lastTransitionTime = Optional.ofNullable((DomainPresenceInfo) packet.get(DOMAIN_PRESENCE_INFO))
+        .map(DomainPresenceInfo::getDomain)
+        .map(DomainResource::getStatus).map(DomainStatus::getConditions).orElse(Collections.emptyList()).stream()
+        .filter(condition -> COMPLETED.equals(condition.getType()) && FALSE.equals(condition.getStatus())).findFirst()
+        .map(DomainCondition::getLastTransitionTime).orElse(null);
+
+    if (lastTransitionTime != null) {
+      String time = packet.getValue(INTROSPECTION_TIME);
+      if (time != null) {
+        OffsetDateTime lastIntrospectionTime = OffsetDateTime.parse(time.replace('_', ':'));
+        if (lastIntrospectionTime.isAfter(lastTransitionTime)) {
+          return true;
+        }
+      }
+    }
+
     return fromPacket(packet).map(MakeRightDomainOperation::wasInspectionRun).orElse(false);
   }
 
diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/Client.java b/operator/src/main/java/oracle/kubernetes/operator/calls/Client.java
@@ -4,6 +4,8 @@
 package oracle.kubernetes.operator.calls;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
 import java.util.concurrent.atomic.AtomicReference;
 
 import io.kubernetes.client.monitoring.Monitoring;
@@ -34,6 +36,13 @@ public static ApiClient getInstance() {
       try {
         LOGGER.fine(MessageKeys.CREATING_API_CLIENT);
         ApiClient client = factory.get();
+        String proxy = System.getenv("HTTPS_PROXY");
+        if (proxy != null) {
+          String[] components = proxy.split(":");
+          client.setHttpClient(client.getHttpClient().newBuilder()
+              .proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(components[0], Integer.valueOf(components[1]))))
+              .build());
+        }
         Monitoring.installMetrics(client);
         Configuration.setDefaultApiClient(client);
         return client;
diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java
@@ -59,8 +59,10 @@
 import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SECRETS_MD_5;
 import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SIT_CONFIG_FILE_PREFIX;
 import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME;
+import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_CLUSTER_SPEC_GENERATION;
 import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_DOMAIN_SPEC_GENERATION;
 import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
+import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_TIME;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_VALIDATION_ERRORS;
 import static oracle.kubernetes.operator.helpers.NamespaceHelper.getOperatorNamespace;
 import static oracle.kubernetes.operator.helpers.StepContextConstants.FLUENTBIT_CONFIGMAP_NAME_SUFFIX;
@@ -327,9 +329,16 @@ class ReadResponseStep extends DefaultResponseStep<V1ConfigMap> {
       public Result onSuccess(Packet packet, KubernetesApiResponse<V1ConfigMap> callResponse) {
         DomainResource domain = DomainPresenceInfo.fromPacket(packet).map(DomainPresenceInfo::getDomain).orElse(null);
         Optional.ofNullable(domain).map(DomainResource::getIntrospectVersion)
-              .ifPresent(value -> addLabel(INTROSPECTION_STATE_LABEL, value));
+            .ifPresent(value -> addLabel(INTROSPECTION_STATE_LABEL, value));
         Optional.ofNullable(domain).map(DomainResource::getMetadata).map(V1ObjectMeta::getGeneration)
-                .ifPresent(value -> addLabel(INTROSPECTION_DOMAIN_SPEC_GENERATION, value.toString()));
+            .ifPresent(value -> addLabel(INTROSPECTION_DOMAIN_SPEC_GENERATION, value.toString()));
+        DomainPresenceInfo.fromPacket(packet).map(DomainPresenceInfo::getReferencedClusters)
+            .ifPresent(list -> list.forEach(cluster -> Optional.ofNullable(cluster.getMetadata())
+                .map(V1ObjectMeta::getGeneration)
+                .ifPresent(value -> addLabel(INTROSPECTION_CLUSTER_SPEC_GENERATION + "."
+                    + cluster.getMetadata().getName(), value.toString()))));
+        Optional.ofNullable((String) packet.get(INTROSPECTION_TIME))
+                .ifPresent(value -> addLabel(INTROSPECTION_TIME, value));
         V1ConfigMap existingMap = withoutTransientData(callResponse.getObject());
         if (existingMap == null) {
           return doNext(createConfigMap(getNext()), packet);
@@ -918,6 +927,11 @@ private void recordIntrospectVersionAndGeneration(V1ConfigMap result, Packet pac
                       () -> packet.remove(INTROSPECTION_STATE_LABEL));
       Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_DOMAIN_SPEC_GENERATION))
               .ifPresent(generation -> packet.put(INTROSPECTION_DOMAIN_SPEC_GENERATION, generation));
+      Optional.ofNullable(labels).ifPresent(x -> x.entrySet().stream()
+              .filter(entry -> entry.getKey().startsWith(INTROSPECTION_CLUSTER_SPEC_GENERATION))
+              .forEach(entry -> packet.put(entry.getKey(), entry.getValue())));
+      Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_TIME))
+              .ifPresent(value -> packet.put(INTROSPECTION_TIME, value));
     }
 
     private String getTopologyYaml(Map<String, String> data) {
diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java
@@ -64,8 +64,10 @@
 import static oracle.kubernetes.operator.DomainStatusUpdater.createIntrospectionFailureSteps;
 import static oracle.kubernetes.operator.DomainStatusUpdater.createRemoveFailuresStep;
 import static oracle.kubernetes.operator.DomainStatusUpdater.createRemoveSelectedFailuresStep;
+import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_CLUSTER_SPEC_GENERATION;
 import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_DOMAIN_SPEC_GENERATION;
 import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
+import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_TIME;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECTION_COMPLETE;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECTOR_JOB;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECT_REQUESTED;
@@ -359,7 +361,8 @@ private Collection<String> getRunningServerNames() {
       }
 
       private boolean isBringingUpNewDomain(Packet packet) {
-        return getNumRunningServers() == 0 && creatingServers(info) && isDomainGenerationChanged(packet);
+        return getNumRunningServers() == 0 && creatingServers(info)
+           && (isDomainGenerationChanged(packet) || isAnyClusterGenerationChanged(packet));
       }
 
       private int getNumRunningServers() {
@@ -379,6 +382,31 @@ private String getDomainGeneration() {
               .orElse("");
       }
 
+      private boolean isAnyClusterGenerationChanged(Packet packet) {
+        List<ClusterResource> referencedClusters = info.getReferencedClusters();
+        if (referencedClusters.size() != packet.entrySet().stream()
+            .filter(entry -> entry.getKey().startsWith(INTROSPECTION_CLUSTER_SPEC_GENERATION)).count()) {
+          return true;
+        }
+
+        for (ClusterResource clusterResource : referencedClusters) {
+          if (Optional.ofNullable(packet.get(INTROSPECTION_CLUSTER_SPEC_GENERATION
+                  + "." + clusterResource.getMetadata().getName()))
+                  .map(gen -> !gen.equals(getClusterGeneration(clusterResource))).orElse(true)) {
+            return true;
+          }
+        }
+
+        return false;
+      }
+
+      private String getClusterGeneration(ClusterResource clusterResource) {
+        return Optional.ofNullable(clusterResource.getMetadata())
+                .map(V1ObjectMeta::getGeneration)
+                .map(Object::toString)
+                .orElse("");
+      }
+
       // Returns true if an introspection was requested. Clears the flag in any case.
       private boolean isIntrospectionRequested(Packet packet) {
         return packet.remove(DOMAIN_INTROSPECT_REQUESTED) != null;
@@ -509,6 +537,9 @@ private class ReadPodLogStep extends Step {
 
         String jobPodName = JobHelper.getName(jobPod);
 
+        Optional.ofNullable(jobPod).map(V1Pod::getMetadata)
+            .map(V1ObjectMeta::getCreationTimestamp).map(OffsetDateTime::toString)
+                .ifPresent(creationTime -> packet.put(INTROSPECTION_TIME, creationTime.replace(':', '_')));
         return doNext(readDomainIntrospectorPodLog(jobPodName, containerName, getNext()), packet);
       }
 

Original file line number	Diff line number	Diff line change
`@@ -289,7 +289,6 @@ private boolean isActive(DomainPresenceInfo dpi) {`
`289`	`289`	`}`
`290`	`290`
`291`	`291`	`private void activateDomain(DomainProcessor dp, DomainPresenceInfo info) {`
`292`		`- info.setPopulated(true);`
`293`	`292`	`EventItem eventItem = getEventItem(info);`
`294`	`293`	`MakeRightDomainOperation makeRight = dp.createMakeRightOperation(info).withExplicitRecheck();`
`295`	`294`	`if (eventItem != null) {`