[FLINK-36535][autoscaler] Optimize the scale down logic based on historical parallelism to reduce the rescale frequency

1. Using the maximum parallelism within the window instead of the latest parallelism when scaling down
2. Never scale down when (currentTime - triggerTime) < scale-down.interval

Author: 1996fanrui

flink-autoscaler-plugin-jdbc/src/main/java/org/apache/flink/autoscaler/jdbc/state/JdbcAutoScalerStateStore.java

@@ -237,7 +237,7 @@ public DelayedScaleDown getDelayedScaleDown(Context jobContext) {
         try {
             return deserializeDelayedScaleDown(delayedScaleDown.get());
         } catch (JacksonException e) {
-            LOG.error(
+            LOG.warn(
                     "Could not deserialize delayed scale down, possibly the format changed. Discarding...",
                     e);
             jdbcStateStore.removeSerializedState(getSerializeKey(jobContext), DELAYED_SCALE_DOWN);
@@ -330,13 +330,11 @@ private static ConfigChanges deserializeConfigOverrides(String configOverrides)
 
     private static String serializeDelayedScaleDown(DelayedScaleDown delayedScaleDown)
             throws JacksonException {
-        return YAML_MAPPER.writeValueAsString(delayedScaleDown.getFirstTriggerTime());
+        return YAML_MAPPER.writeValueAsString(delayedScaleDown);
     }
 
     private static DelayedScaleDown deserializeDelayedScaleDown(String delayedScaleDown)
             throws JacksonException {
-        Map<JobVertexID, Instant> firstTriggerTime =
-                YAML_MAPPER.readValue(delayedScaleDown, new TypeReference<>() {});
-        return new DelayedScaleDown(firstTriggerTime);
+        return YAML_MAPPER.readValue(delayedScaleDown, new TypeReference<>() {});
     }
 }

flink-autoscaler/src/main/java/org/apache/flink/autoscaler/DelayedScaleDown.java

@@ -19,51 +19,81 @@
 
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Data;
 import lombok.Getter;
 
+import javax.annotation.Nonnull;
+
 import java.time.Instant;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
 
 /** All delayed scale down requests. */
 public class DelayedScaleDown {
 
-    @Getter private final Map<JobVertexID, Instant> firstTriggerTime;
+    /** The delayed scale down info for vertex. */
+    @Data
+    public static class VertexDelayedScaleDownInfo {
+        private final Instant firstTriggerTime;
+        private int maxRecommendedParallelism;
+
+        @JsonCreator
+        public VertexDelayedScaleDownInfo(
+                @JsonProperty("firstTriggerTime") Instant firstTriggerTime,
+                @JsonProperty("maxRecommendedParallelism") int maxRecommendedParallelism) {
+            this.firstTriggerTime = firstTriggerTime;
+            this.maxRecommendedParallelism = maxRecommendedParallelism;
+        }
+    }
+
+    @Getter private final Map<JobVertexID, VertexDelayedScaleDownInfo> delayedVertices;
 
     // Have any scale down request been updated? It doesn't need to be stored, it is only used to
     // determine whether DelayedScaleDown needs to be stored.
-    @Getter private boolean isUpdated = false;
+    @JsonIgnore @Getter private boolean updated = false;
 
     public DelayedScaleDown() {
-        this.firstTriggerTime = new HashMap<>();
-    }
-
-    public DelayedScaleDown(Map<JobVertexID, Instant> firstTriggerTime) {
-        this.firstTriggerTime = firstTriggerTime;
+        this.delayedVertices = new HashMap<>();
     }
 
-    Optional<Instant> getFirstTriggerTimeForVertex(JobVertexID vertex) {
-        return Optional.ofNullable(firstTriggerTime.get(vertex));
-    }
+    /** Trigger a scale down, and return the corresponding {@link VertexDelayedScaleDownInfo}. */
+    @Nonnull
+    public VertexDelayedScaleDownInfo triggerScaleDown(
+            JobVertexID vertex, Instant triggerTime, int parallelism) {
+        var vertexInfo = delayedVertices.get(vertex);
+        if (vertexInfo == null) {
+            // It's the first trigger
+            vertexInfo = new VertexDelayedScaleDownInfo(triggerTime, parallelism);
+            delayedVertices.put(vertex, vertexInfo);
+            updated = true;
+        } else if (parallelism > vertexInfo.getMaxRecommendedParallelism()) {
+            // Not the first trigger, but the maxRecommendedParallelism needs to be updated.
+            vertexInfo.setMaxRecommendedParallelism(parallelism);
+            updated = true;
+        }
 
-    void updateTriggerTime(JobVertexID vertex, Instant instant) {
-        firstTriggerTime.put(vertex, instant);
-        isUpdated = true;
+        return vertexInfo;
     }
 
+    // Clear the delayed scale down for corresponding vertex when the recommended parallelism is
+    // greater than or equal to the currentParallelism.
     void clearVertex(JobVertexID vertex) {
-        Instant removed = firstTriggerTime.remove(vertex);
+        VertexDelayedScaleDownInfo removed = delayedVertices.remove(vertex);
         if (removed != null) {
-            isUpdated = true;
+            updated = true;
         }
     }
 
+    // Clear all delayed scale down when rescale happens.
     void clearAll() {
-        if (firstTriggerTime.isEmpty()) {
+        if (delayedVertices.isEmpty()) {
             return;
         }
-        firstTriggerTime.clear();
-        isUpdated = true;
+        delayedVertices.clear();
+        updated = true;
     }
 }

flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobVertexScaler.java

@@ -88,31 +88,22 @@ public JobVertexScaler(AutoScalerEventHandler<KEY, Context> autoScalerEventHandl
         this.autoScalerEventHandler = autoScalerEventHandler;
     }
 
-    /** The parallelism change type of {@link ParallelismChange}. */
-    public enum ParallelismChangeType {
-        NO_CHANGE,
-        REQUIRED_CHANGE,
-        OPTIONAL_CHANGE;
-    }
-
-    /**
-     * The rescaling will be triggered if any vertex's ParallelismChange is required. This means
-     * that if all vertices' ParallelismChange is optional, rescaling will be ignored.
-     */
+    /** The rescaling will be triggered if any vertex's {@link ParallelismResult} is changed. */
     @Getter
-    public static class ParallelismChange {
+    public static class ParallelismResult {
 
-        private static final ParallelismChange NO_CHANGE =
-                new ParallelismChange(ParallelismChangeType.NO_CHANGE, -1);
+        private static final ParallelismResult NO_CHANGE = new ParallelismResult(-1);
 
-        private final ParallelismChangeType changeType;
         private final int newParallelism;
 
-        private ParallelismChange(ParallelismChangeType changeType, int newParallelism) {
-            this.changeType = changeType;
+        private ParallelismResult(int newParallelism) {
             this.newParallelism = newParallelism;
         }
 
+        public boolean isNoChange() {
+            return this == NO_CHANGE;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o) {
@@ -121,39 +112,33 @@ public boolean equals(Object o) {
             if (o == null || getClass() != o.getClass()) {
                 return false;
             }
-            ParallelismChange that = (ParallelismChange) o;
-            return changeType == that.changeType && newParallelism == that.newParallelism;
+            ParallelismResult that = (ParallelismResult) o;
+            return newParallelism == that.newParallelism;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(changeType, newParallelism);
+            return Objects.hash(newParallelism);
         }
 
         @Override
         public String toString() {
-            return "ParallelismChange{"
-                    + "changeType="
-                    + changeType
-                    + ", newParallelism="
-                    + newParallelism
-                    + '}';
-        }
-
-        public static ParallelismChange required(int newParallelism) {
-            return new ParallelismChange(ParallelismChangeType.REQUIRED_CHANGE, newParallelism);
+            return isNoChange()
+                    ? "NoParallelismChange"
+                    : "ParallelismChange{newParallelism=" + newParallelism + '}';
         }
 
-        public static ParallelismChange optional(int newParallelism) {
-            return new ParallelismChange(ParallelismChangeType.OPTIONAL_CHANGE, newParallelism);
+        public static ParallelismResult build(int newParallelism) {
+            checkArgument(newParallelism > 0, "The parallelism should be greater than 0.");
+            return new ParallelismResult(newParallelism);
         }
 
-        public static ParallelismChange noChange() {
+        public static ParallelismResult noChange() {
             return NO_CHANGE;
         }
     }
 
-    public ParallelismChange computeScaleTargetParallelism(
+    public ParallelismResult computeScaleTargetParallelism(
             Context context,
             JobVertexID vertex,
             Collection<ShipStrategy> inputShipStrategies,
@@ -168,7 +153,7 @@ public ParallelismChange computeScaleTargetParallelism(
             LOG.warn(
                     "True processing rate is not available for {}, cannot compute new parallelism",
                     vertex);
-            return ParallelismChange.noChange();
+            return ParallelismResult.noChange();
         }
 
         double targetCapacity =
@@ -178,7 +163,7 @@ public ParallelismChange computeScaleTargetParallelism(
             LOG.warn(
                     "Target data rate is not available for {}, cannot compute new parallelism",
                     vertex);
-            return ParallelismChange.noChange();
+            return ParallelismResult.noChange();
         }
 
         LOG.debug("Target processing capacity for {} is {}", vertex, targetCapacity);
@@ -222,7 +207,7 @@ public ParallelismChange computeScaleTargetParallelism(
             // Clear delayed scale down request if the new parallelism is equal to
             // currentParallelism.
             delayedScaleDown.clearVertex(vertex);
-            return ParallelismChange.noChange();
+            return ParallelismResult.noChange();
         }
 
         // We record our expectations for this scaling operation
@@ -241,7 +226,7 @@ public ParallelismChange computeScaleTargetParallelism(
                 delayedScaleDown);
     }
 
-    private ParallelismChange detectBlockScaling(
+    private ParallelismResult detectBlockScaling(
             Context context,
             JobVertexID vertex,
             Configuration conf,
@@ -263,7 +248,7 @@ private ParallelismChange detectBlockScaling(
 
             // If we don't have past scaling actions for this vertex, don't block scale up.
             if (history.isEmpty()) {
-                return ParallelismChange.required(newParallelism);
+                return ParallelismResult.build(newParallelism);
             }
 
             var lastSummary = history.get(history.lastKey());
@@ -272,38 +257,40 @@ private ParallelismChange detectBlockScaling(
                     && detectIneffectiveScaleUp(
                             context, vertex, conf, evaluatedMetrics, lastSummary)) {
                 // Block scale up when last rescale is ineffective.
-                return ParallelismChange.noChange();
+                return ParallelismResult.noChange();
             }
 
-            return ParallelismChange.required(newParallelism);
+            return ParallelismResult.build(newParallelism);
         } else {
             return applyScaleDownInterval(delayedScaleDown, vertex, conf, newParallelism);
         }
     }
 
-    private ParallelismChange applyScaleDownInterval(
+    private ParallelismResult applyScaleDownInterval(
             DelayedScaleDown delayedScaleDown,
             JobVertexID vertex,
             Configuration conf,
             int newParallelism) {
         var scaleDownInterval = conf.get(SCALE_DOWN_INTERVAL);
         if (scaleDownInterval.toMillis() <= 0) {
             // The scale down interval is disable, so don't block scaling.
-            return ParallelismChange.required(newParallelism);
+            return ParallelismResult.build(newParallelism);
         }
 
-        var firstTriggerTime = delayedScaleDown.getFirstTriggerTimeForVertex(vertex);
-        if (firstTriggerTime.isEmpty()) {
-            LOG.info("The scale down of {} is delayed by {}.", vertex, scaleDownInterval);
-            delayedScaleDown.updateTriggerTime(vertex, clock.instant());
-            return ParallelismChange.optional(newParallelism);
-        }
+        var now = clock.instant();
+        var delayedScaleDownInfo = delayedScaleDown.triggerScaleDown(vertex, now, newParallelism);
 
-        if (clock.instant().isBefore(firstTriggerTime.get().plus(scaleDownInterval))) {
-            LOG.debug("Try to skip immediate scale down within scale-down interval for {}", vertex);
-            return ParallelismChange.optional(newParallelism);
+        if (now.isBefore(delayedScaleDownInfo.getFirstTriggerTime().plus(scaleDownInterval))) {
+            if (now.equals(delayedScaleDownInfo.getFirstTriggerTime())) {
+                LOG.info("The scale down of {} is delayed by {}.", vertex, scaleDownInterval);
+            } else {
+                LOG.debug(
+                        "Try to skip immediate scale down within scale-down interval for {}",
+                        vertex);
+            }
+            return ParallelismResult.noChange();
         } else {
-            return ParallelismChange.required(newParallelism);
+            return ParallelismResult.build(delayedScaleDownInfo.getMaxRecommendedParallelism());
         }
     }
 

flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java

@@ -45,13 +45,10 @@
 import java.time.Instant;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
 
-import static org.apache.flink.autoscaler.JobVertexScaler.ParallelismChangeType.NO_CHANGE;
-import static org.apache.flink.autoscaler.JobVertexScaler.ParallelismChangeType.REQUIRED_CHANGE;
 import static org.apache.flink.autoscaler.config.AutoScalerOptions.EXCLUDED_PERIODS;
 import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
 import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALING_EVENT_INTERVAL;
@@ -181,15 +178,15 @@ private void updateRecommendedParallelism(
     }
 
     @VisibleForTesting
-    static boolean allRequiredVerticesWithinUtilizationTarget(
+    static boolean allChangedVerticesWithinUtilizationTarget(
             Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluatedMetrics,
-            Set<JobVertexID> requiredVertices) {
-        // All vertices' ParallelismChange is optional, rescaling will be ignored.
-        if (requiredVertices.isEmpty()) {
+            Set<JobVertexID> changedVertices) {
+        // No any vertex is changed.
+        if (changedVertices.isEmpty()) {
             return true;
         }
 
-        for (JobVertexID vertex : requiredVertices) {
+        for (var vertex : changedVertices) {
             var metrics = evaluatedMetrics.get(vertex);
 
             double trueProcessingRate = metrics.get(TRUE_PROCESSING_RATE).getAverage();
@@ -234,7 +231,6 @@ Map<JobVertexID, ScalingSummary> computeScalingSummary(
         }
 
         var out = new HashMap<JobVertexID, ScalingSummary>();
-        var requiredVertices = new HashSet<JobVertexID>();
 
         var excludeVertexIdList =
                 context.getConfiguration().get(AutoScalerOptions.VERTEX_EXCLUDE_IDS);
@@ -250,7 +246,7 @@ Map<JobVertexID, ScalingSummary> computeScalingSummary(
                                 var currentParallelism =
                                         (int) metrics.get(ScalingMetric.PARALLELISM).getCurrent();
 
-                                var parallelismChange =
+                                var parallelismResult =
                                         jobVertexScaler.computeScaleTargetParallelism(
                                                 context,
                                                 v,
@@ -260,24 +256,21 @@ Map<JobVertexID, ScalingSummary> computeScalingSummary(
                                                         v, Collections.emptySortedMap()),
                                                 restartTime,
                                                 delayedScaleDown);
-                                if (NO_CHANGE == parallelismChange.getChangeType()) {
+                                if (parallelismResult.isNoChange()) {
                                     return;
-                                } else if (REQUIRED_CHANGE == parallelismChange.getChangeType()) {
-                                    requiredVertices.add(v);
                                 }
                                 out.put(
                                         v,
                                         new ScalingSummary(
                                                 currentParallelism,
-                                                parallelismChange.getNewParallelism(),
+                                                parallelismResult.getNewParallelism(),
                                                 metrics));
                             }
                         });
 
-        // If the Utilization of all required tasks is within range, we can skip scaling.
-        // It means that if only optional tasks are out of scope, we still need to ignore scale.
-        if (allRequiredVerticesWithinUtilizationTarget(
-                evaluatedMetrics.getVertexMetrics(), requiredVertices)) {
+        // If the Utilization of all tasks is within range, we can skip scaling.
+        if (allChangedVerticesWithinUtilizationTarget(
+                evaluatedMetrics.getVertexMetrics(), out.keySet())) {
             return Map.of();
         }