[FLINK-35815][autoscaler] Autoscaler standalone load config from yaml file (#853)

Author: huyuanfeng2018

docs/content.zh/docs/custom-resource/autoscaler.md

@@ -308,6 +308,12 @@ org.apache.flink.autoscaler.standalone.StandaloneAutoscalerEntrypoint \
 --autoscaler.standalone.jdbc.username root
 ```
 
+### Load StandaloneAutoscaler configuration through configuration file
+
+Can specify the running configuration of StandaloneAutoscaler in the configuration file. The config.yaml and
+flink-conf.yaml files in the environment variable `FLINK_CONF_DIR` directory will be loaded by default.
+NOTE: main parameters will overwrite the configuration in the configuration file
+
 All supported options for autoscaler standalone can be viewed
 [here]({{< ref "docs/operations/configuration#autoscaler-standalone-configuration" >}}).
 

docs/content/docs/custom-resource/autoscaler.md

@@ -308,6 +308,12 @@ org.apache.flink.autoscaler.standalone.StandaloneAutoscalerEntrypoint \
 --autoscaler.standalone.jdbc.username root
 ```
 
+### Load StandaloneAutoscaler configuration through configuration file
+
+Can specify the running configuration of StandaloneAutoscaler in the configuration file. The config.yaml and
+flink-conf.yaml files in the environment variable `FLINK_CONF_DIR` directory will be loaded by default.
+NOTE: main parameters will overwrite the configuration in the configuration file
+
 All supported options for autoscaler standalone can be viewed
 [here]({{< ref "docs/operations/configuration#autoscaler-standalone-configuration" >}}).
 

flink-autoscaler-standalone/pom.xml

@@ -215,6 +215,15 @@ under the License.
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <environmentVariables>
+                        <FLINK_CONF_DIR>src/test/resources</FLINK_CONF_DIR>
+                    </environmentVariables>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

flink-autoscaler-standalone/src/main/java/org/apache/flink/autoscaler/standalone/StandaloneAutoscalerEntrypoint.java

@@ -18,6 +18,7 @@
 package org.apache.flink.autoscaler.standalone;
 
 import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.java.utils.ParameterTool;
 import org.apache.flink.autoscaler.JobAutoScaler;
 import org.apache.flink.autoscaler.JobAutoScalerContext;
@@ -31,6 +32,7 @@
 import org.apache.flink.autoscaler.state.AutoScalerStateStore;
 import org.apache.flink.client.program.rest.RestClusterClient;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneClientHAServices;
 
 import org.slf4j.Logger;
@@ -48,7 +50,7 @@ public class StandaloneAutoscalerEntrypoint {
 
     public static <KEY, Context extends JobAutoScalerContext<KEY>> void main(String[] args)
             throws Exception {
-        var conf = ParameterTool.fromArgs(args).getConfiguration();
+        var conf = loadConfiguration(args);
         LOG.info("The standalone autoscaler is started, configuration: {}", conf);
 
         // Initialize JobListFetcher and JobAutoScaler.
@@ -94,4 +96,10 @@ JobAutoScaler<KEY, Context> createJobAutoscaler(
                 new RescaleApiScalingRealizer<>(eventHandler),
                 stateStore);
     }
+
+    @VisibleForTesting
+    static Configuration loadConfiguration(String[] args) {
+        return GlobalConfiguration.loadConfiguration(
+                ParameterTool.fromArgs(args).getConfiguration());
+    }
 }

flink-autoscaler-standalone/src/test/java/org/apache/flink/autoscaler/standalone/StandaloneAutoscalerEntrypointTest.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.standalone;
+
+import org.apache.flink.configuration.Configuration;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+
+import static org.apache.flink.autoscaler.standalone.config.AutoscalerStandaloneOptions.CONTROL_LOOP_INTERVAL;
+import static org.apache.flink.autoscaler.standalone.config.AutoscalerStandaloneOptions.CONTROL_LOOP_PARALLELISM;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+class StandaloneAutoscalerEntrypointTest {
+    @Test
+    public void testLoadConfiguration() {
+        // Test for loading configuration from file.
+        Configuration conf = StandaloneAutoscalerEntrypoint.loadConfiguration(new String[0]);
+        assertNotNull(conf);
+        assertEquals(Duration.ofMinutes(1), conf.get(CONTROL_LOOP_INTERVAL));
+        assertEquals(20, conf.get(CONTROL_LOOP_PARALLELISM));
+        // Test for args override
+        String[] args =
+                new String[] {
+                    "--autoscaler.standalone.control-loop.interval",
+                    "2min",
+                    "--autoscaler" + ".standalone.control-loop.parallelism",
+                    "10"
+                };
+        Configuration confOverride = StandaloneAutoscalerEntrypoint.loadConfiguration(args);
+        assertNotNull(confOverride);
+        assertEquals(Duration.ofMinutes(2), confOverride.get(CONTROL_LOOP_INTERVAL));
+        assertEquals(10, confOverride.get(CONTROL_LOOP_PARALLELISM));
+    }
+}

flink-autoscaler-standalone/src/test/resources/config.yaml

@@ -0,0 +1,20 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+autoscaler.standalone.control-loop.interval: 60s
+autoscaler.standalone.control-loop.parallelism: 20