Merge branch 'master' into rbac

Author: xxubai

amoro-ams/pom.xml

@@ -450,14 +450,12 @@
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
-            <version>1.20.1</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>junit-jupiter</artifactId>
-            <version>1.20.1</version>
             <scope>test</scope>
         </dependency>
 
@@ -471,14 +469,12 @@
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>k3s</artifactId>
-            <version>1.20.1</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>mysql</artifactId>
-            <version>1.20.1</version>
             <scope>test</scope>
         </dependency>
 

amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java

@@ -581,6 +581,23 @@ public class AmoroManagementConf {
           .defaultValue(Duration.ofSeconds(3))
           .withDescription("Optimizer polling task timeout.");
 
+  public static final ConfigOption<Duration> OPTIMIZER_GROUP_MIN_PARALLELISM_CHECK_INTERVAL =
+      ConfigOptions.key("optimizer-group.min-parallelism-check-interval")
+          .durationType()
+          .defaultValue(Duration.ofMinutes(5))
+          .withDescription(
+              "The interval for checking and ensuring the optimizer group meets its minimum parallelism requirement. "
+                  + "When the current parallelism falls below the configured min-parallelism, "
+                  + "the system will attempt to scale out optimizers at this interval. "
+                  + "The actual scale-out timing is calculated as: consecutive keeping attempts * this interval.");
+
+  public static final ConfigOption<Integer> OPTIMIZER_GROUP_MAX_KEEPING_ATTEMPTS =
+      ConfigOptions.key("optimizer-group.max-keeping-attempts")
+          .intType()
+          .defaultValue(3)
+          .withDescription(
+              "The maximum number of consecutive attempts to keep the optimizer group at its current parallelism.");
+
   public static final ConfigOption<Duration> OPTIMIZING_REFRESH_GROUP_INTERVAL =
       ConfigOptions.key("self-optimizing.refresh-group-interval")
           .durationType()

amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java

@@ -236,6 +236,7 @@ public void transitionToFollower() {
   }
 
   public void startOptimizingService() throws Exception {
+
     // Load process factories and build action coordinators from default table runtime factory.
     TableProcessFactoryManager tableProcessFactoryManager = new TableProcessFactoryManager();
     tableProcessFactoryManager.initialize();
@@ -246,6 +247,8 @@ public void startOptimizingService() throws Exception {
 
     List<ActionCoordinator> actionCoordinators = defaultRuntimeFactory.supportedCoordinators();
     ExecuteEngineManager executeEngineManager = new ExecuteEngineManager();
+    processFactories.forEach(
+        c -> c.availableExecuteEngines(executeEngineManager.installedPlugins()));
 
     tableService = new DefaultTableService(serviceConfig, catalogManager, defaultRuntimeFactory);
     processService = new ProcessService(tableService, actionCoordinators, executeEngineManager);