Adjust default data region num per node from 5 to to cpu core / 2 (apache#14348)

* finish

Signed-off-by: OneSizeFitQuorum <[email protected]>

* fix it & ut

Signed-off-by: OneSizeFitQuorum <[email protected]>

---------

Signed-off-by: OneSizeFitQuorum <[email protected]>

Author: OneSizeFitsQuorum

integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java

@@ -162,12 +162,10 @@ public void showClusterAndNodesTest() throws Exception {
           clusterParameters.getSchemaReplicationFactor());
       Assert.assertEquals(
           expectedParameters.getDataRegionPerDataNode(),
-          clusterParameters.getDataRegionPerDataNode(),
-          0.01);
+          clusterParameters.getDataRegionPerDataNode());
       Assert.assertEquals(
           expectedParameters.getSchemaRegionPerDataNode(),
-          clusterParameters.getSchemaRegionPerDataNode(),
-          0.01);
+          clusterParameters.getSchemaRegionPerDataNode());
       Assert.assertEquals(
           expectedParameters.getDiskSpaceWarningThreshold(),
           clusterParameters.getDiskSpaceWarningThreshold(),

integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java

@@ -323,8 +323,8 @@ public static TClusterParameters generateClusterParameters() {
     clusterParameters.setTimePartitionInterval(604800000);
     clusterParameters.setDataReplicationFactor(1);
     clusterParameters.setSchemaReplicationFactor(1);
-    clusterParameters.setDataRegionPerDataNode(5.0);
-    clusterParameters.setSchemaRegionPerDataNode(1.0);
+    clusterParameters.setDataRegionPerDataNode(0);
+    clusterParameters.setSchemaRegionPerDataNode(1);
     clusterParameters.setDiskSpaceWarningThreshold(0.01);
     clusterParameters.setReadConsistencyLevel("strong");
     clusterParameters.setTimestampPrecision("ms");

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java

@@ -91,7 +91,7 @@ public class ConfigNodeConfig {
   private int defaultSchemaRegionGroupNumPerDatabase = 1;
 
   /** The maximum number of SchemaRegions expected to be managed by each DataNode. */
-  private double schemaRegionPerDataNode = schemaReplicationFactor;
+  private int schemaRegionPerDataNode = 1;
 
   /** The policy of extension DataRegionGroup for each Database. */
   private RegionGroupExtensionPolicy dataRegionGroupExtensionPolicy =
@@ -104,8 +104,14 @@ public class ConfigNodeConfig {
    */
   private int defaultDataRegionGroupNumPerDatabase = 2;
 
-  /** The maximum number of DataRegions expected to be managed by each DataNode. */
-  private double dataRegionPerDataNode = 5.0;
+  /**
+   * The maximum number of DataRegions expected to be managed by each DataNode. Set to 0 means that
+   * each dataNode automatically has the number of CPU cores / 2 regions.
+   */
+  private int dataRegionPerDataNode = 0;
+
+  /** each dataNode automatically has the number of CPU cores / 2 regions. */
+  private double dataRegionPerDataNodeProportion = 0.5;
 
   /** RegionGroup allocate policy. */
   private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy =
@@ -479,11 +485,11 @@ public void setDefaultDataRegionGroupNumPerDatabase(int defaultDataRegionGroupNu
     this.defaultDataRegionGroupNumPerDatabase = defaultDataRegionGroupNumPerDatabase;
   }
 
-  public double getSchemaRegionPerDataNode() {
+  public int getSchemaRegionPerDataNode() {
     return schemaRegionPerDataNode;
   }
 
-  public void setSchemaRegionPerDataNode(double schemaRegionPerDataNode) {
+  public void setSchemaRegionPerDataNode(int schemaRegionPerDataNode) {
     this.schemaRegionPerDataNode = schemaRegionPerDataNode;
   }
 
@@ -495,14 +501,18 @@ public void setDataRegionConsensusProtocolClass(String dataRegionConsensusProtoc
     this.dataRegionConsensusProtocolClass = dataRegionConsensusProtocolClass;
   }
 
-  public double getDataRegionPerDataNode() {
+  public int getDataRegionPerDataNode() {
     return dataRegionPerDataNode;
   }
 
-  public void setDataRegionPerDataNode(double dataRegionPerDataNode) {
+  public void setDataRegionPerDataNode(int dataRegionPerDataNode) {
     this.dataRegionPerDataNode = dataRegionPerDataNode;
   }
 
+  public double getDataRegionPerDataNodeProportion() {
+    return dataRegionPerDataNodeProportion;
+  }
+
   public RegionBalancer.RegionGroupAllocatePolicy getRegionGroupAllocatePolicy() {
     return regionGroupAllocatePolicy;
   }

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java

@@ -238,12 +238,13 @@ private void loadProperties(TrimProperties properties) throws BadNodeUrlExceptio
                 .trim()));
 
     conf.setSchemaRegionPerDataNode(
-        Double.parseDouble(
-            properties
-                .getProperty(
-                    "schema_region_per_data_node",
-                    String.valueOf(conf.getSchemaRegionPerDataNode()))
-                .trim()));
+        (int)
+            Double.parseDouble(
+                properties
+                    .getProperty(
+                        "schema_region_per_data_node",
+                        String.valueOf(conf.getSchemaRegionPerDataNode()))
+                    .trim()));
 
     conf.setDataRegionGroupExtensionPolicy(
         RegionGroupExtensionPolicy.parse(
@@ -258,11 +259,13 @@ private void loadProperties(TrimProperties properties) throws BadNodeUrlExceptio
                 String.valueOf(conf.getDefaultDataRegionGroupNumPerDatabase()).trim())));
 
     conf.setDataRegionPerDataNode(
-        Double.parseDouble(
-            properties
-                .getProperty(
-                    "data_region_per_data_node", String.valueOf(conf.getDataRegionPerDataNode()))
-                .trim()));
+        (int)
+            Double.parseDouble(
+                properties
+                    .getProperty(
+                        "data_region_per_data_node",
+                        String.valueOf(conf.getDataRegionPerDataNode()))
+                    .trim()));
 
     try {
       conf.setRegionAllocateStrategy(

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/PartiteGraphPlacementRegionGroupAllocator.java

@@ -67,10 +67,9 @@ public TRegionReplicaSet generateOptimalRegionReplicasDistribution(
       int replicationFactor,
       TConsensusGroupId consensusGroupId) {
     this.regionPerDataNode =
-        (int)
-            (consensusGroupId.getType().equals(TConsensusGroupType.DataRegion)
-                ? ConfigNodeDescriptor.getInstance().getConf().getDataRegionPerDataNode()
-                : ConfigNodeDescriptor.getInstance().getConf().getSchemaRegionPerDataNode());
+        consensusGroupId.getType().equals(TConsensusGroupType.DataRegion)
+            ? ConfigNodeDescriptor.getInstance().getConf().getDataRegionPerDataNode()
+            : ConfigNodeDescriptor.getInstance().getConf().getSchemaRegionPerDataNode();
     prepare(replicationFactor, availableDataNodeMap, allocatedRegionGroups);
 
     // Select alpha nodes set

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java

@@ -123,8 +123,8 @@ public class ClusterSchemaManager {
   private static final Logger LOGGER = LoggerFactory.getLogger(ClusterSchemaManager.class);
 
   private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
-  private static final double SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode();
-  private static final double DATA_REGION_PER_DATA_NODE = CONF.getDataRegionPerDataNode();
+  private static final int SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode();
+  private static final int DATA_REGION_PER_DATA_NODE = CONF.getDataRegionPerDataNode();
 
   private final IManager configManager;
   private final ClusterSchemaInfo clusterSchemaInfo;
@@ -466,6 +466,7 @@ public synchronized void adjustMaxRegionGroupNum() {
     }
 
     int dataNodeNum = getNodeManager().getRegisteredDataNodeCount();
+    int totalCpuCoreNum = getNodeManager().getDataNodeCpuCoreCount();
     int databaseNum = databaseSchemaMap.size();
 
     for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) {
@@ -519,8 +520,10 @@ public synchronized void adjustMaxRegionGroupNum() {
         int maxDataRegionGroupNum =
             calcMaxRegionGroupNum(
                 databaseSchema.getMinDataRegionGroupNum(),
-                DATA_REGION_PER_DATA_NODE,
-                dataNodeNum,
+                DATA_REGION_PER_DATA_NODE == 0
+                    ? CONF.getDataRegionPerDataNodeProportion()
+                    : DATA_REGION_PER_DATA_NODE,
+                DATA_REGION_PER_DATA_NODE == 0 ? totalCpuCoreNum : dataNodeNum,
                 databaseNum,
                 databaseSchema.getDataReplicationFactor(),
                 allocatedDataRegionGroupCount);

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java

@@ -290,15 +290,6 @@ public int getRegisteredDataNodeCount() {
     return result;
   }
 
-  public int getDataNodeCpuCoreCount(int dataNodeId) {
-    try {
-      return registeredDataNodes.get(dataNodeId).getResource().getCpuCoreNum();
-    } catch (Exception e) {
-      LOGGER.warn("Get DataNode {} cpu core fail, will be treated as zero.", dataNodeId, e);
-      return 0;
-    }
-  }
-
   /** Return the number of total cpu cores in online DataNodes. */
   public int getDataNodeTotalCpuCoreCount() {
     int result = 0;

iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/region/AllocatorScatterWidthManualTest.java

@@ -24,7 +24,6 @@
 import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
 
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -51,8 +50,7 @@ public class AllocatorScatterWidthManualTest {
   private static final IRegionGroupAllocator ALLOCATOR = new GreedyRegionGroupAllocator();
 
   private static final int TEST_DATA_NODE_NUM = 50;
-  private static final int DATA_REGION_PER_DATA_NODE =
-      (int) ConfigNodeDescriptor.getInstance().getConf().getDataRegionPerDataNode();
+  private static final int DATA_REGION_PER_DATA_NODE = 5;
   private static final int DATA_REPLICATION_FACTOR = 3;
 
   private static final Map<Integer, TDataNodeConfiguration> AVAILABLE_DATA_NODE_MAP =

iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/region/GreedyCopySetRegionGroupAllocatorTest.java

@@ -24,7 +24,6 @@
 import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
 
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -54,8 +53,7 @@ public class GreedyCopySetRegionGroupAllocatorTest {
   private static final Random RANDOM = new Random();
   private static final int TEST_DATABASE_NUM = 3;
   private static final int TEST_DATA_NODE_NUM = 21;
-  private static final int DATA_REGION_PER_DATA_NODE =
-      (int) ConfigNodeDescriptor.getInstance().getConf().getDataRegionPerDataNode();
+  private static final int DATA_REGION_PER_DATA_NODE = 5;
   private static final Map<Integer, TDataNodeConfiguration> AVAILABLE_DATA_NODE_MAP =
       new HashMap<>();
   private static final Map<Integer, Double> FREE_SPACE_MAP = new HashMap<>();

iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template

@@ -620,8 +620,8 @@ default_schema_region_group_num_per_database=1
 # Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema,
 # this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases.
 # effectiveMode: restart
-# Datatype: Double
-schema_region_per_data_node=1.0
+# Datatype: Integer
+schema_region_per_data_node=1
 
 # The policy of extension DataRegionGroup for each Database.
 # These policies are currently supported:
@@ -641,11 +641,12 @@ default_data_region_group_num_per_database=2
 
 # Only take effect when set data_region_group_extension_policy=AUTO.
 # This parameter is the maximum number of DataRegions expected to be managed by each DataNode.
+# Set to 0 means that each node automatically has the number of CPU cores / 2 regions
 # Notice: Since each Database requires at least two DataRegionGroups to manage its data,
 # this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases.
 # effectiveMode: restart
-# Datatype: Double
-data_region_per_data_node=5.0
+# Datatype: Integer
+data_region_per_data_node=0
 
 # Whether to enable auto leader balance for Ratis consensus protocol.
 # The ConfigNode-leader will balance the leader of Ratis-RegionGroups by leader_distribution_policy if set true.