Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,12 @@ public CommonConfig setSeriesSlotNum(int seriesSlotNum) {
return this;
}

@Override
public CommonConfig setDataPartitionAllocationStrategy(String dataPartitionAllocationStrategy) {
setProperty("data_partition_allocation_strategy", dataPartitionAllocationStrategy);
return this;
}

@Override
public CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass) {
setProperty("series_partition_executor_class", seriesPartitionExecutorClass);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,12 @@ public CommonConfig setSeriesSlotNum(int seriesSlotNum) {
return this;
}

@Override
public CommonConfig setDataPartitionAllocationStrategy(String dataPartitionAllocationStrategy) {
cnConfig.setDataPartitionAllocationStrategy(dataPartitionAllocationStrategy);
return this;
}

@Override
public CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass) {
cnConfig.setSeriesPartitionExecutorClass(seriesPartitionExecutorClass);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,11 @@ public CommonConfig setSeriesSlotNum(int seriesSlotNum) {
return this;
}

@Override
public CommonConfig setDataPartitionAllocationStrategy(String dataPartitionAllocationStrategy) {
return this;
}

@Override
public CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass) {
return this;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ CommonConfig setEnableAutoLeaderBalanceForIoTConsensus(

CommonConfig setSeriesSlotNum(int seriesSlotNum);

CommonConfig setDataPartitionAllocationStrategy(String dataPartitionAllocationStrategy);

CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass);

CommonConfig setSchemaMemoryAllocate(String schemaMemoryAllocate);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@

@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
public class IoTDBPartitionInheritPolicyIT {
public class IoTDBPartitionInheritStrategyIT {

private static final String testDataRegionConsensusProtocolClass =
ConsensusFactory.RATIS_CONSENSUS;
Expand Down Expand Up @@ -91,7 +91,7 @@ public static void tearDown() {
}

@Test
public void testDataPartitionInheritPolicy() throws Exception {
public void testDataPartitionInheritStrategy() throws Exception {
final long baseStartTime = 1000;
Map<TSeriesPartitionSlot, TConsensusGroupId> dataAllotTable1 = new ConcurrentHashMap<>();

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.it.partition;

import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.rpc.TSStatusCode;

import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;

@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
public class IoTDBPartitionShuffleStrategyIT {

private static final String testDataRegionConsensusProtocolClass =
ConsensusFactory.RATIS_CONSENSUS;
private static final int testReplicationFactor = 1;
private static final String testDataPartitionAllocationStrategy = "SHUFFLE";
private static final int testSeriesSlotNum = 1000;
private static final long testTimePartitionInterval = 604800000;
private static final double testDataRegionPerDataNode = 5.0;

private static final String database = "root.database";
private static final int testTimePartitionSlotsNum = 100;

@BeforeClass
public static void setUp() throws Exception {
EnvFactory.getEnv()
.getConfig()
.getCommonConfig()
.setDataRegionConsensusProtocolClass(testDataRegionConsensusProtocolClass)
.setDataReplicationFactor(testReplicationFactor)
.setTimePartitionInterval(testTimePartitionInterval)
.setSeriesSlotNum(testSeriesSlotNum)
.setDataPartitionAllocationStrategy(testDataPartitionAllocationStrategy)
.setDataRegionPerDataNode(testDataRegionPerDataNode);

// Init 1C1D environment
EnvFactory.getEnv().initClusterEnvironment(1, 1);

// Set Database
try (SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
TSStatus status = client.setDatabase(new TDatabaseSchema(database));
Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
}
}

@AfterClass
public static void tearDown() {
EnvFactory.getEnv().cleanClusterEnvironment();
}

@Test
public void testDataPartitionShuffleStrategy() throws Exception {
List<Integer> randomTimeSlotList = new ArrayList<>();
for (int i = 0; i < testTimePartitionSlotsNum; i++) {
randomTimeSlotList.add(i);
}
Collections.shuffle(randomTimeSlotList);
for (int timeSlotId : randomTimeSlotList) {
// To test the shuffle strategy, we merely need to use a random time slot order
ConfigNodeTestUtils.getOrCreateDataPartitionWithRetry(
database, 0, testSeriesSlotNum, timeSlotId, timeSlotId + 1, testTimePartitionInterval);
}
TDataPartitionTableResp dataPartitionTableResp;
try (SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
dataPartitionTableResp =
client.getDataPartitionTable(
new TDataPartitionReq(
ConfigNodeTestUtils.constructPartitionSlotsMap(
database,
0,
testSeriesSlotNum,
0,
testTimePartitionSlotsNum,
testTimePartitionInterval)));
}
Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TConsensusGroupId>>>>
partitionTable = dataPartitionTableResp.getDataPartitionTable();
for (long currentStartTime = testTimePartitionInterval;
currentStartTime < testTimePartitionInterval * testTimePartitionSlotsNum;
currentStartTime += testTimePartitionInterval) {
TTimePartitionSlot precedingTimeSlot =
new TTimePartitionSlot(currentStartTime - testTimePartitionInterval);
TTimePartitionSlot currentTimeSlot = new TTimePartitionSlot(currentStartTime);
for (int seriesSlotId = 0; seriesSlotId < testSeriesSlotNum; seriesSlotId++) {
TSeriesPartitionSlot seriesPartitionSlot = new TSeriesPartitionSlot(seriesSlotId);
List<TConsensusGroupId> precedingRegionGroupIds =
partitionTable.get(database).get(seriesPartitionSlot).get(precedingTimeSlot);
List<TConsensusGroupId> currentRegionGroupIds =
partitionTable.get(database).get(seriesPartitionSlot).get(currentTimeSlot);
Assert.assertEquals(precedingRegionGroupIds.size(), currentRegionGroupIds.size());
for (int i = 0; i < precedingRegionGroupIds.size(); i++) {
// Ensure that the RegionGroupId is different in two adjacent TimePartitionSlots
Assert.assertNotEquals(precedingRegionGroupIds.get(i), currentRegionGroupIds.get(i));
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ public class ConfigNodeConfig {
private String seriesPartitionExecutorClass =
"org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor";

private String dataPartitionAllocationStrategy = "INHERIT";

/** The policy of extension SchemaRegionGroup for each Database. */
private RegionGroupExtensionPolicy schemaRegionGroupExtensionPolicy =
RegionGroupExtensionPolicy.AUTO;
Expand Down Expand Up @@ -423,6 +425,14 @@ public void setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass)
this.seriesPartitionExecutorClass = seriesPartitionExecutorClass;
}

public String getDataPartitionAllocationStrategy() {
return dataPartitionAllocationStrategy;
}

public void setDataPartitionAllocationStrategy(String dataPartitionAllocationStrategy) {
this.dataPartitionAllocationStrategy = dataPartitionAllocationStrategy;
}

public int getCnRpcMaxConcurrentClientNum() {
return rpcMaxConcurrentClientNum;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,10 @@ private void loadProperties(TrimProperties properties) throws BadNodeUrlExceptio
properties.getProperty(
"series_partition_executor_class", conf.getSeriesPartitionExecutorClass()));

conf.setDataPartitionAllocationStrategy(
properties.getProperty(
"data_partition_allocation_strategy", conf.getDataPartitionAllocationStrategy()));

conf.setConfigNodeConsensusProtocolClass(
properties.getProperty(
"config_node_consensus_protocol_class", conf.getConfigNodeConsensusProtocolClass()));
Expand Down
Loading
Loading