Skip to content

Commit af0f757

Browse files
swamirishiswamirishi
committed
HDDS-11650. Address review comments
Change-Id: I7ead428f7ff82968a0f1e5058bbf65f3b807bdb9
1 parent 5b3d27a commit af0f757

File tree

33 files changed

+54
-97
lines changed

33 files changed

+54
-97
lines changed

‎hadoop-hdds/container-service/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
7171
<groupId>org.apache.ozone</groupId>
7272
<artifactId>hdds-server-framework</artifactId>
7373
</dependency>
74-
<dependency>
75-
<groupId>org.apache.ozone</groupId>
76-
<artifactId>hdds-server-framework</artifactId>
77-
<type>test-jar</type>
78-
<scope>test</scope>
79-
</dependency>
8074
<dependency>
8175
<groupId>org.apache.ozone</groupId>
8276
<artifactId>hdds-client</artifactId>

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@
2727

2828
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
2929
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
30+
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
3031
import org.apache.hadoop.hdds.utils.db.Table;
3132
import org.apache.hadoop.ozone.container.common.interfaces.Container;
3233
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
3334
import org.apache.hadoop.ozone.container.common.utils.ContainerLogger;
3435
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
35-
3636
import org.slf4j.Logger;
3737
import org.slf4j.LoggerFactory;
3838

@@ -71,6 +71,11 @@ public class ContainerSet implements Iterable<Container<?>> {
7171
private long recoveringTimeout;
7272
private final Table<Long, State> containerIdsTable;
7373

74+
@VisibleForTesting
75+
public ContainerSet(long recoveringTimeout) {
76+
this(DBTestUtils.getInMemoryTableForTest(), recoveringTimeout);
77+
}
78+
7479
public ContainerSet(Table<Long, State> continerIdsTable, long recoveringTimeout) {
7580
this(continerIdsTable, recoveringTimeout, false);
7681
}

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
2929
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
3030
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
31-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
3231
import org.apache.hadoop.hdfs.util.Canceler;
3332
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
3433
import org.apache.hadoop.io.retry.RetryPolicies;
@@ -337,7 +336,7 @@ public static ContainerDispatcher getNoopContainerDispatcher() {
337336
}
338337

339338
private static final ContainerController EMPTY_CONTAINER_CONTROLLER
340-
= new ContainerController(new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000), Collections.emptyMap());
339+
= new ContainerController(new ContainerSet(1000), Collections.emptyMap());
341340

342341
public static ContainerController getEmptyContainerController() {
343342
return EMPTY_CONTAINER_CONTROLLER;

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
3333
import org.apache.hadoop.hdds.utils.db.BatchOperation;
3434
import org.apache.hadoop.hdds.utils.db.CodecBuffer;
35-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
3635
import org.apache.hadoop.hdds.utils.db.Table;
3736
import org.apache.hadoop.hdds.utils.db.TableIterator;
3837
import org.apache.hadoop.ozone.common.Checksum;
@@ -430,7 +429,7 @@ public void testPendingDeleteBlockReset(ContainerTestVersionInfo versionInfo)
430429
dnConf.setBlockDeletionLimit(blockDeleteLimit);
431430
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
432431
conf.setFromObject(dnConf);
433-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
432+
ContainerSet containerSet = new ContainerSet(1000);
434433

435434
// Create one container with no actual pending delete blocks, but an
436435
// incorrect metadata value indicating it has enough pending deletes to
@@ -538,7 +537,7 @@ public void testBlockDeletion(ContainerTestVersionInfo versionInfo)
538537
dnConf.setBlockDeletionLimit(2);
539538
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
540539
conf.setFromObject(dnConf);
541-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
540+
ContainerSet containerSet = new ContainerSet(1000);
542541
createToDeleteBlocks(containerSet, 1, 3, 1);
543542
ContainerMetrics metrics = ContainerMetrics.create(conf);
544543
KeyValueHandler keyValueHandler =
@@ -664,7 +663,7 @@ public void testWithUnrecordedBlocks(ContainerTestVersionInfo versionInfo)
664663
dnConf.setBlockDeletionLimit(2);
665664
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
666665
conf.setFromObject(dnConf);
667-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
666+
ContainerSet containerSet = new ContainerSet(1000);
668667

669668
createToDeleteBlocks(containerSet, numOfContainers, numOfBlocksPerContainer,
670669
numOfChunksPerBlock);
@@ -774,7 +773,7 @@ public void testShutdownService(ContainerTestVersionInfo versionInfo)
774773
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
775774
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 10);
776775

777-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
776+
ContainerSet containerSet = new ContainerSet(1000);
778777
// Create 1 container with 100 blocks
779778
createToDeleteBlocks(containerSet, 1, 100, 1);
780779
ContainerMetrics metrics = ContainerMetrics.create(conf);
@@ -805,7 +804,7 @@ public void testBlockDeletionTimeout(ContainerTestVersionInfo versionInfo)
805804
blockLimitPerInterval = dnConf.getBlockDeletionLimit();
806805
conf.setFromObject(dnConf);
807806

808-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
807+
ContainerSet containerSet = new ContainerSet(1000);
809808
createToDeleteBlocks(containerSet, 1, 3, 1);
810809
ContainerMetrics metrics = ContainerMetrics.create(conf);
811810
KeyValueHandler keyValueHandler =
@@ -907,7 +906,7 @@ public void testContainerThrottle(ContainerTestVersionInfo versionInfo)
907906
dnConf.setBlockDeletionLimit(1);
908907
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
909908
conf.setFromObject(dnConf);
910-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
909+
ContainerSet containerSet = new ContainerSet(1000);
911910

912911
int containerCount = 2;
913912
int chunksPerBlock = 10;
@@ -967,7 +966,7 @@ public void testContainerMaxLockHoldingTime(
967966
dnConf.setBlockDeletingMaxLockHoldingTime(Duration.ofMillis(-1));
968967
dnConf.setBlockDeletionLimit(3);
969968
conf.setFromObject(dnConf);
970-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
969+
ContainerSet containerSet = new ContainerSet(1000);
971970

972971
int containerCount = 1;
973972
int chunksPerBlock = 10;
@@ -1031,7 +1030,7 @@ public void testBlockThrottle(ContainerTestVersionInfo versionInfo)
10311030
dnConf.setBlockDeletionLimit(10);
10321031
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
10331032
conf.setFromObject(dnConf);
1034-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
1033+
ContainerSet containerSet = new ContainerSet(1000);
10351034
ContainerMetrics metrics = ContainerMetrics.create(conf);
10361035
KeyValueHandler keyValueHandler =
10371036
new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
2424
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
2525
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
26-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
2726
import org.apache.hadoop.hdds.utils.db.Table;
2827
import org.apache.hadoop.ozone.OzoneConfigKeys;
2928
import org.apache.hadoop.ozone.OzoneConsts;
@@ -572,7 +571,7 @@ private void runBlockDeletingService(KeyValueHandler keyValueHandler)
572571
}
573572

574573
private ContainerSet makeContainerSet() throws Exception {
575-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
574+
ContainerSet containerSet = new ContainerSet(1000);
576575
KeyValueContainer container = new KeyValueContainer(newKvData(), conf);
577576
containerSet.addContainer(container);
578577

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
2626
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
2727
import org.apache.hadoop.hdds.utils.db.BatchOperation;
28-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
2928
import org.apache.hadoop.hdds.utils.db.Table;
3029
import org.apache.hadoop.ozone.OzoneConfigKeys;
3130
import org.apache.hadoop.ozone.OzoneConsts;
@@ -136,7 +135,7 @@ public void setup() throws Exception {
136135
blockManager = new BlockManagerImpl(conf);
137136
chunkManager = new FilePerBlockStrategy(true, blockManager, volumeSet);
138137

139-
containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
138+
containerSet = new ContainerSet(1000);
140139
keyValueHandler = new KeyValueHandler(conf, datanodeUuid,
141140
containerSet, volumeSet, ContainerMetrics.create(conf), c -> { });
142141
ozoneContainer = mock(OzoneContainer.class);

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
2424
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
2525
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
26-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
2726
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
2827
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
2928
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@ -147,7 +146,7 @@ private List<Long> createTestContainers(
147146
public void testScrubbingStaleRecoveringContainers(
148147
ContainerTestVersionInfo versionInfo) throws Exception {
149148
initVersionInfo(versionInfo);
150-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 10);
149+
ContainerSet containerSet = new ContainerSet(10);
151150
containerSet.setClock(testClock);
152151
StaleRecoveringContainerScrubbingService srcss =
153152
new StaleRecoveringContainerScrubbingService(

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import java.util.concurrent.TimeUnit;
3131
import org.apache.commons.io.FileUtils;
3232
import org.apache.commons.lang3.RandomUtils;
33-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
3433
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
3534
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
3635
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -86,7 +85,7 @@ public void testRandomChoosingPolicy(ContainerLayoutVersion layout)
8685
RandomContainerDeletionChoosingPolicy.class.getName());
8786
List<StorageLocation> pathLists = new LinkedList<>();
8887
pathLists.add(StorageLocation.parse(containerDir.getAbsolutePath()));
89-
containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
88+
containerSet = new ContainerSet(1000);
9089

9190
int numContainers = 10;
9291
for (int i = 0; i < numContainers; i++) {
@@ -149,7 +148,7 @@ public void testTopNOrderedChoosingPolicy(ContainerLayoutVersion layout)
149148
TopNOrderedContainerDeletionChoosingPolicy.class.getName());
150149
List<StorageLocation> pathLists = new LinkedList<>();
151150
pathLists.add(StorageLocation.parse(containerDir.getAbsolutePath()));
152-
containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
151+
containerSet = new ContainerSet(1000);
153152

154153
int numContainers = 10;
155154
Random random = new Random();

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
4242
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
4343
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
44-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
4544
import org.apache.hadoop.hdds.utils.db.Table;
4645
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
4746
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -153,7 +152,7 @@ public static void shutdown() throws IOException {
153152

154153
@BeforeEach
155154
public void setupPaths() throws IOException {
156-
containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
155+
containerSet = new ContainerSet(1000);
157156
volumeSet = new MutableVolumeSet(DATANODE_UUID, conf, null,
158157
StorageVolume.VolumeType.DATA_VOLUME, null);
159158
// Initialize volume directories.

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
2323
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
2424
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
25-
import org.apache.hadoop.hdds.utils.db.DBTestUtils;
2625
import org.apache.hadoop.ozone.container.common.interfaces.Container;
2726

2827
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@@ -69,7 +68,7 @@ private void setLayoutVersion(ContainerLayoutVersion layoutVersion) {
6968
public void testAddGetRemoveContainer(ContainerLayoutVersion layout)
7069
throws StorageContainerException {
7170
setLayoutVersion(layout);
72-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
71+
ContainerSet containerSet = new ContainerSet(1000);
7372
long containerId = 100L;
7473
ContainerProtos.ContainerDataProto.State state = ContainerProtos
7574
.ContainerDataProto.State.CLOSED;
@@ -158,7 +157,7 @@ public void testIteratorPerVolume(ContainerLayoutVersion layout)
158157
HddsVolume vol2 = mock(HddsVolume.class);
159158
when(vol2.getStorageID()).thenReturn("uuid-2");
160159

161-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
160+
ContainerSet containerSet = new ContainerSet(1000);
162161
for (int i = 0; i < 10; i++) {
163162
KeyValueContainerData kvData = new KeyValueContainerData(i,
164163
layout,
@@ -201,7 +200,7 @@ public void iteratorIsOrderedByScanTime(ContainerLayoutVersion layout)
201200
HddsVolume vol = mock(HddsVolume.class);
202201
when(vol.getStorageID()).thenReturn("uuid-1");
203202
Random random = new Random();
204-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
203+
ContainerSet containerSet = new ContainerSet(1000);
205204
int containerCount = 50;
206205
for (int i = 0; i < containerCount; i++) {
207206
KeyValueContainerData kvData = new KeyValueContainerData(i,
@@ -299,7 +298,7 @@ private static void assertContainerIds(int startId, int count,
299298
}
300299

301300
private ContainerSet createContainerSet() throws StorageContainerException {
302-
ContainerSet containerSet = new ContainerSet(DBTestUtils.getInMemoryTableForTest(), 1000);
301+
ContainerSet containerSet = new ContainerSet(1000);
303302
for (int i = FIRST_ID; i < FIRST_ID + 10; i++) {
304303
KeyValueContainerData kvData = new KeyValueContainerData(i,
305304
layoutVersion,

0 commit comments

Comments
 (0)