Skip to content

Commit e00a96c

Browse files
mdhslmdhsl
authored
[DB] Add forceReadBackFromStorage and commits in obs store unit tests… (#319)
1 parent 0830d34 commit e00a96c

File tree

2 files changed

+39
-10
lines changed

2 files changed

+39
-10
lines changed

‎sensorhub-core/src/testFixtures/java/org/sensorhub/impl/datastore/AbstractTestFeatureStore.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -860,7 +860,8 @@ public void testAddAndRemoveByTimeRange() throws Exception
860860
.withValidTimeDuring(timeFilter)
861861
.build());
862862
System.out.println(count + " features removed");
863-
863+
864+
forceReadBackFromStorage();
864865
// generate truth by removing entries from allFeatures map
865866
var it = allFeatures.values().iterator();
866867
while (it.hasNext())
@@ -916,8 +917,11 @@ public void testSelectByParentID() throws Exception
916917
{
917918
useAdd = true;
918919
BigId group1Id = addFeatureCollection("col1", "collection 1");
920+
forceReadBackFromStorage();
919921
BigId group2Id = addFeatureCollection("col2", "collection 2");
922+
forceReadBackFromStorage();
920923
BigId group3Id = addFeatureCollection("col3", "collection 3");
924+
forceReadBackFromStorage();
921925
addGeoFeaturesPoint2D(group1Id, 0, 20);
922926
addNonGeoFeatures(group2Id, 40, 35);
923927
forceReadBackFromStorage();
@@ -945,8 +949,11 @@ public void testSelectByParentIDAndTime() throws Exception
945949
{
946950
useAdd = true;
947951
BigId group1Id = addFeatureCollection("col1", "collection 1");
952+
forceReadBackFromStorage();
948953
BigId group2Id = addFeatureCollection("col2", "collection 2");
954+
forceReadBackFromStorage();
949955
BigId group3Id = addFeatureCollection("col3", "collection 3");
956+
forceReadBackFromStorage();
950957
addGeoFeaturesPoint2D(group1Id, 0, 20);
951958
addNonGeoFeatures(group2Id, 40, 35);
952959
addTemporalGeoFeatures(group3Id, 100, 46);
@@ -1082,8 +1089,11 @@ public void testSelectByParentUID() throws Exception
10821089
{
10831090
useAdd = true;
10841091
BigId group1Id = addFeatureCollection("col1", "collection 1");
1092+
forceReadBackFromStorage();
10851093
BigId group2Id = addFeatureCollection("col2", "collection 2");
1094+
forceReadBackFromStorage();
10861095
BigId group3Id = addFeatureCollection("col3", "collection 3");
1096+
forceReadBackFromStorage();
10871097
addGeoFeaturesPoint2D(group1Id, 0, 20);
10881098
addNonGeoFeatures(group2Id, 40, 35);
10891099
addTemporalGeoFeatures(group3Id, 100, 46);

‎sensorhub-core/src/testFixtures/java/org/sensorhub/impl/datastore/AbstractTestObsStore.java‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -214,9 +214,10 @@ public void testGetNumRecordsOneDataStream() throws Exception
214214
{
215215
int totalObs = 0, numObs;
216216
var dsID = addSimpleDataStream(bigId(10), "out1");
217-
217+
218218
// add obs w/o FOI
219219
addSimpleObsWithoutResultTime(dsID, BigId.NONE, Instant.parse("2000-01-01T00:00:00Z"), numObs=100);
220+
forceReadBackFromStorage();
220221
assertEquals(totalObs += numObs, obsStore.getNumRecords());
221222

222223
forceReadBackFromStorage();
@@ -233,13 +234,13 @@ public void testGetNumRecordsTwoDataStreams() throws Exception
233234

234235
// add obs with proc1
235236
addSimpleObsWithoutResultTime(ds1, BigId.NONE, Instant.parse("2000-06-21T14:36:12Z"), numObs=100);
237+
forceReadBackFromStorage();
236238
assertEquals(totalObs += numObs, obsStore.getNumRecords());
237239

238240
// add obs with proc2
239241
addSimpleObsWithoutResultTime(ds2, BigId.NONE, Instant.parse("1970-01-01T00:00:00Z"), numObs=50);
240-
assertEquals(totalObs += numObs, obsStore.getNumRecords());
241-
242242
forceReadBackFromStorage();
243+
assertEquals(totalObs += numObs, obsStore.getNumRecords());
243244
assertEquals(totalObs, obsStore.getNumRecords());
244245
}
245246

@@ -252,14 +253,14 @@ public void testAddAndGetByKeyOneDataStream() throws Exception
252253

253254
// add obs w/o FOI
254255
addSimpleObsWithoutResultTime(dsID, BigId.NONE, Instant.parse("2000-01-01T00:00:00Z"), numObs=100);
255-
checkGetObs(totalObs += numObs);
256256
forceReadBackFromStorage();
257+
checkGetObs(totalObs += numObs);
257258
checkGetObs(totalObs);
258259

259260
// add obs with FOI
260261
addSimpleObsWithoutResultTime(dsID, bigId(1001), Instant.parse("9080-02-01T00:00:00Z"), numObs=30);
261-
checkGetObs(totalObs += numObs);
262262
forceReadBackFromStorage();
263+
checkGetObs(totalObs += numObs);
263264
checkGetObs(totalObs);
264265
}
265266

@@ -345,6 +346,11 @@ protected void checkRemoveAllKeys() throws Exception
345346
long t0 = System.currentTimeMillis();
346347
allObs.forEach((k, f) -> {
347348
obsStore.remove(k);
349+
try {
350+
obsStore.commit();
351+
} catch (DataStoreException e) {
352+
throw new RuntimeException(e);
353+
}
348354
assertFalse(obsStore.containsKey(k));
349355
assertTrue(obsStore.get(k) == null);
350356
});
@@ -363,16 +369,14 @@ public void testAddAndRemoveByKey() throws Exception
363369
var dsID = addSimpleDataStream(bigId(10), "out1");
364370

365371
addSimpleObsWithoutResultTime(dsID, BigId.NONE, Instant.parse("1900-01-01T00:00:00Z"), 100);
372+
forceReadBackFromStorage();
366373
checkRemoveAllKeys();
367374

368375
addSimpleObsWithoutResultTime(dsID, bigId(563), Instant.parse("2900-01-01T00:00:00Z"), 100);
369376
forceReadBackFromStorage();
370377
checkRemoveAllKeys();
371378

372-
forceReadBackFromStorage();
373379
addSimpleObsWithoutResultTime(dsID, bigId(1003), Instant.parse("0001-01-01T00:00:00Z"), 100);
374-
checkRemoveAllKeys();
375-
376380
forceReadBackFromStorage();
377381
checkRemoveAllKeys();
378382
}
@@ -516,6 +520,8 @@ public void testSelectObsByDataStreamIDAndTime() throws Exception
516520
Instant startTime2 = Instant.parse("2019-05-31T10:46:03.258Z");
517521
Map<BigId, IObsData> obsBatch2 = addSimpleObsWithoutResultTime(dsID, bigId(104), startTime2, 100, 10000);
518522

523+
forceReadBackFromStorage();
524+
519525
// correct system ID and all times
520526
filter = new ObsFilter.Builder()
521527
.withDataStreams(dsID)
@@ -545,7 +551,6 @@ public void testSelectObsByDataStreamIDAndTime() throws Exception
545551
.withDataStreams(dsID)
546552
.withPhenomenonTimeDuring(startTime2, startTime2.plus(1, ChronoUnit.DAYS))
547553
.build();
548-
forceReadBackFromStorage();
549554
resultStream = obsStore.selectEntries(filter);
550555
checkSelectedEntries(resultStream, obsBatch2, filter);
551556

@@ -589,6 +594,8 @@ public void testSelectObsByDataStreamIDAndFoiID() throws Exception
589594
Instant startProc2Batch3 = Instant.parse("2020-05-31T10:46:03.258Z");
590595
Map<BigId, IObsData> proc2Batch3 = addSimpleObsWithoutResultTime(ds2, bigId(104), startProc2Batch3, 50, 24*3600*1000L);
591596

597+
forceReadBackFromStorage();
598+
592599
// proc1 and all times
593600
filter = new ObsFilter.Builder()
594601
.withDataStreams(ds1)
@@ -683,6 +690,7 @@ public void testSelectObsByDataStreamIDAndPredicates() throws Exception
683690
Instant startProc2Batch1 = Instant.parse("2018-02-11T08:12:06.897Z");
684691
addSimpleObsWithoutResultTime(ds2, bigId(23), startProc2Batch1, 10, 10*24*3600*1000L);
685692

693+
forceReadBackFromStorage();
686694
// proc1 and predicate to select NO FOI
687695
filter = new ObsFilter.Builder()
688696
.withDataStreams(ds1)
@@ -720,6 +728,7 @@ public void testSelectObsByDataStreamFilter() throws Exception
720728
Instant startBatch2 = Instant.parse("2018-02-11T08:11:48.125Z");
721729
Map<BigId, IObsData> obsBatch2 = addSimpleObsWithoutResultTime(ds2, bigId(23), startBatch2, 10, 1200);
722730

731+
forceReadBackFromStorage();
723732
// datastream 2 by ID
724733
filter = new ObsFilter.Builder()
725734
.withDataStreams(ds2)
@@ -859,6 +868,11 @@ protected void addAndCheckObsConcurrent(List<ObsBatch> series)
859868
int numObs = totalObs;
860869
long t0 = System.currentTimeMillis();
861870
addObsConcurrent(series).thenRun(() -> {
871+
try {
872+
forceReadBackFromStorage();
873+
} catch (Exception e) {
874+
throw new RuntimeException(e);
875+
}
862876
readAndCheckSeries(series);
863877

864878
double dt = System.currentTimeMillis() - t0;
@@ -975,6 +989,11 @@ public void testConcurrentReadWrite() throws Throwable
975989
// wait for end of writes and check datastore content
976990
long t0 = System.currentTimeMillis();
977991
addObsConcurrent(obsSeries).thenRun(() -> {
992+
try {
993+
forceReadBackFromStorage();
994+
} catch (Exception e) {
995+
throw new RuntimeException(e);
996+
}
978997
readAndCheckSeries(obsSeries);
979998

980999
double dt = System.currentTimeMillis() - t0;

0 commit comments

Comments
 (0)