Skip to content

Commit 7f7d633

Browse files
jjezrajjezra
authored
Test concurrent indexing sessions without sync lock (#3200)
Since synchronized lock is optional, it would be nice to test concurrent indexing sessions (by records and by source index) that do not use this lock. These indexing sessions will be getting many conflicts, but the index' integrity is expected to be maintained. Resolve #3199
1 parent b470cbc commit 7f7d633

File tree

3 files changed

+105
-8
lines changed

3 files changed

+105
-8
lines changed

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerIndexFromIndexTest.java

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import com.apple.foundationdb.record.IndexBuildProto;
2424
import com.apple.foundationdb.record.RecordCoreException;
2525
import com.apple.foundationdb.record.TestRecords1Proto;
26+
import com.apple.foundationdb.record.logging.KeyValueLogMessage;
27+
import com.apple.foundationdb.record.logging.LogMessageKeys;
2628
import com.apple.foundationdb.record.metadata.Index;
2729
import com.apple.foundationdb.record.metadata.IndexOptions;
2830
import com.apple.foundationdb.record.metadata.IndexTypes;
@@ -33,6 +35,8 @@
3335
import org.junit.jupiter.api.Test;
3436
import org.junit.jupiter.params.ParameterizedTest;
3537
import org.junit.jupiter.params.provider.ValueSource;
38+
import org.slf4j.Logger;
39+
import org.slf4j.LoggerFactory;
3640

3741
import javax.annotation.Nullable;
3842
import java.util.List;
@@ -52,6 +56,8 @@
5256
* Tests for building indexes from other indexes with {@link OnlineIndexer}.
5357
*/
5458
class OnlineIndexerIndexFromIndexTest extends OnlineIndexerTest {
59+
private static final Logger LOGGER = LoggerFactory.getLogger(OnlineIndexerIndexFromIndexTest.class);
60+
5561

5662
private void populateData(final long numRecords, final long numOtherRecords) {
5763
openSimpleMetaData();
@@ -1150,4 +1156,45 @@ void testIndexFromIndexBlock() {
11501156
assertReadable(tgtIndex);
11511157
scrubAndValidate(List.of(tgtIndex));
11521158
}
1159+
1160+
@Test
1161+
void testIndexFromIndexIgnoreSyncLock() {
1162+
1163+
final long numRecords = 180;
1164+
1165+
Index srcIndex = new Index("src_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
1166+
Index tgtIndex = new Index("tgt_index", field("num_value_3_indexed"), IndexTypes.VALUE);
1167+
FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(srcIndex, tgtIndex);
1168+
1169+
populateData(numRecords);
1170+
1171+
openSimpleMetaData(hook);
1172+
buildIndexClean(srcIndex);
1173+
disableAll(List.of(tgtIndex));
1174+
1175+
openSimpleMetaData(hook);
1176+
1177+
IntStream.rangeClosed(0, 4).parallel().forEach(id -> {
1178+
snooze(100 - id);
1179+
try {
1180+
try (OnlineIndexer indexBuilder = newIndexerBuilder(tgtIndex)
1181+
.setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder()
1182+
.setSourceIndex("src_index")
1183+
.forbidRecordScan())
1184+
.setLimit(5)
1185+
.setUseSynchronizedSession(id == 0)
1186+
.setMaxRetries(100) // enough to avoid giving up
1187+
.build()) {
1188+
indexBuilder.buildIndex(true);
1189+
}
1190+
} catch (IndexingBase.UnexpectedReadableException ex) {
1191+
LOGGER.info(KeyValueLogMessage.of("Ignoring lock, got exception",
1192+
LogMessageKeys.SESSION_ID, id,
1193+
LogMessageKeys.ERROR, ex.getMessage()));
1194+
}
1195+
});
1196+
1197+
assertReadable(List.of(tgtIndex));
1198+
scrubAndValidate(List.of(tgtIndex));
1199+
}
11531200
}

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMultiTargetTest.java

Lines changed: 49 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import com.apple.foundationdb.record.IndexBuildProto;
2424
import com.apple.foundationdb.record.RecordCoreException;
2525
import com.apple.foundationdb.record.TestRecords1Proto;
26+
import com.apple.foundationdb.record.logging.KeyValueLogMessage;
27+
import com.apple.foundationdb.record.logging.LogMessageKeys;
2628
import com.apple.foundationdb.record.metadata.Index;
2729
import com.apple.foundationdb.record.metadata.IndexOptions;
2830
import com.apple.foundationdb.record.metadata.IndexTypes;
@@ -35,6 +37,8 @@
3537
import org.junit.jupiter.api.Test;
3638
import org.junit.jupiter.params.ParameterizedTest;
3739
import org.junit.jupiter.params.provider.ValueSource;
40+
import org.slf4j.Logger;
41+
import org.slf4j.LoggerFactory;
3842

3943
import javax.annotation.Nonnull;
4044
import java.util.ArrayList;
@@ -46,6 +50,7 @@
4650
import java.util.concurrent.atomic.AtomicBoolean;
4751
import java.util.concurrent.atomic.AtomicLong;
4852
import java.util.stream.Collectors;
53+
import java.util.stream.IntStream;
4954
import java.util.stream.LongStream;
5055

5156
import static com.apple.foundationdb.record.metadata.Key.Expressions.field;
@@ -58,6 +63,7 @@
5863
*/
5964
@Tag(Tags.Slow)
6065
class OnlineIndexerMultiTargetTest extends OnlineIndexerTest {
66+
private static final Logger LOGGER = LoggerFactory.getLogger(OnlineIndexerMultiTargetTest.class);
6167

6268
private void populateOtherData(final long numRecords) {
6369
List<TestRecords1Proto.MyOtherRecord> records = LongStream.range(0, numRecords).mapToObj(val ->
@@ -882,14 +888,6 @@ void testMultiTargetIndexingBlockerExpiration() {
882888
scrubAndValidate(indexes);
883889
}
884890

885-
void snooze(int millis) {
886-
try {
887-
Thread.sleep(millis);
888-
} catch (InterruptedException e) {
889-
throw new RuntimeException(e);
890-
}
891-
}
892-
893891
@Test
894892
void testForbidConversionOfActiveMultiTarget() throws InterruptedException {
895893
// Do not let a conversion of few indexes of an active multi-target session
@@ -1018,4 +1016,47 @@ void testForbidConversionOfActiveMultiTargetToMutual() throws InterruptedExcepti
10181016
// happy indexes assertion
10191017
assertReadable(indexes);
10201018
}
1019+
1020+
@ParameterizedTest
1021+
@BooleanSource
1022+
void testMultiTargetIgnoringSyncLock(boolean reverseScan) {
1023+
// Simply build the index
1024+
1025+
final long numRecords = 180;
1026+
1027+
List<Index> indexes = new ArrayList<>();
1028+
indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
1029+
indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
1030+
indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
1031+
indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
1032+
1033+
populateData(numRecords);
1034+
1035+
FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
1036+
openSimpleMetaData(hook);
1037+
disableAll(indexes);
1038+
1039+
IntStream.rangeClosed(0, 4).parallel().forEach(id -> {
1040+
snooze(100 - id);
1041+
try {
1042+
try (OnlineIndexer indexBuilder = newIndexerBuilder(indexes)
1043+
.setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder()
1044+
.setReverseScanOrder(reverseScan))
1045+
.setLimit(5)
1046+
.setUseSynchronizedSession(id == 0)
1047+
.setMaxRetries(100) // enough to avoid giving up
1048+
.build()) {
1049+
indexBuilder.buildIndex(true);
1050+
}
1051+
} catch (IndexingBase.UnexpectedReadableException ex) {
1052+
LOGGER.info(KeyValueLogMessage.of("Ignoring lock, got exception",
1053+
LogMessageKeys.SESSION_ID, id,
1054+
LogMessageKeys.ERROR, ex.getMessage()));
1055+
}
1056+
});
1057+
1058+
assertReadable(indexes);
1059+
scrubAndValidate(indexes);
1060+
}
1061+
10211062
}

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,4 +367,13 @@ protected <M extends Message> List<Tuple> getBoundariesList(List<M> records, fin
367367
boundaries.add(null);
368368
return boundaries;
369369
}
370+
371+
protected void snooze(int millis) {
372+
try {
373+
Thread.sleep(millis);
374+
} catch (InterruptedException e) {
375+
Thread.currentThread().interrupt(); //set the flag back to true
376+
throw new RuntimeException(e);
377+
}
378+
}
370379
}

0 commit comments

Comments
 (0)