Skip to content

Commit b737b04

Browse files
jjezrajjezra
authored
Index scrubbing should support range id & range reset (#3298)
Resolve #3297 To allow users run different types of index scrubbing sessions, we should add the next items to the scrubbing policy: - scrubbing range reset: This will force the scrubber to start from scratch (default is false) - scrubbing range id: user can request to use another range set, not to disturb existing one (default, backward compatible, is range 0) - new scrubbing API function `eraseAllIndexingScrubbingData` can be used to delete all scrubbing data. This can be used as a scrubbing state cleanup.
1 parent d218cbb commit b737b04

File tree

14 files changed

+875
-263
lines changed

14 files changed

+875
-263
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/logging/LogMessageKeys.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,8 @@ public enum LogMessageKeys {
241241
RANGE_BYTES,
242242
RANGE_START,
243243
RANGE_END,
244+
RANGE_ID,
245+
RANGE_RESET,
244246
// meta-data evolution
245247
FIELD_NAME,
246248
OLD_FIELD_NAME,
@@ -307,6 +309,7 @@ public enum LogMessageKeys {
307309
FAILED_TRANSACTIONS_COUNT_IN_RUNNER,
308310
TOTAL_RECORDS_SCANNED,
309311
TOTAL_RECORDS_SCANNED_DURING_FAILURES,
312+
SCRUB_TYPE,
310313

311314
// time limits milliseconds
312315
TIME_LIMIT_MILLIS("time_limit_milliseconds"),

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3916,7 +3916,7 @@ public CompletableFuture<IndexBuildState> getIndexBuildStateAsync(Index index) {
39163916
@API(API.Status.INTERNAL)
39173917
@Nonnull
39183918
public CompletableFuture<IndexBuildProto.IndexBuildIndexingStamp> loadIndexingTypeStampAsync(Index index) {
3919-
byte[] stampKey = OnlineIndexer.indexBuildTypeSubspace(this, index).pack();
3919+
byte[] stampKey = IndexingSubspaces.indexBuildTypeSubspace(this, index).pack();
39203920
return ensureContextActive().get(stampKey).thenApply(serializedStamp -> {
39213921
if (serializedStamp == null) {
39223922
return null;
@@ -3944,7 +3944,7 @@ public CompletableFuture<IndexBuildProto.IndexBuildIndexingStamp> loadIndexingTy
39443944
*/
39453945
@API(API.Status.INTERNAL)
39463946
public void saveIndexingTypeStamp(Index index, IndexBuildProto.IndexBuildIndexingStamp stamp) {
3947-
byte[] stampKey = OnlineIndexer.indexBuildTypeSubspace(this, index).pack();
3947+
byte[] stampKey = IndexingSubspaces.indexBuildTypeSubspace(this, index).pack();
39483948
ensureContextActive().set(stampKey, stamp.toByteArray());
39493949
}
39503950

@@ -4710,14 +4710,9 @@ void clearIndexData(@Nonnull Index index) {
47104710
IndexingRangeSet.forIndexBuild(this, index).clear();
47114711
// clear even if non-unique in case the index was previously unique
47124712
context.clear(indexUniquenessViolationsSubspace(index).range());
4713-
// Under the index build subspace, there are 3 lower level subspaces, the lock space, the scanned records
4714-
// subspace, and the type/stamp subspace. We are not supposed to clear the lock subspace, which is used to
4715-
// run online index jobs which may invoke this method. We should clear:
4716-
// * the scanned records subspace. Which, roughly speaking, counts how many records of this store are covered in
4717-
// index range subspace.
4718-
// * the type/stamp subspace. Which indicates which type of indexing is in progress.
4719-
context.clear(Range.startsWith(OnlineIndexer.indexBuildScannedRecordsSubspace(this, index).pack()));
4720-
context.clear(Range.startsWith(OnlineIndexer.indexBuildTypeSubspace(this, index).pack()));
4713+
// Under the index build subspace, there are multiple lower level subspaces - the lock subspace and few others. We are
4714+
// not supposed to clear the lock subspace, which might have been used to an online index job that had invoked this method.
4715+
IndexingSubspaces.eraseAllIndexingDataButTheLock(context, this, index);
47214716
}
47224717

47234718
@SuppressWarnings("PMD.CloseResource")

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexBuildState.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public static CompletableFuture<IndexBuildState> loadIndexBuildStateAsync(FDBRec
9999
@Nonnull
100100
public static CompletableFuture<Long> loadRecordsScannedAsync(FDBRecordStoreBase<?> store, Index index) {
101101
return store.getContext().ensureActive()
102-
.get(OnlineIndexer.indexBuildScannedRecordsSubspace(store, index).getKey())
102+
.get(IndexingSubspaces.indexBuildScannedRecordsSubspace(store, index).getKey())
103103
.thenApply(FDBRecordStore::decodeRecordCount);
104104
}
105105

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.apple.foundationdb.record.RecordMetaData;
3131
import com.apple.foundationdb.record.RecordMetaDataProvider;
3232
import com.apple.foundationdb.record.TupleRange;
33+
import com.apple.foundationdb.record.logging.KeyValueLogMessage;
3334
import com.apple.foundationdb.record.logging.LogMessageKeys;
3435
import com.apple.foundationdb.record.metadata.Index;
3536
import com.apple.foundationdb.record.metadata.MetaDataException;
@@ -86,6 +87,9 @@ List<Object> indexingLogMessageKeyValues() {
8687
return Arrays.asList(
8788
LogMessageKeys.INDEXING_METHOD, scrubberName,
8889
LogMessageKeys.ALLOW_REPAIR, scrubbingPolicy.allowRepair(),
90+
LogMessageKeys.RANGE_ID, scrubbingPolicy.getRangeId(),
91+
LogMessageKeys.RANGE_RESET, scrubbingPolicy.isRangeReset(),
92+
LogMessageKeys.SCRUB_TYPE, scrubbingType,
8993
LogMessageKeys.SCAN_LIMIT, scrubbingPolicy.getEntriesScanLimit()
9094
);
9195
}
@@ -157,6 +161,7 @@ private <T> CompletableFuture<Boolean> indexScrubRangeOnly(final @Nonnull FDBRec
157161
if (range == null) {
158162
// Here: no more missing ranges - all done
159163
// This scrubbing is done. Clear the rangeSet - the next time scrubbing is called it will start from scratch
164+
logScrubberRangeReset("range exhausted");
160165
rangeSet.clear();
161166
return AsyncUtil.READY_FALSE;
162167
}
@@ -228,14 +233,50 @@ private void reportIssues(List<IndexScrubbingTools.Issue> issueList, Throwable e
228233
IndexingRangeSet getRangeset(FDBRecordStore store, Index index) {
229234
switch (scrubbingType) {
230235
case MISSING:
231-
return IndexingRangeSet.forScrubbingRecords(store, index);
236+
return IndexingRangeSet.forScrubbingRecords(store, index, scrubbingPolicy.getRangeId());
232237
case DANGLING:
233-
return IndexingRangeSet.forScrubbingIndex(store, index);
238+
return IndexingRangeSet.forScrubbingIndex(store, index, scrubbingPolicy.getRangeId());
234239
default:
235240
throw new RecordCoreArgumentException("Unpredicted scrubbing type ");
236241
}
237242
}
238243

244+
@Nonnull
245+
@SuppressWarnings("PMD.CloseResource")
246+
@Override
247+
protected CompletableFuture<Void> setScrubberTypeOrThrow(FDBRecordStore store) {
248+
// HERE: The index must be readable, checked by the caller.
249+
IndexBuildProto.IndexBuildIndexingStamp indexingTypeStamp = getIndexingTypeStamp(store);
250+
validateOrThrowEx(indexingTypeStamp.getMethod().equals(IndexBuildProto.IndexBuildIndexingStamp.Method.SCRUB_REPAIR),
251+
"Not a scrubber type-stamp");
252+
253+
final Index index = common.getIndex(); // Note: multi targets mode is not supported (yet)
254+
final IndexingRangeSet rangeSet = getRangeset(store, index);
255+
if (scrubbingPolicy.isRangeReset()) {
256+
logScrubberRangeReset("forced reset");
257+
rangeSet.clear();
258+
return AsyncUtil.DONE;
259+
}
260+
return rangeSet.firstMissingRangeAsync()
261+
.thenAccept(recordRange -> {
262+
if (recordRange == null) {
263+
// Here: no un-scrubbed range is available for this call. Erase the 'ranges' data to allow
264+
// a new, fresh records re-scrubbing.
265+
logScrubberRangeReset("range exhausted detected");
266+
rangeSet.clear();
267+
}
268+
});
269+
}
270+
271+
private void logScrubberRangeReset(String reason) {
272+
if (LOGGER.isInfoEnabled()) {
273+
LOGGER.info(KeyValueLogMessage.build("Reset index scrubbing range")
274+
.addKeysAndValues(common.indexLogMessageKeyValues())
275+
.addKeyAndValue(LogMessageKeys.REASON, reason)
276+
.toString());
277+
}
278+
}
279+
239280
@Override
240281
CompletableFuture<Void> rebuildIndexInternalAsync(final FDBRecordStore store) {
241282
throw new UnsupportedOperationException();

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingBase.java

Lines changed: 7 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import com.apple.foundationdb.FDBError;
2424
import com.apple.foundationdb.FDBException;
2525
import com.apple.foundationdb.MutationType;
26-
import com.apple.foundationdb.Range;
2726
import com.apple.foundationdb.annotation.API;
2827
import com.apple.foundationdb.async.AsyncUtil;
2928
import com.apple.foundationdb.async.MoreAsyncUtil;
@@ -91,12 +90,6 @@
9190
@API(API.Status.INTERNAL)
9291
public abstract class IndexingBase {
9392

94-
private static final Object INDEX_BUILD_LOCK_KEY = 0L;
95-
private static final Object INDEX_BUILD_SCANNED_RECORDS = 1L;
96-
private static final Object INDEX_BUILD_TYPE_VERSION = 2L;
97-
private static final Object INDEX_SCRUBBED_INDEX_RANGES = 3L;
98-
private static final Object INDEX_SCRUBBED_RECORDS_RANGES = 4L;
99-
10093
@Nonnull
10194
private static final Logger LOGGER = LoggerFactory.getLogger(IndexingBase.class);
10295
@Nonnull
@@ -136,38 +129,6 @@ protected FDBDatabaseRunner getRunner() {
136129
return common.getRunner();
137130
}
138131

139-
@Nonnull
140-
private static Subspace indexBuildSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index, Object key) {
141-
return store.getUntypedRecordStore().indexBuildSubspace(index).subspace(Tuple.from(key));
142-
}
143-
144-
@Nonnull
145-
protected static Subspace indexBuildLockSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index) {
146-
return indexBuildSubspace(store, index, INDEX_BUILD_LOCK_KEY);
147-
}
148-
149-
@Nonnull
150-
protected static Subspace indexBuildScannedRecordsSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index) {
151-
return indexBuildSubspace(store, index, INDEX_BUILD_SCANNED_RECORDS);
152-
}
153-
154-
@Nonnull
155-
protected static Subspace indexBuildTypeSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index) {
156-
return indexBuildSubspace(store, index, INDEX_BUILD_TYPE_VERSION);
157-
}
158-
159-
@Nonnull
160-
public static Subspace indexScrubIndexRangeSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index) {
161-
// This subspace holds the scrubbed ranges of the index itself (when looking for dangling entries)
162-
return indexBuildSubspace(store, index, INDEX_SCRUBBED_INDEX_RANGES);
163-
}
164-
165-
@Nonnull
166-
public static Subspace indexScrubRecordsRangeSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index) {
167-
// This subspace hods the scrubbed ranges of the records (when looking for missing index entries)
168-
return indexBuildSubspace(store, index, INDEX_SCRUBBED_RECORDS_RANGES);
169-
}
170-
171132
@SuppressWarnings("squid:S1452")
172133
protected CompletableFuture<FDBRecordStore> openRecordStore(@Nonnull FDBRecordContext context) {
173134
return common.getRecordStoreBuilder().copyBuilder().setContext(context).openAsync();
@@ -204,7 +165,7 @@ public CompletableFuture<Void> buildIndexAsync(boolean markReadable, boolean use
204165
}
205166
if (useSyncLock) {
206167
buildIndexAsyncFuture = runner
207-
.runAsync(context -> openRecordStore(context).thenApply(store -> indexBuildLockSubspace(store, index)),
168+
.runAsync(context -> openRecordStore(context).thenApply(store -> IndexingSubspaces.indexBuildLockSubspace(store, index)),
208169
common.indexLogMessageKeyValues("IndexingBase::indexBuildLockSubspace"))
209170
.thenCompose(lockSubspace -> runner.startSynchronizedSessionAsync(lockSubspace, common.config.getLeaseLengthMillis()))
210171
.thenCompose(synchronizedRunner -> {
@@ -517,7 +478,7 @@ private static IndexBuildProto.IndexBuildIndexingStamp blocklessStampOf(IndexBui
517478

518479
CompletableFuture<Void> throwIfSyncedLock(String otherIndexName, FDBRecordStore store, IndexBuildProto.IndexBuildIndexingStamp newStamp, IndexBuildProto.IndexBuildIndexingStamp savedStamp) {
519480
final Index otherIndex = store.getRecordMetaData().getIndex(otherIndexName);
520-
final Subspace mainLockSubspace = indexBuildLockSubspace(store, otherIndex);
481+
final Subspace mainLockSubspace = IndexingSubspaces.indexBuildLockSubspace(store, otherIndex);
521482
return SynchronizedSession.checkActiveSessionExists(store.ensureContextActive(), mainLockSubspace)
522483
.thenApply(hasActiveSession -> {
523484
if (Boolean.TRUE.equals(hasActiveSession)) {
@@ -576,44 +537,10 @@ private CompletableFuture<Void> throwUnlessNoRecordWasScanned(boolean noRecordSc
576537
}
577538

578539
@Nonnull
579-
@SuppressWarnings("PMD.CloseResource")
580-
private CompletableFuture<Void> setScrubberTypeOrThrow(FDBRecordStore store) {
581-
// HERE: The index must be readable, checked by the caller
582-
// if scrubber had already run and still have missing ranges, do nothing
583-
// else: clear ranges and overwrite type-stamp
584-
IndexBuildProto.IndexBuildIndexingStamp indexingTypeStamp = getIndexingTypeStamp(store);
585-
validateOrThrowEx(indexingTypeStamp.getMethod().equals(IndexBuildProto.IndexBuildIndexingStamp.Method.SCRUB_REPAIR),
586-
"Not a scrubber type-stamp");
587-
588-
final Index index = common.getIndex(); // Note: the scrubbers do not support multi target (yet)
589-
IndexingRangeSet indexRangeSet = IndexingRangeSet.forScrubbingIndex(store, index);
590-
IndexingRangeSet recordsRangeSet = IndexingRangeSet.forScrubbingRecords(store, index);
591-
final CompletableFuture<Range> indexRangeFuture = indexRangeSet.firstMissingRangeAsync();
592-
final CompletableFuture<Range> recordRangeFuture = recordsRangeSet.firstMissingRangeAsync();
593-
return indexRangeFuture.thenCompose(indexRange -> {
594-
if (indexRange == null) {
595-
// Here: no un-scrubbed index range was left for this call. We will
596-
// erase the 'ranges' data to allow a fresh index re-scrubbing.
597-
if (LOGGER.isDebugEnabled()) {
598-
LOGGER.debug(KeyValueLogMessage.build("Reset scrubber's index range")
599-
.addKeysAndValues(common.indexLogMessageKeyValues())
600-
.toString());
601-
}
602-
indexRangeSet.clear();
603-
}
604-
return recordRangeFuture.thenAccept(recordRange -> {
605-
if (recordRange == null) {
606-
// Here: no un-scrubbed records range was left for this call. We will
607-
// erase the 'ranges' data to allow a fresh records re-scrubbing.
608-
if (LOGGER.isDebugEnabled()) {
609-
LOGGER.debug(KeyValueLogMessage.build("Reset scrubber's records range")
610-
.addKeysAndValues(common.indexLogMessageKeyValues())
611-
.toString());
612-
}
613-
recordsRangeSet.clear();
614-
}
615-
});
616-
});
540+
protected CompletableFuture<Void> setScrubberTypeOrThrow(FDBRecordStore store) {
541+
// This path should never be reached
542+
throw new ValidationException("Called setScrubberTypeOrThrow in a non-scrubbing path",
543+
"isScrubber", isScrubber);
617544
}
618545

619546
@Nonnull
@@ -792,7 +719,7 @@ protected <T> CompletableFuture<Void> iterateRangeOnly(@Nonnull FDBRecordStore
792719
}
793720
if (common.isTrackProgress()) {
794721
for (Index index: common.getTargetIndexes()) {
795-
final Subspace scannedRecordsSubspace = indexBuildScannedRecordsSubspace(store, index);
722+
final Subspace scannedRecordsSubspace = IndexingSubspaces.indexBuildScannedRecordsSubspace(store, index);
796723
store.context.ensureActive().mutate(MutationType.ADD, scannedRecordsSubspace.getKey(),
797724
FDBRecordStore.encodeRecordCount(recordsScannedInTransaction));
798725
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingScrubDangling.java

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ List<Object> indexingLogMessageKeyValues() {
8888
return Arrays.asList(
8989
LogMessageKeys.INDEXING_METHOD, "scrub dangling index entries",
9090
LogMessageKeys.ALLOW_REPAIR, scrubbingPolicy.allowRepair(),
91+
LogMessageKeys.RANGE_ID, scrubbingPolicy.getRangeId(),
92+
LogMessageKeys.RANGE_RESET, scrubbingPolicy.isRangeReset(),
9193
LogMessageKeys.SCAN_LIMIT, scrubbingPolicy.getEntriesScanLimit()
9294
);
9395
}
@@ -147,11 +149,17 @@ private CompletableFuture<Boolean> scrubIndexRangeOnly(@Nonnull FDBRecordStore s
147149
validateOrThrowEx(store.getIndexState(index).isScannable(), "scrubbed index is not readable");
148150

149151
final ScanProperties scanProperties = scanPropertiesWithLimits(true);
150-
final IndexingRangeSet rangeSet = IndexingRangeSet.forScrubbingIndex(store, index);
152+
final IndexingRangeSet rangeSet = IndexingRangeSet.forScrubbingIndex(store, index, scrubbingPolicy.getRangeId());
151153
return rangeSet.firstMissingRangeAsync().thenCompose(range -> {
152154
if (range == null) {
153155
// Here: no more missing ranges - all done
154156
// To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
157+
if (LOGGER.isInfoEnabled()) {
158+
LOGGER.info(KeyValueLogMessage.build("Reset index scrubbing range")
159+
.addKeysAndValues(common.indexLogMessageKeyValues())
160+
.addKeyAndValue(LogMessageKeys.REASON, "range exhausted")
161+
.toString());
162+
}
155163
rangeSet.clear();
156164
return AsyncUtil.READY_FALSE;
157165
}
@@ -239,6 +247,44 @@ private void scrubDanglingEntry(@Nonnull FDBRecordStore store, @Nonnull IndexEnt
239247
}
240248
}
241249

250+
@Nonnull
251+
@SuppressWarnings("PMD.CloseResource")
252+
@Override
253+
protected CompletableFuture<Void> setScrubberTypeOrThrow(FDBRecordStore store) {
254+
// Note: this duplicated function should be eliminated with this obsolete module (for legacy mode) is deleted
255+
// HERE: The index must be readable, checked by the caller
256+
IndexBuildProto.IndexBuildIndexingStamp indexingTypeStamp = getIndexingTypeStamp(store);
257+
validateOrThrowEx(indexingTypeStamp.getMethod().equals(IndexBuildProto.IndexBuildIndexingStamp.Method.SCRUB_REPAIR),
258+
"Not a scrubber type-stamp");
259+
260+
final Index index = common.getIndex(); // Note: the scrubbers do not support multi target (yet)
261+
IndexingRangeSet indexRangeSet = IndexingRangeSet.forScrubbingIndex(store, index, scrubbingPolicy.getRangeId());
262+
if (scrubbingPolicy.isRangeReset()) {
263+
indexRangeSet.clear();
264+
if (LOGGER.isInfoEnabled()) {
265+
LOGGER.info(KeyValueLogMessage.build("Reset index scrubbing range")
266+
.addKeysAndValues(common.indexLogMessageKeyValues())
267+
.addKeyAndValue(LogMessageKeys.REASON, "forced reset")
268+
.toString());
269+
}
270+
return AsyncUtil.DONE;
271+
}
272+
return indexRangeSet.firstMissingRangeAsync()
273+
.thenAccept(indexRange -> {
274+
if (indexRange == null) {
275+
// Here: no un-scrubbed records range was left for this call. We will
276+
// erase the 'ranges' data to allow a fresh records re-scrubbing.
277+
if (LOGGER.isInfoEnabled()) {
278+
LOGGER.info(KeyValueLogMessage.build("Reset index scrubbing range")
279+
.addKeysAndValues(common.indexLogMessageKeyValues())
280+
.addKeyAndValue(LogMessageKeys.REASON, "range exhausted detected")
281+
.toString());
282+
}
283+
indexRangeSet.clear();
284+
}
285+
});
286+
}
287+
242288
@Override
243289
CompletableFuture<Void> rebuildIndexInternalAsync(final FDBRecordStore store) {
244290
throw new UnsupportedOperationException();

0 commit comments

Comments
 (0)