Skip to content

Commit 58ff6a6

Browse files
jseddingreschke
jsedding
authored and
reschke
committed
OAK-11934 - segment preloading for PersistentCache (#2569)
1 parent 3c063ba commit 58ff6a6

File tree

10 files changed

+1023
-28
lines changed

10 files changed

+1023
-28
lines changed

‎oak-segment-tar/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,12 @@
368368
<artifactId>org.apache.sling.testing.osgi-mock.junit4</artifactId>
369369
<scope>test</scope>
370370
</dependency>
371+
<dependency>
372+
<groupId>org.awaitility</groupId>
373+
<artifactId>awaitility</artifactId>
374+
<version>4.2.1</version>
375+
<scope>test</scope>
376+
</dependency>
371377
<dependency>
372378
<groupId>org.osgi</groupId>
373379
<artifactId>org.osgi.service.event</artifactId>

‎oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,12 @@
2828
import java.util.Set;
2929
import java.util.UUID;
3030
import java.util.function.Consumer;
31+
import java.util.function.Supplier;
3132

3233
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
3334
import org.apache.jackrabbit.oak.commons.Buffer;
35+
import org.apache.jackrabbit.oak.commons.conditions.Validate;
36+
import org.apache.jackrabbit.oak.commons.pio.Closer;
3437
import org.apache.jackrabbit.oak.segment.CachingSegmentReader;
3538
import org.apache.jackrabbit.oak.segment.RecordType;
3639
import org.apache.jackrabbit.oak.segment.Revisions;
@@ -48,13 +51,16 @@
4851
import org.apache.jackrabbit.oak.segment.SegmentStore;
4952
import org.apache.jackrabbit.oak.segment.SegmentTracker;
5053
import org.apache.jackrabbit.oak.segment.SegmentWriter;
54+
import org.apache.jackrabbit.oak.segment.file.preloader.SegmentPreloader;
5155
import org.apache.jackrabbit.oak.segment.file.tar.EntryRecovery;
5256
import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
5357
import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
5458
import org.apache.jackrabbit.oak.segment.file.tar.TarRecovery;
5559
import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
5660
import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitor;
5761
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
62+
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCache;
63+
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCachePreloadingConfiguration;
5864
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
5965
import org.apache.jackrabbit.oak.stats.StatsOptions;
6066
import org.jetbrains.annotations.NotNull;
@@ -89,6 +95,8 @@ public abstract class AbstractFileStore implements SegmentStore, Closeable {
8995
*/
9096
private static final int MAX_STORE_VERSION = 2;
9197

98+
protected final @Nullable PersistentCache persistentCache;
99+
92100
static ManifestChecker newManifestChecker(SegmentNodeStorePersistence persistence, boolean strictVersionCheck) throws IOException {
93101
return ManifestChecker.newManifestChecker(
94102
persistence.getManifestFile(),
@@ -130,7 +138,7 @@ public void recoverEntry(UUID uuid, byte[] data, EntryRecovery entryRecovery) th
130138
protected final IOMonitor ioMonitor;
131139

132140
protected final RemoteStoreMonitor remoteStoreMonitor;
133-
141+
134142
protected final int binariesInlineThreshold;
135143

136144
AbstractFileStore(final FileStoreBuilder builder) {
@@ -156,6 +164,18 @@ public SegmentId newSegmentId(long msb, long lsb) {
156164
this.remoteStoreMonitor = builder.getRemoteStoreMonitor();
157165
this.segmentBufferMonitor = new SegmentBufferMonitor(builder.getStatsProvider());
158166
this.binariesInlineThreshold = builder.getBinariesInlineThreshold();
167+
this.persistentCache = initializePersistentCache(builder, this::getTarFiles);
168+
}
169+
170+
private static @Nullable PersistentCache initializePersistentCache(FileStoreBuilder builder, Supplier<TarFiles> tarFilesSupplier) {
171+
PersistentCache persistentCache = builder.getPersistentCache();
172+
PersistentCachePreloadingConfiguration preloadingConfig = builder.getPreloadingConfiguration();
173+
if (preloadingConfig != null) {
174+
Validate.checkState(persistentCache != null,
175+
"PersistentCache must be configured when using a PersistentCachePreloadConfiguration");
176+
persistentCache = SegmentPreloader.decorate(persistentCache, preloadingConfig, tarFilesSupplier);
177+
}
178+
return persistentCache;
159179
}
160180

161181
static SegmentNotFoundException asSegmentNotFoundException(Exception e, SegmentId id) {
@@ -165,6 +185,8 @@ static SegmentNotFoundException asSegmentNotFoundException(Exception e, SegmentI
165185
return new SegmentNotFoundException(id, e);
166186
}
167187

188+
abstract TarFiles getTarFiles();
189+
168190
@NotNull
169191
public CacheStatsMBean getSegmentCacheStats() {
170192
return segmentCache.getCacheStats();
@@ -192,7 +214,7 @@ public SegmentReader getReader() {
192214
public SegmentIdProvider getSegmentIdProvider() {
193215
return tracker;
194216
}
195-
217+
196218
public int getBinariesInlineThreshold() {
197219
return binariesInlineThreshold;
198220
}
@@ -281,6 +303,25 @@ public void consume(int number, RecordType type, int offset) {
281303
return binaryReferences;
282304
}
283305

306+
@Override
307+
public void close() {
308+
doClose();
309+
System.gc(); // for any memory-mappings that are no longer used
310+
log.info("TarMK closed: {}", directory);
311+
}
312+
313+
protected void doClose() {
314+
Closer closer = Closer.create();
315+
registerCloseables(closer);
316+
closeAndLogOnFail(closer);
317+
}
318+
319+
protected void registerCloseables(Closer closer) {
320+
if (persistentCache instanceof Closeable) {
321+
closer.register((Closeable) persistentCache);
322+
}
323+
}
324+
284325
static void closeAndLogOnFail(Closeable closeable) {
285326
if (closeable != null) {
286327
try {
@@ -292,11 +333,19 @@ static void closeAndLogOnFail(Closeable closeable) {
292333
}
293334
}
294335

295-
Segment readSegmentUncached(TarFiles tarFiles, SegmentId id) {
296-
Buffer buffer = tarFiles.readSegment(id.getMostSignificantBits(), id.getLeastSignificantBits());
336+
Segment readSegmentUncached(SegmentId id) {
337+
Buffer buffer;
338+
if (persistentCache != null) {
339+
buffer = persistentCache.readSegment(id.getMostSignificantBits(), id.getLeastSignificantBits(),
340+
() -> getTarFiles().readSegment(id.getMostSignificantBits(), id.getLeastSignificantBits()));
341+
} else {
342+
buffer = getTarFiles().readSegment(id.getMostSignificantBits(), id.getLeastSignificantBits());
343+
}
344+
297345
if (buffer == null) {
298346
throw new SegmentNotFoundException(id);
299347
}
348+
300349
segmentBufferMonitor.trackAllocation(buffer);
301350
return new Segment(tracker, id, buffer);
302351
}

‎oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,11 @@ FileStore bind(TarRevisions revisions) throws IOException {
249249
}
250250
}
251251

252+
@Override
253+
TarFiles getTarFiles() {
254+
return tarFiles;
255+
}
256+
252257
@NotNull
253258
private Supplier<RecordId> initialNode() {
254259
return new Supplier<RecordId>() {
@@ -483,21 +488,23 @@ public void close() {
483488
log.warn("Unable to flush the store", e);
484489
}
485490

486-
Closer closer = Closer.create();
487-
closer.register(repositoryLock::unlock);
488-
closer.register(tarFiles) ;
489-
closer.register(revisions);
490-
491-
closeAndLogOnFail(closer);
491+
doClose();
492492
}
493493

494494
// Try removing pending files in case the scheduler didn't have a chance to run yet
495495
System.gc(); // for any memory-mappings that are no longer used
496496
fileReaper.reap();
497-
498497
log.info("TarMK closed: {}", directory);
499498
}
500499

500+
@Override
501+
protected void registerCloseables(Closer closer) {
502+
closer.register(repositoryLock::unlock);
503+
closer.register(tarFiles) ;
504+
closer.register(revisions);
505+
super.registerCloseables(closer);
506+
}
507+
501508
@Override
502509
public boolean containsSegment(SegmentId id) {
503510
try (ShutDownCloser ignored = shutDown.keepAlive()) {
@@ -509,7 +516,7 @@ public boolean containsSegment(SegmentId id) {
509516
@NotNull
510517
public Segment readSegment(final SegmentId id) {
511518
try (ShutDownCloser ignored = shutDown.keepAlive()) {
512-
return segmentCache.getSegment(id, () -> readSegmentUncached(tarFiles, id));
519+
return segmentCache.getSegment(id, () -> readSegmentUncached(id));
513520
} catch (ExecutionException | UncheckedExecutionException e) {
514521
if (e.getCause() instanceof RepositoryNotReachableException) {
515522
RepositoryNotReachableException re = (RepositoryNotReachableException) e.getCause();
@@ -573,6 +580,10 @@ public void writeSegment(SegmentId id, byte[] buffer, int offset, int length) th
573580
if (!eagerSegmentCaching && segment != null) {
574581
segmentCache.putSegment(segment);
575582
}
583+
584+
if (persistentCache != null) {
585+
persistentCache.writeSegment(id.getMostSignificantBits(), id.getLeastSignificantBits(), Buffer.wrap(buffer, offset, length));
586+
}
576587
}
577588
}
578589

‎oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@
4848
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
4949
import org.apache.jackrabbit.oak.segment.spi.monitor.*;
5050
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
51+
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCache;
52+
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCachePreloadingConfiguration;
5153
import org.apache.jackrabbit.oak.segment.tool.iotrace.IOTraceLogWriter;
5254
import org.apache.jackrabbit.oak.segment.tool.iotrace.IOTraceMonitor;
5355
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
@@ -94,7 +96,7 @@ public class FileStoreBuilder {
9496
private boolean memoryMapping = MEMORY_MAPPING_DEFAULT;
9597

9698
private boolean offHeapAccess = getBoolean("access.off.heap");
97-
99+
98100
private int binariesInlineThreshold = Segment.MEDIUM_LIMIT;
99101

100102
private SegmentNodeStorePersistence persistence;
@@ -108,6 +110,12 @@ public class FileStoreBuilder {
108110
@Nullable
109111
private EvictingWriteCacheManager cacheManager;
110112

113+
@Nullable
114+
private PersistentCache persistentCache;
115+
116+
@Nullable
117+
private PersistentCachePreloadingConfiguration preloadingConfig;
118+
111119
private class FileStoreGCListener extends DelegatingGCMonitor implements GCListener {
112120
@Override
113121
public void compactionSucceeded(@NotNull GCGeneration newGeneration) {
@@ -256,6 +264,34 @@ public FileStoreBuilder withNodeDeduplicationCacheSize(int nodeDeduplicationCach
256264
return this;
257265
}
258266

267+
/**
268+
* The {@link PersistentCache} instance to be used, if any. By default no {@code PersistentCache}
269+
* is configured.
270+
*
271+
* @param persistentCache the persistent cache
272+
* @return this instance
273+
*/
274+
@NotNull
275+
public FileStoreBuilder withPersistentCache(@NotNull PersistentCache persistentCache) {
276+
this.persistentCache = requireNonNull(persistentCache);
277+
return this;
278+
}
279+
280+
/**
281+
* The {@link PersistentCachePreloadingConfiguration} to be used for preloading segments.
282+
* This configuration must be used together with a {@link PersistentCache}, configured
283+
* via {@link #withPersistentCache(PersistentCache)}. By default, no segment preloading
284+
* is configured.
285+
*
286+
* @param preloadingConfig the configuration for persistent cache preloading
287+
* @return this instance
288+
*/
289+
@NotNull
290+
public FileStoreBuilder withPersistentCachePreloading(@NotNull PersistentCachePreloadingConfiguration preloadingConfig) {
291+
this.preloadingConfig = requireNonNull(preloadingConfig);
292+
return this;
293+
}
294+
259295
/**
260296
* Turn memory mapping on or off
261297
*
@@ -397,7 +433,7 @@ public FileStoreBuilder withEagerSegmentCaching(boolean eagerSegmentCaching) {
397433
this.eagerSegmentCaching = eagerSegmentCaching;
398434
return this;
399435
}
400-
436+
401437
/**
402438
* Sets the threshold under which binaries are inlined in data segments.
403439
* @param binariesInlineThreshold the threshold
@@ -585,11 +621,19 @@ boolean getStrictVersionCheck() {
585621
boolean getEagerSegmentCaching() {
586622
return eagerSegmentCaching;
587623
}
588-
624+
589625
int getBinariesInlineThreshold() {
590626
return binariesInlineThreshold;
591627
}
592628

629+
@Nullable PersistentCache getPersistentCache() {
630+
return persistentCache;
631+
}
632+
633+
@Nullable PersistentCachePreloadingConfiguration getPreloadingConfiguration() {
634+
return preloadingConfig;
635+
}
636+
593637
@Override
594638
public String toString() {
595639
return "FileStoreBuilder{" +
@@ -607,6 +651,8 @@ public String toString() {
607651
", memoryMapping=" + memoryMapping +
608652
", offHeapAccess=" + offHeapAccess +
609653
", gcOptions=" + gcOptions +
654+
", persistentCache=" + persistentCache +
655+
", persistentCachePreloadingConfiguration=" + preloadingConfig +
610656
'}';
611657
}
612658

‎oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import java.util.Map;
2828
import java.util.Set;
2929
import java.util.UUID;
30-
import java.util.concurrent.Callable;
3130
import java.util.concurrent.ExecutionException;
3231
import java.util.function.Consumer;
3332

@@ -94,6 +93,11 @@ ReadOnlyFileStore bind(@NotNull ReadOnlyRevisions revisions) throws IOException
9493
return this;
9594
}
9695

96+
@Override
97+
TarFiles getTarFiles() {
98+
return tarFiles;
99+
}
100+
97101
/**
98102
* Go to the specified {@code revision}
99103
*
@@ -120,25 +124,17 @@ public boolean containsSegment(SegmentId id) {
120124
@NotNull
121125
public Segment readSegment(final SegmentId id) {
122126
try {
123-
return segmentCache.getSegment(id, new Callable<Segment>() {
124-
@Override
125-
public Segment call() throws Exception {
126-
return readSegmentUncached(tarFiles, id);
127-
}
128-
});
127+
return segmentCache.getSegment(id, () -> readSegmentUncached(id));
129128
} catch (ExecutionException | UncheckedExecutionException e) {
130129
throw asSegmentNotFoundException(e, id);
131130
}
132131
}
133132

134133
@Override
135-
public void close() {
136-
Closer closer = Closer.create();
134+
protected void registerCloseables(Closer closer) {
137135
closer.register(tarFiles);
138136
closer.register(revisions);
139-
closeAndLogOnFail(closer);
140-
System.gc(); // for any memory-mappings that are no longer used
141-
log.info("TarMK closed: {}", directory);
137+
super.registerCloseables(closer);
142138
}
143139

144140
@NotNull

0 commit comments

Comments
 (0)