Skip to content

Commit a10b405

Browse files
kenwenzelkenwenzel
authored
eclipse-rdf4jGH-5260 LMDB Store: Add valueEvictionInterval config (eclipse-rdf4j#5261)
2 parents 1b25c0c + 1936428 commit a10b405

File tree

5 files changed

+198
-7
lines changed

5 files changed

+198
-7
lines changed

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,6 @@ class ValueStore extends AbstractValueFactory {
9494

9595
private final static Logger logger = LoggerFactory.getLogger(ValueStore.class);
9696

97-
private static final long VALUE_EVICTION_INTERVAL = 60000; // 60 seconds
98-
9997
private static final byte URI_VALUE = 0x0; // 00
10098

10199
private static final byte LITERAL_VALUE = 0x1; // 01
@@ -186,18 +184,20 @@ class ValueStore extends AbstractValueFactory {
186184

187185
private final ConcurrentCleaner cleaner = new ConcurrentCleaner();
188186

187+
private final long valueEvictionInterval;
188+
189189
ValueStore(File dir, LmdbStoreConfig config) throws IOException {
190190
this.dir = dir;
191191
this.forceSync = config.getForceSync();
192192
this.autoGrow = config.getAutoGrow();
193193
this.mapSize = config.getValueDBSize();
194+
this.valueEvictionInterval = config.getValueEvictionInterval();
194195
open();
195196

196197
valueCache = new LmdbValue[config.getValueCacheSize()];
197198
valueIDCache = new ConcurrentCache<>(config.getValueIDCacheSize());
198199
namespaceCache = new ConcurrentCache<>(config.getNamespaceCacheSize());
199200
namespaceIDCache = new ConcurrentCache<>(config.getNamespaceIDCacheSize());
200-
201201
setNewRevision();
202202

203203
// read maximum id from store
@@ -935,6 +935,10 @@ private static boolean isCommonVocabulary(IRI nv) {
935935
}
936936

937937
public void gcIds(Collection<Long> ids, Collection<Long> nextIds) throws IOException {
938+
if (!enableGC()) {
939+
return;
940+
}
941+
938942
if (!ids.isEmpty()) {
939943
// wrap into read txn as resizeMap expects an active surrounding read txn
940944
readTransaction(env, (stack1, txn1) -> {
@@ -967,9 +971,9 @@ public void gcIds(Collection<Long> ids, Collection<Long> nextIds) throws IOExcep
967971

968972
deleteValueToIdMappings(stack, writeTxn, finalIds, finalNextIds);
969973

970-
invalidateRevisionOnCommit = true;
974+
invalidateRevisionOnCommit = enableGC();
971975
if (nextValueEvictionTime < 0) {
972-
nextValueEvictionTime = System.currentTimeMillis() + VALUE_EVICTION_INTERVAL;
976+
nextValueEvictionTime = System.currentTimeMillis() + this.valueEvictionInterval;
973977
}
974978
return null;
975979
});
@@ -1180,7 +1184,7 @@ void endTransaction(boolean commit) throws IOException {
11801184
unusedRevisionIds.add(revisionId);
11811185
}
11821186
if (nextValueEvictionTime < 0) {
1183-
nextValueEvictionTime = System.currentTimeMillis() + VALUE_EVICTION_INTERVAL;
1187+
nextValueEvictionTime = System.currentTimeMillis() + this.valueEvictionInterval;
11841188
}
11851189
});
11861190
setNewRevision();
@@ -1614,6 +1618,10 @@ public LmdbLiteral getLmdbLiteral(Literal l) {
16141618
}
16151619
}
16161620

1621+
private boolean enableGC() {
1622+
return this.valueEvictionInterval > 0;
1623+
}
1624+
16171625
public void forceEvictionOfValues() {
16181626
nextValueEvictionTime = 0L;
16191627
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfig.java‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
*******************************************************************************/
1111
package org.eclipse.rdf4j.sail.lmdb.config;
1212

13+
import java.time.Duration;
14+
1315
import org.eclipse.rdf4j.model.Model;
1416
import org.eclipse.rdf4j.model.Resource;
1517
import org.eclipse.rdf4j.model.ValueFactory;
@@ -71,6 +73,8 @@ public class LmdbStoreConfig extends BaseSailConfig {
7173

7274
private boolean autoGrow = true;
7375

76+
private long valueEvictionInterval = Duration.ofSeconds(60).toMillis();
77+
7478
/*--------------*
7579
* Constructors *
7680
*--------------*/
@@ -92,7 +96,6 @@ public LmdbStoreConfig(String tripleIndexes, boolean forceSync) {
9296
/*---------*
9397
* Methods *
9498
*---------*/
95-
9699
public String getTripleIndexes() {
97100
return tripleIndexes;
98101
}
@@ -178,6 +181,15 @@ public LmdbStoreConfig setAutoGrow(boolean autoGrow) {
178181
return this;
179182
}
180183

184+
public long getValueEvictionInterval() {
185+
return valueEvictionInterval;
186+
}
187+
188+
public LmdbStoreConfig setValueEvictionInterval(long valueEvictionInterval) {
189+
this.valueEvictionInterval = valueEvictionInterval;
190+
return this;
191+
}
192+
181193
@Override
182194
public Resource export(Model m) {
183195
Resource implNode = super.export(m);
@@ -211,6 +223,9 @@ public Resource export(Model m) {
211223
if (!autoGrow) {
212224
m.add(implNode, LmdbStoreSchema.AUTO_GROW, vf.createLiteral(false));
213225
}
226+
if (valueEvictionInterval != Duration.ofSeconds(60).toMillis()) {
227+
m.add(implNode, LmdbStoreSchema.VALUE_EVICTION_INTERVAL, vf.createLiteral(valueEvictionInterval));
228+
}
214229
return implNode;
215230
}
216231

@@ -304,6 +319,17 @@ public void parse(Model m, Resource implNode) throws SailConfigException {
304319
"Boolean value required for " + LmdbStoreSchema.AUTO_GROW + " property, found " + lit);
305320
}
306321
});
322+
323+
Models.objectLiteral(m.getStatements(implNode, LmdbStoreSchema.VALUE_EVICTION_INTERVAL, null))
324+
.ifPresent(lit -> {
325+
try {
326+
setValueEvictionInterval(lit.longValue());
327+
} catch (NumberFormatException e) {
328+
throw new SailConfigException(
329+
"Long value required for " + LmdbStoreSchema.VALUE_EVICTION_INTERVAL
330+
+ " property, found " + lit);
331+
}
332+
});
307333
} catch (ModelException e) {
308334
throw new SailConfigException(e.getMessage(), e);
309335
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreSchema.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,11 @@ public class LmdbStoreSchema {
7171
*/
7272
public final static IRI AUTO_GROW;
7373

74+
/**
75+
* <tt>http://rdf4j.org/config/sail/lmdb#valueEvictionInterval</tt>
76+
*/
77+
public final static IRI VALUE_EVICTION_INTERVAL;
78+
7479
static {
7580
ValueFactory factory = SimpleValueFactory.getInstance();
7681
TRIPLE_INDEXES = factory.createIRI(NAMESPACE, "tripleIndexes");
@@ -82,5 +87,6 @@ public class LmdbStoreSchema {
8287
NAMESPACE_CACHE_SIZE = factory.createIRI(NAMESPACE, "namespaceCacheSize");
8388
NAMESPACE_ID_CACHE_SIZE = factory.createIRI(NAMESPACE, "namespaceIDCacheSize");
8489
AUTO_GROW = factory.createIRI(NAMESPACE, "autoGrow");
90+
VALUE_EVICTION_INTERVAL = factory.createIRI(NAMESPACE, "valueEvictionInterval");
8591
}
8692
}

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/ValueStoreTest.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,54 @@ public void testGcURIs() throws Exception {
221221
}
222222
}
223223

224+
@Test
225+
public void testDisableGc() throws Exception {
226+
final Random random = new Random(1337);
227+
final LmdbValue values[] = new LmdbValue[1000];
228+
229+
final ValueStore valueStore = new ValueStore(
230+
new File(dataDir, "values"), new LmdbStoreConfig().setValueEvictionInterval(-1));
231+
232+
valueStore.startTransaction(true);
233+
for (int i = 0; i < values.length; i++) {
234+
values[i] = valueStore.createLiteral("This is a random literal:" + random.nextLong());
235+
valueStore.storeValue(values[i]);
236+
}
237+
valueStore.commit();
238+
239+
final ValueStoreRevision revBefore = valueStore.getRevision();
240+
241+
valueStore.startTransaction(true);
242+
Set<Long> ids = new HashSet<>();
243+
for (int i = 0; i < 30; i++) {
244+
ids.add(values[i].getInternalID());
245+
}
246+
valueStore.gcIds(ids, new HashSet<>());
247+
valueStore.commit();
248+
249+
final ValueStoreRevision revAfter = valueStore.getRevision();
250+
251+
assertEquals("revisions must NOT change since GC is disabled", revBefore, revAfter);
252+
253+
Arrays.fill(values, null);
254+
valueStore.unusedRevisionIds.add(revBefore.getRevisionId());
255+
256+
valueStore.forceEvictionOfValues();
257+
valueStore.startTransaction(true);
258+
valueStore.commit();
259+
260+
valueStore.startTransaction(true);
261+
for (int i = 0; i < 30; i++) {
262+
LmdbValue value = valueStore.createLiteral("This is a random literal:" + random.nextLong());
263+
values[i] = value;
264+
valueStore.storeValue(value);
265+
ids.remove(value.getInternalID());
266+
}
267+
valueStore.commit();
268+
269+
assertNotEquals("IDs should NOT have been reused since GC is disabled", Collections.emptySet(), ids);
270+
}
271+
224272
@AfterEach
225273
public void after() throws Exception {
226274
valueStore.close();

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfigTest.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*******************************************************************************
2+
* Copyright (c) 2025 Eclipse RDF4J contributors.
3+
*
4+
* All rights reserved. This program and the accompanying materials
5+
* are made available under the terms of the Eclipse Distribution License v1.0
6+
* which accompanies this distribution, and is available at
7+
* http://www.eclipse.org/org/documents/edl-v10.php.
8+
*
9+
* SPDX-License-Identifier: BSD-3-Clause
10+
*******************************************************************************/
11+
package org.eclipse.rdf4j.sail.lmdb.config;
12+
13+
import static org.assertj.core.api.Assertions.assertThat;
14+
import static org.eclipse.rdf4j.model.util.Values.bnode;
15+
import static org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig.VALUE_CACHE_SIZE;
16+
17+
import org.eclipse.rdf4j.model.BNode;
18+
import org.eclipse.rdf4j.model.IRI;
19+
import org.eclipse.rdf4j.model.Literal;
20+
import org.eclipse.rdf4j.model.Model;
21+
import org.eclipse.rdf4j.model.Resource;
22+
import org.eclipse.rdf4j.model.impl.LinkedHashModel;
23+
import org.eclipse.rdf4j.model.util.ModelBuilder;
24+
import org.eclipse.rdf4j.model.util.Values;
25+
import org.junit.jupiter.params.ParameterizedTest;
26+
import org.junit.jupiter.params.provider.ValueSource;
27+
28+
class LmdbStoreConfigTest {
29+
30+
@ParameterizedTest
31+
@ValueSource(longs = { 1, 205454, 0, -1231 })
32+
void testThatLmdbStoreConfigParseAndExportValueEvictionInterval(final long valueEvictionInterval) {
33+
testParseAndExport(
34+
LmdbStoreSchema.VALUE_EVICTION_INTERVAL,
35+
Values.literal(valueEvictionInterval),
36+
LmdbStoreConfig::getValueEvictionInterval,
37+
valueEvictionInterval,
38+
true
39+
);
40+
}
41+
42+
@ParameterizedTest
43+
@ValueSource(booleans = { true, false })
44+
void testThatLmdbStoreConfigParseAndExportAutoGrow(final boolean autoGrow) {
45+
testParseAndExport(
46+
LmdbStoreSchema.AUTO_GROW,
47+
Values.literal(autoGrow),
48+
LmdbStoreConfig::getAutoGrow,
49+
autoGrow,
50+
!autoGrow
51+
);
52+
}
53+
54+
@ParameterizedTest
55+
@ValueSource(ints = { 1, 205454, 0, -1231 })
56+
void testThatLmdbStoreConfigParseAndExportValueCacheSize(final int valueCacheSize) {
57+
testParseAndExport(
58+
LmdbStoreSchema.VALUE_CACHE_SIZE,
59+
Values.literal(valueCacheSize >= 0 ? valueCacheSize : VALUE_CACHE_SIZE),
60+
LmdbStoreConfig::getValueCacheSize,
61+
valueCacheSize >= 0 ? valueCacheSize : VALUE_CACHE_SIZE,
62+
true
63+
);
64+
}
65+
66+
// TODO: Add more tests for other properties
67+
68+
/**
69+
* Generic method to test parsing and exporting of config properties.
70+
*
71+
* @param property The schema property to test
72+
* @param value The literal value to use in the test
73+
* @param getter Function to get the value from the config object
74+
* @param expectedValue The expected value after parsing
75+
* @param expectedContains The expected result of the contains check
76+
* @param <T> The type of the value being tested
77+
*/
78+
private <T> void testParseAndExport(
79+
IRI property,
80+
Literal value,
81+
java.util.function.Function<LmdbStoreConfig, T> getter,
82+
T expectedValue,
83+
boolean expectedContains
84+
) {
85+
final BNode implNode = bnode();
86+
final LmdbStoreConfig lmdbStoreConfig = new LmdbStoreConfig();
87+
final Model configModel = new ModelBuilder()
88+
.add(implNode, property, value)
89+
.build();
90+
91+
// Parse the config
92+
lmdbStoreConfig.parse(configModel, implNode);
93+
assertThat(getter.apply(lmdbStoreConfig)).isEqualTo(expectedValue);
94+
95+
// Export the config
96+
final Model exportedModel = new LinkedHashModel();
97+
final Resource exportImplNode = lmdbStoreConfig.export(exportedModel);
98+
99+
// Verify the export
100+
assertThat(exportedModel.contains(exportImplNode, property, value))
101+
.isEqualTo(expectedContains);
102+
}
103+
}

0 commit comments

Comments
 (0)