Skip to content

Commit 6af3ef8

Browse files
pengpeng-lupengpeng-lu
committed
Merge branch 'main' into distinct_from
2 parents 0c45afe + 80d5e7c commit 6af3ef8

File tree

63 files changed

+1960
-183
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+1960
-183
lines changed

‎docs/sphinx/source/ReleaseNotes.md

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,79 @@ This document contains a log of changes to the FoundationDB Record Layer. It aim
55

66
As the [versioning guide](Versioning.md) details, it cannot always be determined solely by looking at the version numbers whether one Record Layer version contains all changes included in another. In particular, bug fixes and backwards-compatible changes might be back-ported to or introduced as patches against older versions. To track when a patch version has been included in the main release train, some releases will say as a note that they contain all changes from a specific patch.
77

8+
## 4.5
9+
10+
### 4.5.3.0
11+
12+
13+
14+
**[Full Changelog (4.5.2.0...4.5.3.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.5.2.0...4.5.3.0)**
15+
16+
#### Mixed Mode Test Results
17+
18+
Mixed mode testing run against the following previous versions:
19+
20+
❌`4.4.7.0`, ❌`4.4.8.0`, ❌`4.4.9.0`, ❌`4.4.10.0`, ❌`4.4.11.0`, ✅`4.4.12.0`, ✅`4.4.14.0`, ✅`4.4.15.0`, ✅`4.5.1.0`, ✅`4.5.2.0`
21+
22+
[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/16629945151)
23+
24+
25+
26+
### 4.5.2.0
27+
28+
<h4> Bug Fixes </h4>
29+
30+
* Fix some software rot in the sqlline CLI - [PR #3502](https://github.com/FoundationDB/fdb-record-layer/pull/3502)
31+
* `NULL` handling fixes in plan generator - [PR #3506](https://github.com/FoundationDB/fdb-record-layer/pull/3506)
32+
33+
34+
**[Full Changelog (4.5.1.0...4.5.2.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.5.1.0...4.5.2.0)**
35+
36+
#### Mixed Mode Test Results
37+
38+
Mixed mode testing run against the following previous versions:
39+
40+
✅`4.4.4.0`, ❌`4.4.7.0`, ❌`4.4.8.0`, ❌`4.4.9.0`, ❌`4.4.10.0`, ❌`4.4.11.0`, ✅`4.4.12.0`, ✅`4.4.14.0`, ✅`4.4.15.0`, ✅`4.5.1.0`
41+
42+
[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/16627768059)
43+
44+
45+
46+
### 4.5.1.0
47+
48+
<h4> Breaking Changes </h4>
49+
50+
* Update to minor version 4.5 in recognition of incompatibility with older 4.3 versions - [PR #3508](https://github.com/FoundationDB/fdb-record-layer/pull/3508)
51+
* Remove QueryHashable - [PR #3392](https://github.com/FoundationDB/fdb-record-layer/pull/3392)
52+
<h4> Bug Fixes </h4>
53+
54+
* Enable AggregateCursorContinuation serialize to the new implementation - [PR #3495](https://github.com/FoundationDB/fdb-record-layer/pull/3495)
55+
56+
<details>
57+
<summary>
58+
59+
<h4> Build/Test/Documentation/Style Improvements (click to expand) </h4>
60+
61+
</summary>
62+
63+
* Introduce option to setup transaction before executing queries - [PR #3471](https://github.com/FoundationDB/fdb-record-layer/pull/3471)
64+
* Omit portions of stack trace that are just junit in test summary - [PR #3499](https://github.com/FoundationDB/fdb-record-layer/pull/3499)
65+
66+
</details>
67+
68+
69+
**[Full Changelog (4.4.15.0...4.5.1.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.4.15.0...4.5.1.0)**
70+
71+
#### Mixed Mode Test Results
72+
73+
Mixed mode testing run against the following previous versions:
74+
75+
✅`4.4.3.0`, ✅`4.4.4.0`, ❌`4.4.7.0`, ❌`4.4.8.0`, ❌`4.4.9.0`, ❌`4.4.10.0`, ❌`4.4.11.0`, ✅`4.4.12.0`, ✅`4.4.14.0`, ✅`4.4.15.0`
76+
77+
[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/16597262218)
78+
79+
80+
881
## 4.4
982

1083
### 4.4.15.0

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1138,7 +1138,10 @@ private enum UnaryPhysicalOperator {
11381138
IS_NOT_NULL_EI(Comparisons.Type.NOT_NULL, Type.TypeCode.ENUM, Objects::nonNull),
11391139

11401140
IS_NULL_ID(Comparisons.Type.IS_NULL, Type.TypeCode.UUID, Objects::isNull),
1141-
IS_NOT_NULL_ID(Comparisons.Type.NOT_NULL, Type.TypeCode.UUID, Objects::nonNull);
1141+
IS_NOT_NULL_ID(Comparisons.Type.NOT_NULL, Type.TypeCode.UUID, Objects::nonNull),
1142+
1143+
IS_NULL_NT(Comparisons.Type.IS_NULL, Type.TypeCode.NULL, Objects::isNull),
1144+
IS_NOT_NULL_NT(Comparisons.Type.NOT_NULL, Type.TypeCode.NULL, Objects::nonNull);
11421145

11431146
@Nonnull
11441147
private static final Supplier<BiMap<UnaryPhysicalOperator, PUnaryPhysicalOperator>> protoEnumBiMapSupplier =

‎fdb-record-layer-core/src/main/proto/record_query_plan.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,6 +1196,9 @@ message PUnaryRelOpValue {
11961196

11971197
IS_NULL_ID = 19;
11981198
IS_NOT_NULL_ID = 20;
1199+
1200+
IS_NULL_NT = 21;
1201+
IS_NOT_NULL_NT = 22;
11991202
}
12001203
optional PRelOpValue super = 1;
12011204
optional PUnaryPhysicalOperator operator = 2;

‎fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ int deleteDocument(Tuple groupingKey, Integer partitionId, Tuple primaryKey) thr
299299
@Nullable final LucenePrimaryKeySegmentIndex segmentIndex = directoryManager.getDirectory(groupingKey, partitionId).getPrimaryKeySegmentIndex();
300300

301301
if (segmentIndex != null) {
302-
final DirectoryReader directoryReader = directoryManager.getDirectoryReader(groupingKey, partitionId);
302+
final DirectoryReader directoryReader = directoryManager.getWriterReader(groupingKey, partitionId);
303303
final LucenePrimaryKeySegmentIndex.DocumentIndexEntry documentIndexEntry = segmentIndex.findDocument(directoryReader, primaryKey);
304304
if (documentIndexEntry != null) {
305305
state.context.ensureActive().clear(documentIndexEntry.entryKey); // TODO: Only if valid?
@@ -702,14 +702,15 @@ private CompletableFuture<IndexOperationResult> getLuceneInfoForAllPartitions(fi
702702
@SuppressWarnings("PMD.CloseResource") // the indexReader and directory are closed by the directoryManager
703703
private CompletableFuture<LuceneMetadataInfo.LuceneInfo> getLuceneInfo(final Tuple groupingKey, final Integer partitionId) {
704704
try {
705-
final IndexReader indexReader = directoryManager.getIndexReader(groupingKey, partitionId);
706-
final FDBDirectory directory = getDirectory(groupingKey, partitionId);
707-
final CompletableFuture<Integer> fieldInfosFuture = directory.getFieldInfosCount();
708-
return directory.listAllAsync()
709-
.thenCombine(fieldInfosFuture, (fileList, fieldInfosCount) ->
710-
new LuceneMetadataInfo.LuceneInfo(
711-
indexReader.numDocs(),
712-
fileList, fieldInfosCount));
705+
try (IndexReader indexReader = directoryManager.getIndexReader(groupingKey, partitionId)) {
706+
final FDBDirectory directory = getDirectory(groupingKey, partitionId);
707+
final CompletableFuture<Integer> fieldInfosFuture = directory.getFieldInfosCount();
708+
return directory.listAllAsync()
709+
.thenCombine(fieldInfosFuture, (fileList, fieldInfosCount) ->
710+
new LuceneMetadataInfo.LuceneInfo(
711+
indexReader.numDocs(),
712+
fileList, fieldInfosCount));
713+
}
713714
} catch (IOException e) {
714715
return CompletableFuture.failedFuture(e);
715716
}

‎fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ private boolean isMissingIndexKey(FDBIndexableRecord<Message> rec, Integer parti
198198
throw LuceneExceptions.toRecordCoreException("failed getIndexWriter", e);
199199
}
200200
try {
201-
DirectoryReader directoryReader = directoryManager.getDirectoryReader(groupingKey, partitionId);
201+
DirectoryReader directoryReader = directoryManager.getWriterReader(groupingKey, partitionId);
202202
final LucenePrimaryKeySegmentIndex.DocumentIndexEntry documentIndexEntry = segmentIndex.findDocument(directoryReader, rec.getPrimaryKey());
203203
if (documentIndexEntry == null) {
204204
// Here: the document had not been found in the PK segment index

‎fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryManager.java

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -84,22 +84,18 @@ public class FDBDirectoryManager implements AutoCloseable {
8484
@Nonnull
8585
private final Map<Tuple, FDBDirectoryWrapper> createdDirectories;
8686
private final int mergeDirectoryCount;
87-
@Nullable
88-
private final Exception exceptionAtCreation;
8987
@Nonnull
9088
protected final LuceneAnalyzerWrapper writerAnalyzer;
9189
@Nonnull
9290
private final LuceneAnalyzerCombinationProvider analyzerSelector;
91+
@Nullable
92+
protected final Exception exceptionAtCreation;
9393

9494
protected FDBDirectoryManager(@Nonnull IndexMaintainerState state) {
9595
this.state = state;
9696
this.createdDirectories = new ConcurrentHashMap<>();
9797
this.mergeDirectoryCount = getMergeDirectoryCount(state);
98-
if (FDBTieredMergePolicy.usesCreationStack()) {
99-
this.exceptionAtCreation = new Exception();
100-
} else {
101-
this.exceptionAtCreation = null;
102-
}
98+
this.exceptionAtCreation = FDBTieredMergePolicy.usesCreationStack() ? new Exception() : null;
10399
final var fieldInfos = LuceneIndexExpressions.getDocumentFieldDerivations(state.index, state.store.getRecordMetaData());
104100
this.analyzerSelector = LuceneAnalyzerRegistryImpl.instance().getLuceneAnalyzerCombinationProvider(state.index, LuceneAnalyzerType.FULL_TEXT, fieldInfos);
105101
this.writerAnalyzer = analyzerSelector.provideIndexAnalyzer();
@@ -206,7 +202,7 @@ public void mergeIndexWithContext(@Nonnull final Tuple groupingKey,
206202
@Nonnull final AgilityContext agilityContext) {
207203
try (FDBDirectoryWrapper directoryWrapper = createDirectoryWrapper(groupingKey, partitionId, agilityContext)) {
208204
try {
209-
directoryWrapper.mergeIndex(exceptionAtCreation);
205+
directoryWrapper.mergeIndex();
210206
if (LOGGER.isDebugEnabled()) {
211207
LOGGER.debug(KeyValueLogMessage.of("Lucene merge success",
212208
LuceneLogMessageKeys.GROUP, groupingKey,
@@ -312,7 +308,8 @@ private FDBDirectoryWrapper createDirectoryWrapper(@Nullable Tuple groupingKey,
312308
}
313309

314310
protected @Nonnull FDBDirectoryWrapper createNewDirectoryWrapper(final IndexMaintainerState state, final Tuple key, final int mergeDirectoryCount, final AgilityContext agilityContext, final int blockCacheMaximumSize) {
315-
return new FDBDirectoryWrapper(state, key, mergeDirectoryCount, agilityContext, blockCacheMaximumSize, writerAnalyzer);
311+
return new FDBDirectoryWrapper(state, key, mergeDirectoryCount, agilityContext, blockCacheMaximumSize,
312+
writerAnalyzer, exceptionAtCreation);
316313
}
317314

318315
private int getBlockCacheMaximumSize() {
@@ -367,11 +364,11 @@ public IndexReader getIndexReader(@Nullable Tuple groupingKey, @Nullable Integer
367364

368365
@Nonnull
369366
public IndexWriter getIndexWriter(@Nullable Tuple groupingKey, @Nullable Integer partitionId) throws IOException {
370-
return getDirectoryWrapper(groupingKey, partitionId).getWriter(exceptionAtCreation);
367+
return getDirectoryWrapper(groupingKey, partitionId).getWriter();
371368
}
372369

373-
public DirectoryReader getDirectoryReader(@Nullable Tuple groupingKey, @Nullable Integer partititonId) throws IOException {
374-
return getDirectoryWrapper(groupingKey, partititonId).getWriterReader(false);
370+
public DirectoryReader getWriterReader(@Nullable Tuple groupingKey, @Nullable Integer partititonId) throws IOException {
371+
return getDirectoryWrapper(groupingKey, partititonId).getWriterReader();
375372
}
376373

377374
@Nonnull

0 commit comments

Comments
 (0)