Skip to content

Commit b4633a0

Browse files
pengpeng-lupengpeng-lu
committed
merge main
2 parents 347e151 + 889cca7 commit b4633a0

File tree

11 files changed

+393
-113
lines changed

11 files changed

+393
-113
lines changed

‎.github/workflows/mixed_mode_test.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Mixed Mode Test
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
tag:
7+
description: 'Tag to test from'
8+
required: true
9+
10+
jobs:
11+
gradle:
12+
runs-on: ubuntu-latest
13+
permissions:
14+
pull-requests: write
15+
steps:
16+
- name: Checkout sources
17+
uses: actions/[email protected]
18+
with:
19+
ref: ${{ inputs.tag }}
20+
ssh-key: ${{ secrets.DEPLOY_KEY }}
21+
- name: Fetch Main
22+
run: git fetch --depth=1 origin main
23+
- name: Setup Base Environment
24+
uses: ./actions/setup-base-env
25+
- name: Setup FDB
26+
uses: ./actions/setup-fdb
27+
28+
# Push a version bump back to main. There are failure scenarios that can result
29+
# in published artifacts but an erroneous build, so it's safer to bump the version
30+
# at the beginning
31+
- name: Configure git
32+
run: |
33+
git config --global user.name 'FoundationDB CI'
34+
git config --global user.email '[email protected]'
35+
- name: Run Gradle Test
36+
uses: ./actions/gradle-test
37+
with:
38+
gradle_command: mixedModeTest
39+
gradle_args: -PreleaseBuild=false -PpublishBuild=false
40+
- name: Checkout Main
41+
run: git checkout main
42+
- name: Update release notes
43+
run: python build/publish-mixed-mode-results.py ${{ inputs.tag }} --release-notes docs/ReleaseNotes.md --commit --run-link ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
44+
- name: Push release notes update
45+
run: git push
46+

‎build/publish-mixed-mode-results.py

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/usr/bin/env python3
2+
3+
#
4+
# publish_mixed_mode_results.py
5+
#
6+
# This source file is part of the FoundationDB open source project
7+
#
8+
# Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
9+
#
10+
# Licensed under the Apache License, Version 2.0 (the "License");
11+
# you may not use this file except in compliance with the License.
12+
# You may obtain a copy of the License at
13+
#
14+
# http://www.apache.org/licenses/LICENSE-2.0
15+
#
16+
# Unless required by applicable law or agreed to in writing, software
17+
# distributed under the License is distributed on an "AS IS" BASIS,
18+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19+
# See the License for the specific language governing permissions and
20+
# limitations under the License.
21+
#
22+
23+
# This script generates a list of mixed-mode testing results generated by the task
24+
# ./gradlew mixedModeTest
25+
26+
import argparse
27+
import subprocess
28+
import sys
29+
30+
def run(command):
31+
try:
32+
process = subprocess.run(command, check=True, capture_output=True, text=True)
33+
return process.stdout
34+
except subprocess.CalledProcessError as e:
35+
print("Failed: " + str(e.cmd))
36+
print(e.stdout)
37+
print(e.stderr)
38+
exit(e.returncode)
39+
40+
def get_results(results_path):
41+
results = {}
42+
with open(results_path) as f:
43+
for line in f:
44+
split = line.strip().split(' ')
45+
if len(split) != 2:
46+
raise Exception("Line is not valid: " + line)
47+
result = split[0]
48+
version = split[1]
49+
if result == 'FAILURE':
50+
results[version] = result
51+
elif result == 'SUCCESS' and version not in results:
52+
results[version] = result
53+
return results
54+
55+
def emoji(result_word):
56+
if result_word == 'FAILURE':
57+
return '❌'
58+
elif result_word == 'SUCCESS':
59+
return '✅'
60+
else:
61+
raise Exception('Invalid result type: ' + result_word)
62+
63+
def generate_markdown(version, results, header_size):
64+
sorted_keys = sorted(results.keys(), key=lambda raw: [int(part) for part in raw.split('.')])
65+
66+
return header_size + " Mixed Mode Test Results\n\nMixed mode testing run against the following previous versions:\n\n" + \
67+
', '.join([emoji(results[version]) + '`' + version + '`' for version in sorted_keys])
68+
69+
def update_release_notes_file(markdown, version, filename):
70+
with open(filename, 'r') as fin:
71+
lines = fin.read().split('\n')
72+
i = 0
73+
target = '<!-- MIXED_MODE_RESULTS ' + version + ' PLACEHOLDER -->'
74+
while i < len(lines) and not lines[i].startswith(target):
75+
i+= 1
76+
if i == len(lines):
77+
raise Exception('Could not find placeholder in release notes file')
78+
return '\n'.join(lines[:i]
79+
+ [markdown]
80+
+ lines[i+1:])
81+
82+
def commit_updates(filename, version):
83+
subprocess.run(['git', 'commit', '-m', "Recording " + version + "mixed mode test results in release notes", filename],
84+
check=True)
85+
86+
def main(argv):
87+
'''Process the output of a mixedModeTest run and convert it into a short markdown'''
88+
parser = argparse.ArgumentParser()
89+
parser.add_argument('--results-path', help='Path to the results', default='.out/reports/mixed-mode-results.log')
90+
parser.add_argument('--release-notes', help='If provided, the results will be injected into this file')
91+
parser.add_argument('--header-size', help='Markdown header level (e.g. # or ##)', default='####')
92+
parser.add_argument('--run-link', help='A link to the test run that generated the results')
93+
parser.add_argument('--commit', action='store_true', default=False, help='Commit the updates to the release notes')
94+
parser.add_argument('version', help='Version of the server that was tested')
95+
args = parser.parse_args(argv)
96+
97+
markdown = generate_markdown(args.version, get_results(args.results_path), args.header_size)
98+
if args.run_link is not None:
99+
markdown = markdown + "\n\n[See full test run](" + args.run_link +")"
100+
if args.release_notes is None:
101+
print(markdown)
102+
else:
103+
new_content = update_release_notes_file(markdown, args.version, args.release_notes)
104+
with open(args.release_notes, 'w') as fout:
105+
fout.write(new_content)
106+
if args.commit:
107+
commit_updates(args.release_notes, args.version)
108+
print(f'Updated {args.release_notes} with test results for {args.version}')
109+
110+
if __name__ == '__main__':
111+
main(sys.argv[1:])

‎build/update_release_notes.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ def get_new_contents(filename, new_version):
106106
+ [template]
107107
+ ['// end next release', '-->', '']
108108
+ updated_next_release_notes
109+
+ ['', '<!-- MIXED_MODE_RESULTS ' + new_version + ' PLACEHOLDER -->', '']
109110
+ lines[next_release_end+3:])
110111

111112

‎docs/ReleaseNotes.md

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,8 @@ Users performing online updates are encouraged to update from [4.0.559.4](#40559
2121
2222
* **Bug fix** Fix 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
2323
* **Bug fix** Fix 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
24-
* **Bug fix** The relational component now specifies `Locale.ROOT` in all usages of `String.format` which may change number formatting if the default locale does not align [(Issue #3121)](https://github.com/FoundationDB/fdb-record-layer/issues/3121)
2524
* **Bug fix** Fix 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
26-
* **Bug fix** Fix 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
27-
* **Bug fix** Fix 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
25+
* **Bug fix** Fix continuation bug in AggregateCursor when a group is finished [Issue #3172](https://github.com/FoundationDB/fdb-record-layer/issues/3172)
2826
* **Performance** Improvement 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
2927
* **Performance** Improvement 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
3028
* **Performance** Improvement 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
@@ -33,9 +31,9 @@ Users performing online updates are encouraged to update from [4.0.559.4](#40559
3331
* **Feature** Feature 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
3432
* **Feature** Feature 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
3533
* **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
36-
* **Feature** FRL now respects PLAN_CACHE_*_MAX_ENTRIES options [(Issue #3155)](https://github.com/FoundationDB/fdb-record-layer/issues/3155)
34+
* **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
3735
* **Feature** Feature 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
38-
e** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
36+
* **Breaking change** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
3937
* **Breaking change** Change 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
4038
* **Breaking change** Change 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
4139
* **Breaking change** Change 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
@@ -44,6 +42,20 @@ e** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/iss
4442
// end next release
4543
-->
4644

45+
### 4.1.8.0
46+
47+
* **Bug fix** The relational component now specifies `Locale.ROOT` in all usages of `String.format` which may change number formatting if the default locale does not align [(Issue #3121)](https://github.com/FoundationDB/fdb-record-layer/issues/3121)
48+
* **Feature** FRL now respects PLAN_CACHE_*_MAX_ENTRIES options [(Issue #3155)](https://github.com/FoundationDB/fdb-record-layer/issues/3155)
49+
50+
51+
#### Mixed Mode Test Results
52+
53+
Mixed mode testing run against the following previous versions:
54+
55+
❌`4.0.559.1`, ✅`4.0.559.2`, ❌`4.0.559.3`, ❌`4.0.559.4`, ❌`4.0.559.6`, ❌`4.0.561.0`, ❌`4.0.562.0`, ✅`4.0.564.0`, ✅`4.0.565.0`, ✅`4.0.566.0`, ✅`4.0.567.0`, ✅`4.0.568.0`, ✅`4.0.569.0`, ✅`4.0.570.0`, ✅`4.0.571.0`, ✅`4.0.572.0`, ✅`4.0.573.0`, ✅`4.0.574.0`, ✅`4.0.575.0`, ✅`4.1.4.0`, ✅`4.1.5.0`, ✅`4.1.6.0`, ✅`4.1.8.0`
56+
57+
[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/13411580367)
58+
4759
### 4.1.6.0
4860

4961
* **Bug fix** Plan hashes for queries involving recursive CTEs are now stable across JVM invocations [(Issue #3139)](https://github.com/FoundationDB/fdb-record-layer/issues/3139)
@@ -52,6 +64,8 @@ e** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/iss
5264
* **Feature** Allow scrubbing of indexes in READABLE_UNIQUE_PENDING state [(Issue #3135)](https://github.com/FoundationDB/fdb-record-layer/issues/3135)
5365
* **Feature** Support Lucene index scrubbing [(Issue #3008)](https://github.com/FoundationDB/fdb-record-layer/issues/3008)
5466

67+
<!-- MIXED_MODE_RESULTS 4.1.6.0 PLACEHOLDER -->
68+
5569
### 4.1.4.0
5670

5771
* **Bug fix** Ungrouped GROUP BY queries result in infinite continuations when maxRows is 1 [(Issue #3093)](https://github.com/FoundationDB/fdb-record-layer/issues/3093)

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import com.apple.foundationdb.record.RecordCursorResult;
2929
import com.apple.foundationdb.record.RecordCursorStartContinuation;
3030
import com.apple.foundationdb.record.RecordCursorVisitor;
31-
import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase;
3231
import com.apple.foundationdb.record.query.plan.plans.QueryResult;
3332
import com.google.common.base.Verify;
3433
import com.google.protobuf.Message;
@@ -97,6 +96,7 @@ public CompletableFuture<RecordCursorResult<QueryResult>> onNext() {
9796
if (groupBreak) {
9897
lastInLastGroup = previousValidResult;
9998
} else {
99+
// previousValidResult is the last row before group break, it sets the continuation
100100
previousValidResult = innerResult;
101101
}
102102
return (!groupBreak);
@@ -106,6 +106,28 @@ public CompletableFuture<RecordCursorResult<QueryResult>> onNext() {
106106
if (Verify.verifyNotNull(previousResult).hasNext()) {
107107
// in this case groupBreak = true, return aggregated result and continuation
108108
RecordCursorContinuation continuation = Verify.verifyNotNull(previousValidResult).getContinuation();
109+
/*
110+
* Update the previousValidResult to the next continuation even though it hasn't been returned. This is to return the correct continuation when there are single-element groups.
111+
* Below is an example that shows how continuation(previousValidResult) moves:
112+
* Initial: previousResult = null, previousValidResult = null
113+
row0 groupKey0 groupBreak = False previousValidResult = row0 previousResult = row0
114+
row1 groupKey0 groupBreak = False previousValidResult = row1 previousResult = row1
115+
row2 groupKey1 groupBreak = True previousValidResult = row1 previousResult = row2
116+
* returns result (groupKey0, continuation = row1), and set previousValidResult = row2
117+
*
118+
* Now there are 2 scenarios, 1) the current iteration continues; 2) the current iteration stops
119+
* In scenario 1, the iteration continues, it gets to row3:
120+
row3 groupKey2 groupBreak = True previousValidResult = row2 previousResult = row3
121+
* returns result (groupKey1, continuation = row2), and set previousValidResult = row3
122+
*
123+
* In scenario 2, a new iteration starts from row2 (because the last returned continuation = row1), and set initial previousResult = null, previousValidResult = null:
124+
row2 groupKey1 groupBreak = False previousValidResult = row2 previousResult = row2
125+
* (Note that because a new iteration starts, groupBreak = False for row2.)
126+
row3 groupKey2 groupBreak = True previousValidResult = row2 previousResult = row3
127+
* returns result (groupKey1, continuation = row2), and set previousValidResult = row3
128+
*
129+
* Both scenarios returns the correct result, and continuation are both set to row3 in the end, row2 is scanned twice if a new iteration starts.
130+
*/
109131
previousValidResult = previousResult;
110132
return RecordCursorResult.withNextValue(QueryResult.ofComputed(streamGrouping.getCompletedGroupResult()), continuation);
111133
} else {
@@ -132,7 +154,7 @@ public CompletableFuture<RecordCursorResult<QueryResult>> onNext() {
132154
}
133155
});
134156
}
135-
157+
136158
private boolean isNoRecords() {
137159
return ((previousValidResult == null) && (!Verify.verifyNotNull(previousResult).hasNext()));
138160
}

0 commit comments

Comments
 (0)