Skip to content

Commit 2b95815

Browse files
afoucretafoucret
committed
Merge branch 'main' of https://github.com/elastic/elasticsearch into esql_text_embedding_function
2 parents c4a0871 + 4ca03f7 commit 2b95815

File tree

260 files changed

+11136
-6777
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

260 files changed

+11136
-6777
lines changed

‎.buildkite/pipelines/pull-request-transport-versions.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,10 @@
1-
steps: []
1+
steps:
2+
- label: update-transport-versions
3+
command: '.buildkite/scripts/update-transport-versions.sh'
4+
# New transport versions are always added via the main branch
5+
if: build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == 'main'
6+
agents:
7+
provider: gcp
8+
image: family/elasticsearch-ubuntu-2404
9+
machineType: custom-16-32768
10+
buildDirectory: /dev/shm/bk

‎.buildkite/pipelines/pull-request/validate-transport-version-backport.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
config:
2+
skip-target-branches: "main"
3+
steps:
4+
- label: validate-transport-version-backport
5+
command: .buildkite/scripts/validate-transport-version-backport.sh

‎.buildkite/scripts/update-transport-versions.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
5+
echo "Not a pull request, skipping transport version update"
6+
exit 0
7+
fi
8+
9+
if ! git diff --exit-code; then
10+
echo "Changes are present before updating transport versions, not running"
11+
git status
12+
exit 0
13+
fi
14+
15+
NEW_COMMIT_MESSAGE="[CI] Update transport version definitions"
16+
17+
echo "--- Generating updated transport version definitions"
18+
# Calculate backport branches based on pull request version labels
19+
backport_branches=$(
20+
echo "${GITHUB_PR_LABELS}" \
21+
| tr ',' '\n' \
22+
| grep -E "v[0-9]+\.[0-9]+\.[0-9]+" \
23+
| sed -E 's/^v([0-9]+)\.([0-9]+)\.[0-9]+$/\1.\2/' \
24+
| paste -sd, -
25+
)
26+
27+
if [[ -z "${backport_branches}" ]]; then
28+
echo "Skipping as pull request contains no version labels"
29+
exit 0
30+
fi
31+
32+
.ci/scripts/run-gradle.sh generateTransportVersion --backport-branches="${backport_branches}"
33+
34+
if git diff --exit-code; then
35+
echo "No changes found after updating transport versions. Don't need to auto commit."
36+
exit 0
37+
fi
38+
39+
git config --global user.name elasticsearchmachine
40+
git config --global user.email '[email protected]'
41+
42+
gh pr checkout "${BUILDKITE_PULL_REQUEST}"
43+
git add -A .
44+
git commit -m "$NEW_COMMIT_MESSAGE"
45+
git push
46+
47+
# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
48+
# So, let's just sleep to give the build time to cancel itself without an error
49+
# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
50+
sleep 300
51+
exit 1

‎.buildkite/scripts/validate-transport-version-backport.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
echo "--- Looking for transport version changes"
5+
6+
# Get any changes in this pull request to transport definitions
7+
git fetch origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" --quiet
8+
changed_files=$(git diff --name-only "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" | grep -E "server/src/main/resources/transport/definitions/.*\.csv" || true)
9+
10+
if [[ -z "${changed_files}" ]]; then
11+
echo "No transport version changes detected."
12+
exit 0
13+
fi
14+
15+
# Compare those files against the main branch to ensure they are the same
16+
git fetch origin main --quiet
17+
while IFS= read -r file; do
18+
if ! git diff --quiet origin/main -- "${file}"; then
19+
echo "Changes to transport definition [${file}] missing from main branch."
20+
echo "Transport changes must first be merged to main before being backported."
21+
exit 1
22+
fi
23+
done <<< "${changed_files}"
24+
25+
echo "All transport changes exist in main branch."

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,9 @@ public void setup() {
8686
false,
8787
Map.of(),
8888
System.nanoTime(),
89-
false
89+
false,
90+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
91+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
9092
);
9193

9294
var fields = 10_000;

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,9 @@ private static Configuration configuration() {
365365
false,
366366
Map.of(),
367367
0,
368-
false
368+
false,
369+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
370+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
369371
);
370372
}
371373

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
122122
apply plugin: 'elasticsearch.transport-version-references'
123123
apply plugin: 'elasticsearch.transport-version-resources'
124124
125-
tasks.named('generateTransportVersionDefinition') {
125+
tasks.named('generateTransportVersion') {
126126
currentUpperBoundName = '9.2'
127127
}
128128
"""

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy‎

Lines changed: 77 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
1818
def runGenerateAndValidateTask(String... additionalArgs) {
1919
List<String> args = new ArrayList<>()
2020
args.add(":myserver:validateTransportVersionResources")
21-
args.add(":myserver:generateTransportVersionDefinition")
21+
args.add(":myserver:generateTransportVersion")
2222
args.addAll(additionalArgs);
2323
return gradleRunner(args.toArray())
2424
}
2525

2626
def runGenerateTask(String... additionalArgs) {
2727
List<String> args = new ArrayList<>()
28-
args.add(":myserver:generateTransportVersionDefinition")
28+
args.add(":myserver:generateTransportVersion")
2929
args.addAll(additionalArgs);
3030
return gradleRunner(args.toArray())
3131
}
@@ -35,11 +35,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
3535
}
3636

3737
void assertGenerateSuccess(BuildResult result) {
38-
assert result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS
38+
assert result.task(":myserver:generateTransportVersion").outcome == TaskOutcome.SUCCESS
3939
}
4040

4141
void assertGenerateFailure(BuildResult result, String expectedOutput) {
42-
assert result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.FAILED
42+
assert result.task(":myserver:generateTransportVersion").outcome == TaskOutcome.FAILED
4343
assertOutputContains(result.output, expectedOutput)
4444
}
4545

@@ -277,6 +277,77 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
277277
assertUpperBound("9.2", "second_tv,8124000")
278278
}
279279

280+
def "update flag works with current"() {
281+
given:
282+
referableAndReferencedTransportVersion("new_tv", "8123000")
283+
file("myserver/src/main/resources/transport/latest/9.2.csv").text =
284+
"""
285+
<<<<<<< HEAD
286+
existing_92,8123000
287+
=======
288+
new_tv,8123000
289+
>>>>>> name
290+
""".strip()
291+
292+
when:
293+
def result = runGenerateAndValidateTask("--resolve-conflict").build()
294+
295+
then:
296+
assertGenerateAndValidateSuccess(result)
297+
assertReferableDefinition("existing_92", "8123000,8012001")
298+
assertReferableDefinition("new_tv", "8124000")
299+
assertUpperBound("9.2", "new_tv,8124000")
300+
}
301+
302+
def "update flag works with multiple branches"() {
303+
given:
304+
referableAndReferencedTransportVersion("new_tv", "8123000,8012001,7123001")
305+
file("myserver/src/main/resources/transport/latest/9.2.csv").text =
306+
"""
307+
<<<<<<< HEAD
308+
existing_92,8123000
309+
=======
310+
new_tv,8123000
311+
>>>>>> name
312+
""".strip()
313+
file("myserver/src/main/resources/transport/latest/9.1.csv").text =
314+
"""
315+
<<<<<<< HEAD
316+
existing_92,8012001
317+
=======
318+
new_tv,8012001
319+
>>>>>> name
320+
""".strip()
321+
file("myserver/src/main/resources/transport/latest/8.19.csv").text =
322+
"""
323+
<<<<<<< HEAD
324+
initial_8.19.7,7123001
325+
=======
326+
new_tv,7123001
327+
>>>>>> name
328+
""".strip()
329+
330+
when:
331+
def result = runGenerateAndValidateTask("--resolve-conflict").build()
332+
333+
then:
334+
assertGenerateAndValidateSuccess(result)
335+
assertReferableDefinition("existing_92", "8123000,8012001")
336+
assertUnreferableDefinition("initial_8.19.7", "7123001")
337+
assertReferableDefinition("new_tv", "8124000,8012002,7123002")
338+
assertUpperBound("9.2", "new_tv,8124000")
339+
assertUpperBound("9.1", "new_tv,8012002")
340+
assertUpperBound("8.19", "new_tv,7123002")
341+
}
342+
343+
def "update flag cannot be used with backport branches"() {
344+
when:
345+
def result = runGenerateTask("--resolve-conflict", "--backport-branches=9.1").buildAndFail()
346+
347+
then:
348+
assertGenerateFailure(result, "Cannot use --resolve-conflict with --backport-branches")
349+
}
350+
280351
def "branches param order does not matter"() {
281352
given:
282353
referencedTransportVersion("test_tv")
@@ -449,7 +520,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
449520
referencedTransportVersion("new_tv")
450521
file("myserver/alt_upper_bound.csv").text = "some_tv,8126000"
451522
file("myserver/build.gradle") << """
452-
tasks.named('generateTransportVersionDefinition') {
523+
tasks.named('generateTransportVersion') {
453524
alternateUpperBoundFile = project.file("alt_upper_bound.csv")
454525
}
455526
tasks.named('validateTransportVersionResources') {
@@ -470,7 +541,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
470541
referencedTransportVersion("new_tv")
471542
file("myserver/alt_upper_bound.csv").text = "some_tv,8122100"
472543
file("myserver/build.gradle") << """
473-
tasks.named('generateTransportVersionDefinition') {
544+
tasks.named('generateTransportVersion') {
474545
alternateUpperBoundFile = project.file("alt_upper_bound.csv")
475546
}
476547
tasks.named('validateTransportVersionResources') {

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,13 @@ public void run() throws IOException {
5555
var definition = new TransportVersionDefinition(initialDefinitionName, List.of(id));
5656
resources.writeUnreferableDefinition(definition);
5757
var newUpperBound = new TransportVersionUpperBound(upperBoundName, initialDefinitionName, id);
58-
resources.writeUpperBound(newUpperBound);
58+
resources.writeUpperBound(newUpperBound, false);
5959

6060
if (releaseVersion.getRevision() == 0) {
6161
Version currentVersion = getCurrentVersion().get();
6262
String currentUpperBoundName = getUpperBoundName(currentVersion);
6363
var currentUpperBound = new TransportVersionUpperBound(currentUpperBoundName, initialDefinitionName, id);
64-
resources.writeUpperBound(currentUpperBound);
64+
resources.writeUpperBound(currentUpperBound, false);
6565
}
6666
}
6767

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,14 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask
7272
@Option(option = "increment", description = "The amount to increment the id from the current upper bounds file by")
7373
public abstract Property<Integer> getIncrement();
7474

75+
@Input
76+
@Optional
77+
@Option(
78+
option = "resolve-conflict",
79+
description = "Regenerate the transport version currently being added to upstream to resolve a merge conflict"
80+
)
81+
public abstract Property<Boolean> getResolveConflict();
82+
7583
/**
7684
* The name of the upper bounds file which will be used at runtime on the current branch. Normally
7785
* this equates to VersionProperties.getElasticsearchVersion().
@@ -95,7 +103,7 @@ public void run() throws IOException {
95103
String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
96104

97105
List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
98-
Set<String> targetUpperBoundNames = getTargetUpperBoundNames(upstreamUpperBounds);
106+
Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
99107

100108
getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
101109
if (targetDefinitionName.isEmpty()) {
@@ -121,6 +129,7 @@ private List<TransportVersionId> updateUpperBounds(
121129
throw new IllegalArgumentException("Invalid increment " + increment + ", must be a positive integer");
122130
}
123131
List<TransportVersionId> ids = new ArrayList<>();
132+
boolean stageInGit = getResolveConflict().getOrElse(false);
124133

125134
TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromUpstream(definitionName);
126135
for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) {
@@ -134,12 +143,12 @@ private List<TransportVersionId> updateUpperBounds(
134143
int targetIncrement = upperBoundName.equals(currentUpperBoundName) ? increment : 1;
135144
targetId = createTargetId(existingUpperBound, targetIncrement);
136145
var newUpperBound = new TransportVersionUpperBound(upperBoundName, definitionName, targetId);
137-
resources.writeUpperBound(newUpperBound);
146+
resources.writeUpperBound(newUpperBound, stageInGit);
138147
}
139148
ids.add(targetId);
140149
} else {
141150
// Default case: we're not targeting this branch so reset it
142-
resources.writeUpperBound(existingUpperBound);
151+
resources.writeUpperBound(existingUpperBound, false);
143152
}
144153
}
145154

@@ -180,7 +189,19 @@ private String getTargetDefinitionName(
180189
}
181190
}
182191

183-
private Set<String> getTargetUpperBoundNames(List<TransportVersionUpperBound> upstreamUpperBounds) {
192+
private Set<String> getTargetUpperBoundNames(
193+
TransportVersionResourcesService resources,
194+
List<TransportVersionUpperBound> upstreamUpperBounds,
195+
String targetDefinitionName
196+
) throws IOException {
197+
if (getResolveConflict().getOrElse(false)) {
198+
if (getBackportBranches().isPresent()) {
199+
throw new IllegalArgumentException("Cannot use --resolve-conflict with --backport-branches");
200+
}
201+
202+
return getUpperBoundNamesFromDefinition(resources, upstreamUpperBounds, targetDefinitionName);
203+
}
204+
184205
Set<String> targetUpperBoundNames = new HashSet<>();
185206
targetUpperBoundNames.add(getCurrentUpperBoundName().get());
186207
if (getBackportBranches().isPresent()) {
@@ -204,9 +225,32 @@ private Set<String> getTargetUpperBoundNames(List<TransportVersionUpperBound> up
204225
return targetUpperBoundNames;
205226
}
206227

228+
private Set<String> getUpperBoundNamesFromDefinition(
229+
TransportVersionResourcesService resources,
230+
List<TransportVersionUpperBound> upstreamUpperBounds,
231+
String targetDefinitionName
232+
) throws IOException {
233+
TransportVersionDefinition definition = resources.getReferableDefinition(targetDefinitionName);
234+
Set<String> upperBoundNames = new HashSet<>();
235+
upperBoundNames.add(getCurrentUpperBoundName().get());
236+
237+
// skip the primary id as that is current, which we always add
238+
for (int i = 1; i < definition.ids().size(); ++i) {
239+
TransportVersionId id = definition.ids().get(i);
240+
// we have a small number of upper bound files, so just scan for the ones we want
241+
for (TransportVersionUpperBound upperBound : upstreamUpperBounds) {
242+
if (upperBound.definitionId().base() == id.base()) {
243+
upperBoundNames.add(upperBound.name());
244+
}
245+
}
246+
}
247+
248+
return upperBoundNames;
249+
}
250+
207251
private void resetAllUpperBounds(TransportVersionResourcesService resources) throws IOException {
208252
for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromUpstream()) {
209-
resources.writeUpperBound(upperBound);
253+
resources.writeUpperBound(upperBound, false);
210254
}
211255
}
212256

0 commit comments

Comments
 (0)