Skip to content

Commit aa8505e

Browse files
Tim-BrooksTim-Brooks
committed
Merge remote-tracking branch 'origin/main' into ankikuma-09162025/ReshardSplitRequestOnSourceTwoPass
2 parents 5ba4e73 + fe44986 commit aa8505e

File tree

630 files changed

+9650
-4170
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

630 files changed

+9650
-4170
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public void setup() {
119119
}
120120

121121
private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
122-
var parsed = parser.createStatement(query, new QueryParams(), telemetry, config);
122+
var parsed = parser.createStatement(query, new QueryParams(), telemetry);
123123
var analyzed = analyzer.analyze(parsed);
124124
var optimized = optimizer.optimize(analyzed);
125125
return optimized;

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@
4141
import org.elasticsearch.compute.data.LongBlock;
4242
import org.elasticsearch.compute.data.LongVector;
4343
import org.elasticsearch.compute.data.Page;
44+
import org.elasticsearch.compute.lucene.AlwaysReferencedIndexedByShardId;
45+
import org.elasticsearch.compute.lucene.IndexedByShardIdFromSingleton;
4446
import org.elasticsearch.compute.lucene.LuceneSourceOperator;
45-
import org.elasticsearch.compute.lucene.ShardRefCounted;
4647
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
4748
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorStatus;
4849
import org.elasticsearch.compute.operator.topn.TopNOperator;
@@ -368,7 +369,7 @@ public void benchmark() {
368369
blockFactory,
369370
ByteSizeValue.ofMb(1).getBytes(),
370371
fields(name),
371-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
372+
new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
372373
throw new UnsupportedOperationException("can't load _source here");
373374
}, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
374375
0
@@ -538,7 +539,7 @@ private void setupPages() {
538539
pages.add(
539540
new Page(
540541
new DocVector(
541-
ShardRefCounted.ALWAYS_REFERENCED,
542+
AlwaysReferencedIndexedByShardId.INSTANCE,
542543
blockFactory.newConstantIntBlockWith(0, end - begin).asVector(),
543544
blockFactory.newConstantIntBlockWith(ctx.ord, end - begin).asVector(),
544545
docs.build(),
@@ -575,8 +576,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
575576
pages.add(
576577
new Page(
577578
new DocVector(
578-
579-
ShardRefCounted.ALWAYS_REFERENCED,
579+
AlwaysReferencedIndexedByShardId.INSTANCE,
580580
blockFactory.newConstantIntVector(0, size),
581581
leafs.build(),
582582
docs.build(),
@@ -594,7 +594,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
594594
pages.add(
595595
new Page(
596596
new DocVector(
597-
ShardRefCounted.ALWAYS_REFERENCED,
597+
AlwaysReferencedIndexedByShardId.INSTANCE,
598598
blockFactory.newConstantIntBlockWith(0, size).asVector(),
599599
leafs.build().asBlock().asVector(),
600600
docs.build(),
@@ -621,8 +621,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
621621
pages.add(
622622
new Page(
623623
new DocVector(
624-
625-
ShardRefCounted.ALWAYS_REFERENCED,
624+
AlwaysReferencedIndexedByShardId.INSTANCE,
626625
blockFactory.newConstantIntVector(0, 1),
627626
blockFactory.newConstantIntVector(next.ord, 1),
628627
blockFactory.newConstantIntVector(next.itr.nextInt(), 1),

‎benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
import org.apache.lucene.index.IndexWriter;
1616
import org.apache.lucene.index.IndexWriterConfig;
1717
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
18-
import org.apache.lucene.index.SortedNumericDocValues;
1918
import org.apache.lucene.search.IndexSearcher;
2019
import org.apache.lucene.search.MatchAllDocsQuery;
2120
import org.apache.lucene.search.Query;
@@ -29,6 +28,7 @@
2928
import org.elasticsearch.index.fielddata.FieldDataContext;
3029
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
3130
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
31+
import org.elasticsearch.index.fielddata.SortedNumericLongValues;
3232
import org.elasticsearch.index.mapper.IndexType;
3333
import org.elasticsearch.index.mapper.MappedFieldType;
3434
import org.elasticsearch.index.mapper.MappingLookup;
@@ -179,14 +179,14 @@ private ScoreScript.Factory bareMetalScript() {
179179
return new ScoreScript.LeafFactory() {
180180
@Override
181181
public ScoreScript newInstance(DocReader docReader) throws IOException {
182-
SortedNumericDocValues values = ifd.load(((DocValuesDocReader) docReader).getLeafReaderContext()).getLongValues();
182+
SortedNumericLongValues values = ifd.load(((DocValuesDocReader) docReader).getLeafReaderContext()).getLongValues();
183183
return new ScoreScript(params, null, docReader) {
184184
private int docId;
185185

186186
@Override
187187
public double execute(ExplanationHolder explanation) {
188188
try {
189-
values.advance(docId);
189+
values.advanceExact(docId);
190190
if (values.docValueCount() != 1) {
191191
throw new IllegalArgumentException("script only works when there is exactly one value");
192192
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,4 +311,43 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
311311
then:
312312
result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
313313
}
314+
315+
def "modification checks use PR base branch in CI"() {
316+
given:
317+
// create 9.1 branch
318+
execute("git checkout -b 9.1")
319+
// setup 9.1 upper bound as if 9.2 was just branched, so no patches yet
320+
unreferableTransportVersion("initial_9.1.0", "8013000")
321+
transportVersionUpperBound("9.1", "initial_9.1.0", "8013000")
322+
execute("git add .")
323+
execute("git commit -m initial")
324+
325+
// advance main branch
326+
execute("git checkout main")
327+
unreferableTransportVersion("initial_9.1.0", "8013000")
328+
referableAndReferencedTransportVersion("new_tv", "8124000,8013001")
329+
transportVersionUpperBound("9.2", "new_tv", "8124000")
330+
transportVersionUpperBound("9.1", "new_tv", "8013001")
331+
execute("git add .")
332+
execute("git commit -m update")
333+
334+
// create a 9.1 backport
335+
execute("git checkout 9.1")
336+
execute("git checkout -b test_9.1")
337+
referableAndReferencedTransportVersion("new_tv", "8124000,8013001")
338+
transportVersionUpperBound("9.2", "new_tv", "8124000")
339+
transportVersionUpperBound("9.1", "new_tv", "8013001")
340+
file("myserver/build.gradle") << """
341+
tasks.named('validateTransportVersionResources') {
342+
currentUpperBoundName = '9.1'
343+
}
344+
"""
345+
346+
when:
347+
def result = gradleRunner("validateTransportVersionResources")
348+
.withEnvironment(Map.of("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "9.1")).build()
349+
350+
then:
351+
result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
352+
}
314353
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,23 @@
1414
import org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin;
1515
import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
1616
import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
17+
import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
18+
import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
1719
import org.gradle.api.Action;
1820
import org.gradle.api.Plugin;
1921
import org.gradle.api.Project;
2022
import org.gradle.api.file.Directory;
2123
import org.gradle.api.plugins.JavaPlugin;
24+
import org.gradle.api.provider.Property;
2225
import org.gradle.api.provider.Provider;
2326
import org.gradle.api.tasks.Copy;
2427
import org.gradle.language.base.plugins.LifecycleBasePlugin;
2528

2629
import java.util.Map;
2730
import java.util.Properties;
2831

32+
import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
33+
2934
public class TransportVersionResourcesPlugin implements Plugin<Project> {
3035

3136
public static final String TRANSPORT_REFERENCES_TOPIC = "transportReferences";
@@ -37,6 +42,9 @@ public void apply(Project project) {
3742
project.getPluginManager().apply(PrecommitTaskPlugin.class);
3843
var psService = project.getPlugins().apply(ProjectSubscribeServicePlugin.class).getService();
3944

45+
project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
46+
Property<BuildParameterExtension> buildParams = loadBuildParams(project);
47+
4048
Properties versions = (Properties) project.getExtensions().getByName(VersionPropertiesPlugin.VERSIONS_EXT);
4149
Version currentVersion = Version.fromString(versions.getProperty("elasticsearch"));
4250

@@ -76,6 +84,7 @@ public void apply(Project project) {
7684
t.getShouldValidateDensity().convention(true);
7785
t.getShouldValidatePrimaryIdNotPatch().convention(true);
7886
t.getCurrentUpperBoundName().convention(currentVersion.getMajor() + "." + currentVersion.getMinor());
87+
t.getCI().set(buildParams.get().getCi());
7988
});
8089
project.getTasks().named(PrecommitPlugin.PRECOMMIT_TASK_NAME).configure(t -> t.dependsOn(validateTask));
8190

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,11 @@ private String findUpstreamRef() {
284284
logger.warn("No remotes found. Using 'main' branch as upstream ref for transport version resources");
285285
return "main";
286286
}
287+
// default the branch name to look at to that which a PR in CI is targeting
288+
String branchName = System.getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH");
289+
if (branchName == null || branchName.strip().isEmpty()) {
290+
branchName = "main";
291+
}
287292
List<String> remoteNames = List.of(remotesOutput.split("\n"));
288293
if (remoteNames.contains(UPSTREAM_REMOTE_NAME) == false) {
289294
// our special remote doesn't exist yet, so create it
@@ -299,14 +304,14 @@ private String findUpstreamRef() {
299304
gitCommand("remote", "add", UPSTREAM_REMOTE_NAME, upstreamUrl);
300305
} else {
301306
logger.warn("No elastic github remotes found to copy. Using 'main' branch as upstream ref for transport version resources");
302-
return "main";
307+
return branchName;
303308
}
304309
}
305310

306311
// make sure the remote main ref is up to date
307-
gitCommand("fetch", UPSTREAM_REMOTE_NAME, "main");
312+
gitCommand("fetch", UPSTREAM_REMOTE_NAME, branchName);
308313

309-
return UPSTREAM_REMOTE_NAME + "/main";
314+
return UPSTREAM_REMOTE_NAME + "/" + branchName;
310315
}
311316

312317
// Return the transport version resources paths that exist in upstream

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java‎

Lines changed: 51 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ public Path getResourcesDir() {
6666
@Input
6767
public abstract Property<String> getCurrentUpperBoundName();
6868

69+
@Input
70+
public abstract Property<Boolean> getCI();
71+
6972
private static final Pattern NAME_FORMAT = Pattern.compile("[a-z0-9_]+");
7073

7174
@ServiceReference("transportVersionResources")
@@ -82,13 +85,14 @@ public void validateTransportVersions() throws IOException {
8285
Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
8386
TransportVersionUpperBound currentUpperBound = upperBounds.get(getCurrentUpperBoundName().get());
8487
boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
88+
boolean validateModifications = onReleaseBranch == false || getCI().get();
8589

8690
for (var definition : referableDefinitions.values()) {
87-
validateNamedDefinition(definition, referencedNames);
91+
validateNamedDefinition(definition, referencedNames, validateModifications);
8892
}
8993

9094
for (var definition : unreferableDefinitions.values()) {
91-
validateUnreferableDefinition(definition);
95+
validateUnreferableDefinition(definition, validateModifications);
9296
}
9397

9498
for (var entry : idsByBase.entrySet()) {
@@ -101,10 +105,10 @@ public void validateTransportVersions() throws IOException {
101105

102106
if (onReleaseBranch) {
103107
// on release branches we only check the current upper bound, others may be inaccurate
104-
validateUpperBound(currentUpperBound, allDefinitions, idsByBase);
108+
validateUpperBound(currentUpperBound, allDefinitions, idsByBase, validateModifications);
105109
} else {
106110
for (var upperBound : upperBounds.values()) {
107-
validateUpperBound(upperBound, allDefinitions, idsByBase);
111+
validateUpperBound(upperBound, allDefinitions, idsByBase, validateModifications);
108112
}
109113

110114
validatePrimaryIds(resources, upperBounds, allDefinitions);
@@ -126,7 +130,11 @@ private Map<String, TransportVersionDefinition> collectAllDefinitions(
126130
return allDefinitions;
127131
}
128132

129-
private void validateNamedDefinition(TransportVersionDefinition definition, Set<String> referencedNames) {
133+
private void validateNamedDefinition(
134+
TransportVersionDefinition definition,
135+
Set<String> referencedNames,
136+
boolean validateModifications
137+
) {
130138
if (referencedNames.contains(definition.name()) == false) {
131139
throwDefinitionFailure(definition, "is not referenced");
132140
}
@@ -152,35 +160,39 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set<
152160
}
153161
}
154162
}
155-
// validate any modifications
156-
TransportVersionDefinition originalDefinition = getResources().get().getReferableDefinitionFromGitBase(definition.name());
157-
if (originalDefinition != null) {
158-
validateIdenticalPrimaryId(definition, originalDefinition);
159-
for (int i = 1; i < originalDefinition.ids().size(); ++i) {
160-
TransportVersionId originalId = originalDefinition.ids().get(i);
161-
162-
// we have a very small number of ids in a definition, so just search linearly
163-
boolean found = false;
164-
for (int j = 1; j < definition.ids().size(); ++j) {
165-
TransportVersionId id = definition.ids().get(j);
166-
if (id.base() == originalId.base()) {
167-
found = true;
168-
if (id.complete() != originalId.complete()) {
169-
throwDefinitionFailure(definition, "has modified patch id from " + originalId + " to " + id);
163+
164+
if (validateModifications) {
165+
TransportVersionDefinition originalDefinition = getResources().get().getReferableDefinitionFromGitBase(definition.name());
166+
if (originalDefinition != null) {
167+
validateIdenticalPrimaryId(definition, originalDefinition);
168+
for (int i = 1; i < originalDefinition.ids().size(); ++i) {
169+
TransportVersionId originalId = originalDefinition.ids().get(i);
170+
171+
// we have a very small number of ids in a definition, so just search linearly
172+
boolean found = false;
173+
for (int j = 1; j < definition.ids().size(); ++j) {
174+
TransportVersionId id = definition.ids().get(j);
175+
if (id.base() == originalId.base()) {
176+
found = true;
177+
if (id.complete() != originalId.complete()) {
178+
throwDefinitionFailure(definition, "has modified patch id from " + originalId + " to " + id);
179+
}
170180
}
171181
}
172-
}
173-
if (found == false) {
174-
throwDefinitionFailure(definition, "has removed id " + originalId);
182+
if (found == false) {
183+
throwDefinitionFailure(definition, "has removed id " + originalId);
184+
}
175185
}
176186
}
177187
}
178188
}
179189

180-
private void validateUnreferableDefinition(TransportVersionDefinition definition) {
181-
TransportVersionDefinition originalDefinition = getResources().get().getUnreferableDefinitionFromGitBase(definition.name());
182-
if (originalDefinition != null) {
183-
validateIdenticalPrimaryId(definition, originalDefinition);
190+
private void validateUnreferableDefinition(TransportVersionDefinition definition, boolean validateModifications) {
191+
if (validateModifications) {
192+
TransportVersionDefinition originalDefinition = getResources().get().getUnreferableDefinitionFromGitBase(definition.name());
193+
if (originalDefinition != null) {
194+
validateIdenticalPrimaryId(definition, originalDefinition);
195+
}
184196
}
185197
if (definition.ids().isEmpty()) {
186198
throwDefinitionFailure(definition, "does not contain any ids");
@@ -204,7 +216,8 @@ private void validateIdenticalPrimaryId(TransportVersionDefinition definition, T
204216
private void validateUpperBound(
205217
TransportVersionUpperBound upperBound,
206218
Map<String, TransportVersionDefinition> definitions,
207-
Map<Integer, List<IdAndDefinition>> idsByBase
219+
Map<Integer, List<IdAndDefinition>> idsByBase,
220+
boolean validateModifications
208221
) {
209222
TransportVersionDefinition upperBoundDefinition = definitions.get(upperBound.definitionName());
210223
if (upperBoundDefinition == null) {
@@ -240,13 +253,16 @@ private void validateUpperBound(
240253
);
241254
}
242255

243-
TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromGitBase(upperBound.name());
244-
if (existingUpperBound != null && getShouldValidatePrimaryIdNotPatch().get()) {
245-
if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) {
246-
throwUpperBoundFailure(
247-
upperBound,
248-
"modifies base id from " + existingUpperBound.definitionId().base() + " to " + upperBound.definitionId().base()
249-
);
256+
if (validateModifications) {
257+
TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromGitBase(upperBound.name());
258+
if (existingUpperBound != null && getShouldValidatePrimaryIdNotPatch().get()) {
259+
if (upperBound.definitionId().patch() != 0
260+
&& upperBound.definitionId().base() != existingUpperBound.definitionId().base()) {
261+
throwUpperBoundFailure(
262+
upperBound,
263+
"modifies base id from " + existingUpperBound.definitionId().base() + " to " + upperBound.definitionId().base()
264+
);
265+
}
250266
}
251267
}
252268
}

‎docs/changelog/132757.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132757
2+
summary: Late materialization after TopN (Node level)
3+
area: ES|QL
4+
type: feature
5+
issues: []

‎docs/changelog/135674.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135674
2+
summary: Send cross cluster api key signature as headers
3+
area: Security
4+
type: enhancement
5+
issues: []

‎docs/changelog/135701.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135701
2+
summary: Add Google Model Garden's Meta, Mistral, Hugging Face and Ai21 providers support to Inference Plugin
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

0 commit comments

Comments
 (0)