Skip to content

Commit 850e2c1

Browse files
DaveCTurnerDaveCTurner
committed
Merge branch 'main' into 2025/03/08/snapshot-finalization-threading
2 parents e355340 + fdfe63e commit 850e2c1

File tree

83 files changed

+1713
-2748
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+1713
-2748
lines changed

‎docs/changelog/123757.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 123757
2+
summary: Fix concurrency issue in `ScriptSortBuilder`
3+
area: Search
4+
type: bug
5+
issues: []

‎docs/changelog/124394.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124394
2+
summary: Avoid `NamedWritable` in block serialization
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/124424.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124424
2+
summary: Lazy collection copying during node transform
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
import java.nio.file.Paths;
2424
import java.util.ArrayList;
2525
import java.util.Arrays;
26+
import java.util.HashMap;
2627
import java.util.HashSet;
27-
import java.util.LinkedHashMap;
2828
import java.util.List;
2929
import java.util.Map;
3030
import java.util.Objects;
@@ -55,7 +55,7 @@ public String toString() {
5555
}
5656

5757
static List<ExclusivePath> buildExclusivePathList(List<ExclusiveFileEntitlement> exclusiveFileEntitlements, PathLookup pathLookup) {
58-
Map<String, ExclusivePath> exclusivePaths = new LinkedHashMap<>();
58+
Map<String, ExclusivePath> exclusivePaths = new HashMap<>();
5959
for (ExclusiveFileEntitlement efe : exclusiveFileEntitlements) {
6060
for (FilesEntitlement.FileData fd : efe.filesEntitlement().filesData()) {
6161
if (fd.exclusive()) {

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@
3838
import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED;
3939
import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.SERVER_COMPONENT_NAME;
4040
import static org.hamcrest.Matchers.aMapWithSize;
41-
import static org.hamcrest.Matchers.equalTo;
41+
import static org.hamcrest.Matchers.allOf;
42+
import static org.hamcrest.Matchers.containsString;
4243
import static org.hamcrest.Matchers.is;
4344
import static org.hamcrest.Matchers.sameInstance;
4445

@@ -493,9 +494,11 @@ public void testFilesEntitlementsWithExclusive() {
493494
);
494495
assertThat(
495496
iae.getMessage(),
496-
equalTo(
497-
"Path [/base/test] is already exclusive to [plugin1][test.module1],"
498-
+ " cannot add exclusive access for [plugin2][test.module2]"
497+
allOf(
498+
containsString("Path [/base/test] is already exclusive"),
499+
containsString("[plugin1][test.module1]"),
500+
containsString("[plugin2][test.module2]"),
501+
containsString("cannot add exclusive access")
499502
)
500503
);
501504

‎modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,3 +482,118 @@
482482
}]
483483
- match: { error.root_cause.0.type: "illegal_argument_exception" }
484484
- match: { error.root_cause.0.reason: "script score function must not produce negative scores, but got: [-9.0]"}
485+
486+
---
487+
488+
"Script Sort + _score":
489+
- do:
490+
index:
491+
index: test
492+
id: "1"
493+
body: { "test": "a", "num1": 1.0, "type" : "first" }
494+
- do:
495+
index:
496+
index: test
497+
id: "2"
498+
body: { "test": "b", "num1": 2.0, "type" : "first" }
499+
- do:
500+
index:
501+
index: test
502+
id: "3"
503+
body: { "test": "c", "num1": 3.0, "type" : "first" }
504+
- do:
505+
index:
506+
index: test
507+
id: "4"
508+
body: { "test": "d", "num1": 4.0, "type" : "second" }
509+
- do:
510+
index:
511+
index: test
512+
id: "5"
513+
body: { "test": "e", "num1": 5.0, "type" : "second" }
514+
- do:
515+
indices.refresh: {}
516+
517+
- do:
518+
search:
519+
rest_total_hits_as_int: true
520+
index: test
521+
body:
522+
sort: [
523+
{
524+
_script: {
525+
script: {
526+
lang: "painless",
527+
source: "doc['num1'].value + _score"
528+
},
529+
type: "number"
530+
}
531+
}
532+
]
533+
534+
- match: { hits.total: 5 }
535+
- match: { hits.hits.0.sort.0: 2.0 }
536+
- match: { hits.hits.1.sort.0: 3.0 }
537+
- match: { hits.hits.2.sort.0: 4.0 }
538+
- match: { hits.hits.3.sort.0: 5.0 }
539+
- match: { hits.hits.4.sort.0: 6.0 }
540+
541+
- do:
542+
search:
543+
rest_total_hits_as_int: true
544+
index: test
545+
body:
546+
sort: [
547+
{
548+
_script: {
549+
script: {
550+
lang: "painless",
551+
source: "doc['test.keyword'].value + '-' + _score"
552+
},
553+
type: "string"
554+
}
555+
}
556+
]
557+
558+
- match: { hits.total: 5 }
559+
- match: { hits.hits.0.sort.0: "a-1.0" }
560+
- match: { hits.hits.1.sort.0: "b-1.0" }
561+
- match: { hits.hits.2.sort.0: "c-1.0" }
562+
- match: { hits.hits.3.sort.0: "d-1.0" }
563+
- match: { hits.hits.4.sort.0: "e-1.0" }
564+
565+
- do:
566+
search:
567+
rest_total_hits_as_int: true
568+
index: test
569+
body:
570+
aggs:
571+
test:
572+
terms:
573+
field: type.keyword
574+
aggs:
575+
top_hits:
576+
top_hits:
577+
sort: [
578+
{
579+
_script: {
580+
script: {
581+
lang: "painless",
582+
source: "doc['test.keyword'].value + '-' + _score"
583+
},
584+
type: "string"
585+
}
586+
},
587+
"_score"
588+
]
589+
size: 1
590+
591+
- match: { hits.total: 5 }
592+
- match: { aggregations.test.buckets.0.key: "first" }
593+
- match: { aggregations.test.buckets.0.top_hits.hits.total: 3 }
594+
- match: { aggregations.test.buckets.0.top_hits.hits.hits.0.sort.0: "a-1.0" }
595+
- match: { aggregations.test.buckets.0.top_hits.hits.hits.0.sort.1: 1.0 }
596+
- match: { aggregations.test.buckets.1.key: "second" }
597+
- match: { aggregations.test.buckets.1.top_hits.hits.total: 2 }
598+
- match: { aggregations.test.buckets.1.top_hits.hits.hits.0.sort.0: "d-1.0" }
599+
- match: { aggregations.test.buckets.1.top_hits.hits.hits.0.sort.1: 1.0 }

‎muted-tests.yml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -330,12 +330,6 @@ tests:
330330
- class: org.elasticsearch.smoketest.MlWithSecurityIT
331331
method: test {yaml=ml/3rd_party_deployment/Test start deployment fails while model download in progress}
332332
issue: https://github.com/elastic/elasticsearch/issues/120814
333-
- class: org.elasticsearch.search.query.QueryPhaseTimeoutTests
334-
method: testScorerTimeoutPoints
335-
issue: https://github.com/elastic/elasticsearch/issues/124140
336-
- class: org.elasticsearch.search.query.QueryPhaseTimeoutTests
337-
method: testScorerTimeoutTerms
338-
issue: https://github.com/elastic/elasticsearch/issues/124141
339333
- class: org.elasticsearch.smoketest.MlWithSecurityIT
340334
method: test {yaml=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable is missing}
341335
issue: https://github.com/elastic/elasticsearch/issues/124168
@@ -366,6 +360,9 @@ tests:
366360
- class: org.elasticsearch.indices.stats.IndexStatsIT
367361
method: testFilterCacheStats
368362
issue: https://github.com/elastic/elasticsearch/issues/124447
363+
- class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
364+
method: test {yaml=data_stream/190_failure_store_redirection/Redirect ingest failure in data stream to failure store}
365+
issue: https://github.com/elastic/elasticsearch/issues/124518
369366

370367
# Examples:
371368
#

‎server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.common.document.DocumentField;
1919
import org.elasticsearch.common.settings.Settings;
2020
import org.elasticsearch.index.IndexSettings;
21+
import org.elasticsearch.index.fielddata.ScriptDocValues;
2122
import org.elasticsearch.index.query.MatchAllQueryBuilder;
2223
import org.elasticsearch.index.query.QueryBuilders;
2324
import org.elasticsearch.index.seqno.SequenceNumbers;
@@ -64,6 +65,7 @@
6465
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
6566
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
6667
import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
68+
import static org.elasticsearch.script.MockScriptPlugin.NAME;
6769
import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
6870
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
6971
import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
@@ -102,7 +104,12 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
102104
public static class CustomScriptPlugin extends MockScriptPlugin {
103105
@Override
104106
protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
105-
return Collections.singletonMap("5", script -> "5");
107+
return Map.of("5", script -> "5", "doc['sort'].value", CustomScriptPlugin::sortDoubleScript);
108+
}
109+
110+
private static Double sortDoubleScript(Map<String, Object> vars) {
111+
Map<?, ?> doc = (Map) vars.get("doc");
112+
return ((Number) ((ScriptDocValues<?>) doc.get("sort")).get(0)).doubleValue();
106113
}
107114

108115
@Override
@@ -1268,6 +1275,41 @@ public void testWithRescore() {
12681275
);
12691276
}
12701277

1278+
public void testScriptSorting() {
1279+
Script script = new Script(ScriptType.INLINE, NAME, "doc['sort'].value", Collections.emptyMap());
1280+
assertNoFailuresAndResponse(
1281+
prepareSearch("idx").addAggregation(
1282+
terms("terms").executionHint(randomExecutionHint())
1283+
.field(TERMS_AGGS_FIELD)
1284+
.subAggregation(topHits("hits").sort(SortBuilders.scriptSort(script, ScriptSortType.NUMBER).order(SortOrder.DESC)))
1285+
),
1286+
response -> {
1287+
Terms terms = response.getAggregations().get("terms");
1288+
assertThat(terms, notNullValue());
1289+
assertThat(terms.getName(), equalTo("terms"));
1290+
assertThat(terms.getBuckets().size(), equalTo(5));
1291+
1292+
double higestSortValue = 0;
1293+
for (int i = 0; i < 5; i++) {
1294+
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
1295+
assertThat(bucket, notNullValue());
1296+
assertThat(key(bucket), equalTo("val" + i));
1297+
assertThat(bucket.getDocCount(), equalTo(10L));
1298+
TopHits topHits = bucket.getAggregations().get("hits");
1299+
SearchHits hits = topHits.getHits();
1300+
assertThat(hits.getTotalHits().value(), equalTo(10L));
1301+
assertThat(hits.getHits().length, equalTo(3));
1302+
higestSortValue += 10;
1303+
assertThat((Double) hits.getAt(0).getSortValues()[0], equalTo(higestSortValue));
1304+
assertThat((Double) hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
1305+
assertThat((Double) hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2));
1306+
1307+
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
1308+
}
1309+
}
1310+
);
1311+
}
1312+
12711313
public static class FetchPlugin extends Plugin implements SearchPlugin {
12721314
@Override
12731315
public List<FetchSubPhase> getFetchSubPhases(FetchPhaseConstructionContext context) {

‎server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -87,21 +87,21 @@ public static class CustomScriptPlugin extends MockScriptPlugin {
8787
@Override
8888
protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
8989
Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
90-
scripts.put("doc['number'].value", vars -> sortDoubleScript(vars));
91-
scripts.put("doc['keyword'].value", vars -> sortStringScript(vars));
90+
scripts.put("doc['number'].value", CustomScriptPlugin::sortDoubleScript);
91+
scripts.put("doc['keyword'].value", CustomScriptPlugin::sortStringScript);
9292
return scripts;
9393
}
9494

95-
static Double sortDoubleScript(Map<String, Object> vars) {
95+
private static Double sortDoubleScript(Map<String, Object> vars) {
9696
Map<?, ?> doc = (Map) vars.get("doc");
97-
Double index = ((Number) ((ScriptDocValues<?>) doc.get("number")).get(0)).doubleValue();
98-
return index;
97+
Double score = (Double) vars.get("_score");
98+
return ((Number) ((ScriptDocValues<?>) doc.get("number")).get(0)).doubleValue() + score;
9999
}
100100

101-
static String sortStringScript(Map<String, Object> vars) {
101+
private static String sortStringScript(Map<String, Object> vars) {
102102
Map<?, ?> doc = (Map) vars.get("doc");
103-
String value = ((String) ((ScriptDocValues<?>) doc.get("keyword")).get(0));
104-
return value;
103+
Double score = (Double) vars.get("_score");
104+
return ((ScriptDocValues<?>) doc.get("keyword")).get(0) + ",_score=" + score;
105105
}
106106
}
107107

@@ -1665,14 +1665,14 @@ public void testCustomFormat() throws Exception {
16651665
);
16661666
}
16671667

1668-
public void testScriptFieldSort() throws Exception {
1668+
public void testScriptFieldSort() {
16691669
assertAcked(prepareCreate("test").setMapping("keyword", "type=keyword", "number", "type=integer"));
16701670
ensureGreen();
16711671
final int numDocs = randomIntBetween(10, 20);
16721672
IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
16731673
List<String> keywords = new ArrayList<>();
16741674
for (int i = 0; i < numDocs; ++i) {
1675-
indexReqs[i] = prepareIndex("test").setSource("number", i, "keyword", Integer.toString(i));
1675+
indexReqs[i] = prepareIndex("test").setSource("number", i, "keyword", Integer.toString(i), "version", i + "." + i);
16761676
keywords.add(Integer.toString(i));
16771677
}
16781678
Collections.sort(keywords);
@@ -1686,7 +1686,7 @@ public void testScriptFieldSort() throws Exception {
16861686
.addSort(SortBuilders.scriptSort(script, ScriptSortBuilder.ScriptSortType.NUMBER))
16871687
.addSort(SortBuilders.scoreSort()),
16881688
response -> {
1689-
double expectedValue = 0;
1689+
double expectedValue = 1; // start from 1 because it includes _score, 1.0f for all docs
16901690
for (SearchHit hit : response.getHits()) {
16911691
assertThat(hit.getSortValues().length, equalTo(2));
16921692
assertThat(hit.getSortValues()[0], equalTo(expectedValue++));
@@ -1707,7 +1707,7 @@ public void testScriptFieldSort() throws Exception {
17071707
int expectedValue = 0;
17081708
for (SearchHit hit : response.getHits()) {
17091709
assertThat(hit.getSortValues().length, equalTo(2));
1710-
assertThat(hit.getSortValues()[0], equalTo(keywords.get(expectedValue++)));
1710+
assertThat(hit.getSortValues()[0], equalTo(keywords.get(expectedValue++) + ",_score=1.0"));
17111711
assertThat(hit.getSortValues()[1], equalTo(1f));
17121712
}
17131713
}

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ static TransportVersion def(int id) {
179179
public static final TransportVersion INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM = def(9_023_0_00);
180180
public static final TransportVersion MAX_OPERATION_SIZE_REJECTIONS_ADDED = def(9_024_0_00);
181181
public static final TransportVersion RETRY_ILM_ASYNC_ACTION_REQUIRE_ERROR = def(9_025_0_00);
182+
public static final TransportVersion ESQL_SERIALIZE_BLOCK_TYPE_CODE = def(9_026_0_00);
182183

183184
/*
184185
* STOP! READ THIS FIRST! No, really,

0 commit comments

Comments
 (0)