Skip to content

Commit 7ee7dac

Browse files
breskebybreskeby
authored
Merge branch 'main' into update-gradle-wrapper-920
2 parents 46c8dc9 + 88a6827 commit 7ee7dac

File tree

23 files changed

+688
-299
lines changed

23 files changed

+688
-299
lines changed

‎docs/changelog/136086.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 136086
2+
summary: Delay automaton creation in `BinaryDvConfirmedQuery` to avoid OOM on queries
3+
against `WildCard` fields
4+
area: Search
5+
type: bug
6+
issues: []

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,9 @@ public void testGetAdditionalIndexSettings() throws Exception {
7171
String dataStreamName = "logs-app1";
7272

7373
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
74-
Settings settings = Settings.EMPTY;
74+
Settings settings = Settings.builder()
75+
.put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabledSetting)
76+
.build();
7577
String mapping = """
7678
{
7779
"_doc": {
@@ -117,11 +119,8 @@ public void testGetAdditionalIndexSettings() throws Exception {
117119
Settings result = additionalSettings.build();
118120
// The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
119121
// (in production the index.mode setting is usually provided in an index or component template)
120-
result = builder().put(result)
121-
.put("index.mode", "time_series")
122-
.put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabledSetting)
123-
.build();
124-
assertThat(result.size(), equalTo(5));
122+
result = builder().put(result).put("index.mode", "time_series").build();
123+
assertThat(result.size(), equalTo(4));
125124
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
126125
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
127126
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
@@ -187,7 +186,9 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
187186
String dataStreamName = "logs-app1";
188187

189188
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
190-
Settings settings = Settings.EMPTY;
189+
Settings settings = Settings.builder()
190+
.put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabledSetting)
191+
.build();
191192
String mapping1 = """
192193
{
193194
"_doc": {
@@ -249,11 +250,8 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
249250
Settings result = additionalSettings.build();
250251
// The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
251252
// (in production the index.mode setting is usually provided in an index or component template)
252-
result = builder().put(result)
253-
.put("index.mode", "time_series")
254-
.put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabledSetting)
255-
.build();
256-
assertThat(result.size(), equalTo(5));
253+
result = builder().put(result).put("index.mode", "time_series").build();
254+
assertThat(result.size(), equalTo(4));
257255
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
258256
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
259257
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));

‎muted-tests.yml‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -612,15 +612,9 @@ tests:
612612
- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithPartialResultsIT
613613
method: testOneRemoteClusterPartial
614614
issue: https://github.com/elastic/elasticsearch/issues/124055
615-
- class: org.elasticsearch.datastreams.DataStreamIndexSettingsProviderTests
616-
method: testGetAdditionalIndexSettingsMappingsMerging
617-
issue: https://github.com/elastic/elasticsearch/issues/135884
618615
- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
619616
method: testTopNPushedToLuceneOnSortedIndex
620617
issue: https://github.com/elastic/elasticsearch/issues/135939
621-
- class: org.elasticsearch.datastreams.DataStreamIndexSettingsProviderTests
622-
method: testGetAdditionalIndexSettings
623-
issue: https://github.com/elastic/elasticsearch/issues/135972
624618
- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
625619
method: test {yaml=indices.get_sample/10_basic/Test get sample for index with no sample config}
626620
issue: https://github.com/elastic/elasticsearch/issues/135975
@@ -696,6 +690,18 @@ tests:
696690
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
697691
method: test {csv-spec:semantic_text.Repeat}
698692
issue: https://github.com/elastic/elasticsearch/issues/136150
693+
- class: org.elasticsearch.xpack.esql.inference.InferenceFunctionEvaluatorTests
694+
method: testFoldTextEmbeddingFunction
695+
issue: https://github.com/elastic/elasticsearch/issues/136154
696+
- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1UnavailableRemotesIT
697+
method: testEsqlRcs1UnavailableRemoteScenarios
698+
issue: https://github.com/elastic/elasticsearch/issues/136157
699+
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
700+
method: test {csv-spec:comparison.IntToInt}
701+
issue: https://github.com/elastic/elasticsearch/issues/136160
702+
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
703+
method: test {csv-spec:math.LeastGreatestMany}
704+
issue: https://github.com/elastic/elasticsearch/issues/136161
699705

700706
# Examples:
701707
#

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,12 +204,26 @@ public void testIndicesDontExist() throws IOException {
204204
ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo"))));
205205
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
206206
assertThat(e.getMessage(), containsString("verification_exception"));
207-
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]")));
207+
assertThat(
208+
e.getMessage(),
209+
anyOf(
210+
containsString("Unknown index [foo]"),
211+
containsString("Unknown index [*:foo]"),
212+
containsString("Unknown index [remote_cluster:foo]")
213+
)
214+
);
208215

209216
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
210217
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
211218
assertThat(e.getMessage(), containsString("verification_exception"));
212-
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]")));
219+
assertThat(
220+
e.getMessage(),
221+
anyOf(
222+
containsString("Unknown index [foo*]"),
223+
containsString("Unknown index [*:foo*]"),
224+
containsString("Unknown index [remote_cluster:foo*]")
225+
)
226+
);
213227

214228
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo, test1")));
215229
assertEquals(404, e.getResponse().getStatusLine().getStatusCode());

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.common.breaker.CircuitBreaker;
2222
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
2323
import org.elasticsearch.common.bytes.BytesReference;
24+
import org.elasticsearch.common.collect.Iterators;
2425
import org.elasticsearch.common.lucene.BytesRefs;
2526
import org.elasticsearch.common.regex.Regex;
2627
import org.elasticsearch.common.settings.Settings;
@@ -574,23 +575,19 @@ public static List<List<Object>> getValuesList(EsqlQueryResponse results) {
574575
}
575576

576577
public static List<List<Object>> getValuesList(Iterator<Iterator<Object>> values) {
577-
var valuesList = new ArrayList<List<Object>>();
578-
values.forEachRemaining(row -> {
579-
var rowValues = new ArrayList<>();
580-
row.forEachRemaining(rowValues::add);
581-
valuesList.add(rowValues);
582-
});
583-
return valuesList;
578+
return toList(Iterators.map(values, EsqlTestUtils::toList));
584579
}
585580

586581
public static List<List<Object>> getValuesList(Iterable<Iterable<Object>> values) {
587-
var valuesList = new ArrayList<List<Object>>();
588-
values.iterator().forEachRemaining(row -> {
589-
var rowValues = new ArrayList<>();
590-
row.iterator().forEachRemaining(rowValues::add);
591-
valuesList.add(rowValues);
592-
});
593-
return valuesList;
582+
return toList(Iterators.map(values.iterator(), row -> toList(row.iterator())));
583+
}
584+
585+
private static <T> List<T> toList(Iterator<T> iterator) {
586+
var list = new ArrayList<T>();
587+
while (iterator.hasNext()) {
588+
list.add(iterator.next());
589+
}
590+
return list;
594591
}
595592

596593
public static List<String> withDefaultLimitWarning(List<String> warnings) {

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterLookupJoinIT.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
1212
import org.elasticsearch.client.internal.Client;
1313
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.index.IndexMode;
15+
import org.elasticsearch.index.IndexSettings;
16+
import org.elasticsearch.index.query.TermQueryBuilder;
1417
import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
1518
import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
1619
import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction;
@@ -28,6 +31,7 @@
2831

2932
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
3033
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
34+
import static org.elasticsearch.xpack.esql.action.EsqlQueryRequest.syncEsqlQueryRequest;
3135
import static org.hamcrest.Matchers.containsString;
3236
import static org.hamcrest.Matchers.empty;
3337
import static org.hamcrest.Matchers.equalTo;
@@ -557,6 +561,57 @@ public void testLookupJoinFieldTypes() throws IOException {
557561
}
558562
}
559563

564+
public void testAlwaysAppliesTheFilter() throws IOException {
565+
setupClusters(3);
566+
setSkipUnavailable(REMOTE_CLUSTER_1, false);
567+
setSkipUnavailable(REMOTE_CLUSTER_2, false);
568+
569+
var defaultSettings = Settings.builder();
570+
createIndexWithDocument(LOCAL_CLUSTER, "data", defaultSettings, Map.of("key", 1, "f1", 1));
571+
createIndexWithDocument(REMOTE_CLUSTER_1, "data", defaultSettings, Map.of("key", 2, "f2", 2));
572+
createIndexWithDocument(REMOTE_CLUSTER_2, "data", defaultSettings, Map.of("key", 3, "f3", 3));
573+
574+
try (var r = runQuery(syncEsqlQueryRequest().query("FROM data,*:data | WHERE f1 == 1").filter(new TermQueryBuilder("f2", 2)))) {
575+
assertThat(getValuesList(r), hasSize(0));
576+
}
577+
}
578+
579+
public void testLookupJoinRetryAnalysis() throws IOException {
580+
setupClusters(3);
581+
setSkipUnavailable(REMOTE_CLUSTER_1, false);
582+
setSkipUnavailable(REMOTE_CLUSTER_2, false);
583+
584+
var defaultSettings = Settings.builder();
585+
createIndexWithDocument(LOCAL_CLUSTER, "data", defaultSettings, Map.of("key", 1, "f1", 1));
586+
createIndexWithDocument(REMOTE_CLUSTER_1, "data", defaultSettings, Map.of("key", 2, "f2", 2));
587+
createIndexWithDocument(REMOTE_CLUSTER_2, "data", defaultSettings, Map.of("key", 3, "f3", 3));
588+
589+
var lookupSettings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOOKUP);
590+
createIndexWithDocument(LOCAL_CLUSTER, "lookup", lookupSettings, Map.of("key", 1, "location", "local"));
591+
createIndexWithDocument(REMOTE_CLUSTER_1, "lookup", lookupSettings, Map.of("key", 2, "location", "remote-1"));
592+
// lookup is intentionally absent on REMOTE_CLUSTER_2
593+
594+
// The following query uses filter f2=2 that narrows down execution only to REMOTE_CLUSTER_1 index however,
595+
// later it uses `WHERE f1 == 1` esql condition that to an attribute present only on the local cluster index.
596+
// This causes analysis to fail and retry the entire query without a filter.
597+
// The second analysis executes against all cluster indices and should discover that lookup is absent on REMOTE_CLUSTER_2.
598+
expectThrows(
599+
VerificationException.class,
600+
containsString("lookup index [lookup] is not available in remote cluster [remote-b]"),
601+
() -> runQuery(
602+
syncEsqlQueryRequest().query("FROM data,*:data | LOOKUP JOIN lookup ON key | WHERE f1 == 1")
603+
.filter(new TermQueryBuilder("f2", 2))
604+
)
605+
);
606+
}
607+
608+
private void createIndexWithDocument(String clusterAlias, String indexName, Settings.Builder settings, Map<String, Object> source) {
609+
var client = client(clusterAlias);
610+
client.admin().indices().prepareCreate(indexName).setSettings(settings).get();
611+
client.prepareIndex(indexName).setSource(source).get();
612+
client.admin().indices().prepareRefresh(indexName).get();
613+
}
614+
560615
protected Map<String, Object> setupClustersAndLookups() throws IOException {
561616
var setupData = setupClusters(2);
562617
populateLookupIndex(LOCAL_CLUSTER, "values_lookup", 10);

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueriesWithInvalidLicenseIT.java‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -65,24 +65,19 @@ public void testQueryAgainstNonMatchingClusterWildcardPattern() {
6565

6666
// since this wildcarded expression does not resolve to a valid remote cluster, it is not considered
6767
// a cross-cluster search and thus should not throw a license error
68-
String q = "FROM xremote*:events";
69-
{
70-
String limit1 = q + " | STATS count(*)";
71-
try (EsqlQueryResponse resp = runQuery(limit1, requestIncludeMeta)) {
72-
assertThat(resp.columns().size(), equalTo(1));
73-
EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
74-
assertThat(executionInfo.isCrossClusterSearch(), is(false));
75-
assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
76-
}
68+
try (EsqlQueryResponse resp = runQuery("FROM xremote*:events | STATS count(*)", requestIncludeMeta)) {
69+
assertThat(resp.columns().size(), equalTo(1));
70+
EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
71+
assertThat(executionInfo.isCrossClusterSearch(), is(false));
72+
assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
73+
}
7774

78-
String limit0 = q + " | LIMIT 0";
79-
try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) {
80-
assertThat(resp.columns().size(), equalTo(1));
81-
assertThat(getValuesList(resp).size(), equalTo(0));
82-
EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
83-
assertThat(executionInfo.isCrossClusterSearch(), is(false));
84-
assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
85-
}
75+
try (EsqlQueryResponse resp = runQuery("FROM xremote*:events | LIMIT 0", requestIncludeMeta)) {
76+
assertThat(resp.columns().size(), equalTo(1));
77+
assertThat(getValuesList(resp).size(), equalTo(0));
78+
EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
79+
assertThat(executionInfo.isCrossClusterSearch(), is(false));
80+
assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
8681
}
8782
}
8883

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryIT.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -383,24 +383,24 @@ protected void testSearchesAgainstNonMatchingIndices(boolean exceptionWithSkipUn
383383
// an error is thrown if there is a concrete index that does not match
384384
{
385385
String q = "FROM nomatch*,cluster-a:nomatch";
386-
String expectedError = "Unknown index [cluster-a:nomatch,nomatch*]";
386+
String expectedError = "Unknown index [nomatch*,cluster-a:nomatch]";
387387
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
388388
}
389389

390390
// an error is thrown if there are no matching indices at all - local with wildcard, remote with wildcard
391391
{
392392
String q = "FROM nomatch*,cluster-a:nomatch*";
393-
String expectedError = "Unknown index [cluster-a:nomatch*,nomatch*]";
393+
String expectedError = "Unknown index [nomatch*,cluster-a:nomatch*]";
394394
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
395395
}
396396
{
397397
String q = "FROM nomatch,cluster-a:nomatch";
398-
String expectedError = "Unknown index [cluster-a:nomatch,nomatch]";
398+
String expectedError = "Unknown index [nomatch,cluster-a:nomatch]";
399399
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
400400
}
401401
{
402402
String q = "FROM nomatch,cluster-a:nomatch*";
403-
String expectedError = "Unknown index [cluster-a:nomatch*,nomatch]";
403+
String expectedError = "Unknown index [nomatch,cluster-a:nomatch*]";
404404
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
405405
}
406406

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,14 @@ public PreAnalysis preAnalyze(LogicalPlan plan) {
4444

4545
protected PreAnalysis doPreAnalyze(LogicalPlan plan) {
4646
Holder<IndexMode> indexMode = new Holder<>();
47-
Holder<IndexPattern> index = new Holder<>();
47+
Holder<IndexPattern> indexPattern = new Holder<>();
4848
List<IndexPattern> lookupIndices = new ArrayList<>();
4949
plan.forEachUp(UnresolvedRelation.class, p -> {
5050
if (p.indexMode() == IndexMode.LOOKUP) {
5151
lookupIndices.add(p.indexPattern());
5252
} else if (indexMode.get() == null || indexMode.get() == p.indexMode()) {
5353
indexMode.set(p.indexMode());
54-
index.set(p.indexPattern());
54+
indexPattern.set(p.indexPattern());
5555
} else {
5656
throw new IllegalStateException("index mode is already set");
5757
}
@@ -92,7 +92,7 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) {
9292

9393
return new PreAnalysis(
9494
indexMode.get(),
95-
index.get(),
95+
indexPattern.get(),
9696
unresolvedEnriches,
9797
lookupIndices,
9898
indexMode.get() == IndexMode.TIME_SERIES || supportsAggregateMetricDouble.get(),

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -155,30 +155,6 @@ static void updateExecutionInfoToReturnEmptyResult(EsqlExecutionInfo executionIn
155155
}
156156
}
157157

158-
static String createIndexExpressionFromAvailableClusters(EsqlExecutionInfo executionInfo) {
159-
StringBuilder sb = new StringBuilder();
160-
for (String clusterAlias : executionInfo.clusterAliases()) {
161-
EsqlExecutionInfo.Cluster cluster = executionInfo.getCluster(clusterAlias);
162-
// Exclude clusters which are either skipped or have no indices matching wildcard, or filtered out.
163-
if (cluster.getStatus() != Cluster.Status.SKIPPED && cluster.getStatus() != Cluster.Status.SUCCESSFUL) {
164-
if (cluster.getClusterAlias().equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) {
165-
sb.append(executionInfo.getCluster(clusterAlias).getIndexExpression()).append(',');
166-
} else {
167-
String indexExpression = executionInfo.getCluster(clusterAlias).getIndexExpression();
168-
for (String index : indexExpression.split(",")) {
169-
sb.append(clusterAlias).append(':').append(index).append(',');
170-
}
171-
}
172-
}
173-
}
174-
175-
if (sb.length() > 0) {
176-
return sb.substring(0, sb.length() - 1);
177-
} else {
178-
return "";
179-
}
180-
}
181-
182158
static String createQualifiedLookupIndexExpressionFromAvailableClusters(EsqlExecutionInfo executionInfo, String localPattern) {
183159
if (executionInfo.getClusters().isEmpty()) {
184160
return localPattern;

0 commit comments

Comments
 (0)