Skip to content

Commit a9cf685

Browse files
bpinteabpintea
authored
[8.19] ESQL: Prevent search functions work with a non-STANDARD index (#130638) (#130704)
* ESQL: Prevent search functions work with a non-STANDARD index (#130638) This introduces verifications to prevent search functions work on fields introduced by a LOOKUP JOIN righthand-side index. This should be a temporary fix until we can either push these filters down also on the righthand-side of a JOIN or have these functions execute within the engine. Closes #130561 Closes #129778 (cherry picked from commit 5c4d17d) * 8.19 test addaptation
1 parent 8f80eab commit a9cf685

File tree

12 files changed

+219
-146
lines changed

12 files changed

+219
-146
lines changed

‎docs/changelog/130638.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
pr: 130638
2+
summary: Prevent search functions work with a non-STANDARD index
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 130561
7+
- 129778

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,14 @@
88
package org.elasticsearch.xpack.esql.plugin;
99

1010
import org.elasticsearch.action.index.IndexRequestBuilder;
11+
import org.elasticsearch.client.internal.IndicesAdminClient;
1112
import org.elasticsearch.cluster.metadata.IndexMetadata;
1213
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.index.IndexSettings;
1315
import org.elasticsearch.xcontent.XContentBuilder;
1416
import org.elasticsearch.xcontent.XContentFactory;
1517
import org.elasticsearch.xpack.esql.EsqlTestUtils;
18+
import org.elasticsearch.xpack.esql.VerificationException;
1619
import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
1720
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
1821
import org.junit.Before;
@@ -25,7 +28,9 @@
2528
import java.util.Locale;
2629
import java.util.Map;
2730

31+
import static org.elasticsearch.index.IndexMode.LOOKUP;
2832
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
33+
import static org.hamcrest.CoreMatchers.containsString;
2934

3035
public class KnnFunctionIT extends AbstractEsqlIntegTestCase {
3136

@@ -109,6 +114,26 @@ public void testKnnNonPushedDown() {
109114
}
110115
}
111116

117+
public void testKnnWithLookupJoin() {
118+
float[] queryVector = new float[numDims];
119+
Arrays.fill(queryVector, 1.0f);
120+
121+
var query = String.format(Locale.ROOT, """
122+
FROM test
123+
| LOOKUP JOIN test_lookup ON id
124+
| WHERE KNN(lookup_vector, %s, {"k": 5}) OR id > 10
125+
""", Arrays.toString(queryVector));
126+
127+
var error = expectThrows(VerificationException.class, () -> run(query));
128+
assertThat(
129+
error.getMessage(),
130+
containsString(
131+
"line 3:13: [KNN] function cannot operate on [lookup_vector], supplied by an index [test_lookup] in non-STANDARD "
132+
+ "mode [lookup]"
133+
)
134+
);
135+
}
136+
112137
@Before
113138
public void setup() throws IOException {
114139
assumeTrue("Needs KNN support", EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled());
@@ -152,5 +177,31 @@ public void setup() throws IOException {
152177
}
153178

154179
indexRandom(true, docs);
180+
181+
createAndPopulateLookupIndex(client, "test_lookup");
182+
}
183+
184+
private void createAndPopulateLookupIndex(IndicesAdminClient client, String lookupIndexName) throws IOException {
185+
XContentBuilder mapping = XContentFactory.jsonBuilder()
186+
.startObject()
187+
.startObject("properties")
188+
.startObject("id")
189+
.field("type", "integer")
190+
.endObject()
191+
.startObject("lookup_vector")
192+
.field("type", "dense_vector")
193+
.field("similarity", "l2_norm")
194+
.endObject()
195+
.endObject()
196+
.endObject();
197+
198+
Settings.Builder settingsBuilder = Settings.builder()
199+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
200+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
201+
.put(IndexSettings.MODE.getKey(), LOOKUP.getName());
202+
203+
var createRequest = client.prepareCreate(lookupIndexName).setMapping(mapping).setSettings(settingsBuilder.build());
204+
assertAcked(createRequest);
205+
155206
}
156207
}

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.ElasticsearchException;
1111
import org.elasticsearch.action.index.IndexRequest;
1212
import org.elasticsearch.action.support.WriteRequest;
13+
import org.elasticsearch.client.internal.IndicesAdminClient;
1314
import org.elasticsearch.common.settings.Settings;
1415
import org.elasticsearch.xpack.esql.VerificationException;
1516
import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
@@ -18,6 +19,7 @@
1819
import org.junit.Before;
1920

2021
import java.util.List;
22+
import java.util.function.Consumer;
2123

2224
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
2325
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
@@ -28,7 +30,7 @@ public class MatchFunctionIT extends AbstractEsqlIntegTestCase {
2830

2931
@Before
3032
public void setupIndex() {
31-
createAndPopulateIndex();
33+
createAndPopulateIndex(this::ensureYellow);
3234
}
3335

3436
public void testSimpleWhereMatch() {
@@ -298,13 +300,30 @@ public void testMatchWithinEval() {
298300
assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE and STATS commands"));
299301
}
300302

301-
private void createAndPopulateIndex() {
303+
public void testMatchWithLookupJoin() {
304+
var query = """
305+
FROM test
306+
| LOOKUP JOIN test_lookup ON id
307+
| WHERE id > 0 AND MATCH(lookup_content, "fox")
308+
""";
309+
310+
var error = expectThrows(VerificationException.class, () -> run(query));
311+
assertThat(
312+
error.getMessage(),
313+
containsString(
314+
"line 3:26: [MATCH] function cannot operate on [lookup_content], supplied by an index [test_lookup] "
315+
+ "in non-STANDARD mode [lookup]"
316+
)
317+
);
318+
}
319+
320+
static void createAndPopulateIndex(Consumer<String[]> ensureYellow) {
302321
var indexName = "test";
303322
var client = client().admin().indices();
304-
var CreateRequest = client.prepareCreate(indexName)
323+
var createRequest = client.prepareCreate(indexName)
305324
.setSettings(Settings.builder().put("index.number_of_shards", 1))
306325
.setMapping("id", "type=integer", "content", "type=text");
307-
assertAcked(CreateRequest);
326+
assertAcked(createRequest);
308327
client().prepareBulk()
309328
.add(new IndexRequest(indexName).id("1").source("id", 1, "content", "This is a brown fox"))
310329
.add(new IndexRequest(indexName).id("2").source("id", 2, "content", "This is a brown dog"))
@@ -314,6 +333,17 @@ private void createAndPopulateIndex() {
314333
.add(new IndexRequest(indexName).id("6").source("id", 6, "content", "The quick brown fox jumps over the lazy dog"))
315334
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
316335
.get();
317-
ensureYellow(indexName);
336+
337+
var lookupIndexName = "test_lookup";
338+
createAndPopulateLookupIndex(client, lookupIndexName);
339+
340+
ensureYellow.accept(new String[] { indexName, lookupIndexName });
341+
}
342+
343+
static void createAndPopulateLookupIndex(IndicesAdminClient client, String lookupIndexName) {
344+
var createRequest = client.prepareCreate(lookupIndexName)
345+
.setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.mode", "lookup"))
346+
.setMapping("id", "type=integer", "lookup_content", "type=text");
347+
assertAcked(createRequest);
318348
}
319349
}

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,6 @@
88
package org.elasticsearch.xpack.esql.plugin;
99

1010
import org.elasticsearch.ElasticsearchException;
11-
import org.elasticsearch.action.index.IndexRequest;
12-
import org.elasticsearch.action.support.WriteRequest;
13-
import org.elasticsearch.common.settings.Settings;
1411
import org.elasticsearch.index.query.QueryBuilder;
1512
import org.elasticsearch.index.query.QueryBuilders;
1613
import org.elasticsearch.xpack.esql.VerificationException;
@@ -22,15 +19,14 @@
2219

2320
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
2421
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
25-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
2622
import static org.hamcrest.CoreMatchers.containsString;
2723

2824
//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug")
2925
public class MatchOperatorIT extends AbstractEsqlIntegTestCase {
3026

3127
@Before
3228
public void setupIndex() {
33-
createAndPopulateIndex();
29+
MatchFunctionIT.createAndPopulateIndex(this::ensureYellow);
3430
}
3531

3632
public void testSimpleWhereMatch() {
@@ -375,22 +371,20 @@ public void testMatchWithNonTextField() {
375371
}
376372
}
377373

378-
private void createAndPopulateIndex() {
379-
var indexName = "test";
380-
var client = client().admin().indices();
381-
var CreateRequest = client.prepareCreate(indexName)
382-
.setSettings(Settings.builder().put("index.number_of_shards", 1))
383-
.setMapping("id", "type=integer", "content", "type=text");
384-
assertAcked(CreateRequest);
385-
client().prepareBulk()
386-
.add(new IndexRequest(indexName).id("1").source("id", 1, "content", "This is a brown fox"))
387-
.add(new IndexRequest(indexName).id("2").source("id", 2, "content", "This is a brown dog"))
388-
.add(new IndexRequest(indexName).id("3").source("id", 3, "content", "This dog is really brown"))
389-
.add(new IndexRequest(indexName).id("4").source("id", 4, "content", "The dog is brown but this document is very very long"))
390-
.add(new IndexRequest(indexName).id("5").source("id", 5, "content", "There is also a white cat"))
391-
.add(new IndexRequest(indexName).id("6").source("id", 6, "content", "The quick brown fox jumps over the lazy dog"))
392-
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
393-
.get();
394-
ensureYellow(indexName);
374+
public void testMatchOperatorWithLookupJoin() {
375+
var query = """
376+
FROM test
377+
| LOOKUP JOIN test_lookup ON id
378+
| WHERE id > 0 AND lookup_content : "fox"
379+
""";
380+
381+
var error = expectThrows(VerificationException.class, () -> run(query));
382+
assertThat(
383+
error.getMessage(),
384+
containsString(
385+
"line 3:20: [:] operator cannot operate on [lookup_content], supplied by an index [test_lookup] "
386+
+ "in non-STANDARD mode [lookup]"
387+
)
388+
);
395389
}
396390
}

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchPhraseFunctionIT.java‎

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,6 @@
88
package org.elasticsearch.xpack.esql.plugin;
99

1010
import org.elasticsearch.ElasticsearchException;
11-
import org.elasticsearch.action.index.IndexRequest;
12-
import org.elasticsearch.action.support.WriteRequest;
13-
import org.elasticsearch.common.settings.Settings;
1411
import org.elasticsearch.xpack.esql.VerificationException;
1512
import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
1613
import org.hamcrest.Matchers;
@@ -19,16 +16,16 @@
1916
import java.util.Collections;
2017
import java.util.List;
2118

22-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
2319
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
20+
import static org.elasticsearch.xpack.esql.plugin.MatchFunctionIT.createAndPopulateIndex;
2421
import static org.hamcrest.CoreMatchers.containsString;
2522

2623
//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug")
2724
public class MatchPhraseFunctionIT extends AbstractEsqlIntegTestCase {
2825

2926
@Before
3027
public void setupIndex() {
31-
createAndPopulateIndex();
28+
createAndPopulateIndex(this::ensureYellow);
3229
}
3330

3431
public void testSimpleWhereMatchPhrase() {
@@ -325,22 +322,20 @@ public void testMatchPhraseWithinEval() {
325322
assertThat(error.getMessage(), containsString("[MatchPhrase] function is only supported in WHERE and STATS commands"));
326323
}
327324

328-
private void createAndPopulateIndex() {
329-
var indexName = "test";
330-
var client = client().admin().indices();
331-
var CreateRequest = client.prepareCreate(indexName)
332-
.setSettings(Settings.builder().put("index.number_of_shards", 1))
333-
.setMapping("id", "type=integer", "content", "type=text");
334-
assertAcked(CreateRequest);
335-
client().prepareBulk()
336-
.add(new IndexRequest(indexName).id("1").source("id", 1, "content", "This is a brown fox"))
337-
.add(new IndexRequest(indexName).id("2").source("id", 2, "content", "This is a brown dog"))
338-
.add(new IndexRequest(indexName).id("3").source("id", 3, "content", "This dog is really brown"))
339-
.add(new IndexRequest(indexName).id("4").source("id", 4, "content", "The dog is brown but this document is very very long"))
340-
.add(new IndexRequest(indexName).id("5").source("id", 5, "content", "There is also a white cat"))
341-
.add(new IndexRequest(indexName).id("6").source("id", 6, "content", "The quick brown fox jumps over the lazy dog"))
342-
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
343-
.get();
344-
ensureYellow(indexName);
325+
public void testMatchPhraseWithLookupJoin() {
326+
var query = """
327+
FROM test
328+
| LOOKUP JOIN test_lookup ON id
329+
| WHERE id > 0 AND MATCH_PHRASE(lookup_content, "fox")
330+
""";
331+
332+
var error = expectThrows(VerificationException.class, () -> run(query));
333+
assertThat(
334+
error.getMessage(),
335+
containsString(
336+
"line 3:33: [MatchPhrase] function cannot operate on [lookup_content], supplied by an index [test_lookup] "
337+
+ "in non-STANDARD mode [lookup]"
338+
)
339+
);
345340
}
346341
}

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,17 @@
1717
import org.junit.Before;
1818

1919
import java.util.List;
20+
import java.util.function.Consumer;
2021

2122
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
23+
import static org.elasticsearch.xpack.esql.plugin.MatchFunctionIT.createAndPopulateLookupIndex;
2224
import static org.hamcrest.CoreMatchers.containsString;
2325

2426
public class QueryStringIT extends AbstractEsqlIntegTestCase {
2527

2628
@Before
2729
public void setupIndex() {
28-
createAndPopulateIndex();
30+
createAndPopulateIndex(this::ensureYellow);
2931
}
3032

3133
public void testSimpleQueryString() {
@@ -92,7 +94,7 @@ public void testInvalidQueryStringLexicalError() {
9294
);
9395
}
9496

95-
private void createAndPopulateIndex() {
97+
static void createAndPopulateIndex(Consumer<String[]> ensureYellow) {
9698
var indexName = "test";
9799
var client = client().admin().indices();
98100
var CreateRequest = client.prepareCreate(indexName)
@@ -136,7 +138,11 @@ private void createAndPopulateIndex() {
136138
)
137139
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
138140
.get();
139-
ensureYellow(indexName);
141+
142+
var lookupIndexName = "test_lookup";
143+
createAndPopulateLookupIndex(client, lookupIndexName);
144+
145+
ensureYellow.accept(new String[] { indexName, lookupIndexName });
140146
}
141147

142148
public void testWhereQstrWithScoring() {
@@ -233,4 +239,15 @@ AND abs(id) > 0
233239
assertValuesInAnyOrder(resp.values(), List.of(List.of(5, 1.0), List.of(4, 1.0)));
234240
}
235241
}
242+
243+
public void testWhereQstrWithLookupJoin() {
244+
var query = """
245+
FROM test
246+
| LOOKUP JOIN test_lookup ON id
247+
| WHERE id > 0 AND QSTR("lookup_content: fox")
248+
""";
249+
250+
var error = expectThrows(VerificationException.class, () -> run(query));
251+
assertThat(error.getMessage(), containsString("line 3:3: [QSTR] function cannot be used after LOOKUP"));
252+
}
236253
}

0 commit comments

Comments
 (0)