Skip to content

Commit 5b7c642

Browse files
idegtiarenkoidegtiarenko
authored
Merge branch 'main' into fix_vault_field_name
2 parents 11a6464 + 5b6c5c9 commit 5b7c642

File tree

21 files changed

+2163
-1986
lines changed

21 files changed

+2163
-1986
lines changed

‎x-pack/plugin/core/template-resources/src/main/resources/[email protected]‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@
4242
"jobtype": {
4343
"type": "keyword"
4444
},
45+
"scheduled_report_id": {
46+
"type": "keyword"
47+
},
4548
"payload": {
4649
"type": "object",
4750
"enabled": false

‎x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/GenerativeForkIT.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.qa.single_node;
9+
10+
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
11+
12+
import org.elasticsearch.test.TestClustersThreadFilter;
13+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
14+
import org.elasticsearch.xpack.esql.CsvSpecReader;
15+
import org.elasticsearch.xpack.esql.qa.rest.generative.GenerativeForkRestTest;
16+
import org.junit.ClassRule;
17+
18+
@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
19+
public class GenerativeForkIT extends GenerativeForkRestTest {
20+
@ClassRule
21+
public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
22+
23+
@Override
24+
protected String getTestRestCluster() {
25+
return cluster.getHttpAddresses();
26+
}
27+
28+
public GenerativeForkIT(
29+
String fileName,
30+
String groupName,
31+
String testName,
32+
Integer lineNumber,
33+
CsvSpecReader.CsvTestCase testCase,
34+
String instructions,
35+
Mode mode
36+
) {
37+
super(fileName, groupName, testName, lineNumber, testCase, instructions, mode);
38+
}
39+
40+
@Override
41+
protected boolean enableRoundingDoubleValuesOnAsserting() {
42+
// This suite runs with more than one node and three shards in serverless
43+
return cluster.getNumNodes() > 1;
44+
}
45+
46+
@Override
47+
protected boolean supportsSourceFieldMapping() {
48+
return cluster.getNumNodes() == 1;
49+
}
50+
}

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,15 +259,19 @@ protected boolean supportsSourceFieldMapping() throws IOException {
259259
return true;
260260
}
261261

262-
protected final void doTest() throws Throwable {
262+
protected void doTest() throws Throwable {
263+
doTest(testCase.query);
264+
}
265+
266+
protected final void doTest(String query) throws Throwable {
263267
RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values()));
264268

265-
if (testCase.query.toUpperCase(Locale.ROOT).contains("LOOKUP_\uD83D\uDC14")) {
269+
if (query.toUpperCase(Locale.ROOT).contains("LOOKUP_\uD83D\uDC14")) {
266270
builder.tables(tables());
267271
}
268272

269273
Map<?, ?> prevTooks = supportsTook() ? tooks() : null;
270-
Map<String, Object> answer = runEsql(builder.query(testCase.query), testCase.assertWarnings(deduplicateExactWarnings()));
274+
Map<String, Object> answer = runEsql(builder.query(query), testCase.assertWarnings(deduplicateExactWarnings()));
271275

272276
var expectedColumnsWithValues = loadCsvSpecValues(testCase.expectedResults);
273277

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.qa.rest.generative;
9+
10+
import org.elasticsearch.xpack.esql.CsvSpecReader;
11+
import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
12+
13+
import java.io.IOException;
14+
import java.util.List;
15+
16+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.*;
17+
18+
/**
19+
* Tests for FORK. We generate tests for FORK from existing CSV tests.
20+
* We append a `| FORK (WHERE true) (WHERE true) | WHERE _fork == "fork1" | DROP _fork` suffix to existing
21+
* CSV test cases. This will produce a query that executes multiple FORK branches but expects the same results
22+
* as the initial CSV test case.
23+
* For now, we skip tests that already require FORK, since multiple FORK commands are not allowed.
24+
*/
25+
public abstract class GenerativeForkRestTest extends EsqlSpecTestCase {
26+
public GenerativeForkRestTest(
27+
String fileName,
28+
String groupName,
29+
String testName,
30+
Integer lineNumber,
31+
CsvSpecReader.CsvTestCase testCase,
32+
String instructions,
33+
Mode mode
34+
) {
35+
super(fileName, groupName, testName, lineNumber, testCase, instructions, mode);
36+
}
37+
38+
@Override
39+
protected void doTest() throws Throwable {
40+
String query = testCase.query + " | FORK (WHERE true) (WHERE true) | WHERE _fork == \"fork1\" | DROP _fork";
41+
doTest(query);
42+
}
43+
44+
@Override
45+
protected void shouldSkipTest(String testName) throws IOException {
46+
super.shouldSkipTest(testName);
47+
48+
assumeFalse(
49+
"Tests using FORK or RRF already are skipped since we don't support multiple FORKs",
50+
testCase.requiredCapabilities.contains(FORK_V7.capabilityName()) || testCase.requiredCapabilities.contains(RRF.capabilityName())
51+
);
52+
53+
assumeFalse(
54+
"Tests using INSIST are not supported for now",
55+
testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName())
56+
);
57+
58+
assumeFalse(
59+
"Tests using implicit_casting_date_and_date_nanos are not supported for now",
60+
testCase.requiredCapabilities.contains(IMPLICIT_CASTING_DATE_AND_DATE_NANOS.capabilityName())
61+
);
62+
63+
assumeTrue("Cluster needs to support FORK", hasCapabilities(client(), List.of(FORK_V7.capabilityName())));
64+
}
65+
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ completion using a ROW source operator
66
required_capability: completion
77

88
ROW prompt="Who is Victor Hugo?"
9-
| COMPLETION prompt WITH test_completion AS completion_output
9+
| COMPLETION completion_output = prompt WITH test_completion
1010
;
1111

1212
prompt:keyword | completion_output:keyword
@@ -18,7 +18,7 @@ completion using a ROW source operator and prompt is a multi-valued field
1818
required_capability: completion
1919

2020
ROW prompt=["Answer the following question:", "Who is Victor Hugo?"]
21-
| COMPLETION prompt WITH test_completion AS completion_output
21+
| COMPLETION completion_output = prompt WITH test_completion
2222
;
2323

2424
prompt:keyword | completion_output:keyword

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ForkIT.java‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.action.support.WriteRequest;
1212
import org.elasticsearch.common.settings.Settings;
1313
import org.elasticsearch.compute.operator.DriverProfile;
14+
import org.elasticsearch.test.junit.annotations.TestLogging;
1415
import org.elasticsearch.xpack.esql.VerificationException;
1516
import org.elasticsearch.xpack.esql.parser.ParsingException;
1617
import org.junit.Before;
@@ -26,13 +27,13 @@
2627
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
2728
import static org.hamcrest.Matchers.equalTo;
2829

29-
// @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug")
30+
@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug")
3031
public class ForkIT extends AbstractEsqlIntegTestCase {
3132

3233
@Before
3334
public void setupIndex() {
3435
assumeTrue("requires FORK capability", EsqlCapabilities.Cap.FORK.isEnabled());
35-
createAndPopulateIndex();
36+
createAndPopulateIndices();
3637
}
3738

3839
public void testSimple() {
@@ -706,6 +707,52 @@ public void testWithLookUpAfterFork() {
706707
}
707708
}
708709

710+
public void testWithUnionTypesBeforeFork() {
711+
var query = """
712+
FROM test,test-other
713+
| EVAL x = id::keyword
714+
| EVAL id = id::keyword
715+
| EVAL content = content::keyword
716+
| FORK (WHERE x == "2")
717+
(WHERE x == "1")
718+
| SORT _fork, x, content
719+
| KEEP content, id, x, _fork
720+
""";
721+
722+
try (var resp = run(query)) {
723+
assertColumnNames(resp.columns(), List.of("content", "id", "x", "_fork"));
724+
Iterable<Iterable<Object>> expectedValues = List.of(
725+
List.of("This is a brown dog", "2", "2", "fork1"),
726+
List.of("This is a brown dog", "2", "2", "fork1"),
727+
List.of("This is a brown fox", "1", "1", "fork2"),
728+
List.of("This is a brown fox", "1", "1", "fork2")
729+
);
730+
assertValues(resp.values(), expectedValues);
731+
}
732+
}
733+
734+
public void testWithUnionTypesInBranches() {
735+
var query = """
736+
FROM test,test-other
737+
| EVAL content = content::keyword
738+
| FORK (EVAL x = id::keyword | WHERE x == "2" | EVAL id = x::integer)
739+
(EVAL x = "a" | WHERE id::keyword == "1" | EVAL id = id::integer)
740+
| SORT _fork, x
741+
| KEEP content, id, x, _fork
742+
""";
743+
744+
try (var resp = run(query)) {
745+
assertColumnNames(resp.columns(), List.of("content", "id", "x", "_fork"));
746+
Iterable<Iterable<Object>> expectedValues = List.of(
747+
List.of("This is a brown dog", 2, "2", "fork1"),
748+
List.of("This is a brown dog", 2, "2", "fork1"),
749+
List.of("This is a brown fox", 1, "a", "fork2"),
750+
List.of("This is a brown fox", 1, "a", "fork2")
751+
);
752+
assertValues(resp.values(), expectedValues);
753+
}
754+
}
755+
709756
public void testWithEvalWithConflictingTypes() {
710757
var query = """
711758
FROM test
@@ -833,7 +880,7 @@ public void testProfile() {
833880
}
834881
}
835882

836-
private void createAndPopulateIndex() {
883+
private void createAndPopulateIndices() {
837884
var indexName = "test";
838885
var client = client().admin().indices();
839886
var createRequest = client.prepareCreate(indexName)
@@ -867,6 +914,20 @@ private void createAndPopulateIndex() {
867914
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
868915
.get();
869916
ensureYellow(lookupIndex);
917+
918+
var otherTestIndex = "test-other";
919+
920+
createRequest = client.prepareCreate(otherTestIndex)
921+
.setSettings(Settings.builder().put("index.number_of_shards", 1))
922+
.setMapping("id", "type=keyword", "content", "type=keyword");
923+
assertAcked(createRequest);
924+
client().prepareBulk()
925+
.add(new IndexRequest(otherTestIndex).id("1").source("id", "1", "content", "This is a brown fox"))
926+
.add(new IndexRequest(otherTestIndex).id("2").source("id", "2", "content", "This is a brown dog"))
927+
.add(new IndexRequest(otherTestIndex).id("3").source("id", "3", "content", "This dog is really brown"))
928+
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
929+
.get();
930+
ensureYellow(indexName);
870931
}
871932

872933
static Iterator<Iterator<Object>> valuesFilter(Iterator<Iterator<Object>> values, Predicate<Iterator<Object>> filter) {

0 commit comments

Comments
 (0)