Skip to content

Commit 6fd3357

Browse files
craigtavernercraigtaverner
committed
Get CsvTests working on views
1 parent f4f3ffb commit 6fd3357

File tree

2 files changed

+86
-29
lines changed

2 files changed

+86
-29
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ public class CsvTestsDataLoader {
7878
private static final TestDataset LANGUAGES = new TestDataset("languages");
7979
private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup").withSetting("lookup-settings.json");
8080
private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key")
81-
.withData("languages_non_unique_key.csv");
81+
.withData("languages_non_unique_key.csv")
82+
.withDynamicTypeMapping(Map.of("country", "text"));
8283
private static final TestDataset LANGUAGES_NESTED_FIELDS = new TestDataset(
8384
"languages_nested_fields",
8485
"mapping-languages_nested_fields.json",
@@ -615,16 +616,20 @@ private static void loadEnrichPolicy(RestClient client, String policyName, Strin
615616
}
616617

617618
private static void loadView(RestClient client, String viewName, String viewFilename, Logger logger) throws IOException {
618-
logger.info("Loading view [{}] from file [{}]", viewName, viewFilename);
619-
URL viewFile = getResource("/" + viewFilename);
620-
String viewQuery = readTextFile(viewFile);
619+
String viewQuery = loadViewQuery(viewName, viewFilename, logger);
621620
Request request = new Request("PUT", "/_query/view/" + viewName);
622621
request.setJsonEntity("{\"query\":\"" + viewQuery.replace("\"", "\\\"").replace("\n", " ") + "\"}");
623622
Response response = client.performRequest(request);
624623
logger.info("View creation response: {}", response.getStatusLine());
625624
getView(client, viewName, logger);
626625
}
627626

627+
static String loadViewQuery(String viewName, String viewFilename, Logger logger) throws IOException {
628+
logger.info("Loading view [{}] from file [{}]", viewName, viewFilename);
629+
URL viewFile = getResource("/" + viewFilename);
630+
return readTextFile(viewFile);
631+
}
632+
628633
private static boolean getView(RestClient client, String viewName, Logger logger) throws IOException {
629634
Request request = new Request("GET", "/_query/view/" + viewName);
630635
try {
@@ -951,15 +956,16 @@ public record TestDataset(
951956
String dataFileName,
952957
String settingFileName,
953958
boolean allowSubFields,
954-
@Nullable Map<String, String> typeMapping,
959+
@Nullable Map<String, String> typeMapping, // Override mappings read from mappings file
960+
@Nullable Map<String, String> dynamicTypeMapping, // Define mappings not in the mapping files, but available from field-caps
955961
boolean requiresInferenceEndpoint
956962
) {
957963
public TestDataset(String indexName, String mappingFileName, String dataFileName) {
958-
this(indexName, mappingFileName, dataFileName, null, true, null, false);
964+
this(indexName, mappingFileName, dataFileName, null, true, null, null, false);
959965
}
960966

961967
public TestDataset(String indexName) {
962-
this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, false);
968+
this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, null, false);
963969
}
964970

965971
public TestDataset withIndex(String indexName) {
@@ -970,6 +976,7 @@ public TestDataset withIndex(String indexName) {
970976
settingFileName,
971977
allowSubFields,
972978
typeMapping,
979+
dynamicTypeMapping,
973980
requiresInferenceEndpoint
974981
);
975982
}
@@ -982,6 +989,7 @@ public TestDataset withData(String dataFileName) {
982989
settingFileName,
983990
allowSubFields,
984991
typeMapping,
992+
dynamicTypeMapping,
985993
requiresInferenceEndpoint
986994
);
987995
}
@@ -994,6 +1002,7 @@ public TestDataset noData() {
9941002
settingFileName,
9951003
allowSubFields,
9961004
typeMapping,
1005+
dynamicTypeMapping,
9971006
requiresInferenceEndpoint
9981007
);
9991008
}
@@ -1006,6 +1015,7 @@ public TestDataset withSetting(String settingFileName) {
10061015
settingFileName,
10071016
allowSubFields,
10081017
typeMapping,
1018+
dynamicTypeMapping,
10091019
requiresInferenceEndpoint
10101020
);
10111021
}
@@ -1018,6 +1028,7 @@ public TestDataset noSubfields() {
10181028
settingFileName,
10191029
false,
10201030
typeMapping,
1031+
dynamicTypeMapping,
10211032
requiresInferenceEndpoint
10221033
);
10231034
}
@@ -1030,12 +1041,35 @@ public TestDataset withTypeMapping(Map<String, String> typeMapping) {
10301041
settingFileName,
10311042
allowSubFields,
10321043
typeMapping,
1044+
dynamicTypeMapping,
1045+
requiresInferenceEndpoint
1046+
);
1047+
}
1048+
1049+
public TestDataset withDynamicTypeMapping(Map<String, String> dynamicTypeMapping) {
1050+
return new TestDataset(
1051+
indexName,
1052+
mappingFileName,
1053+
dataFileName,
1054+
settingFileName,
1055+
allowSubFields,
1056+
typeMapping,
1057+
dynamicTypeMapping,
10331058
requiresInferenceEndpoint
10341059
);
10351060
}
10361061

10371062
public TestDataset withInferenceEndpoint(boolean needsInference) {
1038-
return new TestDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference);
1063+
return new TestDataset(
1064+
indexName,
1065+
mappingFileName,
1066+
dataFileName,
1067+
settingFileName,
1068+
allowSubFields,
1069+
typeMapping,
1070+
dynamicTypeMapping,
1071+
needsInference
1072+
);
10391073
}
10401074

10411075
private Settings readSettingsFile() throws IOException {

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java‎

Lines changed: 44 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@
9898
import org.elasticsearch.xpack.esql.session.Result;
9999
import org.elasticsearch.xpack.esql.stats.DisabledSearchStats;
100100
import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
101+
import org.elasticsearch.xpack.esql.view.InMemoryViewService;
102+
import org.elasticsearch.xpack.esql.view.View;
101103
import org.junit.After;
102104
import org.junit.Before;
103105

@@ -121,6 +123,8 @@
121123
import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues;
122124
import static org.elasticsearch.xpack.esql.CsvTestUtils.loadPageFromCsv;
123125
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET_MAP;
126+
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.VIEW_CONFIGS;
127+
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadViewQuery;
124128
import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_PLANNER_SETTINGS;
125129
import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
126130
import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
@@ -336,10 +340,6 @@ public final void test() throws Throwable {
336340
"CSV tests cannot currently handle TEXT_EMBEDDING function",
337341
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.TEXT_EMBEDDING_FUNCTION.capabilityName())
338342
);
339-
assumeFalse(
340-
"CSV tests cannot currently handle VIEWS",
341-
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.VIEWS_V1.capabilityName())
342-
);
343343
assumeFalse(
344344
"CSV tests cannot currently handle multi_match function that depends on Lucene",
345345
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.capabilityName())
@@ -429,21 +429,30 @@ private static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndex
429429

430430
private static Map<String, EsField> createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) {
431431
var mapping = new TreeMap<>(loadMapping(dataset.mappingFileName()));
432-
if (dataset.typeMapping() == null) {
433-
return mapping;
432+
if (dataset.typeMapping() != null) {
433+
for (var entry : dataset.typeMapping().entrySet()) {
434+
if (mapping.containsKey(entry.getKey())) {
435+
DataType dataType = DataType.fromTypeName(entry.getValue());
436+
EsField field = mapping.get(entry.getKey());
437+
EsField editedField = new EsField(
438+
field.getName(),
439+
dataType,
440+
field.getProperties(),
441+
field.isAggregatable(),
442+
field.getTimeSeriesFieldType()
443+
);
444+
mapping.put(entry.getKey(), editedField);
445+
}
446+
}
434447
}
435-
for (var entry : dataset.typeMapping().entrySet()) {
436-
if (mapping.containsKey(entry.getKey())) {
437-
DataType dataType = DataType.fromTypeName(entry.getValue());
438-
EsField field = mapping.get(entry.getKey());
439-
EsField editedField = new EsField(
440-
field.getName(),
441-
dataType,
442-
field.getProperties(),
443-
field.isAggregatable(),
444-
field.getTimeSeriesFieldType()
445-
);
446-
mapping.put(entry.getKey(), editedField);
448+
// Add dynamic mappings, but only if they are not already mapped
449+
if (dataset.dynamicTypeMapping() != null) {
450+
for (var entry : dataset.dynamicTypeMapping().entrySet()) {
451+
if (mapping.containsKey(entry.getKey()) == false) {
452+
DataType dataType = DataType.fromTypeName(entry.getValue());
453+
EsField editedField = new EsField(entry.getKey(), dataType, Map.of(), false, EsField.TimeSeriesFieldType.NONE);
454+
mapping.put(entry.getKey(), editedField);
455+
}
447456
}
448457
}
449458
return mapping;
@@ -537,7 +546,21 @@ private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.MultiInd
537546
return plan;
538547
}
539548

540-
private static CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan parsed) {
549+
private LogicalPlan resolveViews(LogicalPlan parsed) {
550+
InMemoryViewService viewService = new InMemoryViewService(functionRegistry);
551+
for (var viewConfig : VIEW_CONFIGS) {
552+
try {
553+
String viewQuery = loadViewQuery(viewConfig.viewName(), viewConfig.viewFileName(), LOGGER);
554+
viewService.put(viewConfig.viewName(), new View(viewQuery), ActionListener.noop(), configuration);
555+
} catch (IOException e) {
556+
logger.error("Failed to load view '" + viewConfig + "': " + e.getMessage());
557+
throw new RuntimeException(e);
558+
}
559+
}
560+
return viewService.replaceViews(parsed, new PlanTelemetry(functionRegistry), configuration);
561+
}
562+
563+
private CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan parsed) {
541564
var preAnalysis = new PreAnalyzer().preAnalyze(parsed);
542565
if (preAnalysis.indexPattern() == null) {
543566
// If the data set doesn't matter we'll just grab one we know works. Employees is fine.
@@ -582,8 +605,8 @@ private static TestPhysicalOperationProviders testOperationProviders(
582605
}
583606

584607
private ActualResults executePlan(BigArrays bigArrays) throws Exception {
585-
LogicalPlan parsed = parser.createStatement(testCase.query, EsqlTestUtils.TEST_CFG);
586-
var testDatasets = testDatasets(parsed);
608+
LogicalPlan parsed = resolveViews(parser.createStatement(testCase.query, configuration));
609+
var testDatasets = testDatasets(resolveViews(parsed));
587610
LogicalPlan analyzed = analyzedPlan(parsed, testDatasets);
588611

589612
FoldContext foldCtx = FoldContext.small();

0 commit comments

Comments
 (0)