Skip to content

Commit 7a59a68

Browse files
craigtavernercraigtaverner
committed
Get CsvTests working on views
1 parent b37e1c7 commit 7a59a68

File tree

2 files changed

+86
-29
lines changed

2 files changed

+86
-29
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ public class CsvTestsDataLoader {
7878
private static final TestDataset LANGUAGES = new TestDataset("languages");
7979
private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup").withSetting("lookup-settings.json");
8080
private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key")
81-
.withData("languages_non_unique_key.csv");
81+
.withData("languages_non_unique_key.csv")
82+
.withDynamicTypeMapping(Map.of("country", "text"));
8283
private static final TestDataset LANGUAGES_NESTED_FIELDS = new TestDataset(
8384
"languages_nested_fields",
8485
"mapping-languages_nested_fields.json",
@@ -615,16 +616,20 @@ private static void loadEnrichPolicy(RestClient client, String policyName, Strin
615616
}
616617

617618
private static void loadView(RestClient client, String viewName, String viewFilename, Logger logger) throws IOException {
618-
logger.info("Loading view [{}] from file [{}]", viewName, viewFilename);
619-
URL viewFile = getResource("/" + viewFilename);
620-
String viewQuery = readTextFile(viewFile);
619+
String viewQuery = loadViewQuery(viewName, viewFilename, logger);
621620
Request request = new Request("PUT", "/_query/view/" + viewName);
622621
request.setJsonEntity("{\"query\":\"" + viewQuery.replace("\"", "\\\"").replace("\n", " ") + "\"}");
623622
Response response = client.performRequest(request);
624623
logger.info("View creation response: {}", response.getStatusLine());
625624
getView(client, viewName, logger);
626625
}
627626

627+
static String loadViewQuery(String viewName, String viewFilename, Logger logger) throws IOException {
628+
logger.info("Loading view [{}] from file [{}]", viewName, viewFilename);
629+
URL viewFile = getResource("/" + viewFilename);
630+
return readTextFile(viewFile);
631+
}
632+
628633
private static boolean getView(RestClient client, String viewName, Logger logger) throws IOException {
629634
Request request = new Request("GET", "/_query/view/" + viewName);
630635
try {
@@ -951,15 +956,16 @@ public record TestDataset(
951956
String dataFileName,
952957
String settingFileName,
953958
boolean allowSubFields,
954-
@Nullable Map<String, String> typeMapping,
959+
@Nullable Map<String, String> typeMapping, // Override mappings read from mappings file
960+
@Nullable Map<String, String> dynamicTypeMapping, // Define mappings not in the mapping files, but available from field-caps
955961
boolean requiresInferenceEndpoint
956962
) {
957963
public TestDataset(String indexName, String mappingFileName, String dataFileName) {
958-
this(indexName, mappingFileName, dataFileName, null, true, null, false);
964+
this(indexName, mappingFileName, dataFileName, null, true, null, null, false);
959965
}
960966

961967
public TestDataset(String indexName) {
962-
this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, false);
968+
this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, null, false);
963969
}
964970

965971
public TestDataset withIndex(String indexName) {
@@ -970,6 +976,7 @@ public TestDataset withIndex(String indexName) {
970976
settingFileName,
971977
allowSubFields,
972978
typeMapping,
979+
dynamicTypeMapping,
973980
requiresInferenceEndpoint
974981
);
975982
}
@@ -982,6 +989,7 @@ public TestDataset withData(String dataFileName) {
982989
settingFileName,
983990
allowSubFields,
984991
typeMapping,
992+
dynamicTypeMapping,
985993
requiresInferenceEndpoint
986994
);
987995
}
@@ -994,6 +1002,7 @@ public TestDataset noData() {
9941002
settingFileName,
9951003
allowSubFields,
9961004
typeMapping,
1005+
dynamicTypeMapping,
9971006
requiresInferenceEndpoint
9981007
);
9991008
}
@@ -1006,6 +1015,7 @@ public TestDataset withSetting(String settingFileName) {
10061015
settingFileName,
10071016
allowSubFields,
10081017
typeMapping,
1018+
dynamicTypeMapping,
10091019
requiresInferenceEndpoint
10101020
);
10111021
}
@@ -1018,6 +1028,7 @@ public TestDataset noSubfields() {
10181028
settingFileName,
10191029
false,
10201030
typeMapping,
1031+
dynamicTypeMapping,
10211032
requiresInferenceEndpoint
10221033
);
10231034
}
@@ -1030,12 +1041,35 @@ public TestDataset withTypeMapping(Map<String, String> typeMapping) {
10301041
settingFileName,
10311042
allowSubFields,
10321043
typeMapping,
1044+
dynamicTypeMapping,
1045+
requiresInferenceEndpoint
1046+
);
1047+
}
1048+
1049+
public TestDataset withDynamicTypeMapping(Map<String, String> dynamicTypeMapping) {
1050+
return new TestDataset(
1051+
indexName,
1052+
mappingFileName,
1053+
dataFileName,
1054+
settingFileName,
1055+
allowSubFields,
1056+
typeMapping,
1057+
dynamicTypeMapping,
10331058
requiresInferenceEndpoint
10341059
);
10351060
}
10361061

10371062
public TestDataset withInferenceEndpoint(boolean needsInference) {
1038-
return new TestDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference);
1063+
return new TestDataset(
1064+
indexName,
1065+
mappingFileName,
1066+
dataFileName,
1067+
settingFileName,
1068+
allowSubFields,
1069+
typeMapping,
1070+
dynamicTypeMapping,
1071+
needsInference
1072+
);
10391073
}
10401074

10411075
private Settings readSettingsFile() throws IOException {

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java‎

Lines changed: 44 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@
100100
import org.elasticsearch.xpack.esql.session.Result;
101101
import org.elasticsearch.xpack.esql.stats.DisabledSearchStats;
102102
import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
103+
import org.elasticsearch.xpack.esql.view.InMemoryViewService;
104+
import org.elasticsearch.xpack.esql.view.View;
103105
import org.junit.After;
104106
import org.junit.Before;
105107

@@ -123,6 +125,8 @@
123125
import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues;
124126
import static org.elasticsearch.xpack.esql.CsvTestUtils.loadPageFromCsv;
125127
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET_MAP;
128+
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.VIEW_CONFIGS;
129+
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadViewQuery;
126130
import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
127131
import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
128132
import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyInferenceResolution;
@@ -337,10 +341,6 @@ public final void test() throws Throwable {
337341
"CSV tests cannot currently handle TEXT_EMBEDDING function",
338342
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.TEXT_EMBEDDING_FUNCTION.capabilityName())
339343
);
340-
assumeFalse(
341-
"CSV tests cannot currently handle VIEWS",
342-
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.VIEWS_V1.capabilityName())
343-
);
344344
assumeFalse(
345345
"CSV tests cannot currently handle multi_match function that depends on Lucene",
346346
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.capabilityName())
@@ -430,21 +430,30 @@ private static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndex
430430

431431
private static Map<String, EsField> createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) {
432432
var mapping = new TreeMap<>(loadMapping(dataset.mappingFileName()));
433-
if (dataset.typeMapping() == null) {
434-
return mapping;
433+
if (dataset.typeMapping() != null) {
434+
for (var entry : dataset.typeMapping().entrySet()) {
435+
if (mapping.containsKey(entry.getKey())) {
436+
DataType dataType = DataType.fromTypeName(entry.getValue());
437+
EsField field = mapping.get(entry.getKey());
438+
EsField editedField = new EsField(
439+
field.getName(),
440+
dataType,
441+
field.getProperties(),
442+
field.isAggregatable(),
443+
field.getTimeSeriesFieldType()
444+
);
445+
mapping.put(entry.getKey(), editedField);
446+
}
447+
}
435448
}
436-
for (var entry : dataset.typeMapping().entrySet()) {
437-
if (mapping.containsKey(entry.getKey())) {
438-
DataType dataType = DataType.fromTypeName(entry.getValue());
439-
EsField field = mapping.get(entry.getKey());
440-
EsField editedField = new EsField(
441-
field.getName(),
442-
dataType,
443-
field.getProperties(),
444-
field.isAggregatable(),
445-
field.getTimeSeriesFieldType()
446-
);
447-
mapping.put(entry.getKey(), editedField);
449+
// Add dynamic mappings, but only if they are not already mapped
450+
if (dataset.dynamicTypeMapping() != null) {
451+
for (var entry : dataset.dynamicTypeMapping().entrySet()) {
452+
if (mapping.containsKey(entry.getKey()) == false) {
453+
DataType dataType = DataType.fromTypeName(entry.getValue());
454+
EsField editedField = new EsField(entry.getKey(), dataType, Map.of(), false, EsField.TimeSeriesFieldType.NONE);
455+
mapping.put(entry.getKey(), editedField);
456+
}
448457
}
449458
}
450459
return mapping;
@@ -538,7 +547,21 @@ private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.MultiInd
538547
return plan;
539548
}
540549

541-
private static CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan parsed) {
550+
private LogicalPlan resolveViews(LogicalPlan parsed) {
551+
InMemoryViewService viewService = new InMemoryViewService(functionRegistry);
552+
for (var viewConfig : VIEW_CONFIGS) {
553+
try {
554+
String viewQuery = loadViewQuery(viewConfig.viewName(), viewConfig.viewFileName(), LOGGER);
555+
viewService.put(viewConfig.viewName(), new View(viewQuery), ActionListener.noop(), configuration);
556+
} catch (IOException e) {
557+
logger.error("Failed to load view '" + viewConfig + "': " + e.getMessage());
558+
throw new RuntimeException(e);
559+
}
560+
}
561+
return viewService.replaceViews(parsed, new PlanTelemetry(functionRegistry), configuration);
562+
}
563+
564+
private CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan parsed) {
542565
var preAnalysis = new PreAnalyzer().preAnalyze(parsed);
543566
if (preAnalysis.indexPattern() == null) {
544567
// If the data set doesn't matter we'll just grab one we know works. Employees is fine.
@@ -583,8 +606,8 @@ private static TestPhysicalOperationProviders testOperationProviders(
583606
}
584607

585608
private ActualResults executePlan(BigArrays bigArrays) throws Exception {
586-
LogicalPlan parsed = parser.createStatement(testCase.query, EsqlTestUtils.TEST_CFG);
587-
var testDatasets = testDatasets(parsed);
609+
LogicalPlan parsed = resolveViews(parser.createStatement(testCase.query, configuration));
610+
var testDatasets = testDatasets(resolveViews(parsed));
588611
LogicalPlan analyzed = analyzedPlan(parsed, testDatasets);
589612

590613
FoldContext foldCtx = FoldContext.small();

0 commit comments

Comments
 (0)