Add CSV tests and necessary infra for dense_vector field type

Author: carlosdelest

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java

@@ -35,6 +35,7 @@
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.Type;
+import static org.elasticsearch.xpack.esql.CsvTestUtils.Type.DENSE_VECTOR;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.Type.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.logMetaData;
 import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC_DATE_TIME_FORMATTER;
@@ -145,6 +146,10 @@ private static void assertMetadata(
                     // Type.asType translates all bytes references into keywords
                     continue;
                 }
+                if (blockType == Type.DOUBLE && expectedType == DENSE_VECTOR) {
+                    // DENSE_VECTOR is internally represented as a double block
+                    continue;
+                }
                 if (blockType == Type.NULL) {
                     // Null pages don't have any real type information beyond "it's all null, man"
                     continue;

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java

@@ -486,6 +486,7 @@ public enum Type {
             x -> x == null ? null : stringToAggregateMetricDoubleLiteral(x),
             AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral.class
         ),
+        DENSE_VECTOR(Double::parseDouble, Double.class),
         UNSUPPORTED(Type::convertUnsupported, Void.class);
 
         private static Void convertUnsupported(String s) {
@@ -528,6 +529,8 @@ private static Void convertUnsupported(String s) {
             LOOKUP.put("DATE", DATETIME);
             LOOKUP.put("DT", DATETIME);
             LOOKUP.put("V", VERSION);
+
+            LOOKUP.put("DENSE_VECTOR", DENSE_VECTOR);
         }
 
         private final Function<String, Object> converter;

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -140,6 +140,7 @@ public class CsvTestsDataLoader {
     private static final TestDataset SEMANTIC_TEXT = new TestDataset("semantic_text").withInferenceEndpoint(true);
     private static final TestDataset LOGS = new TestDataset("logs");
     private static final TestDataset MV_TEXT = new TestDataset("mv_text");
+    private static final TestDataset DENSE_VECTOR = new TestDataset("dense_vector");
 
     public static final Map<String, TestDataset> CSV_DATASET_MAP = Map.ofEntries(
         Map.entry(EMPLOYEES.indexName, EMPLOYEES),
@@ -198,7 +199,8 @@ public class CsvTestsDataLoader {
         Map.entry(BOOKS.indexName, BOOKS),
         Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT),
         Map.entry(LOGS.indexName, LOGS),
-        Map.entry(MV_TEXT.indexName, MV_TEXT)
+        Map.entry(MV_TEXT.indexName, MV_TEXT),
+        Map.entry(DENSE_VECTOR.indexName, DENSE_VECTOR)
     );
 
     private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");

x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/dense_vector.csv

@@ -0,0 +1,3 @@
+id:l, vector:dense_vector
+0,    [1.0, 2.0, 3.0]
+1,    [4.0, 5.0, 6.0]

x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec

@@ -0,0 +1,13 @@
+
+retrieveDenseVectorData
+required_capability: dense_vector_field_type
+
+FROM dense_vector
+| KEEP id, vector
+| SORT id
+;
+
+id:l | vector:dense_vector
+0    | [1.0, 2.0, 3.0]
+1    | [4.0, 5.0, 6.0]
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-dense_vector.json

@@ -0,0 +1,11 @@
+{
+  "properties": {
+    "id": {
+      "type": "long"
+    },
+    "vector": {
+      "type": "dense_vector",
+      "similarity": "l2_norm"
+    }
+  }
+}