ES|QL: generative tests - fix identifiers and source field mapping

Author: luigidellaquila

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java

@@ -160,11 +160,11 @@ private static String grok(List<Column> previousOutput) {
             }
             result.append("%{WORD:");
             if (randomBoolean()) {
-                result.append(randomAlphaOfLength(5));
+                result.append(randomIdentifier());
             } else {
                 String fieldName = randomRawName(previousOutput);
                 if (fieldName.isEmpty()) { // it's a bug, managed later, skipping for now
-                    fieldName = randomAlphaOfLength(5);
+                    fieldName = randomIdentifier();
                 }
                 result.append(fieldName);
             }
@@ -188,11 +188,11 @@ private static String dissect(List<Column> previousOutput) {
             }
             result.append("%{");
             if (randomBoolean()) {
-                result.append(randomAlphaOfLength(5));
+                result.append(randomIdentifier());
             } else {
                 String fieldName = randomRawName(previousOutput);
                 if (fieldName.isEmpty()) { // it's a bug, managed later, skipping for now
-                    fieldName = randomAlphaOfLength(5);
+                    fieldName = randomIdentifier();
                 }
                 result.append(fieldName);
             }
@@ -302,7 +302,7 @@ private static String rename(List<Column> previousOutput) {
 
             String newName;
             if (names.isEmpty() || randomBoolean()) {
-                newName = randomAlphaOfLength(5);
+                newName = randomIdentifier();
                 names.add(newName);
             } else {
                 newName = names.get(randomIntBetween(0, names.size() - 1));
@@ -376,7 +376,7 @@ private static String eval(List<Column> previousOutput) {
         for (int i = 0; i < nFields; i++) {
             String name;
             if (randomBoolean()) {
-                name = randomAlphaOfLength(randomIntBetween(3, 10));
+                name = randomIdentifier();
             } else {
                 name = randomName(previousOutput);
             }
@@ -402,7 +402,7 @@ private static String stats(List<Column> previousOutput) {
         for (int i = 0; i < nStats; i++) {
             String name;
             if (randomBoolean()) {
-                name = randomAlphaOfLength(randomIntBetween(3, 10));
+                name = randomIdentifier();
             } else {
                 name = randomName(previousOutput);
             }
@@ -501,7 +501,7 @@ private static String row() {
         StringBuilder cmd = new StringBuilder("row ");
         int nFields = randomIntBetween(1, 10);
         for (int i = 0; i < nFields; i++) {
-            String name = randomAlphaOfLength(randomIntBetween(3, 10));
+            String name = randomIdentifier();
             String expression = constantExpression();
             if (i > 0) {
                 cmd.append(",");
@@ -526,4 +526,8 @@ private static String constantExpression() {
 
     }
 
+    private static String randomIdentifier() {
+        return randomAlphaOfLength(randomIntBetween(8, 12));
+    }
+
 }

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java

@@ -25,6 +25,7 @@
 
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET_MAP;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.ENRICH_POLICIES;
+import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.availableDatasetsForEs;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs;
 
 public abstract class GenerativeRestTest extends ESRestTestCase {
@@ -41,7 +42,6 @@ public abstract class GenerativeRestTest extends ESRestTestCase {
         "Unbounded sort not supported yet",
         "The field names are too complex to process", // field_caps problem
         "must be \\[any type except counter types\\]", // TODO refine the generation of count()
-        "mismatched input .* expecting", // identifier generator needs to be refined, this happens when an identifier is a reserved keyword
 
         // warnings
         "Field '.*' shadowed by field at line .*",
@@ -85,7 +85,7 @@ public static void wipeTestData() throws IOException {
         }
     }
 
-    public void test() {
+    public void test() throws IOException {
         List<String> indices = availableIndices();
         List<LookupIdx> lookupIndices = lookupIndices();
         List<CsvTestsDataLoader.EnrichConfig> policies = availableEnrichPolicies();
@@ -142,14 +142,11 @@ private List<EsqlQueryGenerator.Column> outputSchema(Map<String, Object> a) {
         return cols.stream().map(x -> new EsqlQueryGenerator.Column(x.get("name"), x.get("type"))).collect(Collectors.toList());
     }
 
-    private List<String> availableIndices() {
-        return new ArrayList<>(
-            CSV_DATASET_MAP.entrySet()
-                .stream()
-                .filter(x -> x.getValue().requiresInferenceEndpoint() == false)
-                .map(Map.Entry::getKey)
-                .toList()
-        );
+    private List<String> availableIndices() throws IOException {
+        return availableDatasetsForEs(client(), true, supportsSourceFieldMapping()).stream()
+            .filter(x -> x.requiresInferenceEndpoint() == false)
+            .map(x -> x.indexName())
+            .toList();
     }
 
     record LookupIdx(String idxName, String key, String keyType) {}

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -315,7 +315,7 @@ private static boolean isLookupDataset(TestDataset dataset) throws IOException {
         return (mode != null && mode.equalsIgnoreCase("lookup"));
     }
 
-    private static boolean isSourceMappingDataset(TestDataset dataset) throws IOException {
+    public static boolean isSourceMappingDataset(TestDataset dataset) throws IOException {
         if (dataset.mappingFileName() == null) {
             return true;
         }