ES|QL: more complex expressions in generative tests (#135766)

Let generative tests create more complex expressions, both in EVAL and
in STATS

Author: luigidellaquila

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java

@@ -66,6 +66,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query
         "The incoming YAML document exceeds the limit:", // still to investigate, but it seems to be specific to the test framework
         "Data too large", // Circuit breaker exceptions eg. https://github.com/elastic/elasticsearch/issues/130072
         "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/131509
+        "long overflow", // https://github.com/elastic/elasticsearch/issues/135759
         "can't build page out of released blocks", // https://github.com/elastic/elasticsearch/issues/135679
 
         // Awaiting fixes for correctness

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java

@@ -343,23 +343,43 @@ yield switch ((randomIntBetween(0, 2))) {
     }
 
     public static String agg(List<Column> previousOutput) {
-        String name = randomNumericOrDateField(previousOutput);
+        String name = randomNumericField(previousOutput);
+        // complex with numerics
         if (name != null && randomBoolean()) {
-            // numerics only
-            return switch (randomIntBetween(0, 1)) {
-                case 0 -> "max(" + name + ")";
-                default -> "min(" + name + ")";
-                // TODO more numerics
-            };
+            int ops = randomIntBetween(1, 3);
+            StringBuilder result = new StringBuilder();
+            for (int i = 0; i < ops; i++) {
+                if (i > 0) {
+                    result.append(" + ");
+                }
+                String agg = switch (randomIntBetween(0, 5)) {
+                    case 0 -> "max(" + name + ")";
+                    case 1 -> "min(" + name + ")";
+                    case 2 -> "avg(" + name + ")";
+                    case 3 -> "median(" + name + ")";
+                    case 4 -> "sum(" + name + ")";
+                    default -> "count(" + name + ")";
+                    // TODO more numerics
+                };
+                result.append(agg);
+            }
+            return result.toString();
         }
         // all types
         name = randomBoolean() ? randomStringField(previousOutput) : randomNumericOrDateField(previousOutput);
         if (name == null) {
             return "count(*)";
         }
-        return switch (randomIntBetween(0, 2)) {
+        if (randomBoolean()) {
+            String exp = expression(previousOutput, false);
+            name = exp == null ? name : exp;
+        }
+        return switch (randomIntBetween(0, 5)) {
             case 0 -> "count(*)";
             case 1 -> "count(" + name + ")";
+            case 2 -> "absent(" + name + ")";
+            case 3 -> "present(" + name + ")";
+            case 4 -> "values(" + name + ")";
             default -> "count_distinct(" + name + ")";
         };
     }
@@ -416,9 +436,50 @@ public static String randomName(List<Column> cols, Set<String> allowedTypes) {
         return items.get(randomIntBetween(0, items.size() - 1));
     }
 
-    public static String expression(List<Column> previousOutput) {
-        // TODO improve!!!
-        return constantExpression();
+    /**
+     * @param previousOutput columns that can be used in the expression
+     * @param allowConstants if set to true, this will never return a constant expression.
+     *                       If no expression can be generated, it will return null
+     * @return an expression or null
+     */
+    public static String expression(List<Column> previousOutput, boolean allowConstants) {
+        if (randomBoolean() && allowConstants) {
+            return constantExpression();
+        }
+        if (randomBoolean()) {
+            StringBuilder result = new StringBuilder();
+            for (int i = 0; i < randomIntBetween(1, 3); i++) {
+                String field = randomNumericField(previousOutput);
+                if (field == null) {
+                    return allowConstants ? constantExpression() : null;
+                }
+                if (i > 0) {
+                    result.append(" + ");
+                }
+                result.append(field);
+            }
+            return result.toString();
+        }
+        if (randomBoolean()) {
+            String field = randomKeywordField(previousOutput);
+            if (field == null) {
+                return allowConstants ? constantExpression() : null;
+            }
+            return switch (randomIntBetween(0, 3)) {
+                case 0 -> "substring(" + field + ", 1, 3)";
+                case 1 -> "to_lower(" + field + ")";
+                case 2 -> "to_upper(" + field + ")";
+                default -> "length(" + field + ")";
+            };
+        }
+        if (randomBoolean() || allowConstants == false) {
+            String field = randomStringField(previousOutput);
+            if (field == null || randomBoolean()) {
+                field = randomNumericOrDateField(previousOutput);
+            }
+            return field;
+        }
+        return allowConstants ? constantExpression() : null;
     }
 
     public static String indexPattern(String indexName) {

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java

@@ -15,6 +15,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
 import static org.elasticsearch.test.ESTestCase.randomIntBetween;
@@ -34,6 +35,7 @@ public CommandDescription generate(
     ) {
         StringBuilder cmd = new StringBuilder(" | eval ");
         int nFields = randomIntBetween(1, 10);
+        Map<String, Column> usablePrevious = previousOutput.stream().collect(Collectors.toMap(Column::name, c -> c));
         // TODO pass newly created fields to next expressions
         var newColumns = new ArrayList<>();
         for (int i = 0; i < nFields; i++) {
@@ -46,7 +48,7 @@ public CommandDescription generate(
                     name = EsqlQueryGenerator.randomIdentifier();
                 }
             }
-            String expression = EsqlQueryGenerator.expression(previousOutput);
+            String expression = EsqlQueryGenerator.expression(usablePrevious.values().stream().toList(), true);
             if (i > 0) {
                 cmd.append(",");
             }
@@ -56,6 +58,11 @@ public CommandDescription generate(
             newColumns.add(unquote(name));
             cmd.append(" = ");
             cmd.append(expression);
+
+            // there could be collisions in many ways, remove all of them
+            usablePrevious.remove(name);
+            usablePrevious.remove("`" + name + "`");
+            usablePrevious.remove(unquote(name));
         }
         String cmdString = cmd.toString();
         return new CommandDescription(EVAL, this, cmdString, Map.ofEntries(Map.entry(NEW_COLUMNS, newColumns)));