diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index ee1fe6d3b2c1d..e7ffa3aeb4893 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -66,6 +66,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "The incoming YAML document exceeds the limit:", // still to investigate, but it seems to be specific to the test framework "Data too large", // Circuit breaker exceptions eg. https://github.com/elastic/elasticsearch/issues/130072 "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/131509 + "long overflow", // https://github.com/elastic/elasticsearch/issues/135759 "can't build page out of released blocks", // https://github.com/elastic/elasticsearch/issues/135679 // Awaiting fixes for correctness diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java index cdb262c91dfdb..a235422cf005b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java @@ -343,23 +343,43 @@ yield switch ((randomIntBetween(0, 2))) { } public static String agg(List previousOutput) { - String name = randomNumericOrDateField(previousOutput); + String name = randomNumericField(previousOutput); + // complex with numerics if (name != null && randomBoolean()) { - // numerics only - return switch (randomIntBetween(0, 1)) { - case 0 -> "max(" + name + ")"; - default -> "min(" + name + ")"; - // TODO more numerics - }; + int ops = randomIntBetween(1, 3); + StringBuilder result = new StringBuilder(); + for (int i = 0; i < ops; i++) { + if (i > 0) { + result.append(" + "); + } + String agg = switch (randomIntBetween(0, 5)) { + case 0 -> "max(" + name + ")"; + case 1 -> "min(" + name + ")"; + case 2 -> "avg(" + name + ")"; + case 3 -> "median(" + name + ")"; + case 4 -> "sum(" + name + ")"; + default -> "count(" + name + ")"; + // TODO more numerics + }; + result.append(agg); + } + return result.toString(); } // all types name = randomBoolean() ? randomStringField(previousOutput) : randomNumericOrDateField(previousOutput); if (name == null) { return "count(*)"; } - return switch (randomIntBetween(0, 2)) { + if (randomBoolean()) { + String exp = expression(previousOutput, false); + name = exp == null ? name : exp; + } + return switch (randomIntBetween(0, 5)) { case 0 -> "count(*)"; case 1 -> "count(" + name + ")"; + case 2 -> "absent(" + name + ")"; + case 3 -> "present(" + name + ")"; + case 4 -> "values(" + name + ")"; default -> "count_distinct(" + name + ")"; }; } @@ -416,9 +436,50 @@ public static String randomName(List cols, Set allowedTypes) { return items.get(randomIntBetween(0, items.size() - 1)); } - public static String expression(List previousOutput) { - // TODO improve!!! - return constantExpression(); + /** + * @param previousOutput columns that can be used in the expression + * @param allowConstants if set to true, this will never return a constant expression. + * If no expression can be generated, it will return null + * @return an expression or null + */ + public static String expression(List previousOutput, boolean allowConstants) { + if (randomBoolean() && allowConstants) { + return constantExpression(); + } + if (randomBoolean()) { + StringBuilder result = new StringBuilder(); + for (int i = 0; i < randomIntBetween(1, 3); i++) { + String field = randomNumericField(previousOutput); + if (field == null) { + return allowConstants ? constantExpression() : null; + } + if (i > 0) { + result.append(" + "); + } + result.append(field); + } + return result.toString(); + } + if (randomBoolean()) { + String field = randomKeywordField(previousOutput); + if (field == null) { + return allowConstants ? constantExpression() : null; + } + return switch (randomIntBetween(0, 3)) { + case 0 -> "substring(" + field + ", 1, 3)"; + case 1 -> "to_lower(" + field + ")"; + case 2 -> "to_upper(" + field + ")"; + default -> "length(" + field + ")"; + }; + } + if (randomBoolean() || allowConstants == false) { + String field = randomStringField(previousOutput); + if (field == null || randomBoolean()) { + field = randomNumericOrDateField(previousOutput); + } + return field; + } + return allowConstants ? constantExpression() : null; } public static String indexPattern(String indexName) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java index 8c2841dda3f04..95bb374ce0bd9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomIntBetween; @@ -34,6 +35,7 @@ public CommandDescription generate( ) { StringBuilder cmd = new StringBuilder(" | eval "); int nFields = randomIntBetween(1, 10); + Map usablePrevious = previousOutput.stream().collect(Collectors.toMap(Column::name, c -> c)); // TODO pass newly created fields to next expressions var newColumns = new ArrayList<>(); for (int i = 0; i < nFields; i++) { @@ -46,7 +48,7 @@ public CommandDescription generate( name = EsqlQueryGenerator.randomIdentifier(); } } - String expression = EsqlQueryGenerator.expression(previousOutput); + String expression = EsqlQueryGenerator.expression(usablePrevious.values().stream().toList(), true); if (i > 0) { cmd.append(","); } @@ -56,6 +58,11 @@ public CommandDescription generate( newColumns.add(unquote(name)); cmd.append(" = "); cmd.append(expression); + + // there could be collisions in many ways, remove all of them + usablePrevious.remove(name); + usablePrevious.remove("`" + name + "`"); + usablePrevious.remove(unquote(name)); } String cmdString = cmd.toString(); return new CommandDescription(EVAL, this, cmdString, Map.ofEntries(Map.entry(NEW_COLUMNS, newColumns)));