ESQL: TO_LOWER process all values (#124676)

Make `TO_LOWER` and `TO_UPPER` process all values it received.

This is quite large because it borrows a lot of code from the regular
evaluator generator to generate conversions so we can use the Locale.
That change propagates to the order of some parameters and to the
`toString` and a few more places.

Closes #124002

Author: nik9000

docs/changelog/124676.yaml

@@ -0,0 +1,6 @@
+pr: 124676
+summary: TO_LOWER processes all values
+area: ES|QL
+type: bug
+issues:
+ - 124002

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java

@@ -23,7 +23,6 @@
 import javax.lang.model.type.TypeMirror;
 import javax.lang.model.util.Elements;
 
-import static org.elasticsearch.compute.gen.Methods.appendMethod;
 import static org.elasticsearch.compute.gen.Methods.buildFromFactory;
 import static org.elasticsearch.compute.gen.Methods.getMethod;
 import static org.elasticsearch.compute.gen.Types.ABSTRACT_CONVERT_FUNCTION_EVALUATOR;
@@ -41,27 +40,34 @@
 public class ConvertEvaluatorImplementer {
 
     private final TypeElement declarationType;
-    private final ExecutableElement processFunction;
+    private final EvaluatorImplementer.ProcessFunction processFunction;
     private final String extraName;
     private final ClassName implementation;
     private final TypeName argumentType;
-    private final TypeName resultType;
     private final List<TypeMirror> warnExceptions;
 
     public ConvertEvaluatorImplementer(
         Elements elements,
+        javax.lang.model.util.Types types,
         ExecutableElement processFunction,
         String extraName,
         List<TypeMirror> warnExceptions
     ) {
         this.declarationType = (TypeElement) processFunction.getEnclosingElement();
-        this.processFunction = processFunction;
-        if (processFunction.getParameters().size() != 1) {
-            throw new IllegalArgumentException("processing function should have exactly one parameter");
+        this.processFunction = new EvaluatorImplementer.ProcessFunction(types, processFunction, warnExceptions);
+
+        if (this.processFunction.args.getFirst() instanceof EvaluatorImplementer.StandardProcessFunctionArg == false) {
+            throw new IllegalArgumentException("first argument must be the field to process");
+        }
+        for (int a = 1; a < this.processFunction.args.size(); a++) {
+            if (this.processFunction.args.get(a) instanceof EvaluatorImplementer.FixedProcessFunctionArg == false) {
+                throw new IllegalArgumentException("fixed function args supported after the first");
+                // TODO support more function types when we need them
+            }
         }
+
         this.extraName = extraName;
         this.argumentType = TypeName.get(processFunction.getParameters().get(0).asType());
-        this.resultType = TypeName.get(processFunction.getReturnType());
         this.warnExceptions = warnExceptions;
 
         this.implementation = ClassName.get(
@@ -87,29 +93,36 @@ private TypeSpec type() {
         builder.addModifiers(Modifier.PUBLIC, Modifier.FINAL);
         builder.superclass(ABSTRACT_CONVERT_FUNCTION_EVALUATOR);
 
+        for (EvaluatorImplementer.ProcessFunctionArg a : processFunction.args) {
+            a.declareField(builder);
+        }
         builder.addMethod(ctor());
-        builder.addMethod(name());
+        builder.addMethod(next());
         builder.addMethod(evalVector());
         builder.addMethod(evalValue(true));
         builder.addMethod(evalBlock());
         builder.addMethod(evalValue(false));
+        builder.addMethod(processFunction.toStringMethod(implementation));
+        builder.addMethod(processFunction.close());
         builder.addType(factory());
         return builder.build();
     }
 
     private MethodSpec ctor() {
         MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
-        builder.addParameter(EXPRESSION_EVALUATOR, "field");
         builder.addParameter(SOURCE, "source");
+        builder.addStatement("super(driverContext, source)");
+        for (EvaluatorImplementer.ProcessFunctionArg a : processFunction.args) {
+            a.implementCtor(builder);
+        }
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
-        builder.addStatement("super(driverContext, field, source)");
         return builder.build();
     }
 
-    private MethodSpec name() {
-        MethodSpec.Builder builder = MethodSpec.methodBuilder("name").addModifiers(Modifier.PUBLIC);
-        builder.addAnnotation(Override.class).returns(String.class);
-        builder.addStatement("return $S", declarationType.getSimpleName() + extraName);
+    private MethodSpec next() {
+        MethodSpec.Builder builder = MethodSpec.methodBuilder("next").addAnnotation(Override.class).addModifiers(Modifier.PUBLIC);
+        builder.returns(EXPRESSION_EVALUATOR);
+        builder.addStatement("return $N", ((EvaluatorImplementer.StandardProcessFunctionArg) processFunction.args.getFirst()).name());
         return builder.build();
     }
 
@@ -129,7 +142,7 @@ private MethodSpec evalVector() {
         builder.beginControlFlow("if (vector.isConstant())");
         {
             catchingWarnExceptions(builder, () -> {
-                var constVectType = blockType(resultType);
+                var constVectType = processFunction.resultDataType(true);
                 builder.addStatement(
                     "return driverContext.blockFactory().newConstant$TWith($N, positionCount)",
                     constVectType,
@@ -139,7 +152,7 @@ private MethodSpec evalVector() {
         }
         builder.endControlFlow();
 
-        ClassName resultBuilderType = builderType(blockType(resultType));
+        ClassName resultBuilderType = builderType(processFunction.resultDataType(true));
         builder.beginControlFlow(
             "try ($T builder = driverContext.blockFactory().$L(positionCount))",
             resultBuilderType,
@@ -150,7 +163,11 @@ private MethodSpec evalVector() {
             {
                 catchingWarnExceptions(
                     builder,
-                    () -> builder.addStatement("builder.$L($N)", appendMethod(resultType), evalValueCall("vector", "p", scratchPadName)),
+                    () -> builder.addStatement(
+                        "builder.$L($N)",
+                        processFunction.appendMethod(),
+                        evalValueCall("vector", "p", scratchPadName)
+                    ),
                     () -> builder.addStatement("builder.appendNull()")
                 );
             }
@@ -185,7 +202,7 @@ private MethodSpec evalBlock() {
         TypeName blockType = blockType(argumentType);
         builder.addStatement("$T block = ($T) b", blockType, blockType);
         builder.addStatement("int positionCount = block.getPositionCount()");
-        TypeName resultBuilderType = builderType(blockType(resultType));
+        TypeName resultBuilderType = builderType(processFunction.resultDataType(true));
         builder.beginControlFlow(
             "try ($T builder = driverContext.blockFactory().$L(positionCount))",
             resultBuilderType,
@@ -196,19 +213,18 @@ private MethodSpec evalBlock() {
             builder.addStatement("BytesRef $N = new BytesRef()", scratchPadName);
         }
 
-        String appendMethod = appendMethod(resultType);
+        String appendMethod = processFunction.appendMethod();
         builder.beginControlFlow("for (int p = 0; p < positionCount; p++)");
         {
             builder.addStatement("int valueCount = block.getValueCount(p)");
             builder.addStatement("int start = block.getFirstValueIndex(p)");
             builder.addStatement("int end = start + valueCount");
             builder.addStatement("boolean positionOpened = false");
             builder.addStatement("boolean valuesAppended = false");
-            // builder.addStatement("builder.beginPositionEntry()");
             builder.beginControlFlow("for (int i = start; i < end; i++)");
             {
                 catchingWarnExceptions(builder, () -> {
-                    builder.addStatement("$T value = $N", resultType, evalValueCall("block", "i", scratchPadName));
+                    builder.addStatement("$T value = $N", processFunction.returnType(), evalValueCall("block", "i", scratchPadName));
                     builder.beginControlFlow("if (positionOpened == false && valueCount > 1)");
                     {
                         builder.addStatement("builder.beginPositionEntry()");
@@ -253,8 +269,8 @@ private String evalValueCall(String container, String index, String scratchPad)
 
     private MethodSpec evalValue(boolean forVector) {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("evalValue")
-            .addModifiers(Modifier.PRIVATE, Modifier.STATIC)
-            .returns(resultType);
+            .addModifiers(Modifier.PRIVATE)
+            .returns(processFunction.returnType());
 
         if (forVector) {
             builder.addParameter(vectorType(argumentType), "container");
@@ -269,8 +285,17 @@ private MethodSpec evalValue(boolean forVector) {
             builder.addStatement("$T value = container.$N(index)", argumentType, getMethod(argumentType));
         }
 
-        builder.addStatement("return $T.$N(value)", declarationType, processFunction.getSimpleName());
-
+        StringBuilder pattern = new StringBuilder();
+        List<Object> args = new ArrayList<>();
+        pattern.append("return $T.$N(value");
+        args.add(declarationType);
+        args.add(processFunction.function.getSimpleName());
+        for (int a = 1; a < processFunction.args.size(); a++) {
+            pattern.append(", ");
+            processFunction.args.get(a).buildInvocation(pattern, args, false /* block style parameter should be unused */);
+        }
+        pattern.append(")");
+        builder.addStatement(pattern.toString(), args.toArray());
         return builder.build();
     }
 
@@ -280,42 +305,11 @@ private TypeSpec factory() {
         builder.addModifiers(Modifier.PUBLIC, Modifier.STATIC);
 
         builder.addField(SOURCE, "source", Modifier.PRIVATE, Modifier.FINAL);
-        builder.addField(EXPRESSION_EVALUATOR_FACTORY, "field", Modifier.PRIVATE, Modifier.FINAL);
-
-        builder.addMethod(factoryCtor());
-        builder.addMethod(factoryGet());
-        builder.addMethod(factoryToString());
-        return builder.build();
-    }
-
-    private MethodSpec factoryCtor() {
-        MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
-        builder.addParameter(EXPRESSION_EVALUATOR_FACTORY, "field");
-        builder.addParameter(SOURCE, "source");
-        builder.addStatement("this.field = field");
-        builder.addStatement("this.source = source");
-        return builder.build();
-    }
-
-    private MethodSpec factoryGet() {
-        MethodSpec.Builder builder = MethodSpec.methodBuilder("get").addAnnotation(Override.class);
-        builder.addModifiers(Modifier.PUBLIC);
-        builder.addParameter(DRIVER_CONTEXT, "context");
-        builder.returns(implementation);
-
-        List<String> args = new ArrayList<>();
-        args.add("field.get(context)");
-        args.add("source");
-        args.add("context");
-        builder.addStatement("return new $T($L)", implementation, args.stream().collect(Collectors.joining(", ")));
-        return builder.build();
-    }
+        processFunction.args.forEach(a -> a.declareFactoryField(builder));
 
-    private MethodSpec factoryToString() {
-        MethodSpec.Builder builder = MethodSpec.methodBuilder("toString").addAnnotation(Override.class);
-        builder.addModifiers(Modifier.PUBLIC);
-        builder.returns(String.class);
-        builder.addStatement("return $S + field + $S", declarationType.getSimpleName() + extraName + "Evaluator[field=", "]");
+        builder.addMethod(processFunction.factoryCtor());
+        builder.addMethod(processFunction.factoryGet(implementation));
+        builder.addMethod(processFunction.toStringMethod(implementation));
         return builder.build();
     }
 }