ESQL: Fix case insensitive comparisons to "" (elastic#127532) (elastic#128580)

This fixes the compute engine side of case insensitive ==. You can
trigger it by writing:
```
FROM foo
| WHERE TO_LOWER(field) == ""
```

But *only* when we can't push the comparison to lucene - like if `field`
is not indexed or is a `text` field.

Closes elastic#127431

Author: nik9000

docs/changelog/127532.yaml

@@ -0,0 +1,6 @@
+pr: 127532
+summary: Fix case insensitive comparisons to ""
+area: ES|QL
+type: bug
+issues:
+ - 127431

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -888,7 +888,12 @@ public enum Cap {
          * {@link org.elasticsearch.compute.lucene.LuceneQueryEvaluator} rewrites the query before executing it in Lucene. This
          * provides support for KQL in a STATS ... BY command that uses a KQL query for filter, for example.
          */
-        LUCENE_QUERY_EVALUATOR_QUERY_REWRITE;
+        LUCENE_QUERY_EVALUATOR_QUERY_REWRITE,
+
+        /**
+         * Guards a bug fix matching {@code TO_LOWER(f) == ""}.
+         */
+        TO_LOWER_EMPTY_STRING;
 
         private final boolean enabled;
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
 
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -83,6 +84,10 @@ protected TypeResolution resolveType() {
     }
 
     public static Automaton automaton(BytesRef val) {
+        if (val.length == 0) {
+            // toCaseInsensitiveString doesn't match empty strings properly so let's do it ourselves
+            return Automata.makeEmptyString();
+        }
         return AutomatonQueries.toCaseInsensitiveString(val.utf8ToString());
     }
 

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java

@@ -26,6 +26,8 @@ public void testFold() {
         assertTrue(insensitiveEquals(l("foo*"), l("FOO*")).fold(FoldContext.small()));
         assertTrue(insensitiveEquals(l("foo?bar"), l("foo?bar")).fold(FoldContext.small()));
         assertTrue(insensitiveEquals(l("foo?bar"), l("FOO?BAR")).fold(FoldContext.small()));
+        assertTrue(insensitiveEquals(l(""), l("")).fold(FoldContext.small()));
+
         assertFalse(insensitiveEquals(l("Foo"), l("fo*")).fold(FoldContext.small()));
         assertFalse(insensitiveEquals(l("Fox"), l("fo?")).fold(FoldContext.small()));
         assertFalse(insensitiveEquals(l("Foo"), l("*OO")).fold(FoldContext.small()));
@@ -60,6 +62,8 @@ public void testProcess() {
         assertTrue(InsensitiveEquals.process(BytesRefs.toBytesRef("foo*"), BytesRefs.toBytesRef("FOO*")));
         assertTrue(InsensitiveEquals.process(BytesRefs.toBytesRef("foo?bar"), BytesRefs.toBytesRef("foo?bar")));
         assertTrue(InsensitiveEquals.process(BytesRefs.toBytesRef("foo?bar"), BytesRefs.toBytesRef("FOO?BAR")));
+        assertTrue(InsensitiveEquals.process(BytesRefs.toBytesRef(""), BytesRefs.toBytesRef("")));
+
         assertFalse(InsensitiveEquals.process(BytesRefs.toBytesRef("Foo"), BytesRefs.toBytesRef("fo*")));
         assertFalse(InsensitiveEquals.process(BytesRefs.toBytesRef("Fox"), BytesRefs.toBytesRef("fo?")));
         assertFalse(InsensitiveEquals.process(BytesRefs.toBytesRef("Foo"), BytesRefs.toBytesRef("*OO")));

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/210_empty_string.yml

@@ -0,0 +1,79 @@
+---
+setup:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ query_monitoring ]
+      reason: "uses query monitoring"
+
+  - do:
+      bulk:
+        index: "test"
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2023-10-23T13:55:01.543Z", "message": "" }
+          - { "index": { } }
+          - { "@timestamp": "2023-10-23T13:55:01.544Z" }
+          - { "index": { } }
+          - { "@timestamp": "2023-10-23T13:55:01.545Z", "message": "a" }
+
+---
+keyword equals empty string:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | WHERE message.keyword == "" | SORT @timestamp ASC | KEEP @timestamp | LIMIT 10'
+
+  - match: { columns.0.name: "@timestamp" }
+  - match: { columns.0.type: "date" }
+  - length: { values: 1 }
+  - match: { values.0.0: 2023-10-23T13:55:01.543Z }
+
+---
+keyword to_lower equals empty string:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | WHERE TO_LOWER(message.keyword) == "" | SORT @timestamp ASC | KEEP @timestamp | LIMIT 10'
+
+  - match: { columns.0.name: "@timestamp" }
+  - match: { columns.0.type: "date" }
+  - length: { values: 1 }
+  - match: { values.0.0: 2023-10-23T13:55:01.543Z }
+
+---
+text equals empty string:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | WHERE message == "" | SORT @timestamp ASC | KEEP @timestamp | LIMIT 10'
+
+  - match: { columns.0.name: "@timestamp" }
+  - match: { columns.0.type: "date" }
+  - length: { values: 1 }
+  - match: { values.0.0: 2023-10-23T13:55:01.543Z }
+
+---
+text to_lower equals empty string:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ to_lower_empty_string ]
+      reason: "bug"
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | WHERE TO_LOWER(message) == "" | SORT @timestamp ASC | KEEP @timestamp | LIMIT 10'
+
+  - match: { columns.0.name: "@timestamp" }
+  - match: { columns.0.type: "date" }
+  - length: { values: 1 }
+  - match: { values.0.0: 2023-10-23T13:55:01.543Z }