Use standard tokenizer in ES|QL categorize

Author: jan-elastic

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java

@@ -210,6 +210,23 @@ public static CategorizationAnalyzerConfig buildStandardCategorizationAnalyzer(L
             .build();
     }
 
+    /**
+     * Create a <code>categorization_analyzer</code> that will be used by the ES|QL categorize function.
+     * The only difference from the DSL analyzer is the tokenizer (standard instead of ml_standard).
+     * This means the results are slightly different from the categorize text aggregation and the ML job,
+     * however you can use these tokens for looking up messages in indices generated with the standard
+     * tokenizer. The latter is considered more important.
+     */
+    public static CategorizationAnalyzerConfig buildStandardEsqlCategorizationAnalyzer(List<String> categorizationFilters) {
+
+        return new CategorizationAnalyzerConfig.Builder().addCharFilter("first_line_with_letters")
+            .addCategorizationFilters(categorizationFilters)
+            .setTokenizer("standard")
+            .addDateWordsTokenFilter()
+            .addLimitFilter()
+            .build();
+    }
+
     private final String analyzer;
     private final List<NameOrDefinition> charFilters;
     private final NameOrDefinition tokenizer;

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/CategorizeBlockHash.java

@@ -48,9 +48,8 @@
  */
 public class CategorizeBlockHash extends BlockHash {
 
-    private static final CategorizationAnalyzerConfig ANALYZER_CONFIG = CategorizationAnalyzerConfig.buildStandardCategorizationAnalyzer(
-        List.of()
-    );
+    private static final CategorizationAnalyzerConfig ANALYZER_CONFIG = CategorizationAnalyzerConfig
+        .buildStandardEsqlCategorizationAnalyzer(List.of());
     private static final int NULL_ORD = 0;
 
     private final int channel;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/categorize.csv-spec

@@ -256,9 +256,9 @@ FROM hosts
 ;
 
 COUNT():long | category:keyword
+           2 | .*?DB.+?servers.*?
            2 | .*?Gateway.+?instances.*?
            5 | .*?Kubernetes.+?cluster.*?
-           2 | .*?servers.*?
            1 | null
 
 // Note: DB is removed from "DB servers", because the ml_standard
@@ -302,9 +302,9 @@ FROM sample_data
 ;
 
 COUNT():long | category:keyword
-           3 | .*?Connected.+?to.*?
-           3 | .*?Connection.+?error.*?
-           1 | .*?Disconnected.*?
+3            | .*?Connected.+?to.+?👍🏽.+?😊.*?
+3            | .*?Connection.+?error.+?👍🏽.+?😊.*?
+1            | .*?Disconnected.+?👍🏽.+?😊.*?
 ;
 
 on REVERSE(CONCAT())
@@ -316,9 +316,9 @@ FROM sample_data
 ;
 
 COUNT():long | category:keyword
-           1 | .*?detcennocsiD.*?
-           3 | .*?ot.+?detcennoC.*?
-           3 | .*?rorre.+?noitcennoC.*?
+1            | .*?😊.+?👍🏽.+?detcennocsiD.*?
+3            | .*?😊.+?👍🏽.+?ot.+?detcennoC.*?
+3            | .*?😊.+?👍🏽.+?rorre.+?noitcennoC.*?
 ;
 
 and then TO_LOWER