Use standard tokenizer in ES|QL categorize

Author: jan-elastic

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java

@@ -210,6 +210,23 @@ public static CategorizationAnalyzerConfig buildStandardCategorizationAnalyzer(L
             .build();
     }
 
+    /**
+     * Create a <code>categorization_analyzer</code> that will be used by the ES|QL categorize function.
+     * The only difference from the DSL analyzer is the tokenizer (standard instead of ml_standard).
+     * This means the results are slightly different from the categorize text aggregation and the ML job,
+     * however you can use these tokens for looking up messages in indices generated with the standard
+     * tokenizer. The latter is considered more important.
+     */
+    public static CategorizationAnalyzerConfig buildStandardEsqlCategorizationAnalyzer(List<String> categorizationFilters) {
+
+        return new CategorizationAnalyzerConfig.Builder().addCharFilter("first_line_with_letters")
+            .addCategorizationFilters(categorizationFilters)
+            .setTokenizer("standard")
+            .addDateWordsTokenFilter()
+            .addLimitFilter()
+            .build();
+    }
+
     private final String analyzer;
     private final List<NameOrDefinition> charFilters;
     private final NameOrDefinition tokenizer;

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/CategorizeBlockHash.java

@@ -48,7 +48,7 @@
  */
 public class CategorizeBlockHash extends BlockHash {
 
-    private static final CategorizationAnalyzerConfig ANALYZER_CONFIG = CategorizationAnalyzerConfig.buildStandardCategorizationAnalyzer(
+    private static final CategorizationAnalyzerConfig ANALYZER_CONFIG = CategorizationAnalyzerConfig.buildStandardEsqlCategorizationAnalyzer(
         List.of()
     );
     private static final int NULL_ORD = 0;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/categorize.csv-spec

@@ -256,9 +256,9 @@ FROM hosts
 ;
 
 COUNT():long | category:keyword
+           2 | .*?DB.+?servers.*?
            2 | .*?Gateway.+?instances.*?
            5 | .*?Kubernetes.+?cluster.*?
-           2 | .*?servers.*?
            1 | null
 
 // Note: DB is removed from "DB servers", because the ml_standard
@@ -302,9 +302,9 @@ FROM sample_data
 ;
 
 COUNT():long | category:keyword
-           3 | .*?Connected.+?to.*?
-           3 | .*?Connection.+?error.*?
-           1 | .*?Disconnected.*?
+3            | .*?Connected.+?to.+?👍🏽.+?😊.*?
+3            | .*?Connection.+?error.+?👍🏽.+?😊.*?
+1            | .*?Disconnected.+?👍🏽.+?😊.*?
 ;
 
 on REVERSE(CONCAT())
@@ -316,9 +316,9 @@ FROM sample_data
 ;
 
 COUNT():long | category:keyword
-           1 | .*?detcennocsiD.*?
-           3 | .*?ot.+?detcennoC.*?
-           3 | .*?rorre.+?noitcennoC.*?
+1            | .*?😊.+?👍🏽.+?detcennocsiD.*?
+3            | .*?😊.+?👍🏽.+?ot.+?detcennoC.*?
+3            | .*?😊.+?👍🏽.+?rorre.+?noitcennoC.*?
 ;
 
 and then TO_LOWER

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java

@@ -192,7 +192,7 @@ public class CsvTests extends ESTestCase {
 
     @ParametersFactory(argumentFormatting = "%2$s.%3$s")
     public static List<Object[]> readScriptSpec() throws Exception {
-        List<URL> urls = classpathResources("/*.csv-spec");
+        List<URL> urls = classpathResources("/categorize.csv-spec");
         assertThat("Not enough specs found " + urls, urls, hasSize(greaterThan(0)));
         return SpecReader.readScriptSpec(urls, specParser());
     }