ES|QL Category Text v1

x-pack/plugin/esql/build.gradle

@@ -11,7 +11,7 @@ esplugin {
   name 'x-pack-esql'
   description 'The plugin that powers ESQL for Elasticsearch'
   classname 'org.elasticsearch.xpack.esql.plugin.EsqlPlugin'
-  extendedPlugins = ['x-pack-esql-core', 'lang-painless']
+  extendedPlugins = ['x-pack-esql-core', 'lang-painless', 'x-pack-ml']
 }
 
 base {
@@ -22,6 +22,7 @@ dependencies {
   compileOnly project(path: xpackModule('core'))
   compileOnly project(':modules:lang-painless:spi')
   compileOnly project(xpackModule('esql-core'))
+  compileOnly project(xpackModule('ml'))
   implementation project('compute')
   implementation project('compute:ann')
   implementation project(':libs:elasticsearch-dissect')

x-pack/plugin/esql/compute/build.gradle

@@ -11,11 +11,14 @@ base {
 dependencies {
   compileOnly project(':server')
   compileOnly project('ann')
+  compileOnly project(xpackModule('ml'))
   annotationProcessor project('gen')
   implementation 'com.carrotsearch:hppc:0.8.1'
 
   testImplementation project(':test:framework')
   testImplementation(project(xpackModule('esql-core')))
+  testImplementation(project(xpackModule('core')))
+  testImplementation(project(xpackModule('ml')))
 }
 
 def projectDirectory = project.layout.projectDirectory

x-pack/plugin/esql/compute/src/main/java/module-info.java

@@ -7,6 +7,7 @@
 
 module org.elasticsearch.compute {
 
+    requires org.apache.lucene.analysis.common;
     requires org.apache.lucene.core;
     requires org.elasticsearch.base;
     requires org.elasticsearch.server;
@@ -15,6 +16,7 @@
     // required due to dependency on org.elasticsearch.common.util.concurrent.AbstractAsyncTask
     requires org.apache.logging.log4j;
     requires org.elasticsearch.logging;
+    requires org.elasticsearch.ml;
     requires org.elasticsearch.tdigest;
     requires org.elasticsearch.geo;
     requires hppc;

x-pack/plugin/ml/src/main/java/module-info.java

@@ -37,6 +37,8 @@
 
     exports org.elasticsearch.xpack.ml;
     exports org.elasticsearch.xpack.ml.action;
+    exports org.elasticsearch.xpack.ml.aggs.categorization;
     exports org.elasticsearch.xpack.ml.autoscaling;
+    exports org.elasticsearch.xpack.ml.job.categorization;
     exports org.elasticsearch.xpack.ml.notifications;
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizationBytesRefHash.java

@@ -12,7 +12,7 @@
 import org.elasticsearch.common.util.BytesRefHash;
 import org.elasticsearch.core.Releasable;
 
-class CategorizationBytesRefHash implements Releasable {
+public class CategorizationBytesRefHash implements Releasable {
 
     private final BytesRefHash bytesRefHash;
 

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java

@@ -94,11 +94,11 @@ protected CategorizeTextAggregator(
                 true
             );
         }
-        this.categorizers = bigArrays().newObjectArray(1);
+        this.categorizers = context.bigArrays().newObjectArray(1);
         this.similarityThreshold = similarityThreshold;
-        this.bucketOrds = LongKeyedBucketOrds.build(bigArrays(), CardinalityUpperBound.MANY);
+        this.bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), CardinalityUpperBound.MANY);
         this.bucketCountThresholds = bucketCountThresholds;
-        this.bytesRefHash = new CategorizationBytesRefHash(new BytesRefHash(2048, bigArrays()));
+        this.bytesRefHash = new CategorizationBytesRefHash(new BytesRefHash(2048, context.bigArrays()));
         // TODO: make it possible to choose a language instead of or as well as English for the part-of-speech dictionary
         this.partOfSpeechDictionary = CategorizationPartOfSpeechDictionary.getInstance();
     }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/TokenListCategorizer.java

@@ -14,6 +14,8 @@
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.search.aggregations.AggregationReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.xpack.ml.aggs.categorization.TokenListCategory.TokenAndWeight;
@@ -40,6 +42,25 @@
  */
 public class TokenListCategorizer implements Accountable {
 
+    /**
+     * TokenListCategorizer that takes ownership of the CategorizationBytesRefHash and releases it when closed.
+     */
+    public static class CloseableTokenListCategorizer extends TokenListCategorizer implements Releasable {
+
+        public CloseableTokenListCategorizer(
+            CategorizationBytesRefHash bytesRefHash,
+            CategorizationPartOfSpeechDictionary partOfSpeechDictionary,
+            float threshold
+        ) {
+            super(bytesRefHash, partOfSpeechDictionary, threshold);
+        }
+
+        @Override
+        public void close() {
+            Releasables.close(super.bytesRefHash);
+        }
+    }
+
     public static final int MAX_TOKENS = 100;
     private static final long SHALLOW_SIZE = shallowSizeOfInstance(TokenListCategorizer.class);
     private static final long SHALLOW_SIZE_OF_ARRAY_LIST = shallowSizeOfInstance(ArrayList.class);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/AbstractMlTokenizer.java

@@ -16,8 +16,11 @@
 
 public abstract class AbstractMlTokenizer extends Tokenizer {
 
+    @SuppressWarnings("this-escape")
     protected final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    @SuppressWarnings("this-escape")
     protected final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    @SuppressWarnings("this-escape")
     protected final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
 
     /**

x-pack/plugin/security/qa/multi-cluster/build.gradle

@@ -23,6 +23,8 @@ dependencies {
   // esql with enrich
   clusterModules project(':x-pack:plugin:esql')
   clusterModules project(':x-pack:plugin:enrich')
+  clusterModules project(':x-pack:plugin:autoscaling')
+  clusterModules project(':x-pack:plugin:ml')
   clusterModules(project(":modules:ingest-common"))
 }
 

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java

@@ -56,11 +56,14 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe
         fulfillingCluster = ElasticsearchCluster.local()
             .name("fulfilling-cluster")
             .nodes(3)
+            .module("x-pack-autoscaling")
             .module("x-pack-esql")
             .module("x-pack-enrich")
+            .module("x-pack-ml")
             .module("ingest-common")
             .apply(commonClusterConfig)
             .setting("remote_cluster.port", "0")
+            .setting("xpack.ml.enabled", "false")
             .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
             .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
             .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
@@ -73,10 +76,13 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe
 
         queryCluster = ElasticsearchCluster.local()
             .name("query-cluster")
+            .module("x-pack-autoscaling")
             .module("x-pack-esql")
             .module("x-pack-enrich")
+            .module("x-pack-ml")
             .module("ingest-common")
             .apply(commonClusterConfig)
+            .setting("xpack.ml.enabled", "false")
             .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
             .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
             .setting("xpack.security.authc.token.enabled", "true")

x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle

@@ -23,11 +23,14 @@ def fulfillingCluster = testClusters.register('fulfilling-cluster') {
   module ':modules:data-streams'
   module ':x-pack:plugin:mapper-constant-keyword'
   module ':x-pack:plugin:async-search'
+  module ':x-pack:plugin:autoscaling'
   module ':x-pack:plugin:esql-core'
   module ':x-pack:plugin:esql'
+  module ':x-pack:plugin:ml'
   module ':modules:ingest-common'
   module ':x-pack:plugin:enrich'
   user username: "test_user", password: "x-pack-test-password"
+  setting 'xpack.ml.enabled', 'false'
 }
 
 def queryingCluster = testClusters.register('querying-cluster') {
@@ -38,13 +41,15 @@ def queryingCluster = testClusters.register('querying-cluster') {
   module ':modules:data-streams'
   module ':x-pack:plugin:mapper-constant-keyword'
   module ':x-pack:plugin:async-search'
+  module ':x-pack:plugin:autoscaling'
   module ':x-pack:plugin:esql-core'
   module ':x-pack:plugin:esql'
+  module ':x-pack:plugin:ml'
   module ':modules:ingest-common'
   module ':x-pack:plugin:enrich'
   setting 'cluster.remote.connections_per_cluster', "1"
   user username: "test_user", password: "x-pack-test-password"
-
+  setting 'xpack.ml.enabled', 'false'
   setting 'cluster.remote.my_remote_cluster.skip_unavailable', 'false'
   if (proxyMode) {
     setting 'cluster.remote.my_remote_cluster.mode', 'proxy'