diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 26cf53b334b1e..0225664918b7b 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -11,7 +11,7 @@ esplugin { name 'x-pack-esql' description 'The plugin that powers ESQL for Elasticsearch' classname 'org.elasticsearch.xpack.esql.plugin.EsqlPlugin' - extendedPlugins = ['x-pack-esql-core', 'lang-painless'] + extendedPlugins = ['x-pack-esql-core', 'lang-painless', 'x-pack-ml'] } base { @@ -22,6 +22,7 @@ dependencies { compileOnly project(path: xpackModule('core')) compileOnly project(':modules:lang-painless:spi') compileOnly project(xpackModule('esql-core')) + compileOnly project(xpackModule('ml')) implementation project('compute') implementation project('compute:ann') implementation project(':libs:elasticsearch-dissect') diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle index 971bfd39c231f..81d1a6f5360ca 100644 --- a/x-pack/plugin/esql/compute/build.gradle +++ b/x-pack/plugin/esql/compute/build.gradle @@ -11,11 +11,14 @@ base { dependencies { compileOnly project(':server') compileOnly project('ann') + compileOnly project(xpackModule('ml')) annotationProcessor project('gen') implementation 'com.carrotsearch:hppc:0.8.1' testImplementation project(':test:framework') testImplementation(project(xpackModule('esql-core'))) + testImplementation(project(xpackModule('core'))) + testImplementation(project(xpackModule('ml'))) } def projectDirectory = project.layout.projectDirectory diff --git a/x-pack/plugin/esql/compute/src/main/java/module-info.java b/x-pack/plugin/esql/compute/src/main/java/module-info.java index dc8cda0fbe3c8..1739c90467c2c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/module-info.java +++ b/x-pack/plugin/esql/compute/src/main/java/module-info.java @@ -7,6 +7,7 @@ module org.elasticsearch.compute { + requires org.apache.lucene.analysis.common; requires org.apache.lucene.core; requires org.elasticsearch.base; requires org.elasticsearch.server; @@ -15,6 +16,7 @@ // required due to dependency on org.elasticsearch.common.util.concurrent.AbstractAsyncTask requires org.apache.logging.log4j; requires org.elasticsearch.logging; + requires org.elasticsearch.ml; requires org.elasticsearch.tdigest; requires org.elasticsearch.geo; requires hppc; diff --git a/x-pack/plugin/ml/src/main/java/module-info.java b/x-pack/plugin/ml/src/main/java/module-info.java index 0f3fdd836feca..4984fa8912e28 100644 --- a/x-pack/plugin/ml/src/main/java/module-info.java +++ b/x-pack/plugin/ml/src/main/java/module-info.java @@ -37,6 +37,8 @@ exports org.elasticsearch.xpack.ml; exports org.elasticsearch.xpack.ml.action; + exports org.elasticsearch.xpack.ml.aggs.categorization; exports org.elasticsearch.xpack.ml.autoscaling; + exports org.elasticsearch.xpack.ml.job.categorization; exports org.elasticsearch.xpack.ml.notifications; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizationBytesRefHash.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizationBytesRefHash.java index 58feb24480f87..955790ab6a649 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizationBytesRefHash.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizationBytesRefHash.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.util.BytesRefHash; import org.elasticsearch.core.Releasable; -class CategorizationBytesRefHash implements Releasable { +public class CategorizationBytesRefHash implements Releasable { private final BytesRefHash bytesRefHash; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java index cedaced0f57ee..e55736cf43607 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java @@ -94,11 +94,11 @@ protected CategorizeTextAggregator( true ); } - this.categorizers = bigArrays().newObjectArray(1); + this.categorizers = context.bigArrays().newObjectArray(1); this.similarityThreshold = similarityThreshold; - this.bucketOrds = LongKeyedBucketOrds.build(bigArrays(), CardinalityUpperBound.MANY); + this.bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), CardinalityUpperBound.MANY); this.bucketCountThresholds = bucketCountThresholds; - this.bytesRefHash = new CategorizationBytesRefHash(new BytesRefHash(2048, bigArrays())); + this.bytesRefHash = new CategorizationBytesRefHash(new BytesRefHash(2048, context.bigArrays())); // TODO: make it possible to choose a language instead of or as well as English for the part-of-speech dictionary this.partOfSpeechDictionary = CategorizationPartOfSpeechDictionary.getInstance(); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/TokenListCategorizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/TokenListCategorizer.java index e1f2404ee56b5..d0088edcb0805 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/TokenListCategorizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/TokenListCategorizer.java @@ -14,6 +14,8 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.xpack.ml.aggs.categorization.TokenListCategory.TokenAndWeight; @@ -40,6 +42,25 @@ */ public class TokenListCategorizer implements Accountable { + /** + * TokenListCategorizer that takes ownership of the CategorizationBytesRefHash and releases it when closed. + */ + public static class CloseableTokenListCategorizer extends TokenListCategorizer implements Releasable { + + public CloseableTokenListCategorizer( + CategorizationBytesRefHash bytesRefHash, + CategorizationPartOfSpeechDictionary partOfSpeechDictionary, + float threshold + ) { + super(bytesRefHash, partOfSpeechDictionary, threshold); + } + + @Override + public void close() { + Releasables.close(super.bytesRefHash); + } + } + public static final int MAX_TOKENS = 100; private static final long SHALLOW_SIZE = shallowSizeOfInstance(TokenListCategorizer.class); private static final long SHALLOW_SIZE_OF_ARRAY_LIST = shallowSizeOfInstance(ArrayList.class); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/AbstractMlTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/AbstractMlTokenizer.java index c701216b1984b..d7e7683ce0071 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/AbstractMlTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/AbstractMlTokenizer.java @@ -16,8 +16,11 @@ public abstract class AbstractMlTokenizer extends Tokenizer { + @SuppressWarnings("this-escape") protected final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + @SuppressWarnings("this-escape") protected final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + @SuppressWarnings("this-escape") protected final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); /** diff --git a/x-pack/plugin/security/qa/multi-cluster/build.gradle b/x-pack/plugin/security/qa/multi-cluster/build.gradle index 625b6806ab520..c7b8f81bb7876 100644 --- a/x-pack/plugin/security/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/security/qa/multi-cluster/build.gradle @@ -23,6 +23,8 @@ dependencies { // esql with enrich clusterModules project(':x-pack:plugin:esql') clusterModules project(':x-pack:plugin:enrich') + clusterModules project(':x-pack:plugin:autoscaling') + clusterModules project(':x-pack:plugin:ml') clusterModules(project(":modules:ingest-common")) } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java index f5f9410a145cc..1a236ccb6aa06 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java @@ -56,11 +56,14 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe fulfillingCluster = ElasticsearchCluster.local() .name("fulfilling-cluster") .nodes(3) + .module("x-pack-autoscaling") .module("x-pack-esql") .module("x-pack-enrich") + .module("x-pack-ml") .module("ingest-common") .apply(commonClusterConfig) .setting("remote_cluster.port", "0") + .setting("xpack.ml.enabled", "false") .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") @@ -73,10 +76,13 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe queryCluster = ElasticsearchCluster.local() .name("query-cluster") + .module("x-pack-autoscaling") .module("x-pack-esql") .module("x-pack-enrich") + .module("x-pack-ml") .module("ingest-common") .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") .setting("xpack.security.authc.token.enabled", "true") diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle index b5b8495870259..6d41c4eddf31c 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle @@ -23,11 +23,14 @@ def fulfillingCluster = testClusters.register('fulfilling-cluster') { module ':modules:data-streams' module ':x-pack:plugin:mapper-constant-keyword' module ':x-pack:plugin:async-search' + module ':x-pack:plugin:autoscaling' module ':x-pack:plugin:esql-core' module ':x-pack:plugin:esql' + module ':x-pack:plugin:ml' module ':modules:ingest-common' module ':x-pack:plugin:enrich' user username: "test_user", password: "x-pack-test-password" + setting 'xpack.ml.enabled', 'false' } def queryingCluster = testClusters.register('querying-cluster') { @@ -38,13 +41,15 @@ def queryingCluster = testClusters.register('querying-cluster') { module ':modules:data-streams' module ':x-pack:plugin:mapper-constant-keyword' module ':x-pack:plugin:async-search' + module ':x-pack:plugin:autoscaling' module ':x-pack:plugin:esql-core' module ':x-pack:plugin:esql' + module ':x-pack:plugin:ml' module ':modules:ingest-common' module ':x-pack:plugin:enrich' setting 'cluster.remote.connections_per_cluster', "1" user username: "test_user", password: "x-pack-test-password" - + setting 'xpack.ml.enabled', 'false' setting 'cluster.remote.my_remote_cluster.skip_unavailable', 'false' if (proxyMode) { setting 'cluster.remote.my_remote_cluster.mode', 'proxy'