Merge 322ed68 into openjdk23-bundle

Author: elasticsearchmachine

docs/changelog/112768.yaml

@@ -0,0 +1,5 @@
+pr: 112768
+summary: Deduplicate Kuromoji User Dictionary
+area: Search
+type: enhancement
+issues: []

docs/plugins/analysis-kuromoji.asciidoc

@@ -133,6 +133,11 @@ unknown words. It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -221,7 +226,8 @@ PUT kuromoji_sample
             "type": "kuromoji_tokenizer",
             "mode": "extended",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ja.txt"
+            "user_dictionary": "userdict_ja.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {

docs/plugins/analysis-nori.asciidoc

@@ -58,6 +58,11 @@ It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -104,7 +109,8 @@ PUT nori_sample
             "type": "nori_tokenizer",
             "decompound_mode": "mixed",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ko.txt"
+            "user_dictionary": "userdict_ko.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {
@@ -299,7 +305,6 @@ Which responds with:
 }
 --------------------------------------------------
 
-
 [[analysis-nori-speech]]
 ==== `nori_part_of_speech` token filter
 

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java

@@ -46,7 +46,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
         assert msg instanceof HttpObject;
         if (msg instanceof HttpRequest request) {
             var preReq = HttpHeadersAuthenticatorUtils.asHttpPreRequest(request);
-            aggregating = decider.test(preReq) && IGNORE_TEST.test(preReq);
+            aggregating = (decider.test(preReq) && IGNORE_TEST.test(preReq)) || request.decoderResult().isFailure();
         }
         if (aggregating || msg instanceof FullHttpRequest) {
             super.channelRead(ctx, msg);

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java

@@ -17,6 +17,7 @@
 import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.QueryStringDecoder;
 import io.netty.handler.codec.http.cookie.Cookie;
 import io.netty.handler.codec.http.cookie.ServerCookieDecoder;
 import io.netty.handler.codec.http.cookie.ServerCookieEncoder;
@@ -48,6 +49,7 @@ public class Netty4HttpRequest implements HttpRequest {
     private final Exception inboundException;
     private final boolean pooled;
     private final int sequence;
+    private final QueryStringDecoder queryStringDecoder;
 
     Netty4HttpRequest(int sequence, io.netty.handler.codec.http.HttpRequest request, Netty4HttpRequestBodyStream contentStream) {
         this(
@@ -94,6 +96,7 @@ private Netty4HttpRequest(
         this.pooled = pooled;
         this.released = released;
         this.inboundException = inboundException;
+        this.queryStringDecoder = new QueryStringDecoder(request.uri());
     }
 
     @Override
@@ -106,6 +109,11 @@ public String uri() {
         return request.uri();
     }
 
+    @Override
+    public String rawPath() {
+        return queryStringDecoder.rawPath();
+    }
+
     @Override
     public HttpBody body() {
         assert released.get() == false;

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java

@@ -375,9 +375,8 @@ protected HttpMessage createMessage(String[] initialLine) throws Exception {
             final HttpObjectAggregator aggregator = new Netty4HttpAggregator(
                 handlingSettings.maxContentLength(),
                 httpPreRequest -> enabled.get() == false
-                    || (httpPreRequest.uri().contains("_bulk") == false
-                        || httpPreRequest.uri().contains("_bulk_update")
-                        || httpPreRequest.uri().contains("/_xpack/monitoring/_bulk"))
+                    || ((httpPreRequest.rawPath().endsWith("/_bulk") == false)
+                        || httpPreRequest.rawPath().startsWith("/_xpack/monitoring/_bulk"))
             );
             aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
             ch.pipeline()

muted-tests.yml

@@ -215,9 +215,6 @@ tests:
 - class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT
   method: test {case-functions.testSelectInsertWithLcaseAndLengthWithOrderBy}
   issue: https://github.com/elastic/elasticsearch/issues/112642
-- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT
-  method: testHistogramAggregation
-  issue: https://github.com/elastic/elasticsearch/issues/113109
 - class: org.elasticsearch.action.admin.cluster.node.stats.NodeStatsTests
   method: testChunking
   issue: https://github.com/elastic/elasticsearch/issues/113139
@@ -262,15 +259,9 @@ tests:
 - class: org.elasticsearch.index.mapper.DoubleRangeFieldMapperTests
   method: testSyntheticSourceKeepAll
   issue: https://github.com/elastic/elasticsearch/issues/113234
-- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT
-  method: testTermsQuery
-  issue: https://github.com/elastic/elasticsearch/issues/113246
 - class: org.elasticsearch.integration.KibanaUserRoleIntegTests
   method: testGetMappings
   issue: https://github.com/elastic/elasticsearch/issues/113260
-- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT
-  method: testMatchAllQuery
-  issue: https://github.com/elastic/elasticsearch/issues/113265
 - class: org.elasticsearch.xpack.security.authz.SecurityScrollTests
   method: testSearchAndClearScroll
   issue: https://github.com/elastic/elasticsearch/issues/113285
@@ -286,6 +277,41 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {stats.DocsStatsGroupByMultipleValues}
   issue: https://github.com/elastic/elasticsearch/issues/113296
+- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
+  issue: https://github.com/elastic/elasticsearch/issues/113298
+- class: org.elasticsearch.integration.KibanaUserRoleIntegTests
+  method: testGetIndex
+  issue: https://github.com/elastic/elasticsearch/issues/113311
+- class: org.elasticsearch.packaging.test.WindowsServiceTests
+  method: test81JavaOptsInJvmOptions
+  issue: https://github.com/elastic/elasticsearch/issues/113313
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=esql/50_index_patterns/disjoint_mappings}
+  issue: https://github.com/elastic/elasticsearch/issues/113315
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=wildcard/10_wildcard_basic/Query_string wildcard query}
+  issue: https://github.com/elastic/elasticsearch/issues/113316
+- class: org.elasticsearch.index.mapper.LongRangeFieldMapperTests
+  method: testSyntheticSourceKeepAll
+  issue: https://github.com/elastic/elasticsearch/issues/113324
+- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
+  method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item}
+  issue: https://github.com/elastic/elasticsearch/issues/113325
+- class: org.elasticsearch.index.mapper.IntegerRangeFieldMapperTests
+  method: testSyntheticSourceKeepArrays
+  issue: https://github.com/elastic/elasticsearch/issues/113326
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/transforms_force_delete/Test force deleting a running transform}
+  issue: https://github.com/elastic/elasticsearch/issues/113327
+- class: org.elasticsearch.integration.KibanaUserRoleIntegTests
+  method: testValidateQuery
+  issue: https://github.com/elastic/elasticsearch/issues/113328
+- class: org.elasticsearch.index.mapper.LongRangeFieldMapperTests
+  method: testSyntheticSourceKeepArrays
+  issue: https://github.com/elastic/elasticsearch/issues/113335
+- class: org.elasticsearch.xpack.security.support.SecurityIndexManagerIntegTests
+  method: testOnIndexAvailableForSearchIndexAlreadyAvailable
+  issue: https://github.com/elastic/elasticsearch/issues/113336
 
 # Examples:
 #

plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java

@@ -33,6 +33,7 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
     private static final String NBEST_COST = "nbest_cost";
     private static final String NBEST_EXAMPLES = "nbest_examples";
     private static final String DISCARD_COMPOUND_TOKEN = "discard_compound_token";
+    private static final String LENIENT = "lenient";
 
     private final UserDictionary userDictionary;
     private final Mode mode;
@@ -58,14 +59,23 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting
                 "It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" + " with [" + USER_DICT_RULES_OPTION + "]"
             );
         }
-        List<String> ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, false, true);
+        List<String> ruleList = Analysis.getWordList(
+            env,
+            settings,
+            USER_DICT_PATH_OPTION,
+            USER_DICT_RULES_OPTION,
+            LENIENT,
+            false,  // typically don't want to remove comments as deduplication will provide better feedback
+            true
+        );
         if (ruleList == null || ruleList.isEmpty()) {
             return null;
         }
         StringBuilder sb = new StringBuilder();
         for (String line : ruleList) {
             sb.append(line).append(System.lineSeparator());
         }
+
         try (Reader rulesReader = new StringReader(sb.toString())) {
             return UserDictionary.open(rulesReader);
         } catch (IOException e) {

plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java

@@ -445,7 +445,26 @@ public void testKuromojiAnalyzerDuplicateUserDictRule() throws Exception {
             )
             .build();
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings));
-        assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [3]"));
+        assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [4]"));
+    }
+
+    public void testKuromojiAnalyzerDuplicateUserDictRuleDeduplication() throws Exception {
+        Settings settings = Settings.builder()
+            .put("index.analysis.analyzer.my_analyzer.type", "kuromoji")
+            .put("index.analysis.analyzer.my_analyzer.lenient", "true")
+            .putList(
+                "index.analysis.analyzer.my_analyzer.user_dictionary_rules",
+                "c++,c++,w,w",
+                "#comment",
+                "制限スピード,制限スピード,セイゲンスピード,テスト名詞",
+                "制限スピード,制限スピード,セイゲンスピード,テスト名詞"
+            )
+            .build();
+        TestAnalysis analysis = createTestAnalysis(settings);
+        Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer");
+        try (TokenStream stream = analyzer.tokenStream("", "制限スピード")) {
+            assertTokenStreamContents(stream, new String[] { "制限スピード" });
+        }
     }
 
     public void testDiscardCompoundToken() throws Exception {

plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java

@@ -31,6 +31,7 @@
 public class NoriTokenizerFactory extends AbstractTokenizerFactory {
     private static final String USER_DICT_PATH_OPTION = "user_dictionary";
     private static final String USER_DICT_RULES_OPTION = "user_dictionary_rules";
+    private static final String LENIENT = "lenient";
 
     private final UserDictionary userDictionary;
     private final KoreanTokenizer.DecompoundMode decompoundMode;
@@ -54,7 +55,8 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting
             settings,
             USER_DICT_PATH_OPTION,
             USER_DICT_RULES_OPTION,
-            true,
+            LENIENT,
+            false,  // typically don't want to remove comments as deduplication will provide better feedback
             isSupportDuplicateCheck(indexSettings)
         );
         if (ruleList == null || ruleList.isEmpty()) {