Merge branch 'opensearch-project:main' into feat/hunspell-ref-path-core

Author: shayush622

.ci/bwcVersions

@@ -48,6 +48,7 @@ BWC_VERSION:
   - "2.19.3"
   - "2.19.4"
   - "2.19.5"
+  - "2.19.6"
   - "3.0.0"
   - "3.1.0"
   - "3.2.0"

CHANGELOG.md

@@ -29,8 +29,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use env variable (OPENSEARCH_FIPS_MODE) to enable opensearch to run in FIPS enforced mode instead of checking for existence of bcFIPS jars ([#20625](https://github.com/opensearch-project/OpenSearch/pull/20625))
 - Update streaming flag to use search request context ([#20530](https://github.com/opensearch-project/OpenSearch/pull/20530))
 - Move pull-based ingestion classes from experimental to publicAPI ([#20704](https://github.com/opensearch-project/OpenSearch/pull/20704))
+- Disable concurrent search for filter duplicates in significant_text ([#20857](https://github.com/opensearch-project/OpenSearch/pull/20857))
 
 ### Fixed
+- Relax index template pattern overlap check to use minimum-string heuristic, allowing distinguishable multi-wildcard patterns at the same priority ([#20702](https://github.com/opensearch-project/OpenSearch/pull/20702))
 - Fix `AutoForceMergeMetrics` silently dropping tags due to unreassigned `addTag()` return value ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))
 - Fix flaky test failures in ShardsLimitAllocationDeciderIT ([#20375](https://github.com/opensearch-project/OpenSearch/pull/20375))
 - Prevent criteria update for context aware indices ([#20250](https://github.com/opensearch-project/OpenSearch/pull/20250))
@@ -51,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix ShardSearchFailure in transport-grpc ([#20641](https://github.com/opensearch-project/OpenSearch/pull/20641))
 - Fix TLS cert hot-reload for Arrow Flight transport ([#20732](https://github.com/opensearch-project/OpenSearch/pull/20732))
 - Fix misleading heap usage cancellation message in SearchBackpressureService ([#20779](https://github.com/opensearch-project/OpenSearch/pull/20779))
+- Fix task details JSON logs with nested JSON in metadata are not properly escaped ([#20802](https://github.com/opensearch-project/OpenSearch/pull/20802))
 - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775))
 - Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823))
 - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842))

libs/core/src/main/java/org/opensearch/Version.java

@@ -127,6 +127,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_19_3 = new Version(2190399, org.apache.lucene.util.Version.LUCENE_9_12_2);
     public static final Version V_2_19_4 = new Version(2190499, org.apache.lucene.util.Version.LUCENE_9_12_3);
     public static final Version V_2_19_5 = new Version(2190599, org.apache.lucene.util.Version.LUCENE_9_12_3);
+    public static final Version V_2_19_6 = new Version(2190699, org.apache.lucene.util.Version.LUCENE_9_12_3);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0);
     public static final Version V_3_1_0 = new Version(3010099, org.apache.lucene.util.Version.LUCENE_10_2_1);
     public static final Version V_3_2_0 = new Version(3020099, org.apache.lucene.util.Version.LUCENE_10_2_2);

qa/logging-config/src/test/java/org/opensearch/common/logging/OpenSearchJsonLayoutTests.java

@@ -122,8 +122,8 @@ public void testLayoutWithAdditionalFields() {
                 "\"cluster.name\": \"${sys:opensearch.logs.cluster_name}\", " +
                 "\"node.name\": \"%node_name\", " +
                 "\"message\": \"%notEmpty{%enc{%marker}{JSON} }%enc{%.-10000m}{JSON}\"" +
-                "%notEmpty{, \"x-opaque-id\": \"%OpenSearchMessageField{x-opaque-id}\"}" +
-                "%notEmpty{, \"someOtherField\": \"%OpenSearchMessageField{someOtherField}\"}" +
+                "%notEmpty{, \"x-opaque-id\": \"%enc{%OpenSearchMessageField{x-opaque-id}}{JSON}\"}" +
+                "%notEmpty{, \"someOtherField\": \"%enc{%OpenSearchMessageField{someOtherField}}{JSON}\"}" +
                 "%notEmpty{, %node_and_cluster_id }" +
                 "%exceptionAsJson }" + System.lineSeparator()));
     }
@@ -143,7 +143,7 @@ public void testLayoutWithAdditionalFieldOverride() {
                 "\"component\": \"%c{1.}\", " +
                 "\"cluster.name\": \"${sys:opensearch.logs.cluster_name}\", " +
                 "\"node.name\": \"%node_name\"" +
-                "%notEmpty{, \"message\": \"%OpenSearchMessageField{message}\"}" +
+                "%notEmpty{, \"message\": \"%enc{%OpenSearchMessageField{message}}{JSON}\"}" +
                 "%notEmpty{, %node_and_cluster_id }" +
                 "%exceptionAsJson }" + System.lineSeparator()));
     }

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml

@@ -102,6 +102,51 @@
               number_of_shards:   1
               number_of_replicas: 0
 
+---
+"Put distinguishable multi-wildcard templates at the same priority":
+  - skip:
+      features: allowed_warnings
+      version: " - 3.5.99"
+      reason: "template overlap check was relaxed in 3.6"
+
+  - do:
+      allowed_warnings:
+        - "index template [app-test-some] has index patterns [app-test-*-some-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some] will take precedence during new index creation"
+      indices.put_index_template:
+        name: app-test-some
+        body:
+          index_patterns: ["app-test-*-some-*"]
+          priority: 0
+
+  # The second template has a literal segment after the first wildcard that differs from the first
+  # template (-some_other- vs -some-), so the two patterns do not practically overlap and both
+  # templates should be accepted at the same priority.
+  - do:
+      allowed_warnings:
+        - "index template [app-test-some-other] has index patterns [app-test-*-some_other-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some-other] will take precedence during new index creation"
+      indices.put_index_template:
+        name: app-test-some-other
+        body:
+          index_patterns: ["app-test-*-some_other-*"]
+          priority: 0
+
+  - do:
+      indices.get_index_template:
+        name: app-test-some
+  - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some-*"]}
+
+  - do:
+      indices.get_index_template:
+        name: app-test-some-other
+  - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some_other-*"]}
+
+  - do:
+      indices.delete_index_template:
+        name: app-test-some
+  - do:
+      indices.delete_index_template:
+        name: app-test-some-other
+
 ---
 "Put index template without index_patterns":
 

server/src/internalClusterTest/java/org/opensearch/indexing/IndexActionIT.java

@@ -44,6 +44,7 @@
 import org.opensearch.index.VersionType;
 import org.opensearch.index.mapper.MapperParsingException;
 import org.opensearch.indices.InvalidIndexNameException;
+import org.opensearch.search.SearchHit;
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.opensearch.test.hamcrest.OpenSearchAssertions;
 
@@ -97,12 +98,22 @@ public void testAutoGenerateIdNoDuplicates() throws Exception {
                     logger.debug("running search");
                     SearchResponse response = client().prepareSearch(indexName).get();
                     if (response.getHits().getTotalHits().value() != numOfDocs) {
+                        // Fetch all docs to identify the unexpected documents
+                        SearchResponse allDocs = client().prepareSearch(indexName)
+                            .setSize((int) response.getHits().getTotalHits().value())
+                            .get();
+                        StringBuilder docDetails = new StringBuilder();
+                        for (SearchHit hit : allDocs.getHits().getHits()) {
+                            docDetails.append("\n  id=").append(hit.getId()).append(" source=").append(hit.getSourceAsString());
+                        }
                         final String message = "Count is "
                             + response.getHits().getTotalHits().value()
                             + " but "
                             + numOfDocs
                             + " was expected. "
-                            + OpenSearchAssertions.formatShardStatus(response);
+                            + OpenSearchAssertions.formatShardStatus(response)
+                            + "\nAll documents:"
+                            + docDetails;
                         logger.error("{}. search response: \n{}", message, response);
                         fail(message);
                     }