Merge origin/main into vshards_p2

Signed-off-by: Atri Sharma <atri.jiit@gmail.com>

Author: atris

.ci/bwcVersions

@@ -48,6 +48,7 @@ BWC_VERSION:
   - "2.19.3"
   - "2.19.4"
   - "2.19.5"
+  - "2.19.6"
   - "3.0.0"
   - "3.1.0"
   - "3.2.0"

.github/workflows/links.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v6
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v2.7.0
+        uses: lycheeverse/lychee-action@v2.8.0
         with:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true

CHANGELOG.md

@@ -21,15 +21,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - WLM group custom search settings - groundwork and timeout ([#20536](https://github.com/opensearch-project/OpenSearch/issues/20536))
 - Expose JVM runtime metrics via telemetry framework ([#20844](https://github.com/opensearch-project/OpenSearch/pull/20844))
 - Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503))
+- Add support for enabling pluggable data formats, starting with phase-1 of decoupling shard from engine, and introducing basic abstractions ([#20675](https://github.com/opensearch-project/OpenSearch/pull/20675))
 
 ### Changed
 - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))
 - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570))
 - Use env variable (OPENSEARCH_FIPS_MODE) to enable opensearch to run in FIPS enforced mode instead of checking for existence of bcFIPS jars ([#20625](https://github.com/opensearch-project/OpenSearch/pull/20625))
 - Update streaming flag to use search request context ([#20530](https://github.com/opensearch-project/OpenSearch/pull/20530))
 - Move pull-based ingestion classes from experimental to publicAPI ([#20704](https://github.com/opensearch-project/OpenSearch/pull/20704))
+- Disable concurrent search for filter duplicates in significant_text ([#20857](https://github.com/opensearch-project/OpenSearch/pull/20857))
 
 ### Fixed
+- Relax index template pattern overlap check to use minimum-string heuristic, allowing distinguishable multi-wildcard patterns at the same priority ([#20702](https://github.com/opensearch-project/OpenSearch/pull/20702))
 - Fix `AutoForceMergeMetrics` silently dropping tags due to unreassigned `addTag()` return value ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))
 - Fix flaky test failures in ShardsLimitAllocationDeciderIT ([#20375](https://github.com/opensearch-project/OpenSearch/pull/20375))
 - Prevent criteria update for context aware indices ([#20250](https://github.com/opensearch-project/OpenSearch/pull/20250))
@@ -50,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix ShardSearchFailure in transport-grpc ([#20641](https://github.com/opensearch-project/OpenSearch/pull/20641))
 - Fix TLS cert hot-reload for Arrow Flight transport ([#20732](https://github.com/opensearch-project/OpenSearch/pull/20732))
 - Fix misleading heap usage cancellation message in SearchBackpressureService ([#20779](https://github.com/opensearch-project/OpenSearch/pull/20779))
+- Fix task details JSON logs with nested JSON in metadata are not properly escaped ([#20802](https://github.com/opensearch-project/OpenSearch/pull/20802))
 - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775))
 - Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823))
 - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842))
@@ -74,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.jline:jline` from 3.30.6 to 4.0.0 ([#20800](https://github.com/opensearch-project/OpenSearch/pull/20800))
 - Bump `com.netflix.nebula.ospackage-base` from 12.2.0 to 12.3.0 ([#20799](https://github.com/opensearch-project/OpenSearch/pull/20799))
 - Bump `com.netflix.nebula:gradle-info-plugin` to 16.2.1 ([#20825](https://github.com/opensearch-project/OpenSearch/pull/20825))
+- Bump `lycheeverse/lychee-action` from 2.7.0 to 2.8.0 ([#20888](https://github.com/opensearch-project/OpenSearch/pull/20888))
 
 ### Removed
 

buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -106,7 +106,7 @@ private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionI
                 bwcVersion,
                 distributionProject.name,
                 distributionProject.getProjectPath(),
-                distributionProject.getDistFile(),
+                distributionProject,
                 buildBwcTaskProvider
             );
 
@@ -131,14 +131,24 @@ private void registerBwcArtifacts(Project bwcProject, DistributionProject distri
     }
 
     private void registerDistributionArchiveArtifact(Project bwcProject, DistributionProject distributionProject, String buildBwcTask) {
-        String artifactFileName = distributionProject.getDistFile().getName();
+        String artifactFileName = distributionProject.getExpectedDistFile().getName();
         String artifactName = "opensearch";
 
         String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName.substring(artifactFileName.length() - 3);
         int archIndex = artifactFileName.indexOf("x64");
 
+        Provider<File> artifactFileProvider = providerFactory.provider(() -> {
+            if (distributionProject.getExpectedDistFile().exists()) {
+                return distributionProject.getExpectedDistFile();
+            } else if (distributionProject.getFallbackDistFile().exists()) {
+                return distributionProject.getFallbackDistFile();
+            }
+            // File doesn't exist, validation will fail elsewhere but we must return a File here
+            return distributionProject.getExpectedDistFile();
+        });
+
         bwcProject.getConfigurations().create(distributionProject.name);
-        bwcProject.getArtifacts().add(distributionProject.name, distributionProject.getDistFile(), artifact -> {
+        bwcProject.getArtifacts().add(distributionProject.name, artifactFileProvider, artifact -> {
             artifact.setName(artifactName);
             artifact.builtBy(buildBwcTask);
             artifact.setType(suffix);
@@ -152,6 +162,10 @@ private void registerDistributionArchiveArtifact(Project bwcProject, Distributio
         });
     }
 
+    static Version nextPatchVersion(Version version) {
+        return new Version(version.getMajor(), version.getMinor(), version.getRevision() + 1);
+    }
+
     private static List<DistributionProject> resolveArchiveProjects(File checkoutDir, Version bwcVersion) {
         List<String> projects = new ArrayList<>();
         // All active BWC branches publish rpm and deb packages
@@ -174,6 +188,8 @@ private static List<DistributionProject> resolveArchiveProjects(File checkoutDir
             projects.addAll(asList("zip", "tar"));
         }
 
+        Version fallbackVersion = nextPatchVersion(bwcVersion);
+
         return projects.stream().map(name -> {
             String baseDir = "distribution" + (name.endsWith("zip") || name.endsWith("tar") ? "/archives" : "/packages");
             String classifier = "";
@@ -199,7 +215,7 @@ private static List<DistributionProject> resolveArchiveProjects(File checkoutDir
             } else {
                 extension = name.substring(4);
             }
-            return new DistributionProject(name, baseDir, bwcVersion, classifier, extension, checkoutDir);
+            return new DistributionProject(name, baseDir, bwcVersion, fallbackVersion, classifier, extension, checkoutDir);
         }).collect(Collectors.toList());
     }
 
@@ -215,15 +231,15 @@ static void createBuildBwcTask(
         Provider<Version> bwcVersion,
         String projectName,
         String projectPath,
-        File projectArtifact,
+        DistributionProject distributionProject,
         TaskProvider<Task> bwcTaskProvider
     ) {
         String bwcTaskName = buildBwcTaskName(projectName);
         bwcSetupExtension.bwcTask(bwcTaskName, new Action<LoggedExec>() {
             @Override
             public void execute(LoggedExec c) {
                 c.getInputs().file(new File(project.getBuildDir(), "refspec"));
-                c.getOutputs().files(projectArtifact);
+                c.getOutputs().files(distributionProject.getExpectedDistFile(), distributionProject.getFallbackDistFile());
                 c.getOutputs().cacheIf("BWC distribution caching is disabled on 'master' branch", task -> {
                     String gitBranch = System.getenv("GIT_BRANCH");
                     return BuildParams.isCi() && (gitBranch == null || gitBranch.endsWith("master") == false);
@@ -235,9 +251,24 @@ public void execute(LoggedExec c) {
                 c.doLast(new Action<Task>() {
                     @Override
                     public void execute(Task task) {
-                        if (projectArtifact.exists() == false) {
+                        if (distributionProject.getExpectedDistFile().exists() == false
+                            && distributionProject.getFallbackDistFile().exists()) {
+                            project.getLogger()
+                                .warn(
+                                    "BWC branch produced {} instead of {}. "
+                                        + "Version.java on main may need updating to include version {}.",
+                                    distributionProject.getFallbackDistFile().getName(),
+                                    distributionProject.getExpectedDistFile().getName(),
+                                    distributionProject.getFallbackVersion()
+                                );
+                        } else if (distributionProject.getExpectedDistFile().exists() == false) {
                             throw new InvalidUserDataException(
-                                "Building " + bwcVersion.get() + " didn't generate expected file " + projectArtifact
+                                "Building "
+                                    + bwcVersion.get()
+                                    + " didn't generate expected file "
+                                    + distributionProject.getExpectedDistFile()
+                                    + " or fallback "
+                                    + distributionProject.getFallbackDistFile()
                             );
                         }
                     }
@@ -251,20 +282,48 @@ public void execute(Task task) {
      * Represents an archive project (distribution/archives/*)
      * we build from a bwc Version in a cloned repository
      */
-    private static class DistributionProject {
-        private final String name;
-        private String projectPath;
-        private File distFile;
-        private File expandedDistDir;
-
-        DistributionProject(String name, String baseDir, Version version, String classifier, String extension, File checkoutDir) {
+    static class DistributionProject {
+        final String name;
+        private final String projectPath;
+        private final File expectedDistFile;
+        private final File fallbackDistFile;
+        private final Version fallbackVersion;
+        private final File expandedDistDir;
+
+        DistributionProject(
+            String name,
+            String baseDir,
+            Version expectedVersion,
+            Version fallbackVersion,
+            String classifier,
+            String extension,
+            File checkoutDir
+        ) {
             this.name = name;
             this.projectPath = baseDir + "/" + name;
+            this.expectedDistFile = buildDistFile(name, baseDir, expectedVersion, classifier, extension, checkoutDir);
+            this.fallbackDistFile = buildDistFile(name, baseDir, fallbackVersion, classifier, extension, checkoutDir);
+            this.fallbackVersion = fallbackVersion;
+            if (name.endsWith("zip") || name.endsWith("tar")) {
+                this.expandedDistDir = new File(checkoutDir, baseDir + "/" + name + "/build/install");
+            } else {
+                this.expandedDistDir = null;
+            }
+        }
+
+        private static File buildDistFile(
+            String name,
+            String baseDir,
+            Version version,
+            String classifier,
+            String extension,
+            File checkoutDir
+        ) {
             if (version.onOrAfter("1.1.0")) {
                 // Deb uses underscores (I don't know why...):
                 // https://github.com/opensearch-project/OpenSearch/blob/f6d9a86f0e2e8241fd58b7e8b6cdeaf931b5108f/distribution/packages/build.gradle#L139
                 final String separator = name.equals("deb") ? "_" : "-";
-                this.distFile = new File(
+                return new File(
                     checkoutDir,
                     baseDir
                         + "/"
@@ -278,23 +337,27 @@ private static class DistributionProject {
                         + extension
                 );
             } else {
-                this.distFile = new File(
+                return new File(
                     checkoutDir,
                     baseDir + "/" + name + "/build/distributions/opensearch-" + version + "-SNAPSHOT" + classifier + "." + extension
                 );
             }
-            // we only ported this down to the 7.x branch.
-            if (version.onOrAfter("7.10.0") && (name.endsWith("zip") || name.endsWith("tar"))) {
-                this.expandedDistDir = new File(checkoutDir, baseDir + "/" + name + "/build/install");
-            }
         }
 
         public String getProjectPath() {
             return projectPath;
         }
 
-        public File getDistFile() {
-            return distFile;
+        public File getExpectedDistFile() {
+            return expectedDistFile;
+        }
+
+        public File getFallbackDistFile() {
+            return fallbackDistFile;
+        }
+
+        public Version getFallbackVersion() {
+            return fallbackVersion;
         }
 
         public File getExpandedDistDirectory() {

libs/core/src/main/java/org/opensearch/Version.java

@@ -127,6 +127,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_19_3 = new Version(2190399, org.apache.lucene.util.Version.LUCENE_9_12_2);
     public static final Version V_2_19_4 = new Version(2190499, org.apache.lucene.util.Version.LUCENE_9_12_3);
     public static final Version V_2_19_5 = new Version(2190599, org.apache.lucene.util.Version.LUCENE_9_12_3);
+    public static final Version V_2_19_6 = new Version(2190699, org.apache.lucene.util.Version.LUCENE_9_12_3);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0);
     public static final Version V_3_1_0 = new Version(3010099, org.apache.lucene.util.Version.LUCENE_10_2_1);
     public static final Version V_3_2_0 = new Version(3020099, org.apache.lucene.util.Version.LUCENE_10_2_2);
@@ -135,6 +136,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_3_3_2 = new Version(3030299, org.apache.lucene.util.Version.LUCENE_10_3_1);
     public static final Version V_3_4_0 = new Version(3040099, org.apache.lucene.util.Version.LUCENE_10_3_2);
     public static final Version V_3_5_0 = new Version(3050099, org.apache.lucene.util.Version.LUCENE_10_3_2);
+    public static final Version V_3_5_1 = new Version(3050199, org.apache.lucene.util.Version.LUCENE_10_3_2);
     public static final Version V_3_6_0 = new Version(3060099, org.apache.lucene.util.Version.LUCENE_10_4_0);
     public static final Version CURRENT = V_3_6_0;
 

qa/logging-config/src/test/java/org/opensearch/common/logging/OpenSearchJsonLayoutTests.java

@@ -122,8 +122,8 @@ public void testLayoutWithAdditionalFields() {
                 "\"cluster.name\": \"${sys:opensearch.logs.cluster_name}\", " +
                 "\"node.name\": \"%node_name\", " +
                 "\"message\": \"%notEmpty{%enc{%marker}{JSON} }%enc{%.-10000m}{JSON}\"" +
-                "%notEmpty{, \"x-opaque-id\": \"%OpenSearchMessageField{x-opaque-id}\"}" +
-                "%notEmpty{, \"someOtherField\": \"%OpenSearchMessageField{someOtherField}\"}" +
+                "%notEmpty{, \"x-opaque-id\": \"%enc{%OpenSearchMessageField{x-opaque-id}}{JSON}\"}" +
+                "%notEmpty{, \"someOtherField\": \"%enc{%OpenSearchMessageField{someOtherField}}{JSON}\"}" +
                 "%notEmpty{, %node_and_cluster_id }" +
                 "%exceptionAsJson }" + System.lineSeparator()));
     }
@@ -143,7 +143,7 @@ public void testLayoutWithAdditionalFieldOverride() {
                 "\"component\": \"%c{1.}\", " +
                 "\"cluster.name\": \"${sys:opensearch.logs.cluster_name}\", " +
                 "\"node.name\": \"%node_name\"" +
-                "%notEmpty{, \"message\": \"%OpenSearchMessageField{message}\"}" +
+                "%notEmpty{, \"message\": \"%enc{%OpenSearchMessageField{message}}{JSON}\"}" +
                 "%notEmpty{, %node_and_cluster_id }" +
                 "%exceptionAsJson }" + System.lineSeparator()));
     }

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml

@@ -102,6 +102,51 @@
               number_of_shards:   1
               number_of_replicas: 0
 
+---
+"Put distinguishable multi-wildcard templates at the same priority":
+  - skip:
+      features: allowed_warnings
+      version: " - 3.5.99"
+      reason: "template overlap check was relaxed in 3.6"
+
+  - do:
+      allowed_warnings:
+        - "index template [app-test-some] has index patterns [app-test-*-some-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some] will take precedence during new index creation"
+      indices.put_index_template:
+        name: app-test-some
+        body:
+          index_patterns: ["app-test-*-some-*"]
+          priority: 0
+
+  # The second template has a literal segment after the first wildcard that differs from the first
+  # template (-some_other- vs -some-), so the two patterns do not practically overlap and both
+  # templates should be accepted at the same priority.
+  - do:
+      allowed_warnings:
+        - "index template [app-test-some-other] has index patterns [app-test-*-some_other-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some-other] will take precedence during new index creation"
+      indices.put_index_template:
+        name: app-test-some-other
+        body:
+          index_patterns: ["app-test-*-some_other-*"]
+          priority: 0
+
+  - do:
+      indices.get_index_template:
+        name: app-test-some
+  - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some-*"]}
+
+  - do:
+      indices.get_index_template:
+        name: app-test-some-other
+  - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some_other-*"]}
+
+  - do:
+      indices.delete_index_template:
+        name: app-test-some
+  - do:
+      indices.delete_index_template:
+        name: app-test-some-other
+
 ---
 "Put index template without index_patterns":