Merge branch 'main' into main

HarishNarasimhanK · Harish Narasimhan · commit fd80fda6f62f · 2026-03-21T18:47:55.000Z
Signed-off-by: 𝐇𝐚𝐫𝐢𝐬𝐡 𝐍𝐚𝐫𝐚𝐬𝐢𝐦𝐡𝐚𝐧 𝐊 &lt;163456392+HarishNarasimhanK@users.noreply.github.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -59,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823))
 - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842))
 - Handle dependencies between analyzers ([#19248](https://github.com/opensearch-project/OpenSearch/pull/19248))
+- Fix `_field_caps` returning empty results and corrupted field names for `disable_objects: true` mappings ([#20800](https://github.com/opensearch-project/OpenSearch/pull/20800))
 - Reduce ClusterState retention in retry closures of TransportClusterManagerNodeAction ([#20858](https://github.com/opensearch-project/OpenSearch/pull/20858))
 
 ### Dependencies
diff --git a/gradle/run.gradle b/gradle/run.gradle
@@ -60,7 +60,9 @@ testClusters {
       for (String p : installedPlugins) {
         // check if its a local plugin first
         if (project.findProject(':plugins:' + p) != null) {
-          plugin('plugins:' + p)
+          plugin(':plugins:' + p)
+        } else if (project.findProject(':sandbox:plugins:' + p) != null) {
+          plugin(':sandbox:plugins:' + p)
         } else {
           // attempt to fetch it from maven
           project.repositories.mavenLocal()
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/40_disable_objects.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/40_disable_objects.yml
@@ -0,0 +1,139 @@
+---
+"Field caps with disable_objects returns leaf fields":
+
+  - skip:
+      version: " - 3.4.99"
+      reason: "disable_objects was introduced in 3.5.0"
+
+  - do:
+      indices.create:
+        index: test_disable_objects
+        body:
+          mappings:
+            properties:
+              attributes:
+                type: object
+                disable_objects: true
+
+  - do:
+      index:
+        index: test_disable_objects
+        id: "1"
+        body:
+          attributes:
+            foo:
+              bar: baz
+
+  - do:
+      indices.refresh:
+        index: test_disable_objects
+
+  - do:
+      field_caps:
+        index: test_disable_objects
+        fields: "*"
+
+  # Leaf field must exist
+  - is_true: fields.attributes\.foo\.bar
+
+  # Parent object field must have type object
+  - match: { fields.attributes.object.type: object }
+
+  # Intermediate path must not exist (since disable_objects is set)
+  - is_false: fields.attributes\.foo
+
+---
+"Field caps with disable_objects does not corrupt field names after second document indexed":
+
+  - skip:
+      version: " - 3.4.99"
+      reason: "disable_objects was introduced in 3.5.0"
+
+  - do:
+      indices.create:
+        index: test_disable_objects_merge
+        body:
+          mappings:
+            properties:
+              attributes:
+                type: object
+                disable_objects: true
+
+  # doc 1: creates the disable_objects leaf field
+  - do:
+      index:
+        index: test_disable_objects_merge
+        id: "1"
+        body:
+          attributes:
+            foo:
+              bar: baz
+
+  # doc 2: unrelated - must not corrupt the existing mapping
+  - do:
+      index:
+        index: test_disable_objects_merge
+        id: "2"
+        body:
+          key: 1
+
+  - do:
+      indices.refresh:
+        index: test_disable_objects_merge
+
+  - do:
+      field_caps:
+        index: test_disable_objects_merge
+        fields: "*"
+
+  # Leaf field must exist
+  - is_true: fields.attributes\.foo\.bar
+
+  # Corrupted field name must not exist
+  - is_false: fields.attributes\.foo\.foo\.bar
+
+  # The unrelated field must also exist
+  - is_true: fields.key
+
+---
+"Field caps with normal nested object returns intermediate paths":
+
+  - do:
+      indices.create:
+        index: test_normal_nested
+        body:
+          mappings:
+            properties:
+              user:
+                type: object
+                properties:
+                  address:
+                    type: object
+                    properties:
+                      city:
+                        type: keyword
+
+  - do:
+      index:
+        index: test_normal_nested
+        id: "1"
+        body:
+          user:
+            address:
+              city: Chemnitz
+
+  - do:
+      indices.refresh:
+        index: test_normal_nested
+
+  - do:
+      field_caps:
+        index: test_normal_nested
+        fields: "*"
+
+  # Leaf field must exist
+  - is_true: fields.user\.address\.city
+
+  # Intermediate object paths must also exist
+  - match: { fields.user\.address.object.type: object }
+  - match: { fields.user.object.type: object }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
@@ -34,6 +34,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import static org.opensearch.index.remote.RemoteStorePressureSettings.MIN_CONSECUTIVE_FAILURES_LIMIT;
@@ -44,28 +45,23 @@ public class RemoteStoreBackpressureAndResiliencyIT extends AbstractRemoteStoreM
     public void testWritesRejectedDueToConsecutiveFailureBreach() throws Exception {
         // Here the doc size of the request remains same throughout the test. After initial indexing, all remote store interactions
         // fail leading to consecutive failure limit getting exceeded and leading to rejections.
-        validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 10, ByteSizeUnit.KB.toIntBytes(1), 15, "failure_streak_count");
+        validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 10, ByteSizeUnit.KB.toIntBytes(1), "failure_streak_count");
     }
 
     public void testWritesRejectedDueToBytesLagBreach() throws Exception {
         // Initially indexing happens with doc size of 2 bytes, then all remote store interactions start failing. Now, the
         // indexing happens with doc size of 1KB leading to bytes lag limit getting exceeded and leading to rejections.
-        validateBackpressure(ByteSizeUnit.BYTES.toIntBytes(2), 30, ByteSizeUnit.KB.toIntBytes(1), 15, "bytes_lag");
+        validateBackpressure(ByteSizeUnit.BYTES.toIntBytes(2), 30, ByteSizeUnit.KB.toIntBytes(1), "bytes_lag");
     }
 
     public void testWritesRejectedDueToTimeLagBreach() throws Exception {
         // Initially indexing happens with doc size of 1KB, then all remote store interactions start failing. Now, the
         // indexing happens with doc size of 1 byte leading to time lag limit getting exceeded and leading to rejections.
-        validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 20, ByteSizeUnit.BYTES.toIntBytes(1), 3, "time_lag");
+        validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 20, ByteSizeUnit.BYTES.toIntBytes(1), "time_lag");
     }
 
-    private void validateBackpressure(
-        int initialDocSize,
-        int initialDocsToIndex,
-        int onFailureDocSize,
-        int onFailureDocsToIndex,
-        String breachMode
-    ) throws Exception {
+    private void validateBackpressure(int initialDocSize, int initialDocsToIndex, int onFailureDocSize, String breachMode)
+        throws Exception {
         Path location = randomRepoPath().toAbsolutePath();
         String dataNodeName = setup(location, 0d, "metadata", Long.MAX_VALUE);
 
@@ -92,10 +88,17 @@ private void validateBackpressure(
 
         jsonString = generateString(onFailureDocSize);
         BytesReference onFailureSource = new BytesArray(jsonString);
-        OpenSearchRejectedExecutionException ex = assertThrows(
-            OpenSearchRejectedExecutionException.class,
-            () -> indexDocAndRefresh(onFailureSource, onFailureDocsToIndex)
-        );
+        AtomicReference<OpenSearchRejectedExecutionException> exRef = new AtomicReference<>();
+        assertBusy(() -> {
+            try {
+                indexDocAndRefresh(onFailureSource, 1);
+            } catch (OpenSearchRejectedExecutionException e) {
+                exRef.set(e);
+                return;
+            }
+            fail("Expected OpenSearchRejectedExecutionException to be thrown");
+        }, 30, TimeUnit.SECONDS);
+        OpenSearchRejectedExecutionException ex = exRef.get();
         assertTrue(ex.getMessage().contains("rejected execution on primary shard"));
         assertTrue(ex.getMessage().contains(breachMode));
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/wlm/WorkloadManagementIT.java b/server/src/internalClusterTest/java/org/opensearch/wlm/WorkloadManagementIT.java
@@ -112,6 +112,7 @@ public final void cleanupNodeSettings() {
         );
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17154")
     public void testHighCPUInEnforcedMode() throws InterruptedException {
         Settings request = Settings.builder().put(WorkloadManagementSettings.WLM_MODE_SETTING.getKey(), ENABLED).build();
         assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
@@ -129,6 +130,7 @@ public void testHighCPUInEnforcedMode() throws InterruptedException {
         updateWorkloadGroupInClusterState(DELETE, workloadGroup);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17154")
     public void testHighCPUInMonitorMode() throws InterruptedException {
         WorkloadGroup workloadGroup = new WorkloadGroup(
             "name",
@@ -143,6 +145,7 @@ public void testHighCPUInMonitorMode() throws InterruptedException {
         updateWorkloadGroupInClusterState(DELETE, workloadGroup);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17154")
     public void testHighMemoryInEnforcedMode() throws InterruptedException {
         Settings request = Settings.builder().put(WorkloadManagementSettings.WLM_MODE_SETTING.getKey(), ENABLED).build();
         assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
@@ -157,6 +160,7 @@ public void testHighMemoryInEnforcedMode() throws InterruptedException {
         updateWorkloadGroupInClusterState(DELETE, workloadGroup);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17154")
     public void testHighMemoryInMonitorMode() throws InterruptedException {
         WorkloadGroup workloadGroup = new WorkloadGroup(
             "name",
diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
@@ -180,6 +180,12 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd
                     if (mapperService.fieldType(parentField) == null) {
                         // no field type, it must be an object field
                         ObjectMapper mapper = mapperService.getObjectMapper(parentField);
+                        if (mapper == null) {
+                            // parentField is part of a literal dotted field name under a disable_objects=true parent
+                            // No ObjectMapper exists for this intermediate path so skip it and continue up the chain
+                            dotIndex = parentField.lastIndexOf('.');
+                            continue;
+                        }
                         String type = mapper.nested().isNested() ? "nested" : "object";
                         IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
                             parentField,
diff --git a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
@@ -127,12 +127,14 @@ public ParametrizedFieldMapper merge(Mapper mergeWith) {
         Conflicts conflicts = new Conflicts(name());
         builder.merge((FieldMapper) mergeWith, conflicts);
         conflicts.check();
-        return builder.build(new BuilderContext(Settings.EMPTY, parentPath(name())));
+        return builder.build(new BuilderContext(Settings.EMPTY, parentPath(name(), simpleName())));
     }
 
-    private static ContentPath parentPath(String name) {
-        int endPos = name.lastIndexOf(".");
-        if (endPos == -1) {
+    private static ContentPath parentPath(String name, String simpleName) {
+        // Use simpleName to compute the parent path so that fields whose simpleName contains dots
+        // (because of disable_objects) get the correct parent path
+        int endPos = name.length() - simpleName.length() - 1;
+        if (endPos < 0) {
             return new ContentPath(0);
         }
         return new ContentPath(name.substring(0, endPos));
@@ -619,7 +621,7 @@ private void merge(FieldMapper in, Conflicts conflicts) {
                 param.merge(in, conflicts);
             }
             for (Mapper newSubField : in.multiFields) {
-                multiFieldsBuilder.update(newSubField, parentPath(newSubField.name()));
+                multiFieldsBuilder.update(newSubField, parentPath(newSubField.name(), newSubField.simpleName()));
             }
             this.copyTo.reset(in.copyTo);
             validate();
diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java
@@ -243,13 +243,18 @@ public CompletableFuture<IndexInput> asyncLoadIndexInput(Executor executor) {
                         return createIndexInput(fileCache, streamReader, request);
                     } catch (Exception e) {
                         fileCache.remove(request.getFilePath());
-                        throw new CompletionException(e);
+                        throw (e instanceof RuntimeException) ? (RuntimeException) e : new CompletionException(e);
                     }
                 }, executor).handle((indexInput, throwable) -> {
-                    fileCache.decRef(request.getFilePath());
                     if (throwable != null) {
-                        result.completeExceptionally(throwable);
+                        // On failure, the entry was already removed from the cache in the
+                        // catch block above, so we must not decRef here.
+                        Throwable cause = (throwable instanceof CompletionException && throwable.getCause() != null)
+                            ? throwable.getCause()
+                            : throwable;
+                        result.completeExceptionally(cause);
                     } else {
+                        fileCache.decRef(request.getFilePath());
                         result.complete(indexInput);
                     }
                     return null;
diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexActionTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexActionTests.java
