Merge branch '9.2' into backport/9.2/pr-137375

nielsbauman · nielsbauman · commit de247f375c82 · 2025-10-31T14:07:14.000+01:00
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -44,6 +44,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
@@ -178,7 +179,11 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                     try {
                         // we are very generious here, as the upload can take
                         // a long time depending on its size
-                        pb.start().waitFor(30, java.util.concurrent.TimeUnit.MINUTES);
+                        long timeoutSec = calculateUploadWaitTimeoutSeconds(uploadFile);
+                        boolean completedInTime = pb.start().waitFor(timeoutSec, TimeUnit.SECONDS);
+                        if (completedInTime == false) {
+                            System.out.println("Timed out waiting for buildkite artifact upload after " + timeoutSec + " seconds");
+                        }
                     } catch (InterruptedException e) {
                         System.out.println("Failed to upload buildkite artifact " + e.getMessage());
                     }
@@ -278,5 +283,14 @@ private static void createBuildArchiveTar(List<File> files, File projectDir, Fil
         private static String calculateArchivePath(Path path, Path projectPath) {
             return path.startsWith(projectPath) ? projectPath.relativize(path).toString() : path.getFileName().toString();
         }
+
+        private static long calculateUploadWaitTimeoutSeconds(File file) {
+            long fileSizeBytes = file.length();
+            long fileSizeMB = fileSizeBytes / (1024 * 1024);
+
+            // Allocate 4 seconds per MB (assumes ~250 KB/s upload speed)
+            // with min 10 seconds and max 30 minutes
+            return Math.max(10, Math.min(1800, fileSizeMB * 4));
+        }
     }
 }
diff --git a/docs/changelog/137047.yaml b/docs/changelog/137047.yaml
@@ -0,0 +1,5 @@
+pr: 137047
+summary: Reject invalid `reverse_nested` aggs
+area: Aggregations
+type: bug
+issues: []
diff --git a/docs/changelog/137399.yaml b/docs/changelog/137399.yaml
@@ -0,0 +1,5 @@
+pr: 137399
+summary: Allow allocating clones over low watermark
+area: Allocation
+type: bug
+issues: []
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/nested.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/nested.yml
@@ -194,3 +194,33 @@ setup:
   - match: { aggregations.courses.highpass_filter.unnest.department.buckets.0.doc_count: 1 }
   - match: { aggregations.courses.highpass_filter.unnest.department.buckets.1.key: math }
   - match: { aggregations.courses.highpass_filter.unnest.department.buckets.1.doc_count: 1 }
+---
+"Illegal reverse nested aggregation to a child nested object":
+  - requires:
+        capabilities:
+          - method: POST
+            path: /_search
+            capabilities: [ reject_invalid_reverse_nesting ]
+        test_runner_features: [ capabilities ]
+        reason: "search does not yet reject invalid reverse nesting paths"
+  - do:
+      catch: /Reverse nested path \[courses.sessions\] is not a parent of the current nested scope \[courses\]/
+      search:
+        index: test
+        body:
+          {
+            "aggs": {
+              "parent_nested": {
+                "nested": {
+                  "path": "courses"
+                },
+                "aggs": {
+                  "invalid_reverse_nested": {
+                    "reverse_nested": {
+                      "path": "courses.sessions"
+                    }
+                  }
+                }
+              }
+            }
+          }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java
@@ -12,12 +12,18 @@
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
+import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterInfoServiceUtils;
 import org.elasticsearch.cluster.DiskUsageIntegTestCase;
 import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -53,6 +59,7 @@
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING;
 import static org.elasticsearch.index.store.Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -103,6 +110,62 @@ public void testHighWatermarkNotExceeded() throws Exception {
         assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, contains(in(shardSizes.getSmallestShardIds())));
     }
 
+    public void testAllocateCloneIgnoresLowWatermark() throws Exception {
+        final var lowWatermarkBytes = randomLongBetween(WATERMARK_BYTES + 1, WATERMARK_BYTES * 5);
+
+        internalCluster().startMasterOnlyNode(
+            Settings.builder()
+                .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), lowWatermarkBytes + "b")
+                .build()
+        );
+        final var dataNodeName = internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+
+        final InternalClusterInfoService clusterInfoService = getInternalClusterInfoService();
+        internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> {
+            ClusterInfoServiceUtils.refresh(clusterInfoService);
+        });
+
+        final var sourceIndexName = "source-" + randomIdentifier();
+        createIndex(sourceIndexName, indexSettings(1, 0).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build());
+        final var shardSizes = createReasonableSizedShards(sourceIndexName);
+
+        updateIndexSettings(Settings.builder().put("blocks.write", true), sourceIndexName);
+
+        final var totalSpace = randomLongBetween(
+            /* do not exceed the high watermark */
+            shardSizes.getSmallestShardSize() + WATERMARK_BYTES + 1,
+            /* but make it so that naively duplicating the shard would exceed the low watermark, or else it's not a meaningful test */
+            2 * shardSizes.getSmallestShardSize() + lowWatermarkBytes
+        );
+
+        getTestFileStore(dataNodeName).setTotalSpace(totalSpace);
+        refreshDiskUsage();
+
+        final var targetIndexName = "target-" + randomIdentifier();
+        final var resizeRequest = new ResizeRequest(targetIndexName, sourceIndexName);
+        resizeRequest.setResizeType(ResizeType.CLONE);
+        resizeRequest.masterNodeTimeout(TEST_REQUEST_TIMEOUT);
+        resizeRequest.ackTimeout(TEST_REQUEST_TIMEOUT);
+        resizeRequest.setWaitForActiveShards(ActiveShardCount.ALL);
+        resizeRequest.getTargetIndexRequest()
+            .settings(
+                Settings.builder().put(resizeRequest.getTargetIndexRequest().settings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            );
+
+        safeAwait(
+            SubscribableListener.<CreateIndexResponse>newForked(l -> indicesAdmin().resizeIndex(resizeRequest, l))
+                .andThenAccept(
+                    createIndexResponse -> assertThat(
+                        true,
+                        allOf(equalTo(createIndexResponse.isAcknowledged()), equalTo(createIndexResponse.isShardsAcknowledged()))
+                    )
+                )
+        );
+
+        ensureGreen(sourceIndexName, targetIndexName);
+    }
+
     public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Exception {
         internalCluster().startMasterOnlyNode();
         internalCluster().startDataOnlyNode();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
@@ -470,15 +470,19 @@ public void testReverseNestedAggWithoutNestedAgg() {
     public void testNonExistingNestedField() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("idx2").setQuery(matchAllQuery())
-                .addAggregation(nested("nested2", "nested1.nested2").subAggregation(reverseNested("incorrect").path("nested3"))),
+                .addAggregation(nested("nested2", "nested1.nested2"))
+                .addAggregation(nested("incorrect", "nested1.incorrect")),
             response -> {
 
                 SingleBucketAggregation nested = response.getAggregations().get("nested2");
                 assertThat(nested, notNullValue());
                 assertThat(nested.getName(), equalTo("nested2"));
+                assertThat(nested.getDocCount(), is(27L));
 
-                SingleBucketAggregation reverseNested = nested.getAggregations().get("incorrect");
-                assertThat(reverseNested.getDocCount(), is(0L));
+                SingleBucketAggregation incorrect = response.getAggregations().get("incorrect");
+                assertThat(incorrect, notNullValue());
+                assertThat(incorrect.getName(), equalTo("incorrect"));
+                assertThat(incorrect.getDocCount(), is(0L));
             }
         );
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -219,10 +219,11 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
         }
 
         // flag that determines whether the low threshold checks below can be skipped. We use this for a primary shard that is freshly
-        // allocated and empty.
-        boolean skipLowThresholdChecks = shardRouting.primary()
+        // allocated and either empty or the result of cloning another shard.
+        final var isNewCloneTarget = isNewCloneTarget(shardRouting, allocation);
+        final var skipLowThresholdChecks = shardRouting.primary()
             && shardRouting.active() == false
-            && shardRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE;
+            && (isNewCloneTarget || shardRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE);
 
         if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLowStage(total).getBytes()) {
             if (skipLowThresholdChecks == false) {
@@ -283,6 +284,18 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
         }
 
         // Secondly, check that allocating the shard to this node doesn't put it above the high watermark
+
+        if (isNewCloneTarget) {
+            // The clone will be a hard-linked copy of the original shard so will not meaningfully increase disk usage
+            return allocation.decision(
+                Decision.YES,
+                NAME,
+                "enough disk for freshly-cloned shard on node, free: [%s], used: [%s]",
+                freeBytesValue,
+                Strings.format1Decimals(usedDiskPercentage, "%")
+            );
+        }
+
         final long shardSize = getExpectedShardSize(shardRouting, 0L, allocation);
         assert shardSize >= 0 : shardSize;
         long freeBytesAfterShard = freeBytes - shardSize;
@@ -326,6 +339,23 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
         );
     }
 
+    private static boolean isNewCloneTarget(ShardRouting shardRouting, RoutingAllocation allocation) {
+        if (shardRouting.unassigned() == false
+            || shardRouting.primary() == false
+            || shardRouting.recoverySource() != RecoverySource.LocalShardsRecoverySource.INSTANCE) {
+            return false;
+        }
+
+        final var targetMetadata = allocation.metadata().indexMetadata(shardRouting.index());
+        final var sourceIndex = targetMetadata.getResizeSourceIndex();
+        if (sourceIndex == null) {
+            return false;
+        }
+
+        final var sourceMetadata = allocation.metadata().indexMetadata(sourceIndex);
+        return sourceMetadata != null && sourceMetadata.getNumberOfShards() == targetMetadata.getNumberOfShards();
+    }
+
     @Override
     public Decision canForceAllocateDuringReplace(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         Map<String, DiskUsage> usages = allocation.clusterInfo().getNodeMostAvailableDiskUsages();
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
@@ -60,6 +60,7 @@ private SearchCapabilities() {}
     private static final String BUCKET_SCRIPT_PARENT_MULTI_BUCKET_ERROR = "bucket_script_parent_multi_bucket_error";
     private static final String EXCLUDE_SOURCE_VECTORS_SETTING = "exclude_source_vectors_setting";
     private static final String CLUSTER_STATS_EXTENDED_USAGE = "extended-search-usage-stats";
+    private static final String REJECT_INVALID_REVERSE_NESTING = "reject_invalid_reverse_nesting";
 
     public static final Set<String> CAPABILITIES;
     static {
@@ -90,6 +91,7 @@ private SearchCapabilities() {}
         capabilities.add(BUCKET_SCRIPT_PARENT_MULTI_BUCKET_ERROR);
         capabilities.add(EXCLUDE_SOURCE_VECTORS_SETTING);
         capabilities.add(CLUSTER_STATS_EXTENDED_USAGE);
+        capabilities.add(REJECT_INVALID_REVERSE_NESTING);
         CAPABILITIES = Set.copyOf(capabilities);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
@@ -84,6 +84,18 @@ protected AggregatorFactory doBuild(AggregationContext context, AggregatorFactor
             throw new IllegalArgumentException("Reverse nested aggregation [" + name + "] can only be used inside a [nested] aggregation");
         }
 
+        if (path != null) {
+            NestedObjectMapper currentParent = context.nestedScope().getObjectMapper();
+            if (currentParent != null) {
+                String parentPath = currentParent.fullPath();
+                if (parentPath.equals(path) == false && parentPath.startsWith(path + ".") == false) {
+                    throw new IllegalArgumentException(
+                        "Reverse nested path [" + path + "] is not a parent of the current nested scope [" + parentPath + "]"
+                    );
+                }
+            }
+        }
+
         NestedObjectMapper nestedMapper = null;
         if (path != null) {
             nestedMapper = context.nestedLookup().getNestedMappers().get(path);
diff --git a/server/src/main/resources/transport/definitions/referable/esql_resolve_fields_response_used.csv b/server/src/main/resources/transport/definitions/referable/esql_resolve_fields_response_used.csv
@@ -0,0 +1 @@
+9204000,9185005
diff --git a/server/src/main/resources/transport/definitions/referable/ilm_searchable_snapshot_opt_out_clone.csv b/server/src/main/resources/transport/definitions/referable/ilm_searchable_snapshot_opt_out_clone.csv
@@ -1 +1 @@
-9209000,9185006
+9186005
diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv
@@ -1 +1 @@
-ilm_searchable_snapshot_opt_out_clone,9185006
+ilm_searchable_snapshot_opt_out_clone,9186005
diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv
@@ -1 +1 @@
-ilm_searchable_snapshot_opt_out_clone,9209000
+esql_resolve_fields_response_used,9204000
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
@@ -247,4 +247,29 @@ public void testNestedUnderTerms() throws IOException {
     protected List<ObjectMapper> objectMappers() {
         return NestedAggregatorTests.MOCK_OBJECT_MAPPERS;
     }
+
+    public void testErrorOnInvalidReverseNestedWithPath() throws IOException {
+        AggregationBuilder aggregationBuilder = nested("n1", NESTED_OBJECT).subAggregation(
+            nested("n2", NESTED_OBJECT + ".child").subAggregation(
+                reverseNested("r1").path(NESTED_OBJECT).subAggregation(reverseNested("r2").path(NESTED_OBJECT + ".child"))
+            )
+        );
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter iw = newRandomIndexWriterWithLogDocMergePolicy(directory)) {
+                // No docs needed
+            }
+            try (DirectoryReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
+                IllegalArgumentException e = assertThrows(
+                    IllegalArgumentException.class,
+                    () -> searchAndReduce(indexReader, new AggTestConfig(aggregationBuilder))
+                );
+                assertThat(
+                    e.getMessage(),
+                    equalTo("Reverse nested path [nested_object.child] is not a parent of the current nested scope [nested_object]")
+                );
+
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.NodeRoles;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.autoscaling.action.GetAutoscalingCapacityAction;
 import org.elasticsearch.xpack.autoscaling.action.PutAutoscalingPolicyAction;
@@ -400,13 +399,7 @@ public void testScaleWhileShrinking() throws Exception {
         );
     }
 
-    @TestLogging(
-        reason = "Debugging https://github.com/elastic/elasticsearch/issues/96764",
-        value = "org.elasticsearch.cluster.InternalClusterInfoService:TRACE"
-            + ",org.elasticsearch.xpack.autoscaling.action:TRACE"
-            + ",org.elasticsearch.cluster.routing.allocation:DEBUG"
-    )
-    public void testScaleDuringSplitOrClone() throws Exception {
+    public void testScaleDuringSplit() throws Exception {
         internalCluster().startMasterOnlyNode();
         final String dataNode1Name = internalCluster().startDataOnlyNode();
 
@@ -460,9 +453,8 @@ public void testScaleDuringSplitOrClone() throws Exception {
         );
 
         updateIndexSettings(Settings.builder().put("index.blocks.write", true), indexName);
-        ResizeType resizeType = randomFrom(ResizeType.CLONE, ResizeType.SPLIT);
         String cloneName = "clone-" + indexName;
-        int resizedShardCount = resizeType == ResizeType.CLONE ? 1 : between(2, 10);
+        int resizedShardCount = between(2, 10);
         assertAcked(
             indicesAdmin().prepareResizeIndex(indexName, cloneName)
                 .setSettings(
@@ -472,7 +464,7 @@ public void testScaleDuringSplitOrClone() throws Exception {
                         .build()
                 )
                 .setWaitForActiveShards(ActiveShardCount.NONE)
-                .setResizeType(resizeType)
+                .setResizeType(ResizeType.SPLIT)
         );
 
         // * 2 since worst case is no hard links, see DiskThresholdDecider.getExpectedShardSize.
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
@@ -976,8 +976,13 @@ Builder underConstruction() {
         }
     }
 
-    private static class DataTypesTransportVersions {
+    public static class DataTypesTransportVersions {
 
+        /**
+         * The first transport version after the PR that introduced geotile/geohash/geohex, resp.
+         * after 9.1. We didn't require transport versions at that point in time, as geotile/hash/hex require
+         * using specific functions to even occur in query plans.
+         */
         public static final TransportVersion INDEX_SOURCE = TransportVersion.fromName("index_source");
 
         public static final TransportVersion ESQL_DENSE_VECTOR_CREATED_VERSION = TransportVersion.fromName(
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java

Original file line number	Diff line number	Diff line change
`@@ -60,6 +60,7 @@ private SearchCapabilities() {}`
`60`	`60`	`private static final String BUCKET_SCRIPT_PARENT_MULTI_BUCKET_ERROR = "bucket_script_parent_multi_bucket_error";`
`61`	`61`	`private static final String EXCLUDE_SOURCE_VECTORS_SETTING = "exclude_source_vectors_setting";`
`62`	`62`	`private static final String CLUSTER_STATS_EXTENDED_USAGE = "extended-search-usage-stats";`
	`63`	`+ private static final String REJECT_INVALID_REVERSE_NESTING = "reject_invalid_reverse_nesting";`
`63`	`64`
`64`	`65`	`public static final Set<String> CAPABILITIES;`
`65`	`66`	`static {`
`@@ -90,6 +91,7 @@ private SearchCapabilities() {}`
`90`	`91`	`capabilities.add(BUCKET_SCRIPT_PARENT_MULTI_BUCKET_ERROR);`
`91`	`92`	`capabilities.add(EXCLUDE_SOURCE_VECTORS_SETTING);`
`92`	`93`	`capabilities.add(CLUSTER_STATS_EXTENDED_USAGE);`
	`94`	`+ capabilities.add(REJECT_INVALID_REVERSE_NESTING);`
`93`	`95`	`CAPABILITIES = Set.copyOf(capabilities);`
`94`	`96`	`}`
`95`	`97`	`}`