Merge branch 'main' into make_inlinestats_tests_snapshot

Author: astefan

docs/changelog/133424.yaml

@@ -0,0 +1,5 @@
+pr: 133424
+summary: Ensuring only a single request executor object is created
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/133604.yaml

@@ -0,0 +1,5 @@
+pr: 133604
+summary: Update `DefBootstrap` to handle Error from `ClassValue`
+area: Infra/Scripting
+type: bug
+issues: []

docs/changelog/133611.yaml

@@ -0,0 +1,6 @@
+pr: 133611
+summary: Allow trailing empty string field names in paths of flattened field
+area: Mapping
+type: bug
+issues:
+ - 130139

modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java

@@ -201,7 +201,10 @@ protected MethodHandle computeValue(Class<?> receiverType) {
                     try {
                         return lookup(flavor, name, receiverType).asType(type);
                     } catch (Throwable t) {
-                        Def.rethrow(t);
+                        // ClassValue.getFromHashMap wraps checked exceptions as Error, so we
+                        // use a sentinel class [PainlessWrappedException] here to work around
+                        // this issue and later unwrap the original exception
+                        Def.rethrow(t instanceof Exception ? new PainlessWrappedException((Exception) t) : t);
                         throw new AssertionError();
                     }
                 }

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java

@@ -46,6 +46,9 @@ public interface PainlessScript {
      * @return The generated ScriptException.
      */
     default ScriptException convertToScriptException(Throwable t, Map<String, List<String>> extraMetadata) {
+        if (t instanceof PainlessWrappedException) {
+            t = t.getCause();
+        }
         // create a script stack: this is just the script portion
         List<String> scriptStack = new ArrayList<>();
         ScriptException.Position pos = null;

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessWrappedException.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.painless;
+
+/**
+ * Checked exceptions are wrapped in {@link ClassValue}#getFromHashMap in Error
+ * which leads to unexpected behavior in Painless. This class is used as a
+ * workaround for that exception wrapping.
+ */
+public class PainlessWrappedException extends Error {
+
+    /**
+     * Constructor.
+     * @param cause The {@link Exception} cause.
+     */
+    public PainlessWrappedException(final Exception cause) {
+        super(cause);
+    }
+}

modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.PainlessError;
 import org.elasticsearch.painless.PainlessExplainError;
+import org.elasticsearch.painless.PainlessWrappedException;
 import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.ScriptClassInfo.MethodArgument;
 import org.elasticsearch.painless.ir.BinaryImplNode;
@@ -415,6 +416,7 @@ protected static void injectSandboxExceptions(FunctionNode irFunctionNode) {
 
             for (Class<? extends Throwable> throwable : List.of(
                 PainlessError.class,
+                PainlessWrappedException.class,
                 LinkageError.class,
                 OutOfMemoryError.class,
                 StackOverflowError.class,

modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java

@@ -139,9 +139,11 @@ public void testMegamorphic() throws Throwable {
         map.put("a", "b");
         assertEquals(2, (int) handle.invokeExact((Object) map));
 
-        final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> {
+        final PainlessWrappedException pwe = expectThrows(PainlessWrappedException.class, () -> {
             Integer.toString((int) handle.invokeExact(new Object()));
         });
+        assertTrue(pwe.getCause() instanceof IllegalArgumentException);
+        IllegalArgumentException iae = (IllegalArgumentException) pwe.getCause();
         assertEquals("dynamic method [java.lang.Object, size/0] not found", iae.getMessage());
         assertTrue("Does not fail inside ClassValue.computeValue()", Arrays.stream(iae.getStackTrace()).anyMatch(e -> {
             return e.getMethodName().equals("computeValue") && e.getClassName().startsWith("org.elasticsearch.painless.DefBootstrap$PIC$");

server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java

@@ -22,12 +22,13 @@
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -39,6 +40,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
@@ -98,8 +100,9 @@ public void accept(ActionListener<Response> listener) {
                 assert totalShardCopyCount == 0 && successShardCopyCount == 0 && allFailures.isEmpty() : "shouldn't call this twice";
 
                 final ClusterState clusterState = clusterService.state();
-                final ProjectMetadata project = projectResolver.getProjectMetadata(clusterState);
-                final List<ShardId> shards = shards(request, project, clusterState.routingTable(project.id()));
+                final ProjectState projectState = projectResolver.getProjectState(clusterState);
+                final ProjectMetadata project = projectState.metadata();
+                final List<ShardId> shards = shards(request, projectState);
                 final Map<String, IndexMetadata> indexMetadataByName = project.indices();
 
                 try (var refs = new RefCountingRunnable(() -> finish(listener))) {
@@ -185,17 +188,17 @@ protected void shardExecute(Task task, Request request, ShardId shardId, ActionL
     /**
      * @return all shard ids the request should run on
      */
-    protected List<ShardId> shards(Request request, ProjectMetadata project, RoutingTable indexRoutingTables) {
+    protected List<ShardId> shards(Request request, ProjectState projectState) {
         assert Transports.assertNotTransportThread("may hit all the shards");
         List<ShardId> shardIds = new ArrayList<>();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(project, request);
+
+        OperationRouting operationRouting = clusterService.operationRouting();
+
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), request);
         for (String index : concreteIndices) {
-            IndexMetadata indexMetadata = project.indices().get(index);
-            if (indexMetadata != null) {
-                final IndexRoutingTable indexRoutingTable = indexRoutingTables.indicesRouting().get(index);
-                for (int i = 0; i < indexRoutingTable.size(); i++) {
-                    shardIds.add(indexRoutingTable.shard(i).shardId());
-                }
+            Iterator<IndexShardRoutingTable> iterator = operationRouting.allWritableShards(projectState, index);
+            while (iterator.hasNext()) {
+                shardIds.add(iterator.next().shardId());
             }
         }
         return shardIds;

server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java

@@ -159,11 +159,18 @@ public void checkIndexSplitAllowed() {}
      * @param shardId  shardId to which the current document is routed based on hashing
      * @return Updated shardId
      */
-    protected final int rerouteIfResharding(int shardId) {
+    protected final int rerouteWritesIfResharding(int shardId) {
+        return rerouteFromSplitTargetShard(shardId, IndexReshardingState.Split.TargetShardState.HANDOFF);
+    }
+
+    protected final int rerouteSearchIfResharding(int shardId) {
+        return rerouteFromSplitTargetShard(shardId, IndexReshardingState.Split.TargetShardState.SPLIT);
+    }
+
+    private int rerouteFromSplitTargetShard(int shardId, IndexReshardingState.Split.TargetShardState minimumRequiredState) {
+        assert indexReshardingMetadata == null || indexReshardingMetadata.isSplit() : "Index resharding state is not a split";
         if (indexReshardingMetadata != null && indexReshardingMetadata.getSplit().isTargetShard(shardId)) {
-            assert indexReshardingMetadata.isSplit() : "Index resharding state is not a split";
-            if (indexReshardingMetadata.getSplit()
-                .targetStateAtLeast(shardId, IndexReshardingState.Split.TargetShardState.HANDOFF) == false) {
+            if (indexReshardingMetadata.getSplit().targetStateAtLeast(shardId, minimumRequiredState) == false) {
                 return indexReshardingMetadata.getSplit().sourceShard(shardId);
             }
         }
@@ -217,21 +224,21 @@ public int indexShard(String id, @Nullable String routing, XContentType sourceTy
             }
             checkRoutingRequired(id, routing);
             int shardId = shardId(id, routing);
-            return rerouteIfResharding(shardId);
+            return rerouteWritesIfResharding(shardId);
         }
 
         @Override
         public int updateShard(String id, @Nullable String routing) {
             checkRoutingRequired(id, routing);
             int shardId = shardId(id, routing);
-            return rerouteIfResharding(shardId);
+            return rerouteWritesIfResharding(shardId);
         }
 
         @Override
         public int deleteShard(String id, @Nullable String routing) {
             checkRoutingRequired(id, routing);
             int shardId = shardId(id, routing);
-            return rerouteIfResharding(shardId);
+            return rerouteWritesIfResharding(shardId);
         }
 
         @Override
@@ -262,7 +269,7 @@ protected int shardId(String id, @Nullable String routing) {
 
         @Override
         public void collectSearchShards(String routing, IntConsumer consumer) {
-            consumer.accept(hashToShardId(effectiveRoutingToHash(routing)));
+            consumer.accept(rerouteSearchIfResharding(hashToShardId(effectiveRoutingToHash(routing))));
         }
     }
 
@@ -290,7 +297,7 @@ protected int shardId(String id, @Nullable String routing) {
         public void collectSearchShards(String routing, IntConsumer consumer) {
             int hash = effectiveRoutingToHash(routing);
             for (int i = 0; i < routingPartitionSize; i++) {
-                consumer.accept(hashToShardId(hash + i));
+                consumer.accept(rerouteSearchIfResharding(hashToShardId(hash + i)));
             }
         }
     }
@@ -339,7 +346,7 @@ public int indexShard(String id, @Nullable String routing, XContentType sourceTy
             checkNoRouting(routing);
             hash = hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty);
             int shardId = hashToShardId(hash);
-            return (rerouteIfResharding(shardId));
+            return (rerouteWritesIfResharding(shardId));
         }
 
         public String createId(XContentType sourceType, BytesReference source, byte[] suffix) {
@@ -480,14 +487,14 @@ public int updateShard(String id, @Nullable String routing) {
         public int deleteShard(String id, @Nullable String routing) {
             checkNoRouting(routing);
             int shardId = idToHash(id);
-            return (rerouteIfResharding(shardId));
+            return (rerouteWritesIfResharding(shardId));
         }
 
         @Override
         public int getShard(String id, @Nullable String routing) {
             checkNoRouting(routing);
             int shardId = idToHash(id);
-            return (rerouteIfResharding(shardId));
+            return (rerouteWritesIfResharding(shardId));
         }
 
         private void checkNoRouting(@Nullable String routing) {