Merge branch 'main' into revert-128293

iverase · web-flow · commit f91da1263b0f · 2025-06-11T11:25:18.000+02:00
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -96,7 +96,7 @@ fi
 
 if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
   PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
-  PERF_METRICS_USERNAME=$(vault read -field=es_username /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
+  PERF_METRICS_USERNAME=$(vault read -field=es_user /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
   PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
 
   export PERF_METRICS_HOST
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java
@@ -24,9 +24,11 @@
 import java.util.HexFormat;
 import java.util.Locale;
 import java.util.function.Function;
+import java.util.jar.Attributes;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.stream.Collectors;
 
 import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES;
@@ -60,6 +62,10 @@ public String toString() {
         }
     }
 
+    public static void patchJar(File inputJar, File outputJar, Collection<PatcherInfo> patchers) {
+        patchJar(inputJar, outputJar, patchers, false);
+    }
+
     /**
      * Patches the classes in the input JAR file, using the collection of patchers. Each patcher specifies a target class (its jar entry
      * name) and the SHA256 digest on the class bytes.
@@ -69,8 +75,11 @@ public String toString() {
      * @param inputFile the JAR file to patch
      * @param outputFile the output (patched) JAR file
      * @param patchers list of patcher info (classes to patch (jar entry name + optional SHA256 digest) and ASM visitor to transform them)
+     * @param unsignJar whether to remove class signatures from the JAR Manifest; set this to true when patching a signed JAR,
+     *                  otherwise the patched classes will fail to load at runtime due to mismatched signatures.
+     *                  @see <a href="https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html">Understanding Signing and Verification</a>
      */
-    public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers) {
+    public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers, boolean unsignJar) {
         var classPatchers = patchers.stream().collect(Collectors.toMap(PatcherInfo::jarEntryName, Function.identity()));
         var mismatchedClasses = new ArrayList<MismatchInfo>();
         try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) {
@@ -101,9 +110,23 @@ public static void patchJar(File inputFile, File outputFile, Collection<PatcherI
                         );
                     }
                 } else {
-                    // Read the entry's data and write it to the new JAR
                     try (InputStream is = jarFile.getInputStream(entry)) {
-                        is.transferTo(jos);
+                        if (unsignJar && entryName.equals("META-INF/MANIFEST.MF")) {
+                            var manifest = new Manifest(is);
+                            for (var manifestEntry : manifest.getEntries().entrySet()) {
+                                var nonSignatureAttributes = new Attributes();
+                                for (var attribute : manifestEntry.getValue().entrySet()) {
+                                    if (attribute.getKey().toString().endsWith("Digest") == false) {
+                                        nonSignatureAttributes.put(attribute.getKey(), attribute.getValue());
+                                    }
+                                }
+                                manifestEntry.setValue(nonSignatureAttributes);
+                            }
+                            manifest.write(jos);
+                        } else if (unsignJar == false || entryName.matches("META-INF/.*\\.SF") == false) {
+                            // Read the entry's data and write it to the new JAR
+                            is.transferTo(jos);
+                        }
                     }
                 }
                 jos.closeEntry();
diff --git a/docs/changelog/128613.yaml b/docs/changelog/128613.yaml
@@ -0,0 +1,5 @@
+pr: 128613
+summary: Improve support for bytecode patching signed jars
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -496,12 +496,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test073RunEsAsDifferentUserAndGroupWithoutBindMounting
   issue: https://github.com/elastic/elasticsearch/issues/128996
-- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
-  method: test {lookup-join.LookupJoinOnTimeSeriesIndex ASYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/129078
-- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
-  method: test {lookup-join.LookupJoinOnTimeSeriesIndex SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/129082
 - class: org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT
   method: test {p0=upgraded_cluster/70_ilm/Test Lifecycle Still There And Indices Are Still Managed}
   issue: https://github.com/elastic/elasticsearch/issues/129097
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -45,6 +46,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
@@ -706,12 +708,15 @@ public Metadata.Builder put(Metadata.Builder builder, Metadata.ProjectCustom par
 
             @Override
             public Metadata.Builder remove(Metadata.Builder builder, String name) {
+                @FixForMultiProject
+                final var projectBuilder = builder.getProject(ProjectId.DEFAULT);
                 if (IndexGraveyard.TYPE.equals(name)) {
                     // there must always be at least an empty graveyard
-                    return builder.indexGraveyard(IndexGraveyard.builder().build());
+                    projectBuilder.indexGraveyard(IndexGraveyard.builder().build());
                 } else {
-                    return builder.removeProjectCustom(name);
+                    projectBuilder.removeCustom(name);
                 }
+                return builder;
             }
 
             @Override
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java
@@ -91,13 +91,13 @@ public void testShouldRestoreOnlySnapshotMetadata() throws Exception {
         assertThat(getSnapshot("test-repo", "test-snapshot").state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("update custom persistent metadata");
-        updateClusterState(currentState -> currentState.copyAndUpdateMetadata(metadataBuilder -> {
+        updateClusterState(currentState -> currentState.copyAndUpdateProject(currentState.metadata().getProject().id(), builder -> {
             if (isSnapshotMetadataSet == false || randomBoolean()) {
-                metadataBuilder.putCustom(SnapshotMetadata.TYPE, new SnapshotMetadata("after_snapshot_s"));
+                builder.putCustom(SnapshotMetadata.TYPE, new SnapshotMetadata("after_snapshot_s"));
             } else {
-                metadataBuilder.removeProjectCustom(SnapshotMetadata.TYPE);
+                builder.removeCustom(SnapshotMetadata.TYPE);
             }
-            metadataBuilder.putCustom(ApiMetadata.TYPE, new ApiMetadata("after_snapshot_ns"));
+            builder.putCustom(ApiMetadata.TYPE, new ApiMetadata("after_snapshot_ns"));
         }));
 
         logger.info("restore snapshot");
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.core.Tuple;
@@ -66,6 +68,8 @@ protected void processDataPaths(Terminal terminal, Path[] dataPaths, OptionSet o
             "project scoped custom metadata names: " + oldClusterState.metadata().getProject().customs().keySet()
         );
         final Metadata.Builder metadataBuilder = Metadata.builder(oldClusterState.metadata());
+        @FixForMultiProject
+        final ProjectMetadata.Builder projectBuilder = metadataBuilder.getProject(ProjectId.DEFAULT);
         for (String customToRemove : customsToRemove) {
             @FixForMultiProject
             boolean matched = false;
@@ -82,7 +86,7 @@ protected void processDataPaths(Terminal terminal, Path[] dataPaths, OptionSet o
             }
             for (String customKey : oldClusterState.metadata().getProject().customs().keySet()) {
                 if (Regex.simpleMatch(customToRemove, customKey)) {
-                    metadataBuilder.removeProjectCustom(customKey);
+                    projectBuilder.removeCustom(customKey);
                     if (matched == false) {
                         terminal.println("The following customs will be removed:");
                     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
@@ -1743,18 +1743,6 @@ public Builder putProjectCustom(String type, ProjectCustom custom) {
             return this;
         }
 
-        @Deprecated(forRemoval = true)
-        public Builder removeProjectCustom(String type) {
-            getSingleProject().removeCustom(type);
-            return this;
-        }
-
-        @Deprecated(forRemoval = true)
-        public Builder removeProjectCustomIf(BiPredicate<String, ? super ProjectCustom> p) {
-            getSingleProject().removeCustomIf(p);
-            return this;
-        }
-
         @Deprecated(forRemoval = true)
         public Builder projectCustoms(Map<String, ProjectCustom> projectCustoms) {
             projectCustoms.forEach((key, value) -> Objects.requireNonNull(value, key));
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -681,10 +681,12 @@ private static ClusterState nextState(
         final ClusterState.Builder builder = ClusterState.builder(previousState);
         builder.stateUUID(UUIDs.randomBase64UUID());
         Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata());
+        ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(previousState.metadata().projects().values().iterator().next());
         metadataBuilder.removeCustomIf((ignore, custom) -> custom instanceof TestClusterCustomMetadata);
-        metadataBuilder.removeProjectCustomIf((ignore, custom) -> custom instanceof TestProjectCustomMetadata);
+        projectBuilder.removeCustomIf((ignore, custom) -> custom instanceof TestProjectCustomMetadata);
         clusterCustoms.forEach(clusterCustom -> metadataBuilder.putCustom(clusterCustom.getWriteableName(), clusterCustom));
-        projectCustoms.forEach(projectCustom -> metadataBuilder.putCustom(projectCustom.getWriteableName(), projectCustom));
+        projectCustoms.forEach(projectCustom -> projectBuilder.putCustom(projectCustom.getWriteableName(), projectCustom));
+        metadataBuilder.put(projectBuilder);
         builder.metadata(metadataBuilder);
         return builder.build();
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
@@ -1391,20 +1391,6 @@ public void testBuilderRemoveClusterCustomIf() {
         assertThat(metadata.custom("custom2"), sameInstance(custom2));
     }
 
-    public void testBuilderRemoveProjectCustomIf() {
-        var custom1 = new TestProjectCustomMetadata();
-        var custom2 = new TestProjectCustomMetadata();
-        var builder = Metadata.builder();
-        builder.putCustom("custom1", custom1);
-        builder.putCustom("custom2", custom2);
-
-        builder.removeProjectCustomIf((key, value) -> Objects.equals(key, "custom1"));
-
-        var metadata = builder.build();
-        assertThat(metadata.getProject().custom("custom1"), nullValue());
-        assertThat(metadata.getProject().custom("custom2"), sameInstance(custom2));
-    }
-
     public void testBuilderRejectsDataStreamThatConflictsWithIndex() {
         final String dataStreamName = "my-data-stream";
         IndexMetadata idx = createFirstBackingIndex(dataStreamName).build();
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -933,7 +934,10 @@ private ClusterState insignificantChange(ClusterState clusterState) {
                 if (scope == PersistentTasksExecutor.Scope.CLUSTER) {
                     metadata = Metadata.builder(clusterState.metadata()).removeCustom(ClusterPersistentTasksCustomMetadata.TYPE);
                 } else {
-                    metadata = Metadata.builder(clusterState.metadata()).removeProjectCustom(PersistentTasksCustomMetadata.TYPE);
+                    metadata = Metadata.builder(clusterState.metadata())
+                        .put(
+                            ProjectMetadata.builder(clusterState.metadata().getProject()).removeCustom(PersistentTasksCustomMetadata.TYPE)
+                        );
                 }
                 return builder.metadata(metadata).build();
             }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
@@ -865,10 +865,10 @@ static void removeCCRRelatedMetadataFromClusterState(ClusterService clusterServi
             public ClusterState execute(ClusterState currentState) throws Exception {
                 AutoFollowMetadata empty = new AutoFollowMetadata(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
                 ClusterState.Builder newState = ClusterState.builder(currentState);
-                newState.metadata(
-                    Metadata.builder(currentState.getMetadata())
+                newState.putProjectMetadata(
+                    ProjectMetadata.builder(currentState.metadata().getProject())
                         .putCustom(AutoFollowMetadata.TYPE, empty)
-                        .removeProjectCustom(PersistentTasksCustomMetadata.TYPE)
+                        .removeCustom(PersistentTasksCustomMetadata.TYPE)
                         .build()
                 );
                 return newState.build();
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
 import org.elasticsearch.index.mapper.IndexModeFieldMapper;
@@ -26,8 +25,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.core.Tuple.tuple;
-
 public class MetadataAttribute extends TypedAttribute {
     public static final String TIMESTAMP_FIELD = "@timestamp"; // this is not a true metadata attribute
     public static final String TSID_FIELD = "_tsid";
@@ -40,23 +37,19 @@ public class MetadataAttribute extends TypedAttribute {
         MetadataAttribute::readFrom
     );
 
-    private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
-        "_version",
-        tuple(DataType.LONG, false), // _version field is not searchable
-        INDEX,
-        tuple(DataType.KEYWORD, true),
-        IdFieldMapper.NAME,
-        tuple(DataType.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default
-        IgnoredFieldMapper.NAME,
-        tuple(DataType.KEYWORD, true),
-        SourceFieldMapper.NAME,
-        tuple(DataType.SOURCE, false),
-        IndexModeFieldMapper.NAME,
-        tuple(DataType.KEYWORD, true),
-        SCORE,
-        tuple(DataType.DOUBLE, false)
+    private static final Map<String, MetadataAttributeConfiguration> ATTRIBUTES_MAP = Map.ofEntries(
+        Map.entry("_version", new MetadataAttributeConfiguration(DataType.LONG, false)),
+        Map.entry(INDEX, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
+        // actually _id is searchable, but fielddata access on it is disallowed by default
+        Map.entry(IdFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, false)),
+        Map.entry(IgnoredFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
+        Map.entry(SourceFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.SOURCE, false)),
+        Map.entry(IndexModeFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
+        Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false))
     );
 
+    private record MetadataAttributeConfiguration(DataType dataType, boolean searchable) {}
+
     private final boolean searchable;
 
     public MetadataAttribute(
@@ -160,12 +153,12 @@ public boolean searchable() {
 
     public static MetadataAttribute create(Source source, String name) {
         var t = ATTRIBUTES_MAP.get(name);
-        return t != null ? new MetadataAttribute(source, name, t.v1(), t.v2()) : null;
+        return t != null ? new MetadataAttribute(source, name, t.dataType(), t.searchable()) : null;
     }
 
     public static DataType dataType(String name) {
         var t = ATTRIBUTES_MAP.get(name);
-        return t != null ? t.v1() : null;
+        return t != null ? t.dataType() : null;
     }
 
     public static boolean isSupported(String name) {
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec
@@ -4587,6 +4587,7 @@ emp_no:integer | language_code:integer | language_name:keyword
 
 lookupJoinOnTimeSeriesIndex
 required_capability: join_lookup_v12
+required_capability: lookup_join_on_mixed_numeric_fields
 
 FROM k8s
 | RENAME network.bytes_in AS language_code
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/where-like.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/where-like.csv-spec
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/bulk/BulkInferenceExecutionState.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/bulk/BulkInferenceExecutionState.java
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/bulk/BulkInferenceExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/bulk/BulkInferenceExecutor.java
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/completion/CompletionOperatorOutputBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/completion/CompletionOperatorOutputBuilder.java
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/completion/CompletionOperatorRequestIterator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/inference/completion/CompletionOperatorRequestIterator.java
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/inference/InferenceOperatorTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/inference/InferenceOperatorTestCase.java
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java
