Merge branch '9.0' into backport/9.0/pr-120754

Author: nik9000

docs/changelog/120852.yaml

@@ -0,0 +1,5 @@
+pr: 120852
+summary: Correct line and column numbers of missing named parameters
+area: ES|QL
+type: bug
+issues: []

docs/changelog/121325.yaml

@@ -0,0 +1,5 @@
+pr: 121325
+summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality'
+area: Data streams
+type: bug
+issues: []

docs/internal/Versioning.md

@@ -35,19 +35,19 @@ Every change to the transport protocol is represented by a new transport version
 higher than all previous transport versions, which then becomes the highest version
 recognized by that build of Elasticsearch. The version ids are stored
 as constants in the `TransportVersions` class.
-Each id has a standard pattern `M_NNN_SS_P`, where:
+Each id has a standard pattern `M_NNN_S_PP`, where:
 * `M` is the major version
 * `NNN` is an incrementing id
-* `SS` is used in subsidiary repos amending the default transport protocol
-* `P` is used for patches and backports
+* `S` is used in subsidiary repos amending the default transport protocol
+* `PP` is used for patches and backports
 
 When you make a change to the serialization form of any object,
 you need to create a new sequential constant in `TransportVersions`,
 introduced in the same PR that adds the change, that increments
 the `NNN` component from the previous highest version,
 with other components  set to zero.
-For example, if the previous version number is `8_413_00_1`,
-the next version number should be `8_414_00_0`.
+For example, if the previous version number is `8_413_0_01`,
+the next version number should be `8_414_0_00`.
 
 Once you have defined your constant, you then need to use it
 in serialization code. If the transport version is at or above the new id,
@@ -166,7 +166,7 @@ also has that change, and knows about the patch backport ids and what they mean.
 
 Index version is a single incrementing version number for the index data format,
 metadata, and associated mappings. It is declared the same way as the
-transport version - with the pattern `M_NNN_SS_P`, for the major version, version id,
+transport version - with the pattern `M_NNN_S_PP`, for the major version, version id,
 subsidiary version id, and patch number respectively.
 
 Index version is stored in index metadata when an index is created,

libs/core/src/main/java/org/elasticsearch/core/CheckedSupplier.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.core;
+
+/**
+ * A {@link java.util.function.Supplier}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedSupplier<T, E extends Exception> {
+    T get() throws E;
+}

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java

@@ -52,10 +52,9 @@
  * <p>
  * A bit like Mockito but way more painful.
  */
-class DummyImplementations {
-
-    static class DummyLocaleServiceProvider extends LocaleServiceProvider {
+public class DummyImplementations {
 
+    public static class DummyLocaleServiceProvider extends LocaleServiceProvider {
         @Override
         public Locale[] getAvailableLocales() {
             throw unexpected();

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java

@@ -96,6 +96,9 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
 
     private static final Map<String, CheckAction> checkActions = Stream.concat(
         Stream.<Entry<String, CheckAction>>of(
+            entry("static_reflection", deniedToPlugins(RestEntitlementsCheckAction::staticMethodNeverEntitledViaReflection)),
+            entry("nonstatic_reflection", deniedToPlugins(RestEntitlementsCheckAction::nonstaticMethodNeverEntitledViaReflection)),
+            entry("constructor_reflection", deniedToPlugins(RestEntitlementsCheckAction::constructorNeverEntitledViaReflection)),
             entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
             entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
             entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
@@ -338,6 +341,11 @@ private static void systemExit() {
         System.exit(123);
     }
 
+    private static void staticMethodNeverEntitledViaReflection() throws Exception {
+        Method systemExit = System.class.getMethod("exit", int.class);
+        systemExit.invoke(null, 123);
+    }
+
     private static void createClassLoader() throws IOException {
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
@@ -348,6 +356,11 @@ private static void processBuilder_start() throws IOException {
         new ProcessBuilder("").start();
     }
 
+    private static void nonstaticMethodNeverEntitledViaReflection() throws Exception {
+        Method processBuilderStart = ProcessBuilder.class.getMethod("start");
+        processBuilderStart.invoke(new ProcessBuilder(""));
+    }
+
     private static void processBuilder_startPipeline() throws IOException {
         ProcessBuilder.startPipeline(List.of());
     }
@@ -386,6 +399,10 @@ private static void setHttpsConnectionProperties() {
         new DummyLocaleServiceProvider();
     }
 
+    private static void constructorNeverEntitledViaReflection() throws Exception {
+        DummyLocaleServiceProvider.class.getConstructor().newInstance();
+    }
+
     private static void breakIteratorProvider$() {
         new DummyBreakIteratorProvider();
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -14,6 +14,8 @@
 import com.sun.tools.attach.AttachNotSupportedException;
 import com.sun.tools.attach.VirtualMachine;
 
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.CheckedSupplier;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
 import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
@@ -22,8 +24,10 @@
 import org.elasticsearch.logging.Logger;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -144,30 +148,31 @@ private static String findAgentJar() {
      * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing
      */
     private static void selfTest() {
-        ensureCannotStartProcess();
-        ensureCanCreateTempFile();
+        ensureCannotStartProcess(ProcessBuilder::start);
+        ensureCanCreateTempFile(EntitlementBootstrap::createTempFile);
+
+        // Try again with reflection
+        ensureCannotStartProcess(EntitlementBootstrap::reflectiveStartProcess);
+        ensureCanCreateTempFile(EntitlementBootstrap::reflectiveCreateTempFile);
     }
 
-    private static void ensureCannotStartProcess() {
+    private static void ensureCannotStartProcess(CheckedConsumer<ProcessBuilder, ?> startProcess) {
         try {
             // The command doesn't matter; it doesn't even need to exist
-            new ProcessBuilder("").start();
+            startProcess.accept(new ProcessBuilder(""));
         } catch (NotEntitledException e) {
             logger.debug("Success: Entitlement protection correctly prevented process creation");
             return;
-        } catch (IOException e) {
+        } catch (Exception e) {
             throw new IllegalStateException("Failed entitlement protection self-test", e);
         }
         throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted");
     }
 
-    /**
-     * Originally {@code Security.selfTest}.
-     */
     @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
-    private static void ensureCanCreateTempFile() {
+    private static void ensureCanCreateTempFile(CheckedSupplier<Path, ?> createTempFile) {
         try {
-            Path p = Files.createTempFile(null, null);
+            Path p = createTempFile.get();
             p.toFile().deleteOnExit();
 
             // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally.
@@ -184,5 +189,24 @@ private static void ensureCanCreateTempFile() {
         logger.debug("Success: Entitlement protection correctly permitted temp file creation");
     }
 
+    @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
+    private static Path createTempFile() throws Exception {
+        return Files.createTempFile(null, null);
+    }
+
+    private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception {
+        try {
+            var start = ProcessBuilder.class.getMethod("start");
+            start.invoke(pb);
+        } catch (InvocationTargetException e) {
+            throw (Exception) e.getCause();
+        }
+    }
+
+    private static Path reflectiveCreateTempFile() throws Exception {
+        return (Path) Files.class.getMethod("createTempFile", String.class, String.class, FileAttribute[].class)
+            .invoke(null, null, null, new FileAttribute<?>[0]);
+    }
+
     private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class);
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -9,10 +9,8 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.FileEntitlement;
 
-import java.io.File;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -51,20 +49,10 @@ boolean canRead(Path path) {
         return checkPath(normalize(path), readPaths);
     }
 
-    @SuppressForbidden(reason = "Explicitly checking File apis")
-    boolean canRead(File file) {
-        return checkPath(normalize(file.toPath()), readPaths);
-    }
-
     boolean canWrite(Path path) {
         return checkPath(normalize(path), writePaths);
     }
 
-    @SuppressForbidden(reason = "Explicitly checking File apis")
-    boolean canWrite(File file) {
-        return checkPath(normalize(file.toPath()), writePaths);
-    }
-
     private static String normalize(Path path) {
         return path.toAbsolutePath().normalize().toString();
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -169,23 +169,7 @@ private static void validateEntitlementsPerModule(String sourceName, String modu
     }
 
     public void checkStartProcess(Class<?> callerClass) {
-        neverEntitled(callerClass, "start process");
-    }
-
-    private void neverEntitled(Class<?> callerClass, String operationDescription) {
-        var requestingClass = requestingClass(callerClass);
-        if (isTriviallyAllowed(requestingClass)) {
-            return;
-        }
-
-        throw new NotEntitledException(
-            Strings.format(
-                "Not entitled: caller [%s], module [%s], operation [%s]",
-                callerClass,
-                requestingClass.getModule() == null ? "<none>" : requestingClass.getModule().getName(),
-                operationDescription
-            )
-        );
+        neverEntitled(callerClass, () -> "start process");
     }
 
     /**
@@ -241,31 +225,9 @@ public void checkChangeNetworkHandling(Class<?> callerClass) {
         checkChangeJVMGlobalState(callerClass);
     }
 
-    /**
-     * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions
-     */
-    public void checkReadSensitiveNetworkInformation(Class<?> callerClass) {
-        neverEntitled(callerClass, "access sensitive network information");
-    }
-
     @SuppressForbidden(reason = "Explicitly checking File apis")
     public void checkFileRead(Class<?> callerClass, File file) {
-        var requestingClass = requestingClass(callerClass);
-        if (isTriviallyAllowed(requestingClass)) {
-            return;
-        }
-
-        ModuleEntitlements entitlements = getEntitlements(requestingClass);
-        if (entitlements.fileAccess().canRead(file) == false) {
-            throw new NotEntitledException(
-                Strings.format(
-                    "Not entitled: caller [%s], module [%s], entitlement [file], operation [read], path [%s]",
-                    callerClass,
-                    requestingClass.getModule(),
-                    file
-                )
-            );
-        }
+        checkFileRead(callerClass, file.toPath());
     }
 
     public void checkFileRead(Class<?> callerClass, Path path) {
@@ -289,22 +251,7 @@ public void checkFileRead(Class<?> callerClass, Path path) {
 
     @SuppressForbidden(reason = "Explicitly checking File apis")
     public void checkFileWrite(Class<?> callerClass, File file) {
-        var requestingClass = requestingClass(callerClass);
-        if (isTriviallyAllowed(requestingClass)) {
-            return;
-        }
-
-        ModuleEntitlements entitlements = getEntitlements(requestingClass);
-        if (entitlements.fileAccess().canWrite(file) == false) {
-            throw new NotEntitledException(
-                Strings.format(
-                    "Not entitled: caller [%s], module [%s], entitlement [file], operation [write], path [%s]",
-                    callerClass,
-                    requestingClass.getModule(),
-                    file
-                )
-            );
-        }
+        checkFileWrite(callerClass, file.toPath());
     }
 
     public void checkFileWrite(Class<?> callerClass, Path path) {

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java

@@ -238,7 +238,6 @@ public void testRequestingClassFastPath() throws IOException, ClassNotFoundExcep
     }
 
     public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoundException {
-        var agentsClass = new TestAgent();
         var entitlementsClass = makeClassInItsOwnModule();    // A class in the entitlements library itself
         var requestingClass = makeClassInItsOwnModule();      // This guy is always the right answer
         var instrumentedClass = makeClassInItsOwnModule();    // The class that called the check method
@@ -365,13 +364,6 @@ private static Class<?> makeClassInItsOwnModule() throws IOException, ClassNotFo
         return layer.findLoader("org.example.plugin").loadClass("q.B");
     }
 
-    private static Class<?> makeClassInItsOwnUnnamedModule() throws IOException, ClassNotFoundException {
-        final Path home = createTempDir();
-        Path jar = createMockPluginJar(home);
-        var layer = createLayerForJar(jar, "org.example.plugin");
-        return layer.findLoader("org.example.plugin").loadClass("q.B");
-    }
-
     private static PolicyManager policyManager(String agentsPackageName, Module entitlementsModule) {
         return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", agentsPackageName, entitlementsModule);
     }