Merge branch 'main' into fixChunkTest

Author: elasticmachine

docs/reference/ingest/apis/simulate-ingest.asciidoc

@@ -265,8 +265,8 @@ Definition of a mapping that will be merged into the index's mapping for validat
 
 [[simulate-ingest-api-pre-existing-pipelines-ex]]
 ===== Use pre-existing pipeline definitions
-In this example the index `index` has a default pipeline called `my-pipeline` and a final
-pipeline called `my-final-pipeline`. Since both documents are being ingested into `index`,
+In this example the index `my-index` has a default pipeline called `my-pipeline` and a final
+pipeline called `my-final-pipeline`. Since both documents are being ingested into `my-index`,
 both pipelines are executed using the pipeline definitions that are already in the system.
 
 [source,console]

docs/reference/modules/indices/circuit_breaker.asciidoc

@@ -2,7 +2,16 @@
 === Circuit breaker settings
 [[circuit-breaker-description]]
 // tag::circuit-breaker-description-tag[]
-{es} contains multiple circuit breakers used to prevent operations from causing an OutOfMemoryError. Each breaker specifies a limit for how much memory it can use. Additionally, there is a parent-level breaker that specifies the total amount of memory that can be used across all breakers.
+{es} contains multiple circuit breakers used to prevent operations from using an excessive amount of memory. Each breaker tracks the memory
+used by certain operations and specifies a limit for how much memory it may track. Additionally, there
+is a parent-level breaker that specifies the total amount of memory that may be tracked across all breakers.
+
+When a circuit breaker reaches its limit, {es} will reject further operations. See <<circuit-breaker-errors>> for information about errors
+raised by circuit breakers.
+
+Circuit breakers do not track all memory usage in {es} and therefore provide only incomplete protection against excessive memory usage. If
+{es} uses too much memory then it may suffer from performance issues and nodes may even fail with an `OutOfMemoryError`. See
+<<high-jvm-memory-pressure>> for help with troubleshooting high heap usage.
 
 Except where noted otherwise, these settings can be dynamically updated on a
 live cluster with the <<cluster-update-settings,cluster-update-settings>> API.

libs/entitlement/tools/build.gradle

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+plugins {
+  id 'java'
+}
+
+group = 'org.elasticsearch.entitlement.tools'
+

libs/entitlement/tools/common/build.gradle

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.tools;
+
+import java.io.IOException;
+import java.lang.module.ModuleDescriptor;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class Utils {
+
+    public static Map<String, Set<String>> findModuleExports(FileSystem fs) throws IOException {
+        var modulesExports = new HashMap<String, Set<String>>();
+        try (var stream = Files.walk(fs.getPath("modules"))) {
+            stream.filter(p -> p.getFileName().toString().equals("module-info.class")).forEach(x -> {
+                try (var is = Files.newInputStream(x)) {
+                    var md = ModuleDescriptor.read(is);
+                    modulesExports.put(
+                        md.name(),
+                        md.exports()
+                            .stream()
+                            .filter(e -> e.isQualified() == false)
+                            .map(ModuleDescriptor.Exports::source)
+                            .collect(Collectors.toSet())
+                    );
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        }
+        return modulesExports;
+    }
+
+}

libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java

@@ -0,0 +1,61 @@
+plugins {
+  id 'application'
+}
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.publish'
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /asm-.*/, to: 'asm'
+}
+
+group = 'org.elasticsearch.entitlement.tools'
+
+ext {
+  javaMainClass = "org.elasticsearch.entitlement.tools.securitymanager.scanner.Main"
+}
+
+application {
+  mainClass.set(javaMainClass)
+  applicationDefaultJvmArgs = [
+    '--add-exports', 'java.base/sun.security.util=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.net.spi=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+    '--add-opens', 'java.base/javax.crypto=ALL-UNNAMED',
+    '--add-opens', 'java.base/javax.security.auth=ALL-UNNAMED',
+    '--add-opens', 'java.base/jdk.internal.logger=ALL-UNNAMED',
+    '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+    '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+    '--add-opens', 'java.logging/java.util.logging=ALL-UNNAMED',
+    '--add-opens', 'java.logging/sun.util.logging.internal=ALL-UNNAMED',
+    '--add-opens', 'java.naming/javax.naming.ldap.spi=ALL-UNNAMED',
+    '--add-opens', 'java.rmi/sun.rmi.runtime=ALL-UNNAMED',
+    '--add-opens', 'jdk.dynalink/jdk.dynalink=ALL-UNNAMED',
+    '--add-opens', 'jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED',
+    '--add-opens', 'java.desktop/sun.awt=ALL-UNNAMED',
+    '--add-opens', 'java.sql.rowset/javax.sql.rowset.spi=ALL-UNNAMED',
+    '--add-opens', 'java.sql/java.sql=ALL-UNNAMED',
+    '--add-opens', 'java.xml.crypto/com.sun.org.apache.xml.internal.security.utils=ALL-UNNAMED'
+  ]
+}
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  compileOnly(project(':libs:core'))
+  implementation 'org.ow2.asm:asm:9.7'
+  implementation 'org.ow2.asm:asm-util:9.7'
+  implementation(project(':libs:entitlement:tools:common'))
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreMissingClasses()
+}

libs/entitlement/tools/securitymanager-scanner/build.gradle

@@ -0,0 +1,26 @@
+Copyright (c) 2012 France Télécom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.

libs/entitlement/tools/securitymanager-scanner/licenses/asm-LICENSE.txt

@@ -0,0 +1 @@
+ 

libs/entitlement/tools/securitymanager-scanner/licenses/asm-NOTICE.txt

@@ -0,0 +1,47 @@
+This tool scans the JDK on which it is running, looking for any location where `SecurityManager` is currently used, thus giving us a list of "entry points" inside the JDK where security checks are currently happening.
+
+More in detail, the tool scans for calls to any `SecurityManager` method starting with `check` (e.g. `checkWrite`). The tool treats the generic `checkPermission` method a little bit differently: `checkPermission` accepts a generic `Permission` object, it tries to read the permission type and permission name to give more information about it, trying to match two patterns that are used frequently inside the JDK:
+
+Pattern 1: private static permission field
+
+```java
+private static final RuntimePermission INET_ADDRESS_RESOLVER_PERMISSION =
+new RuntimePermission("inetAddressResolverProvider");
+...
+sm.checkPermission(INET_ADDRESS_RESOLVER_PERMISSION);
+```
+Pattern 2: direct object creation
+
+```java
+sm.checkPermission(new LinkPermission("symbolic"));
+```
+
+The tool will recognize this pattern, and report the permission type and name alongside the `checkPermission` entry point (type `RuntimePermission` and name `inetAddressResolverProvider` in the first case, type `LinkPermission` and name `symbolic` in the second).
+
+This allows to give more information (either a specific type like `LinkPermission`, or a specific name like `inetAddressResolverProvider`) to generic `checkPermission` to help in deciding how to classify the permission check. The 2 patterns work quite well and cover roughly 90% of the cases.
+
+In order to run the tool, use:
+```shell
+./gradlew :libs:entitlement:tools:securitymanager-scanner:run
+```
+The output of the tool is a CSV file, with one line for each entry-point, columns separated by `TAB`
+
+The columns are:
+1. Module name
+2. File name (from source root)
+3. Line number
+4. Fully qualified class name (ASM style, with `/` separators)
+5. Method name
+6. Method descriptor (ASM signature)
+6. Visibility (PUBLIC/PUBLIC-METHOD/PRIVATE)
+7. Check detail 1 (method name, or in case of checkPermission, permission name. Might be `MISSING`)
+8. Check detail 2 (in case of checkPermission, the argument type (`Permission` subtype). Might be `MISSING`)
+
+Examples:
+```
+java.base       sun/nio/ch/DatagramChannelImpl.java     1360    sun/nio/ch/DatagramChannelImpl  connect (Ljava/net/SocketAddress;Z)Ljava/nio/channels/DatagramChannel;  PRIVATE checkConnect
+```
+or
+```
+java.base       java/net/ResponseCache.java     118     java/net/ResponseCache  setDefault      (Ljava/net/ResponseCache;)V     PUBLIC  setResponseCache        java/net/NetPermission
+```

libs/entitlement/tools/securitymanager-scanner/src/README.md

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.tools.securitymanager.scanner;
+
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.entitlement.tools.Utils;
+import org.objectweb.asm.ClassReader;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+public class Main {
+
+    static final Set<String> excludedModules = Set.of("java.desktop");
+
+    private static void identifySMChecksEntryPoints() throws IOException {
+
+        FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/"));
+
+        var moduleExports = Utils.findModuleExports(fs);
+
+        var callers = new HashMap<String, List<SecurityCheckClassVisitor.CallerInfo>>();
+        var visitor = new SecurityCheckClassVisitor(callers);
+
+        try (var stream = Files.walk(fs.getPath("modules"))) {
+            stream.filter(x -> x.toString().endsWith(".class")).forEach(x -> {
+                var moduleName = x.subpath(1, 2).toString();
+                if (excludedModules.contains(moduleName) == false) {
+                    try {
+                        ClassReader cr = new ClassReader(Files.newInputStream(x));
+                        visitor.setCurrentModule(moduleName, moduleExports.get(moduleName));
+                        var path = x.getNameCount() > 3 ? x.subpath(2, x.getNameCount() - 1).toString() : "";
+                        visitor.setCurrentSourcePath(path);
+                        cr.accept(visitor, 0);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            });
+        }
+
+        printToStdout(callers);
+    }
+
+    @SuppressForbidden(reason = "This simple tool just prints to System.out")
+    private static void printToStdout(HashMap<String, List<SecurityCheckClassVisitor.CallerInfo>> callers) {
+        for (var kv : callers.entrySet()) {
+            for (var e : kv.getValue()) {
+                System.out.println(toString(kv.getKey(), e));
+            }
+        }
+    }
+
+    private static final String SEPARATOR = "\t";
+
+    private static String toString(String calleeName, SecurityCheckClassVisitor.CallerInfo callerInfo) {
+        var s = callerInfo.moduleName() + SEPARATOR + callerInfo.source() + SEPARATOR + callerInfo.line() + SEPARATOR + callerInfo
+            .className() + SEPARATOR + callerInfo.methodName() + SEPARATOR + callerInfo.methodDescriptor() + SEPARATOR;
+
+        if (callerInfo.externalAccess().contains(SecurityCheckClassVisitor.ExternalAccess.METHOD)
+            && callerInfo.externalAccess().contains(SecurityCheckClassVisitor.ExternalAccess.CLASS)) {
+            s += "PUBLIC";
+        } else if (callerInfo.externalAccess().contains(SecurityCheckClassVisitor.ExternalAccess.METHOD)) {
+            s += "PUBLIC-METHOD";
+        } else {
+            s += "PRIVATE";
+        }
+
+        if (callerInfo.runtimePermissionType() != null) {
+            s += SEPARATOR + callerInfo.runtimePermissionType();
+        } else if (calleeName.equals("checkPermission")) {
+            s += SEPARATOR + "MISSING"; // missing information
+        } else {
+            s += SEPARATOR + calleeName;
+        }
+
+        if (callerInfo.permissionType() != null) {
+            s += SEPARATOR + callerInfo.permissionType();
+        } else if (calleeName.equals("checkPermission")) {
+            s += SEPARATOR + "MISSING"; // missing information
+        } else {
+            s += SEPARATOR;
+        }
+        return s;
+    }
+
+    public static void main(String[] args) throws IOException {
+        identifySMChecksEntryPoints();
+    }
+}