Merge branch 'main' into data-stream-settings-rest-action

Author: masseyke

docs/changelog/127563.yaml

@@ -0,0 +1,6 @@
+pr: 127563
+summary: "ESQL: Avoid unintended attribute removal"
+area: ES|QL
+type: bug
+issues:
+  - 127468

docs/changelog/128161.yaml

@@ -0,0 +1,6 @@
+pr: 128161
+summary: Fix system data streams incorrectly showing up in the list of template validation
+  problems
+area: Data streams
+type: bug
+issues: []

docs/reference/query-languages/esql/esql-functions-operators.md

@@ -12,56 +12,67 @@ mapped_pages:
 ## Functions overview [esql-functions]
 
 ::::{dropdown} Aggregate functions
+:open:
 :::{include} _snippets/lists/aggregation-functions.md
 :::
 ::::
 
 ::::{dropdown} Grouping functions
+:open:
 :::{include} _snippets/lists/grouping-functions.md
 :::
 ::::
 
 ::::{dropdown} Conditional functions and expressions
+:open:
 :::{include} _snippets/lists/conditional-functions-and-expressions.md
 :::
 ::::
 
 ::::{dropdown} Date and time functions
+:open:
 :::{include} _snippets/lists/date-time-functions.md
 :::
 ::::
 
 ::::{dropdown} IP functions
+:open:
 :::{include} _snippets/lists/ip-functions.md
 :::
 ::::
 
 ::::{dropdown} Math functions
+:open:
 :::{include} _snippets/lists/math-functions.md
 :::
 ::::
 
 ::::{dropdown} Search functions
+:open:
 :::{include} _snippets/lists/search-functions.md
 :::
 ::::
 
 ::::{dropdown} Spatial functions
+:open:
 :::{include} _snippets/lists/spatial-functions.md
 :::
 ::::
 
 ::::{dropdown} String functions
+:open:
 :::{include} _snippets/lists/string-functions.md
 :::
 ::::
 
 ::::{dropdown} Type conversion functions
+:open:
 :::{include} _snippets/lists/type-conversion-functions.md
 :::
 ::::
 
 ::::{dropdown} Multi value functions
+:open:
 :::{include} _snippets/lists/mv-functions.md
 :::
 ::::
@@ -70,6 +81,7 @@ mapped_pages:
 ## Operators overview [esql-operators-overview]
 
 ::::{dropdown} Operators
+:open:
 :::{include} _snippets/lists/operators.md
 :::
 ::::

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -43,14 +43,14 @@ public record BootstrapArgs(
         Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
         PathLookup pathLookup,
         Map<String, Path> sourcePaths,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(scopeResolver);
             requireNonNull(pathLookup);
             requireNonNull(sourcePaths);
-            requireNonNull(suppressFailureLogClasses);
+            requireNonNull(suppressFailureLogPackages);
         }
     }
 
@@ -78,7 +78,7 @@ public static BootstrapArgs bootstrapArgs() {
      * @param tempDir        the temp directory for Elasticsearch
      * @param logsDir        the log directory for Elasticsearch
      * @param pidFile        path to a pid file for Elasticsearch, or {@code null} if one was not specified
-     * @param suppressFailureLogClasses   classes for which we do not need or want to log Entitlements failures
+     * @param suppressFailureLogPackages   packages for which we do not need or want to log Entitlements failures
      */
     public static void bootstrap(
         Policy serverPolicyPatch,
@@ -95,7 +95,7 @@ public static void bootstrap(
         Path logsDir,
         Path tempDir,
         Path pidFile,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
@@ -119,7 +119,7 @@ public static void bootstrap(
                 settingResolver
             ),
             sourcePaths,
-            suppressFailureLogClasses
+            suppressFailureLogPackages
         );
         exportInitializationToAgent();
         loadAgent(findAgentJar());

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -90,7 +90,7 @@ private static PolicyManager createPolicyManager() {
             EntitlementBootstrap.bootstrapArgs().sourcePaths(),
             ENTITLEMENTS_MODULE,
             pathLookup,
-            bootstrapArgs.suppressFailureLogClasses()
+            bootstrapArgs.suppressFailureLogPackages()
         );
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -256,7 +256,7 @@ ModuleEntitlements policyEntitlements(String componentName, Path componentPath,
     private final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, PolicyScope> scopeResolver;
     private final PathLookup pathLookup;
-    private final Set<Class<?>> mutedClasses;
+    private final Set<Package> suppressFailureLogPackages;
 
     public static final String ALL_UNNAMED = "ALL-UNNAMED";
 
@@ -311,7 +311,7 @@ public PolicyManager(
         Map<String, Path> sourcePaths,
         Module entitlementsModule,
         PathLookup pathLookup,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy));
         this.apmAgentEntitlements = apmAgentEntitlements;
@@ -322,7 +322,7 @@ public PolicyManager(
         this.sourcePaths = sourcePaths;
         this.entitlementsModule = entitlementsModule;
         this.pathLookup = requireNonNull(pathLookup);
-        this.mutedClasses = suppressFailureLogClasses;
+        this.suppressFailureLogPackages = suppressFailureLogPackages;
 
         List<ExclusiveFileEntitlement> exclusiveFileEntitlements = new ArrayList<>();
         for (var e : serverEntitlements.entrySet()) {
@@ -688,8 +688,8 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
 
     private void notEntitled(String message, Class<?> callerClass, ModuleEntitlements entitlements) {
         var exception = new NotEntitledException(message);
-        // Don't emit a log for muted classes, e.g. classes containing self tests
-        if (mutedClasses.contains(callerClass) == false) {
+        // Don't emit a log for suppressed packages, e.g. packages containing self tests
+        if (suppressFailureLogPackages.contains(callerClass.getPackage()) == false) {
             entitlements.logger().warn("Not entitled: {}", message, exception);
         }
         throw exception;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java

@@ -81,6 +81,10 @@ public static Map<String, Policy> createPluginPolicies(
         return pluginPolicies;
     }
 
+    /**
+     * @throws PolicyParserException if the supplied policy is formatted incorrectly
+     * @throws IllegalStateException for any other error parsing the patch, such as nonexistent module names
+     */
     public static Policy parseEncodedPolicyIfExists(
         String encodedPolicy,
         String version,
@@ -106,11 +110,8 @@ public static Policy parseEncodedPolicyIfExists(
                         version
                     );
                 }
-            } catch (Exception ex) {
-                logger.warn(
-                    Strings.format("Found a policy patch with invalid content. The patch will not be applied. Layer [%s]", layerName),
-                    ex
-                );
+            } catch (Exception e) {
+                throw new IllegalStateException("Unable to parse policy patch for layer [" + layerName + "]", e);
             }
         }
         return null;

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtilsTests.java

@@ -134,6 +134,7 @@ public void testNoPatchWithVersionMismatch() {
 
     public void testNoPatchWithValidationError() {
 
+        // Nonexistent module names
         var policyPatch = """
             versions:
               - 9.0.0
@@ -149,13 +150,15 @@ public void testNoPatchWithValidationError() {
             StandardCharsets.UTF_8
         );
 
-        var policy = PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of());
-
-        assertThat(policy, nullValue());
+        assertThrows(
+            IllegalStateException.class,
+            () -> PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of())
+        );
     }
 
     public void testNoPatchWithParsingError() {
 
+        // no <version> or <policy> field
         var policyPatch = """
             entitlement-module-name:
               - load_native_libraries
@@ -167,9 +170,10 @@ public void testNoPatchWithParsingError() {
             StandardCharsets.UTF_8
         );
 
-        var policy = PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of());
-
-        assertThat(policy, nullValue());
+        assertThrows(
+            IllegalStateException.class,
+            () -> PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of())
+        );
     }
 
     public void testMergeScopes() {

muted-tests.yml

@@ -438,9 +438,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {lookup-join.MvJoinKeyOnTheLookupIndex ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/128030
-- class: org.elasticsearch.compute.aggregation.PercentileIntGroupingAggregatorFunctionTests
-  method: testManyInitialManyPartialFinalRunner
-  issue: https://github.com/elastic/elasticsearch/issues/128092
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test042KeystorePermissionsAreCorrect
   issue: https://github.com/elastic/elasticsearch/issues/128018
@@ -453,6 +450,18 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test041AmazonCaCertsAreInTheKeystore
   issue: https://github.com/elastic/elasticsearch/issues/128006
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test130JavaHasCorrectOwnership
+  issue: https://github.com/elastic/elasticsearch/issues/128174
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test600Interrupt
+  issue: https://github.com/elastic/elasticsearch/issues/128144
+- class: org.elasticsearch.packaging.test.EnrollmentProcessTests
+  method: test20DockerAutoFormCluster
+  issue: https://github.com/elastic/elasticsearch/issues/128113
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test121CanUseStackLoggingConfig
+  issue: https://github.com/elastic/elasticsearch/issues/128165
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.repository_verify_integrity.json

@@ -5,7 +5,7 @@
       "description":"Verifies the integrity of the contents of a snapshot repository"
     },
     "stability":"experimental",
-    "visibility":"private",
+    "visibility":"public",
     "headers": {
       "accept": [
         "application/json"