Merge branch 'main' into allow-opt-out-ilm

Author: nielsbauman

docs/changelog/136828.yaml

@@ -0,0 +1,5 @@
+pr: 136828
+summary: Can match phase coordinator duration APM metric
+area: Search
+type: enhancement
+issues: []

docs/changelog/136996.yaml

@@ -0,0 +1,5 @@
+pr: 136996
+summary: Add periodic PKC JWK set reloading capability to JWT realm
+area: Security
+type: enhancement
+issues: []

docs/reference/elasticsearch/configuration-reference/security-settings.md

@@ -1523,7 +1523,19 @@ $$$jwt-claim-pattern-principal$$$
 :   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) Specifies the time-to-live for the period of time to cache JWT entries. JWTs can only be cached if client authentication is successful (or disabled). Uses the standard {{es}} [time units](/reference/elasticsearch/rest-apis/api-conventions.md#time-units). If clients use a different JWT for every request, set to `0` to disable the JWT cache. Defaults to `20m`.
 
 `pkc_jwkset_path` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
-:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) The file name or URL to a JSON Web Key Set (JWKS) with the public key material that the JWT Realm uses for verifying token signatures. A value is considered a file name if it does not begin with `https`. The file name is resolved relative to the {{es}} configuration directory. If a URL is provided, then it must begin with `https://` (`http://` is not supported). {{es}} automatically caches the JWK set and will attempt to refresh the JWK set upon signature verification failure, as this might indicate that the JWT Provider has rotated the signing keys.
+:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) The file name or URL to a JSON Web Key Set (JWKS) with the public key material that the JWT Realm uses for verifying token signatures. A value is considered a file name if it does not begin with `https`. The file name is resolved relative to the {{es}} configuration directory. If a URL is provided, then it must begin with `https://` (`http://` is not supported). {{es}} automatically caches the JWK set and will attempt to refresh the JWK set upon signature verification failure, as this might indicate that the JWT Provider has rotated the signing keys. Background JWKS reloading can also be configured with the setting `pkc_jwkset_reload.enabled`. This ensures that rotated keys are automatically discovered and used to verify JWT signatures.
+
+`pkc_jwkset_reload.enabled` {applies_to}`stack: ga 9.3` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
+:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) Indicates whether JWKS background reloading is enabled. Defaults to `false`.
+
+`pkc_jwkset_reload.file_interval` {applies_to}`stack: ga 9.3` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
+:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) Specifies the reload interval for file-based JWKS. Defaults to `5m`.
+
+`pkc_jwkset_reload.url_interval_min` {applies_to}`stack: ga 9.3` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
+:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) Specifies the minimum reload interval for URL-based JWKS. The `Expires` and `Cache-Control` HTTP response headers inform the reload interval. This configuration setting is the lower bound of what is considered, and it is also the default interval in the absence of useful response headers. Defaults to `1h`.
+
+`pkc_jwkset_reload.url_interval_max` {applies_to}`stack: ga 9.3` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
+:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) Specifies the maximum reload interval for URL-based JWKS. This configuration setting is the upper bound of what is considered from header responses (`5d`).
 
 `hmac_jwkset` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
 :   ([Secure](docs-content://deploy-manage/security/secure-settings.md)) Contents of a JSON Web Key Set (JWKS), including the secret key that the JWT realm uses to verify token signatures. This format supports multiple keys and optional attributes, and is preferred over the `hmac_key` setting. Cannot be used in conjunction with the `hmac_key` setting. Refer to [Configure {{es}} to use a JWT realm](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md).

muted-tests.yml

@@ -507,6 +507,12 @@ tests:
 - class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
   method: testSearchableSnapshotAction
   issue: https://github.com/elastic/elasticsearch/issues/137167
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=indices.validate_query/20_query_string/validate_query with query_string parameters}
+  issue: https://github.com/elastic/elasticsearch/issues/137391
+- class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
+  method: testILMDownsampleRollingRestart
+  issue: https://github.com/elastic/elasticsearch/issues/136585
 
 # Examples:
 #

qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/100_sampling_with_reroute.yml

@@ -0,0 +1,186 @@
+---
+"Test get sample with multiple reroutes":
+  - requires:
+      cluster_features: [ "random_sampling" ]
+      reason: requires feature 'random_sampling' to get random samples
+
+  - do:
+      ingest.put_pipeline:
+        id: pipeline1
+        body:  >
+          {
+             "processors" : [
+              {
+                "set" : {
+                  "field": "message",
+                  "value": "set by pipeline1"
+                }
+              },
+              {
+                "reroute" : {
+                  "destination": "foo.bar"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: pipeline2
+        body:  >
+          {
+             "processors" : [
+              {
+                "set" : {
+                  "field": "message",
+                  "value": "set by pipeline2"
+                }
+              },
+              {
+                "reroute" : {
+                  "destination": "foo.bar.baz"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.put_index_template:
+        name: my-template1
+        body:
+          index_patterns: [foo]
+          template:
+            settings:
+              default_pipeline: pipeline1
+              index.number_of_shards: 1
+              index.number_of_replicas: 0
+            mappings:
+              dynamic: strict
+              properties:
+                message:
+                  type: text
+          data_stream: {}
+  - match: { acknowledged: true }
+
+  - do:
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [foo.bar]
+          template:
+            settings:
+              default_pipeline: pipeline2
+              index.number_of_shards: 1
+              index.number_of_replicas: 0
+            mappings:
+              dynamic: strict
+              properties:
+                message:
+                  type: text
+          data_stream: {}
+  - match: { acknowledged: true }
+
+  - do:
+      indices.put_index_template:
+        name: my-template3
+        body:
+          index_patterns: [foo.bar.baz]
+          template:
+            settings:
+              index.number_of_shards: 1
+              index.number_of_replicas: 0
+            mappings:
+              dynamic: strict
+              properties:
+                message:
+                  type: text
+          data_stream: {}
+  - match: { acknowledged: true }
+
+  - do:
+      indices.create_data_stream:
+        name: foo
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: foo.bar
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: foo.bar.baz
+  - is_true: acknowledged
+
+  - do:
+      indices.rollover:
+        alias: foo.bar.baz
+        wait_for_active_shards: 1
+  - match: { rolled_over: true }
+
+  - do:
+      indices.put_sample_configuration:
+        index: foo
+        body:
+          rate: 1.0
+          max_samples: 100
+
+  - do:
+      indices.put_sample_configuration:
+        index: foo.bar
+        body:
+          rate: 1.0
+          max_samples: 100
+
+  - do:
+      indices.put_sample_configuration:
+        index: foo.bar.baz
+        body:
+          rate: 1.0
+          max_samples: 100
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "create":{"_index": "foo" } }'
+          - '{"@timestamp": 123456, "message": "This is the original message"}'
+          - '{ "create":{"_index": "foo" } }'
+          - '{"@timestamp": 123456, "message": "This is the original message"}'
+  - match: { errors: false }
+
+  - do:
+      indices.get_sample:
+        index: foo
+  - length: { sample: 2 }
+  - match: { sample.0.index: "foo" }
+  - match: { sample.0.source.message: "This is the original message" }
+  - match: { sample.1.source.message: "This is the original message" }
+
+  - do:
+      indices.get_sample:
+        index: foo.bar
+  - length: { sample: 2 }
+  - match: { sample.0.index: "foo.bar" }
+  - match: { sample.0.source.message: "This is the original message" }
+  - match: { sample.1.source.message: "This is the original message" }
+
+  - do:
+      indices.get_sample:
+        index: foo.bar.baz
+  - length: { sample: 2 }
+  - match: { sample.0.index: "foo.bar.baz" }
+  - match: { sample.0.source.message: "This is the original message" }
+  - match: { sample.1.source.message: "This is the original message" }
+
+---
+teardown:
+  - requires:
+      cluster_features: [ "random_sampling" ]
+      reason: requires feature 'random_sampling' to get random samples
+
+  - do:
+      indices.delete_data_stream:
+        name: foo*

server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.query.CoordinatorRewriteContext;
 import org.elasticsearch.index.query.CoordinatorRewriteContextProvider;
+import org.elasticsearch.rest.action.search.SearchResponseMetrics;
 import org.elasticsearch.search.CanMatchShardResponse;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.SearchShardTarget;
@@ -58,6 +59,8 @@
  */
 final class CanMatchPreFilterSearchPhase {
 
+    private static final String PHASE_NAME = "can_match";
+
     private final Logger logger;
     private final SearchRequest request;
     private final List<SearchShardIterator> shardsIts;
@@ -136,20 +139,35 @@ public static SubscribableListener<List<SearchShardIterator>> execute(
         TransportSearchAction.SearchTimeProvider timeProvider,
         SearchTask task,
         boolean requireAtLeastOneMatch,
-        CoordinatorRewriteContextProvider coordinatorRewriteContextProvider
+        CoordinatorRewriteContextProvider coordinatorRewriteContextProvider,
+        SearchResponseMetrics searchResponseMetrics
     ) {
         if (shardsIts.isEmpty()) {
             return SubscribableListener.newSucceeded(List.of());
         }
         final SubscribableListener<List<SearchShardIterator>> listener = new SubscribableListener<>();
+        long phaseStartTimeInNanos = System.nanoTime();
+
+        listener.addListener(new ActionListener<>() {
+            @Override
+            public void onResponse(List<SearchShardIterator> shardsIts) {
+                searchResponseMetrics.recordSearchPhaseDuration(PHASE_NAME, System.nanoTime() - phaseStartTimeInNanos);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                // do not record duration of failed phases
+            }
+        });
+
         // Note that the search is failed when this task is rejected by the executor
         executor.execute(new AbstractRunnable() {
             @Override
             public void onFailure(Exception e) {
                 if (logger.isDebugEnabled()) {
                     logger.debug(() -> format("Failed to execute [%s] while running [can_match] phase", request), e);
                 }
-                listener.onFailure(new SearchPhaseExecutionException("can_match", "start", e, ShardSearchFailure.EMPTY_ARRAY));
+                listener.onFailure(new SearchPhaseExecutionException(PHASE_NAME, "start", e, ShardSearchFailure.EMPTY_ARRAY));
             }
 
             @Override
@@ -249,7 +267,7 @@ private synchronized void consumeResult(int shardIndex, boolean canMatch, MinAnd
 
     private void checkNoMissingShards(List<SearchShardIterator> shards) {
         assert assertSearchCoordinationThread();
-        SearchPhase.doCheckNoMissingShards("can_match", request, shards);
+        SearchPhase.doCheckNoMissingShards(PHASE_NAME, request, shards);
     }
 
     private Map<SendingTarget, List<SearchShardIterator>> groupByNode(List<SearchShardIterator> shards) {
@@ -386,7 +404,7 @@ public void onFailure(Exception e) {
             if (logger.isDebugEnabled()) {
                 logger.debug(() -> format("Failed to execute [%s] while running [can_match] phase", request), e);
             }
-            listener.onFailure(new SearchPhaseExecutionException("can_match", "round", e, ShardSearchFailure.EMPTY_ARRAY));
+            listener.onFailure(new SearchPhaseExecutionException(PHASE_NAME, "round", e, ShardSearchFailure.EMPTY_ARRAY));
         }
     }
 

server/src/main/java/org/elasticsearch/action/search/SearchRequest.java

@@ -163,6 +163,11 @@ public boolean allowsRemoteIndices() {
         return true;
     }
 
+    @Override
+    public boolean allowsCrossProject() {
+        return true;
+    }
+
     /**
      * Creates a new sub-search request starting from the original search request that is provided.
      * For internal use only, allows to fork a search request into multiple search requests that will be executed independently.

server/src/main/java/org/elasticsearch/action/search/SearchResponse.java

@@ -609,13 +609,18 @@ public Clusters(StreamInput in) throws IOException {
         }
 
         public Clusters(Map<String, Cluster> clusterInfoMap) {
+            this(clusterInfoMap, true);
+        }
+
+        public Clusters(Map<String, Cluster> clusterInfoMap, boolean ccsMinimizeRoundtrips) {
             assert clusterInfoMap.size() > 0 : "this constructor should not be called with an empty Cluster info map";
             this.total = clusterInfoMap.size();
-            this.clusterInfo = clusterInfoMap;
+            this.clusterInfo = ConcurrentCollections.newConcurrentMap();
+            this.clusterInfo.putAll(clusterInfoMap);
             this.successful = getClusterStateCount(Cluster.Status.SUCCESSFUL);
             this.skipped = getClusterStateCount(Cluster.Status.SKIPPED);
             // should only be called if "details" section of fromXContent is present (for ccsMinimizeRoundtrips)
-            this.ccsMinimizeRoundtrips = true;
+            this.ccsMinimizeRoundtrips = ccsMinimizeRoundtrips;
         }
 
         @Override

server/src/main/java/org/elasticsearch/action/search/SearchShardsRequest.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.LegacyActionRequest;
+import org.elasticsearch.action.ResolvedIndexExpressions;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -43,6 +44,8 @@ public final class SearchShardsRequest extends LegacyActionRequest implements In
 
     private final String clusterAlias;
 
+    private ResolvedIndexExpressions resolvedIndexExpressions;
+
     public SearchShardsRequest(
         String[] indices,
         IndicesOptions indicesOptions,
@@ -179,4 +182,14 @@ public int hashCode() {
         result = 31 * result + Arrays.hashCode(indices);
         return result;
     }
+
+    @Override
+    public void setResolvedIndexExpressions(ResolvedIndexExpressions expressions) {
+        this.resolvedIndexExpressions = expressions;
+    }
+
+    @Override
+    public ResolvedIndexExpressions getResolvedIndexExpressions() {
+        return resolvedIndexExpressions;
+    }
 }