Merge remote-tracking branch 'upstream/main' into bwc-elastic#125599

Author: benwtrent

BUILDING.md

@@ -82,7 +82,7 @@ In case of updating a dependency, ensure to remove the unused entry of the outda
 
 You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
 ```
->./gradlew --write-verification-metadata sha256 precommit
+./gradlew --write-verification-metadata sha256 precommit
 ```
 
 The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations,
@@ -92,10 +92,10 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
->A manual confirmation of the Gradle generated checksums is currently not mandatory.
->If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
->Please replace the content of the `origin` attribute by `official site` in that case.
->
+> [!Tip]  
+> A manual confirmation of the Gradle generated checksums is currently not mandatory.
+> If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
+> Please replace the content of the `origin` attribute by `official site` in that case.
 
 #### Custom plugin and task implementations
 
@@ -229,13 +229,9 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
----
-
-**NOTE**
-
-You should only use that approach locally or on a developer branch for production dependencies as we do
+> [!Note]  
+> You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
----
 
 #### How to use a custom third party artifact?
 
@@ -265,12 +261,9 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
----
-**NOTE**
-
-As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
+> [!Note]  
+> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
-For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
+> For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
 that supports meta-data, it will use the second repository to provide the module.
-Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)
----
+> Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)

CONTRIBUTING.md

@@ -401,7 +401,8 @@ It is important that the only code covered by the Elastic licence is contained
 within the top-level `x-pack` directory. The build will fail its pre-commit
 checks if contributed code does not have the appropriate license headers.
 
-> **NOTE:** If you have imported the project into IntelliJ IDEA the project will
+> [!NOTE]
+> If you have imported the project into IntelliJ IDEA the project will
 > be automatically configured to add the correct license header to new source
 > files based on the source location.
 

docs/changelog/127383.yaml

@@ -0,0 +1,5 @@
+pr: 127383
+summary: Don't push down filters on the right hand side of an inlinejoin
+area: ES|QL
+type: bug
+issues: []

docs/changelog/127573.yaml

@@ -0,0 +1,5 @@
+pr: 127573
+summary: "[Failure store] Introduce default retention for failure indices"
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/127668.yaml

@@ -0,0 +1,6 @@
+pr: 127668
+summary: Fix transport versions
+area: "ES|QL"
+type: bug
+issues:
+ - 127667

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java

@@ -248,7 +248,8 @@ public void testSystemDataStreamRetention() throws Exception {
                                 builder,
                                 withEffectiveRetention,
                                 getDataStreamResponse.getRolloverConfiguration(),
-                                getDataStreamResponse.getGlobalRetention()
+                                getDataStreamResponse.getDataGlobalRetention(),
+                                getDataStreamResponse.getFailuresGlobalRetention()
                             );
                         String serialized = Strings.toString(builder);
                         Map<String, Object> resultMap = XContentHelper.convertToMap(

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java

@@ -147,7 +147,12 @@ public void testDataStreamRetention() throws Exception {
     @SuppressWarnings("unchecked")
     public void testDefaultRetention() throws Exception {
         // Set default global retention
-        updateClusterSettings(Settings.builder().put("data_streams.lifecycle.retention.default", "10s").build());
+        updateClusterSettings(
+            Settings.builder()
+                .put("data_streams.lifecycle.retention.default", "10s")
+                .put("data_streams.lifecycle.retention.failures_default", "10s")
+                .build()
+        );
 
         // Verify that the effective retention matches the default retention
         {
@@ -163,7 +168,7 @@ public void testDefaultRetention() throws Exception {
             assertThat(lifecycle.get("data_retention"), nullValue());
             Map<String, Object> failuresLifecycle = ((Map<String, Map<String, Object>>) dataStream.get("failure_store")).get("lifecycle");
             assertThat(failuresLifecycle.get("effective_retention"), is("10s"));
-            assertThat(failuresLifecycle.get("retention_determined_by"), is("default_global_retention"));
+            assertThat(failuresLifecycle.get("retention_determined_by"), is("default_failures_retention"));
             assertThat(failuresLifecycle.get("data_retention"), nullValue());
         }
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -370,7 +370,8 @@ public int compareTo(IndexInfo o) {
         return new GetDataStreamAction.Response(
             dataStreamInfos,
             request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
-            globalRetentionSettings.get()
+            globalRetentionSettings.get(false),
+            globalRetentionSettings.get(true)
         );
     }
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -45,7 +45,6 @@
 import org.elasticsearch.cluster.SimpleBatchedExecutor;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
@@ -372,13 +371,18 @@ private void run(ProjectState projectState) {
                 continue;
             }
 
+            // Retrieve the effective retention to ensure the same retention is used for this data stream
+            // through all operations.
+            var dataRetention = getEffectiveRetention(dataStream, globalRetentionSettings, false);
+            var failuresRetention = getEffectiveRetention(dataStream, globalRetentionSettings, true);
+
             // the following indices should not be considered for the remainder of this service run, for various reasons.
             Set<Index> indicesToExcludeForRemainingRun = new HashSet<>();
 
             // These are the pre-rollover write indices. They may or may not be the write index after maybeExecuteRollover has executed,
             // depending on rollover criteria, for this reason we exclude them for the remaining run.
-            indicesToExcludeForRemainingRun.add(maybeExecuteRollover(project, dataStream, false));
-            Index failureStoreWriteIndex = maybeExecuteRollover(project, dataStream, true);
+            indicesToExcludeForRemainingRun.add(maybeExecuteRollover(project, dataStream, dataRetention, false));
+            Index failureStoreWriteIndex = maybeExecuteRollover(project, dataStream, failuresRetention, true);
             if (failureStoreWriteIndex != null) {
                 indicesToExcludeForRemainingRun.add(failureStoreWriteIndex);
             }
@@ -394,7 +398,9 @@ private void run(ProjectState projectState) {
             );
 
             try {
-                indicesToExcludeForRemainingRun.addAll(maybeExecuteRetention(project, dataStream, indicesToExcludeForRemainingRun));
+                indicesToExcludeForRemainingRun.addAll(
+                    maybeExecuteRetention(project, dataStream, dataRetention, failuresRetention, indicesToExcludeForRemainingRun)
+                );
             } catch (Exception e) {
                 // individual index errors would be reported via the API action listener for every delete call
                 // we could potentially record errors at a data stream level and expose it via the _data_stream API?
@@ -840,7 +846,12 @@ private void clearErrorStoreForUnmanagedIndices(ProjectMetadata project, DataStr
     }
 
     @Nullable
-    private Index maybeExecuteRollover(ProjectMetadata project, DataStream dataStream, boolean rolloverFailureStore) {
+    private Index maybeExecuteRollover(
+        ProjectMetadata project,
+        DataStream dataStream,
+        TimeValue effectiveRetention,
+        boolean rolloverFailureStore
+    ) {
         Index currentRunWriteIndex = rolloverFailureStore ? dataStream.getWriteFailureIndex() : dataStream.getWriteIndex();
         if (currentRunWriteIndex == null) {
             return null;
@@ -851,7 +862,7 @@ private Index maybeExecuteRollover(ProjectMetadata project, DataStream dataStrea
                 RolloverRequest rolloverRequest = getDefaultRolloverRequest(
                     rolloverConfiguration,
                     dataStream.getName(),
-                    lifecycle.getEffectiveDataRetention(globalRetentionSettings.get(), dataStream.isInternal()),
+                    effectiveRetention,
                     rolloverFailureStore
                 );
                 transportActionsDeduplicator.executeOnce(
@@ -903,10 +914,13 @@ private Index maybeExecuteRollover(ProjectMetadata project, DataStream dataStrea
      * @param indicesToExcludeForRemainingRun Indices to exclude from retention even if it would be time for them to be deleted
      * @return The set of indices that delete requests have been sent for
      */
-    Set<Index> maybeExecuteRetention(ProjectMetadata project, DataStream dataStream, Set<Index> indicesToExcludeForRemainingRun) {
-        DataStreamGlobalRetention globalRetention = dataStream.isSystem() ? null : globalRetentionSettings.get();
-        var dataRetention = getRetention(dataStream, globalRetention, false);
-        var failureRetention = getRetention(dataStream, globalRetention, true);
+    Set<Index> maybeExecuteRetention(
+        ProjectMetadata project,
+        DataStream dataStream,
+        TimeValue dataRetention,
+        TimeValue failureRetention,
+        Set<Index> indicesToExcludeForRemainingRun
+    ) {
         if (dataRetention == null && failureRetention == null) {
             return Set.of();
         }
@@ -1361,11 +1375,15 @@ private static boolean isForceMergeComplete(IndexMetadata backingIndex) {
     }
 
     @Nullable
-    private static TimeValue getRetention(DataStream dataStream, DataStreamGlobalRetention globalRetention, boolean failureStore) {
+    private static TimeValue getEffectiveRetention(
+        DataStream dataStream,
+        DataStreamGlobalRetentionSettings globalRetentionSettings,
+        boolean failureStore
+    ) {
         DataStreamLifecycle lifecycle = failureStore ? dataStream.getFailuresLifecycle() : dataStream.getDataLifecycle();
         return lifecycle == null || lifecycle.enabled() == false
             ? null
-            : lifecycle.getEffectiveDataRetention(globalRetention, dataStream.isInternal());
+            : lifecycle.getEffectiveDataRetention(globalRetentionSettings.get(failureStore), dataStream.isInternal());
     }
 
     /**

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java

@@ -123,7 +123,8 @@ protected void masterOperation(
             new ExplainDataStreamLifecycleAction.Response(
                 explainIndices,
                 request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
-                globalRetentionSettings.get()
+                globalRetentionSettings.get(false),
+                globalRetentionSettings.get(true)
             )
         );
     }