Merge remote-tracking branch 'upstream/main' into convert-processor-tests-and-docs

Author: PeteGillinElastic

build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java

@@ -3,8 +3,7 @@
  * or more contributor license agreements. Licensed under the "Elastic License
  * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
  * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * your election, the "Server Side Public License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 package org.elasticsearch.gradle;
@@ -18,48 +17,58 @@
  * An outputstream to a File that is lazily opened on the first write.
  */
 class LazyFileOutputStream extends OutputStream {
-    private OutputStream delegate;
+    private final File file;
+    private volatile OutputStream delegate;
+    private volatile boolean initialized = false;
+    private final Object lock = new Object();
 
     LazyFileOutputStream(File file) {
-        // use an initial dummy delegate to avoid doing a conditional on every write
-        this.delegate = new OutputStream() {
-            private void bootstrap() throws IOException {
-                file.getParentFile().mkdirs();
-                delegate = new FileOutputStream(file);
-            }
-
-            @Override
-            public void write(int b) throws IOException {
-                bootstrap();
-                delegate.write(b);
-            }
-
-            @Override
-            public void write(byte b[], int off, int len) throws IOException {
-                bootstrap();
-                delegate.write(b, off, len);
-            }
+        this.file = file;
+    }
 
-            @Override
-            public void write(byte b[]) throws IOException {
-                bootstrap();
-                delegate.write(b);
+    private void ensureInitialized() throws IOException {
+        if (initialized == false) {
+            synchronized (lock) {
+                if (initialized == false) {
+                    file.getParentFile().mkdirs();
+                    delegate = new FileOutputStream(file);
+                    initialized = true;
+                }
             }
-        };
+        }
     }
 
     @Override
     public void write(int b) throws IOException {
+        ensureInitialized();
         delegate.write(b);
     }
 
     @Override
     public void write(byte b[], int off, int len) throws IOException {
+        ensureInitialized();
         delegate.write(b, off, len);
     }
 
+    @Override
+    public void write(byte b[]) throws IOException {
+        ensureInitialized();
+        delegate.write(b);
+    }
+
     @Override
     public void close() throws IOException {
-        delegate.close();
+        synchronized (lock) {
+            if (initialized && delegate != null) {
+                delegate.close();
+            }
+        }
+    }
+
+    @Override
+    public void flush() throws IOException {
+        if (initialized && delegate != null) {
+            delegate.flush();
+        }
     }
 }

docs/changelog/132922.yaml

@@ -0,0 +1,5 @@
+pr: 132922
+summary: Change GeoIpCache and EnrichCache to LongAdder
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/133188.yaml

@@ -0,0 +1,6 @@
+pr: 133188
+summary: Don't fail search if bottom doc can't be formatted
+area: Search
+type: bug
+issues:
+ - 125321

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java

@@ -54,7 +54,8 @@ public class TransportUpdateDataStreamSettingsAction extends TransportMasterNode
     UpdateDataStreamSettingsAction.Request,
     UpdateDataStreamSettingsAction.Response> {
     private static final Logger logger = LogManager.getLogger(TransportUpdateDataStreamSettingsAction.class);
-    private static final Set<String> APPLY_TO_BACKING_INDICES = Set.of(
+    private static final Set<String> APPLY_TO_WRITE_INDEX = Set.of("index.number_of_replicas");
+    private static final Set<String> APPLY_TO_ALL_BACKING_INDICES = Set.of(
         "index.lifecycle.name",
         IndexSettings.PREFER_ILM,
         "index.refresh_interval"
@@ -158,7 +159,9 @@ private void updateSingleDataStream(
         logger.debug("updating settings for {}", dataStreamName);
         Set<String> settingsToReject = new HashSet<>();
         for (String settingName : settingsOverrides.keySet()) {
-            if (APPLY_TO_BACKING_INDICES.contains(settingName) == false && APPLY_TO_DATA_STREAM_ONLY.contains(settingName) == false) {
+            if (APPLY_TO_WRITE_INDEX.contains(settingName) == false
+                && APPLY_TO_ALL_BACKING_INDICES.contains(settingName) == false
+                && APPLY_TO_DATA_STREAM_ONLY.contains(settingName) == false) {
                 settingsToReject.add(settingName);
             }
         }
@@ -223,19 +226,26 @@ private void updateSettingsOnIndices(
         TimeValue ackTimeout,
         ActionListener<UpdateDataStreamSettingsAction.DataStreamSettingsResponse> listener
     ) {
-        Map<String, Object> settingsToApply = new HashMap<>();
+        Map<String, Object> settingsToApplyToNonWriteIndices = new HashMap<>();
+        Map<String, Object> settingsToApplyToWriteIndex = new HashMap<>();
         List<String> appliedToDataStreamOnly = new ArrayList<>();
+        List<String> appliedToDataStreamAndWriteIndexOnly = new ArrayList<>();
         List<String> appliedToDataStreamAndBackingIndices = new ArrayList<>();
         Settings effectiveSettings = dataStream.getEffectiveSettings(projectResolver.getProjectMetadata(clusterService.state()));
         for (String settingName : requestSettings.keySet()) {
-            if (APPLY_TO_BACKING_INDICES.contains(settingName)) {
-                settingsToApply.put(settingName, effectiveSettings.get(settingName));
+            if (APPLY_TO_WRITE_INDEX.contains(settingName)) {
+                settingsToApplyToWriteIndex.put(settingName, effectiveSettings.get(settingName));
+                appliedToDataStreamAndWriteIndexOnly.add(settingName);
+            } else if (APPLY_TO_ALL_BACKING_INDICES.contains(settingName)) {
+                settingsToApplyToWriteIndex.put(settingName, effectiveSettings.get(settingName));
+                settingsToApplyToNonWriteIndices.put(settingName, effectiveSettings.get(settingName));
                 appliedToDataStreamAndBackingIndices.add(settingName);
             } else if (APPLY_TO_DATA_STREAM_ONLY.contains(settingName)) {
                 appliedToDataStreamOnly.add(settingName);
             }
         }
         final List<Index> concreteIndices = dataStream.getIndices();
+        final Index writeIndex = dataStream.getWriteIndex();
         final List<UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError> indexSettingErrors = new ArrayList<>();
 
         CountDownActionListener indexCountDownListener = new CountDownActionListener(
@@ -250,6 +260,7 @@ private void updateSettingsOnIndices(
                         settingsFilter.filter(effectiveSettings),
                         new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult(
                             appliedToDataStreamOnly,
+                            appliedToDataStreamAndWriteIndexOnly,
                             appliedToDataStreamAndBackingIndices,
                             indexSettingErrors
                         )
@@ -259,11 +270,13 @@ private void updateSettingsOnIndices(
         );
 
         indexCountDownListener.onResponse(null); // handles the case where there were zero indices
-        Settings applyToIndexSettings = builder().loadFromMap(settingsToApply).build();
+        Settings applyToNonWriteIndexSettings = builder().loadFromMap(settingsToApplyToNonWriteIndices).build();
+        Settings applyToWriteIndexSettings = builder().loadFromMap(settingsToApplyToWriteIndex).build();
         for (Index index : concreteIndices) {
+            Settings settings = index.equals(writeIndex) ? applyToWriteIndexSettings : applyToNonWriteIndexSettings;
             updateSettingsOnSingleIndex(
                 index,
-                applyToIndexSettings,
+                settings,
                 dryRun,
                 masterNodeTimeout,
                 ackTimeout,

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml

@@ -495,3 +495,98 @@ setup:
         index: my-component-only-data-stream-1
   - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" }
   - match: { .$idx0name.settings.index.lifecycle.prefer_ilm: null }
+
+---
+"Test write index only setting":
+  - requires:
+      cluster_features: [ "logs_stream" ]
+      reason: requires setting 'logs_stream' to get or set data stream settings
+  - do:
+      allowed_warnings:
+        - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: [ my-data-stream-* ]
+          data_stream: { }
+          template:
+            settings:
+              number_of_replicas: 0
+              lifecycle.name: my-policy
+
+  - do:
+      indices.create_data_stream:
+        name: my-data-stream-1
+
+  - do:
+      cluster.health:
+        index: "my-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream_settings:
+        name: my-data-stream-1
+  - match: { data_streams.0.name: my-data-stream-1 }
+  - match: { data_streams.0.settings: {} }
+  - match: { data_streams.0.effective_settings.index.number_of_shards: null }
+  - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" }
+
+  - do:
+      indices.rollover:
+        alias: "my-data-stream-1"
+
+  - do:
+      cluster.health:
+        index: "my-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.put_data_stream_settings:
+        name: my-data-stream-1
+        body:
+          index:
+            number_of_replicas: 1
+  - match: { data_streams.0.name: my-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: []}
+  - match: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: []}
+  - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_write_indices: 1  }
+  - match: { data_streams.0.effective_settings.index.number_of_replicas: "1" }
+
+  - do:
+      indices.rollover:
+        alias: "my-data-stream-1"
+
+  - do:
+      cluster.health:
+        index: "my-data-stream-1"
+        wait_for_status: yellow
+
+  - do:
+      indices.get_data_stream_settings:
+        name: my-data-stream-1
+  - match: { data_streams.0.name: my-data-stream-1 }
+  - match: { data_streams.0.effective_settings.index.number_of_replicas: "1" }
+
+  - do:
+      indices.get_data_stream:
+        name: my-data-stream-1
+  - match: { data_streams.0.name: my-data-stream-1 }
+  - match: { data_streams.0.settings.index.number_of_replicas: "1" }
+  - match: { data_streams.0.effective_settings: null }
+
+  - do:
+      indices.get_data_stream:
+        name: my-data-stream-1
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+  - set: { data_streams.0.indices.1.index_name: idx1name }
+  - set: { data_streams.0.indices.2.index_name: idx2name }
+
+  # We expect that index.number_of_replicas only gets updated on the current write index and any future indices. We have
+  # done one rollover since setting the setting, so it ought to be updated in the two most recent indices only:
+  - do:
+      indices.get_settings:
+        index: my-data-stream-1
+  - match: { .$idx0name.settings.index.number_of_replicas: "0" }
+  - match: { .$idx1name.settings.index.number_of_replicas: "1" }
+  - match: { .$idx2name.settings.index.number_of_replicas: "1" }

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java

@@ -17,7 +17,7 @@
 import org.elasticsearch.ingest.geoip.stats.CacheStats;
 
 import java.nio.file.Path;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 
@@ -44,8 +44,8 @@ public String toString() {
 
     private final LongSupplier relativeNanoTimeProvider;
     private final Cache<CacheKey, Object> cache;
-    private final AtomicLong hitsTimeInNanos = new AtomicLong(0);
-    private final AtomicLong missesTimeInNanos = new AtomicLong(0);
+    private final LongAdder hitsTimeInNanos = new LongAdder();
+    private final LongAdder missesTimeInNanos = new LongAdder();
 
     // package private for testing
     GeoIpCache(long maxSize, LongSupplier relativeNanoTimeProvider) {
@@ -80,9 +80,9 @@ <RESPONSE> RESPONSE putIfAbsent(ProjectId projectId, String ip, String databaseP
             // store the result or no-result in the cache
             cache.put(cacheKey, response);
             long databaseRequestAndCachePutTime = relativeNanoTimeProvider.getAsLong() - retrieveStart;
-            missesTimeInNanos.addAndGet(cacheRequestTime + databaseRequestAndCachePutTime);
+            missesTimeInNanos.add(cacheRequestTime + databaseRequestAndCachePutTime);
         } else {
-            hitsTimeInNanos.addAndGet(cacheRequestTime);
+            hitsTimeInNanos.add(cacheRequestTime);
         }
 
         if (response == NO_RESULT) {
@@ -126,8 +126,8 @@ public CacheStats getCacheStats() {
             stats.getHits(),
             stats.getMisses(),
             stats.getEvictions(),
-            TimeValue.nsecToMSec(hitsTimeInNanos.get()),
-            TimeValue.nsecToMSec(missesTimeInNanos.get())
+            TimeValue.nsecToMSec(hitsTimeInNanos.sum()),
+            TimeValue.nsecToMSec(missesTimeInNanos.sum())
         );
     }
 

muted-tests.yml

@@ -621,6 +621,18 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.RandomizedTimeSeriesIT
   method: testGroupBySubset
   issue: https://github.com/elastic/elasticsearch/issues/133220
+- class: org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests
+  method: testSortedDocValuesSingleUniqueValue
+  issue: https://github.com/elastic/elasticsearch/issues/133221
+- class: org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests
+  method: testSortedNumberMergeAwayAllValuesWithSkipper
+  issue: https://github.com/elastic/elasticsearch/issues/133223
+- class: org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests
+  method: testSortedSetDocValuesWithSkipperSmall
+  issue: https://github.com/elastic/elasticsearch/issues/133224
+- class: org.elasticsearch.xpack.esql.action.RandomizedTimeSeriesIT
+  method: testGroupByNothing
+  issue: https://github.com/elastic/elasticsearch/issues/133225
 
 # Examples:
 #