Merge branch 'main' into sort_other_numeric_types

Author: mayya-sharipova

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -0,0 +1,196 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.index.codec.tsdb;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LogByteSizeMergePolicy;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
+import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.profile.AsyncProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Fork(1)
+@Threads(1)
+@Warmup(iterations = 0)
+@Measurement(iterations = 1)
+public class TSDBDocValuesMergeBenchmark {
+
+    static {
+        // For Elasticsearch900Lucene101Codec:
+        LogConfigurator.loadLog4jPlugins();
+        LogConfigurator.configureESLogging();
+        LogConfigurator.setNodeName("test");
+    }
+
+    @Param("20431204")
+    private int nDocs;
+
+    @Param("1000")
+    private int deltaTime;
+
+    @Param("42")
+    private int seed;
+
+    private static final String TIMESTAMP_FIELD = "@timestamp";
+    private static final String HOSTNAME_FIELD = "host.name";
+    private static final long BASE_TIMESTAMP = 1704067200000L;
+
+    private IndexWriter indexWriterWithoutOptimizedMerge;
+    private IndexWriter indexWriterWithOptimizedMerge;
+    private ExecutorService executorService;
+
+    public static void main(String[] args) throws RunnerException {
+        final Options options = new OptionsBuilder().include(TSDBDocValuesMergeBenchmark.class.getSimpleName())
+            .addProfiler(AsyncProfiler.class)
+            .build();
+
+        new Runner(options).run();
+    }
+
+    @Setup(Level.Trial)
+    public void setup() throws IOException {
+        executorService = Executors.newSingleThreadExecutor();
+
+        final Directory tempDirectoryWithoutDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp1-"));
+        final Directory tempDirectoryWithDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp2-"));
+
+        indexWriterWithoutOptimizedMerge = createIndex(tempDirectoryWithoutDocValuesSkipper, false);
+        indexWriterWithOptimizedMerge = createIndex(tempDirectoryWithDocValuesSkipper, true);
+    }
+
+    private IndexWriter createIndex(final Directory directory, final boolean optimizedMergeEnabled) throws IOException {
+        final var iwc = createIndexWriterConfig(optimizedMergeEnabled);
+        long counter1 = 0;
+        long counter2 = 10_000_000;
+        long[] gauge1Values = new long[] { 2, 4, 6, 8, 10, 12, 14, 16 };
+        long[] gauge2Values = new long[] { -2, -4, -6, -8, -10, -12, -14, -16 };
+        int numHosts = 1000;
+        String[] tags = new String[] { "tag_1", "tag_2", "tag_3", "tag_4", "tag_5", "tag_6", "tag_7", "tag_8" };
+
+        final Random random = new Random(seed);
+        IndexWriter indexWriter = new IndexWriter(directory, iwc);
+        for (int i = 0; i < nDocs; i++) {
+            final Document doc = new Document();
+
+            final int batchIndex = i / numHosts;
+            final String hostName = "host-" + batchIndex;
+            // Slightly vary the timestamp in each document
+            final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
+
+            doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
+            doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
+            doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
+            doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
+            doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
+            doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
+            int numTags = tags.length % (i + 1);
+            for (int j = 0; j < numTags; j++) {
+                doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
+            }
+
+            indexWriter.addDocument(doc);
+        }
+        indexWriter.commit();
+        return indexWriter;
+    }
+
+    @Benchmark
+    public void forceMergeWithoutOptimizedMerge() throws IOException {
+        forceMerge(indexWriterWithoutOptimizedMerge);
+    }
+
+    @Benchmark
+    public void forceMergeWithOptimizedMerge() throws IOException {
+        forceMerge(indexWriterWithOptimizedMerge);
+    }
+
+    private void forceMerge(final IndexWriter indexWriter) throws IOException {
+        indexWriter.forceMerge(1);
+    }
+
+    @TearDown(Level.Trial)
+    public void tearDown() {
+        if (executorService != null) {
+            executorService.shutdown();
+            try {
+                if (executorService.awaitTermination(30, TimeUnit.SECONDS) == false) {
+                    executorService.shutdownNow();
+                }
+            } catch (InterruptedException e) {
+                executorService.shutdownNow();
+                Thread.currentThread().interrupt();
+            }
+        }
+    }
+
+    private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeEnabled) {
+        var config = new IndexWriterConfig(new StandardAnalyzer());
+        // NOTE: index sort config matching LogsDB's sort order
+        config.setIndexSort(
+            new Sort(
+                new SortField(HOSTNAME_FIELD, SortField.Type.STRING, false),
+                new SortedNumericSortField(TIMESTAMP_FIELD, SortField.Type.LONG, true)
+            )
+        );
+        config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
+        config.setMergePolicy(new LogByteSizeMergePolicy());
+        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
+        config.setCodec(new Elasticsearch900Lucene101Codec() {
+
+            @Override
+            public DocValuesFormat getDocValuesFormatForField(String field) {
+                return docValuesFormat;
+            }
+        });
+        return config;
+    }
+}

distribution/docker/build.gradle

@@ -448,8 +448,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
       // is functional.
       if (base == DockerBase.IRON_BANK) {
         Map<String, String> buildArgsMap = [
-          'BASE_REGISTRY': 'docker.elastic.co',
-          'BASE_IMAGE'   : 'ubi9/ubi',
+          'BASE_REGISTRY': 'docker.io',
+          'BASE_IMAGE'   : 'redhat/ubi9',
           'BASE_TAG'     : 'latest'
         ]
 

docs/changelog/125403.yaml

@@ -0,0 +1,5 @@
+pr: 125403
+summary: First step optimizing tsdb doc values codec merging
+area: Codec
+type: enhancement
+issues: []

docs/reference/query-languages/esql/esql-commands.md

@@ -802,6 +802,8 @@ RENAME old_name1 AS new_name1[, ..., old_nameN AS new_nameN]
 
 The `RENAME` processing command renames one or more columns. If a column with the new name already exists, it will be replaced by the new column.
 
+A `RENAME` with multiple column renames is equivalent to multiple sequential `RENAME` commands.
+
 **Examples**
 
 ```esql
@@ -818,6 +820,15 @@ FROM employees
 | RENAME first_name AS fn, last_name AS ln
 ```
 
+With multiple `RENAME` commands:
+
+```esql
+FROM employees
+| KEEP first_name, last_name
+| RENAME first_name AS fn
+| RENAME last_name AS ln
+```
+
 
 ## `SORT` [esql-sort]
 

muted-tests.yml

@@ -261,9 +261,6 @@ tests:
 - class: org.elasticsearch.env.NodeEnvironmentTests
   method: testIndexCompatibilityChecks
   issue: https://github.com/elastic/elasticsearch/issues/124388
-- class: org.elasticsearch.xpack.restart.FullClusterRestartIT
-  method: testWatcherWithApiKey {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/124159
 - class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=data_stream/190_failure_store_redirection/Redirect ingest failure in data stream to failure store}
   issue: https://github.com/elastic/elasticsearch/issues/124518
@@ -357,9 +354,6 @@ tests:
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/122707
-- class: org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT
-  method: testDeletionOfFailingToRecoverIndexShouldStopRestore
-  issue: https://github.com/elastic/elasticsearch/issues/126204
 - class: org.elasticsearch.xpack.esql.inference.RerankOperatorTests
   method: testSimpleCircuitBreaking
   issue: https://github.com/elastic/elasticsearch/issues/124337
@@ -450,6 +444,15 @@ tests:
 - class: org.elasticsearch.xpack.security.cli.HttpCertificateCommandTests
   method: testGenerateMultipleCertificateWithNewCA
   issue: https://github.com/elastic/elasticsearch/issues/126471
+- class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
+  method: testILMDownsampleRollingRestart
+  issue: https://github.com/elastic/elasticsearch/issues/126495
+- class: org.elasticsearch.xpack.search.AsyncSearchErrorTraceIT
+  method: testDataNodeLogsStackTraceWhenErrorTraceFalseOrEmpty
+  issue: https://github.com/elastic/elasticsearch/issues/126357
+- class: org.elasticsearch.smoketest.MlWithSecurityIT
+  method: test {yaml=ml/get_trained_model_stats/Test get stats given trained models}
+  issue: https://github.com/elastic/elasticsearch/issues/126510
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_eis.json

@@ -748,7 +748,7 @@ public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Except
 
         logger.info("--> wait for the index to appear");
         // that would mean that recovery process started and failing
-        waitForIndex("test-idx", TimeValue.timeValueSeconds(10));
+        safeGet(clusterAdmin().prepareHealth(SAFE_AWAIT_TIMEOUT, "test-idx").execute());
 
         logger.info("--> delete index");
         cluster().wipeIndices("test-idx");
@@ -1619,14 +1619,6 @@ public void testDeleteSnapshotWhileRestoringFails() throws Exception {
         restoreFut.get();
     }
 
-    private void waitForIndex(final String index, TimeValue timeout) throws Exception {
-        assertBusy(
-            () -> assertTrue("Expected index [" + index + "] to exist", indexExists(index)),
-            timeout.millis(),
-            TimeUnit.MILLISECONDS
-        );
-    }
-
     public void testSnapshotName() throws Exception {
         disableRepoConsistencyCheck("This test does not create any data in the repository");
 

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java

@@ -476,4 +476,5 @@
     exports org.elasticsearch.monitor.metrics;
     exports org.elasticsearch.plugins.internal.rewriter to org.elasticsearch.inference;
     exports org.elasticsearch.lucene.util.automaton;
+    exports org.elasticsearch.index.codec.perfield;
 }

server/src/main/java/module-info.java

@@ -353,7 +353,8 @@ public static ElasticsearchException readException(StreamInput input, int id) th
     public static boolean isRegistered(Class<? extends Throwable> exception, TransportVersion version) {
         ElasticsearchExceptionHandle elasticsearchExceptionHandle = CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE.get(exception);
         if (elasticsearchExceptionHandle != null) {
-            return version.onOrAfter(elasticsearchExceptionHandle.versionAdded);
+            return version.onOrAfter(elasticsearchExceptionHandle.versionAdded)
+                || Arrays.stream(elasticsearchExceptionHandle.patchVersions).anyMatch(version::isPatchFrom);
         }
         return false;
     }
@@ -1983,24 +1984,34 @@ private enum ElasticsearchExceptionHandle {
             183,
             TransportVersions.V_8_16_0
         ),
-        REMOTE_EXCEPTION(RemoteException.class, RemoteException::new, 184, TransportVersions.REMOTE_EXCEPTION);
+        REMOTE_EXCEPTION(
+            RemoteException.class,
+            RemoteException::new,
+            184,
+            TransportVersions.REMOTE_EXCEPTION,
+            TransportVersions.REMOTE_EXCEPTION_8_19
+        );
 
         final Class<? extends ElasticsearchException> exceptionClass;
         final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;
         final int id;
-        final TransportVersion versionAdded;
+        private final TransportVersion versionAdded;
+        private final TransportVersion[] patchVersions;
 
         <E extends ElasticsearchException> ElasticsearchExceptionHandle(
             Class<E> exceptionClass,
             CheckedFunction<StreamInput, E, IOException> constructor,
             int id,
-            TransportVersion versionAdded
+            TransportVersion versionAdded,
+            TransportVersion... patchVersions
         ) {
             // We need the exceptionClass because you can't dig it out of the constructor reliably.
             this.exceptionClass = exceptionClass;
             this.constructor = constructor;
-            this.versionAdded = versionAdded;
+
             this.id = id;
+            this.versionAdded = versionAdded;
+            this.patchVersions = patchVersions;
         }
     }