Merge branch 'main' into esql_mv_stats

Author: craigtaverner

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayIndexOfBenchmark.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.benchmark.bytes;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 4, time = 1)
+@Measurement(iterations = 5, time = 1)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+@Fork(value = 1)
+public class BytesArrayIndexOfBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    static final byte MARKER = (byte) '\n';
+
+    @Param(value = { "64", "127", "128", "4096", "16384", "65536", "1048576" })
+    public int size;
+
+    BytesArray bytesArray;
+
+    @Setup
+    public void setup() {
+        byte[] bytes = new byte[size];
+        bytes[bytes.length - 1] = MARKER;
+        bytesArray = new BytesArray(bytes, 0, bytes.length);
+    }
+
+    @Benchmark
+    public int indexOf() {
+        return bytesArray.indexOf(MARKER, 0);
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public int indexOfPanama() {
+        return bytesArray.indexOf(MARKER, 0);
+    }
+
+    @Benchmark
+    public int withOffsetIndexOf() {
+        return bytesArray.indexOf(MARKER, 1);
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public int withOffsetIndexPanama() {
+        return bytesArray.indexOf(MARKER, 1);
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java

@@ -0,0 +1,228 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.bytes;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
+import org.elasticsearch.common.recycler.Recycler;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+@Warmup(iterations = 3)
+@Measurement(iterations = 3)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(value = 1)
+public class RecyclerBytesStreamOutputBenchmark {
+
+    private final AtomicReference<BytesRef> bytesRef = new AtomicReference<>(new BytesRef(16384));
+    private RecyclerBytesStreamOutput streamOutput;
+    private String shortString;
+    private String longString;
+    private String nonAsciiString;
+    private String veryLongString;
+    private byte[] bytes1;
+    private byte[] bytes2;
+    private byte[] bytes3;
+    private byte[] multiPageBytes;
+    private int[] vints;
+
+    @Setup
+    public void initResults() throws IOException {
+        streamOutput = new RecyclerBytesStreamOutput(new BenchmarkRecycler(bytesRef));
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+
+        bytes1 = new byte[327];
+        bytes2 = new byte[712];
+        bytes3 = new byte[1678];
+        multiPageBytes = new byte[16387 * 4];
+        random.nextBytes(bytes1);
+        random.nextBytes(bytes2);
+        random.nextBytes(bytes3);
+        random.nextBytes(multiPageBytes);
+
+        // We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
+        // impact direct comparisons run-to-run
+
+        shortString = generateAsciiString(20);
+        longString = generateAsciiString(100);
+        nonAsciiString = generateUtf8String(200);
+        veryLongString = generateAsciiString(800);
+        // vint values for benchmarking
+        vints = new int[1000];
+        for (int i = 0; i < vints.length; i++) {
+            if (random.nextBoolean()) {
+                // 1-byte 50% of the time
+                vints[i] = random.nextInt(128);
+            } else if (random.nextBoolean()) {
+                // 2-byte 25% of the time
+                vints[i] = random.nextInt(128, 16384);
+            } else {
+                if (random.nextBoolean()) {
+                    // 3-byte vints
+                    vints[i] = random.nextInt(16384, 2097152);
+                } else {
+                    // All vint variants
+                    vints[i] = random.nextInt();
+                }
+            }
+        }
+    }
+
+    @Benchmark
+    public void writeByte() throws IOException {
+        streamOutput.seek(1);
+        for (byte item : bytes1) {
+            streamOutput.writeByte(item);
+        }
+        for (byte item : bytes2) {
+            streamOutput.writeByte(item);
+        }
+        for (byte item : bytes3) {
+            streamOutput.writeByte(item);
+        }
+    }
+
+    @Benchmark
+    public void writeBytes() throws IOException {
+        streamOutput.seek(1);
+        streamOutput.writeBytes(bytes1, 0, bytes1.length);
+        streamOutput.writeBytes(bytes2, 0, bytes2.length);
+        streamOutput.writeBytes(bytes3, 0, bytes3.length);
+    }
+
+    @Benchmark
+    public void writeBytesAcrossPageBoundary() throws IOException {
+        streamOutput.seek(16384 - 1000);
+        streamOutput.writeBytes(bytes1, 0, bytes1.length);
+        streamOutput.writeBytes(bytes2, 0, bytes2.length);
+        streamOutput.writeBytes(bytes3, 0, bytes3.length);
+    }
+
+    @Benchmark
+    public void writeBytesMultiPage() throws IOException {
+        streamOutput.seek(16384 - 1000);
+        streamOutput.writeBytes(multiPageBytes, 0, multiPageBytes.length);
+    }
+
+    @Benchmark
+    public void writeString() throws IOException {
+        streamOutput.seek(1);
+        streamOutput.writeString(shortString);
+        streamOutput.writeString(longString);
+        streamOutput.writeString(nonAsciiString);
+        streamOutput.writeString(veryLongString);
+    }
+
+    @Benchmark
+    public void writeVInt() throws IOException {
+        streamOutput.seek(1);
+        for (int vint : vints) {
+            streamOutput.writeVInt(vint);
+        }
+    }
+
+    public static String generateAsciiString(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int ascii = random.nextInt(128);
+            sb.append((char) ascii);
+        }
+
+        return sb.toString();
+    }
+
+    public static String generateUtf8String(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int codePoint;
+            int probability = random.nextInt(100);
+
+            if (probability < 85) {
+                // 1-byte UTF-8 (ASCII range)
+                // 0x0000 to 0x007F
+                codePoint = random.nextInt(0x0080);
+            } else if (probability < 95) {
+                // 2-byte UTF-8
+                // 0x0080 to 0x07FF
+                codePoint = random.nextInt(0x0080, 0x0800);
+            } else {
+                // 3-byte UTF-8
+                // 0x0800 to 0xFFFF
+                do {
+                    codePoint = random.nextInt(0x0800, 0x10000);
+                    // Skip surrogate pairs (0xD800-0xDFFF)
+                } while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
+            }
+
+            sb.appendCodePoint(codePoint);
+        }
+
+        return sb.toString();
+    }
+
+    private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
+
+        @Override
+        public V<BytesRef> obtain() {
+            BytesRef recycledBytesRef = bytesRef.getAndSet(null);
+            final BytesRef localBytesRef;
+            final boolean recycled;
+            if (recycledBytesRef != null) {
+                recycled = true;
+                localBytesRef = recycledBytesRef;
+            } else {
+                recycled = false;
+                localBytesRef = new BytesRef(16384);
+            }
+            return new V<>() {
+                @Override
+                public BytesRef v() {
+                    return localBytesRef;
+                }
+
+                @Override
+                public boolean isRecycled() {
+                    return recycled;
+                }
+
+                @Override
+                public void close() {
+                    if (recycled) {
+                        bytesRef.set(localBytesRef);
+                    }
+                }
+            };
+        }
+
+        @Override
+        public int pageSize() {
+            return 16384;
+        }
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -26,7 +26,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
+import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
 import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -63,7 +63,6 @@
 public class TSDBDocValuesMergeBenchmark {
 
     static {
-        // For Elasticsearch900Lucene101Codec:
         LogConfigurator.loadLog4jPlugins();
         LogConfigurator.configureESLogging();
         LogConfigurator.setNodeName("test");
@@ -259,8 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
         var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
-        config.setCodec(new Elasticsearch900Lucene101Codec() {
-
+        config.setCodec(new Elasticsearch92Lucene103Codec() {
             @Override
             public DocValuesFormat getDocValuesFormatForField(String field) {
                 return docValuesFormat;

benchmarks/src/test/java/org/elasticsearch/benchmark/bytes/BytesArrayIndexOfBenchmarkTests.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.bytes;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.ESTestCase;
+import org.openjdk.jmh.annotations.Param;
+
+import java.util.Arrays;
+
+public class BytesArrayIndexOfBenchmarkTests extends ESTestCase {
+
+    final int size;
+
+    public BytesArrayIndexOfBenchmarkTests(int size) {
+        this.size = size;
+    }
+
+    public void testIndexOf() {
+        var bench = new BytesArrayIndexOfBenchmark();
+        bench.size = size;
+        bench.setup();
+        assertEquals(size - 1, bench.indexOf());
+        assertEquals(size - 1, bench.indexOfPanama());
+        assertEquals(size - 1, bench.withOffsetIndexOf());
+        assertEquals(size - 1, bench.withOffsetIndexPanama());
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parametersFactory() {
+        try {
+            var params = BytesArrayIndexOfBenchmark.class.getField("size").getAnnotationsByType(Param.class)[0].value();
+            return () -> Arrays.stream(params).map(Integer::parseInt).map(i -> new Object[] { i }).iterator();
+        } catch (NoSuchFieldException e) {
+            throw new AssertionError(e);
+        }
+    }
+}