Merge branch 'main' into 2025/04/07/remove-TransportResponse-Empty

Author: DaveCTurner

benchmarks/README.md

@@ -82,19 +82,21 @@ To get realistic results, you should exercise care when running benchmarks. Here
 NOTE: Linux only. Sorry Mac and Windows.
 
 Disassembling is fun! Maybe not always useful, but always fun! Generally, you'll want to install `perf` and the JDK's `hsdis`.
-`perf` is generally available via `apg-get install perf` or `pacman -S perf`. `hsdis` you'll want to compile from source. is a little more involved. This worked
+`perf` is generally available via `apg-get install perf` or `pacman -S perf linux-tools`. `hsdis` you'll want to compile from source. is a little more involved. This worked
 on 2020-08-01:
 
 ```
 git clone [email protected]:openjdk/jdk.git
 cd jdk
-git checkout jdk-17-ga
-cd src/utils/hsdis
+git checkout jdk-24-ga
 # Get a known good binutils
 wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.gz
 tar xf binutils-2.35.tar.gz
-make BINUTILS=binutils-2.35 ARCH=amd64
-sudo cp build/linux-amd64/hsdis-amd64.so /usr/lib/jvm/java-17-openjdk/lib/server/
+bash configure --with-hsdis=binutils --with-binutils-src=binutils-2.35 \
+    --with-boot-jdk=~/.gradle/jdks/oracle_corporation-24-amd64-linux.2
+make build-hsdis
+cp ./build/linux-x86_64-server-release/jdk/lib/hsdis-amd64.so \
+    ~/.gradle/jdks/oracle_corporation-24-amd64-linux.2/lib/hsdis.so
 ```
 
 If you want to disassemble a single method do something like this:
@@ -105,6 +107,30 @@ gradlew -p benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDia
 
 If you want `perf` to find the hot methods for you, then do add `-prof perfasm`.
 
+NOTE: `perfasm` will need more access:
+```
+sudo bash
+echo -1 > /proc/sys/kernel/perf_event_paranoid
+exit
+```
+
+If you get warnings like:
+```
+The perf event count is suspiciously low (0).
+```
+then check if you are bumping into [this](https://man.archlinux.org/man/perf-stat.1.en#INTEL_HYBRID_SUPPORT)
+by running:
+```
+perf stat -B dd if=/dev/zero of=/dev/null count=1000000
+```
+
+If you see lines like:
+```
+         765019980      cpu_atom/cycles/                 #    1.728 GHz                         (0.60%)
+        2258845959      cpu_core/cycles/                 #    5.103 GHz                         (99.18%)
+```
+then `perf` is just not going to work for you.
+
 ## Async Profiler
 
 Note: Linux and Mac only. Sorry Windows.

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ParseIpBenchmark.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.compute.operator;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ParseIp;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.net.InetAddress;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 5)
+@Measurement(iterations = 7)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(1)
+public class ParseIpBenchmark {
+    private final BytesRef ip = new BytesRef("192.168.0.1");
+    private final BreakingBytesRefBuilder scratch = ParseIp.buildScratch(new NoopCircuitBreaker("request"));
+
+    @Benchmark
+    public BytesRef leadingZerosRejected() {
+        return ParseIp.leadingZerosRejected(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreDecimal() {
+        return ParseIp.leadingZerosAreDecimal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreOctal() {
+        return ParseIp.leadingZerosAreOctal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef original() {
+        InetAddress inetAddress = InetAddresses.forString(ip.utf8ToString());
+        return new BytesRef(InetAddressPoint.encode(inetAddress));
+    }
+}

docs/changelog/126237.yaml

@@ -0,0 +1,5 @@
+pr: 126237
+summary: Use `FallbackSyntheticSourceBlockLoader` for text fields
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/126338.yaml

@@ -0,0 +1,5 @@
+pr: 126338
+summary: Speed up TO_IP
+area: ES|QL
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java

@@ -313,7 +313,7 @@ public String toString() {
             return Strings.format(
                 "For data stream %s: %s based on [inc/dec cooldowns %s/%s, %d-%d threads, "
                     + "write index %s has all-time/recent/peak loads %g/%g/%g, current shards %d, "
-                    + "using %s value %g for increase gives %d shards%s]",
+                    + "increase calculation gives %d shards using %s load %g for write index%s]",
                 inputs.dataStream,
                 result,
                 inputs.increaseShardsCooldown,
@@ -325,19 +325,19 @@ public String toString() {
                 inputs.writeIndexRecentLoad,
                 inputs.writeIndexPeakLoad,
                 inputs.currentNumberOfWriteIndexShards,
+                increaseCalculation.optimalShardCountForIncrease,
                 inputs.increaseShardsMetric,
                 increaseCalculation.writeIndexLoadForIncrease,
-                increaseCalculation.optimalShardCountForIncrease,
                 decreaseCalculation == null
                     ? ""
                     : Strings.format(
-                        ", and using %s value %g for dec based on %s gives %d shards",
+                        ", decrease calculation gives %d shards using %s load %g for %s",
+                        decreaseCalculation.optimalShardCountForDecrease,
                         inputs.decreaseShardsMetric,
                         decreaseCalculation.maxLoadWithinCooldown.load,
                         decreaseCalculation.maxLoadWithinCooldown.previousIndexWithMaxLoad != null
                             ? decreaseCalculation.maxLoadWithinCooldown.previousIndexWithMaxLoad
-                            : "write index",
-                        decreaseCalculation.optimalShardCountForDecrease
+                            : "write index"
                     )
             );
         }

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -74,6 +74,7 @@
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -1016,10 +1017,53 @@ protected String delegatingTo() {
             if (isStored()) {
                 return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name());
             }
+
+            // _ignored_source field will only be present if text field is not stored
+            // and there is no syntheticSourceDelegate
+            if (isSyntheticSource && syntheticSourceDelegate == null) {
+                return fallbackSyntheticSourceBlockLoader();
+            }
+
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext));
         }
 
+        FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader() {
+            var reader = new FallbackSyntheticSourceBlockLoader.SingleValueReader<BytesRef>(null) {
+                @Override
+                public void convertValue(Object value, List<BytesRef> accumulator) {
+                    if (value != null) {
+                        accumulator.add(new BytesRef(value.toString()));
+                    }
+                }
+
+                @Override
+                protected void parseNonNullValue(XContentParser parser, List<BytesRef> accumulator) throws IOException {
+                    var text = parser.textOrNull();
+
+                    if (text != null) {
+                        accumulator.add(new BytesRef(text));
+                    }
+                }
+
+                @Override
+                public void writeToBlock(List<BytesRef> values, BlockLoader.Builder blockBuilder) {
+                    var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
+
+                    for (var value : values) {
+                        bytesRefBuilder.appendBytesRef(value);
+                    }
+                }
+            };
+
+            return new FallbackSyntheticSourceBlockLoader(reader, name()) {
+                @Override
+                public Builder builder(BlockFactory factory, int expectedCount) {
+                    return factory.bytesRefs(expectedCount);
+                }
+            };
+        }
+
         /**
          * Build an iterator of documents that have the field. This mirrors parseCreateField,
          * using whatever