Merge remote-tracking branch 'upstream/main' into pattern-text-remove-ff

Author: jordan-powers

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java

@@ -0,0 +1,228 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.bytes;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
+import org.elasticsearch.common.recycler.Recycler;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+@Warmup(iterations = 3)
+@Measurement(iterations = 3)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(value = 1)
+public class RecyclerBytesStreamOutputBenchmark {
+
+    private final AtomicReference<BytesRef> bytesRef = new AtomicReference<>(new BytesRef(16384));
+    private RecyclerBytesStreamOutput streamOutput;
+    private String shortString;
+    private String longString;
+    private String nonAsciiString;
+    private String veryLongString;
+    private byte[] bytes1;
+    private byte[] bytes2;
+    private byte[] bytes3;
+    private byte[] multiPageBytes;
+    private int[] vints;
+
+    @Setup
+    public void initResults() throws IOException {
+        streamOutput = new RecyclerBytesStreamOutput(new BenchmarkRecycler(bytesRef));
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+
+        bytes1 = new byte[327];
+        bytes2 = new byte[712];
+        bytes3 = new byte[1678];
+        multiPageBytes = new byte[16387 * 4];
+        random.nextBytes(bytes1);
+        random.nextBytes(bytes2);
+        random.nextBytes(bytes3);
+        random.nextBytes(multiPageBytes);
+
+        // We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
+        // impact direct comparisons run-to-run
+
+        shortString = generateAsciiString(20);
+        longString = generateAsciiString(100);
+        nonAsciiString = generateUtf8String(200);
+        veryLongString = generateAsciiString(800);
+        // vint values for benchmarking
+        vints = new int[1000];
+        for (int i = 0; i < vints.length; i++) {
+            if (random.nextBoolean()) {
+                // 1-byte 50% of the time
+                vints[i] = random.nextInt(128);
+            } else if (random.nextBoolean()) {
+                // 2-byte 25% of the time
+                vints[i] = random.nextInt(128, 16384);
+            } else {
+                if (random.nextBoolean()) {
+                    // 3-byte vints
+                    vints[i] = random.nextInt(16384, 2097152);
+                } else {
+                    // All vint variants
+                    vints[i] = random.nextInt();
+                }
+            }
+        }
+    }
+
+    @Benchmark
+    public void writeByte() throws IOException {
+        streamOutput.seek(1);
+        for (byte item : bytes1) {
+            streamOutput.writeByte(item);
+        }
+        for (byte item : bytes2) {
+            streamOutput.writeByte(item);
+        }
+        for (byte item : bytes3) {
+            streamOutput.writeByte(item);
+        }
+    }
+
+    @Benchmark
+    public void writeBytes() throws IOException {
+        streamOutput.seek(1);
+        streamOutput.writeBytes(bytes1, 0, bytes1.length);
+        streamOutput.writeBytes(bytes2, 0, bytes2.length);
+        streamOutput.writeBytes(bytes3, 0, bytes3.length);
+    }
+
+    @Benchmark
+    public void writeBytesAcrossPageBoundary() throws IOException {
+        streamOutput.seek(16384 - 1000);
+        streamOutput.writeBytes(bytes1, 0, bytes1.length);
+        streamOutput.writeBytes(bytes2, 0, bytes2.length);
+        streamOutput.writeBytes(bytes3, 0, bytes3.length);
+    }
+
+    @Benchmark
+    public void writeBytesMultiPage() throws IOException {
+        streamOutput.seek(16384 - 1000);
+        streamOutput.writeBytes(multiPageBytes, 0, multiPageBytes.length);
+    }
+
+    @Benchmark
+    public void writeString() throws IOException {
+        streamOutput.seek(1);
+        streamOutput.writeString(shortString);
+        streamOutput.writeString(longString);
+        streamOutput.writeString(nonAsciiString);
+        streamOutput.writeString(veryLongString);
+    }
+
+    @Benchmark
+    public void writeVInt() throws IOException {
+        streamOutput.seek(1);
+        for (int vint : vints) {
+            streamOutput.writeVInt(vint);
+        }
+    }
+
+    public static String generateAsciiString(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int ascii = random.nextInt(128);
+            sb.append((char) ascii);
+        }
+
+        return sb.toString();
+    }
+
+    public static String generateUtf8String(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int codePoint;
+            int probability = random.nextInt(100);
+
+            if (probability < 85) {
+                // 1-byte UTF-8 (ASCII range)
+                // 0x0000 to 0x007F
+                codePoint = random.nextInt(0x0080);
+            } else if (probability < 95) {
+                // 2-byte UTF-8
+                // 0x0080 to 0x07FF
+                codePoint = random.nextInt(0x0080, 0x0800);
+            } else {
+                // 3-byte UTF-8
+                // 0x0800 to 0xFFFF
+                do {
+                    codePoint = random.nextInt(0x0800, 0x10000);
+                    // Skip surrogate pairs (0xD800-0xDFFF)
+                } while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
+            }
+
+            sb.appendCodePoint(codePoint);
+        }
+
+        return sb.toString();
+    }
+
+    private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
+
+        @Override
+        public V<BytesRef> obtain() {
+            BytesRef recycledBytesRef = bytesRef.getAndSet(null);
+            final BytesRef localBytesRef;
+            final boolean recycled;
+            if (recycledBytesRef != null) {
+                recycled = true;
+                localBytesRef = recycledBytesRef;
+            } else {
+                recycled = false;
+                localBytesRef = new BytesRef(16384);
+            }
+            return new V<>() {
+                @Override
+                public BytesRef v() {
+                    return localBytesRef;
+                }
+
+                @Override
+                public boolean isRecycled() {
+                    return recycled;
+                }
+
+                @Override
+                public void close() {
+                    if (recycled) {
+                        bytesRef.set(localBytesRef);
+                    }
+                }
+            };
+        }
+
+        @Override
+        public int pageSize() {
+            return 16384;
+        }
+    }
+}

docs/changelog/133630.yaml

@@ -0,0 +1,6 @@
+pr: 134963
+summary: Fix a bug in the GET _transform API that incorrectly claims some Transform configurations are missing
+area: Transform
+type: bug
+issues:
+ - 134263

docs/changelog/134963.yaml

@@ -0,0 +1,5 @@
+pr: 135096
+summary: Tolerate mixed types in datafeed stats sort
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/135096.yaml

@@ -0,0 +1,5 @@
+pr: 135244
+summary: Add reload listener to `SslProfile`
+area: TLS
+type: enhancement
+issues: []

docs/changelog/135244.yaml

@@ -0,0 +1,5 @@
+pr: 135373
+summary: Improve block loader for source only runtime date fields
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/135373.yaml

@@ -36,6 +36,9 @@ In order to ensure consistency and robustness, all new `TransportVersion`s
 must first be created in the `main` branch and then backported to the relevant
 release branches.
 
+_Elastic developers_ - instructions in this section for managing transport
+versions are the same in Serverless.
+
 ### Internal state files
 
 The Elasticsearch server jar contains resource files representing each
@@ -51,9 +54,6 @@ regenerated to resolve the conflict before merging to `main`.
 All of these internal state files are managed by gradle tasks; they should
 not be edited directly.
 
-_Elastic developers_ - please see corresponding documentation for Serverless
-on creating transport versions for Serverless changes.
-
 ### Creating transport versions locally
 
 To create a transport version, declare a reference anywhere in java code. For example:

docs/internal/Versioning.md

@@ -79,6 +79,10 @@ $$$xpack.ml.max_open_jobs$$$
 `xpack.ml.max_open_jobs`
 :   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can run simultaneously on a node. In this context, jobs include both {{anomaly-jobs}} and {{dfanalytics-jobs}}. The maximum number of jobs is also constrained by memory usage. Thus if the estimated memory usage of the jobs would be higher than allowed, fewer jobs will run on a node. Prior to version 7.1, this setting was a per-node non-dynamic setting. It became a cluster-wide dynamic setting in version 7.1. As a result, changes to its value after node startup are used only after every node in the cluster is running version 7.1 or higher. The minimum value is `1`; the maximum value is `512`. Defaults to `512`.
 
+    ::::{note}
+    This is a cluster-wide setting. If it is configured with different values across nodes, Elasticsearch resolves the inconsistency by applying the lowest configured value across the cluster as the effective cluster-wide limit.
+    ::::
+
 `xpack.ml.nightly_maintenance_requests_per_second`
 :   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0`
 

docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md

@@ -6,6 +6,19 @@ stack: preview 9.1.0
 
 The `COMPLETION` command allows you to send prompts and context to a Large Language Model (LLM) directly within your ES|QL queries, to perform text generation tasks.
 
+:::{important}
+**Every row processed by the COMPLETION command generates a separate API call to the LLM endpoint.**
+
+Be careful to test with small datasets first before running on production data or in automated workflows, to avoid unexpected costs.
+
+Best practices:
+
+1. **Start with dry runs**: Validate your query logic and row counts by running without `COMPLETION` initially. Use `| STATS count = COUNT(*)` to check result size.
+2. **Filter first**: Use `WHERE` clauses to limit rows before applying `COMPLETION`.
+3. **Test with `LIMIT`**: Always start with a low [`LIMIT`](/reference/query-languages/esql/commands/limit.md) and gradually increase.
+4. **Monitor usage**: Track your LLM API consumption and costs.
+:::
+
 **Syntax**
 
 ::::{tab-set}