Merge remote-tracking branch 'upstream/main' into fix-usage-of-already-released-null-block

Author: alex-spies

benchmarks/README.md

@@ -82,19 +82,21 @@ To get realistic results, you should exercise care when running benchmarks. Here
 NOTE: Linux only. Sorry Mac and Windows.
 
 Disassembling is fun! Maybe not always useful, but always fun! Generally, you'll want to install `perf` and the JDK's `hsdis`.
-`perf` is generally available via `apg-get install perf` or `pacman -S perf`. `hsdis` you'll want to compile from source. is a little more involved. This worked
+`perf` is generally available via `apg-get install perf` or `pacman -S perf linux-tools`. `hsdis` you'll want to compile from source. is a little more involved. This worked
 on 2020-08-01:
 
 ```
 git clone [email protected]:openjdk/jdk.git
 cd jdk
-git checkout jdk-17-ga
-cd src/utils/hsdis
+git checkout jdk-24-ga
 # Get a known good binutils
 wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.gz
 tar xf binutils-2.35.tar.gz
-make BINUTILS=binutils-2.35 ARCH=amd64
-sudo cp build/linux-amd64/hsdis-amd64.so /usr/lib/jvm/java-17-openjdk/lib/server/
+bash configure --with-hsdis=binutils --with-binutils-src=binutils-2.35 \
+    --with-boot-jdk=~/.gradle/jdks/oracle_corporation-24-amd64-linux.2
+make build-hsdis
+cp ./build/linux-x86_64-server-release/jdk/lib/hsdis-amd64.so \
+    ~/.gradle/jdks/oracle_corporation-24-amd64-linux.2/lib/hsdis.so
 ```
 
 If you want to disassemble a single method do something like this:
@@ -105,6 +107,30 @@ gradlew -p benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDia
 
 If you want `perf` to find the hot methods for you, then do add `-prof perfasm`.
 
+NOTE: `perfasm` will need more access:
+```
+sudo bash
+echo -1 > /proc/sys/kernel/perf_event_paranoid
+exit
+```
+
+If you get warnings like:
+```
+The perf event count is suspiciously low (0).
+```
+then check if you are bumping into [this](https://man.archlinux.org/man/perf-stat.1.en#INTEL_HYBRID_SUPPORT)
+by running:
+```
+perf stat -B dd if=/dev/zero of=/dev/null count=1000000
+```
+
+If you see lines like:
+```
+         765019980      cpu_atom/cycles/                 #    1.728 GHz                         (0.60%)
+        2258845959      cpu_core/cycles/                 #    5.103 GHz                         (99.18%)
+```
+then `perf` is just not going to work for you.
+
 ## Async Profiler
 
 Note: Linux and Mac only. Sorry Windows.

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ParseIpBenchmark.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.compute.operator;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ParseIp;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.net.InetAddress;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 5)
+@Measurement(iterations = 7)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(1)
+public class ParseIpBenchmark {
+    private final BytesRef ip = new BytesRef("192.168.0.1");
+    private final BreakingBytesRefBuilder scratch = ParseIp.buildScratch(new NoopCircuitBreaker("request"));
+
+    @Benchmark
+    public BytesRef leadingZerosRejected() {
+        return ParseIp.leadingZerosRejected(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreDecimal() {
+        return ParseIp.leadingZerosAreDecimal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreOctal() {
+        return ParseIp.leadingZerosAreOctal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef original() {
+        InetAddress inetAddress = InetAddresses.forString(ip.utf8ToString());
+        return new BytesRef(InetAddressPoint.encode(inetAddress));
+    }
+}

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java

@@ -8,8 +8,6 @@
  */
 package org.elasticsearch.plugin.noop;
 
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.search.SearchResponse;
@@ -41,10 +39,10 @@ public class NoopPlugin extends Plugin implements ActionPlugin {
     public static final ActionType<BulkResponse> NOOP_BULK_ACTION = new ActionType<>("mock:data/write/bulk");
 
     @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+    public List<ActionHandler> getActions() {
         return Arrays.asList(
-            new ActionHandler<>(NOOP_BULK_ACTION, TransportNoopBulkAction.class),
-            new ActionHandler<>(NOOP_SEARCH_ACTION, TransportNoopSearchAction.class)
+            new ActionHandler(NOOP_BULK_ACTION, TransportNoopBulkAction.class),
+            new ActionHandler(NOOP_SEARCH_ACTION, TransportNoopSearchAction.class)
         );
     }
 

docs/changelog/126237.yaml

@@ -0,0 +1,5 @@
+pr: 126237
+summary: Use `FallbackSyntheticSourceBlockLoader` for text fields
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/126264.yaml

@@ -0,0 +1,5 @@
+pr: 126264
+summary: '`FileWatchingService` shoudld not throw for missing file'
+area: Infra/Settings
+type: enhancement
+issues: []

docs/changelog/126338.yaml

@@ -0,0 +1,5 @@
+pr: 126338
+summary: Speed up TO_IP
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/126376.yaml

@@ -0,0 +1,6 @@
+pr: 126376
+summary: Set `keyUsage` for generated HTTP certificates and self-signed CA
+area: TLS
+type: bug
+issues:
+ - 117769

docs/changelog/126417.yaml

@@ -0,0 +1,5 @@
+pr: 126417
+summary: Correctly handle nulls in nested paths in the remove processor
+area: Ingest Node
+type: bug
+issues: []

docs/reference/elasticsearch/command-line-tools/certutil.md

@@ -13,7 +13,7 @@ The `elasticsearch-certutil` command simplifies the creation of certificates for
 ```shell
 bin/elasticsearch-certutil
 (
-(ca [--ca-dn <name>] [--days <n>] [--pem])
+(ca [--ca-dn <name>] [--keyusage <key_usages>] [--days <n>] [--pem])
 
 | (cert ([--ca <file_path>] | [--ca-cert <file_path> --ca-key <file_path>])
 [--ca-dn <name>] [--ca-pass <password>] [--days <n>]
@@ -105,6 +105,9 @@ The `http` mode guides you through the process of generating certificates for us
 `--ca-pass <password>`
 :   Specifies the password for an existing CA private key or the generated CA private key. This parameter is only applicable to the `cert` parameter
 
+`--keyusage <key_usages>`
+:   Specifies a comma-separated list of key usage restrictions (as per RFC 5280) that are used for the generated CA certificate. The default value is `keyCertSign,cRLSign`. This parameter may only be used with the `ca` parameter.
+
 `--days <n>`
 :   Specifies an integer value that represents the number of days the generated certificates are valid. The default value is `1095`. This parameter cannot be used with the `csr` or `http` parameters.
 

docs/release-notes/breaking-changes.md

@@ -10,54 +10,98 @@ Breaking changes can impact your Elastic applications, potentially disrupting no
 
 If you are migrating from a version prior to version 9.0, you must first upgrade to the last 8.x version available. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md).
 
-To learn how to upgrade, check out <uprade docs>.
-
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
 ## 9.0.0 [elasticsearch-900-breaking-changes]
 
-Allocation
-:   * Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831)
+Aggregations:
+* Remove date histogram boolean support [#118484](https://github.com/elastic/elasticsearch/pull/118484)
+
+Allocation:
+* Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831)
 * Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207)
-* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: [#88978](https://github.com/elastic/elasticsearch/issues/88978))
+* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978])
 
-Analysis
-:   * Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146)
-* The *german2* stemmer is now an alias for the *german* snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614)
-* The *persian* analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: [#113050](https://github.com/elastic/elasticsearch/issues/113050))
+Analysis:
+* Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146)
+* The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614)
+* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050])
 * The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124)
 
+Authentication:
+* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password
+will prevent node from starting [#118366](https://github.com/elastic/elasticsearch/pull/118366)
 
-Cluster Coordination
-:   * Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903)
+Cluster Coordination:
+* Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903)
 
+Discovery-Plugins:
+* Upgrade `discovery-ec2` to AWS SDK v2 [#122062](https://github.com/elastic/elasticsearch/pull/122062)
 
-Highlighting
-:   * Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943)
+EQL:
+* Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267)
 
+Extract&Transform:
+* Restrict Connector APIs to manage/monitor_connector privileges [#119863](https://github.com/elastic/elasticsearch/pull/119863)
 
-Indices APIs
-:   * Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923)
+Highlighting:
+* Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943)
+
+Indices APIs:
+* Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923)
 * Remove deprecated local attribute from alias APIs [#115393](https://github.com/elastic/elasticsearch/pull/115393)
+* Remove the ability to read frozen indices [#120108](https://github.com/elastic/elasticsearch/pull/120108)
+* Remove unfreeze REST endpoint [#119227](https://github.com/elastic/elasticsearch/pull/119227)
+
+Infra/Core:
+* Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026)
+* Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142)
+* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574])
+* Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103)
 
+Infra/Logging:
+* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251])
+* Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: {es-issue}125445[#125445])
 
-Infra/REST API
-:   * Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: [#89387](https://github.com/elastic/elasticsearch/issues/89387))
+Infra/Metrics:
+* Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926)
 
+Infra/REST API:
+* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387])
 
-Ingest Node
-:   * Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077)
+Ingest Node:
+* Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077)
 * Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112)
 
+Logs:
+* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489])
+
+Machine Learning:
+* Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125)
+
+Mapping:
+* Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825)
+* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596])
+
+Search:
+* Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671)
+* Change Semantic Text To Act Like A Normal Text Field [#120813](https://github.com/elastic/elasticsearch/pull/120813)
+* Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970)
+
+Snapshot/Restore:
+* Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202)
 
-Mapping
-:   * Remove support for type, fields, `copy_to` and boost in metadata field definition [#116944](https://github.com/elastic/elasticsearch/pull/116944)
+TLS:
+* Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600)
+* Remove TLSv1.1 from default protocols [#121731](https://github.com/elastic/elasticsearch/pull/121731)
 
+Transform:
+* Remove `data_frame_transforms` roles [#117519](https://github.com/elastic/elasticsearch/pull/117519)
 
-Search
-:   * Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970)
+Vector Search:
+* Remove old `_knn_search` tech preview API in v9 [#118104](https://github.com/elastic/elasticsearch/pull/118104)
 
+Watcher:
+* Removing support for types field in watcher search [#120748](https://github.com/elastic/elasticsearch/pull/120748)
 
-Snapshot/Restore
-:   * Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202)