Merge branch 'main' into fix-fips-artifacts-in-release-tests

Author: breskeby

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.7", "8.18.3", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -303,8 +303,8 @@ steps:
         env:
           BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.7 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.7
+      - label: "{{matrix.image}} / 8.17.8 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.8
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,7 +317,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.7
+          BWC_VERSION: 8.17.8
 
       - label: "{{matrix.image}} / 8.18.3 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.3
@@ -351,8 +351,8 @@ steps:
         env:
           BWC_VERSION: 8.19.0
 
-      - label: "{{matrix.image}} / 9.0.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.2
+      - label: "{{matrix.image}} / 9.0.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -365,7 +365,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.2
+          BWC_VERSION: 9.0.3
 
       - label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.7 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.7#bwcTest
+      - label: 8.17.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.7
+          BWC_VERSION: 8.17.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.2#bwcTest
+      - label: 9.0.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.2
+          BWC_VERSION: 9.0.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.7", "8.18.3", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -529,7 +529,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.7", "8.18.3", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -16,8 +16,8 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.6"
-  - "8.17.7"
+  - "8.17.8"
   - "8.18.3"
   - "8.19.0"
-  - "9.0.2"
+  - "9.0.3"
   - "9.1.0"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
-  - "8.17.7"
+  - "8.17.8"
   - "8.18.3"
   - "8.19.0"
-  - "9.0.2"
+  - "9.0.3"
   - "9.1.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/OptimizedTextBenchmark.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.xcontent;
+
+import org.elasticsearch.benchmark.index.mapper.MapperServiceFactory;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Benchmark to measure indexing performance of keyword fields. Used to measure performance impact of skipping
+ * UTF-8 to UTF-16 conversion during document parsing.
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Fork(1)
+@Threads(1)
+@Warmup(iterations = 1)
+@Measurement(iterations = 5)
+public class OptimizedTextBenchmark {
+    static {
+        // For Elasticsearch900Lucene101Codec:
+        LogConfigurator.loadLog4jPlugins();
+        LogConfigurator.configureESLogging();
+        LogConfigurator.setNodeName("test");
+    }
+
+    /**
+     * Total number of documents to index.
+     */
+    @Param("1048576")
+    private int nDocs;
+
+    private MapperService mapperService;
+    private SourceToParse[] sources;
+
+    private String randomValue(int length) {
+        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+        Random random = new Random();
+        StringBuilder builder = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            builder.append(CHARS.charAt(random.nextInt(CHARS.length())));
+        }
+        return builder.toString();
+    }
+
+    @Setup(Level.Trial)
+    public void setup() throws IOException {
+        mapperService = MapperServiceFactory.create("""
+            {
+                "_doc": {
+                    "dynamic": false,
+                    "properties": {
+                        "field": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """);
+
+        sources = new SourceToParse[nDocs];
+        for (int i = 0; i < nDocs; i++) {
+            XContentBuilder b = XContentFactory.jsonBuilder();
+            b.startObject().field("field", randomValue(8)).endObject();
+            sources[i] = new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(b), XContentType.JSON);
+        }
+    }
+
+    @Benchmark
+    public void indexDocuments(final Blackhole bh) {
+        final var mapper = mapperService.documentMapper();
+        for (int i = 0; i < nDocs; i++) {
+            bh.consume(mapper.parse(sources[i]));
+        }
+    }
+}

distribution/docker/src/docker/dockerfiles/default/Dockerfile

@@ -29,23 +29,20 @@ RUN microdnf install -y findutils tar gzip
 # The tini GitHub page gives instructions for verifying the binary using
 # gpg, but the keyservers are slow to return the key and this can fail the
 # build. Instead, we check the binary against the published checksum.
-RUN set -eux ; \\
-    tini_bin="" ; \\
-    arch="\$(rpm --query --queryformat='%{ARCH}' rpm)"; \
-    case "\$(arch)" in \\
-        aarch64) tini_bin='tini-arm64' ;; \\
-        x86_64)  tini_bin='tini-amd64' ;; \\
-        *) echo >&2 ; echo >&2 "Unsupported architecture \$arch" ; echo >&2 ; exit 1 ;; \\
+RUN set -eux; \\
+    arch="\$(rpm --query --queryformat='%{ARCH}' rpm)"; \\
+    case "\$arch" in \\
+        aarch64) tini_bin='tini-arm64'; tini_sum='07952557df20bfd2a95f9bef198b445e006171969499a1d361bd9e6f8e5e0e81' ;; \\
+        x86_64)  tini_bin='tini-amd64'; tini_sum='93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c' ;; \\
+        *) echo >&2 "Unsupported architecture \$arch"; exit 1 ;; \\
     esac ; \\
-    curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\
-    curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\
-    sha256sum -c \${tini_bin}.sha256sum ; \\
-    rm \${tini_bin}.sha256sum ; \\
-    mv \${tini_bin} /bin/tini ; \\
+    curl -f --retry 10 -S -L -o /tmp/tini https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}; \\
+    echo "\${tini_sum}  /tmp/tini" | sha256sum -c -; \\
+    mv /tmp/tini /bin/tini; \\
     chmod 0555 /bin/tini
 
 WORKDIR /usr/share/elasticsearch
-RUN arch="\$(rpm --query --queryformat='%{ARCH}' rpm)" && curl -f --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\$(arch).tar.gz
+RUN arch="\$(rpm --query --queryformat='%{ARCH}' rpm)" && curl -f --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\$arch.tar.gz
 RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 && \\
 # Configure the distribution for Docker
     sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\
@@ -111,12 +108,10 @@ ENV SHELL=/bin/bash
 COPY --chmod=0555 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 RUN chmod g=u /etc/passwd && \\
-    chmod 0555 /usr/local/bin/docker-entrypoint.sh && \\
     find / -xdev -perm -4000 -exec chmod ug-s {} + && \\
     chmod 0775 /usr/share/elasticsearch && \\
     chown elasticsearch bin config config/jvm.options.d data logs plugins
 
-
 EXPOSE 9200 9300
 
 LABEL org.label-schema.build-date="${build_date}" \\

docs/changelog/125699.yaml

@@ -0,0 +1,5 @@
+pr: 125699
+summary: Adding `NormalizeForStreamProcessor`
+area: Ingest Node
+type: feature
+issues: []

docs/changelog/127009.yaml

@@ -0,0 +1,6 @@
+pr: 127321
+summary: Granting `kibana_system` reserved role access to "all" privileges to `.adhoc.alerts*`
+  and `.internal.adhoc.alerts*` indices
+area: Authorization
+type: enhancement
+issues: []