Merge branch 'main' into add_min_score_linear_retriever

Author: mridula-s109

.buildkite/pipelines/pull-request/part-1-fips.yml

@@ -1,5 +1,7 @@
 config:
-  allow-labels: "Team:Security"
+  allow-labels:
+    - Team:Security
+    - test-fips
 steps:
   - label: part-1-fips
     command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart1

.buildkite/pipelines/pull-request/part-2-fips.yml

@@ -1,5 +1,7 @@
 config:
-  allow-labels: "Team:Security"
+  allow-labels:
+    - Team:Security
+    - test-fips
 steps:
   - label: part-2-fips
     command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart2

.buildkite/pipelines/pull-request/part-3-fips.yml

@@ -1,5 +1,7 @@
 config:
-  allow-labels: "Team:Security"
+  allow-labels:
+    - Team:Security
+    - test-fips
 steps:
   - label: part-3-fips
     command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart3

.buildkite/pipelines/pull-request/part-4-fips.yml

@@ -1,5 +1,7 @@
 config:
-  allow-labels: "Team:Security"
+  allow-labels:
+    - Team:Security
+    - test-fips
 steps:
   - label: part-4-fips
     command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart4

.buildkite/pipelines/pull-request/part-5-fips.yml

@@ -1,5 +1,7 @@
 config:
-  allow-labels: "Team:Security"
+  allow-labels:
+    - Team:Security
+    - test-fips
 steps:
   - label: part-5-fips
     command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart5

benchmarks/README.md

@@ -82,19 +82,21 @@ To get realistic results, you should exercise care when running benchmarks. Here
 NOTE: Linux only. Sorry Mac and Windows.
 
 Disassembling is fun! Maybe not always useful, but always fun! Generally, you'll want to install `perf` and the JDK's `hsdis`.
-`perf` is generally available via `apg-get install perf` or `pacman -S perf`. `hsdis` you'll want to compile from source. is a little more involved. This worked
+`perf` is generally available via `apg-get install perf` or `pacman -S perf linux-tools`. `hsdis` you'll want to compile from source. is a little more involved. This worked
 on 2020-08-01:
 
 ```
 git clone [email protected]:openjdk/jdk.git
 cd jdk
-git checkout jdk-17-ga
-cd src/utils/hsdis
+git checkout jdk-24-ga
 # Get a known good binutils
 wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.gz
 tar xf binutils-2.35.tar.gz
-make BINUTILS=binutils-2.35 ARCH=amd64
-sudo cp build/linux-amd64/hsdis-amd64.so /usr/lib/jvm/java-17-openjdk/lib/server/
+bash configure --with-hsdis=binutils --with-binutils-src=binutils-2.35 \
+    --with-boot-jdk=~/.gradle/jdks/oracle_corporation-24-amd64-linux.2
+make build-hsdis
+cp ./build/linux-x86_64-server-release/jdk/lib/hsdis-amd64.so \
+    ~/.gradle/jdks/oracle_corporation-24-amd64-linux.2/lib/hsdis.so
 ```
 
 If you want to disassemble a single method do something like this:
@@ -105,6 +107,30 @@ gradlew -p benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDia
 
 If you want `perf` to find the hot methods for you, then do add `-prof perfasm`.
 
+NOTE: `perfasm` will need more access:
+```
+sudo bash
+echo -1 > /proc/sys/kernel/perf_event_paranoid
+exit
+```
+
+If you get warnings like:
+```
+The perf event count is suspiciously low (0).
+```
+then check if you are bumping into [this](https://man.archlinux.org/man/perf-stat.1.en#INTEL_HYBRID_SUPPORT)
+by running:
+```
+perf stat -B dd if=/dev/zero of=/dev/null count=1000000
+```
+
+If you see lines like:
+```
+         765019980      cpu_atom/cycles/                 #    1.728 GHz                         (0.60%)
+        2258845959      cpu_core/cycles/                 #    5.103 GHz                         (99.18%)
+```
+then `perf` is just not going to work for you.
+
 ## Async Profiler
 
 Note: Linux and Mac only. Sorry Windows.

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ParseIpBenchmark.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.compute.operator;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ParseIp;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.net.InetAddress;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 5)
+@Measurement(iterations = 7)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(1)
+public class ParseIpBenchmark {
+    private final BytesRef ip = new BytesRef("192.168.0.1");
+    private final BreakingBytesRefBuilder scratch = ParseIp.buildScratch(new NoopCircuitBreaker("request"));
+
+    @Benchmark
+    public BytesRef leadingZerosRejected() {
+        return ParseIp.leadingZerosRejected(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreDecimal() {
+        return ParseIp.leadingZerosAreDecimal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreOctal() {
+        return ParseIp.leadingZerosAreOctal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef original() {
+        InetAddress inetAddress = InetAddresses.forString(ip.utf8ToString());
+        return new BytesRef(InetAddressPoint.encode(inetAddress));
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java

@@ -73,6 +73,16 @@ private void configureDraRepository(
                 patternLayout.artifact(
                     String.format("/%s/%s/downloads/%s/[module]/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
                 );
+
+                if ("beats".equals(draKey)) {
+                    // we don't have a good pattern here for beats fips specific images
+                    patternLayout.artifact(
+                        String.format("/%s/%s/downloads/%s/metricbeat/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
+                    );
+                    patternLayout.artifact(
+                        String.format("/%s/%s/downloads/%s/filebeat/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
+                    );
+                }
             });
             repo.metadataSources(metadataSources -> metadataSources.artifact());
             repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeVersionByRegex(".*", ".*", includeVersionRegex));

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java

@@ -8,8 +8,6 @@
  */
 package org.elasticsearch.plugin.noop;
 
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.search.SearchResponse;
@@ -41,10 +39,10 @@ public class NoopPlugin extends Plugin implements ActionPlugin {
     public static final ActionType<BulkResponse> NOOP_BULK_ACTION = new ActionType<>("mock:data/write/bulk");
 
     @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+    public List<ActionHandler> getActions() {
         return Arrays.asList(
-            new ActionHandler<>(NOOP_BULK_ACTION, TransportNoopBulkAction.class),
-            new ActionHandler<>(NOOP_SEARCH_ACTION, TransportNoopSearchAction.class)
+            new ActionHandler(NOOP_BULK_ACTION, TransportNoopBulkAction.class),
+            new ActionHandler(NOOP_SEARCH_ACTION, TransportNoopSearchAction.class)
         );
     }
 

distribution/docker/build.gradle

@@ -57,10 +57,14 @@ if (useDra == false) {
             patternLayout {
               if (VersionProperties.isElasticsearchSnapshot()) {
                 artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]'
+                artifact '/[organization]/[revision]/downloads/[organization]/filebeat/[module]-[revision]-[classifier].[ext]'
+                artifact '/[organization]/[revision]/downloads/[organization]/metricbeat/[module]-[revision]-[classifier].[ext]'
               } else {
                 // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
                 // Release builds are always done with a local repo.
                 artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/[module]/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
+                artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/filebeat/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
+                artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/metricbeat/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
               }
             }
           }
@@ -93,9 +97,13 @@ configurations {
   tini
   allPlugins
   filebeat_aarch64
+  filebeat_fips_aarch64
   filebeat_x86_64
+  filebeat_fips_x86_64
   metricbeat_aarch64
+  metricbeat_fips_aarch64
   metricbeat_x86_64
+  metricbeat_fips_x86_64
   fips
 }
 
@@ -111,8 +119,15 @@ dependencies {
   allPlugins project(path: ':plugins', configuration: 'allPlugins')
   filebeat_aarch64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
   filebeat_x86_64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
+  filebeat_fips_aarch64 "beats:filebeat-fips:${VersionProperties.elasticsearch}:[email protected]"
+  filebeat_fips_x86_64 "beats:filebeat-fips:${VersionProperties.elasticsearch}:[email protected]"
+
   metricbeat_aarch64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
   metricbeat_x86_64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
+
+  metricbeat_fips_aarch64 "beats:metricbeat-fips:${VersionProperties.elasticsearch}:[email protected]"
+  metricbeat_fips_x86_64 "beats:metricbeat-fips:${VersionProperties.elasticsearch}:[email protected]"
+
   fips "org.bouncycastle:bc-fips:1.0.2.5"
   fips "org.bouncycastle:bctls-fips:1.0.19"
 }
@@ -301,8 +316,8 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
 
         if (includeBeats) {
-          from configurations.getByName("filebeat_${architecture.classifier}")
-          from configurations.getByName("metricbeat_${architecture.classifier}")
+          from configurations.getByName("filebeat_fips_${architecture.classifier}")
+          from configurations.getByName("metricbeat_fips_${architecture.classifier}")
           // For some reason, the artifact name can differ depending on what repository we used.
           rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
         }