Merge branch 'main' into inner_hits_aggs_concurrency_bug

Author: benchaplin

README.asciidoc

@@ -97,6 +97,15 @@ From the `elastic-start-local` folder, check the connection to Elasticsearch usi
 source .env
 curl $ES_LOCAL_URL -H "Authorization: ApiKey ${ES_LOCAL_API_KEY}"
 ----
+
+To use the password for the `elastic` user, set and export the `ES_LOCAL_PASSWORD` environment variable. For example:
+
+[source,sh]
+----
+source .env
+export ES_LOCAL_PASSWORD
+----
+
 // NOTCONSOLE
 
 === Send requests to Elasticsearch
@@ -112,22 +121,23 @@ Here's an example curl command to create a new Elasticsearch index, using basic
 
 [source,sh]
 ----
-curl -u elastic:$ELASTIC_PASSWORD \
+curl -u elastic:$ES_LOCAL_PASSWORD \
   -X PUT \
   http://localhost:9200/my-new-index \
   -H 'Content-Type: application/json'
 ----
+
 // NOTCONSOLE
 
 ==== Using a language client
 
-To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable.
+To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password stored in the `ES_LOCAL_PASSWORD` environment variable.
 
 You'll use the following connection details:
 
 * **Elasticsearch endpoint**: `http://localhost:9200`
 * **Username**: `elastic`
-* **Password**: `$ELASTIC_PASSWORD` (Value you set in the environment variable)
+* **Password**: `$ES_LOCAL_PASSWORD` (Value you set in the environment variable)
 
 For example, to connect with the Python `elasticsearch` client:
 
@@ -137,7 +147,7 @@ import os
 from elasticsearch import Elasticsearch
 
 username = 'elastic'
-password = os.getenv('ELASTIC_PASSWORD') # Value you set in the environment variable
+password = os.getenv('ES_LOCAL_PASSWORD') # Value you set in the environment variable
 
 client = Elasticsearch(
     "http://localhost:9200",

TESTING.asciidoc

@@ -46,14 +46,8 @@ run it using Gradle:
 ==== Launching and debugging from an IDE
 
 If you want to run and debug Elasticsearch from your IDE, the `./gradlew run` task
-supports a remote debugging option. Run the following from your terminal:
-
----------------------------------------------------------------------------
-./gradlew run --debug-jvm
----------------------------------------------------------------------------
-
-Next start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the IDE to connect to the process and allow debug functionality.
-
+supports a remote debugging option. Start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the
+IDE to connect to the process and allow debug functionality.
 
 As such the IDE needs to be instructed to listen for connections on the debug port.
 Since we might run multiple JVMs as part of configuring and starting the cluster it's
@@ -64,6 +58,12 @@ NOTE: If you have imported the project into IntelliJ according to the instructio
 link:/CONTRIBUTING.md#importing-the-project-into-intellij-idea[CONTRIBUTING.md] then a debug run configuration
 named "Debug Elasticsearch" will be created for you and configured appropriately.
 
+Next run the following from your terminal:
+
+---------------------------------------------------------------------------
+./gradlew run --debug-jvm
+---------------------------------------------------------------------------
+
 ===== Debugging the CLI launcher
 
 The gradle task does not start the Elasticsearch server process directly; like in the Elasticsearch distribution,

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -17,7 +17,7 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.simdvec.internal.vectorization.ES91OSQVectorsScorer;
+import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;

build-conventions/settings.gradle

@@ -8,7 +8,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "3.19.2"
+  id "com.gradle.develocity" version "4.0.1"
 }
 
 rootProject.name = 'build-conventions'

build-tools-internal/settings.gradle

@@ -9,7 +9,7 @@ pluginManagement {
 }
 
 plugins {
-  id "com.gradle.develocity" version "3.19.2"
+  id "com.gradle.develocity" version "4.0.1"
 }
 
 dependencyResolutionManagement {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -14,30 +14,27 @@
  */
 public enum DockerBase {
     // "latest" here is intentional, since the image name specifies "9"
-    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"),
+    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "dockerfiles/default/Dockerfile"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"),
 
     // Chainguard based wolfi image with latest jdk
-    // This is usually updated via renovatebot
-    // spotless:off
     WOLFI(
-        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
+        null,
         "-wolfi",
         "apk",
-        "Dockerfile"
+        "dockerfiles/wolfi/Dockerfile"
     ),
-    // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.
     CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.ess"),
 
     CLOUD_ESS_FIPS(
-        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
+        null,
         "-cloud-ess-fips",
         "apk",
-        "Dockerfile.ess-fips"
+        "dockerfiles/cloud_ess_fips/Dockerfile"
     );
 
     private final String image;

build-tools/settings.gradle

@@ -10,7 +10,7 @@ pluginManagement {
     includeBuild "../build-conventions"
 }
 plugins {
-  id "com.gradle.develocity" version "3.19.2"
+  id "com.gradle.develocity" version "4.0.1"
 }
 include 'reaper'
 

distribution/docker/build.gradle

@@ -482,8 +482,10 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
 
         baseImages = [baseImage]
         buildArgs = buildArgsMap
-      } else {
+      } else if(base.image != null) {
         baseImages = [base.image]
+      } else {
+        baseImages = []
       }
 
       Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(

distribution/docker/src/docker/Dockerfile

@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.5
+ARG BASE_TAG=9.6
 <% } %>
 
 ################################################################################

distribution/docker/src/docker/Dockerfile.ess-fips renamed to distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -24,7 +24,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM ${base_image} AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -103,7 +103,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM ${base_image}
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +