Merge branch 'main' into esql_lookup_join_docs_examples

Author: craigtaverner

.ci/scripts/packaging-test.sh

@@ -3,7 +3,7 @@
 # opensuse 15 has a missing dep for systemd
 
 if which zypper > /dev/null ; then
-    sudo zypper install -y insserv-compat
+    sudo zypper install -y insserv-compat docker-buildx
 fi
 
 if [ -e /etc/sysctl.d/99-gce.conf ]; then

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -14,36 +14,45 @@
  */
 public enum DockerBase {
     // "latest" here is intentional, since the image name specifies "9"
-    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf"),
+    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
-    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
+    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"),
 
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
+    WOLFI(
+        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
         "-wolfi",
-        "apk"
+        "apk",
+        "Dockerfile"
+    ),
+    FIPS(
+        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
+        "-fips",
+        "apk",
+        "Dockerfile"
     ),
-    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.
-    CLOUD_ESS(null, "-cloud-ess", "apk");
+    CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.cloud-ess"),;
 
     private final String image;
     private final String suffix;
     private final String packageManager;
+    private final String dockerfile;
 
     DockerBase(String image, String suffix) {
-        this(image, suffix, "apt-get");
+        this(image, suffix, "apt-get", "dockerfile");
     }
 
-    DockerBase(String image, String suffix, String packageManager) {
+    DockerBase(String image, String suffix, String packageManager, String dockerfile) {
         this.image = image;
         this.suffix = suffix;
         this.packageManager = packageManager;
+        this.dockerfile = dockerfile;
     }
 
     public String getImage() {
@@ -57,4 +66,8 @@ public String getSuffix() {
     public String getPackageManager() {
         return packageManager;
     }
+
+    public String getDockerfile() {
+        return dockerfile;
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java

@@ -170,6 +170,7 @@ private void pullBaseImage(String baseImage) {
                         maybeConfigureDockerConfig(spec);
                         spec.executable("docker");
                         spec.args("pull");
+                        spec.environment("DOCKER_BUILDKIT", "1");
                         spec.args(baseImage);
                     });
 
@@ -205,7 +206,7 @@ public void execute() {
                 maybeConfigureDockerConfig(spec);
 
                 spec.executable("docker");
-
+                spec.environment("DOCKER_BUILDKIT", "1");
                 if (isCrossPlatform) {
                     spec.args("buildx");
                 }

distribution/docker/build.gradle

@@ -201,9 +201,10 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
         from projectDir.resolve("src/docker/config")
       }
     }
-    from(projectDir.resolve("src/docker/Dockerfile")) {
+    from(projectDir.resolve("src/docker/${base.dockerfile}")) {
       expand(varExpansions)
       filter SquashNewlinesFilter
+      rename base.dockerfile, "Dockerfile"
     }
   }
 }
@@ -360,8 +361,7 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     String distributionFolderName = "elasticsearch-${VersionProperties.elasticsearch}"
 
     from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) {
-
-      if (base != DockerBase.IRON_BANK) {
+      if (base != DockerBase.IRON_BANK && base != DockerBase.DEFAULT) {
         // iron bank always needs a COPY with the tarball file path
         eachFile { FileCopyDetails details ->
           if (details.name.equals("Dockerfile")) {
@@ -372,6 +372,17 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
           }
         }
       }
+      if (base == DockerBase.DEFAULT) {
+        // iron bank always needs a COPY with the tarball file path
+        eachFile { FileCopyDetails details ->
+          if (details.name.equals("Dockerfile")) {
+            filter { String contents ->
+              return contents.replaceAll('^RUN *.*artifacts-no-kpi.*$', "COPY $distributionFolderName .")
+                .replaceAll('^RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 &&', "RUN ")
+            }
+          }
+        }
+      }
     }
     into "${project.buildDir}/docker-context/${archiveName}"
 

distribution/docker/src/docker/Dockerfile

@@ -287,7 +287,7 @@ LABEL name="Elasticsearch" \\
   description="You know, for search."
 <% } %>
 
-RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE
+RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 <% if (docker_base == 'iron_bank') { %>
 COPY LICENSE /licenses/LICENSE.addendum
 <% } %>

distribution/docker/src/docker/Dockerfile.default

@@ -0,0 +1,162 @@
+<% /*
+  This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes
+  have to be escaped in order for them to appear in the final Dockerfile. You
+  can also comment out blocks, like this one. See:
+
+  https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html
+
+  We use control-flow tags in this file to conditionally render the content. The
+  layout/presentation here has been adjusted so that it looks reasonable when rendered,
+  at the slight expense of how it looks here.
+
+  Note that this file is also filtered to squash together newlines, so we can
+  add as many newlines here as necessary to improve legibility.
+*/ %>
+
+
+################################################################################
+# Build stage 1 `builder`:
+# Extract Elasticsearch artifact
+################################################################################
+
+FROM ${base_image} AS builder
+
+RUN microdnf install -y findutils tar gzip
+
+# `tini` is a tiny but valid init for containers. This is used to cleanly
+# control how ES and any child processes are shut down.
+#
+# The tini GitHub page gives instructions for verifying the binary using
+# gpg, but the keyservers are slow to return the key and this can fail the
+# build. Instead, we check the binary against the published checksum.
+RUN set -eux ; \\
+    tini_bin="" ; \\
+    arch="\$(rpm --query --queryformat='%{ARCH}' rpm)"; \
+    case "\$(arch)" in \\
+        aarch64) tini_bin='tini-arm64' ;; \\
+        x86_64)  tini_bin='tini-amd64' ;; \\
+        *) echo >&2 ; echo >&2 "Unsupported architecture \$arch" ; echo >&2 ; exit 1 ;; \\
+    esac ; \\
+    curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\
+    curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\
+    sha256sum -c \${tini_bin}.sha256sum ; \\
+    rm \${tini_bin}.sha256sum ; \\
+    mv \${tini_bin} /bin/tini ; \\
+    chmod 0555 /bin/tini
+
+WORKDIR /usr/share/elasticsearch
+RUN arch="\$(rpm --query --queryformat='%{ARCH}' rpm)" && curl -f --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\$(arch).tar.gz
+RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 && \\
+# Configure the distribution for Docker
+    sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\
+# Create required directory
+    mkdir data && \\
+# Reset permissions on all directories
+    find . -type d -exec chmod 0555 {} + && \\
+# keep default elasticsearch log4j config
+    mv config/log4j2.properties config/log4j2.file.properties && \\
+# Reset permissions on all files
+    find . -type f -exec chmod 0444 {} + && \\
+# Make CLI tools executable
+    chmod 0555 bin/* jdk/bin/* jdk/lib/jspawnhelper modules/x-pack-ml/platform/linux-*/bin/* && \\
+# Make some directories writable. `bin` must be writable because
+# plugins can install their own CLI utilities.
+    chmod 0775 bin config config/jvm.options.d data logs plugins && \\
+# Make some files writable
+    find config -type f -exec chmod 0664 {} + && \\
+# Tighten up permissions on the ES home dir (the permissions of the contents are handled below)
+    chmod 0775 . && \\
+# You can't install plugins that include configuration when running as `elasticsearch` and the `config`
+# dir is owned by `root`, because the installed tries to manipulate the permissions on the plugin's
+# config directory.
+    chown 1000:1000 bin config config/jvm.options.d data logs plugins
+
+# The distribution includes a `config` directory, no need to create it
+COPY --chmod=664 config/elasticsearch.yml config/log4j2.properties config/
+
+
+################################################################################
+# Build stage 2 (the actual Elasticsearch image):
+#
+# Copy elasticsearch from stage 1
+# Add entrypoint
+################################################################################
+
+FROM ${base_image}
+
+RUN microdnf install --setopt=tsflags=nodocs -y \\
+    nc shadow-utils zip unzip findutils procps-ng && \\
+    microdnf clean all
+
+RUN groupadd -g 1000 elasticsearch && \\
+    adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\
+    chown -R 0:0 /usr/share/elasticsearch
+
+ENV ELASTIC_CONTAINER=true
+
+COPY --from=builder /bin/tini /bin/tini
+
+WORKDIR /usr/share/elasticsearch
+
+COPY --from=builder --chown=0:0 /usr/share/elasticsearch .
+
+# Replace OpenJDK's built-in CA certificate keystore with the one from the OS
+# vendor. The latter is superior in several ways.
+# REF: https://github.com/elastic/elasticsearch-docker/issues/171
+RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts jdk/lib/security/cacerts
+
+ENV PATH=/usr/share/elasticsearch/bin:\$PATH
+ENV SHELL=/bin/bash
+
+COPY --chmod=0555 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+RUN chmod g=u /etc/passwd && \\
+    chmod 0555 /usr/local/bin/docker-entrypoint.sh && \\
+    find / -xdev -perm -4000 -exec chmod ug-s {} + && \\
+    chmod 0775 /usr/share/elasticsearch && \\
+    chown elasticsearch bin config config/jvm.options.d data logs plugins
+
+
+EXPOSE 9200 9300
+
+LABEL org.label-schema.build-date="${build_date}" \\
+  org.label-schema.license="${license}" \\
+  org.label-schema.name="Elasticsearch" \\
+  org.label-schema.schema-version="1.0" \\
+  org.label-schema.url="https://www.elastic.co/products/elasticsearch" \\
+  org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
+  org.label-schema.vcs-ref="${git_revision}" \\
+  org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \\
+  org.label-schema.vendor="Elastic" \\
+  org.label-schema.version="${version}" \\
+  org.opencontainers.image.created="${build_date}" \\
+  org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
+  org.opencontainers.image.licenses="${license}" \\
+  org.opencontainers.image.revision="${git_revision}" \\
+  org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \\
+  org.opencontainers.image.title="Elasticsearch" \\
+  org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\
+  org.opencontainers.image.vendor="Elastic" \\
+  org.opencontainers.image.version="${version}"
+
+LABEL name="Elasticsearch" \\
+  maintainer="[email protected]" \\
+  vendor="Elastic" \\
+  version="${version}" \\
+  release="1" \\
+  summary="Elasticsearch" \\
+  description="You know, for search."
+
+RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
+
+# Our actual entrypoint is `tini`, a minimal but functional init program. It
+# calls the entrypoint we provide, while correctly forwarding signals.
+ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
+# Dummy overridable parameter parsed by entrypoint
+CMD ["eswrapper"]
+
+USER 1000:0
+
+################################################################################
+# End of multi-stage Dockerfile
+################################################################################

docs/changelog/126273.yaml

@@ -0,0 +1,5 @@
+pr: 126273
+summary: Fix LTR rescorer with model alias
+area: Ranking
+type: bug
+issues: []

docs/changelog/126319.yaml

@@ -0,0 +1,5 @@
+pr: 126319
+summary: COMPLETION command grammar and logical plan
+area: ES|QL
+type: feature
+issues: []

docs/docset.yml

@@ -40,6 +40,8 @@ subs:
   ece:   "Elastic Cloud Enterprise"
   eck:   "Elastic Cloud on Kubernetes"
   ech:   "Elastic Cloud Hosted"
+  es-apis: https://www.elastic.co/docs/api/doc/elasticsearch/
+  es-serverless-apis: https://www.elastic.co/docs/api/doc/elasticsearch-serverless/
   es-issue:   "https://github.com/elastic/elasticsearch/issues/"
   serverless-full:   "Elastic Cloud Serverless"
   serverless-short:   "Serverless"

docs/internal/DistributedArchitectureGuide.md

@@ -22,50 +22,6 @@ See the [Javadocs for `ActionListener`](https://github.com/elastic/elasticsearch
 
 (TODO: add useful starter references and explanations for a range of Listener classes. Reference the Netty section.)
 
-### REST Layer
-
-The REST and Transport layers are bound together through the `ActionModule`. `ActionModule#initRestHandlers` registers all the
-rest actions with a `RestController` that matches incoming requests to particular REST actions. `RestController#registerHandler`
-uses each `Rest*Action`'s `#routes()` implementation to match HTTP requests to that particular `Rest*Action`. Typically, REST
-actions follow the class naming convention `Rest*Action`, which makes them easier to find, but not always; the `#routes()`
-definition can also be helpful in finding a REST action. `RestController#dispatchRequest` eventually calls `#handleRequest` on a
-`RestHandler` implementation. `RestHandler` is the base class for `BaseRestHandler`, which most `Rest*Action` instances extend to
-implement a particular REST action.
-
-`BaseRestHandler#handleRequest` calls into `BaseRestHandler#prepareRequest`, which children `Rest*Action` classes extend to
-define the behavior for a particular action. `RestController#dispatchRequest` passes a `RestChannel` to the `Rest*Action` via
-`RestHandler#handleRequest`: `Rest*Action#prepareRequest` implementations return a `RestChannelConsumer` defining how to execute
-the action and reply on the channel (usually in the form of completing an ActionListener wrapper). `Rest*Action#prepareRequest`
-implementations are responsible for parsing the incoming request, and verifying that the structure of the request is valid.
-`BaseRestHandler#handleRequest` will then check that all the request parameters have been consumed: unexpected request parameters
-result in an error.
-
-### How REST Actions Connect to Transport Actions
-
-The Rest layer uses an implementation of `AbstractClient`. `BaseRestHandler#prepareRequest` takes a `NodeClient`: this client
-knows how to connect to a specified TransportAction. A `Rest*Action` implementation will return a `RestChannelConsumer` that
-most often invokes a call into a method on the `NodeClient` to pass through to the TransportAction. Along the way from
-`BaseRestHandler#prepareRequest` through the `AbstractClient` and `NodeClient` code, `NodeClient#executeLocally` is called: this
-method calls into `TaskManager#registerAndExecute`, registering the operation with the `TaskManager` so it can be found in Task
-API requests, before moving on to execute the specified TransportAction.
-
-`NodeClient` has a `NodeClient#actions` map from `ActionType` to `TransportAction`. `ActionModule#setupActions` registers all the
-core TransportActions, as well as those defined in any plugins that are being used: plugins can override `Plugin#getActions()` to
-define additional TransportActions. Note that not all TransportActions will be mapped back to a REST action: many TransportActions
-are only used for internode operations/communications.
-
-### Transport Layer
-
-(Managed by the TransportService, TransportActions must be registered there, too)
-
-(Executing a TransportAction (either locally via NodeClient or remotely via TransportService) is where most of the authorization & other security logic runs)
-
-(What actions, and why, are registered in TransportService but not NodeClient?)
-
-### Direct Node to Node Transport Layer
-
-(TransportService maps incoming requests to TransportActions)
-
 ### Chunk Encoding
 
 #### XContent