Merge remote-tracking branch 'es/main' into ds_rollover_tsdb_to_logsdb

Author: martijnvg

.ci/scripts/packaging-test.sh

@@ -3,7 +3,7 @@
 # opensuse 15 has a missing dep for systemd
 
 if which zypper > /dev/null ; then
-    sudo zypper install -y insserv-compat
+    sudo zypper install -y insserv-compat docker-buildx
 fi
 
 if [ -e /etc/sysctl.d/99-gce.conf ]; then

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -14,36 +14,45 @@
  */
 public enum DockerBase {
     // "latest" here is intentional, since the image name specifies "9"
-    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf"),
+    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
-    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
+    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"),
 
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
+    WOLFI(
+        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
         "-wolfi",
-        "apk"
+        "apk",
+        "Dockerfile"
+    ),
+    FIPS(
+        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
+        "-fips",
+        "apk",
+        "Dockerfile"
     ),
-    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.
-    CLOUD_ESS(null, "-cloud-ess", "apk");
+    CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.cloud-ess"),;
 
     private final String image;
     private final String suffix;
     private final String packageManager;
+    private final String dockerfile;
 
     DockerBase(String image, String suffix) {
-        this(image, suffix, "apt-get");
+        this(image, suffix, "apt-get", "dockerfile");
     }
 
-    DockerBase(String image, String suffix, String packageManager) {
+    DockerBase(String image, String suffix, String packageManager, String dockerfile) {
         this.image = image;
         this.suffix = suffix;
         this.packageManager = packageManager;
+        this.dockerfile = dockerfile;
     }
 
     public String getImage() {
@@ -57,4 +66,8 @@ public String getSuffix() {
     public String getPackageManager() {
         return packageManager;
     }
+
+    public String getDockerfile() {
+        return dockerfile;
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java

@@ -170,6 +170,7 @@ private void pullBaseImage(String baseImage) {
                         maybeConfigureDockerConfig(spec);
                         spec.executable("docker");
                         spec.args("pull");
+                        spec.environment("DOCKER_BUILDKIT", "1");
                         spec.args(baseImage);
                     });
 
@@ -205,7 +206,7 @@ public void execute() {
                 maybeConfigureDockerConfig(spec);
 
                 spec.executable("docker");
-
+                spec.environment("DOCKER_BUILDKIT", "1");
                 if (isCrossPlatform) {
                     spec.args("buildx");
                 }

distribution/docker/build.gradle

@@ -201,9 +201,10 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
         from projectDir.resolve("src/docker/config")
       }
     }
-    from(projectDir.resolve("src/docker/Dockerfile")) {
+    from(projectDir.resolve("src/docker/${base.dockerfile}")) {
       expand(varExpansions)
       filter SquashNewlinesFilter
+      rename base.dockerfile, "Dockerfile"
     }
   }
 }
@@ -360,8 +361,7 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     String distributionFolderName = "elasticsearch-${VersionProperties.elasticsearch}"
 
     from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) {
-
-      if (base != DockerBase.IRON_BANK) {
+      if (base != DockerBase.IRON_BANK && base != DockerBase.DEFAULT) {
         // iron bank always needs a COPY with the tarball file path
         eachFile { FileCopyDetails details ->
           if (details.name.equals("Dockerfile")) {
@@ -372,6 +372,17 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
           }
         }
       }
+      if (base == DockerBase.DEFAULT) {
+        // iron bank always needs a COPY with the tarball file path
+        eachFile { FileCopyDetails details ->
+          if (details.name.equals("Dockerfile")) {
+            filter { String contents ->
+              return contents.replaceAll('^RUN *.*artifacts-no-kpi.*$', "COPY $distributionFolderName .")
+                .replaceAll('^RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 &&', "RUN ")
+            }
+          }
+        }
+      }
     }
     into "${project.buildDir}/docker-context/${archiveName}"
 

distribution/docker/src/docker/Dockerfile

@@ -287,7 +287,7 @@ LABEL name="Elasticsearch" \\
   description="You know, for search."
 <% } %>
 
-RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE
+RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 <% if (docker_base == 'iron_bank') { %>
 COPY LICENSE /licenses/LICENSE.addendum
 <% } %>

distribution/docker/src/docker/Dockerfile.default

@@ -0,0 +1,162 @@
+<% /*
+  This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes
+  have to be escaped in order for them to appear in the final Dockerfile. You
+  can also comment out blocks, like this one. See:
+
+  https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html
+
+  We use control-flow tags in this file to conditionally render the content. The
+  layout/presentation here has been adjusted so that it looks reasonable when rendered,
+  at the slight expense of how it looks here.
+
+  Note that this file is also filtered to squash together newlines, so we can
+  add as many newlines here as necessary to improve legibility.
+*/ %>
+
+
+################################################################################
+# Build stage 1 `builder`:
+# Extract Elasticsearch artifact
+################################################################################
+
+FROM ${base_image} AS builder
+
+RUN microdnf install -y findutils tar gzip
+
+# `tini` is a tiny but valid init for containers. This is used to cleanly
+# control how ES and any child processes are shut down.
+#
+# The tini GitHub page gives instructions for verifying the binary using
+# gpg, but the keyservers are slow to return the key and this can fail the
+# build. Instead, we check the binary against the published checksum.
+RUN set -eux ; \\
+    tini_bin="" ; \\
+    arch="\$(rpm --query --queryformat='%{ARCH}' rpm)"; \
+    case "\$(arch)" in \\
+        aarch64) tini_bin='tini-arm64' ;; \\
+        x86_64)  tini_bin='tini-amd64' ;; \\
+        *) echo >&2 ; echo >&2 "Unsupported architecture \$arch" ; echo >&2 ; exit 1 ;; \\
+    esac ; \\
+    curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\
+    curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\
+    sha256sum -c \${tini_bin}.sha256sum ; \\
+    rm \${tini_bin}.sha256sum ; \\
+    mv \${tini_bin} /bin/tini ; \\
+    chmod 0555 /bin/tini
+
+WORKDIR /usr/share/elasticsearch
+RUN arch="\$(rpm --query --queryformat='%{ARCH}' rpm)" && curl -f --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\$(arch).tar.gz
+RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 && \\
+# Configure the distribution for Docker
+    sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\
+# Create required directory
+    mkdir data && \\
+# Reset permissions on all directories
+    find . -type d -exec chmod 0555 {} + && \\
+# keep default elasticsearch log4j config
+    mv config/log4j2.properties config/log4j2.file.properties && \\
+# Reset permissions on all files
+    find . -type f -exec chmod 0444 {} + && \\
+# Make CLI tools executable
+    chmod 0555 bin/* jdk/bin/* jdk/lib/jspawnhelper modules/x-pack-ml/platform/linux-*/bin/* && \\
+# Make some directories writable. `bin` must be writable because
+# plugins can install their own CLI utilities.
+    chmod 0775 bin config config/jvm.options.d data logs plugins && \\
+# Make some files writable
+    find config -type f -exec chmod 0664 {} + && \\
+# Tighten up permissions on the ES home dir (the permissions of the contents are handled below)
+    chmod 0775 . && \\
+# You can't install plugins that include configuration when running as `elasticsearch` and the `config`
+# dir is owned by `root`, because the installed tries to manipulate the permissions on the plugin's
+# config directory.
+    chown 1000:1000 bin config config/jvm.options.d data logs plugins
+
+# The distribution includes a `config` directory, no need to create it
+COPY --chmod=664 config/elasticsearch.yml config/log4j2.properties config/
+
+
+################################################################################
+# Build stage 2 (the actual Elasticsearch image):
+#
+# Copy elasticsearch from stage 1
+# Add entrypoint
+################################################################################
+
+FROM ${base_image}
+
+RUN microdnf install --setopt=tsflags=nodocs -y \\
+    nc shadow-utils zip unzip findutils procps-ng && \\
+    microdnf clean all
+
+RUN groupadd -g 1000 elasticsearch && \\
+    adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\
+    chown -R 0:0 /usr/share/elasticsearch
+
+ENV ELASTIC_CONTAINER=true
+
+COPY --from=builder /bin/tini /bin/tini
+
+WORKDIR /usr/share/elasticsearch
+
+COPY --from=builder --chown=0:0 /usr/share/elasticsearch .
+
+# Replace OpenJDK's built-in CA certificate keystore with the one from the OS
+# vendor. The latter is superior in several ways.
+# REF: https://github.com/elastic/elasticsearch-docker/issues/171
+RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts jdk/lib/security/cacerts
+
+ENV PATH=/usr/share/elasticsearch/bin:\$PATH
+ENV SHELL=/bin/bash
+
+COPY --chmod=0555 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+RUN chmod g=u /etc/passwd && \\
+    chmod 0555 /usr/local/bin/docker-entrypoint.sh && \\
+    find / -xdev -perm -4000 -exec chmod ug-s {} + && \\
+    chmod 0775 /usr/share/elasticsearch && \\
+    chown elasticsearch bin config config/jvm.options.d data logs plugins
+
+
+EXPOSE 9200 9300
+
+LABEL org.label-schema.build-date="${build_date}" \\
+  org.label-schema.license="${license}" \\
+  org.label-schema.name="Elasticsearch" \\
+  org.label-schema.schema-version="1.0" \\
+  org.label-schema.url="https://www.elastic.co/products/elasticsearch" \\
+  org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
+  org.label-schema.vcs-ref="${git_revision}" \\
+  org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \\
+  org.label-schema.vendor="Elastic" \\
+  org.label-schema.version="${version}" \\
+  org.opencontainers.image.created="${build_date}" \\
+  org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
+  org.opencontainers.image.licenses="${license}" \\
+  org.opencontainers.image.revision="${git_revision}" \\
+  org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \\
+  org.opencontainers.image.title="Elasticsearch" \\
+  org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\
+  org.opencontainers.image.vendor="Elastic" \\
+  org.opencontainers.image.version="${version}"
+
+LABEL name="Elasticsearch" \\
+  maintainer="[email protected]" \\
+  vendor="Elastic" \\
+  version="${version}" \\
+  release="1" \\
+  summary="Elasticsearch" \\
+  description="You know, for search."
+
+RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
+
+# Our actual entrypoint is `tini`, a minimal but functional init program. It
+# calls the entrypoint we provide, while correctly forwarding signals.
+ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
+# Dummy overridable parameter parsed by entrypoint
+CMD ["eswrapper"]
+
+USER 1000:0
+
+################################################################################
+# End of multi-stage Dockerfile
+################################################################################

docs/docset.yml

@@ -40,6 +40,8 @@ subs:
   ece:   "Elastic Cloud Enterprise"
   eck:   "Elastic Cloud on Kubernetes"
   ech:   "Elastic Cloud Hosted"
+  es-apis: https://www.elastic.co/docs/api/doc/elasticsearch/
+  es-serverless-apis: https://www.elastic.co/docs/api/doc/elasticsearch-serverless/
   es-issue:   "https://github.com/elastic/elasticsearch/issues/"
   serverless-full:   "Elastic Cloud Serverless"
   serverless-short:   "Serverless"

docs/reference/elasticsearch/rest-apis/api-conventions.md

@@ -1,8 +1,10 @@
 ---
 mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html
 applies_to:
   stack: all
+  serverless: all
 navigation_title: API conventions
 ---
 
@@ -51,7 +53,6 @@ For example, the following `traceparent` value would produce the following `trac
 `trace.id`: 0af7651916cd43dd8448eb211c80319c
 ```
 
-
 ## GET and POST requests [get-requests]
 
 A number of {{es}} GET APIs—​most notably the search API—​support a request body. While the GET action makes sense in the context of retrieving information, GET requests with a body are not supported by all HTTP libraries. All {{es}} GET APIs that require a body can also be submitted as POST requests. Alternatively, you can pass the request body as the [`source` query string parameter](#api-request-body-query-string) when using GET.
@@ -369,7 +370,6 @@ Global index templates that match all indices are not applied to hidden indices.
 Direct access to system indices is deprecated and will no longer be allowed in a future major version.
 ::::
 
-
 To view system indices within cluster:
 
 ```console
@@ -380,7 +380,61 @@ GET _cluster/state/metadata?filter_path=metadata.indices.*.system
 When overwriting current cluster state, system indices should be restored as part of their [feature state](docs-content://deploy-manage/tools/snapshot-and-restore.md#feature-state).
 ::::
 
+### Node specification [cluster-nodes]
+
+Some cluster-level APIs may operate on a subset of the nodes which can be specified with node filters.
+For example,  [task management]({{es-apis}}group/endpoint-tasks), [node stats]({{es-apis}}operation/operation-nodes-stats), and [node info]({{es-apis}}operation/operation-nodes-info-1) APIs can all report results from a filtered set of nodes rather than from all nodes.
+
+Node filters are written as a comma-separated list of individual filters, each of which adds or removes nodes from the chosen subset.
+Each filter can be one of the following:
+
+* `_all`, to add all nodes to the subset.
+* `_local`, to add the local node to the subset.
+* `_master`, to add the currently-elected master node to the subset.
+* a node ID or name, to add this node to the subset.
+* an IP address or hostname, to add all matching nodes to the subset.
+* a pattern, using `*` wildcards, which adds all nodes to the subset whose name, address, or hostname matches the pattern.
+* `master:true`, `data:true`, `ingest:true`, `voting_only:true`, `ml:true`, or `coordinating_only:true`, which respectively add to the subset all master-eligible nodes, all data nodes, all ingest nodes, all voting-only nodes, all machine learning nodes, and all coordinating-only nodes.
+* `master:false`, `data:false`, `ingest:false`, `voting_only:false`, `ml:false`, or `coordinating_only:false`, which respectively remove from the subset all master-eligible nodes, all data nodes, all ingest nodes, all voting-only nodes, all machine learning nodes, and all coordinating-only nodes.
+* a pair of patterns, using `*` wildcards, of the form `attrname:attrvalue`, which adds to the subset all nodes with a [custom node attribute](/reference/elasticsearch/configuration-reference/node-settings.md#custom-node-attributes) whose name and value match the respective patterns. Custom node attributes are configured by setting properties in the configuration file of the form `node.attr.attrname: attrvalue`.
 
+Node filters run in the order in which they are given, which is important if using filters that remove nodes from the set.
+For example, `_all,master:false` means all the nodes except the master-eligible ones.
+`master:false,_all` means the same as `_all` because the `_all` filter runs after the `master:false` filter.
+
+If no filters are given, the default is to select all nodes.
+If any filters are specified, they run starting with an empty chosen subset.
+This means that filters such as `master:false` which remove nodes from the chosen subset are only useful if they come after some other filters.
+When used on its own, `master:false` selects no nodes.
+
+Here are some examples of the use of node filters with some [cluster APIs]({{es-apis}}group/endpoint-cluster):
+
+```sh
+# If no filters are given, the default is to select all nodes
+GET /_nodes
+# Explicitly select all nodes
+GET /_nodes/_all
+# Select just the local node
+GET /_nodes/_local
+# Select the elected master node
+GET /_nodes/_master
+# Select nodes by name, which can include wildcards
+GET /_nodes/node_name_goes_here
+GET /_nodes/node_name_goes_*
+# Select nodes by address, which can include wildcards
+GET /_nodes/10.0.0.3,10.0.0.4
+GET /_nodes/10.0.0.*
+# Select nodes by role
+GET /_nodes/_all,master:false
+GET /_nodes/data:true,ingest:true
+GET /_nodes/coordinating_only:true
+GET /_nodes/master:true,voting_only:false
+# Select nodes by custom attribute
+# (for example, with something like `node.attr.rack: 2` in the configuration file)
+GET /_nodes/rack:2
+GET /_nodes/ra*:2
+GET /_nodes/ra*:2*
+```
 
 ## Parameters [api-conventions-parameters]