Skip to content

Commit 8de1110

Browse files
mhl-bmhl-b
authored
Merge branch 'main' into gcp-fixture-multipart-refactor
2 parents 47912f5 + 1739049 commit 8de1110

File tree

60 files changed

+1442
-228
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1442
-228
lines changed

‎.ci/scripts/packaging-test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# opensuse 15 has a missing dep for systemd
44

55
if which zypper > /dev/null ; then
6-
sudo zypper install -y insserv-compat
6+
sudo zypper install -y insserv-compat docker-buildx
77
fi
88

99
if [ -e /etc/sysctl.d/99-gce.conf ]; then

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,36 +14,45 @@
1414
*/
1515
public enum DockerBase {
1616
// "latest" here is intentional, since the image name specifies "9"
17-
DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf"),
17+
DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"),
1818

1919
// The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
20-
IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
20+
IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"),
2121

2222
// Chainguard based wolfi image with latest jdk
2323
// This is usually updated via renovatebot
2424
// spotless:off
25-
WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
25+
WOLFI(
26+
"docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
2627
"-wolfi",
27-
"apk"
28+
"apk",
29+
"Dockerfile"
30+
),
31+
FIPS(
32+
"docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
33+
"-fips",
34+
"apk",
35+
"Dockerfile"
2836
),
29-
FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
3037
// spotless:on
3138
// Based on WOLFI above, with more extras. We don't set a base image because
3239
// we programmatically extend from the wolfi image.
33-
CLOUD_ESS(null, "-cloud-ess", "apk");
40+
CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.cloud-ess"),;
3441

3542
private final String image;
3643
private final String suffix;
3744
private final String packageManager;
45+
private final String dockerfile;
3846

3947
DockerBase(String image, String suffix) {
40-
this(image, suffix, "apt-get");
48+
this(image, suffix, "apt-get", "dockerfile");
4149
}
4250

43-
DockerBase(String image, String suffix, String packageManager) {
51+
DockerBase(String image, String suffix, String packageManager, String dockerfile) {
4452
this.image = image;
4553
this.suffix = suffix;
4654
this.packageManager = packageManager;
55+
this.dockerfile = dockerfile;
4756
}
4857

4958
public String getImage() {
@@ -57,4 +66,8 @@ public String getSuffix() {
5766
public String getPackageManager() {
5867
return packageManager;
5968
}
69+
70+
public String getDockerfile() {
71+
return dockerfile;
72+
}
6073
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ private void pullBaseImage(String baseImage) {
170170
maybeConfigureDockerConfig(spec);
171171
spec.executable("docker");
172172
spec.args("pull");
173+
spec.environment("DOCKER_BUILDKIT", "1");
173174
spec.args(baseImage);
174175
});
175176

@@ -205,7 +206,7 @@ public void execute() {
205206
maybeConfigureDockerConfig(spec);
206207

207208
spec.executable("docker");
208-
209+
spec.environment("DOCKER_BUILDKIT", "1");
209210
if (isCrossPlatform) {
210211
spec.args("buildx");
211212
}

‎distribution/docker/build.gradle‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -201,9 +201,10 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
201201
from projectDir.resolve("src/docker/config")
202202
}
203203
}
204-
from(projectDir.resolve("src/docker/Dockerfile")) {
204+
from(projectDir.resolve("src/docker/${base.dockerfile}")) {
205205
expand(varExpansions)
206206
filter SquashNewlinesFilter
207+
rename base.dockerfile, "Dockerfile"
207208
}
208209
}
209210
}
@@ -360,8 +361,7 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
360361
String distributionFolderName = "elasticsearch-${VersionProperties.elasticsearch}"
361362

362363
from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) {
363-
364-
if (base != DockerBase.IRON_BANK) {
364+
if (base != DockerBase.IRON_BANK && base != DockerBase.DEFAULT) {
365365
// iron bank always needs a COPY with the tarball file path
366366
eachFile { FileCopyDetails details ->
367367
if (details.name.equals("Dockerfile")) {
@@ -372,6 +372,17 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
372372
}
373373
}
374374
}
375+
if (base == DockerBase.DEFAULT) {
376+
// iron bank always needs a COPY with the tarball file path
377+
eachFile { FileCopyDetails details ->
378+
if (details.name.equals("Dockerfile")) {
379+
filter { String contents ->
380+
return contents.replaceAll('^RUN *.*artifacts-no-kpi.*$', "COPY $distributionFolderName .")
381+
.replaceAll('^RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 &&', "RUN ")
382+
}
383+
}
384+
}
385+
}
375386
}
376387
into "${project.buildDir}/docker-context/${archiveName}"
377388

‎distribution/docker/src/docker/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ LABEL name="Elasticsearch" \\
287287
description="You know, for search."
288288
<% } %>
289289

290-
RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE
290+
RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
291291
<% if (docker_base == 'iron_bank') { %>
292292
COPY LICENSE /licenses/LICENSE.addendum
293293
<% } %>

‎distribution/docker/src/docker/Dockerfile.default‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
<% /*
2+
This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes
3+
have to be escaped in order for them to appear in the final Dockerfile. You
4+
can also comment out blocks, like this one. See:
5+
6+
https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html
7+
8+
We use control-flow tags in this file to conditionally render the content. The
9+
layout/presentation here has been adjusted so that it looks reasonable when rendered,
10+
at the slight expense of how it looks here.
11+
12+
Note that this file is also filtered to squash together newlines, so we can
13+
add as many newlines here as necessary to improve legibility.
14+
*/ %>
15+
16+
17+
################################################################################
18+
# Build stage 1 `builder`:
19+
# Extract Elasticsearch artifact
20+
################################################################################
21+
22+
FROM ${base_image} AS builder
23+
24+
RUN microdnf install -y findutils tar gzip
25+
26+
# `tini` is a tiny but valid init for containers. This is used to cleanly
27+
# control how ES and any child processes are shut down.
28+
#
29+
# The tini GitHub page gives instructions for verifying the binary using
30+
# gpg, but the keyservers are slow to return the key and this can fail the
31+
# build. Instead, we check the binary against the published checksum.
32+
RUN set -eux ; \\
33+
tini_bin="" ; \\
34+
arch="\$(rpm --query --queryformat='%{ARCH}' rpm)"; \
35+
case "\$(arch)" in \\
36+
aarch64) tini_bin='tini-arm64' ;; \\
37+
x86_64) tini_bin='tini-amd64' ;; \\
38+
*) echo >&2 ; echo >&2 "Unsupported architecture \$arch" ; echo >&2 ; exit 1 ;; \\
39+
esac ; \\
40+
curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\
41+
curl -f --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\
42+
sha256sum -c \${tini_bin}.sha256sum ; \\
43+
rm \${tini_bin}.sha256sum ; \\
44+
mv \${tini_bin} /bin/tini ; \\
45+
chmod 0555 /bin/tini
46+
47+
WORKDIR /usr/share/elasticsearch
48+
RUN arch="\$(rpm --query --queryformat='%{ARCH}' rpm)" && curl -f --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\$(arch).tar.gz
49+
RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 && \\
50+
# Configure the distribution for Docker
51+
sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\
52+
# Create required directory
53+
mkdir data && \\
54+
# Reset permissions on all directories
55+
find . -type d -exec chmod 0555 {} + && \\
56+
# keep default elasticsearch log4j config
57+
mv config/log4j2.properties config/log4j2.file.properties && \\
58+
# Reset permissions on all files
59+
find . -type f -exec chmod 0444 {} + && \\
60+
# Make CLI tools executable
61+
chmod 0555 bin/* jdk/bin/* jdk/lib/jspawnhelper modules/x-pack-ml/platform/linux-*/bin/* && \\
62+
# Make some directories writable. `bin` must be writable because
63+
# plugins can install their own CLI utilities.
64+
chmod 0775 bin config config/jvm.options.d data logs plugins && \\
65+
# Make some files writable
66+
find config -type f -exec chmod 0664 {} + && \\
67+
# Tighten up permissions on the ES home dir (the permissions of the contents are handled below)
68+
chmod 0775 . && \\
69+
# You can't install plugins that include configuration when running as `elasticsearch` and the `config`
70+
# dir is owned by `root`, because the installed tries to manipulate the permissions on the plugin's
71+
# config directory.
72+
chown 1000:1000 bin config config/jvm.options.d data logs plugins
73+
74+
# The distribution includes a `config` directory, no need to create it
75+
COPY --chmod=664 config/elasticsearch.yml config/log4j2.properties config/
76+
77+
78+
################################################################################
79+
# Build stage 2 (the actual Elasticsearch image):
80+
#
81+
# Copy elasticsearch from stage 1
82+
# Add entrypoint
83+
################################################################################
84+
85+
FROM ${base_image}
86+
87+
RUN microdnf install --setopt=tsflags=nodocs -y \\
88+
nc shadow-utils zip unzip findutils procps-ng && \\
89+
microdnf clean all
90+
91+
RUN groupadd -g 1000 elasticsearch && \\
92+
adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\
93+
chown -R 0:0 /usr/share/elasticsearch
94+
95+
ENV ELASTIC_CONTAINER=true
96+
97+
COPY --from=builder /bin/tini /bin/tini
98+
99+
WORKDIR /usr/share/elasticsearch
100+
101+
COPY --from=builder --chown=0:0 /usr/share/elasticsearch .
102+
103+
# Replace OpenJDK's built-in CA certificate keystore with the one from the OS
104+
# vendor. The latter is superior in several ways.
105+
# REF: https://github.com/elastic/elasticsearch-docker/issues/171
106+
RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts jdk/lib/security/cacerts
107+
108+
ENV PATH=/usr/share/elasticsearch/bin:\$PATH
109+
ENV SHELL=/bin/bash
110+
111+
COPY --chmod=0555 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
112+
113+
RUN chmod g=u /etc/passwd && \\
114+
chmod 0555 /usr/local/bin/docker-entrypoint.sh && \\
115+
find / -xdev -perm -4000 -exec chmod ug-s {} + && \\
116+
chmod 0775 /usr/share/elasticsearch && \\
117+
chown elasticsearch bin config config/jvm.options.d data logs plugins
118+
119+
120+
EXPOSE 9200 9300
121+
122+
LABEL org.label-schema.build-date="${build_date}" \\
123+
org.label-schema.license="${license}" \\
124+
org.label-schema.name="Elasticsearch" \\
125+
org.label-schema.schema-version="1.0" \\
126+
org.label-schema.url="https://www.elastic.co/products/elasticsearch" \\
127+
org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
128+
org.label-schema.vcs-ref="${git_revision}" \\
129+
org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \\
130+
org.label-schema.vendor="Elastic" \\
131+
org.label-schema.version="${version}" \\
132+
org.opencontainers.image.created="${build_date}" \\
133+
org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
134+
org.opencontainers.image.licenses="${license}" \\
135+
org.opencontainers.image.revision="${git_revision}" \\
136+
org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \\
137+
org.opencontainers.image.title="Elasticsearch" \\
138+
org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\
139+
org.opencontainers.image.vendor="Elastic" \\
140+
org.opencontainers.image.version="${version}"
141+
142+
LABEL name="Elasticsearch" \\
143+
maintainer="[email protected]" \\
144+
vendor="Elastic" \\
145+
version="${version}" \\
146+
release="1" \\
147+
summary="Elasticsearch" \\
148+
description="You know, for search."
149+
150+
RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
151+
152+
# Our actual entrypoint is `tini`, a minimal but functional init program. It
153+
# calls the entrypoint we provide, while correctly forwarding signals.
154+
ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
155+
# Dummy overridable parameter parsed by entrypoint
156+
CMD ["eswrapper"]
157+
158+
USER 1000:0
159+
160+
################################################################################
161+
# End of multi-stage Dockerfile
162+
################################################################################

‎docs/docset.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ subs:
4040
ece: "Elastic Cloud Enterprise"
4141
eck: "Elastic Cloud on Kubernetes"
4242
ech: "Elastic Cloud Hosted"
43+
es-apis: https://www.elastic.co/docs/api/doc/elasticsearch/
44+
es-serverless-apis: https://www.elastic.co/docs/api/doc/elasticsearch-serverless/
4345
es-issue: "https://github.com/elastic/elasticsearch/issues/"
4446
serverless-full: "Elastic Cloud Serverless"
4547
serverless-short: "Serverless"

‎docs/reference/elasticsearch/rest-apis/api-conventions.md‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
---
22
mapped_pages:
33
- https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html
4+
- https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html
45
applies_to:
56
stack: all
7+
serverless: all
68
navigation_title: API conventions
79
---
810

@@ -51,7 +53,6 @@ For example, the following `traceparent` value would produce the following `trac
5153
`trace.id`: 0af7651916cd43dd8448eb211c80319c
5254
```
5355

54-
5556
## GET and POST requests [get-requests]
5657

5758
A number of {{es}} GET APIs—​most notably the search API—​support a request body. While the GET action makes sense in the context of retrieving information, GET requests with a body are not supported by all HTTP libraries. All {{es}} GET APIs that require a body can also be submitted as POST requests. Alternatively, you can pass the request body as the [`source` query string parameter](#api-request-body-query-string) when using GET.
@@ -369,7 +370,6 @@ Global index templates that match all indices are not applied to hidden indices.
369370
Direct access to system indices is deprecated and will no longer be allowed in a future major version.
370371
::::
371372

372-
373373
To view system indices within cluster:
374374

375375
```console
@@ -380,7 +380,61 @@ GET _cluster/state/metadata?filter_path=metadata.indices.*.system
380380
When overwriting current cluster state, system indices should be restored as part of their [feature state](docs-content://deploy-manage/tools/snapshot-and-restore.md#feature-state).
381381
::::
382382

383+
### Node specification [cluster-nodes]
384+
385+
Some cluster-level APIs may operate on a subset of the nodes which can be specified with node filters.
386+
For example, [task management]({{es-apis}}group/endpoint-tasks), [node stats]({{es-apis}}operation/operation-nodes-stats), and [node info]({{es-apis}}operation/operation-nodes-info-1) APIs can all report results from a filtered set of nodes rather than from all nodes.
387+
388+
Node filters are written as a comma-separated list of individual filters, each of which adds or removes nodes from the chosen subset.
389+
Each filter can be one of the following:
390+
391+
* `_all`, to add all nodes to the subset.
392+
* `_local`, to add the local node to the subset.
393+
* `_master`, to add the currently-elected master node to the subset.
394+
* a node ID or name, to add this node to the subset.
395+
* an IP address or hostname, to add all matching nodes to the subset.
396+
* a pattern, using `*` wildcards, which adds all nodes to the subset whose name, address, or hostname matches the pattern.
397+
* `master:true`, `data:true`, `ingest:true`, `voting_only:true`, `ml:true`, or `coordinating_only:true`, which respectively add to the subset all master-eligible nodes, all data nodes, all ingest nodes, all voting-only nodes, all machine learning nodes, and all coordinating-only nodes.
398+
* `master:false`, `data:false`, `ingest:false`, `voting_only:false`, `ml:false`, or `coordinating_only:false`, which respectively remove from the subset all master-eligible nodes, all data nodes, all ingest nodes, all voting-only nodes, all machine learning nodes, and all coordinating-only nodes.
399+
* a pair of patterns, using `*` wildcards, of the form `attrname:attrvalue`, which adds to the subset all nodes with a [custom node attribute](/reference/elasticsearch/configuration-reference/node-settings.md#custom-node-attributes) whose name and value match the respective patterns. Custom node attributes are configured by setting properties in the configuration file of the form `node.attr.attrname: attrvalue`.
383400

401+
Node filters run in the order in which they are given, which is important if using filters that remove nodes from the set.
402+
For example, `_all,master:false` means all the nodes except the master-eligible ones.
403+
`master:false,_all` means the same as `_all` because the `_all` filter runs after the `master:false` filter.
404+
405+
If no filters are given, the default is to select all nodes.
406+
If any filters are specified, they run starting with an empty chosen subset.
407+
This means that filters such as `master:false` which remove nodes from the chosen subset are only useful if they come after some other filters.
408+
When used on its own, `master:false` selects no nodes.
409+
410+
Here are some examples of the use of node filters with some [cluster APIs]({{es-apis}}group/endpoint-cluster):
411+
412+
```sh
413+
# If no filters are given, the default is to select all nodes
414+
GET /_nodes
415+
# Explicitly select all nodes
416+
GET /_nodes/_all
417+
# Select just the local node
418+
GET /_nodes/_local
419+
# Select the elected master node
420+
GET /_nodes/_master
421+
# Select nodes by name, which can include wildcards
422+
GET /_nodes/node_name_goes_here
423+
GET /_nodes/node_name_goes_*
424+
# Select nodes by address, which can include wildcards
425+
GET /_nodes/10.0.0.3,10.0.0.4
426+
GET /_nodes/10.0.0.*
427+
# Select nodes by role
428+
GET /_nodes/_all,master:false
429+
GET /_nodes/data:true,ingest:true
430+
GET /_nodes/coordinating_only:true
431+
GET /_nodes/master:true,voting_only:false
432+
# Select nodes by custom attribute
433+
# (for example, with something like `node.attr.rack: 2` in the configuration file)
434+
GET /_nodes/rack:2
435+
GET /_nodes/ra*:2
436+
GET /_nodes/ra*:2*
437+
```
384438

385439
## Parameters [api-conventions-parameters]
386440

0 commit comments

Comments
 (0)