Skip to content

Commit f5c4501

Browse files
kkrik-eskkrik-es
committed
Merge branch 'main' into synthetic-source/second-pass-flag
# Conflicts: # server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java # server/src/main/java/org/elasticsearch/index/IndexSettings.java
2 parents c55f8c9 + a262eb6 commit f5c4501

File tree

242 files changed

+9683
-3583
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

242 files changed

+9683
-3583
lines changed

‎.buildkite/pipelines/pull-request/packaging-tests-unix.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ steps:
55
steps:
66
- label: "{{matrix.image}} / docker / packaging-tests-unix"
77
key: "packaging-tests-unix-docker"
8-
command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
8+
command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess
99
timeout_in_minutes: 300
1010
matrix:
1111
setup:

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,13 @@
3030
import org.elasticsearch.compute.data.BooleanBlock;
3131
import org.elasticsearch.compute.data.BooleanVector;
3232
import org.elasticsearch.compute.data.BytesRefBlock;
33+
import org.elasticsearch.compute.data.BytesRefVector;
3334
import org.elasticsearch.compute.data.DoubleBlock;
3435
import org.elasticsearch.compute.data.ElementType;
3536
import org.elasticsearch.compute.data.IntBlock;
37+
import org.elasticsearch.compute.data.IntVector;
3638
import org.elasticsearch.compute.data.LongBlock;
39+
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
3740
import org.elasticsearch.compute.data.Page;
3841
import org.elasticsearch.compute.operator.AggregationOperator;
3942
import org.elasticsearch.compute.operator.DriverContext;
@@ -78,7 +81,10 @@ public class AggregatorBenchmark {
7881
private static final String DOUBLES = "doubles";
7982
private static final String BOOLEANS = "booleans";
8083
private static final String BYTES_REFS = "bytes_refs";
84+
private static final String ORDINALS = "ordinals";
8185
private static final String TWO_LONGS = "two_" + LONGS;
86+
private static final String TWO_BYTES_REFS = "two_" + BYTES_REFS;
87+
private static final String TWO_ORDINALS = "two_" + ORDINALS;
8288
private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS;
8389
private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS;
8490

@@ -119,7 +125,21 @@ public class AggregatorBenchmark {
119125
}
120126
}
121127

122-
@Param({ NONE, LONGS, INTS, DOUBLES, BOOLEANS, BYTES_REFS, TWO_LONGS, LONGS_AND_BYTES_REFS, TWO_LONGS_AND_BYTES_REFS })
128+
@Param(
129+
{
130+
NONE,
131+
LONGS,
132+
INTS,
133+
DOUBLES,
134+
BOOLEANS,
135+
BYTES_REFS,
136+
ORDINALS,
137+
TWO_LONGS,
138+
TWO_BYTES_REFS,
139+
TWO_ORDINALS,
140+
LONGS_AND_BYTES_REFS,
141+
TWO_LONGS_AND_BYTES_REFS }
142+
)
123143
public String grouping;
124144

125145
@Param({ COUNT, COUNT_DISTINCT, MIN, MAX, SUM })
@@ -144,8 +164,12 @@ private static Operator operator(DriverContext driverContext, String grouping, S
144164
case INTS -> List.of(new BlockHash.GroupSpec(0, ElementType.INT));
145165
case DOUBLES -> List.of(new BlockHash.GroupSpec(0, ElementType.DOUBLE));
146166
case BOOLEANS -> List.of(new BlockHash.GroupSpec(0, ElementType.BOOLEAN));
147-
case BYTES_REFS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF));
167+
case BYTES_REFS, ORDINALS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF));
148168
case TWO_LONGS -> List.of(new BlockHash.GroupSpec(0, ElementType.LONG), new BlockHash.GroupSpec(1, ElementType.LONG));
169+
case TWO_BYTES_REFS, TWO_ORDINALS -> List.of(
170+
new BlockHash.GroupSpec(0, ElementType.BYTES_REF),
171+
new BlockHash.GroupSpec(1, ElementType.BYTES_REF)
172+
);
149173
case LONGS_AND_BYTES_REFS -> List.of(
150174
new BlockHash.GroupSpec(0, ElementType.LONG),
151175
new BlockHash.GroupSpec(1, ElementType.BYTES_REF)
@@ -218,6 +242,10 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
218242
checkGroupingBlock(prefix, LONGS, page.getBlock(0));
219243
checkGroupingBlock(prefix, LONGS, page.getBlock(1));
220244
}
245+
case TWO_BYTES_REFS, TWO_ORDINALS -> {
246+
checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(0));
247+
checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1));
248+
}
221249
case LONGS_AND_BYTES_REFS -> {
222250
checkGroupingBlock(prefix, LONGS, page.getBlock(0));
223251
checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1));
@@ -379,7 +407,7 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo
379407
throw new AssertionError(prefix + "bad group expected [true] but was [" + groups.getBoolean(1) + "]");
380408
}
381409
}
382-
case BYTES_REFS -> {
410+
case BYTES_REFS, ORDINALS -> {
383411
BytesRefBlock groups = (BytesRefBlock) block;
384412
for (int g = 0; g < GROUPS; g++) {
385413
if (false == groups.getBytesRef(g, new BytesRef()).equals(bytesGroup(g))) {
@@ -508,6 +536,8 @@ private static Block dataBlock(BlockFactory blockFactory, String blockType) {
508536
private static List<Block> groupingBlocks(String grouping, String blockType) {
509537
return switch (grouping) {
510538
case TWO_LONGS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(LONGS, blockType));
539+
case TWO_BYTES_REFS -> List.of(groupingBlock(BYTES_REFS, blockType), groupingBlock(BYTES_REFS, blockType));
540+
case TWO_ORDINALS -> List.of(groupingBlock(ORDINALS, blockType), groupingBlock(ORDINALS, blockType));
511541
case LONGS_AND_BYTES_REFS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(BYTES_REFS, blockType));
512542
case TWO_LONGS_AND_BYTES_REFS -> List.of(
513543
groupingBlock(LONGS, blockType),
@@ -570,6 +600,19 @@ private static Block groupingBlock(String grouping, String blockType) {
570600
}
571601
yield builder.build();
572602
}
603+
case ORDINALS -> {
604+
IntVector.Builder ordinals = blockFactory.newIntVectorBuilder(BLOCK_LENGTH * valuesPerGroup);
605+
for (int i = 0; i < BLOCK_LENGTH; i++) {
606+
for (int v = 0; v < valuesPerGroup; v++) {
607+
ordinals.appendInt(i % GROUPS);
608+
}
609+
}
610+
BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH * valuesPerGroup);
611+
for (int i = 0; i < GROUPS; i++) {
612+
bytes.appendBytesRef(bytesGroup(i));
613+
}
614+
yield new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock();
615+
}
573616
default -> throw new UnsupportedOperationException("unsupported grouping [" + grouping + "]");
574617
};
575618
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,6 @@ public enum DockerBase {
2424
// Base image with extras for Cloud
2525
CLOUD("ubuntu:20.04", "-cloud", "apt-get"),
2626

27-
// Based on CLOUD above, with more extras. We don't set a base image because
28-
// we programmatically extend from the Cloud image.
29-
CLOUD_ESS(null, "-cloud-ess", "apt-get"),
30-
3127
// Chainguard based wolfi image with latest jdk
3228
// This is usually updated via renovatebot
3329
// spotless:off
@@ -36,10 +32,9 @@ public enum DockerBase {
3632
"apk"
3733
),
3834
// spotless:on
39-
4035
// Based on WOLFI above, with more extras. We don't set a base image because
41-
// we programmatically extend from the Wolfi image.
42-
WOLFI_ESS(null, "-wolfi-ess", "apk");
36+
// we programmatically extend from the wolfi image.
37+
CLOUD_ESS(null, "-cloud-ess", "apk");
4338

4439
private final String image;
4540
private final String suffix;

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -181,9 +181,6 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
181181
if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) {
182182
return projectName + "wolfi-docker" + archString + "-export";
183183
}
184-
if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS) {
185-
return projectName + "wolfi-ess-docker" + archString + "-export";
186-
}
187184
return projectName + distribution.getType().getName();
188185
}
189186

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ public class InternalElasticsearchDistributionTypes {
2222
public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType();
2323
public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
2424
public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
25-
public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType();
2625

2726
public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
2827
DEB,
@@ -32,7 +31,6 @@ public class InternalElasticsearchDistributionTypes {
3231
DOCKER_IRONBANK,
3332
DOCKER_CLOUD,
3433
DOCKER_CLOUD_ESS,
35-
DOCKER_WOLFI,
36-
DOCKER_WOLFI_ESS
34+
DOCKER_WOLFI
3735
);
3836
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@
5454
import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
5555
import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
5656
import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
57-
import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS;
5857
import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
5958

6059
/**
@@ -153,7 +152,6 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
153152
lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud"));
154153
lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
155154
lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
156-
lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess"));
157155
lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
158156
lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
159157
lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

‎distribution/docker/README.md‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ the [DockerBase] enum.
77
* UBI - the same as the default image, but based upon [RedHat's UBI
88
images][ubi], specifically their minimal flavour.
99
* Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
10-
* Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins
10+
* Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins
1111
that the ES build generates in an archive directory. It also sets an
1212
environment variable that points at this directory. This allows plugins to
1313
be installed from the archive instead of the internet, speeding up
@@ -23,20 +23,13 @@ the [DockerBase] enum.
2323
software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
2424
another UBI build, this time on the regular UBI image, with extra
2525
hardening. See below for more details.
26-
2726
* Cloud - this is mostly the same as the default image, with some notable differences:
2827
* `filebeat` and `metricbeat` are included
2928
* `wget` is included
3029
* The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is
3130
`/app/elasticsearch.sh`. In normal use this file would be bind-mounted
3231
in, but the image ships a stub version of this file so that the image
3332
can still be tested.
34-
* Cloud ESS - this directly extends the Cloud image, and adds all ES plugins
35-
that the ES build generates in an archive directory. It also sets an
36-
environment variable that points at this directory. This allows plugins to
37-
be installed from the archive instead of the internet, speeding up
38-
deployment times.
39-
4033
The long-term goal is for both Cloud images to be retired in favour of the
4134
default image.
4235

‎distribution/docker/build.gradle‎

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
381381
String image = "elasticsearch${base.suffix}"
382382

383383
String namespace = 'elasticsearch'
384-
if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
384+
if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) {
385385
namespace += '-ci'
386386
}
387387

@@ -446,7 +446,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
446446
}
447447
}
448448

449-
void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) {
449+
void addBuildEssDockerImageTask(Architecture architecture) {
450+
DockerBase dockerBase = DockerBase.CLOUD_ESS
450451
String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
451452
String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
452453

@@ -460,22 +461,20 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
460461
from configurations.allPlugins
461462
}
462463

463-
if (dockerBase == DockerBase.WOLFI_ESS) {
464-
// If we're performing a release build, but `build.id` hasn't been set, we can
465-
// infer that we're not at the Docker building stage of the build, and therefore
466-
// we should skip the beats part of the build.
467-
String buildId = providers.systemProperty('build.id').getOrNull()
468-
boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
464+
// If we're performing a release build, but `build.id` hasn't been set, we can
465+
// infer that we're not at the Docker building stage of the build, and therefore
466+
// we should skip the beats part of the build.
467+
String buildId = providers.systemProperty('build.id').getOrNull()
468+
boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
469469

470-
if (includeBeats) {
471-
from configurations.getByName("filebeat_${architecture.classifier}")
472-
from configurations.getByName("metricbeat_${architecture.classifier}")
473-
}
474-
// For some reason, the artifact name can differ depending on what repository we used.
475-
rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
470+
if (includeBeats) {
471+
from configurations.getByName("filebeat_${architecture.classifier}")
472+
from configurations.getByName("metricbeat_${architecture.classifier}")
476473
}
474+
// For some reason, the artifact name can differ depending on what repository we used.
475+
rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
477476

478-
String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix
477+
String baseSuffix = DockerBase.WOLFI.suffix
479478
from(projectDir.resolve("src/docker/Dockerfile.ess")) {
480479
expand(
481480
[
@@ -493,7 +492,7 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
493492
final TaskProvider<DockerBuildTask> buildDockerImageTask =
494493
tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) {
495494

496-
DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI
495+
DockerBase base = DockerBase.WOLFI
497496

498497
TaskProvider<DockerBuildTask> buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage"))
499498
inputs.files(buildBaseTask)
@@ -519,16 +518,15 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
519518

520519
for (final Architecture architecture : Architecture.values()) {
521520
for (final DockerBase base : DockerBase.values()) {
522-
if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
521+
if (base == DockerBase.CLOUD_ESS) {
523522
continue
524523
}
525524
addBuildDockerContextTask(architecture, base)
526525
addTransformDockerContextTask(architecture, base)
527526
addBuildDockerImageTask(architecture, base)
528527
}
529528

530-
addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS)
531-
addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS)
529+
addBuildEssDockerImageTask(architecture)
532530
}
533531

534532
def exportDockerImages = tasks.register("exportDockerImages")
@@ -564,8 +562,7 @@ subprojects { Project subProject ->
564562
(base == DockerBase.CLOUD ? 'cloud.tar' :
565563
(base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
566564
(base == DockerBase.WOLFI ? 'wolfi.tar' :
567-
(base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' :
568-
'docker.tar')))))
565+
'docker.tar'))))
569566
final String artifactName = "elasticsearch${arch}${base.suffix}_test"
570567

571568
final String exportTaskName = taskName("export", architecture, base, 'DockerImage')

‎distribution/docker/src/docker/Dockerfile.ess‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,31 @@ FROM ${base_image} AS builder
22

33
USER root
44

5-
<% if (docker_base == "wolfi_ess") { %>
6-
# Add plugins infrastructure
7-
RUN mkdir -p /opt/plugins/archive
8-
RUN chmod -R 0555 /opt/plugins
9-
10-
COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
11-
RUN set -eux ; \\
12-
for beat in filebeat metricbeat ; do \\
13-
if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
14-
echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
15-
exit 1 ; \\
16-
fi ; \\
17-
if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
18-
echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
19-
exit 1 ; \\
20-
fi ; \\
21-
mkdir -p /opt/\$beat ; \\
22-
tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
23-
done
24-
<% } %>
5+
# Add plugins infrastructure
6+
RUN mkdir -p /opt/plugins/archive
7+
RUN chmod -R 0555 /opt/plugins
8+
9+
COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
10+
RUN set -eux ; \\
11+
for beat in filebeat metricbeat ; do \\
12+
if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
13+
echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
14+
exit 1 ; \\
15+
fi ; \\
16+
if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
17+
echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
18+
exit 1 ; \\
19+
fi ; \\
20+
mkdir -p /opt/\$beat ; \\
21+
tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
22+
done
2523

2624
COPY plugins/*.zip /opt/plugins/archive/
2725

2826
RUN chown 1000:1000 /opt/plugins/archive/*
2927
RUN chmod 0444 /opt/plugins/archive/*
3028

3129
FROM ${base_image}
32-
<% if (docker_base == "wolfi_ess") { %>
3330
USER root
3431

3532
RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %>
@@ -44,8 +41,4 @@ RUN mkdir /app && \\
4441

4542
COPY --from=builder --chown=0:0 /opt /opt
4643
USER 1000:0
47-
<% } else { %>
48-
COPY --from=builder /opt/plugins /opt/plugins
49-
<% } %>
50-
5144
ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive

0 commit comments

Comments
 (0)