Skip to content

Commit ea7d8a8

Browse files
ioanatiaioanatia
authored
Merge branch 'main' into fork_better_branches
2 parents f2fa8c0 + 62636f9 commit ea7d8a8

File tree

596 files changed

+9367
-2628
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

596 files changed

+9367
-2628
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.index.codec.tsdb;
11+
12+
import org.apache.lucene.analysis.standard.StandardAnalyzer;
13+
import org.apache.lucene.codecs.DocValuesFormat;
14+
import org.apache.lucene.document.Document;
15+
import org.apache.lucene.document.SortedDocValuesField;
16+
import org.apache.lucene.document.SortedNumericDocValuesField;
17+
import org.apache.lucene.document.SortedSetDocValuesField;
18+
import org.apache.lucene.index.IndexWriter;
19+
import org.apache.lucene.index.IndexWriterConfig;
20+
import org.apache.lucene.index.LogByteSizeMergePolicy;
21+
import org.apache.lucene.search.Sort;
22+
import org.apache.lucene.search.SortField;
23+
import org.apache.lucene.search.SortedNumericSortField;
24+
import org.apache.lucene.store.Directory;
25+
import org.apache.lucene.store.FSDirectory;
26+
import org.apache.lucene.util.BytesRef;
27+
import org.elasticsearch.cluster.metadata.DataStream;
28+
import org.elasticsearch.common.logging.LogConfigurator;
29+
import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
30+
import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
31+
import org.openjdk.jmh.annotations.Benchmark;
32+
import org.openjdk.jmh.annotations.BenchmarkMode;
33+
import org.openjdk.jmh.annotations.Fork;
34+
import org.openjdk.jmh.annotations.Level;
35+
import org.openjdk.jmh.annotations.Measurement;
36+
import org.openjdk.jmh.annotations.Mode;
37+
import org.openjdk.jmh.annotations.OutputTimeUnit;
38+
import org.openjdk.jmh.annotations.Param;
39+
import org.openjdk.jmh.annotations.Scope;
40+
import org.openjdk.jmh.annotations.Setup;
41+
import org.openjdk.jmh.annotations.State;
42+
import org.openjdk.jmh.annotations.TearDown;
43+
import org.openjdk.jmh.annotations.Threads;
44+
import org.openjdk.jmh.annotations.Warmup;
45+
import org.openjdk.jmh.profile.AsyncProfiler;
46+
import org.openjdk.jmh.runner.Runner;
47+
import org.openjdk.jmh.runner.RunnerException;
48+
import org.openjdk.jmh.runner.options.Options;
49+
import org.openjdk.jmh.runner.options.OptionsBuilder;
50+
51+
import java.io.IOException;
52+
import java.nio.file.Files;
53+
import java.util.Random;
54+
import java.util.concurrent.ExecutorService;
55+
import java.util.concurrent.Executors;
56+
import java.util.concurrent.TimeUnit;
57+
58+
@BenchmarkMode(Mode.SampleTime)
59+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
60+
@State(Scope.Benchmark)
61+
@Fork(1)
62+
@Threads(1)
63+
@Warmup(iterations = 0)
64+
@Measurement(iterations = 1)
65+
public class TSDBDocValuesMergeBenchmark {
66+
67+
static {
68+
// For Elasticsearch900Lucene101Codec:
69+
LogConfigurator.loadLog4jPlugins();
70+
LogConfigurator.configureESLogging();
71+
LogConfigurator.setNodeName("test");
72+
}
73+
74+
@Param("20431204")
75+
private int nDocs;
76+
77+
@Param("1000")
78+
private int deltaTime;
79+
80+
@Param("42")
81+
private int seed;
82+
83+
private static final String TIMESTAMP_FIELD = "@timestamp";
84+
private static final String HOSTNAME_FIELD = "host.name";
85+
private static final long BASE_TIMESTAMP = 1704067200000L;
86+
87+
private IndexWriter indexWriterWithoutOptimizedMerge;
88+
private IndexWriter indexWriterWithOptimizedMerge;
89+
private ExecutorService executorService;
90+
91+
public static void main(String[] args) throws RunnerException {
92+
final Options options = new OptionsBuilder().include(TSDBDocValuesMergeBenchmark.class.getSimpleName())
93+
.addProfiler(AsyncProfiler.class)
94+
.build();
95+
96+
new Runner(options).run();
97+
}
98+
99+
@Setup(Level.Trial)
100+
public void setup() throws IOException {
101+
executorService = Executors.newSingleThreadExecutor();
102+
103+
final Directory tempDirectoryWithoutDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp1-"));
104+
final Directory tempDirectoryWithDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp2-"));
105+
106+
indexWriterWithoutOptimizedMerge = createIndex(tempDirectoryWithoutDocValuesSkipper, false);
107+
indexWriterWithOptimizedMerge = createIndex(tempDirectoryWithDocValuesSkipper, true);
108+
}
109+
110+
private IndexWriter createIndex(final Directory directory, final boolean optimizedMergeEnabled) throws IOException {
111+
final var iwc = createIndexWriterConfig(optimizedMergeEnabled);
112+
long counter1 = 0;
113+
long counter2 = 10_000_000;
114+
long[] gauge1Values = new long[] { 2, 4, 6, 8, 10, 12, 14, 16 };
115+
long[] gauge2Values = new long[] { -2, -4, -6, -8, -10, -12, -14, -16 };
116+
int numHosts = 1000;
117+
String[] tags = new String[] { "tag_1", "tag_2", "tag_3", "tag_4", "tag_5", "tag_6", "tag_7", "tag_8" };
118+
119+
final Random random = new Random(seed);
120+
IndexWriter indexWriter = new IndexWriter(directory, iwc);
121+
for (int i = 0; i < nDocs; i++) {
122+
final Document doc = new Document();
123+
124+
final int batchIndex = i / numHosts;
125+
final String hostName = "host-" + batchIndex;
126+
// Slightly vary the timestamp in each document
127+
final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
128+
129+
doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
130+
doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
131+
doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
132+
doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
133+
doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
134+
doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
135+
int numTags = tags.length % (i + 1);
136+
for (int j = 0; j < numTags; j++) {
137+
doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
138+
}
139+
140+
indexWriter.addDocument(doc);
141+
}
142+
indexWriter.commit();
143+
return indexWriter;
144+
}
145+
146+
@Benchmark
147+
public void forceMergeWithoutOptimizedMerge() throws IOException {
148+
forceMerge(indexWriterWithoutOptimizedMerge);
149+
}
150+
151+
@Benchmark
152+
public void forceMergeWithOptimizedMerge() throws IOException {
153+
forceMerge(indexWriterWithOptimizedMerge);
154+
}
155+
156+
private void forceMerge(final IndexWriter indexWriter) throws IOException {
157+
indexWriter.forceMerge(1);
158+
}
159+
160+
@TearDown(Level.Trial)
161+
public void tearDown() {
162+
if (executorService != null) {
163+
executorService.shutdown();
164+
try {
165+
if (executorService.awaitTermination(30, TimeUnit.SECONDS) == false) {
166+
executorService.shutdownNow();
167+
}
168+
} catch (InterruptedException e) {
169+
executorService.shutdownNow();
170+
Thread.currentThread().interrupt();
171+
}
172+
}
173+
}
174+
175+
private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeEnabled) {
176+
var config = new IndexWriterConfig(new StandardAnalyzer());
177+
// NOTE: index sort config matching LogsDB's sort order
178+
config.setIndexSort(
179+
new Sort(
180+
new SortField(HOSTNAME_FIELD, SortField.Type.STRING, false),
181+
new SortedNumericSortField(TIMESTAMP_FIELD, SortField.Type.LONG, true)
182+
)
183+
);
184+
config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
185+
config.setMergePolicy(new LogByteSizeMergePolicy());
186+
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
187+
config.setCodec(new Elasticsearch900Lucene101Codec() {
188+
189+
@Override
190+
public DocValuesFormat getDocValuesFormatForField(String field) {
191+
return docValuesFormat;
192+
}
193+
});
194+
return config;
195+
}
196+
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,9 @@ echo "Running elasticsearch \$0"
179179

180180
file(distProjectFolder, 'src/config/elasticsearch.properties') << "some propes"
181181
file(distProjectFolder, 'src/config/jvm.options') << """
182-
-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m
183-
-XX:ErrorFile=logs/hs_err_pid%p.log
184-
-XX:HeapDumpPath=data
182+
-Xlog:gc*,gc+age=trace,safepoint:file=gc.log:utctime,level,pid,tags:filecount=32,filesize=64m
183+
-XX:ErrorFile=hs_err_pid%p.log
184+
# -XX:HeapDumpPath=/heap/dump/path
185185
"""
186186
file(distProjectFolder, 'build.gradle') << """
187187
import org.gradle.api.internal.artifacts.ArtifactAttributes;

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1435,30 +1435,57 @@ private void tweakJvmOptions(Path configFileRoot) {
14351435
Path jvmOptions = configFileRoot.resolve("jvm.options");
14361436
try {
14371437
String content = new String(Files.readAllBytes(jvmOptions));
1438-
Map<String, String> expansions = jvmOptionExpansions();
1439-
for (String origin : expansions.keySet()) {
1440-
if (content.contains(origin) == false) {
1441-
throw new IOException("template property " + origin + " not found in template.");
1438+
Map<ReplacementKey, String> expansions = jvmOptionExpansions();
1439+
for (var entry : expansions.entrySet()) {
1440+
ReplacementKey replacement = entry.getKey();
1441+
String key = replacement.key();
1442+
if (content.contains(key) == false) {
1443+
key = replacement.fallback();
1444+
if (content.contains(key) == false) {
1445+
throw new IOException("Template property '" + replacement + "' not found in template:\n" + content);
1446+
}
14421447
}
1443-
content = content.replace(origin, expansions.get(origin));
1448+
content = content.replace(key, entry.getValue());
14441449
}
14451450
Files.write(jvmOptions, content.getBytes());
14461451
} catch (IOException ioException) {
14471452
throw new UncheckedIOException(ioException);
14481453
}
14491454
}
14501455

1451-
private Map<String, String> jvmOptionExpansions() {
1452-
Map<String, String> expansions = new HashMap<>();
1456+
private record ReplacementKey(String key, String fallback) {}
1457+
1458+
private Map<ReplacementKey, String> jvmOptionExpansions() {
1459+
Map<ReplacementKey, String> expansions = new HashMap<>();
14531460
Version version = getVersion();
1454-
String heapDumpOrigin = getVersion().onOrAfter("6.3.0") ? "-XX:HeapDumpPath=data" : "-XX:HeapDumpPath=/heap/dump/path";
1455-
expansions.put(heapDumpOrigin, "-XX:HeapDumpPath=" + confPathLogs);
1456-
if (version.onOrAfter("6.2.0")) {
1457-
expansions.put("logs/gc.log", confPathLogs.resolve("gc.log").toString());
1461+
1462+
ReplacementKey heapDumpPathSub;
1463+
if (version.before("8.19.0") && version.onOrAfter("6.3.0")) {
1464+
heapDumpPathSub = new ReplacementKey("-XX:HeapDumpPath=data", null);
1465+
} else {
1466+
// temporarily fall back to the old substitution so both old and new work during backport
1467+
heapDumpPathSub = new ReplacementKey("# -XX:HeapDumpPath=/heap/dump/path", "-XX:HeapDumpPath=data");
14581468
}
1459-
if (getVersion().getMajor() >= 7) {
1460-
expansions.put("-XX:ErrorFile=logs/hs_err_pid%p.log", "-XX:ErrorFile=" + confPathLogs.resolve("hs_err_pid%p.log"));
1469+
expansions.put(heapDumpPathSub, "-XX:HeapDumpPath=" + confPathLogs);
1470+
1471+
ReplacementKey gcLogSub;
1472+
if (version.before("8.19.0") && version.onOrAfter("6.2.0")) {
1473+
gcLogSub = new ReplacementKey("logs/gc.log", null);
1474+
} else {
1475+
// temporarily check the old substitution first so both old and new work during backport
1476+
gcLogSub = new ReplacementKey("logs/gc.log", "gc.log");
14611477
}
1478+
expansions.put(gcLogSub, confPathLogs.resolve("gc.log").toString());
1479+
1480+
ReplacementKey errorFileSub;
1481+
if (version.before("8.19.0") && version.getMajor() >= 7) {
1482+
errorFileSub = new ReplacementKey("-XX:ErrorFile=logs/hs_err_pid%p.log", null);
1483+
} else {
1484+
// temporarily check the old substitution first so both old and new work during backport
1485+
errorFileSub = new ReplacementKey("-XX:ErrorFile=logs/hs_err_pid%p.log", "-XX:ErrorFile=hs_err_pid%p.log");
1486+
}
1487+
expansions.put(errorFileSub, "-XX:ErrorFile=" + confPathLogs.resolve("hs_err_pid%p.log"));
1488+
14621489
return expansions;
14631490
}
14641491

‎distribution/build.gradle‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,6 @@ subprojects {
531531
final String packagingPathData = "path.data: /var/lib/elasticsearch"
532532
final String pathLogs = "/var/log/elasticsearch"
533533
final String packagingPathLogs = "path.logs: ${pathLogs}"
534-
final String packagingLoggc = "${pathLogs}/gc.log"
535534

536535
String licenseText
537536
if (isTestDistro) {
@@ -576,23 +575,6 @@ subprojects {
576575
'rpm': packagingPathLogs,
577576
'def': '#path.logs: /path/to/logs'
578577
],
579-
'loggc': [
580-
'deb': packagingLoggc,
581-
'rpm': packagingLoggc,
582-
'def': 'logs/gc.log'
583-
],
584-
585-
'heap.dump.path': [
586-
'deb': "-XX:HeapDumpPath=/var/lib/elasticsearch",
587-
'rpm': "-XX:HeapDumpPath=/var/lib/elasticsearch",
588-
'def': "-XX:HeapDumpPath=data"
589-
],
590-
591-
'error.file': [
592-
'deb': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
593-
'rpm': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
594-
'def': "-XX:ErrorFile=logs/hs_err_pid%p.log"
595-
],
596578

597579
'scripts.footer': [
598580
/* Debian needs exit 0 on these scripts so we add it here and preserve

‎distribution/docker/build.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -448,8 +448,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
448448
// is functional.
449449
if (base == DockerBase.IRON_BANK) {
450450
Map<String, String> buildArgsMap = [
451-
'BASE_REGISTRY': 'docker.elastic.co',
452-
'BASE_IMAGE' : 'ubi9/ubi',
451+
'BASE_REGISTRY': 'docker.io',
452+
'BASE_IMAGE' : 'redhat/ubi9',
453453
'BASE_TAG' : 'latest'
454454
]
455455

‎distribution/src/config/jvm.options‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,10 @@
7777

7878
# specify an alternative path for heap dumps; ensure the directory exists and
7979
# has sufficient space
80-
@heap.dump.path@
80+
# -XX:HeapDumpPath=/heap/dump/path
8181

8282
# specify an alternative path for JVM fatal error logs
83-
@error.file@
83+
-XX:ErrorFile=hs_err_pid%p.log
8484

8585
## GC logging
86-
-Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,level,pid,tags:filecount=32,filesize=64m
86+
-Xlog:gc*,gc+age=trace,safepoint:file=gc.log:utctime,level,pid,tags:filecount=32,filesize=64m

‎distribution/tools/cli-launcher/src/main/java/org/elasticsearch/launcher/CliToolLauncher.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public static void main(String[] args) throws Exception {
5858
String toolname = getToolName(pinfo.sysprops());
5959
String libs = pinfo.sysprops().getOrDefault("cli.libs", "");
6060

61-
command = CliToolProvider.load(toolname, libs).create();
61+
command = CliToolProvider.load(pinfo.sysprops(), toolname, libs).create();
6262
Terminal terminal = Terminal.DEFAULT;
6363
Runtime.getRuntime().addShutdownHook(createShutdownHook(terminal, command));
6464

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@
1010
package org.elasticsearch.server.cli;
1111

1212
import org.elasticsearch.common.Strings;
13+
import org.elasticsearch.core.SuppressForbidden;
1314

1415
import java.io.BufferedReader;
1516
import java.io.IOException;
1617
import java.io.InputStream;
1718
import java.io.InputStreamReader;
1819
import java.nio.charset.StandardCharsets;
20+
import java.nio.file.Files;
1921
import java.nio.file.Path;
2022
import java.util.List;
2123
import java.util.Locale;
@@ -106,7 +108,9 @@ private static List<String> flagsFinal(final List<String> userDefinedJvmOptions)
106108
userDefinedJvmOptions.stream(),
107109
Stream.of("-XX:+PrintFlagsFinal", "-version")
108110
).flatMap(Function.identity()).toList();
109-
final Process process = new ProcessBuilder().command(command).start();
111+
final ProcessBuilder builder = new ProcessBuilder().command(command);
112+
setWorkingDir(builder);
113+
final Process process = builder.start();
110114
final List<String> output = readLinesFromInputStream(process.getInputStream());
111115
final List<String> error = readLinesFromInputStream(process.getErrorStream());
112116
final int status = process.waitFor();
@@ -124,6 +128,14 @@ private static List<String> flagsFinal(final List<String> userDefinedJvmOptions)
124128
}
125129
}
126130

131+
@SuppressForbidden(reason = "ProcessBuilder takes File")
132+
private static void setWorkingDir(ProcessBuilder builder) throws IOException {
133+
// The real ES process uses the logs dir as the working directory. Since we don't
134+
// have the logs dir yet, here we use a temp directory for calculating jvm options.
135+
final Path tmpDir = Files.createTempDirectory("final-flags");
136+
builder.directory(tmpDir.toFile());
137+
}
138+
127139
private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
128140
try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
129141
return br.lines().toList();

0 commit comments

Comments
 (0)