Skip to content

Commit bc16b73

Browse files
jbaierajbaiera
committed
Merge branch 'main' into failure-store-sql-support
2 parents 94463ec + 065c583 commit bc16b73

File tree

686 files changed

+8763
-6172
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

686 files changed

+8763
-6172
lines changed

‎.buildkite/pipelines/pull-request/build-benchmark.yml‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,19 @@ steps:
77
command: |
88
.ci/scripts/run-gradle.sh :build-tools-internal:bootstrapPerformanceTests
99
.ci/scripts/install-gradle-profiler.sh
10-
.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-benchmark-{{matrix.part}}.scenarios --project-dir . --output-dir profile-out
11-
mkdir build
12-
tar -czf build/$BUILDKITE_BUILD_NUMBER.tar.bz2 profile-out
10+
.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-{{matrix.part}}.scenarios --measure-config-time --project-dir . --output-dir profile-out
11+
mkdir -p build
12+
buildkite-agent artifact upload "profile-out/*"
13+
cat << EOF | buildkite-agent annotate --style "info"
14+
Benchmark Html Report <a href="artifact://profile-out/benchmark.html">Benchmark Html Report</a>
15+
EOF
1316
matrix:
1417
setup:
1518
part:
16-
- part1
17-
- part2
19+
- esql-tests
1820
env:
1921
BUILD_PERFORMANCE_TEST: "true"
22+
2023
agents:
2124
provider: gcp
2225
image: family/elasticsearch-ubuntu-2004

‎.ci/scripts/install-gradle-profiler.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set -e
44
# profiler version we wanna install
5-
PROFILER_VERSION="0.16.0"
6-
wget https://repo.gradle.org/gradle/ext-releases-local/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
5+
PROFILER_VERSION="0.22.0"
6+
wget https://repo1.maven.org/maven2/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
77
unzip $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
88
mv $WORKSPACE/gradle-profiler-$PROFILER_VERSION $WORKSPACE/gradle-profiler

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.index.codec.tsdb;
11+
12+
import org.apache.lucene.analysis.standard.StandardAnalyzer;
13+
import org.apache.lucene.codecs.DocValuesFormat;
14+
import org.apache.lucene.document.Document;
15+
import org.apache.lucene.document.SortedDocValuesField;
16+
import org.apache.lucene.document.SortedNumericDocValuesField;
17+
import org.apache.lucene.document.SortedSetDocValuesField;
18+
import org.apache.lucene.index.IndexWriter;
19+
import org.apache.lucene.index.IndexWriterConfig;
20+
import org.apache.lucene.index.LogByteSizeMergePolicy;
21+
import org.apache.lucene.search.Sort;
22+
import org.apache.lucene.search.SortField;
23+
import org.apache.lucene.search.SortedNumericSortField;
24+
import org.apache.lucene.store.Directory;
25+
import org.apache.lucene.store.FSDirectory;
26+
import org.apache.lucene.util.BytesRef;
27+
import org.elasticsearch.cluster.metadata.DataStream;
28+
import org.elasticsearch.common.logging.LogConfigurator;
29+
import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
30+
import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
31+
import org.openjdk.jmh.annotations.Benchmark;
32+
import org.openjdk.jmh.annotations.BenchmarkMode;
33+
import org.openjdk.jmh.annotations.Fork;
34+
import org.openjdk.jmh.annotations.Level;
35+
import org.openjdk.jmh.annotations.Measurement;
36+
import org.openjdk.jmh.annotations.Mode;
37+
import org.openjdk.jmh.annotations.OutputTimeUnit;
38+
import org.openjdk.jmh.annotations.Param;
39+
import org.openjdk.jmh.annotations.Scope;
40+
import org.openjdk.jmh.annotations.Setup;
41+
import org.openjdk.jmh.annotations.State;
42+
import org.openjdk.jmh.annotations.TearDown;
43+
import org.openjdk.jmh.annotations.Threads;
44+
import org.openjdk.jmh.annotations.Warmup;
45+
import org.openjdk.jmh.profile.AsyncProfiler;
46+
import org.openjdk.jmh.runner.Runner;
47+
import org.openjdk.jmh.runner.RunnerException;
48+
import org.openjdk.jmh.runner.options.Options;
49+
import org.openjdk.jmh.runner.options.OptionsBuilder;
50+
51+
import java.io.IOException;
52+
import java.nio.file.Files;
53+
import java.util.Random;
54+
import java.util.concurrent.ExecutorService;
55+
import java.util.concurrent.Executors;
56+
import java.util.concurrent.TimeUnit;
57+
58+
@BenchmarkMode(Mode.SampleTime)
59+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
60+
@State(Scope.Benchmark)
61+
@Fork(1)
62+
@Threads(1)
63+
@Warmup(iterations = 0)
64+
@Measurement(iterations = 1)
65+
public class TSDBDocValuesMergeBenchmark {
66+
67+
static {
68+
// For Elasticsearch900Lucene101Codec:
69+
LogConfigurator.loadLog4jPlugins();
70+
LogConfigurator.configureESLogging();
71+
LogConfigurator.setNodeName("test");
72+
}
73+
74+
@Param("20431204")
75+
private int nDocs;
76+
77+
@Param("1000")
78+
private int deltaTime;
79+
80+
@Param("42")
81+
private int seed;
82+
83+
private static final String TIMESTAMP_FIELD = "@timestamp";
84+
private static final String HOSTNAME_FIELD = "host.name";
85+
private static final long BASE_TIMESTAMP = 1704067200000L;
86+
87+
private IndexWriter indexWriterWithoutOptimizedMerge;
88+
private IndexWriter indexWriterWithOptimizedMerge;
89+
private ExecutorService executorService;
90+
91+
public static void main(String[] args) throws RunnerException {
92+
final Options options = new OptionsBuilder().include(TSDBDocValuesMergeBenchmark.class.getSimpleName())
93+
.addProfiler(AsyncProfiler.class)
94+
.build();
95+
96+
new Runner(options).run();
97+
}
98+
99+
@Setup(Level.Trial)
100+
public void setup() throws IOException {
101+
executorService = Executors.newSingleThreadExecutor();
102+
103+
final Directory tempDirectoryWithoutDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp1-"));
104+
final Directory tempDirectoryWithDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp2-"));
105+
106+
indexWriterWithoutOptimizedMerge = createIndex(tempDirectoryWithoutDocValuesSkipper, false);
107+
indexWriterWithOptimizedMerge = createIndex(tempDirectoryWithDocValuesSkipper, true);
108+
}
109+
110+
private IndexWriter createIndex(final Directory directory, final boolean optimizedMergeEnabled) throws IOException {
111+
final var iwc = createIndexWriterConfig(optimizedMergeEnabled);
112+
long counter1 = 0;
113+
long counter2 = 10_000_000;
114+
long[] gauge1Values = new long[] { 2, 4, 6, 8, 10, 12, 14, 16 };
115+
long[] gauge2Values = new long[] { -2, -4, -6, -8, -10, -12, -14, -16 };
116+
int numHosts = 1000;
117+
String[] tags = new String[] { "tag_1", "tag_2", "tag_3", "tag_4", "tag_5", "tag_6", "tag_7", "tag_8" };
118+
119+
final Random random = new Random(seed);
120+
IndexWriter indexWriter = new IndexWriter(directory, iwc);
121+
for (int i = 0; i < nDocs; i++) {
122+
final Document doc = new Document();
123+
124+
final int batchIndex = i / numHosts;
125+
final String hostName = "host-" + batchIndex;
126+
// Slightly vary the timestamp in each document
127+
final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
128+
129+
doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
130+
doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
131+
doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
132+
doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
133+
doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
134+
doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
135+
int numTags = tags.length % (i + 1);
136+
for (int j = 0; j < numTags; j++) {
137+
doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
138+
}
139+
140+
indexWriter.addDocument(doc);
141+
}
142+
indexWriter.commit();
143+
return indexWriter;
144+
}
145+
146+
@Benchmark
147+
public void forceMergeWithoutOptimizedMerge() throws IOException {
148+
forceMerge(indexWriterWithoutOptimizedMerge);
149+
}
150+
151+
@Benchmark
152+
public void forceMergeWithOptimizedMerge() throws IOException {
153+
forceMerge(indexWriterWithOptimizedMerge);
154+
}
155+
156+
private void forceMerge(final IndexWriter indexWriter) throws IOException {
157+
indexWriter.forceMerge(1);
158+
}
159+
160+
@TearDown(Level.Trial)
161+
public void tearDown() {
162+
if (executorService != null) {
163+
executorService.shutdown();
164+
try {
165+
if (executorService.awaitTermination(30, TimeUnit.SECONDS) == false) {
166+
executorService.shutdownNow();
167+
}
168+
} catch (InterruptedException e) {
169+
executorService.shutdownNow();
170+
Thread.currentThread().interrupt();
171+
}
172+
}
173+
}
174+
175+
private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeEnabled) {
176+
var config = new IndexWriterConfig(new StandardAnalyzer());
177+
// NOTE: index sort config matching LogsDB's sort order
178+
config.setIndexSort(
179+
new Sort(
180+
new SortField(HOSTNAME_FIELD, SortField.Type.STRING, false),
181+
new SortedNumericSortField(TIMESTAMP_FIELD, SortField.Type.LONG, true)
182+
)
183+
);
184+
config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
185+
config.setMergePolicy(new LogByteSizeMergePolicy());
186+
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
187+
config.setCodec(new Elasticsearch900Lucene101Codec() {
188+
189+
@Override
190+
public DocValuesFormat getDocValuesFormatForField(String field) {
191+
return docValuesFormat;
192+
}
193+
});
194+
return config;
195+
}
196+
}

‎build-tools-internal/performance/elasticsearch-build-benchmark-part1.scenarios‎

Lines changed: 0 additions & 72 deletions
This file was deleted.

‎build-tools-internal/performance/elasticsearch-build-benchmark-part2.scenarios‎

Lines changed: 0 additions & 38 deletions
This file was deleted.

0 commit comments

Comments
 (0)