Skip to content

Commit 39a411a

Browse files
gmarouligmarouli
authored
Merge branch 'main' into data-lifecycle-rename
2 parents e90c943 + 065c583 commit 39a411a

File tree

442 files changed

+5418
-1020
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

442 files changed

+5418
-1020
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.index.codec.tsdb;
11+
12+
import org.apache.lucene.analysis.standard.StandardAnalyzer;
13+
import org.apache.lucene.codecs.DocValuesFormat;
14+
import org.apache.lucene.document.Document;
15+
import org.apache.lucene.document.SortedDocValuesField;
16+
import org.apache.lucene.document.SortedNumericDocValuesField;
17+
import org.apache.lucene.document.SortedSetDocValuesField;
18+
import org.apache.lucene.index.IndexWriter;
19+
import org.apache.lucene.index.IndexWriterConfig;
20+
import org.apache.lucene.index.LogByteSizeMergePolicy;
21+
import org.apache.lucene.search.Sort;
22+
import org.apache.lucene.search.SortField;
23+
import org.apache.lucene.search.SortedNumericSortField;
24+
import org.apache.lucene.store.Directory;
25+
import org.apache.lucene.store.FSDirectory;
26+
import org.apache.lucene.util.BytesRef;
27+
import org.elasticsearch.cluster.metadata.DataStream;
28+
import org.elasticsearch.common.logging.LogConfigurator;
29+
import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
30+
import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
31+
import org.openjdk.jmh.annotations.Benchmark;
32+
import org.openjdk.jmh.annotations.BenchmarkMode;
33+
import org.openjdk.jmh.annotations.Fork;
34+
import org.openjdk.jmh.annotations.Level;
35+
import org.openjdk.jmh.annotations.Measurement;
36+
import org.openjdk.jmh.annotations.Mode;
37+
import org.openjdk.jmh.annotations.OutputTimeUnit;
38+
import org.openjdk.jmh.annotations.Param;
39+
import org.openjdk.jmh.annotations.Scope;
40+
import org.openjdk.jmh.annotations.Setup;
41+
import org.openjdk.jmh.annotations.State;
42+
import org.openjdk.jmh.annotations.TearDown;
43+
import org.openjdk.jmh.annotations.Threads;
44+
import org.openjdk.jmh.annotations.Warmup;
45+
import org.openjdk.jmh.profile.AsyncProfiler;
46+
import org.openjdk.jmh.runner.Runner;
47+
import org.openjdk.jmh.runner.RunnerException;
48+
import org.openjdk.jmh.runner.options.Options;
49+
import org.openjdk.jmh.runner.options.OptionsBuilder;
50+
51+
import java.io.IOException;
52+
import java.nio.file.Files;
53+
import java.util.Random;
54+
import java.util.concurrent.ExecutorService;
55+
import java.util.concurrent.Executors;
56+
import java.util.concurrent.TimeUnit;
57+
58+
@BenchmarkMode(Mode.SampleTime)
59+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
60+
@State(Scope.Benchmark)
61+
@Fork(1)
62+
@Threads(1)
63+
@Warmup(iterations = 0)
64+
@Measurement(iterations = 1)
65+
public class TSDBDocValuesMergeBenchmark {
66+
67+
static {
68+
// For Elasticsearch900Lucene101Codec:
69+
LogConfigurator.loadLog4jPlugins();
70+
LogConfigurator.configureESLogging();
71+
LogConfigurator.setNodeName("test");
72+
}
73+
74+
@Param("20431204")
75+
private int nDocs;
76+
77+
@Param("1000")
78+
private int deltaTime;
79+
80+
@Param("42")
81+
private int seed;
82+
83+
private static final String TIMESTAMP_FIELD = "@timestamp";
84+
private static final String HOSTNAME_FIELD = "host.name";
85+
private static final long BASE_TIMESTAMP = 1704067200000L;
86+
87+
private IndexWriter indexWriterWithoutOptimizedMerge;
88+
private IndexWriter indexWriterWithOptimizedMerge;
89+
private ExecutorService executorService;
90+
91+
public static void main(String[] args) throws RunnerException {
92+
final Options options = new OptionsBuilder().include(TSDBDocValuesMergeBenchmark.class.getSimpleName())
93+
.addProfiler(AsyncProfiler.class)
94+
.build();
95+
96+
new Runner(options).run();
97+
}
98+
99+
@Setup(Level.Trial)
100+
public void setup() throws IOException {
101+
executorService = Executors.newSingleThreadExecutor();
102+
103+
final Directory tempDirectoryWithoutDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp1-"));
104+
final Directory tempDirectoryWithDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp2-"));
105+
106+
indexWriterWithoutOptimizedMerge = createIndex(tempDirectoryWithoutDocValuesSkipper, false);
107+
indexWriterWithOptimizedMerge = createIndex(tempDirectoryWithDocValuesSkipper, true);
108+
}
109+
110+
private IndexWriter createIndex(final Directory directory, final boolean optimizedMergeEnabled) throws IOException {
111+
final var iwc = createIndexWriterConfig(optimizedMergeEnabled);
112+
long counter1 = 0;
113+
long counter2 = 10_000_000;
114+
long[] gauge1Values = new long[] { 2, 4, 6, 8, 10, 12, 14, 16 };
115+
long[] gauge2Values = new long[] { -2, -4, -6, -8, -10, -12, -14, -16 };
116+
int numHosts = 1000;
117+
String[] tags = new String[] { "tag_1", "tag_2", "tag_3", "tag_4", "tag_5", "tag_6", "tag_7", "tag_8" };
118+
119+
final Random random = new Random(seed);
120+
IndexWriter indexWriter = new IndexWriter(directory, iwc);
121+
for (int i = 0; i < nDocs; i++) {
122+
final Document doc = new Document();
123+
124+
final int batchIndex = i / numHosts;
125+
final String hostName = "host-" + batchIndex;
126+
// Slightly vary the timestamp in each document
127+
final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
128+
129+
doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
130+
doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
131+
doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
132+
doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
133+
doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
134+
doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
135+
int numTags = tags.length % (i + 1);
136+
for (int j = 0; j < numTags; j++) {
137+
doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
138+
}
139+
140+
indexWriter.addDocument(doc);
141+
}
142+
indexWriter.commit();
143+
return indexWriter;
144+
}
145+
146+
@Benchmark
147+
public void forceMergeWithoutOptimizedMerge() throws IOException {
148+
forceMerge(indexWriterWithoutOptimizedMerge);
149+
}
150+
151+
@Benchmark
152+
public void forceMergeWithOptimizedMerge() throws IOException {
153+
forceMerge(indexWriterWithOptimizedMerge);
154+
}
155+
156+
private void forceMerge(final IndexWriter indexWriter) throws IOException {
157+
indexWriter.forceMerge(1);
158+
}
159+
160+
@TearDown(Level.Trial)
161+
public void tearDown() {
162+
if (executorService != null) {
163+
executorService.shutdown();
164+
try {
165+
if (executorService.awaitTermination(30, TimeUnit.SECONDS) == false) {
166+
executorService.shutdownNow();
167+
}
168+
} catch (InterruptedException e) {
169+
executorService.shutdownNow();
170+
Thread.currentThread().interrupt();
171+
}
172+
}
173+
}
174+
175+
private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeEnabled) {
176+
var config = new IndexWriterConfig(new StandardAnalyzer());
177+
// NOTE: index sort config matching LogsDB's sort order
178+
config.setIndexSort(
179+
new Sort(
180+
new SortField(HOSTNAME_FIELD, SortField.Type.STRING, false),
181+
new SortedNumericSortField(TIMESTAMP_FIELD, SortField.Type.LONG, true)
182+
)
183+
);
184+
config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
185+
config.setMergePolicy(new LogByteSizeMergePolicy());
186+
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
187+
config.setCodec(new Elasticsearch900Lucene101Codec() {
188+
189+
@Override
190+
public DocValuesFormat getDocValuesFormatForField(String field) {
191+
return docValuesFormat;
192+
}
193+
});
194+
return config;
195+
}
196+
}

‎docs/changelog/124832.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 124832
2+
summary: List/get query API
3+
area: ES|QL
4+
type: feature
5+
issues:
6+
- 124827

‎docs/changelog/125403.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125403
2+
summary: First step optimizing tsdb doc values codec merging
3+
area: Codec
4+
type: enhancement
5+
issues: []

‎docs/changelog/126409.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 126409
2+
summary: System data streams are not being upgraded in the feature migration API
3+
area: Infra/Core
4+
type: bug
5+
issues:
6+
- 122949

‎docs/reference/elasticsearch/security-privileges.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,9 @@ This section lists the privileges that you can assign to a role.
194194
`monitor_enrich`
195195
: All read-only operations related to managing and executing enrich policies.
196196

197+
`monitor_esql`
198+
: All read-only operations related to ES|QL queries.
199+
197200
`monitor_inference`
198201
: All read-only operations related to {{infer}}.
199202

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
326326
.build(),
327327
Map.of(),
328328
List.of("product"),
329+
"product",
329330
ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
330331
)
331332
);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
275275
.build(),
276276
Map.of(),
277277
Collections.singletonList("test"),
278+
"test",
278279
new ExecutorNames(ThreadPool.Names.SYSTEM_CRITICAL_READ, ThreadPool.Names.SYSTEM_READ, ThreadPool.Names.SYSTEM_WRITE)
279280
)
280281
);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
213213
.build(),
214214
Map.of(),
215215
List.of("product"),
216+
"product",
216217
ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
217218
)
218219
);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1097,6 +1097,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
10971097
.build(),
10981098
Map.of(),
10991099
List.of(),
1100+
"test",
11001101
ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS
11011102
)
11021103
);

‎muted-tests.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -357,9 +357,6 @@ tests:
357357
- class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
358358
method: testSearchWithRandomDisconnects
359359
issue: https://github.com/elastic/elasticsearch/issues/122707
360-
- class: org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT
361-
method: testDeletionOfFailingToRecoverIndexShouldStopRestore
362-
issue: https://github.com/elastic/elasticsearch/issues/126204
363360
- class: org.elasticsearch.xpack.esql.inference.RerankOperatorTests
364361
method: testSimpleCircuitBreaking
365362
issue: https://github.com/elastic/elasticsearch/issues/124337
@@ -450,6 +447,12 @@ tests:
450447
- class: org.elasticsearch.xpack.security.cli.HttpCertificateCommandTests
451448
method: testGenerateMultipleCertificateWithNewCA
452449
issue: https://github.com/elastic/elasticsearch/issues/126471
450+
- class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
451+
method: testILMDownsampleRollingRestart
452+
issue: https://github.com/elastic/elasticsearch/issues/126495
453+
- class: org.elasticsearch.xpack.search.AsyncSearchErrorTraceIT
454+
method: testDataNodeLogsStackTraceWhenErrorTraceFalseOrEmpty
455+
issue: https://github.com/elastic/elasticsearch/issues/126357
453456

454457
# Examples:
455458
#

0 commit comments

Comments
 (0)