Skip to content

Commit b810b0d

Browse files
szybiaszybia
committed
Merge remote-tracking branch 'upstream/main' into geoip-enrich-cache-improv
* upstream/main: (58 commits) Fixing flaky LoggedExec (tests) (elastic#133215) CPS search should not use `skip_unavailable` (elastic#132927) Don't fail search if bottom doc can't be formatted (elastic#133188) Mute org.elasticsearch.xpack.esql.action.RandomizedTimeSeriesIT testGroupByNothing elastic#133225 Mute org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests testSortedSetDocValuesWithSkipperSmall elastic#133224 Mute org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests testSortedNumberMergeAwayAllValuesWithSkipper elastic#133223 Adding support for index.number_of_replicas to data stream settings (elastic#132748) Mute org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests testSortedDocValuesSingleUniqueValue elastic#133221 Fix VectorSimilarityFunctionsIT (elastic#133206) Mute org.elasticsearch.xpack.esql.action.RandomizedTimeSeriesIT testGroupBySubset elastic#133220 Increase the number of FORK branches in ForkGenerator (elastic#132019) Mute org.elasticsearch.index.mapper.blockloader.IpFieldBlockLoaderTests testBlockLoader {preference=Params[syntheticSource=true, preference=STORED]} elastic#133218 Mute org.elasticsearch.index.mapper.blockloader.IpFieldBlockLoaderTests testBlockLoader {preference=Params[syntheticSource=true, preference=DOC_VALUES]} elastic#133217 Mute org.elasticsearch.index.mapper.blockloader.IpFieldBlockLoaderTests testBlockLoader {preference=Params[syntheticSource=true, preference=NONE]} elastic#133216 Set default processor allocation for test clusters (elastic#133204) Add mapper for exponential histograms (elastic#132493) Fix offset handling in Murmur3Hasher (elastic#133193) unmute testDoesNotResolveClosedIndex (elastic#133115) Fix an AWS SDK v2 release note (elastic#133155) Limit the depth of a filter (elastic#133113) ...
2 parents 9c8656a + 8bcecb9 commit b810b0d

File tree

297 files changed

+9585
-3446
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

297 files changed

+9585
-3446
lines changed

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ dependencies {
4747
api(project(':x-pack:plugin:core'))
4848
api(project(':x-pack:plugin:esql'))
4949
api(project(':x-pack:plugin:esql:compute'))
50+
api(project(':x-pack:plugin:mapper-exponential-histogram'))
5051
implementation project(path: ':libs:native')
5152
implementation project(path: ':libs:simdvec')
5253
implementation project(path: ':libs:exponential-histogram')

‎benchmarks/src/main/java/org/elasticsearch/benchmark/common/network/IpAddressesBenchmarks.java‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.common.network;
11+
12+
import org.elasticsearch.common.network.InetAddresses;
13+
import org.elasticsearch.xcontent.Text;
14+
import org.elasticsearch.xcontent.XContentString;
15+
import org.openjdk.jmh.annotations.Benchmark;
16+
import org.openjdk.jmh.annotations.BenchmarkMode;
17+
import org.openjdk.jmh.annotations.Fork;
18+
import org.openjdk.jmh.annotations.Measurement;
19+
import org.openjdk.jmh.annotations.Mode;
20+
import org.openjdk.jmh.annotations.OutputTimeUnit;
21+
import org.openjdk.jmh.annotations.Param;
22+
import org.openjdk.jmh.annotations.Scope;
23+
import org.openjdk.jmh.annotations.Setup;
24+
import org.openjdk.jmh.annotations.State;
25+
import org.openjdk.jmh.annotations.Warmup;
26+
import org.openjdk.jmh.infra.Blackhole;
27+
28+
import java.net.InetAddress;
29+
import java.net.UnknownHostException;
30+
import java.util.Random;
31+
import java.util.concurrent.TimeUnit;
32+
33+
@Warmup(iterations = 2)
34+
@Measurement(iterations = 3)
35+
@BenchmarkMode(Mode.Throughput)
36+
@OutputTimeUnit(TimeUnit.SECONDS)
37+
@State(Scope.Benchmark)
38+
@Fork(1)
39+
public class IpAddressesBenchmarks {
40+
41+
@Param("1000")
42+
private int size;
43+
private String[] ipV6Addresses;
44+
private String[] ipV4Addresses;
45+
private XContentString[] ipV6AddressesBytes;
46+
private XContentString[] ipV4AddressesBytes;
47+
48+
@Setup
49+
public void setup() throws UnknownHostException {
50+
Random random = new Random();
51+
ipV6Addresses = new String[size];
52+
ipV4Addresses = new String[size];
53+
ipV6AddressesBytes = new XContentString[size];
54+
ipV4AddressesBytes = new XContentString[size];
55+
byte[] ipv6Bytes = new byte[16];
56+
byte[] ipv4Bytes = new byte[4];
57+
for (int i = 0; i < size; i++) {
58+
random.nextBytes(ipv6Bytes);
59+
random.nextBytes(ipv4Bytes);
60+
String ipv6String = InetAddresses.toAddrString(InetAddress.getByAddress(ipv6Bytes));
61+
String ipv4String = InetAddresses.toAddrString(InetAddress.getByAddress(ipv4Bytes));
62+
ipV6Addresses[i] = ipv6String;
63+
ipV4Addresses[i] = ipv4String;
64+
ipV6AddressesBytes[i] = new Text(ipv6String);
65+
ipV4AddressesBytes[i] = new Text(ipv4String);
66+
}
67+
}
68+
69+
@Benchmark
70+
public boolean isInetAddressIpv6() {
71+
boolean b = true;
72+
for (int i = 0; i < size; i++) {
73+
b ^= InetAddresses.isInetAddress(ipV6Addresses[i]);
74+
}
75+
return b;
76+
}
77+
78+
@Benchmark
79+
public boolean isInetAddressIpv4() {
80+
boolean b = true;
81+
for (int i = 0; i < size; i++) {
82+
b ^= InetAddresses.isInetAddress(ipV4Addresses[i]);
83+
}
84+
return b;
85+
}
86+
87+
@Benchmark
88+
public void getIpOrHostIpv6(Blackhole blackhole) {
89+
for (int i = 0; i < size; i++) {
90+
blackhole.consume(InetAddresses.getIpOrHost(ipV6Addresses[i]));
91+
}
92+
}
93+
94+
@Benchmark
95+
public void getIpOrHostIpv4(Blackhole blackhole) {
96+
for (int i = 0; i < size; i++) {
97+
blackhole.consume(InetAddresses.forString(ipV4Addresses[i]));
98+
}
99+
}
100+
101+
@Benchmark
102+
public void forStringIpv6String(Blackhole blackhole) {
103+
for (int i = 0; i < size; i++) {
104+
blackhole.consume(InetAddresses.forString(ipV6Addresses[i]));
105+
}
106+
}
107+
108+
@Benchmark
109+
public void forStringIpv4String(Blackhole blackhole) {
110+
for (int i = 0; i < size; i++) {
111+
blackhole.consume(InetAddresses.forString(ipV4Addresses[i]));
112+
}
113+
}
114+
115+
@Benchmark
116+
public void forStringIpv6Bytes(Blackhole blackhole) {
117+
for (int i = 0; i < size; i++) {
118+
blackhole.consume(InetAddresses.forString(ipV6AddressesBytes[i].bytes()));
119+
}
120+
}
121+
122+
@Benchmark
123+
public void forStringIpv4Bytes(Blackhole blackhole) {
124+
for (int i = 0; i < size; i++) {
125+
blackhole.consume(InetAddresses.forString(ipV4AddressesBytes[i].bytes()));
126+
}
127+
}
128+
129+
@Benchmark
130+
public void encodeAsIpv6WithIpv6(Blackhole blackhole) {
131+
for (int i = 0; i < size; i++) {
132+
blackhole.consume(InetAddresses.encodeAsIpv6(ipV6AddressesBytes[i]));
133+
}
134+
}
135+
136+
@Benchmark
137+
public void encodeAsIpv6WithIpv4(Blackhole blackhole) {
138+
for (int i = 0; i < size; i++) {
139+
blackhole.consume(InetAddresses.encodeAsIpv6(ipV4AddressesBytes[i]));
140+
}
141+
}
142+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,15 @@
99

1010
package org.elasticsearch.benchmark.exponentialhistogram;
1111

12+
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.common.io.stream.BytesStreamOutput;
1214
import org.elasticsearch.exponentialhistogram.BucketIterator;
1315
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
1416
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
1517
import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
1618
import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
19+
import org.elasticsearch.xpack.exponentialhistogram.CompressedExponentialHistogram;
20+
import org.elasticsearch.xpack.exponentialhistogram.IndexWithCount;
1721
import org.openjdk.jmh.annotations.Benchmark;
1822
import org.openjdk.jmh.annotations.BenchmarkMode;
1923
import org.openjdk.jmh.annotations.Fork;
@@ -27,6 +31,8 @@
2731
import org.openjdk.jmh.annotations.Threads;
2832
import org.openjdk.jmh.annotations.Warmup;
2933

34+
import java.io.IOException;
35+
import java.util.ArrayList;
3036
import java.util.List;
3137
import java.util.Random;
3238
import java.util.concurrent.ThreadLocalRandom;
@@ -47,6 +53,9 @@ public class ExponentialHistogramMergeBench {
4753
@Param({ "0.01", "0.1", "0.25", "0.5", "1.0", "2.0" })
4854
double mergedHistoSizeFactor;
4955

56+
@Param({ "array-backed", "compressed" })
57+
String histoImplementation;
58+
5059
Random random;
5160
ExponentialHistogramMerger histoMerger;
5261

@@ -81,16 +90,54 @@ public void setUp() {
8190
bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
8291
generator.add(Math.pow(1.001, bucketIndex));
8392
}
84-
toMerge[i] = generator.getAndClear();
85-
cnt = getBucketCount(toMerge[i]);
93+
ExponentialHistogram histogram = generator.getAndClear();
94+
cnt = getBucketCount(histogram);
8695
if (cnt < dataPointSize) {
87-
throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
96+
throw new IllegalStateException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
97+
}
98+
99+
if ("array-backed".equals(histoImplementation)) {
100+
toMerge[i] = histogram;
101+
} else if ("compressed".equals(histoImplementation)) {
102+
toMerge[i] = asCompressedHistogram(histogram);
103+
} else {
104+
throw new IllegalArgumentException("Unknown implementation: " + histoImplementation);
88105
}
89106
}
90107

91108
index = 0;
92109
}
93110

111+
private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogram) {
112+
List<IndexWithCount> negativeBuckets = new ArrayList<>();
113+
List<IndexWithCount> positiveBuckets = new ArrayList<>();
114+
115+
BucketIterator it = histogram.negativeBuckets().iterator();
116+
while (it.hasNext()) {
117+
negativeBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
118+
it.advance();
119+
}
120+
it = histogram.positiveBuckets().iterator();
121+
while (it.hasNext()) {
122+
positiveBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
123+
it.advance();
124+
}
125+
126+
long totalCount = histogram.zeroBucket().count() + histogram.negativeBuckets().valueCount() + histogram.positiveBuckets()
127+
.valueCount();
128+
BytesStreamOutput histoBytes = new BytesStreamOutput();
129+
try {
130+
CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
131+
CompressedExponentialHistogram result = new CompressedExponentialHistogram();
132+
BytesRef data = histoBytes.bytes().toBytesRef();
133+
result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, data);
134+
return result;
135+
} catch (IOException e) {
136+
throw new RuntimeException(e);
137+
}
138+
139+
}
140+
94141
private static int getBucketCount(ExponentialHistogram histo) {
95142
int cnt = 0;
96143
for (BucketIterator it : List.of(histo.negativeBuckets().iterator(), histo.positiveBuckets().iterator())) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
258258
);
259259
config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
260260
config.setMergePolicy(new LogByteSizeMergePolicy());
261-
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
261+
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
262262
config.setCodec(new Elasticsearch900Lucene101Codec() {
263263

264264
@Override

‎build-tools-internal/gradle/wrapper/gradle-wrapper.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
distributionBase=GRADLE_USER_HOME
22
distributionPath=wrapper/dists
3-
distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
4-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
3+
distributionSha256Sum=f759b8dd5204e2e3fa4ca3e73f452f087153cf81bac9561eeb854229cc2c5365
4+
distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-all.zip
55
networkTimeout=10000
66
validateDistributionUrl=true
77
zipStoreBase=GRADLE_USER_HOME

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,10 @@ abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {
5353
}
5454
"""
5555

56-
subProject(":distribution:archives:integ-test-zip") << "configurations.create('extracted')\n"
57-
subProject(":distribution:archives:integ-test-zip") << "configurations.create('default')\n"
56+
subProject(":distribution:archives:integ-test-zip") << """
57+
apply plugin: 'base'
58+
configurations.create('extracted')
59+
"""
5860
}
5961

6062
void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackTests = []) {

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,11 +80,8 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
8080
def "registered distribution provides archives and directory variant"() {
8181
given:
8282
file('someFile.txt') << "some content"
83-
84-
settingsFile << """
85-
include ':consumer'
86-
include ':producer-tar'
87-
"""
83+
subProject("consumer")
84+
subProject("producer-tar")
8885

8986
buildFile << """
9087
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
@@ -154,9 +151,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
154151
def "builds extracted distribution via extractedAssemble"() {
155152
given:
156153
file('someFile.txt') << "some content"
157-
settingsFile << """
158-
include ':producer-tar'
159-
"""
154+
subProject("producer-tar")
160155

161156
buildFile << """
162157
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,5 +117,4 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
117117
result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" +
118118
"distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT")
119119
}
120-
121120
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,15 @@
99

1010
package org.elasticsearch.gradle.internal
1111

12-
import spock.lang.Ignore
12+
13+
import spock.lang.Unroll
1314

1415
import org.apache.commons.compress.archivers.tar.TarArchiveEntry
1516
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
1617
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
1718
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
1819
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
1920
import org.gradle.api.GradleException
20-
import spock.lang.Unroll
2121

2222
import java.nio.file.Files
2323
import java.nio.file.Path
@@ -66,8 +66,12 @@ tasks.register("buildBZip2Tar", SymbolicLinkPreservingTar) { SymbolicLinkPreserv
6666
from fileTree("archiveRoot")
6767
6868
into('config') {
69-
dirMode 0750
70-
fileMode 0660
69+
dirPermissions {
70+
unix(0750)
71+
}
72+
filePermissions {
73+
unix(0660)
74+
}
7175
from "real-folder2"
7276
}
7377
}
@@ -118,8 +122,10 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
118122
preserverTimestamp << [true, false]
119123
}
120124

121-
private boolean assertTar(final File archive, final Function<? super FileInputStream, ? extends InputStream> wrapper, boolean preserveFileTimestamps)
122-
throws IOException {
125+
private boolean assertTar(final File archive,
126+
final Function<? super FileInputStream, ? extends InputStream> wrapper,
127+
boolean preserveFileTimestamps)
128+
throws IOException {
123129
try (TarArchiveInputStream tar = new TarArchiveInputStream(wrapper.apply(new FileInputStream(archive)))) {
124130
TarArchiveEntry entry = tar.getNextTarEntry();
125131
boolean realFolderEntry = false;
@@ -132,7 +138,7 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
132138
if (entry.getName().equals("real-folder/")) {
133139
assert entry.isDirectory()
134140
realFolderEntry = true
135-
} else if (entry.getName().equals("real-folder/file")) {
141+
} else if (entry.getName().equals("real-folder/file")) {
136142
assert entry.isFile()
137143
fileEntry = true
138144
} else if (entry.getName().equals("real-folder/link-to-file")) {
@@ -145,7 +151,7 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
145151
} else if (entry.getName().equals("config/sub/")) {
146152
assert entry.isDirectory()
147153
assert entry.getMode() == 16872
148-
}else if (entry.getName().equals("link-in-folder/")) {
154+
} else if (entry.getName().equals("link-in-folder/")) {
149155
assert entry.isDirectory()
150156
linkInFolderEntry = true
151157
} else if (entry.getName().equals("link-in-folder/link-to-file")) {

‎build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix3/build.gradle‎

Whitespace-only changes.

0 commit comments

Comments
 (0)