Skip to content

Commit dba1d9f

Browse files
tteofilitteofili
committed
Merge branch 'main' of github.com:elastic/elasticsearch into dbbq_bulk_size
2 parents f2e0fdb + 352229b commit dba1d9f

File tree

641 files changed

+39297
-4281
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

641 files changed

+39297
-4281
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 31 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.elasticsearch.cluster.metadata.IndexMetadata;
2727
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
2828
import org.elasticsearch.common.logging.LogConfigurator;
29-
import org.elasticsearch.common.lucene.Lucene;
3029
import org.elasticsearch.common.settings.Settings;
3130
import org.elasticsearch.common.unit.ByteSizeValue;
3231
import org.elasticsearch.common.util.BigArrays;
@@ -227,50 +226,44 @@ private static BlockLoader blockLoader(String name) {
227226
break;
228227
}
229228
ft.freeze();
230-
return new KeywordFieldMapper.KeywordFieldType(
231-
w.name,
232-
ft,
233-
Lucene.KEYWORD_ANALYZER,
234-
Lucene.KEYWORD_ANALYZER,
235-
Lucene.KEYWORD_ANALYZER,
236-
new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(ft.docValuesType() != DocValuesType.NONE),
237-
syntheticSource
238-
).blockLoader(new MappedFieldType.BlockLoaderContext() {
239-
@Override
240-
public String indexName() {
241-
return "benchmark";
242-
}
229+
return new KeywordFieldMapper.KeywordFieldType(w.name, ft, syntheticSource).blockLoader(
230+
new MappedFieldType.BlockLoaderContext() {
231+
@Override
232+
public String indexName() {
233+
return "benchmark";
234+
}
243235

244-
@Override
245-
public IndexSettings indexSettings() {
246-
throw new UnsupportedOperationException();
247-
}
236+
@Override
237+
public IndexSettings indexSettings() {
238+
throw new UnsupportedOperationException();
239+
}
248240

249-
@Override
250-
public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
251-
return MappedFieldType.FieldExtractPreference.NONE;
252-
}
241+
@Override
242+
public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
243+
return MappedFieldType.FieldExtractPreference.NONE;
244+
}
253245

254-
@Override
255-
public SearchLookup lookup() {
256-
throw new UnsupportedOperationException();
257-
}
246+
@Override
247+
public SearchLookup lookup() {
248+
throw new UnsupportedOperationException();
249+
}
258250

259-
@Override
260-
public Set<String> sourcePaths(String name) {
261-
return Set.of(name);
262-
}
251+
@Override
252+
public Set<String> sourcePaths(String name) {
253+
return Set.of(name);
254+
}
263255

264-
@Override
265-
public String parentField(String field) {
266-
throw new UnsupportedOperationException();
267-
}
256+
@Override
257+
public String parentField(String field) {
258+
throw new UnsupportedOperationException();
259+
}
268260

269-
@Override
270-
public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
271-
return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
261+
@Override
262+
public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
263+
return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
264+
}
272265
}
273-
});
266+
);
274267
}
275268
throw new IllegalArgumentException("can't read [" + name + "]");
276269
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
217217
);
218218
yield EvalMapper.toEvaluator(
219219
FOLD_CONTEXT,
220-
new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
220+
new DateTrunc(
221+
Source.EMPTY,
222+
new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION),
223+
timestamp,
224+
configuration()
225+
),
221226
layout(timestamp)
222227
).get(driverContext);
223228
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java‎

Lines changed: 70 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
1919
import org.elasticsearch.common.logging.LogConfigurator;
2020
import org.elasticsearch.core.IOUtils;
21+
import org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat;
2122
import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
23+
import org.elasticsearch.simdvec.ESNextOSQVectorsScorer;
2224
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
2325
import org.openjdk.jmh.annotations.Benchmark;
2426
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -43,7 +45,7 @@
4345
@OutputTimeUnit(TimeUnit.MILLISECONDS)
4446
@State(Scope.Benchmark)
4547
// first iteration is complete garbage, so make sure we really warmup
46-
@Warmup(iterations = 3, time = 1)
48+
@Warmup(iterations = 4, time = 1)
4749
// real iterations. not useful to spend tons of time here, better to fork more
4850
@Measurement(iterations = 5, time = 1)
4951
// engage some noise reduction
@@ -54,25 +56,25 @@ public class OSQScorerBenchmark {
5456
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
5557
}
5658

57-
@Param({ "16", "32", "64", "128", "256" })
58-
int bulkSize;
59-
6059
@Param({ "384", "782", "1024" })
6160
int dims;
6261

62+
@Param({ "1", "2", "4" })
63+
int bits;
64+
6365
int length;
6466

65-
final int numVectors = 16 * 64;
66-
final int numQueries = 10;
67+
int numVectors = ES91OSQVectorsScorer.BULK_SIZE * 10;
68+
int numQueries = 10;
6769

6870
byte[][] binaryVectors;
6971
byte[][] binaryQueries;
7072
OptimizedScalarQuantizer.QuantizationResult result;
7173
float centroidDp;
7274

7375
byte[] scratch;
74-
ES91OSQVectorsScorer scorerMmap;
75-
ES91OSQVectorsScorer scorerNfios;
76+
ESNextOSQVectorsScorer scorerMmap;
77+
ESNextOSQVectorsScorer scorerNfios;
7678

7779
Directory dirMmap;
7880
IndexInput inMmap;
@@ -86,7 +88,13 @@ public class OSQScorerBenchmark {
8688
@Setup
8789
public void setup() throws IOException {
8890
Random random = new Random(123);
89-
this.length = OptimizedScalarQuantizer.discretize(dims, 64) / 8;
91+
92+
this.length = switch (bits) {
93+
case 1 -> ESNextDiskBBQVectorsFormat.QuantEncoding.ONE_BIT_4BIT_QUERY.getDocPackedLength(dims);
94+
case 2 -> ESNextDiskBBQVectorsFormat.QuantEncoding.TWO_BIT_4BIT_QUERY.getDocPackedLength(dims);
95+
case 4 -> ESNextDiskBBQVectorsFormat.QuantEncoding.FOUR_BIT_SYMMETRIC.getDocPackedLength(dims);
96+
default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
97+
};
9098

9199
binaryVectors = new byte[numVectors][length];
92100
for (byte[] binaryVector : binaryVectors) {
@@ -97,9 +105,9 @@ public void setup() throws IOException {
97105
dirNiofs = new NIOFSDirectory(Files.createTempDirectory("vectorDataNFIOS"));
98106
IndexOutput outMmap = dirMmap.createOutput("vectors", IOContext.DEFAULT);
99107
IndexOutput outNfios = dirNiofs.createOutput("vectors", IOContext.DEFAULT);
100-
byte[] correctionBytes = new byte[14 * bulkSize];
101-
for (int i = 0; i < numVectors; i += bulkSize) {
102-
for (int j = 0; j < bulkSize; j++) {
108+
byte[] correctionBytes = new byte[14 * ES91OSQVectorsScorer.BULK_SIZE];
109+
for (int i = 0; i < numVectors; i += ES91OSQVectorsScorer.BULK_SIZE) {
110+
for (int j = 0; j < ES91OSQVectorsScorer.BULK_SIZE; j++) {
103111
outMmap.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
104112
outNfios.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
105113
}
@@ -111,8 +119,14 @@ public void setup() throws IOException {
111119
outNfios.close();
112120
inMmap = dirMmap.openInput("vectors", IOContext.DEFAULT);
113121
inNiofs = dirNiofs.openInput("vectors", IOContext.DEFAULT);
114-
115-
binaryQueries = new byte[numVectors][4 * length];
122+
int binaryQueryLength = switch (bits) {
123+
case 1 -> ESNextDiskBBQVectorsFormat.QuantEncoding.ONE_BIT_4BIT_QUERY.getQueryPackedLength(dims);
124+
case 2 -> ESNextDiskBBQVectorsFormat.QuantEncoding.TWO_BIT_4BIT_QUERY.getQueryPackedLength(dims);
125+
case 4 -> ESNextDiskBBQVectorsFormat.QuantEncoding.FOUR_BIT_SYMMETRIC.getQueryPackedLength(dims);
126+
default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
127+
};
128+
129+
binaryQueries = new byte[numVectors][binaryQueryLength];
116130
for (byte[] binaryVector : binaryVectors) {
117131
random.nextBytes(binaryVector);
118132
}
@@ -125,9 +139,27 @@ public void setup() throws IOException {
125139
centroidDp = random.nextFloat();
126140

127141
scratch = new byte[length];
128-
scorerMmap = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inMmap, bulkSize, dims);
129-
scorerNfios = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inNiofs, bulkSize, dims);
130-
scratchScores = new float[bulkSize];
142+
final int docBits;
143+
final int queryBits = switch (bits) {
144+
case 1 -> {
145+
docBits = 1;
146+
yield 4;
147+
}
148+
case 2 -> {
149+
docBits = 2;
150+
yield 4;
151+
}
152+
case 4 -> {
153+
docBits = 4;
154+
yield 4;
155+
}
156+
default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
157+
};
158+
scorerMmap = ESVectorizationProvider.getInstance()
159+
.newESNextOSQVectorsScorer(inMmap, (byte) queryBits, (byte) docBits, dims, length);
160+
scorerNfios = ESVectorizationProvider.getInstance()
161+
.newESNextOSQVectorsScorer(inNiofs, (byte) queryBits, (byte) docBits, dims, length);
162+
scratchScores = new float[16];
131163
corrections = new float[3];
132164
}
133165

@@ -136,23 +168,29 @@ public void teardown() throws IOException {
136168
IOUtils.close(dirMmap, inMmap, dirNiofs, inNiofs);
137169
}
138170

171+
@Benchmark
139172
public void scoreFromMemorySegmentOnlyVectorMmapScalar(Blackhole bh) throws IOException {
140173
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
141174
}
142175

176+
@Benchmark
177+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
143178
public void scoreFromMemorySegmentOnlyVectorMmapVect(Blackhole bh) throws IOException {
144179
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
145180
}
146181

182+
@Benchmark
147183
public void scoreFromMemorySegmentOnlyVectorNiofsScalar(Blackhole bh) throws IOException {
148184
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
149185
}
150186

187+
@Benchmark
188+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
151189
public void scoreFromMemorySegmentOnlyVectorNiofsVect(Blackhole bh) throws IOException {
152190
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
153191
}
154192

155-
private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
193+
private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
156194
for (int j = 0; j < numQueries; j++) {
157195
in.seek(0);
158196
for (int i = 0; i < numVectors; i++) {
@@ -177,28 +215,34 @@ private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91O
177215
}
178216
}
179217

218+
@Benchmark
180219
public void scoreFromMemorySegmentOnlyVectorBulkMmapScalar(Blackhole bh) throws IOException {
181220
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
182221
}
183222

223+
@Benchmark
224+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
184225
public void scoreFromMemorySegmentOnlyVectorBulkMmapVect(Blackhole bh) throws IOException {
185226
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
186227
}
187228

229+
@Benchmark
188230
public void scoreFromMemorySegmentOnlyVectorBulkNiofsScalar(Blackhole bh) throws IOException {
189231
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
190232
}
191233

234+
@Benchmark
235+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
192236
public void scoreFromMemorySegmentOnlyVectorBulkNiofsVect(Blackhole bh) throws IOException {
193237
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
194238
}
195239

196-
private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
240+
private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
197241
for (int j = 0; j < numQueries; j++) {
198242
in.seek(0);
199-
for (int i = 0; i < numVectors; i += bulkSize) {
200-
scorer.quantizeScoreBulk(binaryQueries[j], bulkSize, scratchScores);
201-
for (int k = 0; k < bulkSize; k++) {
243+
for (int i = 0; i < numVectors; i += 16) {
244+
scorer.quantizeScoreBulk(binaryQueries[j], ES91OSQVectorsScorer.BULK_SIZE, scratchScores);
245+
for (int k = 0; k < ES91OSQVectorsScorer.BULK_SIZE; k++) {
202246
in.readFloats(corrections, 0, corrections.length);
203247
int addition = Short.toUnsignedInt(in.readShort());
204248
float score = scorer.score(
@@ -220,6 +264,7 @@ private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, E
220264
}
221265
}
222266

267+
@Benchmark
223268
public void scoreFromMemorySegmentAllBulkMmapScalar(Blackhole bh) throws IOException {
224269
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
225270
}
@@ -230,6 +275,7 @@ public void scoreFromMemorySegmentAllBulkMmapVect(Blackhole bh) throws IOExcepti
230275
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
231276
}
232277

278+
@Benchmark
233279
public void scoreFromMemorySegmentAllBulkNiofsScalar(Blackhole bh) throws IOException {
234280
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
235281
}
@@ -240,10 +286,10 @@ public void scoreFromMemorySegmentAllBulkNiofsVect(Blackhole bh) throws IOExcept
240286
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
241287
}
242288

243-
private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
289+
private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
244290
for (int j = 0; j < numQueries; j++) {
245291
in.seek(0);
246-
for (int i = 0; i < numVectors; i += bulkSize) {
292+
for (int i = 0; i < numVectors; i += 16) {
247293
scorer.scoreBulk(
248294
binaryQueries[j],
249295
result.lowerInterval(),

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,9 @@ public void apply(Project project) {
4444
Provider<String> agplLicenseURL = licenseCommitProvider.map(
4545
licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX
4646
);
47-
// But stick the Elastic license url in project.ext so we can get it if we need to switch to it
47+
// Stick the license urls in project.ext so we can get them if we need to switch to them
4848
project.getExtensions().getExtraProperties().set("elasticLicenseUrl", elasticLicenseURL);
49+
project.getExtensions().getExtraProperties().set("agplLicenseUrl", agplLicenseURL);
4950

5051
MapProperty<String, Provider<String>> licensesProperty = project.getObjects()
5152
.mapProperty(String.class, (Class<Provider<String>>) (Class<?>) Provider.class)

‎build-tools-internal/gradle/wrapper/gradle-wrapper.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
distributionBase=GRADLE_USER_HOME
22
distributionPath=wrapper/dists
3-
distributionSha256Sum=b84e04fa845fecba48551f425957641074fcc00a88a84d2aae5808743b35fc85
4-
distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-all.zip
3+
distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3
4+
distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip
55
networkTimeout=10000
66
validateDistributionUrl=true
77
zipStoreBase=GRADLE_USER_HOME

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactBasePlugin.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.gradle.api.Plugin;
1313
import org.gradle.api.Project;
14+
import org.gradle.api.plugins.JavaPlugin;
1415
import org.gradle.api.provider.ProviderFactory;
1516

1617
import javax.inject.Inject;
@@ -25,6 +26,7 @@ public InternalTestArtifactBasePlugin(ProviderFactory providerFactory) {
2526

2627
@Override
2728
public void apply(Project project) {
29+
project.getPlugins().apply(JavaPlugin.class);
2830
project.getPlugins().apply(ElasticsearchJavaBasePlugin.class);
2931
project.getExtensions().create("testArtifacts", InternalTestArtifactExtension.class, project, providerFactory);
3032
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,15 @@ public void apply(final Project project) {
5353

5454
// only setup tests to build
5555
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
56-
final SourceSet testSourceSet = sourceSets.create("test");
56+
final SourceSet testSourceSet = sourceSets.maybeCreate("test");
5757

5858
project.getTasks().withType(Test.class).configureEach(test -> {
5959
test.setTestClassesDirs(testSourceSet.getOutput().getClassesDirs());
6060
test.setClasspath(testSourceSet.getRuntimeClasspath());
6161
});
6262

6363
// create a compileOnly configuration as others might expect it
64-
project.getConfigurations().create("compileOnly");
64+
project.getConfigurations().maybeCreate("compileOnly");
6565
RestTestUtil.setupJavaRestTestDependenciesDefaults(project, testSourceSet);
6666

6767
EclipseModel eclipse = project.getExtensions().getByType(EclipseModel.class);

‎build-tools-internal/src/main/resources/minimumGradleVersion‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
9.1.0
1+
9.2.0

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ tasks.named('zipAggregation').configure {
7777
destinationDirectory.set(layout.buildDirectory.dir("distributions"));
7878

7979
dependsOn gradle.includedBuild('build-tools').task(':zipElasticPublication')
80-
from(zipTree(gradle.includedBuild('build-tools').task(':zipElasticPublication').resolveTask().archiveFile.get()))
80+
from(zipTree("build-tools/build/nmcp/zip/publicationElastic.zip"))
8181
}
8282

8383
/**

0 commit comments

Comments
 (0)