Skip to content

Commit 00a317b

Browse files
mridula-s109mridula-s109
authored
Merge branch 'main' into topsnippet/foldable
2 parents 9aae38f + d8aae4c commit 00a317b

File tree

516 files changed

+26651
-9434
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

516 files changed

+26651
-9434
lines changed

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ dependencies {
5151
api(project(':x-pack:plugin:logsdb'))
5252
implementation project(path: ':libs:native')
5353
implementation project(path: ':libs:simdvec')
54+
implementation (testFixtures(project(path: ':libs:simdvec')))
5455
implementation project(path: ':libs:swisshash')
5556
implementation project(path: ':libs:exponential-histogram')
5657
implementation(project(':x-pack:plugin:searchable-snapshots')) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ public void setup() {
126126
minimumVersion,
127127
UNMAPPED_FIELDS.defaultValue()
128128
),
129-
new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
129+
new Verifier(new Metrics(functionRegistry, true, true), new XPackLicenseState(() -> 0L))
130130
);
131131
defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small(), minimumVersion));
132132
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/TopNBenchmark.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.compute.data.ElementType;
2424
import org.elasticsearch.compute.data.Page;
2525
import org.elasticsearch.compute.operator.Operator;
26+
import org.elasticsearch.compute.operator.topn.SharedMinCompetitive;
2627
import org.elasticsearch.compute.operator.topn.TopNEncoder;
2728
import org.elasticsearch.compute.operator.topn.TopNOperator;
2829
import org.elasticsearch.indices.breaker.CircuitBreakerMetrics;
@@ -56,6 +57,10 @@
5657
@State(Scope.Thread)
5758
@Fork(1)
5859
public class TopNBenchmark {
60+
static {
61+
LogConfigurator.configureESLogging();
62+
}
63+
5964
private static final BlockFactory blockFactory = BlockFactory.getInstance(
6065
new NoopCircuitBreaker("noop"),
6166
BigArrays.NON_RECYCLING_INSTANCE
@@ -133,6 +138,19 @@ private static Operator operator(String data, int topCount, boolean sortedInput)
133138
List.of(),
134139
ClusterSettings.createBuiltInClusterSettings()
135140
);
141+
SharedMinCompetitive.Supplier minCompetitive = new SharedMinCompetitive.Supplier(
142+
blockFactory.breaker(),
143+
IntStream.range(0, encoders.size())
144+
.mapToObj(
145+
i -> new SharedMinCompetitive.KeyConfig(
146+
elementTypes.get(i),
147+
encoders.get(i),
148+
sortOrders.get(i).asc(),
149+
sortOrders.get(i).nullsFirst()
150+
)
151+
)
152+
.toList()
153+
);
136154
return new TopNOperator(
137155
blockFactory,
138156
breakerService.getBreaker(CircuitBreaker.REQUEST),
@@ -141,7 +159,8 @@ private static Operator operator(String data, int topCount, boolean sortedInput)
141159
encoders,
142160
sortOrders,
143161
8 * 1024,
144-
sortedInput ? TopNOperator.InputOrdering.SORTED : TopNOperator.InputOrdering.NOT_SORTED
162+
sortedInput ? TopNOperator.InputOrdering.SORTED : TopNOperator.InputOrdering.NOT_SORTED,
163+
minCompetitive // This is optional, but doesn't add much overhead either way
145164
);
146165
}
147166

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 49 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ private static BlockLoader blockLoader(String name) {
237237
break;
238238
}
239239
ft.freeze();
240-
return new KeywordFieldMapper.KeywordFieldType(w.name, ft, syntheticSource).blockLoader(blContext());
240+
return new KeywordFieldMapper.KeywordFieldType(w.name, ft, syntheticSource).blockLoader(new BenchContext());
241241
}
242242
throw new IllegalArgumentException("can't read [" + name + "]");
243243
}
@@ -279,7 +279,7 @@ private static BlockLoader numericBlockLoader(WhereAndBaseName w, NumberFieldMap
279279
null,
280280
null,
281281
false
282-
).blockLoader(blContext());
282+
).blockLoader(new BenchContext());
283283
}
284284

285285
/**
@@ -586,52 +586,60 @@ public void teardownIndex() throws IOException {
586586
IOUtils.close(reader, directory);
587587
}
588588

589-
private static MappedFieldType.BlockLoaderContext blContext() {
590-
return new MappedFieldType.BlockLoaderContext() {
591-
@Override
592-
public String indexName() {
593-
return "benchmark";
594-
}
589+
private static class BenchContext implements MappedFieldType.BlockLoaderContext {
590+
@Override
591+
public String indexName() {
592+
return "benchmark";
593+
}
595594

596-
@Override
597-
public IndexSettings indexSettings() {
598-
throw new UnsupportedOperationException();
599-
}
595+
@Override
596+
public IndexSettings indexSettings() {
597+
throw new UnsupportedOperationException();
598+
}
600599

601-
@Override
602-
public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
603-
return MappedFieldType.FieldExtractPreference.NONE;
604-
}
600+
@Override
601+
public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
602+
return MappedFieldType.FieldExtractPreference.NONE;
603+
}
605604

606-
@Override
607-
public SearchLookup lookup() {
608-
throw new UnsupportedOperationException();
609-
}
605+
@Override
606+
public SearchLookup lookup() {
607+
throw new UnsupportedOperationException();
608+
}
610609

611-
@Override
612-
public Set<String> sourcePaths(String name) {
613-
return Set.of(name);
614-
}
610+
@Override
611+
public Set<String> sourcePaths(String name) {
612+
return Set.of(name);
613+
}
615614

616-
@Override
617-
public String parentField(String field) {
618-
throw new UnsupportedOperationException();
619-
}
615+
@Override
616+
public String parentField(String field) {
617+
throw new UnsupportedOperationException();
618+
}
620619

621-
@Override
622-
public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
623-
return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
624-
}
620+
@Override
621+
public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
622+
return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
623+
}
625624

626-
@Override
627-
public MappingLookup mappingLookup() {
628-
return null;
629-
}
625+
@Override
626+
public MappingLookup mappingLookup() {
627+
return null;
628+
}
630629

631-
@Override
632-
public BlockLoaderFunctionConfig blockLoaderFunctionConfig() {
633-
return null;
634-
}
635-
};
630+
@Override
631+
public BlockLoaderFunctionConfig blockLoaderFunctionConfig() {
632+
return null;
633+
}
634+
635+
@Override
636+
public ByteSizeValue ordinalsByteSize() {
637+
return DEFAULT_ORDINALS_BYTE_SIZE;
638+
}
639+
640+
@Override
641+
public ByteSizeValue scriptByteSize() {
642+
return DEFAULT_SCRIPT_BYTE_SIZE;
643+
}
636644
}
637645
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/BenchmarkUtils.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@
1010
package org.elasticsearch.benchmark.vector.scorer;
1111

1212
import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
13+
import org.apache.lucene.codecs.lucene95.OffHeapByteVectorValues;
1314
import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;
1415
import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorScorer;
1516
import org.apache.lucene.codecs.lucene99.OffHeapQuantizedByteVectorValues;
17+
import org.apache.lucene.index.ByteVectorValues;
1618
import org.apache.lucene.index.FloatVectorValues;
1719
import org.apache.lucene.index.VectorSimilarityFunction;
1820
import org.apache.lucene.store.Directory;
@@ -54,13 +56,21 @@ static void writeInt7VectorData(Directory dir, byte[][] vectors, float[] offsets
5456
static void writeFloatVectorData(Directory dir, float[][] vectors) throws IOException {
5557
try (IndexOutput out = dir.createOutput("vector.data", IOContext.DEFAULT)) {
5658
ByteBuffer buffer = ByteBuffer.allocate(vectors[0].length * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN);
57-
for (int v = 0; v < vectors.length; v++) {
58-
buffer.asFloatBuffer().put(vectors[v]);
59+
for (float[] vector : vectors) {
60+
buffer.asFloatBuffer().put(vector);
5961
out.writeBytes(buffer.array(), buffer.capacity());
6062
}
6163
}
6264
}
6365

66+
static void writeByteVectorData(Directory dir, byte[][] vectors) throws IOException {
67+
try (IndexOutput out = dir.createOutput("vector.data", IOContext.DEFAULT)) {
68+
for (byte[] vector : vectors) {
69+
out.writeBytes(vector, vector.length);
70+
}
71+
}
72+
}
73+
6474
static VectorScorerFactory getScorerFactoryOrDie() {
6575
var optionalVectorScorerFactory = VectorScorerFactory.instance();
6676
if (optionalVectorScorerFactory.isEmpty()) {
@@ -85,6 +95,11 @@ static FloatVectorValues floatVectorValues(int dims, int size, IndexInput in, Ve
8595
return new OffHeapFloatVectorValues.DenseOffHeapVectorValues(dims, size, slice, dims * Float.BYTES, null, sim);
8696
}
8797

98+
static ByteVectorValues byteVectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
99+
var slice = in.slice("values", 0, in.length());
100+
return new OffHeapByteVectorValues.DenseOffHeapVectorValues(dims, size, slice, dims, null, sim);
101+
}
102+
88103
static QuantizedByteVectorValues quantizedVectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim)
89104
throws IOException {
90105
var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
@@ -96,6 +111,10 @@ static RandomVectorScorerSupplier luceneScoreSupplier(FloatVectorValues values,
96111
return FlatVectorScorerUtil.getLucene99FlatVectorsScorer().getRandomVectorScorerSupplier(sim, values);
97112
}
98113

114+
static RandomVectorScorerSupplier luceneScoreSupplier(ByteVectorValues values, VectorSimilarityFunction sim) throws IOException {
115+
return FlatVectorScorerUtil.getLucene99FlatVectorsScorer().getRandomVectorScorerSupplier(sim, values);
116+
}
117+
99118
static RandomVectorScorerSupplier luceneScoreSupplier(QuantizedByteVectorValues values, VectorSimilarityFunction sim)
100119
throws IOException {
101120
return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorerSupplier(sim, values);
@@ -105,6 +124,10 @@ static RandomVectorScorer luceneScorer(FloatVectorValues values, VectorSimilarit
105124
return FlatVectorScorerUtil.getLucene99FlatVectorsScorer().getRandomVectorScorer(sim, values, queryVec);
106125
}
107126

127+
static RandomVectorScorer luceneScorer(ByteVectorValues values, VectorSimilarityFunction sim, byte[] queryVec) throws IOException {
128+
return FlatVectorScorerUtil.getLucene99FlatVectorsScorer().getRandomVectorScorer(sim, values, queryVec);
129+
}
130+
108131
static RandomVectorScorer luceneScorer(QuantizedByteVectorValues values, VectorSimilarityFunction sim, float[] queryVec)
109132
throws IOException {
110133
return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorer(sim, values, queryVec);

0 commit comments

Comments
 (0)