Skip to content

Commit c120f4f

Browse files
lukewhitinglukewhiting
committed
Merge branch 'main' of github.com:elastic/elasticsearch into es-12416-streams-enable-check-for-indices
2 parents db5100c + 249efde commit c120f4f

File tree

7 files changed

+313
-11
lines changed

7 files changed

+313
-11
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.elasticsearch.common.logging.LogConfigurator;
12+
import org.elasticsearch.index.codec.vectors.BQSpaceUtils;
13+
import org.elasticsearch.index.codec.vectors.BQVectorUtils;
14+
import org.openjdk.jmh.annotations.Benchmark;
15+
import org.openjdk.jmh.annotations.BenchmarkMode;
16+
import org.openjdk.jmh.annotations.Fork;
17+
import org.openjdk.jmh.annotations.Measurement;
18+
import org.openjdk.jmh.annotations.Mode;
19+
import org.openjdk.jmh.annotations.OutputTimeUnit;
20+
import org.openjdk.jmh.annotations.Param;
21+
import org.openjdk.jmh.annotations.Scope;
22+
import org.openjdk.jmh.annotations.Setup;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
import org.openjdk.jmh.infra.Blackhole;
26+
27+
import java.io.IOException;
28+
import java.util.Random;
29+
import java.util.concurrent.TimeUnit;
30+
31+
@BenchmarkMode(Mode.Throughput)
32+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
33+
@State(Scope.Benchmark)
34+
// first iteration is complete garbage, so make sure we really warmup
35+
@Warmup(iterations = 4, time = 1)
36+
// real iterations. not useful to spend tons of time here, better to fork more
37+
@Measurement(iterations = 5, time = 1)
38+
// engage some noise reduction
39+
@Fork(value = 1)
40+
public class TransposeHalfByteBenchmark {
41+
42+
static {
43+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
44+
}
45+
46+
@Param({ "384", "782", "1024" })
47+
int dims;
48+
49+
int length;
50+
51+
int numVectors = 1000;
52+
53+
int[][] qVectors;
54+
byte[] packed;
55+
56+
@Setup
57+
public void setup() throws IOException {
58+
Random random = new Random(123);
59+
60+
this.length = 4 * BQVectorUtils.discretize(dims, 64) / 8;
61+
this.packed = new byte[length];
62+
63+
qVectors = new int[numVectors][dims];
64+
for (int[] qVector : qVectors) {
65+
for (int i = 0; i < dims; i++) {
66+
qVector[i] = random.nextInt(16);
67+
}
68+
}
69+
}
70+
71+
@Benchmark
72+
public void transposeHalfByte(Blackhole bh) {
73+
for (int i = 0; i < numVectors; i++) {
74+
BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
75+
bh.consume(packed);
76+
}
77+
}
78+
79+
@Benchmark
80+
public void transposeHalfByteLegacy(Blackhole bh) {
81+
for (int i = 0; i < numVectors; i++) {
82+
BQSpaceUtils.transposeHalfByteLegacy(qVectors[i], packed);
83+
bh.consume(packed);
84+
}
85+
}
86+
}

‎muted-tests.yml‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -486,9 +486,6 @@ tests:
486486
- class: org.elasticsearch.xpack.ml.integration.RevertModelSnapshotIT
487487
method: testRevertModelSnapshot_DeleteInterveningResults
488488
issue: https://github.com/elastic/elasticsearch/issues/132349
489-
- class: org.elasticsearch.xpack.sql.action.SqlSearchPageTimeoutIT
490-
method: testSearchContextIsCleanedUpAfterPageTimeoutForAggregationQueries
491-
issue: https://github.com/elastic/elasticsearch/issues/132619
492489
- class: org.elasticsearch.xpack.ml.integration.TextEmbeddingQueryIT
493490
method: testHybridSearch
494491
issue: https://github.com/elastic/elasticsearch/issues/132703
@@ -513,6 +510,54 @@ tests:
513510
- class: org.elasticsearch.gradle.internal.transport.TransportVersionManagementPluginFuncTest
514511
method: cannot change committed ids to a branch
515512
issue: https://github.com/elastic/elasticsearch/issues/132790
513+
- class: org.elasticsearch.xpack.esql.CsvTests
514+
method: test {csv-spec:spatial.ConvertFromStringParseError}
515+
issue: https://github.com/elastic/elasticsearch/issues/132805
516+
- class: org.elasticsearch.xpack.esql.CsvTests
517+
method: test {csv-spec:floats.ScalbWithHugeScaleFactor}
518+
issue: https://github.com/elastic/elasticsearch/issues/132806
519+
- class: org.elasticsearch.reservedstate.service.FileSettingsServiceIT
520+
method: testSettingsAppliedOnStart
521+
issue: https://github.com/elastic/elasticsearch/issues/131210
522+
- class: org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapperTests
523+
method: testPruningScenarios
524+
issue: https://github.com/elastic/elasticsearch/issues/132810
525+
- class: org.elasticsearch.xpack.esql.CsvTests
526+
method: test {csv-spec:spatial_shapes.ConvertFromStringParseError}
527+
issue: https://github.com/elastic/elasticsearch/issues/132812
528+
- class: org.elasticsearch.xpack.esql.CsvTests
529+
method: test {csv-spec:string.MvStringNotEqualsLong}
530+
issue: https://github.com/elastic/elasticsearch/issues/132813
531+
- class: org.elasticsearch.xpack.esql.CsvTests
532+
method: test {csv-spec:spatial.ConvertCartesianFromStringParseError}
533+
issue: https://github.com/elastic/elasticsearch/issues/132814
534+
- class: org.elasticsearch.xpack.esql.CsvTests
535+
method: test {csv-spec:spatial_shapes.ConvertCartesianShapeFromStringParseError}
536+
issue: https://github.com/elastic/elasticsearch/issues/132815
537+
- class: org.elasticsearch.xpack.esql.CsvTests
538+
method: test {csv-spec:string.RepeatNegative}
539+
issue: https://github.com/elastic/elasticsearch/issues/132818
540+
- class: org.elasticsearch.xpack.esql.CsvTests
541+
method: test {csv-spec:stats.WeightedAvgWeightMvWarning}
542+
issue: https://github.com/elastic/elasticsearch/issues/132821
543+
- class: org.elasticsearch.xpack.esql.CsvTests
544+
method: test {csv-spec:ip.PushDownIPWithIn}
545+
issue: https://github.com/elastic/elasticsearch/issues/132822
546+
- class: org.elasticsearch.xpack.esql.CsvTests
547+
method: test {csv-spec:floats.ScalbWithHugeconstantFirstArgument}
548+
issue: https://github.com/elastic/elasticsearch/issues/132823
549+
- class: org.elasticsearch.xpack.esql.CsvTests
550+
method: test {csv-spec:boolean.MvSliceWarnings}
551+
issue: https://github.com/elastic/elasticsearch/issues/132824
552+
- class: org.elasticsearch.xpack.esql.CsvTests
553+
method: test {csv-spec:change_point.Values null column}
554+
issue: https://github.com/elastic/elasticsearch/issues/132825
555+
- class: org.elasticsearch.xpack.esql.CsvTests
556+
method: test {csv-spec:string.MvStringEqualsLongString}
557+
issue: https://github.com/elastic/elasticsearch/issues/132826
558+
- class: org.elasticsearch.xpack.esql.CsvTests
559+
method: test {csv-spec:stats.WeightedAvgFieldMvWarning}
560+
issue: https://github.com/elastic/elasticsearch/issues/132827
516561

517562
# Examples:
518563
#

‎server/src/main/java/org/elasticsearch/index/codec/vectors/BQSpaceUtils.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,57 @@ public class BQSpaceUtils {
3737
* @param quantQueryByte the byte array to store the transposed query vector
3838
*/
3939
public static void transposeHalfByte(byte[] q, byte[] quantQueryByte) {
40+
int limit = q.length - 7;
41+
int i = 0;
42+
int index = 0;
43+
for (; i < limit; i += 8, index++) {
44+
assert q[i] >= 0 && q[i] <= 15;
45+
assert q[i + 1] >= 0 && q[i + 1] <= 15;
46+
assert q[i + 2] >= 0 && q[i + 2] <= 15;
47+
assert q[i + 3] >= 0 && q[i + 3] <= 15;
48+
assert q[i + 4] >= 0 && q[i + 4] <= 15;
49+
assert q[i + 5] >= 0 && q[i + 5] <= 15;
50+
assert q[i + 6] >= 0 && q[i + 6] <= 15;
51+
assert q[i + 7] >= 0 && q[i + 7] <= 15;
52+
int lowerByte = (q[i] & 1) << 7 | (q[i + 1] & 1) << 6 | (q[i + 2] & 1) << 5 | (q[i + 3] & 1) << 4 | (q[i + 4] & 1) << 3 | (q[i
53+
+ 5] & 1) << 2 | (q[i + 6] & 1) << 1 | (q[i + 7] & 1);
54+
int lowerMiddleByte = ((q[i] >> 1) & 1) << 7 | ((q[i + 1] >> 1) & 1) << 6 | ((q[i + 2] >> 1) & 1) << 5 | ((q[i + 3] >> 1) & 1)
55+
<< 4 | ((q[i + 4] >> 1) & 1) << 3 | ((q[i + 5] >> 1) & 1) << 2 | ((q[i + 6] >> 1) & 1) << 1 | ((q[i + 7] >> 1) & 1);
56+
int upperMiddleByte = ((q[i] >> 2) & 1) << 7 | ((q[i + 1] >> 2) & 1) << 6 | ((q[i + 2] >> 2) & 1) << 5 | ((q[i + 3] >> 2) & 1)
57+
<< 4 | ((q[i + 4] >> 2) & 1) << 3 | ((q[i + 5] >> 2) & 1) << 2 | ((q[i + 6] >> 2) & 1) << 1 | ((q[i + 7] >> 2) & 1);
58+
int upperByte = ((q[i] >> 3) & 1) << 7 | ((q[i + 1] >> 3) & 1) << 6 | ((q[i + 2] >> 3) & 1) << 5 | ((q[i + 3] >> 3) & 1) << 4
59+
| ((q[i + 4] >> 3) & 1) << 3 | ((q[i + 5] >> 3) & 1) << 2 | ((q[i + 6] >> 3) & 1) << 1 | ((q[i + 7] >> 3) & 1);
60+
quantQueryByte[index] = (byte) lowerByte;
61+
quantQueryByte[index + quantQueryByte.length / 4] = (byte) lowerMiddleByte;
62+
quantQueryByte[index + quantQueryByte.length / 2] = (byte) upperMiddleByte;
63+
quantQueryByte[index + 3 * quantQueryByte.length / 4] = (byte) upperByte;
64+
}
65+
if (i == q.length) {
66+
return; // all done
67+
}
68+
int lowerByte = 0;
69+
int lowerMiddleByte = 0;
70+
int upperMiddleByte = 0;
71+
int upperByte = 0;
72+
for (int j = 7; i < q.length; j--, i++) {
73+
lowerByte |= (q[i] & 1) << j;
74+
lowerMiddleByte |= ((q[i] >> 1) & 1) << j;
75+
upperMiddleByte |= ((q[i] >> 2) & 1) << j;
76+
upperByte |= ((q[i] >> 3) & 1) << j;
77+
}
78+
quantQueryByte[index] = (byte) lowerByte;
79+
quantQueryByte[index + quantQueryByte.length / 4] = (byte) lowerMiddleByte;
80+
quantQueryByte[index + quantQueryByte.length / 2] = (byte) upperMiddleByte;
81+
quantQueryByte[index + 3 * quantQueryByte.length / 4] = (byte) upperByte;
82+
}
83+
84+
/**
85+
* Same as {@link #transposeHalfByte(byte[], byte[])} but with more readable but slower code.
86+
*
87+
* @param q the query vector, assumed to be half-byte quantized with values between 0 and 15
88+
* @param quantQueryByte the byte array to store the transposed query vector
89+
* */
90+
public static void transposeHalfByteLegacy(byte[] q, byte[] quantQueryByte) {
4091
for (int i = 0; i < q.length;) {
4192
assert q[i] >= 0 && q[i] <= 15;
4293
int lowerByte = 0;
@@ -66,6 +117,57 @@ public static void transposeHalfByte(byte[] q, byte[] quantQueryByte) {
66117
* @param quantQueryByte the byte array to store the transposed query vector
67118
* */
68119
public static void transposeHalfByte(int[] q, byte[] quantQueryByte) {
120+
int limit = q.length - 7;
121+
int i = 0;
122+
int index = 0;
123+
for (; i < limit; i += 8, index++) {
124+
assert q[i] >= 0 && q[i] <= 15;
125+
assert q[i + 1] >= 0 && q[i + 1] <= 15;
126+
assert q[i + 2] >= 0 && q[i + 2] <= 15;
127+
assert q[i + 3] >= 0 && q[i + 3] <= 15;
128+
assert q[i + 4] >= 0 && q[i + 4] <= 15;
129+
assert q[i + 5] >= 0 && q[i + 5] <= 15;
130+
assert q[i + 6] >= 0 && q[i + 6] <= 15;
131+
assert q[i + 7] >= 0 && q[i + 7] <= 15;
132+
int lowerByte = (q[i] & 1) << 7 | (q[i + 1] & 1) << 6 | (q[i + 2] & 1) << 5 | (q[i + 3] & 1) << 4 | (q[i + 4] & 1) << 3 | (q[i
133+
+ 5] & 1) << 2 | (q[i + 6] & 1) << 1 | (q[i + 7] & 1);
134+
int lowerMiddleByte = ((q[i] >> 1) & 1) << 7 | ((q[i + 1] >> 1) & 1) << 6 | ((q[i + 2] >> 1) & 1) << 5 | ((q[i + 3] >> 1) & 1)
135+
<< 4 | ((q[i + 4] >> 1) & 1) << 3 | ((q[i + 5] >> 1) & 1) << 2 | ((q[i + 6] >> 1) & 1) << 1 | ((q[i + 7] >> 1) & 1);
136+
int upperMiddleByte = ((q[i] >> 2) & 1) << 7 | ((q[i + 1] >> 2) & 1) << 6 | ((q[i + 2] >> 2) & 1) << 5 | ((q[i + 3] >> 2) & 1)
137+
<< 4 | ((q[i + 4] >> 2) & 1) << 3 | ((q[i + 5] >> 2) & 1) << 2 | ((q[i + 6] >> 2) & 1) << 1 | ((q[i + 7] >> 2) & 1);
138+
int upperByte = ((q[i] >> 3) & 1) << 7 | ((q[i + 1] >> 3) & 1) << 6 | ((q[i + 2] >> 3) & 1) << 5 | ((q[i + 3] >> 3) & 1) << 4
139+
| ((q[i + 4] >> 3) & 1) << 3 | ((q[i + 5] >> 3) & 1) << 2 | ((q[i + 6] >> 3) & 1) << 1 | ((q[i + 7] >> 3) & 1);
140+
quantQueryByte[index] = (byte) lowerByte;
141+
quantQueryByte[index + quantQueryByte.length / 4] = (byte) lowerMiddleByte;
142+
quantQueryByte[index + quantQueryByte.length / 2] = (byte) upperMiddleByte;
143+
quantQueryByte[index + 3 * quantQueryByte.length / 4] = (byte) upperByte;
144+
}
145+
if (i == q.length) {
146+
return; // all done
147+
}
148+
int lowerByte = 0;
149+
int lowerMiddleByte = 0;
150+
int upperMiddleByte = 0;
151+
int upperByte = 0;
152+
for (int j = 7; i < q.length; j--, i++) {
153+
lowerByte |= (q[i] & 1) << j;
154+
lowerMiddleByte |= ((q[i] >> 1) & 1) << j;
155+
upperMiddleByte |= ((q[i] >> 2) & 1) << j;
156+
upperByte |= ((q[i] >> 3) & 1) << j;
157+
}
158+
quantQueryByte[index] = (byte) lowerByte;
159+
quantQueryByte[index + quantQueryByte.length / 4] = (byte) lowerMiddleByte;
160+
quantQueryByte[index + quantQueryByte.length / 2] = (byte) upperMiddleByte;
161+
quantQueryByte[index + 3 * quantQueryByte.length / 4] = (byte) upperByte;
162+
}
163+
164+
/**
165+
* Same as {@link #transposeHalfByte(int[], byte[])} but with more readable but slower code.
166+
*
167+
* @param q the query vector, assumed to be half-byte quantized with values between 0 and 15
168+
* @param quantQueryByte the byte array to store the transposed query vector
169+
* */
170+
public static void transposeHalfByteLegacy(int[] q, byte[] quantQueryByte) {
69171
for (int i = 0; i < q.length;) {
70172
assert q[i] >= 0 && q[i] <= 15;
71173
int lowerByte = 0;

‎server/src/test/java/org/elasticsearch/index/codec/vectors/BQSpaceUtilsTests.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.codec.vectors;
11+
12+
import org.elasticsearch.test.ESTestCase;
13+
14+
public class BQSpaceUtilsTests extends ESTestCase {
15+
16+
public void testIntegerTransposeHalfByte() {
17+
int dims = randomIntBetween(16, 2048);
18+
int[] toPack = new int[dims];
19+
for (int i = 0; i < dims; i++) {
20+
toPack[i] = randomInt(15);
21+
}
22+
int length = 4 * BQVectorUtils.discretize(dims, 64) / 8;
23+
byte[] packed = new byte[length];
24+
byte[] packedLegacy = new byte[length];
25+
BQSpaceUtils.transposeHalfByteLegacy(toPack, packedLegacy);
26+
BQSpaceUtils.transposeHalfByte(toPack, packed);
27+
assertArrayEquals(packedLegacy, packed);
28+
}
29+
30+
public void testByteTransposeHalfByte() {
31+
int dims = randomIntBetween(16, 2048);
32+
byte[] toPack = new byte[dims];
33+
for (int i = 0; i < dims; i++) {
34+
toPack[i] = (byte) randomInt(15);
35+
}
36+
int length = 4 * BQVectorUtils.discretize(dims, 64) / 8;
37+
byte[] packed = new byte[length];
38+
byte[] packedLegacy = new byte[length];
39+
BQSpaceUtils.transposeHalfByteLegacy(toPack, packedLegacy);
40+
BQSpaceUtils.transposeHalfByte(toPack, packed);
41+
assertArrayEquals(packedLegacy, packed);
42+
}
43+
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,15 @@ yellow | [255.0, 255.0, 0.0]
7070
knnWithPrefilter
7171
required_capability: knn_function_v3
7272

73-
from colors metadata _score
74-
| where knn(rgb_vector, [128,128,0], 10) and (match(color, "olive") or match(color, "green"))
75-
| sort _score desc, color asc
76-
| keep color, rgb_vector
73+
from colors
74+
| where knn(rgb_vector, [120,180,0], 10) and (match(color, "olive") or match(color, "green"))
75+
| sort color asc
76+
| keep color
7777
;
7878

79-
color:text | rgb_vector:dense_vector
80-
olive | [128.0, 128.0, 0.0]
81-
green | [0.0, 128.0, 0.0]
79+
color:text
80+
green
81+
olive
8282
;
8383

8484
knnWithNegatedPrefilter

0 commit comments

Comments
 (0)