Skip to content

Commit 8032b48

Browse files
markjhoymarkjhoy
authored
Merge branch 'main' into markjhoy/add-sparse_vector_index_options-support_semantic_text
2 parents 59d5833 + fc0ea64 commit 8032b48

File tree

179 files changed

+7060
-3623
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

179 files changed

+7060
-3623
lines changed

‎.buildkite/pipelines/intake.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ steps:
6565
timeout_in_minutes: 300
6666
matrix:
6767
setup:
68-
BWC_VERSION: ["8.17.10", "8.18.5", "9.0.5", "9.1.1", "9.2.0"]
68+
BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
6969
agents:
7070
provider: gcp
7171
image: family/elasticsearch-ubuntu-2404

‎.buildkite/pipelines/periodic-packaging.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,22 @@ steps:
334334
env:
335335
BWC_VERSION: 8.18.5
336336

337+
- label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
338+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
339+
timeout_in_minutes: 300
340+
matrix:
341+
setup:
342+
image:
343+
- rocky-8
344+
- ubuntu-2404
345+
agents:
346+
provider: gcp
347+
image: family/elasticsearch-{{matrix.image}}
348+
machineType: custom-16-32768
349+
buildDirectory: /dev/shm/bk
350+
env:
351+
BWC_VERSION: 8.19.0
352+
337353
- label: "{{matrix.image}} / 9.0.5 / packaging-tests-upgrade"
338354
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.5
339355
timeout_in_minutes: 300

‎.buildkite/pipelines/periodic.yml‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,25 @@ steps:
363363
- signal_reason: agent_stop
364364
limit: 3
365365

366+
- label: 8.19.0 / bwc
367+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.0#bwcTest
368+
timeout_in_minutes: 300
369+
agents:
370+
provider: gcp
371+
image: family/elasticsearch-ubuntu-2404
372+
machineType: n1-standard-32
373+
buildDirectory: /dev/shm/bk
374+
preemptible: true
375+
env:
376+
BWC_VERSION: 8.19.0
377+
retry:
378+
automatic:
379+
- exit_status: "-1"
380+
limit: 3
381+
signal_reason: none
382+
- signal_reason: agent_stop
383+
limit: 3
384+
366385
- label: 9.0.5 / bwc
367386
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.5#bwcTest
368387
timeout_in_minutes: 300
@@ -487,7 +506,7 @@ steps:
487506
setup:
488507
ES_RUNTIME_JAVA:
489508
- openjdk21
490-
BWC_VERSION: ["8.17.10", "8.18.5", "9.0.5", "9.1.1", "9.2.0"]
509+
BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
491510
agents:
492511
provider: gcp
493512
image: family/elasticsearch-ubuntu-2404
@@ -531,7 +550,7 @@ steps:
531550
ES_RUNTIME_JAVA:
532551
- openjdk21
533552
- openjdk23
534-
BWC_VERSION: ["8.17.10", "8.18.5", "9.0.5", "9.1.1", "9.2.0"]
553+
BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
535554
agents:
536555
provider: gcp
537556
image: family/elasticsearch-ubuntu-2404

‎.ci/bwcVersions‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ BWC_VERSION:
1818
- "8.16.6"
1919
- "8.17.10"
2020
- "8.18.5"
21+
- "8.19.0"
2122
- "9.0.5"
2223
- "9.1.1"
2324
- "9.2.0"

‎.ci/snapshotBwcVersions‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
BWC_VERSION:
22
- "8.17.10"
33
- "8.18.5"
4+
- "8.19.0"
45
- "9.0.5"
56
- "9.1.1"
67
- "9.2.0"

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int7ScorerBenchmark.java‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.apache.lucene.index.VectorSimilarityFunction;
12+
import org.apache.lucene.store.Directory;
13+
import org.apache.lucene.store.IOContext;
14+
import org.apache.lucene.store.IndexInput;
15+
import org.apache.lucene.store.IndexOutput;
16+
import org.apache.lucene.store.MMapDirectory;
17+
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
18+
import org.elasticsearch.common.logging.LogConfigurator;
19+
import org.elasticsearch.core.IOUtils;
20+
import org.elasticsearch.simdvec.ES91Int4VectorsScorer;
21+
import org.elasticsearch.simdvec.ES92Int7VectorsScorer;
22+
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
23+
import org.openjdk.jmh.annotations.Benchmark;
24+
import org.openjdk.jmh.annotations.BenchmarkMode;
25+
import org.openjdk.jmh.annotations.Fork;
26+
import org.openjdk.jmh.annotations.Measurement;
27+
import org.openjdk.jmh.annotations.Mode;
28+
import org.openjdk.jmh.annotations.OutputTimeUnit;
29+
import org.openjdk.jmh.annotations.Param;
30+
import org.openjdk.jmh.annotations.Scope;
31+
import org.openjdk.jmh.annotations.Setup;
32+
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.TearDown;
34+
import org.openjdk.jmh.annotations.Warmup;
35+
import org.openjdk.jmh.infra.Blackhole;
36+
37+
import java.io.IOException;
38+
import java.nio.file.Files;
39+
import java.util.concurrent.ThreadLocalRandom;
40+
import java.util.concurrent.TimeUnit;
41+
42+
@BenchmarkMode(Mode.Throughput)
43+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
44+
@State(Scope.Benchmark)
45+
// first iteration is complete garbage, so make sure we really warmup
46+
@Warmup(iterations = 4, time = 1)
47+
// real iterations. not useful to spend tons of time here, better to fork more
48+
@Measurement(iterations = 5, time = 1)
49+
// engage some noise reduction
50+
@Fork(value = 1)
51+
public class Int7ScorerBenchmark {
52+
53+
static {
54+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
55+
}
56+
57+
@Param({ "384", "782", "1024" })
58+
int dims;
59+
60+
int numVectors = 20 * ES92Int7VectorsScorer.BULK_SIZE;
61+
int numQueries = 5;
62+
63+
byte[] scratch;
64+
byte[][] binaryVectors;
65+
byte[][] binaryQueries;
66+
float[] scores = new float[ES92Int7VectorsScorer.BULK_SIZE];
67+
68+
ES92Int7VectorsScorer scorer;
69+
Directory dir;
70+
IndexInput in;
71+
72+
OptimizedScalarQuantizer.QuantizationResult queryCorrections;
73+
float centroidDp;
74+
75+
@Setup
76+
public void setup() throws IOException {
77+
binaryVectors = new byte[numVectors][dims];
78+
dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
79+
try (IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT)) {
80+
for (byte[] binaryVector : binaryVectors) {
81+
for (int i = 0; i < dims; i++) {
82+
// 4-bit quantization
83+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(128);
84+
}
85+
out.writeBytes(binaryVector, 0, binaryVector.length);
86+
ThreadLocalRandom.current().nextBytes(binaryVector);
87+
out.writeBytes(binaryVector, 0, 16); // corrections
88+
}
89+
}
90+
91+
queryCorrections = new OptimizedScalarQuantizer.QuantizationResult(
92+
ThreadLocalRandom.current().nextFloat(),
93+
ThreadLocalRandom.current().nextFloat(),
94+
ThreadLocalRandom.current().nextFloat(),
95+
Short.toUnsignedInt((short) ThreadLocalRandom.current().nextInt())
96+
);
97+
centroidDp = ThreadLocalRandom.current().nextFloat();
98+
99+
in = dir.openInput("vectors", IOContext.DEFAULT);
100+
binaryQueries = new byte[numVectors][dims];
101+
for (byte[] binaryVector : binaryVectors) {
102+
for (int i = 0; i < dims; i++) {
103+
// 7-bit quantization
104+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(128);
105+
}
106+
}
107+
108+
scratch = new byte[dims];
109+
scorer = ESVectorizationProvider.getInstance().newES92Int7VectorsScorer(in, dims);
110+
}
111+
112+
@TearDown
113+
public void teardown() throws IOException {
114+
IOUtils.close(dir, in);
115+
}
116+
117+
@Benchmark
118+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
119+
public void scoreFromMemorySegment(Blackhole bh) throws IOException {
120+
for (int j = 0; j < numQueries; j++) {
121+
in.seek(0);
122+
for (int i = 0; i < numVectors; i++) {
123+
bh.consume(
124+
scorer.score(
125+
binaryQueries[j],
126+
queryCorrections.lowerInterval(),
127+
queryCorrections.upperInterval(),
128+
queryCorrections.quantizedComponentSum(),
129+
queryCorrections.additionalCorrection(),
130+
VectorSimilarityFunction.EUCLIDEAN,
131+
centroidDp
132+
)
133+
);
134+
}
135+
}
136+
}
137+
138+
@Benchmark
139+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
140+
public void scoreFromMemorySegmentBulk(Blackhole bh) throws IOException {
141+
for (int j = 0; j < numQueries; j++) {
142+
in.seek(0);
143+
for (int i = 0; i < numVectors; i += ES91Int4VectorsScorer.BULK_SIZE) {
144+
scorer.scoreBulk(
145+
binaryQueries[j],
146+
queryCorrections.lowerInterval(),
147+
queryCorrections.upperInterval(),
148+
queryCorrections.quantizedComponentSum(),
149+
queryCorrections.additionalCorrection(),
150+
VectorSimilarityFunction.EUCLIDEAN,
151+
centroidDp,
152+
scores
153+
);
154+
for (float score : scores) {
155+
bh.consume(score);
156+
}
157+
}
158+
}
159+
}
160+
}

‎build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,9 @@ public void apply(Project project) {
5757
task.into("META-INF", copy -> copy.from(testBuildInfoTask));
5858
});
5959

60-
if (project.getRootProject().getName().equals("elasticsearch")) {
61-
project.getTasks()
62-
.withType(Test.class)
63-
.matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
64-
.configureEach(test -> {
65-
test.systemProperty("es.entitlement.enableForTests", "true");
66-
});
67-
}
60+
project.getTasks()
61+
.withType(Test.class)
62+
.matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
63+
.configureEach(test -> test.getSystemProperties().putIfAbsent("es.entitlement.enableForTests", "true"));
6864
}
6965
}

‎distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
# Extract Elasticsearch artifact
2525
################################################################################
2626
27-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:69f3df4cc5fd08b194a1a44dda2ff9f6665ac2b59063a2b2aedb8948ebd87f97 AS builder
27+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ea864ffab2b828ec157b2480c8b8dbd6f27da682c59a93d4b75a46163922b237 AS builder
2828
2929
# Install required packages to extract the Elasticsearch distribution
3030
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -103,7 +103,7 @@ WORKDIR /usr/share/elasticsearch/config
103103
# Add entrypoint
104104
################################################################################
105105

106-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:69f3df4cc5fd08b194a1a44dda2ff9f6665ac2b59063a2b2aedb8948ebd87f97
106+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ea864ffab2b828ec157b2480c8b8dbd6f27da682c59a93d4b75a46163922b237
107107

108108
RUN <%= retry.loop(package_manager,
109109
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
# Extract Elasticsearch artifact
2525
################################################################################
2626
27-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:1c4caa90ee9cf26c9143e44074f50ba9bb17636823bde4397751a5e0d846bfd6 AS builder
27+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:9ded4d2364e7f263cada56b0b9ca3ef643e8dac958a79df3d18c2a9f0a33fbc7 AS builder
2828
2929
# Install required packages to extract the Elasticsearch distribution
3030
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -79,7 +79,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
7979
# Add entrypoint
8080
################################################################################
8181

82-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:1c4caa90ee9cf26c9143e44074f50ba9bb17636823bde4397751a5e0d846bfd6
82+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:9ded4d2364e7f263cada56b0b9ca3ef643e8dac958a79df3d18c2a9f0a33fbc7
8383

8484
RUN <%= retry.loop(package_manager,
8585
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎distribution/docker/src/docker/iron_bank/hardening_manifest.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ tags:
1111
# Build args passed to Dockerfile ARGs
1212
args:
1313
BASE_IMAGE: "redhat/ubi/ubi9"
14-
BASE_TAG: "9.5"
14+
BASE_TAG: "9.6"
1515
# Docker image labels
1616
labels:
1717
org.opencontainers.image.title: "elasticsearch"

0 commit comments

Comments
 (0)