Skip to content

Commit cbf06b7

Browse files
ywangdywangd
committed
Merge remote-tracking branch 'origin/main' into repositories-service-project-aware
2 parents f62e3fe + 59a10bd commit cbf06b7

File tree

316 files changed

+11452
-5771
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

316 files changed

+11452
-5771
lines changed

‎.buildkite/scripts/run-bc-upgrade-tests.sh‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -56,20 +56,17 @@ echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or s
5656

5757
cat <<EOF | buildkite-agent pipeline upload
5858
steps:
59-
- label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
60-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
61-
timeout_in_minutes: 300
62-
agents:
63-
provider: gcp
64-
image: family/elasticsearch-ubuntu-2004
65-
machineType: n1-standard-32
66-
buildDirectory: /dev/shm/bk
67-
preemptible: true
68-
retry:
69-
automatic:
70-
- exit_status: "-1"
71-
limit: 3
72-
signal_reason: none
73-
- signal_reason: agent_stop
74-
limit: 3
59+
- group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
60+
steps:
61+
- label: "bc-upgrade-tests-part{{matrix.PART}}"
62+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.PART}}
63+
timeout_in_minutes: 300
64+
agents:
65+
provider: gcp
66+
image: family/elasticsearch-ubuntu-2004
67+
machineType: n1-standard-32
68+
buildDirectory: /dev/shm/bk
69+
matrix:
70+
setup:
71+
PART: ["1", "2", "3", "4", "5", "6"]
7572
EOF

‎.buildkite/scripts/run-pr-upgrade-tests.sh‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,22 +24,19 @@ VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-to
2424

2525
echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
2626

27-
cat <<EOF | buildkite-agent pipeline upload
27+
cat << EOF | buildkite-agent pipeline upload
2828
steps:
29-
- label: pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH
30-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
31-
timeout_in_minutes: 300
32-
agents:
33-
provider: gcp
34-
image: family/elasticsearch-ubuntu-2004
35-
machineType: n1-standard-32
36-
buildDirectory: /dev/shm/bk
37-
preemptible: true
38-
retry:
39-
automatic:
40-
- exit_status: "-1"
41-
limit: 3
42-
signal_reason: none
43-
- signal_reason: agent_stop
44-
limit: 3
29+
- group: "pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH"
30+
steps:
31+
- label: "pr-upgrade-part-{{matrix.PART}}"
32+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.PART}}
33+
timeout_in_minutes: 300
34+
agents:
35+
provider: gcp
36+
image: family/elasticsearch-ubuntu-2004
37+
machineType: n1-standard-32
38+
buildDirectory: /dev/shm/bk
39+
matrix:
40+
setup:
41+
PART: ["1", "2", "3", "4", "5", "6"]
4542
EOF

‎.github/workflows/docs-build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ jobs:
1616
deployments: write
1717
id-token: write
1818
contents: read
19-
pull-requests: read
19+
pull-requests: write

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int4ScorerBenchmark.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.apache.lucene.store.Directory;
12+
import org.apache.lucene.store.IOContext;
13+
import org.apache.lucene.store.IndexInput;
14+
import org.apache.lucene.store.IndexOutput;
15+
import org.apache.lucene.store.MMapDirectory;
16+
import org.apache.lucene.util.VectorUtil;
17+
import org.elasticsearch.common.logging.LogConfigurator;
18+
import org.elasticsearch.core.IOUtils;
19+
import org.elasticsearch.simdvec.ES91Int4VectorsScorer;
20+
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
21+
import org.openjdk.jmh.annotations.Benchmark;
22+
import org.openjdk.jmh.annotations.BenchmarkMode;
23+
import org.openjdk.jmh.annotations.Fork;
24+
import org.openjdk.jmh.annotations.Measurement;
25+
import org.openjdk.jmh.annotations.Mode;
26+
import org.openjdk.jmh.annotations.OutputTimeUnit;
27+
import org.openjdk.jmh.annotations.Param;
28+
import org.openjdk.jmh.annotations.Scope;
29+
import org.openjdk.jmh.annotations.Setup;
30+
import org.openjdk.jmh.annotations.State;
31+
import org.openjdk.jmh.annotations.TearDown;
32+
import org.openjdk.jmh.annotations.Warmup;
33+
import org.openjdk.jmh.infra.Blackhole;
34+
35+
import java.io.IOException;
36+
import java.nio.file.Files;
37+
import java.util.concurrent.ThreadLocalRandom;
38+
import java.util.concurrent.TimeUnit;
39+
40+
@BenchmarkMode(Mode.Throughput)
41+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
42+
@State(Scope.Benchmark)
43+
// first iteration is complete garbage, so make sure we really warmup
44+
@Warmup(iterations = 4, time = 1)
45+
// real iterations. not useful to spend tons of time here, better to fork more
46+
@Measurement(iterations = 5, time = 1)
47+
// engage some noise reduction
48+
@Fork(value = 1)
49+
public class Int4ScorerBenchmark {
50+
51+
static {
52+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
53+
}
54+
55+
@Param({ "384", "702", "1024" })
56+
int dims;
57+
58+
int numVectors = 200;
59+
int numQueries = 10;
60+
61+
byte[] scratch;
62+
byte[][] binaryVectors;
63+
byte[][] binaryQueries;
64+
65+
ES91Int4VectorsScorer scorer;
66+
Directory dir;
67+
IndexInput in;
68+
69+
@Setup
70+
public void setup() throws IOException {
71+
binaryVectors = new byte[numVectors][dims];
72+
dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
73+
try (IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT)) {
74+
for (byte[] binaryVector : binaryVectors) {
75+
for (int i = 0; i < dims; i++) {
76+
// 4-bit quantization
77+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(16);
78+
}
79+
out.writeBytes(binaryVector, 0, binaryVector.length);
80+
}
81+
}
82+
83+
in = dir.openInput("vectors", IOContext.DEFAULT);
84+
binaryQueries = new byte[numVectors][dims];
85+
for (byte[] binaryVector : binaryVectors) {
86+
for (int i = 0; i < dims; i++) {
87+
// 4-bit quantization
88+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(16);
89+
}
90+
}
91+
92+
scratch = new byte[dims];
93+
scorer = ESVectorizationProvider.getInstance().newES91Int4VectorsScorer(in, dims);
94+
}
95+
96+
@TearDown
97+
public void teardown() throws IOException {
98+
IOUtils.close(dir, in);
99+
}
100+
101+
@Benchmark
102+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
103+
public void scoreFromArray(Blackhole bh) throws IOException {
104+
for (int j = 0; j < numQueries; j++) {
105+
in.seek(0);
106+
for (int i = 0; i < numVectors; i++) {
107+
in.readBytes(scratch, 0, dims);
108+
bh.consume(VectorUtil.int4DotProduct(binaryQueries[j], scratch));
109+
}
110+
}
111+
}
112+
113+
@Benchmark
114+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
115+
public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
116+
for (int j = 0; j < numQueries; j++) {
117+
in.seek(0);
118+
for (int i = 0; i < numVectors; i++) {
119+
bh.consume(scorer.int4DotProduct(binaryQueries[j]));
120+
}
121+
}
122+
}
123+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.elasticsearch.gradle.internal;
1111

1212
import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
13-
import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
1413
import org.gradle.api.Plugin;
1514
import org.gradle.api.Project;
1615
import org.gradle.api.artifacts.Configuration;
@@ -28,14 +27,14 @@ public void apply(final Project project) {
2827
var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
2928

3029
depsInfo.configure(t -> {
31-
t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
32-
t.setCompileOnlyConfiguration(
33-
project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
34-
);
35-
t.getConventionMapping().map("mappings", () -> {
36-
var depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
37-
return depLic.get().getMappings();
38-
});
30+
var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
31+
t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
32+
t.getClasspath().from(runtimeConfiguration);
33+
var compileOnlyConfiguration = project.getConfigurations()
34+
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
35+
t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
36+
t.getClasspath().from(compileOnlyConfiguration);
37+
3938
});
4039
Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
4140
dependenciesInfoFilesConfiguration.setCanBeResolved(false);

0 commit comments

Comments
 (0)