Skip to content

Commit dbaa908

Browse files
limotovalimotova
committed
Merge branch 'main' into add-stddev-population
2 parents 01e00e5 + d3049e0 commit dbaa908

File tree

252 files changed

+7051
-2842
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

252 files changed

+7051
-2842
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int4ScorerBenchmark.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.apache.lucene.store.Directory;
12+
import org.apache.lucene.store.IOContext;
13+
import org.apache.lucene.store.IndexInput;
14+
import org.apache.lucene.store.IndexOutput;
15+
import org.apache.lucene.store.MMapDirectory;
16+
import org.apache.lucene.util.VectorUtil;
17+
import org.elasticsearch.common.logging.LogConfigurator;
18+
import org.elasticsearch.core.IOUtils;
19+
import org.elasticsearch.simdvec.ES91Int4VectorsScorer;
20+
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
21+
import org.openjdk.jmh.annotations.Benchmark;
22+
import org.openjdk.jmh.annotations.BenchmarkMode;
23+
import org.openjdk.jmh.annotations.Fork;
24+
import org.openjdk.jmh.annotations.Measurement;
25+
import org.openjdk.jmh.annotations.Mode;
26+
import org.openjdk.jmh.annotations.OutputTimeUnit;
27+
import org.openjdk.jmh.annotations.Param;
28+
import org.openjdk.jmh.annotations.Scope;
29+
import org.openjdk.jmh.annotations.Setup;
30+
import org.openjdk.jmh.annotations.State;
31+
import org.openjdk.jmh.annotations.TearDown;
32+
import org.openjdk.jmh.annotations.Warmup;
33+
import org.openjdk.jmh.infra.Blackhole;
34+
35+
import java.io.IOException;
36+
import java.nio.file.Files;
37+
import java.util.concurrent.ThreadLocalRandom;
38+
import java.util.concurrent.TimeUnit;
39+
40+
@BenchmarkMode(Mode.Throughput)
41+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
42+
@State(Scope.Benchmark)
43+
// first iteration is complete garbage, so make sure we really warmup
44+
@Warmup(iterations = 4, time = 1)
45+
// real iterations. not useful to spend tons of time here, better to fork more
46+
@Measurement(iterations = 5, time = 1)
47+
// engage some noise reduction
48+
@Fork(value = 1)
49+
public class Int4ScorerBenchmark {
50+
51+
static {
52+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
53+
}
54+
55+
@Param({ "384", "702", "1024" })
56+
int dims;
57+
58+
int numVectors = 200;
59+
int numQueries = 10;
60+
61+
byte[] scratch;
62+
byte[][] binaryVectors;
63+
byte[][] binaryQueries;
64+
65+
ES91Int4VectorsScorer scorer;
66+
Directory dir;
67+
IndexInput in;
68+
69+
@Setup
70+
public void setup() throws IOException {
71+
binaryVectors = new byte[numVectors][dims];
72+
dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
73+
try (IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT)) {
74+
for (byte[] binaryVector : binaryVectors) {
75+
for (int i = 0; i < dims; i++) {
76+
// 4-bit quantization
77+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(16);
78+
}
79+
out.writeBytes(binaryVector, 0, binaryVector.length);
80+
}
81+
}
82+
83+
in = dir.openInput("vectors", IOContext.DEFAULT);
84+
binaryQueries = new byte[numVectors][dims];
85+
for (byte[] binaryVector : binaryVectors) {
86+
for (int i = 0; i < dims; i++) {
87+
// 4-bit quantization
88+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(16);
89+
}
90+
}
91+
92+
scratch = new byte[dims];
93+
scorer = ESVectorizationProvider.getInstance().newES91Int4VectorsScorer(in, dims);
94+
}
95+
96+
@TearDown
97+
public void teardown() throws IOException {
98+
IOUtils.close(dir, in);
99+
}
100+
101+
@Benchmark
102+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
103+
public void scoreFromArray(Blackhole bh) throws IOException {
104+
for (int j = 0; j < numQueries; j++) {
105+
in.seek(0);
106+
for (int i = 0; i < numVectors; i++) {
107+
in.readBytes(scratch, 0, dims);
108+
bh.consume(VectorUtil.int4DotProduct(binaryQueries[j], scratch));
109+
}
110+
}
111+
}
112+
113+
@Benchmark
114+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
115+
public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
116+
for (int j = 0; j < numQueries; j++) {
117+
in.seek(0);
118+
for (int i = 0; i < numVectors; i++) {
119+
bh.consume(scorer.int4DotProduct(binaryQueries[j]));
120+
}
121+
}
122+
}
123+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.elasticsearch.gradle.internal;
1111

1212
import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
13-
import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
1413
import org.gradle.api.Plugin;
1514
import org.gradle.api.Project;
1615
import org.gradle.api.artifacts.Configuration;
@@ -28,14 +27,14 @@ public void apply(final Project project) {
2827
var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
2928

3029
depsInfo.configure(t -> {
31-
t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
32-
t.setCompileOnlyConfiguration(
33-
project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
34-
);
35-
t.getConventionMapping().map("mappings", () -> {
36-
var depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
37-
return depLic.get().getMappings();
38-
});
30+
var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
31+
t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
32+
t.getClasspath().from(runtimeConfiguration);
33+
var compileOnlyConfiguration = project.getConfigurations()
34+
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
35+
t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
36+
t.getClasspath().from(compileOnlyConfiguration);
37+
3938
});
4039
Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
4140
dependenciesInfoFilesConfiguration.setCanBeResolved(false);

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java‎

Lines changed: 64 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,22 @@
1111

1212
import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
1313
import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer;
14-
import org.gradle.api.artifacts.Configuration;
15-
import org.gradle.api.artifacts.Dependency;
16-
import org.gradle.api.artifacts.DependencySet;
17-
import org.gradle.api.artifacts.ModuleVersionIdentifier;
14+
import org.gradle.api.artifacts.ArtifactCollection;
1815
import org.gradle.api.artifacts.ProjectDependency;
16+
import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
17+
import org.gradle.api.file.ConfigurableFileCollection;
1918
import org.gradle.api.file.DirectoryProperty;
2019
import org.gradle.api.file.ProjectLayout;
2120
import org.gradle.api.internal.ConventionTask;
2221
import org.gradle.api.model.ObjectFactory;
22+
import org.gradle.api.provider.MapProperty;
23+
import org.gradle.api.provider.Property;
24+
import org.gradle.api.provider.Provider;
2325
import org.gradle.api.provider.ProviderFactory;
2426
import org.gradle.api.tasks.Input;
2527
import org.gradle.api.tasks.InputDirectory;
2628
import org.gradle.api.tasks.InputFiles;
29+
import org.gradle.api.tasks.Internal;
2730
import org.gradle.api.tasks.Optional;
2831
import org.gradle.api.tasks.OutputFile;
2932
import org.gradle.api.tasks.TaskAction;
@@ -34,6 +37,7 @@
3437
import java.nio.file.StandardOpenOption;
3538
import java.util.Arrays;
3639
import java.util.LinkedHashMap;
40+
import java.util.Map;
3741
import java.util.Set;
3842
import java.util.regex.Pattern;
3943
import java.util.stream.Collectors;
@@ -51,31 +55,58 @@
5155
* <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
5256
* </ul>
5357
*/
54-
public class DependenciesInfoTask extends ConventionTask {
58+
public abstract class DependenciesInfoTask extends ConventionTask {
5559

56-
private final DirectoryProperty licensesDir;
60+
@Inject
61+
public abstract ProviderFactory getProviderFactory();
5762

58-
@OutputFile
59-
private File outputFile;
63+
/**
64+
* We have to use ArtifactCollection instead of ResolvedArtifactResult here as we're running
65+
* into a an issue in Gradle: https://github.com/gradle/gradle/issues/27582
66+
*/
6067

61-
private LinkedHashMap<String, String> mappings;
68+
@Internal
69+
abstract Property<ArtifactCollection> getRuntimeArtifacts();
6270

63-
public Configuration getRuntimeConfiguration() {
64-
return runtimeConfiguration;
71+
@Input
72+
public Provider<Set<ModuleComponentIdentifier>> getRuntimeModules() {
73+
return mapToModuleComponentIdentifiers(getRuntimeArtifacts().get());
6574
}
6675

67-
public void setRuntimeConfiguration(Configuration runtimeConfiguration) {
68-
this.runtimeConfiguration = runtimeConfiguration;
69-
}
76+
@Internal
77+
abstract Property<ArtifactCollection> getCompileOnlyArtifacts();
7078

71-
public Configuration getCompileOnlyConfiguration() {
72-
return compileOnlyConfiguration;
79+
@Input
80+
public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
81+
return mapToModuleComponentIdentifiers(getCompileOnlyArtifacts().get());
7382
}
7483

75-
public void setCompileOnlyConfiguration(Configuration compileOnlyConfiguration) {
76-
this.compileOnlyConfiguration = compileOnlyConfiguration;
84+
/**
85+
* We need to track file inputs here from the configurations we inspect to ensure we dont miss any
86+
* artifact transforms that might be applied and fail due to missing task dependency to jar
87+
* generating tasks.
88+
* */
89+
@InputFiles
90+
abstract ConfigurableFileCollection getClasspath();
91+
92+
private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
93+
return getProviderFactory().provider(
94+
() -> artifacts.getArtifacts()
95+
.stream()
96+
.map(r -> r.getId())
97+
.filter(id -> id instanceof ModuleComponentIdentifier)
98+
.map(id -> (ModuleComponentIdentifier) id)
99+
.collect(Collectors.toSet())
100+
);
77101
}
78102

103+
private final DirectoryProperty licensesDir;
104+
105+
@OutputFile
106+
private File outputFile;
107+
108+
private LinkedHashMap<String, String> mappings;
109+
79110
/**
80111
* Directory to read license files
81112
*/
@@ -102,17 +133,6 @@ public void setOutputFile(File outputFile) {
102133
this.outputFile = outputFile;
103134
}
104135

105-
/**
106-
* Dependencies to gather information from.
107-
*/
108-
@InputFiles
109-
private Configuration runtimeConfiguration;
110-
/**
111-
* We subtract compile-only dependencies.
112-
*/
113-
@InputFiles
114-
private Configuration compileOnlyConfiguration;
115-
116136
@Inject
117137
public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFactory, ProviderFactory providerFactory) {
118138
this.licensesDir = objectFactory.directoryProperty();
@@ -123,22 +143,18 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
123143

124144
@TaskAction
125145
public void generateDependenciesInfo() throws IOException {
126-
final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies();
127-
// we have to resolve the transitive dependencies and create a group:artifactId:version map
128-
129-
final Set<String> compileOnlyArtifacts = compileOnlyConfiguration.getResolvedConfiguration()
130-
.getResolvedArtifacts()
131-
.stream()
132-
.map(r -> {
133-
ModuleVersionIdentifier id = r.getModuleVersion().getId();
134-
return id.getGroup() + ":" + id.getName() + ":" + id.getVersion();
135-
})
136-
.collect(Collectors.toSet());
137146

147+
final Set<String> compileOnlyIds = getCompileOnlyModules().map(
148+
set -> set.stream()
149+
.map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
150+
.collect(Collectors.toSet())
151+
).get();
138152
final StringBuilder output = new StringBuilder();
139-
for (final Dependency dep : runtimeDependencies) {
153+
Map<String, String> mappings = getMappings().get();
154+
for (final ModuleComponentIdentifier dep : getRuntimeModules().get()) {
140155
// we do not need compile-only dependencies here
141-
if (compileOnlyArtifacts.contains(dep.getGroup() + ":" + dep.getName() + ":" + dep.getVersion())) {
156+
String moduleName = dep.getModuleIdentifier().getName();
157+
if (compileOnlyIds.contains(dep.getGroup() + ":" + moduleName + ":" + dep.getVersion())) {
142158
continue;
143159
}
144160

@@ -147,25 +163,20 @@ public void generateDependenciesInfo() throws IOException {
147163
continue;
148164
}
149165

150-
final String url = createURL(dep.getGroup(), dep.getName(), dep.getVersion());
151-
final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dep.getName());
152-
getLogger().info("mapped dependency " + dep.getGroup() + ":" + dep.getName() + " to " + dependencyName + " for license info");
166+
final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
167+
final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
168+
getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
153169

154170
final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
155-
output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
171+
output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
156172
}
157173

158174
Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
159175
}
160176

161177
@Input
162-
public LinkedHashMap<String, String> getMappings() {
163-
return mappings;
164-
}
165-
166-
public void setMappings(LinkedHashMap<String, String> mappings) {
167-
this.mappings = mappings;
168-
}
178+
@Optional
179+
public abstract MapProperty<String, String> getMappings();
169180

170181
/**
171182
* Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>

‎docs/changelog/125143.yaml‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎docs/changelog/129089.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129089
2+
summary: Update `sparse_vector` field mapping to include default setting for token pruning
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎docs/changelog/129325.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 129325
2+
summary: Check for model deployment in inference endpoints before stopping
3+
area: Machine Learning
4+
type: bug
5+
issues:
6+
- 128549

‎docs/changelog/129413.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129413
2+
summary: '`SageMaker` Elastic Payload'
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/129557.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129557
2+
summary: Pushdown for LIKE (LIST)
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/129659.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129659
2+
summary: Simplified RRF Retriever
3+
area: Search
4+
type: enhancement
5+
issues: []

0 commit comments

Comments
 (0)