Skip to content

Commit 9ca82dc

Browse files
prwhelanprwhelan
committed
Merge branch 'main' of github.com:prwhelan/elasticsearch into sagemaker-elastic-all-2
2 parents 1d67191 + 7d4bbcc commit 9ca82dc

File tree

22 files changed

+182
-129
lines changed

22 files changed

+182
-129
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.elasticsearch.gradle.internal;
1111

1212
import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
13-
import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
1413
import org.gradle.api.Plugin;
1514
import org.gradle.api.Project;
1615
import org.gradle.api.artifacts.Configuration;
@@ -28,14 +27,14 @@ public void apply(final Project project) {
2827
var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
2928

3029
depsInfo.configure(t -> {
31-
t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
32-
t.setCompileOnlyConfiguration(
33-
project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
34-
);
35-
t.getConventionMapping().map("mappings", () -> {
36-
var depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
37-
return depLic.get().getMappings();
38-
});
30+
var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
31+
t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
32+
t.getClasspath().from(runtimeConfiguration);
33+
var compileOnlyConfiguration = project.getConfigurations()
34+
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
35+
t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
36+
t.getClasspath().from(compileOnlyConfiguration);
37+
3938
});
4039
Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
4140
dependenciesInfoFilesConfiguration.setCanBeResolved(false);

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java‎

Lines changed: 64 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,22 @@
1111

1212
import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
1313
import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer;
14-
import org.gradle.api.artifacts.Configuration;
15-
import org.gradle.api.artifacts.Dependency;
16-
import org.gradle.api.artifacts.DependencySet;
17-
import org.gradle.api.artifacts.ModuleVersionIdentifier;
14+
import org.gradle.api.artifacts.ArtifactCollection;
1815
import org.gradle.api.artifacts.ProjectDependency;
16+
import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
17+
import org.gradle.api.file.ConfigurableFileCollection;
1918
import org.gradle.api.file.DirectoryProperty;
2019
import org.gradle.api.file.ProjectLayout;
2120
import org.gradle.api.internal.ConventionTask;
2221
import org.gradle.api.model.ObjectFactory;
22+
import org.gradle.api.provider.MapProperty;
23+
import org.gradle.api.provider.Property;
24+
import org.gradle.api.provider.Provider;
2325
import org.gradle.api.provider.ProviderFactory;
2426
import org.gradle.api.tasks.Input;
2527
import org.gradle.api.tasks.InputDirectory;
2628
import org.gradle.api.tasks.InputFiles;
29+
import org.gradle.api.tasks.Internal;
2730
import org.gradle.api.tasks.Optional;
2831
import org.gradle.api.tasks.OutputFile;
2932
import org.gradle.api.tasks.TaskAction;
@@ -34,6 +37,7 @@
3437
import java.nio.file.StandardOpenOption;
3538
import java.util.Arrays;
3639
import java.util.LinkedHashMap;
40+
import java.util.Map;
3741
import java.util.Set;
3842
import java.util.regex.Pattern;
3943
import java.util.stream.Collectors;
@@ -51,31 +55,58 @@
5155
* <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
5256
* </ul>
5357
*/
54-
public class DependenciesInfoTask extends ConventionTask {
58+
public abstract class DependenciesInfoTask extends ConventionTask {
5559

56-
private final DirectoryProperty licensesDir;
60+
@Inject
61+
public abstract ProviderFactory getProviderFactory();
5762

58-
@OutputFile
59-
private File outputFile;
63+
/**
64+
* We have to use ArtifactCollection instead of ResolvedArtifactResult here as we're running
65+
* into a an issue in Gradle: https://github.com/gradle/gradle/issues/27582
66+
*/
6067

61-
private LinkedHashMap<String, String> mappings;
68+
@Internal
69+
abstract Property<ArtifactCollection> getRuntimeArtifacts();
6270

63-
public Configuration getRuntimeConfiguration() {
64-
return runtimeConfiguration;
71+
@Input
72+
public Provider<Set<ModuleComponentIdentifier>> getRuntimeModules() {
73+
return mapToModuleComponentIdentifiers(getRuntimeArtifacts().get());
6574
}
6675

67-
public void setRuntimeConfiguration(Configuration runtimeConfiguration) {
68-
this.runtimeConfiguration = runtimeConfiguration;
69-
}
76+
@Internal
77+
abstract Property<ArtifactCollection> getCompileOnlyArtifacts();
7078

71-
public Configuration getCompileOnlyConfiguration() {
72-
return compileOnlyConfiguration;
79+
@Input
80+
public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
81+
return mapToModuleComponentIdentifiers(getCompileOnlyArtifacts().get());
7382
}
7483

75-
public void setCompileOnlyConfiguration(Configuration compileOnlyConfiguration) {
76-
this.compileOnlyConfiguration = compileOnlyConfiguration;
84+
/**
85+
* We need to track file inputs here from the configurations we inspect to ensure we dont miss any
86+
* artifact transforms that might be applied and fail due to missing task dependency to jar
87+
* generating tasks.
88+
* */
89+
@InputFiles
90+
abstract ConfigurableFileCollection getClasspath();
91+
92+
private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
93+
return getProviderFactory().provider(
94+
() -> artifacts.getArtifacts()
95+
.stream()
96+
.map(r -> r.getId())
97+
.filter(id -> id instanceof ModuleComponentIdentifier)
98+
.map(id -> (ModuleComponentIdentifier) id)
99+
.collect(Collectors.toSet())
100+
);
77101
}
78102

103+
private final DirectoryProperty licensesDir;
104+
105+
@OutputFile
106+
private File outputFile;
107+
108+
private LinkedHashMap<String, String> mappings;
109+
79110
/**
80111
* Directory to read license files
81112
*/
@@ -102,17 +133,6 @@ public void setOutputFile(File outputFile) {
102133
this.outputFile = outputFile;
103134
}
104135

105-
/**
106-
* Dependencies to gather information from.
107-
*/
108-
@InputFiles
109-
private Configuration runtimeConfiguration;
110-
/**
111-
* We subtract compile-only dependencies.
112-
*/
113-
@InputFiles
114-
private Configuration compileOnlyConfiguration;
115-
116136
@Inject
117137
public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFactory, ProviderFactory providerFactory) {
118138
this.licensesDir = objectFactory.directoryProperty();
@@ -123,22 +143,18 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
123143

124144
@TaskAction
125145
public void generateDependenciesInfo() throws IOException {
126-
final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies();
127-
// we have to resolve the transitive dependencies and create a group:artifactId:version map
128-
129-
final Set<String> compileOnlyArtifacts = compileOnlyConfiguration.getResolvedConfiguration()
130-
.getResolvedArtifacts()
131-
.stream()
132-
.map(r -> {
133-
ModuleVersionIdentifier id = r.getModuleVersion().getId();
134-
return id.getGroup() + ":" + id.getName() + ":" + id.getVersion();
135-
})
136-
.collect(Collectors.toSet());
137146

147+
final Set<String> compileOnlyIds = getCompileOnlyModules().map(
148+
set -> set.stream()
149+
.map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
150+
.collect(Collectors.toSet())
151+
).get();
138152
final StringBuilder output = new StringBuilder();
139-
for (final Dependency dep : runtimeDependencies) {
153+
Map<String, String> mappings = getMappings().get();
154+
for (final ModuleComponentIdentifier dep : getRuntimeModules().get()) {
140155
// we do not need compile-only dependencies here
141-
if (compileOnlyArtifacts.contains(dep.getGroup() + ":" + dep.getName() + ":" + dep.getVersion())) {
156+
String moduleName = dep.getModuleIdentifier().getName();
157+
if (compileOnlyIds.contains(dep.getGroup() + ":" + moduleName + ":" + dep.getVersion())) {
142158
continue;
143159
}
144160

@@ -147,25 +163,20 @@ public void generateDependenciesInfo() throws IOException {
147163
continue;
148164
}
149165

150-
final String url = createURL(dep.getGroup(), dep.getName(), dep.getVersion());
151-
final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dep.getName());
152-
getLogger().info("mapped dependency " + dep.getGroup() + ":" + dep.getName() + " to " + dependencyName + " for license info");
166+
final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
167+
final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
168+
getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
153169

154170
final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
155-
output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
171+
output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
156172
}
157173

158174
Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
159175
}
160176

161177
@Input
162-
public LinkedHashMap<String, String> getMappings() {
163-
return mappings;
164-
}
165-
166-
public void setMappings(LinkedHashMap<String, String> mappings) {
167-
this.mappings = mappings;
168-
}
178+
@Optional
179+
public abstract MapProperty<String, String> getMappings();
169180

170181
/**
171182
* Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>

‎muted-tests.yml‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -559,24 +559,27 @@ tests:
559559
- class: org.elasticsearch.test.apmintegration.TracesApmIT
560560
method: testApmIntegration
561561
issue: https://github.com/elastic/elasticsearch/issues/129651
562-
- class: org.elasticsearch.search.query.RescoreKnnVectorQueryIT
563-
method: testKnnSearchRescore
564-
issue: https://github.com/elastic/elasticsearch/issues/129713
565562
- class: org.elasticsearch.snapshots.SnapshotShutdownIT
566563
method: testSnapshotShutdownProgressTracker
567564
issue: https://github.com/elastic/elasticsearch/issues/129752
568565
- class: org.elasticsearch.xpack.security.SecurityRolesMultiProjectIT
569566
method: testUpdatingFileBasedRoleAffectsAllProjects
570567
issue: https://github.com/elastic/elasticsearch/issues/129775
571-
- class: org.elasticsearch.search.query.RescoreKnnVectorQueryIT
572-
method: testKnnQueryRescore
573-
issue: https://github.com/elastic/elasticsearch/issues/129809
574-
- class: org.elasticsearch.search.query.RescoreKnnVectorQueryIT
575-
method: testKnnRetriever
576-
issue: https://github.com/elastic/elasticsearch/issues/129818
568+
- class: org.elasticsearch.qa.verify_version_constants.VerifyVersionConstantsIT
569+
method: testLuceneVersionConstant
570+
issue: https://github.com/elastic/elasticsearch/issues/125638
571+
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
572+
method: test
573+
issue: https://github.com/elastic/elasticsearch/issues/129819
577574
- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceDiskSpaceTests
578575
method: testAbortingOrRunningMergeTaskHoldsUpBudget
579576
issue: https://github.com/elastic/elasticsearch/issues/129823
577+
- class: org.elasticsearch.index.store.FsDirectoryFactoryTests
578+
method: testPreload
579+
issue: https://github.com/elastic/elasticsearch/issues/129852
580+
- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT
581+
method: test {yaml=rrf/950_pinned_interaction/rrf with pinned retriever as a sub-retriever}
582+
issue: https://github.com/elastic/elasticsearch/issues/129845
580583

581584
# Examples:
582585
#

‎server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
8585
public void setup() throws IOException {
8686
String type = randomFrom(
8787
Arrays.stream(VectorIndexType.values())
88-
.filter(VectorIndexType::isQuantized)
88+
.filter(t -> t.isQuantized() && t.isEnabled())
8989
.map(t -> t.name().toLowerCase(Locale.ROOT))
9090
.collect(Collectors.toCollection(ArrayList::new))
9191
);

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,8 @@ static TransportVersion def(int id) {
201201
public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE_8_19 = def(8_841_0_53);
202202
public static final TransportVersion STREAMS_LOGS_SUPPORT_8_19 = def(8_841_0_54);
203203
public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE_8_19 = def(8_841_0_55);
204-
public static final TransportVersion ML_INFERENCE_SAGEMAKER_ELASTIC_8_19 = def(8_841_0_56);
204+
public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER_8_19 = def(8_841_0_56);
205+
public static final TransportVersion ML_INFERENCE_SAGEMAKER_ELASTIC_8_19 = def(8_841_0_57);
205206

206207
public static final TransportVersion V_9_0_0 = def(9_000_0_09);
207208
public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);

‎server/src/main/java/org/elasticsearch/cluster/metadata/StreamsMetadata.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,13 @@ public String getWriteableName() {
5959

6060
@Override
6161
public TransportVersion getMinimalSupportedVersion() {
62-
return TransportVersions.STREAMS_LOGS_SUPPORT;
62+
return TransportVersions.STREAMS_LOGS_SUPPORT_8_19;
63+
}
64+
65+
@Override
66+
public boolean supportsVersion(TransportVersion version) {
67+
return version.onOrAfter(TransportVersions.STREAMS_LOGS_SUPPORT)
68+
|| version.isPatchFrom(TransportVersions.STREAMS_LOGS_SUPPORT_8_19);
6369
}
6470

6571
public static NamedDiff<Metadata.ProjectCustom> readDiffFrom(StreamInput in) throws IOException {

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1655,11 +1655,16 @@ public boolean supportsElementType(ElementType elementType) {
16551655
public boolean supportsDimension(int dims) {
16561656
return true;
16571657
}
1658+
1659+
@Override
1660+
public boolean isEnabled() {
1661+
return IVF_FORMAT.isEnabled();
1662+
}
16581663
};
16591664

16601665
public static Optional<VectorIndexType> fromString(String type) {
16611666
return Stream.of(VectorIndexType.values())
1662-
.filter(vectorIndexType -> vectorIndexType != VectorIndexType.BBQ_IVF || IVF_FORMAT.isEnabled())
1667+
.filter(VectorIndexType::isEnabled)
16631668
.filter(vectorIndexType -> vectorIndexType.name.equals(type))
16641669
.findFirst();
16651670
}
@@ -1686,6 +1691,10 @@ public boolean isQuantized() {
16861691
return quantized;
16871692
}
16881693

1694+
public boolean isEnabled() {
1695+
return true;
1696+
}
1697+
16891698
public String getName() {
16901699
return name;
16911700
}

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplingQueryBuilder.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,11 +139,15 @@ public String getWriteableName() {
139139
return NAME;
140140
}
141141

142-
/**
143-
* The minimal version of the recipient this object can be sent to
144-
*/
142+
@Override
143+
public boolean supportsVersion(TransportVersion version) {
144+
return version.onOrAfter(TransportVersions.RANDOM_SAMPLER_QUERY_BUILDER)
145+
|| version.isPatchFrom(TransportVersions.RANDOM_SAMPLER_QUERY_BUILDER_8_19);
146+
}
147+
145148
@Override
146149
public TransportVersion getMinimalSupportedVersion() {
147-
return TransportVersions.RANDOM_SAMPLER_QUERY_BUILDER;
150+
assert false : "must not be called when overriding supportsVersion";
151+
throw new UnsupportedOperationException("must not be called when overriding supportsVersion");
148152
}
149153
}

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ public enum FeatureFlag {
2020
SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null),
2121
DOC_VALUES_SKIPPER("es.doc_values_skipper_feature_flag_enabled=true", Version.fromString("8.18.1"), null),
2222
USE_LUCENE101_POSTINGS_FORMAT("es.use_lucene101_postings_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null),
23-
INFERENCE_CUSTOM_SERVICE_ENABLED("es.inference_custom_service_feature_flag_enabled=true", Version.fromString("8.19.0"), null),
2423
IVF_FORMAT("es.ivf_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null),
2524
LOGS_STREAM("es.logs_stream_feature_flag_enabled=true", Version.fromString("9.1.0"), null);
2625

‎x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/BaseMockEISAuthServerTest.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.elasticsearch.common.util.concurrent.ThreadContext;
1515
import org.elasticsearch.core.TimeValue;
1616
import org.elasticsearch.test.cluster.ElasticsearchCluster;
17-
import org.elasticsearch.test.cluster.FeatureFlag;
1817
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
1918
import org.elasticsearch.test.rest.ESRestTestCase;
2019
import org.junit.ClassRule;
@@ -45,7 +44,6 @@ public class BaseMockEISAuthServerTest extends ESRestTestCase {
4544
// This plugin is located in the inference/qa/test-service-plugin package, look for TestInferenceServicePlugin
4645
.plugin("inference-service-test")
4746
.user("x_pack_rest_user", "x-pack-test-password")
48-
.feature(FeatureFlag.INFERENCE_CUSTOM_SERVICE_ENABLED)
4947
.build();
5048

5149
// The reason we're doing this is to make sure the mock server is initialized first so we can get the address before communicating

0 commit comments

Comments
 (0)