Skip to content

Commit d6fc39c

Browse files
mark-vieiramark-vieira
authored
Fixes for module projects in new tests clusters and auto security config (#92533) (#92535)
Fix an issue where the build cannot resolve a module dependency for the current module project. Also add partial support for security auto- configuration in test clusters.
1 parent ef1c742 commit d6fc39c

File tree

8 files changed

+99
-43
lines changed

8 files changed

+99
-43
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java‎

Lines changed: 59 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin;
1919
import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
2020
import org.elasticsearch.gradle.internal.info.BuildParams;
21+
import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin;
2122
import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
2223
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
2324
import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
@@ -41,7 +42,8 @@
4142
import org.gradle.api.tasks.util.PatternFilterable;
4243

4344
import java.util.Collection;
44-
import java.util.HashSet;
45+
import java.util.Iterator;
46+
import java.util.LinkedHashSet;
4547
import java.util.List;
4648
import java.util.Map;
4749
import java.util.Optional;
@@ -89,19 +91,18 @@ public void apply(Project project) {
8991
});
9092

9193
// Create configures for module and plugin dependencies
92-
Configuration modulesConfiguration = createPluginConfiguration(project, MODULES_CONFIGURATION, true);
93-
Configuration pluginsConfiguration = createPluginConfiguration(project, PLUGINS_CONFIGURATION, false);
94-
Configuration extractedPluginsConfiguration = createPluginConfiguration(project, EXTRACTED_PLUGINS_CONFIGURATION, true);
94+
Configuration modulesConfiguration = createPluginConfiguration(project, MODULES_CONFIGURATION, true, false);
95+
Configuration pluginsConfiguration = createPluginConfiguration(project, PLUGINS_CONFIGURATION, false, false);
96+
Configuration extractedPluginsConfiguration = createPluginConfiguration(project, EXTRACTED_PLUGINS_CONFIGURATION, true, true);
9597
extractedPluginsConfiguration.extendsFrom(pluginsConfiguration);
9698
configureArtifactTransforms(project);
9799

98100
// For plugin and module projects, register the current project plugin bundle as a dependency
99101
project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
100102
if (GradleUtils.isModuleProject(project.getPath())) {
101-
project.getDependencies()
102-
.add(modulesConfiguration.getName(), project.getDependencies().project(Map.of("path", project.getPath())));
103+
project.getDependencies().add(MODULES_CONFIGURATION, getExplodedBundleDependency(project, project.getPath()));
103104
} else {
104-
project.getDependencies().add(pluginsConfiguration.getName(), project.files(project.getTasks().named("bundlePlugin")));
105+
project.getDependencies().add(PLUGINS_CONFIGURATION, getBundleZipTaskDependency(project, project.getPath()));
105106
}
106107

107108
});
@@ -158,15 +159,15 @@ private FileTree getDistributionFiles(ElasticsearchDistribution distribution, Ac
158159
}
159160

160161
private void registerConfigurationInputs(Task task, Configuration configuration) {
161-
task.getInputs()
162-
.files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"))))
163-
.withPropertyName(configuration.getName() + "-classpath")
164-
.withNormalizer(ClasspathNormalizer.class);
165-
166162
task.getInputs()
167163
.files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar") == false)))
168164
.withPropertyName(configuration.getName() + "-files")
169165
.withPathSensitivity(PathSensitivity.RELATIVE);
166+
167+
task.getInputs()
168+
.files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"))))
169+
.withPropertyName(configuration.getName() + "-classpath")
170+
.withNormalizer(ClasspathNormalizer.class);
170171
}
171172

172173
private void registerDistributionInputs(Task task, ElasticsearchDistribution distribution) {
@@ -192,36 +193,64 @@ private Optional<String> findModulePath(Project project, String pluginName) {
192193
.map(Project::getPath);
193194
}
194195

195-
private Configuration createPluginConfiguration(Project project, String name, boolean useExploded) {
196+
private Configuration createPluginConfiguration(Project project, String name, boolean useExploded, boolean isExtended) {
196197
return project.getConfigurations().create(name, c -> {
197198
if (useExploded) {
198199
c.attributes(a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE));
199200
} else {
200201
c.attributes(a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE));
201202
}
202-
c.withDependencies(dependencies -> {
203-
// Add dependencies of any modules
204-
Collection<Dependency> additionalDependencies = new HashSet<>();
205-
for (Dependency dependency : dependencies) {
206-
if (dependency instanceof ProjectDependency projectDependency) {
207-
List<String> extendedPlugins = projectDependency.getDependencyProject()
208-
.getExtensions()
209-
.getByType(PluginPropertiesExtension.class)
210-
.getExtendedPlugins();
211-
212-
for (String extendedPlugin : extendedPlugins) {
213-
findModulePath(project, extendedPlugin).ifPresent(
214-
modulePath -> additionalDependencies.add(project.getDependencies().project(Map.of("path", modulePath)))
215-
);
203+
if (isExtended == false) {
204+
c.withDependencies(dependencies -> {
205+
// Add dependencies of any modules
206+
Collection<Dependency> additionalDependencies = new LinkedHashSet<>();
207+
for (Iterator<Dependency> iterator = dependencies.iterator(); iterator.hasNext();) {
208+
Dependency dependency = iterator.next();
209+
if (dependency instanceof ProjectDependency projectDependency) {
210+
Project dependencyProject = projectDependency.getDependencyProject();
211+
List<String> extendedPlugins = dependencyProject.getExtensions()
212+
.getByType(PluginPropertiesExtension.class)
213+
.getExtendedPlugins();
214+
215+
// Replace project dependency with explicit dependency on exploded configuration to workaround variant bug
216+
if (projectDependency.getTargetConfiguration() == null) {
217+
iterator.remove();
218+
additionalDependencies.add(
219+
useExploded
220+
? getExplodedBundleDependency(project, dependencyProject.getPath())
221+
: getBundleZipTaskDependency(project, dependencyProject.getPath())
222+
);
223+
}
224+
225+
for (String extendedPlugin : extendedPlugins) {
226+
findModulePath(project, extendedPlugin).ifPresent(
227+
modulePath -> additionalDependencies.add(
228+
useExploded
229+
? getExplodedBundleDependency(project, modulePath)
230+
: getBundleZipTaskDependency(project, modulePath)
231+
)
232+
);
233+
}
216234
}
217235
}
218-
}
219236

220-
dependencies.addAll(additionalDependencies);
221-
});
237+
dependencies.addAll(additionalDependencies);
238+
});
239+
}
222240
});
223241
}
224242

243+
private Dependency getExplodedBundleDependency(Project project, String projectPath) {
244+
return project.getDependencies()
245+
.project(Map.of("path", projectPath, "configuration", BasePluginBuildPlugin.EXPLODED_BUNDLE_CONFIG));
246+
}
247+
248+
private Dependency getBundleZipTaskDependency(Project project, String projectPath) {
249+
Project dependencyProject = project.findProject(projectPath);
250+
return project.getDependencies()
251+
.create(project.files(dependencyProject.getTasks().named(BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME)));
252+
}
253+
225254
private void configureArtifactTransforms(Project project) {
226255
project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
227256
transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE);

‎modules/aggregations/build.gradle‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import org.elasticsearch.gradle.Version
21
import org.elasticsearch.gradle.internal.info.BuildParams
32

43
/*
@@ -8,8 +7,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams
87
* in compliance with, at your election, the Elastic License 2.0 or the Server
98
* Side Public License, v 1.
109
*/
11-
apply plugin: 'elasticsearch.legacy-yaml-rest-test'
12-
apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
10+
apply plugin: 'elasticsearch.internal-yaml-rest-test'
11+
apply plugin: 'elasticsearch.yaml-rest-compat-test'
1312
apply plugin: 'elasticsearch.internal-cluster-test'
1413

1514
esplugin {
@@ -52,11 +51,7 @@ artifacts {
5251
restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
5352
}
5453

55-
testClusters.configureEach {
56-
module ':modules:lang-painless'
57-
requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
58-
}
59-
6054
dependencies {
6155
compileOnly(project(':modules:lang-painless:spi'))
56+
clusterModules(project(':modules:lang-painless'))
6257
}

‎modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,20 @@
1010
import com.carrotsearch.randomizedtesting.annotations.Name;
1111
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
1212

13+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
14+
import org.elasticsearch.test.cluster.FeatureFlag;
1315
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
1416
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
17+
import org.junit.ClassRule;
1518

1619
public class AggregationsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
20+
@ClassRule
21+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
22+
.module("aggregations")
23+
.module("lang-painless")
24+
.feature(FeatureFlag.TIME_SERIES_MODE)
25+
.build();
26+
1727
public AggregationsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
1828
super(testCandidate);
1929
}
@@ -22,4 +32,9 @@ public AggregationsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
2232
public static Iterable<Object[]> parameters() throws Exception {
2333
return ESClientYamlSuiteTestCase.createParameters();
2434
}
35+
36+
@Override
37+
protected String getTestRestCluster() {
38+
return cluster.getHttpAddresses();
39+
}
2540
}

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSettingsProvider.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public Map<String, String> get(LocalNodeSpec nodeSpec) {
2626
settings.put("node.portsfile", "true");
2727
settings.put("http.port", "0");
2828
settings.put("transport.port", "0");
29+
settings.put("network.host", "_local_");
2930

3031
if (nodeSpec.getDistributionType() == DistributionType.INTEG_TEST) {
3132
settings.put("xpack.security.enabled", "false");

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ private void installPlugins() {
341341
+ project
342342
+ "':\n\n"
343343
+ "dependencies {\n"
344-
+ " clusterModules "
344+
+ " clusterPlugins "
345345
+ "project(':plugins:"
346346
+ pluginName
347347
+ "')"

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.concurrent.atomic.AtomicBoolean;
3232
import java.util.concurrent.atomic.AtomicLong;
3333
import java.util.stream.Collectors;
34+
import java.util.stream.Stream;
3435

3536
public class LocalClusterHandle implements ClusterHandle {
3637
private static final Logger LOGGER = LogManager.getLogger(LocalClusterHandle.class);
@@ -124,11 +125,17 @@ private void waitUntilReady() {
124125
try {
125126
Retry.retryUntilTrue(CLUSTER_UP_TIMEOUT, Duration.ZERO, () -> {
126127
Node node = nodes.get(0);
127-
String scheme = node.getSpec().isSettingTrue("xpack.security.http.ssl.enabled") ? "https" : "http";
128+
boolean securityEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.enabled", "true"));
129+
boolean sslEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.http.ssl.enabled", "false"));
130+
boolean securityAutoConfigured = isSecurityAutoConfigured(node);
131+
String scheme = securityEnabled && (sslEnabled || securityAutoConfigured) ? "https" : "http";
128132
WaitForHttpResource wait = new WaitForHttpResource(scheme, node.getHttpAddress(), nodes.size());
129133
User credentials = node.getSpec().getUsers().get(0);
130134
wait.setUsername(credentials.getUsername());
131135
wait.setPassword(credentials.getPassword());
136+
if (securityAutoConfigured) {
137+
wait.setCertificateAuthorities(node.getWorkingDir().resolve("config/certs/http_ca.crt").toFile());
138+
}
132139
return wait.wait(500);
133140
});
134141
} catch (TimeoutException e) {
@@ -138,6 +145,15 @@ private void waitUntilReady() {
138145
}
139146
}
140147

148+
private boolean isSecurityAutoConfigured(Node node) {
149+
Path configFile = node.getWorkingDir().resolve("config").resolve("elasticsearch.yml");
150+
try (Stream<String> lines = Files.lines(configFile)) {
151+
return lines.anyMatch(l -> l.contains("BEGIN SECURITY AUTO CONFIGURATION"));
152+
} catch (IOException e) {
153+
throw new UncheckedIOException(e);
154+
}
155+
}
156+
141157
private void writeUnicastHostsFile() {
142158
String transportUris = execute(() -> nodes.parallelStream().map(Node::getTransportEndpoint).collect(Collectors.joining("\n")));
143159
nodes.forEach(node -> {

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,8 +147,8 @@ public boolean isSecurityEnabled() {
147147
);
148148
}
149149

150-
public boolean isSettingTrue(String setting) {
151-
return Boolean.parseBoolean(resolveSettings().getOrDefault(setting, "false"));
150+
public String getSetting(String setting, String defaultValue) {
151+
return resolveSettings().getOrDefault(setting, defaultValue);
152152
}
153153

154154
/**

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public DistributionDescriptor resolve(Version version, DistributionType type) {
3737
+ "':\n\n"
3838
+ "tasks.named('"
3939
+ taskName
40-
+ "').configure {\n"
40+
+ "') {\n"
4141
+ " usesDefaultDistribution()\n"
4242
+ "}"
4343
);

0 commit comments

Comments
 (0)