Skip to content

Commit 240351a

Browse files
smalyshevsmalyshev
authored
Merge branch 'main' into remote-lookup-join
2 parents a9a19f8 + a6c4c08 commit 240351a

File tree

55 files changed

+3707
-2654
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+3707
-2654
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java‎

Lines changed: 104 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,11 @@
3434
import java.io.File;
3535
import java.util.List;
3636
import java.util.Map;
37+
import java.util.stream.Stream;
3738

3839
import javax.inject.Inject;
3940

41+
import static java.util.stream.Collectors.joining;
4042
import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
4143
import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
4244
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
@@ -173,6 +175,16 @@ public void execute(Task t) {
173175
// we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
174176
nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
175177

178+
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
179+
SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
180+
SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
181+
if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
182+
FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
183+
FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
184+
FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
185+
test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
186+
}
187+
176188
test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("tests.").get());
177189
test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("es.").get());
178190

@@ -205,46 +217,122 @@ public void execute(Task t) {
205217
}
206218

207219
/*
208-
* If this project builds a shadow JAR than any unit tests should test against that artifact instead of
220+
* If this project builds a shadow JAR then any unit tests should test against that artifact instead of
209221
* compiled class output and dependency jars. This better emulates the runtime environment of consumers.
210222
*/
211223
project.getPluginManager().withPlugin("com.gradleup.shadow", p -> {
212224
if (test.getName().equals(JavaPlugin.TEST_TASK_NAME)) {
213225
// Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these
214-
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
215-
FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
216226
// Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR
217227
Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.CONFIGURATION_NAME);
218228
// Add the shadow JAR artifact itself
219229
FileCollection shadowJar = project.files(project.getTasks().named("shadowJar"));
220-
FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
230+
FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
231+
FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
221232
test.setClasspath(testRuntime.minus(mainRuntime).plus(shadowConfig).plus(shadowJar));
222233
}
223234
});
224235
});
225-
configureImmutableCollectionsPatch(project);
236+
configureJavaBaseModuleOptions(project);
237+
configureEntitlements(project);
238+
}
239+
240+
/**
241+
* Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
242+
*/
243+
private void configureJavaBaseModuleOptions(Project project) {
244+
project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
245+
FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
246+
if (patchedImmutableCollections != null) {
247+
test.getInputs().files(patchedImmutableCollections);
248+
test.systemProperty("tests.hackImmutableCollections", "true");
249+
}
250+
251+
FileCollection entitlementBridge = entitlementBridge(project);
252+
if (entitlementBridge != null) {
253+
test.getInputs().files(entitlementBridge);
254+
}
255+
256+
test.getJvmArgumentProviders().add(() -> {
257+
String javaBasePatch = Stream.concat(
258+
singleFilePath(patchedImmutableCollections).map(str -> str + "/java.base"),
259+
singleFilePath(entitlementBridge)
260+
).collect(joining(File.pathSeparator));
261+
262+
return javaBasePatch.isEmpty()
263+
? List.of()
264+
: List.of("--patch-module=java.base=" + javaBasePatch, "--add-opens=java.base/java.util=ALL-UNNAMED");
265+
});
266+
});
226267
}
227268

228-
private void configureImmutableCollectionsPatch(Project project) {
269+
private Stream<String> singleFilePath(FileCollection collection) {
270+
return Stream.ofNullable(collection).filter(fc -> fc.isEmpty() == false).map(FileCollection::getSingleFile).map(File::toString);
271+
}
272+
273+
private static FileCollection patchedImmutableCollections(Project project) {
229274
String patchProject = ":test:immutable-collections-patch";
230275
if (project.findProject(patchProject) == null) {
231-
return; // build tests may not have this project, just skip
276+
return null; // build tests may not have this project, just skip
232277
}
233278
String configurationName = "immutableCollectionsPatch";
234279
FileCollection patchedFileCollection = project.getConfigurations()
235280
.create(configurationName, config -> config.setCanBeConsumed(false));
236281
var deps = project.getDependencies();
237282
deps.add(configurationName, deps.project(Map.of("path", patchProject, "configuration", "patch")));
238-
project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
239-
test.getInputs().files(patchedFileCollection);
240-
test.systemProperty("tests.hackImmutableCollections", "true");
241-
test.getJvmArgumentProviders()
242-
.add(
243-
() -> List.of(
244-
"--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base",
245-
"--add-opens=java.base/java.util=ALL-UNNAMED"
246-
)
283+
return patchedFileCollection;
284+
}
285+
286+
private static FileCollection entitlementBridge(Project project) {
287+
return project.getConfigurations().findByName("entitlementBridge");
288+
}
289+
290+
/**
291+
* Sets the required JVM options and system properties to enable entitlement enforcement on tests.
292+
* <p>
293+
* One command line option is set in {@link #configureJavaBaseModuleOptions} out of necessity,
294+
* since the command line can have only one {@code --patch-module} option for a given module.
295+
*/
296+
private static void configureEntitlements(Project project) {
297+
Configuration agentConfig = project.getConfigurations().create("entitlementAgent");
298+
Project agent = project.findProject(":libs:entitlement:agent");
299+
if (agent != null) {
300+
agentConfig.defaultDependencies(
301+
deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:agent"))); }
302+
);
303+
}
304+
FileCollection agentFiles = agentConfig;
305+
306+
Configuration bridgeConfig = project.getConfigurations().create("entitlementBridge");
307+
Project bridge = project.findProject(":libs:entitlement:bridge");
308+
if (bridge != null) {
309+
bridgeConfig.defaultDependencies(
310+
deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:bridge"))); }
311+
);
312+
}
313+
FileCollection bridgeFiles = bridgeConfig;
314+
315+
project.getTasks().withType(Test.class).configureEach(test -> {
316+
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
317+
318+
// Agent
319+
if (agentFiles.isEmpty() == false) {
320+
test.getInputs().files(agentFiles);
321+
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
322+
test.systemProperty("jdk.attach.allowAttachSelf", true);
323+
}
324+
325+
// Bridge
326+
if (bridgeFiles.isEmpty() == false) {
327+
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
328+
test.getInputs().files(bridgeFiles);
329+
// Tests may not be modular, but the JDK still is
330+
test.jvmArgs(
331+
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
332+
+ modulesContainingEntitlementInstrumentation
247333
);
334+
}
248335
});
249336
}
337+
250338
}

‎build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) {
188188
}
189189

190190
private String getResource(String name) throws Exception {
191-
return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8);
191+
return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8).replace("\r", "");
192192
}
193193

194194
private void writeResource(String name, String contents) throws Exception {

‎build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@
1818
import org.gradle.api.provider.ProviderFactory;
1919
import org.gradle.api.tasks.SourceSet;
2020
import org.gradle.api.tasks.SourceSetContainer;
21+
import org.gradle.api.tasks.testing.Test;
2122
import org.gradle.language.jvm.tasks.ProcessResources;
2223

24+
import java.util.List;
25+
2326
import javax.inject.Inject;
2427

2528
/**
@@ -53,5 +56,11 @@ public void apply(Project project) {
5356
project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> {
5457
task.into("META-INF", copy -> copy.from(testBuildInfoTask));
5558
});
59+
60+
if (project.getRootProject().getName().equals("elasticsearch")) {
61+
project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
62+
test.systemProperty("es.entitlement.enableForTests", "true");
63+
});
64+
}
5665
}
5766
}

‎docs/changelog/127797.yaml‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎libs/build.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,14 @@ configure(childProjects.values()) {
4545
*/
4646
apply plugin: 'elasticsearch.build'
4747
}
48+
49+
// This is for any code potentially included in the server at runtime.
50+
// Omit oddball libraries that aren't in server.
51+
def nonServerLibs = ['plugin-scanner']
52+
if (false == nonServerLibs.contains(project.name)) {
53+
project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
54+
test.systemProperty('es.entitlement.enableForTests', 'true')
55+
})
56+
}
57+
4858
}

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyCheckerImpl.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ private void neverEntitled(Class<?> callerClass, Supplier<String> operationDescr
135135
Strings.format(
136136
"component [%s], module [%s], class [%s], operation [%s]",
137137
entitlements.componentName(),
138-
PolicyCheckerImpl.getModuleName(requestingClass),
138+
entitlements.moduleName(),
139139
requestingClass,
140140
operationDescription.get()
141141
),
@@ -247,7 +247,7 @@ public void checkFileRead(Class<?> callerClass, Path path, boolean followLinks)
247247
Strings.format(
248248
"component [%s], module [%s], class [%s], entitlement [file], operation [read], path [%s]",
249249
entitlements.componentName(),
250-
PolicyCheckerImpl.getModuleName(requestingClass),
250+
entitlements.moduleName(),
251251
requestingClass,
252252
realPath == null ? path : Strings.format("%s -> %s", path, realPath)
253253
),
@@ -279,7 +279,7 @@ public void checkFileWrite(Class<?> callerClass, Path path) {
279279
Strings.format(
280280
"component [%s], module [%s], class [%s], entitlement [file], operation [write], path [%s]",
281281
entitlements.componentName(),
282-
PolicyCheckerImpl.getModuleName(requestingClass),
282+
entitlements.moduleName(),
283283
requestingClass,
284284
path
285285
),
@@ -383,7 +383,7 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
383383
() -> Strings.format(
384384
"Entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
385385
entitlements.componentName(),
386-
PolicyCheckerImpl.getModuleName(requestingClass),
386+
entitlements.moduleName(),
387387
requestingClass,
388388
property
389389
)
@@ -394,7 +394,7 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
394394
Strings.format(
395395
"component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
396396
entitlements.componentName(),
397-
PolicyCheckerImpl.getModuleName(requestingClass),
397+
entitlements.moduleName(),
398398
requestingClass,
399399
property
400400
),
@@ -447,7 +447,7 @@ private void checkFlagEntitlement(
447447
Strings.format(
448448
"component [%s], module [%s], class [%s], entitlement [%s]",
449449
classEntitlements.componentName(),
450-
PolicyCheckerImpl.getModuleName(requestingClass),
450+
classEntitlements.moduleName(),
451451
requestingClass,
452452
PolicyParser.buildEntitlementNameFromClass(entitlementClass)
453453
),
@@ -460,7 +460,7 @@ private void checkFlagEntitlement(
460460
() -> Strings.format(
461461
"Entitled: component [%s], module [%s], class [%s], entitlement [%s]",
462462
classEntitlements.componentName(),
463-
PolicyCheckerImpl.getModuleName(requestingClass),
463+
classEntitlements.moduleName(),
464464
requestingClass,
465465
PolicyParser.buildEntitlementNameFromClass(entitlementClass)
466466
)

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,9 @@ public enum ComponentKind {
118118
*
119119
* @param componentName the plugin name or else one of the special component names like "(server)".
120120
*/
121-
record ModuleEntitlements(
121+
protected record ModuleEntitlements(
122122
String componentName,
123+
String moduleName,
123124
Map<Class<? extends Entitlement>, List<Entitlement>> entitlementsByType,
124125
FileAccessTree fileAccess,
125126
Logger logger
@@ -148,7 +149,13 @@ private FileAccessTree getDefaultFileAccess(Collection<Path> componentPaths) {
148149

149150
// pkg private for testing
150151
ModuleEntitlements defaultEntitlements(String componentName, Collection<Path> componentPaths, String moduleName) {
151-
return new ModuleEntitlements(componentName, Map.of(), getDefaultFileAccess(componentPaths), getLogger(componentName, moduleName));
152+
return new ModuleEntitlements(
153+
componentName,
154+
moduleName,
155+
Map.of(),
156+
getDefaultFileAccess(componentPaths),
157+
getLogger(componentName, moduleName)
158+
);
152159
}
153160

154161
// pkg private for testing
@@ -166,6 +173,7 @@ ModuleEntitlements policyEntitlements(
166173
}
167174
return new ModuleEntitlements(
168175
componentName,
176+
moduleName,
169177
entitlements.stream().collect(groupingBy(Entitlement::getClass)),
170178
FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, componentPaths, exclusivePaths),
171179
getLogger(componentName, moduleName)
@@ -293,11 +301,11 @@ private static Logger getLogger(String componentName, String moduleName) {
293301
*/
294302
private static final ConcurrentHashMap<String, Logger> MODULE_LOGGERS = new ConcurrentHashMap<>();
295303

296-
ModuleEntitlements getEntitlements(Class<?> requestingClass) {
304+
protected ModuleEntitlements getEntitlements(Class<?> requestingClass) {
297305
return moduleEntitlementsMap.computeIfAbsent(requestingClass.getModule(), m -> computeEntitlements(requestingClass));
298306
}
299307

300-
private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
308+
protected final ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
301309
var policyScope = scopeResolver.apply(requestingClass);
302310
var componentName = policyScope.componentName();
303311
var moduleName = policyScope.moduleName();
@@ -336,8 +344,7 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
336344
}
337345
}
338346

339-
// pkg private for testing
340-
static Collection<Path> getComponentPathsFromClass(Class<?> requestingClass) {
347+
protected Collection<Path> getComponentPathsFromClass(Class<?> requestingClass) {
341348
var codeSource = requestingClass.getProtectionDomain().getCodeSource();
342349
if (codeSource == null) {
343350
return List.of();

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,6 @@ public void testGetEntitlements() {
8989
AtomicReference<PolicyScope> policyScope = new AtomicReference<>();
9090

9191
// A common policy with a variety of entitlements to test
92-
Collection<Path> thisSourcePaths = PolicyManager.getComponentPathsFromClass(getClass());
9392
var plugin1SourcePaths = List.of(Path.of("modules", "plugin1"));
9493
var policyManager = new PolicyManager(
9594
new Policy("server", List.of(new Scope("org.example.httpclient", List.of(new OutboundNetworkEntitlement())))),
@@ -99,6 +98,7 @@ public void testGetEntitlements() {
9998
Map.of("plugin1", plugin1SourcePaths),
10099
TEST_PATH_LOOKUP
101100
);
101+
Collection<Path> thisSourcePaths = policyManager.getComponentPathsFromClass(getClass());
102102

103103
// "Unspecified" below means that the module is not named in the policy
104104

‎modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.ingest.Processor;
1616
import org.elasticsearch.ingest.RandomDocumentPicks;
1717
import org.elasticsearch.test.ESTestCase;
18+
import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
1819
import org.junit.Before;
1920

2021
import java.io.InputStream;
@@ -38,6 +39,7 @@
3839
import static org.hamcrest.Matchers.notNullValue;
3940
import static org.hamcrest.Matchers.nullValue;
4041

42+
@WithoutEntitlements // ES-12084
4143
public class AttachmentProcessorTests extends ESTestCase {
4244

4345
private Processor processor;

‎modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.apache.tika.metadata.Metadata;
1515
import org.elasticsearch.core.PathUtils;
1616
import org.elasticsearch.test.ESTestCase;
17+
import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
1718

1819
import java.nio.file.DirectoryStream;
1920
import java.nio.file.Files;
@@ -25,6 +26,7 @@
2526
* comes back and no exception.
2627
*/
2728
@SuppressFileSystems("ExtrasFS") // don't try to parse extraN
29+
@WithoutEntitlements // ES-12084
2830
public class TikaDocTests extends ESTestCase {
2931

3032
/** some test files from tika test suite, zipped up */

0 commit comments

Comments
 (0)