diff --git a/component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesServiceConfiguration.java b/component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesConfiguration.java similarity index 85% rename from component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesServiceConfiguration.java rename to component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesConfiguration.java index d367dce515f68..f7f01ec74d5b3 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesServiceConfiguration.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesConfiguration.java @@ -26,9 +26,9 @@ @Target(TYPE) @Retention(RUNTIME) -@ConfigurationType("dynamicDependenciesServiceConfiguration") -@Documentation("Mark a model (complex object) as being the configuration used in services annotated with @DynamicDependencies.") -public @interface DynamicDependenciesServiceConfiguration { +@ConfigurationType("dynamicDependenciesConfiguration") +@Documentation("Mark a model (complex object) as being the configuration expected to compute dynamic dependencies.") +public @interface DynamicDependenciesConfiguration { String value() default "default"; } \ No newline at end of file diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java index 56957e149196f..1396c41a75441 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java @@ -113,6 +113,7 @@ import org.apache.xbean.finder.archive.FileArchive; import org.apache.xbean.finder.archive.FilteredArchive; import org.apache.xbean.finder.archive.JarArchive; +import org.apache.xbean.finder.filter.ContainsFilter; import org.apache.xbean.finder.filter.ExcludeIncludeFilter; import org.apache.xbean.finder.filter.Filter; import org.apache.xbean.finder.filter.FilterList; @@ -312,6 +313,8 @@ public String[] categories() { // + tcomp "runtime" indeed (invisible from the components but required for the runtime private final Filter classesFilter; + private final Filter resourcesFilter; + private final ParameterModelService parameterModelService; private final InternationalizationServiceFactory internationalizationServiceFactory; @@ -427,6 +430,13 @@ public ComponentManager(final Path m2, final String dependenciesResource, final .map(PrefixFilter::new) .toArray(Filter[]::new)); + resourcesFilter = new FilterList(Stream.concat( + Stream.of("META-INF/services/"), + additionalParentResources()) + .distinct() + .map(ContainsFilter::new) + .toArray(Filter[]::new)); + jsonpProvider = loadJsonProvider(); jsonbProvider = loadJsonbProvider(); // these factories have memory caches so ensure we reuse them properly @@ -460,6 +470,7 @@ public ComponentManager(final Path m2, final String dependenciesResource, final migrationHandlerFactory = new MigrationHandlerFactory(reflections); final Predicate isContainerClass = name -> isContainerClass(classesFilter, name); + final Predicate isParentResource = name -> isContainerResource(resourcesFilter, name); final ContainerManager.ClassLoaderConfiguration defaultClassLoaderConfiguration = ContainerManager.ClassLoaderConfiguration .builder() @@ -467,6 +478,7 @@ public ComponentManager(final Path m2, final String dependenciesResource, final .parentClassesFilter(isContainerClass) .classesFilter(isContainerClass.negate()) .supportsResourceDependencies(true) + .parentResourcesFilter(isParentResource) .create(); this.container = new ContainerManager(ContainerManager.DependenciesResolutionConfiguration .builder() @@ -611,6 +623,16 @@ private Stream additionalContainerClasses() { .orElseGet(Stream::empty)); } + private Stream additionalParentResources() { + return Stream + .concat(customizers.stream().flatMap(Customizer::parentResources), + ofNullable( + System.getProperty("talend.component.manager.classloader.container.parentResources")) + .map(s -> s.split(",")) + .map(Stream::of) + .orElseGet(Stream::empty)); + } + public static Path findM2() { return new MavenRepositoryDefaultResolver().discover(); } @@ -899,16 +921,30 @@ private void autoDiscoverPlugins0(final boolean callers, final boolean classpath // common for studio until job generation is updated to build a tcomp friendly bundle if (classpath && !Boolean.getBoolean("component.manager.classpath.skip")) { try { + final String markerValue = "TALEND-INF/dependencies.txt"; final Enumeration componentMarkers = - Thread.currentThread().getContextClassLoader().getResources("TALEND-INF/dependencies.txt"); + Thread.currentThread().getContextClassLoader().getResources(markerValue); while (componentMarkers.hasMoreElements()) { - File file = Files.toFile(componentMarkers.nextElement()); - if (file.getName().equals("dependencies.txt") && file.getParentFile() != null - && file.getParentFile().getName().equals("TALEND-INF")) { - file = file.getParentFile().getParentFile(); - } - if (!hasPlugin(container.buildAutoIdFromName(file.getName()))) { - addPlugin(file.getAbsolutePath()); + final URL marker = componentMarkers.nextElement(); + File file = Files.toFile(marker); + if (file != null) { + if (file.getName().equals("dependencies.txt") && file.getParentFile() != null + && file.getParentFile().getName().equals("TALEND-INF")) { + file = file.getParentFile().getParentFile(); + } + if (!hasPlugin(container.buildAutoIdFromName(file.getName()))) { + addPlugin(file.getAbsolutePath()); + } + } else { + // lookup nested jar + if (marker != null && "jar".equals(marker.getProtocol())) { + final String urlFile = marker.getFile(); + final String jarPath = urlFile.substring(0, urlFile.lastIndexOf("!")); + final String jarFilePath = jarPath.substring(jarPath.lastIndexOf("/") + 1); + if (!hasPlugin(container.buildAutoIdFromName(jarFilePath))) { + addPlugin(jarPath); + } + } } } } catch (final IOException e) { @@ -1045,6 +1081,10 @@ protected boolean isContainerClass(final Filter filter, final String name) { return name != null && filter.accept(name); } + protected boolean isContainerResource(final Filter filter, final String name) { + return name != null && filter.accept(name); + } + @Override public void close() { container.close(); @@ -1299,11 +1339,14 @@ public void onCreate(final Container container) { final AnnotationFinder finder; Archive archive = null; + final String rootModule = container.getRootModule(); + final boolean nested = rootModule != null && rootModule.startsWith("nested:"); try { String alreadyScannedClasses = null; Filter filter = KnownClassesFilter.INSTANCE; - try (final InputStream containerFilterConfig = - container.getLoader().getResourceAsStream("TALEND-INF/scanning.properties")) { + try (final InputStream containerFilterConfig = nested + ? loader.getNestedResource(rootModule + "!/TALEND-INF/scanning.properties") + : loader.getResourceAsStream("TALEND-INF/scanning.properties")) { if (containerFilterConfig != null) { final Properties config = new Properties(); config.load(containerFilterConfig); @@ -1778,8 +1821,10 @@ private Archive toArchive(final String module, final String moduleId, final Conf } } info(module + " (" + moduleId + ") is not a file, will try to look it up from a nested maven repository"); - final URL nestedJar = - loader.getParent().getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY + module); + URL nestedJar = loader.getParent().getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY + module); + if (nestedJar == null) { + nestedJar = loader.getParent().getResource(module); + } if (nestedJar != null) { InputStream nestedStream = null; final JarInputStream jarStream; @@ -2207,6 +2252,13 @@ public interface Customizer { */ Stream containerClassesAndPackages(); + /** + * @return + */ + default Stream parentResources() { + return Stream.empty(); + } + /** * @return advanced toggle to ignore built-in beam exclusions and let this customizer override them. */ diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/service/ResolverImpl.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/service/ResolverImpl.java index 1ba69e875a6c5..cf699aa45e5e3 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/service/ResolverImpl.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/service/ResolverImpl.java @@ -67,7 +67,7 @@ public ClassLoaderDescriptor mapDescriptorToClassLoader(final InputStream descri } catch (final MalformedURLException e) { throw new IllegalStateException(e); } - } else if (loader.getResource("MAVEN-INF/repository/" + path) != null) { + } else if (loader.getResource(path) != null) { nested.add(path); resolved.add(artifact.toCoordinate()); } // else will be missing diff --git a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java index 585e197c1ce3e..69d12bd624f95 100644 --- a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java +++ b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java @@ -60,7 +60,9 @@ import javax.management.ReflectionException; import org.apache.xbean.finder.util.Files; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -98,6 +100,16 @@ private ComponentManager newManager() { return newManager(new File("target/test-dependencies")); } + @BeforeAll + static void setup() { + System.setProperty("talend.component.manager.m2.fallback", "true"); + } + + @AfterAll + static void teardown() { + System.clearProperty("talend.component.manager.m2.fallback"); + } + @Test void doubleClose() { final ComponentManager instance = ComponentManager.instance(); @@ -417,7 +429,7 @@ void extendFamilyInNestedRepo(@TempDir final File temporaryFolder) throws Except final URLClassLoader parentLoader = new URLClassLoader(new URL[] { fatJar.toURI().toURL() }, thread.getContextClassLoader()); thread.setContextClassLoader(parentLoader); - try (final ComponentManager manager = newManager(new File("target/missing_" + UUID.randomUUID().toString()))) { + try (final ComponentManager manager = newManager(pluginFolder)) { try { manager.addPlugin(plugin2.getAbsolutePath()); @@ -429,7 +441,7 @@ void extendFamilyInNestedRepo(@TempDir final File temporaryFolder) throws Except .map(File::getName) .sorted() .toArray(String[]::new); - assertEquals(1, dependencies.length); // ignored transitive deps, enables the new root to control it + assertEquals(2, dependencies.length); // ignored transitive deps, enables the new root to control it assertEquals("main.jar", dependencies[0]); // transitive-1.0.0.jar is nested } finally { if (!transitive.delete()) { diff --git a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/ResolverImplTest.java b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/ResolverImplTest.java index 0f704c2a90daf..4f57d58d58b26 100644 --- a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/ResolverImplTest.java +++ b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/ResolverImplTest.java @@ -99,7 +99,7 @@ void resolvefromDescriptor() throws IOException { private void addDepToJar(final String dep, final JarOutputStream out) { final String[] segments = dep.split(":"); - final String path = "MAVEN-INF/repository/" + segments[0].replace(".", "/") + "/" + segments[1] + "/" + final String path = segments[0].replace(".", "/") + "/" + segments[1] + "/" + segments[3] + "/" + segments[1] + "-" + segments[3] + "." + segments[2]; // create folders for this m2 embedded deps @@ -114,7 +114,7 @@ private void addDepToJar(final String dep, final JarOutputStream out) { } } // add the dep - final File jar = new File("target/test-dependencies", path.substring("MAVEN-INF/repository/".length())); + final File jar = new File("target/test-dependencies", path); try { out.putNextEntry(new ZipEntry(path)); Files.copy(jar.toPath(), out); diff --git a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/xbean/NestedJarArchiveTest.java b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/xbean/NestedJarArchiveTest.java index bdc1b8cd5d305..7c60166964dfd 100644 --- a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/xbean/NestedJarArchiveTest.java +++ b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/xbean/NestedJarArchiveTest.java @@ -53,9 +53,9 @@ void xbeanNestedScanning(final TestInfo info, @TempDir final File temporaryFolde final File jar = createPlugin(temporaryFolder, info.getTestMethod().get().getName()); final ConfigurableClassLoader configurableClassLoader = new ConfigurableClassLoader("", new URL[0], new URLClassLoader(new URL[] { jar.toURI().toURL() }, Thread.currentThread().getContextClassLoader()), - n -> true, n -> true, new String[] { "com/foo/bar/1.0/bar-1.0.jar" }, new String[0]); + n -> true, n -> true, new String[] { "BOOT-INF/lib/com/foo/bar/1.0/bar-1.0.jar" }, new String[0]); try (final JarInputStream jis = new JarInputStream( - configurableClassLoader.getResourceAsStream("MAVEN-INF/repository/com/foo/bar/1.0/bar-1.0.jar"))) { + configurableClassLoader.getResourceAsStream("BOOT-INF/lib/com/foo/bar/1.0/bar-1.0.jar"))) { assertNotNull(jis, "test is wrongly setup, no nested jar, fix the createPlugin() method please"); final AnnotationFinder finder = new AnnotationFinder(new NestedJarArchive(null, jis, configurableClassLoader)); @@ -72,7 +72,7 @@ private File createPlugin(final File pluginFolder, final String name) throws IOE final File target = new File(pluginFolder, name); target.getParentFile().mkdirs(); try (final JarOutputStream outputStream = new JarOutputStream(new FileOutputStream(target))) { - outputStream.putNextEntry(new ZipEntry("MAVEN-INF/repository/com/foo/bar/1.0/bar-1.0.jar")); + outputStream.putNextEntry(new ZipEntry("BOOT-INF/lib/com/foo/bar/1.0/bar-1.0.jar")); try (final JarOutputStream nestedStream = new JarOutputStream(outputStream)) { final String packageName = "org/talend/test/generated/" + name.replace(".jar", ""); { // the factory (declaration) diff --git a/container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java b/container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java index 3ded8250b3ccd..0187468d3d3e0 100644 --- a/container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java +++ b/container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java @@ -28,6 +28,8 @@ import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.lang.instrument.ClassFileTransformer; @@ -38,7 +40,10 @@ import java.net.URLClassLoader; import java.net.URLConnection; import java.net.URLStreamHandler; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.security.CodeSource; import java.security.cert.Certificate; import java.util.ArrayList; @@ -55,6 +60,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Predicate; +import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.jar.JarInputStream; import java.util.jar.Manifest; @@ -92,6 +98,9 @@ public class ConfigurableClassLoader extends URLClassLoader { @Getter private final Predicate childFirstFilter; + @Getter + private final Predicate resourcesFilter; + private final Map> resources = new HashMap<>(); private final Collection transformers = new ArrayList<>(); @@ -114,7 +123,16 @@ public class ConfigurableClassLoader extends URLClassLoader { public ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent, final Predicate parentFilter, final Predicate childFirstFilter, final String[] nestedDependencies, final String[] jvmPrefixes) { - this(id, urls, parent, parentFilter, childFirstFilter, emptyMap(), jvmPrefixes); + this(id, urls, parent, parentFilter, childFirstFilter, emptyMap(), jvmPrefixes, (name) -> false); + if (nestedDependencies != null) { + loadNestedDependencies(parent, nestedDependencies); + } + } + + public ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent, + final Predicate parentFilter, final Predicate childFirstFilter, + final String[] nestedDependencies, final String[] jvmPrefixes, final Predicate resourcesFilter) { + this(id, urls, parent, parentFilter, childFirstFilter, emptyMap(), jvmPrefixes, resourcesFilter); if (nestedDependencies != null) { loadNestedDependencies(parent, nestedDependencies); } @@ -122,12 +140,14 @@ public ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoa private ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent, final Predicate parentFilter, final Predicate childFirstFilter, - final Map> resources, final String[] jvmPrefixes) { + final Map> resources, final String[] jvmPrefixes, + final Predicate resourcesFilter) { super(urls, parent); this.id = id; this.creationUrls = urls; this.parentFilter = parentFilter; this.childFirstFilter = childFirstFilter; + this.resourcesFilter = resourcesFilter; this.resources.putAll(resources); this.fullPathJvmPrefixes = @@ -150,7 +170,7 @@ private ConfigurableClassLoader(final String id, final URL[] urls, final ClassLo private void loadNestedDependencies(final ClassLoader parent, final String[] nestedDependencies) { final byte[] buffer = new byte[8192]; // should be good for most cases final ByteArrayOutputStream out = new ByteArrayOutputStream(buffer.length); - Stream.of(nestedDependencies).map(d -> NESTED_MAVEN_REPOSITORY + d).forEach(resource -> { + Stream.of(nestedDependencies).forEach(resource -> { final URL url = ofNullable(super.findResource(resource)).orElseGet(() -> parent.getResource(resource)); if (url == null) { throw new IllegalArgumentException("Didn't find " + resource + " in " + asList(nestedDependencies)); @@ -229,7 +249,7 @@ public void registerTransformer(final ClassFileTransformer transformer) { public synchronized URLClassLoader createTemporaryCopy() { final ConfigurableClassLoader self = this; return temporaryCopy == null ? temporaryCopy = new ConfigurableClassLoader(id, creationUrls, getParent(), - parentFilter, childFirstFilter, resources, fullPathJvmPrefixes) { + parentFilter, childFirstFilter, resources, fullPathJvmPrefixes, resourcesFilter) { @Override public synchronized void close() throws IOException { @@ -459,10 +479,17 @@ public Enumeration findResources(final String name) throws IOException { } private boolean isNestedDependencyResource(final String name) { - return name != null && name.startsWith(NESTED_MAVEN_REPOSITORY); + return name.startsWith(NESTED_MAVEN_REPOSITORY) || name.endsWith(".jar"); // TODO: improve coz not at all + // precise } private boolean isInJvm(final URL resource) { + // Services and parent allowed resources that should always be found by top level classloader. + // By default, META-INF/services/ is always allowed otherwise SPI won't work properly in nested environments. + // Warning: selection shouldn't be too generic! Use very specific paths only like jndi.properties. + if (resourcesFilter.test(resource.getFile())) { + return true; + } final Path path = toPath(resource); if (path == null) { return false; @@ -502,6 +529,63 @@ public List findContainedResources(final String name) { } } + /** + * Opens a stream to a resource located in a nested JAR. + * + * @param nestedUrl The full "nested:" URL, e.g. + * nested:/path/to/outer.jar/!path/inner.jar!/file.txt + * @return InputStream to the resource (must be closed by caller) + */ + public InputStream getNestedResource(final String nestedUrl) throws IOException { + if (nestedUrl == null || !nestedUrl.startsWith("nested:")) { + throw new IllegalArgumentException("Invalid nested URL: " + nestedUrl); + } + final String path = nestedUrl.substring("nested:".length()); + // Find the "/!" and "!/" separators + final int firstBang = path.indexOf("/!"); + final int secondBang = path.indexOf("!/", firstBang + 2); + if (firstBang < 0 || secondBang < 0) { + throw new IllegalArgumentException("Malformed nested URL: " + nestedUrl); + } + final Path outerJarPath = Paths.get(path.substring(0, firstBang)); // before /! + final String innerJarPath = path.substring(firstBang + 2, secondBang); // between /! and !/ + final String resourcePath = path.substring(secondBang + 2); // after the last !/ + // Open the outer JAR file + try (JarFile outerJar = new JarFile(outerJarPath.toFile())) { + JarEntry innerEntry = outerJar.getJarEntry(innerJarPath); + if (innerEntry == null) { + throw new FileNotFoundException("Inner JAR not found: " + innerJarPath); + } + // Copy the inner JAR to a temporary file + final Path tempInnerJar = Files.createTempFile("nested-inner-", ".jar"); + try (InputStream innerStream = outerJar.getInputStream(innerEntry)) { + Files.copy(innerStream, tempInnerJar, StandardCopyOption.REPLACE_EXISTING); + + JarFile innerJar = new JarFile(tempInnerJar.toFile()); + final JarEntry resourceEntry = innerJar.getJarEntry(resourcePath); + if (resourceEntry == null) { + throw new FileNotFoundException("Resource not found: " + resourcePath); + } + + // Return a stream that cleans up automatically when closed + return new FilterInputStream(innerJar.getInputStream(resourceEntry)) { + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + Files.deleteIfExists(tempInnerJar); + } + } + }; + } catch (IOException e) { + Files.deleteIfExists(tempInnerJar); + throw e; + } + } + } + private URL nestedResourceToURL(final String name, final Resource nestedResource) { try { return new URL("nested", null, -1, nestedResource.entry + "!/" + name, new Handler(nestedResource)); diff --git a/container/container-core/src/main/java/org/talend/sdk/component/container/Container.java b/container/container-core/src/main/java/org/talend/sdk/component/container/Container.java index 67e33c498479a..f2d5a5a88af98 100644 --- a/container/container-core/src/main/java/org/talend/sdk/component/container/Container.java +++ b/container/container-core/src/main/java/org/talend/sdk/component/container/Container.java @@ -101,7 +101,7 @@ public Container(final String id, final String rootModule, final Artifact[] depe this.dependencies = dependencies; this.localDependencyRelativeResolver = localDependencyRelativeResolver; this.lastModifiedTimestamp.set(new Date(0)); - this.hasNestedRepository = hasNestedRepository; + this.hasNestedRepository = rootModule.startsWith("nested:") || hasNestedRepository; ofNullable(initializer).ifPresent(i -> i.accept(this)); this.classloaderProvider = () -> { @@ -143,9 +143,12 @@ public Container(final String id, final String rootModule, final Artifact[] depe .distinct() .toArray(String[]::new) : null; - final ConfigurableClassLoader loader = new ConfigurableClassLoader(id, urls, - overrideClassLoaderConfig.getParent(), overrideClassLoaderConfig.getParentClassesFilter(), - overrideClassLoaderConfig.getClassesFilter(), rawNestedDependencies, jvmMarkers); + final Predicate parentFilter = + this.hasNestedRepository ? (name) -> true : overrideClassLoaderConfig.getParentClassesFilter(); + final ConfigurableClassLoader loader = + new ConfigurableClassLoader(id, urls, overrideClassLoaderConfig.getParent(), parentFilter, + overrideClassLoaderConfig.getClassesFilter(), rawNestedDependencies, jvmMarkers, + overrideClassLoaderConfig.getParentResourcesFilter()); transformers.forEach(loader::registerTransformer); activeSpecificTransformers(loader); return loader; @@ -171,7 +174,7 @@ private boolean findNestedDependency(final ContainerManager.ClassLoaderConfigura } final URL url = overrideClassLoaderConfig .getParent() - .getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY + depPath); + .getResource(depPath); return url != null; } @@ -307,6 +310,10 @@ public Date getCreated() { return created.get(); } + public boolean hasNestedRepository() { + return hasNestedRepository; + } + public void registerTransformer(final ClassFileTransformer transformer) { transformers.add(transformer); } diff --git a/container/container-core/src/main/java/org/talend/sdk/component/container/ContainerManager.java b/container/container-core/src/main/java/org/talend/sdk/component/container/ContainerManager.java index 393401aeadf87..593fe282101b5 100644 --- a/container/container-core/src/main/java/org/talend/sdk/component/container/ContainerManager.java +++ b/container/container-core/src/main/java/org/talend/sdk/component/container/ContainerManager.java @@ -29,6 +29,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -116,6 +119,7 @@ public ContainerManager(final DependenciesResolutionConfiguration dependenciesRe if (log.isDebugEnabled()) { log.debug("Using root repository: " + this.rootRepositoryLocation.toAbsolutePath()); + getSystemInformation(); } final String nestedPluginMappingResource = ofNullable(classLoaderConfiguration.getNestedPluginMappingResource()) @@ -124,11 +128,18 @@ public ContainerManager(final DependenciesResolutionConfiguration dependenciesRe ofNullable(classLoaderConfiguration.getParent()).orElseGet(ContainerManager.class::getClassLoader), ofNullable(classLoaderConfiguration.getClassesFilter()).orElseGet(() -> name -> true), ofNullable(classLoaderConfiguration.getParentClassesFilter()).orElseGet(() -> name -> true), + ofNullable(classLoaderConfiguration.getParentResourcesFilter()).orElseGet(() -> name -> true), classLoaderConfiguration.isSupportsResourceDependencies(), nestedPluginMappingResource); if (classLoaderConfiguration.isSupportsResourceDependencies()) { try (final InputStream mappingStream = classLoaderConfiguration.getParent().getResourceAsStream(nestedPluginMappingResource)) { if (mappingStream != null) { + if (log.isDebugEnabled()) { + final URL plug = classLoaderConfiguration.getParent().getResource(nestedPluginMappingResource); + if (plug != null) { + log.debug("[sysinfo] plugins mapping " + plug.toString()); + } + } final Properties properties = new Properties() { { @@ -155,10 +166,13 @@ public ContainerManager(final DependenciesResolutionConfiguration dependenciesRe this.jvmMarkers = Stream .concat(Stream.concat(Stream.of(getJre()), getComponentModules()), getCustomJvmMarkers()) .toArray(String[]::new); - this.hasNestedRepository = - this.classLoaderConfiguration.isSupportsResourceDependencies() && this.classLoaderConfiguration - .getParent() - .getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY) != null; + final URL nestedMvn = this.classLoaderConfiguration + .getParent() + .getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY); + this.hasNestedRepository = this.classLoaderConfiguration.isSupportsResourceDependencies() && nestedMvn != null; + if (log.isDebugEnabled() && hasNestedRepository) { + log.debug("[sysinfo] nested maven repository: " + nestedMvn); + } } public File getRootRepositoryLocation() { @@ -394,6 +408,22 @@ private String getJre() { .orElseThrow(IllegalArgumentException::new); } + private void getSystemInformation() { + try { + final RuntimeMXBean rt = ManagementFactory.getRuntimeMXBean(); + log.debug("[sysinfo] JVM arguments: " + rt.getInputArguments()); + try { + log.debug("[sysinfo] Boot classpath: " + rt.getBootClassPath()); + } catch (Exception e) { + // nop, will fail in some cases for boot classpath + } + log.debug("[sysinfo] Runtime classpath: " + rt.getClassPath()); + log.debug("[sysinfo] Runtime arguments: " + System.getProperty("sun.java.command")); + } catch (Exception e) { + log.debug("Unable to get JVM information: " + e.getMessage(), e); + } + } + @Override public void close() { lifecycle.closeIfNeeded(() -> { @@ -426,6 +456,8 @@ public static class ClassLoaderConfiguration { private final Predicate parentClassesFilter; + private final Predicate parentResourcesFilter; + // is nested jar in jar supported (1 level only) private final boolean supportsResourceDependencies; diff --git a/container/container-core/src/main/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolver.java b/container/container-core/src/main/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolver.java index c061f2d95caec..d115b7663437d 100644 --- a/container/container-core/src/main/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolver.java +++ b/container/container-core/src/main/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolver.java @@ -66,22 +66,21 @@ public Stream resolve(final ClassLoader rootLoader, final String artif .filter(Files::exists) .map(this::findDependenciesFile) .orElseGet(() -> { - final boolean isNested; - try (final InputStream stream = rootLoader - .getResourceAsStream(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY + artifact)) { + boolean isNested; + try (final InputStream stream = rootLoader.getResourceAsStream(artifact)) { isNested = stream != null; } catch (final IOException e) { log.debug(e.getMessage(), e); return ""; } - if (isNested) { // we reuse ConfigurableClassLoader just to not - // rewrite the logic but it is NOT a plugin! - try (final ConfigurableClassLoader configurableClassLoader = + if (isNested) { + try (final ConfigurableClassLoader ccl = new ConfigurableClassLoader("", new URL[0], rootLoader, name -> true, name -> true, new String[] { artifact }, new String[0])) { - try (final InputStream deps = - configurableClassLoader.getResourceAsStream(dependenciesListFile)) { + try (final InputStream deps = artifact.startsWith("nested:") + ? ccl.getNestedResource(artifact + "!/" + dependenciesListFile) + : ccl.getResourceAsStream(dependenciesListFile)) { return ofNullable(deps).map(s -> { try { return slurp(s); diff --git a/container/container-core/src/test/java/org/talend/sdk/component/classloader/ConfigurableClassLoaderTest.java b/container/container-core/src/test/java/org/talend/sdk/component/classloader/ConfigurableClassLoaderTest.java index 639b7b091d9e9..0401356ad2a10 100644 --- a/container/container-core/src/test/java/org/talend/sdk/component/classloader/ConfigurableClassLoaderTest.java +++ b/container/container-core/src/test/java/org/talend/sdk/component/classloader/ConfigurableClassLoaderTest.java @@ -193,8 +193,7 @@ void nestedJars(@TempDir final File temporaryFolder) throws IOException { final URL url = loader.getResource(resource); assertNotNull(url); assertEquals("nested", url.getProtocol()); - assertEquals( - "MAVEN-INF/repository/org/apache/tomee/ziplock/8.0.14/ziplock-8.0.14.jar!/org/apache/ziplock/JarLocation.class", + assertEquals("org/apache/tomee/ziplock/8.0.14/ziplock-8.0.14.jar!/org/apache/ziplock/JarLocation.class", url.getFile()); final byte[] bytes = slurp(url.openStream()); assertEquals(4666, bytes.length, mavenJarSizeMargin); @@ -377,7 +376,7 @@ private File createNestedJar(final File temporaryFolder, final String... deps) t Stream.of(deps).forEach(s -> { final String[] segments = s.split(":"); - final String path = "MAVEN-INF/repository/" + segments[0].replace(".", "/") + "/" + segments[1] + "/" + final String path = segments[0].replace(".", "/") + "/" + segments[1] + "/" + segments[3] + "/" + segments[1] + "-" + segments[3] + "." + segments[2]; { // create folders for this m2 embedded deps @@ -393,8 +392,7 @@ private File createNestedJar(final File temporaryFolder, final String... deps) t } } { // add the dep - final File jar = - new File(Constants.DEPENDENCIES_LOCATION, path.substring("MAVEN-INF/repository/".length())); + final File jar = new File(Constants.DEPENDENCIES_LOCATION, path); try { out.putNextEntry(new ZipEntry(path)); Files.copy(jar.toPath(), out); diff --git a/container/container-core/src/test/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolverTest.java b/container/container-core/src/test/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolverTest.java index aa02eedb0176d..b6950ab38d6e7 100644 --- a/container/container-core/src/test/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolverTest.java +++ b/container/container-core/src/test/java/org/talend/sdk/component/dependencies/maven/MvnDependencyListLocalRepositoryResolverTest.java @@ -33,6 +33,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.talend.sdk.component.classloader.ConfigurableClassLoader; import org.talend.sdk.component.test.dependencies.DependenciesTxtBuilder; class MvnDependencyListLocalRepositoryResolverTest { @@ -58,7 +59,7 @@ void nestedDependency(@TempDir final File temporaryFolder) throws IOException { new URLClassLoader(new URL[] { file.toURI().toURL() }, getSystemClassLoader())) { final List toResolve = new MvnDependencyListLocalRepositoryResolver("TALEND-INF/dependencies.txt", d -> null) - .resolve(tempLoader, "foo/bar/dummy/1.0.0/dummy-1.0.0.jar") + .resolve(tempLoader, "MAVEN-INF/repository/foo/bar/dummy/1.0.0/dummy-1.0.0.jar") .map(Artifact::toPath) .collect(toList()); assertEquals(asList("org/apache/tomee/ziplock/8.0.14/ziplock-8.0.14.jar", @@ -71,8 +72,7 @@ void nestedDependencyWithJira(@TempDir final File temporaryFolder) throws IOExce final File file = new File(temporaryFolder, UUID.randomUUID().toString() + ".jar"); file.getParentFile().mkdirs(); try (final JarOutputStream enclosing = new JarOutputStream(new FileOutputStream(file))) { - enclosing.putNextEntry( - new ZipEntry("MAVEN-INF/repository/foo/bar/dummy/1.0.0-TCOMP-2285/dummy-1.0.0-TCOMP-2285.jar")); + enclosing.putNextEntry(new ZipEntry("BOOT-INF/lib/dummy-1.0.0-TCOMP-2285.jar")); try (final JarOutputStream nested = new JarOutputStream(enclosing)) { nested.putNextEntry(new ZipEntry("TALEND-INF/dependencies.txt")); nested @@ -85,10 +85,13 @@ void nestedDependencyWithJira(@TempDir final File temporaryFolder) throws IOExce } try (final URLClassLoader tempLoader = - new URLClassLoader(new URL[] { file.toURI().toURL() }, getSystemClassLoader())) { + new URLClassLoader(new URL[] { file.toURI().toURL() }, getSystemClassLoader()); + final ConfigurableClassLoader ccl = new ConfigurableClassLoader("test", + new URL[] {}, getSystemClassLoader(), name -> true, name -> true, + new String[] {}, new String[0])) { final List toResolve = new MvnDependencyListLocalRepositoryResolver("TALEND-INF/dependencies.txt", d -> null) - .resolve(tempLoader, "foo/bar/dummy/1.0.0-TCOMP-2285/dummy-1.0.0-TCOMP-2285.jar") + .resolve(tempLoader, "BOOT-INF/lib/dummy-1.0.0-TCOMP-2285.jar") .map(Artifact::toPath) .collect(toList()); assertEquals(asList("org/apache/tomee/ziplock/8.0.14/ziplock-8.0.14.jar", diff --git a/sample-parent/sample-connector/src/main/java/org/talend/sdk/component/test/connectors/config/DynamicDependenciesConf.java b/sample-parent/sample-connector/src/main/java/org/talend/sdk/component/test/connectors/config/DynamicDependenciesConf.java index 22258c28bebca..4d182f743ac96 100644 --- a/sample-parent/sample-connector/src/main/java/org/talend/sdk/component/test/connectors/config/DynamicDependenciesConf.java +++ b/sample-parent/sample-connector/src/main/java/org/talend/sdk/component/test/connectors/config/DynamicDependenciesConf.java @@ -18,14 +18,14 @@ import java.io.Serializable; import org.talend.sdk.component.api.configuration.Option; -import org.talend.sdk.component.api.configuration.type.DynamicDependenciesServiceConfiguration; +import org.talend.sdk.component.api.configuration.type.DynamicDependenciesConfiguration; import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; import org.talend.sdk.component.api.meta.Documentation; import lombok.Data; @Data -@DynamicDependenciesServiceConfiguration +@DynamicDependenciesConfiguration @GridLayout({ @GridLayout.Row({ "group" }), @GridLayout.Row({ "artifact" })