feat(QTDI-2134): dynamic dependencies - nested loading

component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesConfiguration.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright (C) 2006-2025 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.talend.sdk.component.api.configuration.type;
+
+import static java.lang.annotation.ElementType.TYPE;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import org.talend.sdk.component.api.configuration.type.meta.ConfigurationType;
+import org.talend.sdk.component.api.meta.Documentation;
+
+@Target(TYPE)
+@Retention(RUNTIME)
+@ConfigurationType("dynamicDependenciesConfiguration")
+@Documentation("Mark a model (complex object) as being the configuration expected to compute dynamic dependencies.")
+public @interface DynamicDependenciesConfiguration {
+
+    String value() default "default";
+}

component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java

@@ -899,16 +899,30 @@ private void autoDiscoverPlugins0(final boolean callers, final boolean classpath
         // common for studio until job generation is updated to build a tcomp friendly bundle
         if (classpath && !Boolean.getBoolean("component.manager.classpath.skip")) {
             try {
+                final String markerValue = "TALEND-INF/dependencies.txt";
                 final Enumeration<URL> componentMarkers =
-                        Thread.currentThread().getContextClassLoader().getResources("TALEND-INF/dependencies.txt");
+                        Thread.currentThread().getContextClassLoader().getResources(markerValue);
                 while (componentMarkers.hasMoreElements()) {
-                    File file = Files.toFile(componentMarkers.nextElement());
-                    if (file.getName().equals("dependencies.txt") && file.getParentFile() != null
-                            && file.getParentFile().getName().equals("TALEND-INF")) {
-                        file = file.getParentFile().getParentFile();
-                    }
-                    if (!hasPlugin(container.buildAutoIdFromName(file.getName()))) {
-                        addPlugin(file.getAbsolutePath());
+                    final URL marker = componentMarkers.nextElement();
+                    File file = Files.toFile(marker);
+                    if (file != null) {
+                        if (file.getName().equals("dependencies.txt") && file.getParentFile() != null
+                                && file.getParentFile().getName().equals("TALEND-INF")) {
+                            file = file.getParentFile().getParentFile();
+                        }
+                        if (!hasPlugin(container.buildAutoIdFromName(file.getName()))) {
+                            addPlugin(file.getAbsolutePath());
+                        }
+                    } else {
+                        // lookup nested jar
+                        if (marker != null && "jar".equals(marker.getProtocol())) {
+                            final String urlFile = marker.getFile();
+                            final String jarPath = urlFile.substring(0, urlFile.lastIndexOf("!"));
+                            final String jarFilePath = jarPath.substring(jarPath.lastIndexOf("/") + 1);
+                            if (!hasPlugin(container.buildAutoIdFromName(jarFilePath))) {
+                                addPlugin(jarPath);
+                            }
+                        }
                     }
                 }
             } catch (final IOException e) {
@@ -1299,11 +1313,14 @@ public void onCreate(final Container container) {
 
             final AnnotationFinder finder;
             Archive archive = null;
+            final String rootModule = container.getRootModule();
+            final boolean nested = rootModule != null && rootModule.startsWith("nested:");
             try {
                 String alreadyScannedClasses = null;
                 Filter filter = KnownClassesFilter.INSTANCE;
-                try (final InputStream containerFilterConfig =
-                        container.getLoader().getResourceAsStream("TALEND-INF/scanning.properties")) {
+                try (final InputStream containerFilterConfig = nested
+                        ? loader.getNestedResource(rootModule + "!/TALEND-INF/scanning.properties")
+                        : loader.getResourceAsStream("TALEND-INF/scanning.properties")) {
                     if (containerFilterConfig != null) {
                         final Properties config = new Properties();
                         config.load(containerFilterConfig);
@@ -1778,8 +1795,10 @@ private Archive toArchive(final String module, final String moduleId, final Conf
                 }
             }
             info(module + " (" + moduleId + ") is not a file, will try to look it up from a nested maven repository");
-            final URL nestedJar =
-                    loader.getParent().getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY + module);
+            URL nestedJar = loader.getParent().getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY + module);
+            if (nestedJar == null) {
+                nestedJar = loader.getParent().getResource(module);
+            }
             if (nestedJar != null) {
                 InputStream nestedStream = null;
                 final JarInputStream jarStream;

component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/service/ResolverImpl.java

@@ -67,7 +67,7 @@ public ClassLoaderDescriptor mapDescriptorToClassLoader(final InputStream descri
                             } catch (final MalformedURLException e) {
                                 throw new IllegalStateException(e);
                             }
-                        } else if (loader.getResource("MAVEN-INF/repository/" + path) != null) {
+                        } else if (loader.getResource(path) != null) {
                             nested.add(path);
                             resolved.add(artifact.toCoordinate());
                         } // else will be missing

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java

@@ -60,7 +60,9 @@
 import javax.management.ReflectionException;
 
 import org.apache.xbean.finder.util.Files;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -98,6 +100,16 @@ private ComponentManager newManager() {
         return newManager(new File("target/test-dependencies"));
     }
 
+    @BeforeAll
+    static void setup() {
+        System.setProperty("talend.component.manager.m2.fallback", "true");
+    }
+
+    @AfterAll
+    static void teardown() {
+        System.clearProperty("talend.component.manager.m2.fallback");
+    }
+
     @Test
     void doubleClose() {
         final ComponentManager instance = ComponentManager.instance();
@@ -417,7 +429,7 @@ void extendFamilyInNestedRepo(@TempDir final File temporaryFolder) throws Except
         final URLClassLoader parentLoader =
                 new URLClassLoader(new URL[] { fatJar.toURI().toURL() }, thread.getContextClassLoader());
         thread.setContextClassLoader(parentLoader);
-        try (final ComponentManager manager = newManager(new File("target/missing_" + UUID.randomUUID().toString()))) {
+        try (final ComponentManager manager = newManager(pluginFolder)) {
             try {
                 manager.addPlugin(plugin2.getAbsolutePath());
 
@@ -429,7 +441,7 @@ void extendFamilyInNestedRepo(@TempDir final File temporaryFolder) throws Except
                         .map(File::getName)
                         .sorted()
                         .toArray(String[]::new);
-                assertEquals(1, dependencies.length); // ignored transitive deps, enables the new root to control it
+                assertEquals(2, dependencies.length); // ignored transitive deps, enables the new root to control it
                 assertEquals("main.jar", dependencies[0]); // transitive-1.0.0.jar is nested
             } finally {
                 if (!transitive.delete()) {

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/ResolverImplTest.java

@@ -99,7 +99,7 @@ void resolvefromDescriptor() throws IOException {
 
     private void addDepToJar(final String dep, final JarOutputStream out) {
         final String[] segments = dep.split(":");
-        final String path = "MAVEN-INF/repository/" + segments[0].replace(".", "/") + "/" + segments[1] + "/"
+        final String path = segments[0].replace(".", "/") + "/" + segments[1] + "/"
                 + segments[3] + "/" + segments[1] + "-" + segments[3] + "." + segments[2];
 
         // create folders for this m2 embedded deps
@@ -114,7 +114,7 @@ private void addDepToJar(final String dep, final JarOutputStream out) {
             }
         }
         // add the dep
-        final File jar = new File("target/test-dependencies", path.substring("MAVEN-INF/repository/".length()));
+        final File jar = new File("target/test-dependencies", path);
         try {
             out.putNextEntry(new ZipEntry(path));
             Files.copy(jar.toPath(), out);

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/xbean/NestedJarArchiveTest.java

@@ -53,9 +53,9 @@ void xbeanNestedScanning(final TestInfo info, @TempDir final File temporaryFolde
         final File jar = createPlugin(temporaryFolder, info.getTestMethod().get().getName());
         final ConfigurableClassLoader configurableClassLoader = new ConfigurableClassLoader("", new URL[0],
                 new URLClassLoader(new URL[] { jar.toURI().toURL() }, Thread.currentThread().getContextClassLoader()),
-                n -> true, n -> true, new String[] { "com/foo/bar/1.0/bar-1.0.jar" }, new String[0]);
+                n -> true, n -> true, new String[] { "BOOT-INF/lib/com/foo/bar/1.0/bar-1.0.jar" }, new String[0]);
         try (final JarInputStream jis = new JarInputStream(
-                configurableClassLoader.getResourceAsStream("MAVEN-INF/repository/com/foo/bar/1.0/bar-1.0.jar"))) {
+                configurableClassLoader.getResourceAsStream("BOOT-INF/lib/com/foo/bar/1.0/bar-1.0.jar"))) {
             assertNotNull(jis, "test is wrongly setup, no nested jar, fix the createPlugin() method please");
             final AnnotationFinder finder =
                     new AnnotationFinder(new NestedJarArchive(null, jis, configurableClassLoader));
@@ -72,7 +72,7 @@ private File createPlugin(final File pluginFolder, final String name) throws IOE
         final File target = new File(pluginFolder, name);
         target.getParentFile().mkdirs();
         try (final JarOutputStream outputStream = new JarOutputStream(new FileOutputStream(target))) {
-            outputStream.putNextEntry(new ZipEntry("MAVEN-INF/repository/com/foo/bar/1.0/bar-1.0.jar"));
+            outputStream.putNextEntry(new ZipEntry("BOOT-INF/lib/com/foo/bar/1.0/bar-1.0.jar"));
             try (final JarOutputStream nestedStream = new JarOutputStream(outputStream)) {
                 final String packageName = "org/talend/test/generated/" + name.replace(".jar", "");
                 { // the factory (declaration)

container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java

@@ -28,6 +28,8 @@
 import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.instrument.ClassFileTransformer;
@@ -38,7 +40,10 @@
 import java.net.URLClassLoader;
 import java.net.URLConnection;
 import java.net.URLStreamHandler;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.security.CodeSource;
 import java.security.cert.Certificate;
 import java.util.ArrayList;
@@ -55,6 +60,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Predicate;
+import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import java.util.jar.JarInputStream;
 import java.util.jar.Manifest;
@@ -150,7 +156,7 @@ private ConfigurableClassLoader(final String id, final URL[] urls, final ClassLo
     private void loadNestedDependencies(final ClassLoader parent, final String[] nestedDependencies) {
         final byte[] buffer = new byte[8192]; // should be good for most cases
         final ByteArrayOutputStream out = new ByteArrayOutputStream(buffer.length);
-        Stream.of(nestedDependencies).map(d -> NESTED_MAVEN_REPOSITORY + d).forEach(resource -> {
+        Stream.of(nestedDependencies).forEach(resource -> {
             final URL url = ofNullable(super.findResource(resource)).orElseGet(() -> parent.getResource(resource));
             if (url == null) {
                 throw new IllegalArgumentException("Didn't find " + resource + " in " + asList(nestedDependencies));
@@ -459,7 +465,7 @@ public Enumeration<URL> findResources(final String name) throws IOException {
     }
 
     private boolean isNestedDependencyResource(final String name) {
-        return name != null && name.startsWith(NESTED_MAVEN_REPOSITORY);
+        return name.startsWith(NESTED_MAVEN_REPOSITORY) || name.endsWith(".jar");
     }
 
     private boolean isInJvm(final URL resource) {
@@ -502,6 +508,60 @@ public List<InputStream> findContainedResources(final String name) {
         }
     }
 
+    /**
+     * Opens a stream to a resource located in a nested JAR.
+     *
+     * @param nestedUrl The full "nested:" URL, e.g.
+     * nested:/path/to/outer.jar/!path/inner.jar!/file.txt
+     * @return InputStream to the resource (must be closed by caller)
+     */
+    public InputStream getNestedResource(final String nestedUrl) throws IOException {
+        if (nestedUrl == null || !nestedUrl.startsWith("nested:")) {
+            throw new IllegalArgumentException("Invalid nested URL: " + nestedUrl);
+        }
+        final String path = nestedUrl.substring("nested:".length());
+        // Find the "/!" and "!/" separators
+        final int firstBang = path.indexOf("/!");
+        final int secondBang = path.indexOf("!/", firstBang + 2);
+        if (firstBang < 0 || secondBang < 0) {
+            throw new IllegalArgumentException("Malformed nested URL: " + nestedUrl);
+        }
+        final Path outerJarPath = Paths.get(path.substring(0, firstBang)); // before /!
+        final String innerJarPath = path.substring(firstBang + 2, secondBang); // between /! and !/
+        final String resourcePath = path.substring(secondBang + 2); // after the last !/
+        // Open the outer JAR file
+        try (JarFile outerJar = new JarFile(outerJarPath.toFile())) {
+            JarEntry innerEntry = outerJar.getJarEntry(innerJarPath);
+            if (innerEntry == null) {
+                throw new FileNotFoundException("Inner JAR not found: " + innerJarPath);
+            }
+            // Copy the inner JAR to a temporary file
+            final Path tempInnerJar = Files.createTempFile("nested-inner-", ".jar");
+            try (InputStream innerStream = outerJar.getInputStream(innerEntry)) {
+                Files.copy(innerStream, tempInnerJar, StandardCopyOption.REPLACE_EXISTING);
+
+                try (JarFile innerJar = new JarFile(tempInnerJar.toFile())) {
+                    final JarEntry resourceEntry = innerJar.getJarEntry(resourcePath);
+                    if (resourceEntry == null) {
+                        throw new FileNotFoundException("Resource not found: " + resourcePath);
+                    }
+
+                    // Return a stream that cleans up automatically when closed
+                    return new FilterInputStream(innerJar.getInputStream(resourceEntry)) {
+                        @Override
+                        public void close() throws IOException {
+                            super.close();
+                            Files.deleteIfExists(tempInnerJar);
+                        }
+                    };
+                }
+            } catch (IOException e) {
+                Files.deleteIfExists(tempInnerJar);
+                throw e;
+            }
+        }
+    }
+
     private URL nestedResourceToURL(final String name, final Resource nestedResource) {
         try {
             return new URL("nested", null, -1, nestedResource.entry + "!/" + name, new Handler(nestedResource));

container/container-core/src/main/java/org/talend/sdk/component/container/Container.java

@@ -101,7 +101,7 @@ public Container(final String id, final String rootModule, final Artifact[] depe
         this.dependencies = dependencies;
         this.localDependencyRelativeResolver = localDependencyRelativeResolver;
         this.lastModifiedTimestamp.set(new Date(0));
-        this.hasNestedRepository = hasNestedRepository;
+        this.hasNestedRepository = rootModule.startsWith("nested:") || hasNestedRepository;
         ofNullable(initializer).ifPresent(i -> i.accept(this));
 
         this.classloaderProvider = () -> {
@@ -143,9 +143,11 @@ public Container(final String id, final String rootModule, final Artifact[] depe
                                     .distinct()
                                     .toArray(String[]::new)
                             : null;
-            final ConfigurableClassLoader loader = new ConfigurableClassLoader(id, urls,
-                    overrideClassLoaderConfig.getParent(), overrideClassLoaderConfig.getParentClassesFilter(),
-                    overrideClassLoaderConfig.getClassesFilter(), rawNestedDependencies, jvmMarkers);
+            final Predicate<String> parentFilter =
+                    this.hasNestedRepository ? (name) -> true : overrideClassLoaderConfig.getParentClassesFilter();
+            final ConfigurableClassLoader loader =
+                    new ConfigurableClassLoader(id, urls, overrideClassLoaderConfig.getParent(), parentFilter,
+                            overrideClassLoaderConfig.getClassesFilter(), rawNestedDependencies, jvmMarkers);
             transformers.forEach(loader::registerTransformer);
             activeSpecificTransformers(loader);
             return loader;
@@ -171,7 +173,7 @@ private boolean findNestedDependency(final ContainerManager.ClassLoaderConfigura
         }
         final URL url = overrideClassLoaderConfig
                 .getParent()
-                .getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY + depPath);
+                .getResource(depPath);
         return url != null;
     }
 
@@ -307,6 +309,10 @@ public Date getCreated() {
         return created.get();
     }
 
+    public boolean hasNestedRepository() {
+        return hasNestedRepository;
+    }
+
     public void registerTransformer(final ClassFileTransformer transformer) {
         transformers.add(transformer);
     }