IGNITE-26536 Extract code deployment class loader to separate module (#7083)

Author: Pochatkin

modules/client/build.gradle

@@ -57,6 +57,7 @@ dependencies {
     testImplementation project(':ignite-cluster-management')
     testImplementation project(':ignite-compute')
     testImplementation project(':ignite-code-deployment')
+    testImplementation project(':ignite-code-deployment-classloader')
     testImplementation project(':ignite-eventlog')
     testImplementation testFixtures(project(':ignite-api'))
     testImplementation testFixtures(project(':ignite-core'))

modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java

@@ -68,7 +68,7 @@
 import org.apache.ignite.internal.compute.TaskStateImpl;
 import org.apache.ignite.internal.compute.events.ComputeEventMetadata;
 import org.apache.ignite.internal.compute.events.ComputeEventMetadataBuilder;
-import org.apache.ignite.internal.compute.loader.JobClassLoader;
+import org.apache.ignite.internal.deployunit.loader.UnitsClassLoader;
 import org.apache.ignite.internal.hlc.HybridTimestamp;
 import org.apache.ignite.internal.network.ClusterNodeImpl;
 import org.apache.ignite.internal.network.InternalClusterNode;
@@ -132,7 +132,7 @@ public CompletableFuture<JobExecution<ComputeJobDataHolder>> executeAsyncWithFai
         }
 
         if (jobClassName.startsWith("org.apache.ignite")) {
-            JobClassLoader jobClassLoader = new JobClassLoader(List.of(), this.getClass().getClassLoader());
+            UnitsClassLoader jobClassLoader = new UnitsClassLoader(List.of(), this.getClass().getClassLoader());
             Class<ComputeJob<Object, Object>> jobClass = ComputeUtils.jobClass(jobClassLoader, jobClassName);
             ComputeJob<Object, Object> job = ComputeUtils.instantiateJob(jobClass);
             CompletableFuture<Object> jobFut = job.executeAsync(

modules/code-deployment-classloader/build.gradle

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+apply from: 'jobs.gradle'
+
+description = 'ignite-code-deployment-classloader'
+
+dependencies {
+    implementation libs.jetbrains.annotations
+
+    implementation project(':ignite-core')
+    implementation project(':ignite-api')
+    implementation project(':ignite-code-deployment')
+
+    testImplementation project(':ignite-core')
+    testImplementation testFixtures(project(':ignite-core'))
+}

modules/code-deployment-classloader/jobs.gradle

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+sourceSets {
+    jobs
+    unit1
+    unit2
+}
+
+def registerJarTask(SourceSet sourceSet, String baseName) {
+    tasks.register(sourceSet.jarTaskName, Jar) {
+        group = 'build'
+        archiveBaseName = baseName
+        archiveVersion = '1.0-SNAPSHOT'
+        from sourceSet.output
+    }
+}
+
+registerJarTask(sourceSets.unit1, 'ignite-unit-test-job1')
+registerJarTask(sourceSets.unit2, 'ignite-unit-test-job2')
+
+processTestResources {
+    into('units/unit1') {
+        into('1.0.0') {
+            from unit1Jar
+        }
+        into('2.0.0') {
+            from unit2Jar
+        }
+        into('3.0.1') {
+            from unit1Jar
+            from unit2Jar
+        }
+        into('3.0.2') {
+            into('subdir') {
+                from unit2Jar
+            }
+            from unit1Jar
+        }
+    }
+    into('units/unit2') {
+        into('1.0.0') {
+            from unit1Jar
+        }
+        into('2.0.0') {
+            from unit2Jar
+        }
+    }
+
+    File corruptedJarFile = file("$destinationDir/units/unit1/4.0.0/unit1-1.0-corrupted.jar")
+    doLast {
+        // Generate a 100 bytes long file with random contents to simulate corrupted jar
+        byte[] randomBytes = new byte[100]
+        new Random().nextBytes(randomBytes)
+        corruptedJarFile.parentFile.mkdirs()
+        corruptedJarFile.bytes = randomBytes
+    }
+}
+
+dependencies {
+    jobsImplementation project(':ignite-api')
+    jobsImplementation project(':ignite-core')
+    unit1Implementation project(':ignite-api')
+    unit2Implementation project(':ignite-api')
+}

…compute/ClassLoaderExceptionsMapper.java …/loader/ClassLoaderExceptionsMapper.javamodules/compute/src/main/java/org/apache/ignite/internal/compute/ClassLoaderExceptionsMapper.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/ClassLoaderExceptionsMapper.java modules/compute/src/main/java/org/apache/ignite/internal/compute/ClassLoaderExceptionsMapper.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/ClassLoaderExceptionsMapper.java

@@ -15,13 +15,12 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.compute;
+package org.apache.ignite.internal.deployunit.loader;
 
 import static org.apache.ignite.internal.util.ExceptionUtils.unwrapCompletionThrowable;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
-import org.apache.ignite.internal.compute.loader.JobContext;
 import org.apache.ignite.internal.deployunit.exception.DeploymentUnitNotFoundException;
 import org.apache.ignite.internal.deployunit.exception.DeploymentUnitUnavailableException;
 
@@ -34,20 +33,20 @@ class ClassLoaderExceptionsMapper {
     private static final String DEPLOYMENT_UNIT_NOT_AVAILABLE_MSG = "%s. Deployment unit %s:%s can't be used: "
             + "[clusterStatus = %s, nodeStatus = %s]";
 
-    static CompletableFuture<JobContext> mapClassLoaderExceptions(
-            CompletableFuture<JobContext> future,
-            String jobClassName
+    static CompletableFuture<UnitsClassLoaderContext> mapClassLoaderExceptions(
+            CompletableFuture<UnitsClassLoaderContext> future,
+            String acquireId
     ) {
         return future.handle((v, e) -> {
-            if (e instanceof Exception) {
-                throw new CompletionException(mapException((Exception) unwrapCompletionThrowable((Exception) e), jobClassName));
+            if (e != null) {
+                throw new CompletionException(mapException(unwrapCompletionThrowable(e), acquireId));
             } else {
                 return v;
             }
         });
     }
 
-    private static Exception mapException(Exception e, String jobClassName) {
+    private static Throwable mapException(Throwable e, String jobClassName) {
         if (e instanceof DeploymentUnitNotFoundException) {
             return new ClassNotFoundException(
                     String.format(

…ernal/compute/loader/JobClassLoader.java …/deployunit/loader/UnitsClassLoader.javamodules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClassLoader.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoader.java modules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClassLoader.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoader.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.compute.loader;
+package org.apache.ignite.internal.deployunit.loader;
 
 import java.net.URL;
 import java.security.AccessController;
@@ -32,22 +32,22 @@
 /**
  * Implementation of {@link ClassLoader} that loads classes from specified component directories.
  */
-public class JobClassLoader implements AutoCloseable {
-    private static final IgniteLogger LOG = Loggers.forClass(JobClassLoader.class);
+public class UnitsClassLoader implements AutoCloseable {
+    private static final IgniteLogger LOG = Loggers.forClass(UnitsClassLoader.class);
 
     private final List<DisposableDeploymentUnit> units;
 
     private final ClassLoader parent;
 
-    private volatile JobClassLoaderImpl impl;
+    private volatile UnitsClassLoaderImpl impl;
 
     /**
-     * Creates new instance of {@link JobClassLoader}.
+     * Creates new instance of {@link UnitsClassLoader}.
      *
      * @param units Units to load classes from.
      * @param parent Parent class loader.
      */
-    public JobClassLoader(List<DisposableDeploymentUnit> units, ClassLoader parent) {
+    public UnitsClassLoader(List<DisposableDeploymentUnit> units, ClassLoader parent) {
         this.units = units;
         this.parent = parent;
     }
@@ -77,18 +77,18 @@ public ClassLoader classLoader() {
         return impl;
     }
 
-    private JobClassLoaderImpl createClassLoader() {
-        return AccessController.doPrivileged((PrivilegedAction<JobClassLoaderImpl>) () -> {
+    private UnitsClassLoaderImpl createClassLoader() {
+        return AccessController.doPrivileged((PrivilegedAction<UnitsClassLoaderImpl>) () -> {
             URL[] classpath = units.stream()
                     .map(DisposableDeploymentUnit::path)
-                    .flatMap(JobClasspath::collectClasspath)
+                    .flatMap(UnitsClasspath::collectClasspath)
                     .toArray(URL[]::new);
 
             if (LOG.isDebugEnabled()) {
                 LOG.debug("Created class loader with classpath: {}", Arrays.toString(classpath));
             }
 
-            return new JobClassLoaderImpl(units, classpath, parent);
+            return new UnitsClassLoaderImpl(units, classpath, parent);
         });
     }
 
@@ -105,7 +105,7 @@ public void close() {
         }
 
         try {
-            JobClassLoaderImpl impl0 = impl;
+            UnitsClassLoaderImpl impl0 = impl;
 
             if (impl0 != null) {
                 impl0.close();

…/internal/compute/loader/JobContext.java …unit/loader/UnitsClassLoaderContext.javamodules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobContext.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoaderContext.java modules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobContext.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoaderContext.java

@@ -15,25 +15,25 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.compute.loader;
+package org.apache.ignite.internal.deployunit.loader;
 
 import java.util.function.Consumer;
 
 /**
  * Job context.
  */
-public class JobContext implements AutoCloseable {
+public class UnitsClassLoaderContext implements AutoCloseable {
 
-    private final JobClassLoader classLoader;
+    private final UnitsClassLoader classLoader;
 
-    private final Consumer<JobContext> onClose;
+    private final Consumer<UnitsClassLoaderContext> onClose;
 
-    public JobContext(JobClassLoader classLoader, Consumer<JobContext> onClose) {
+    public UnitsClassLoaderContext(UnitsClassLoader classLoader, Consumer<UnitsClassLoaderContext> onClose) {
         this.classLoader = classLoader;
         this.onClose = onClose;
     }
 
-    public JobClassLoader classLoader() {
+    public UnitsClassLoader classLoader() {
         return classLoader;
     }
 

…ompute/loader/JobClassLoaderFactory.java …unit/loader/UnitsClassLoaderFactory.javamodules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClassLoaderFactory.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoaderFactory.java modules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClassLoaderFactory.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoaderFactory.java

@@ -15,22 +15,22 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.compute.loader;
+package org.apache.ignite.internal.deployunit.loader;
 
 import java.util.List;
 import org.apache.ignite.internal.deployunit.DisposableDeploymentUnit;
 
 /**
- * Creates a class loader for a job.
+ * Deployment units classloader factory.
  */
-public class JobClassLoaderFactory {
+public class UnitsClassLoaderFactory {
     /**
      * Create a class loader for the specified units.
      *
      * @param units The units of the job.
      * @return The class loader.
      */
-    public JobClassLoader createClassLoader(List<DisposableDeploymentUnit> units) {
-        return new JobClassLoader(units, getClass().getClassLoader());
+    public UnitsClassLoader createClassLoader(List<DisposableDeploymentUnit> units) {
+        return new UnitsClassLoader(units, getClass().getClassLoader());
     }
 }

…l/compute/loader/JobClassLoaderImpl.java …loyunit/loader/UnitsClassLoaderImpl.javamodules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClassLoaderImpl.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoaderImpl.java modules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClassLoaderImpl.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClassLoaderImpl.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.compute.loader;
+package org.apache.ignite.internal.deployunit.loader;
 
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -24,7 +24,7 @@
 import java.util.regex.Pattern;
 import org.apache.ignite.internal.deployunit.DisposableDeploymentUnit;
 
-class JobClassLoaderImpl extends URLClassLoader {
+class UnitsClassLoaderImpl extends URLClassLoader {
     /**
      * Pattern to match system packages.
      */
@@ -38,13 +38,13 @@ class JobClassLoaderImpl extends URLClassLoader {
     private final ClassLoader parent;
 
     /**
-     * Creates new instance of {@link JobClassLoader}.
+     * Creates new instance of {@link UnitsClassLoader}.
      *
      * @param urls URLs to load classes from.
      * @param units Units to load classes from.
      * @param parent Parent class loader.
      */
-    JobClassLoaderImpl(List<DisposableDeploymentUnit> units, URL[] urls, ClassLoader parent) {
+    public UnitsClassLoaderImpl(List<DisposableDeploymentUnit> units, URL[] urls, ClassLoader parent) {
         super("compute-job", urls, parent);
         this.units = units;
         this.parent = parent;

…nternal/compute/loader/JobClasspath.java …al/deployunit/loader/UnitsClasspath.javamodules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClasspath.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClasspath.java modules/compute/src/main/java/org/apache/ignite/internal/compute/loader/JobClasspath.java renamed to modules/code-deployment-classloader/src/main/java/org/apache/ignite/internal/deployunit/loader/UnitsClasspath.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.compute.loader;
+package org.apache.ignite.internal.deployunit.loader;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
@@ -31,7 +31,7 @@
 import org.apache.ignite.compute.ComputeException;
 import org.apache.ignite.lang.ErrorGroups.Compute;
 
-class JobClasspath {
+class UnitsClasspath {
     static Stream<URL> collectClasspath(Path unitDir) {
         if (Files.notExists(unitDir)) {
             throw new IllegalArgumentException("Unit does not exist: " + unitDir);