IGNITE-26706 Migration Tools: Allow coping with unnecessary third-party classes from SqlFunctionClasses (#6768)

Author: tmgodinho

migration-tools/modules/migration-tools-commons-tests/build.gradle

@@ -23,6 +23,7 @@ description = 'migration-tools-commons-tests'
 
 configurations {
     unpackDependencies { transitive = false}
+    fullSampleClusterDependencies
 }
 
 dependencies {
@@ -37,6 +38,8 @@ dependencies {
     implementation files(layout.buildDirectory.dir("generated/unpackClassesFromDependencies"))
 
     unpackDependencies libs.ignite2.core
+
+    fullSampleClusterDependencies project(':migration-tools-e2e-implementations-custom-classes')
 }
 
 def unpackTask = tasks.register('unpackClassesFromDependencies', Copy) {
@@ -53,10 +56,30 @@ def unpackTask = tasks.register('unpackClassesFromDependencies', Copy) {
             "org/apache/ignite/cache/query/annotations/QuerySqlField\$Group.class"
 }
 
+def createFullSampleClusterResources = tasks.register('createFullSampleClusterResources') {
+    description = 'Generates the "fullsamplecluster" resource file.'
+
+    def outputFile = file("$buildDir/resources/main/fullsamplecluster")
+    outputs.file(outputFile)
+
+    doLast {
+        def depList = configurations.fullSampleClusterDependencies.resolvedConfiguration.resolvedArtifacts.collect { artifact ->
+            artifact.file.path
+        }
+
+        outputFile.parentFile.mkdirs()
+        outputFile.text = depList.join('\n')
+    }
+}
+
 compileJava {
     dependsOn unpackTask
 }
 
+processResources {
+    dependsOn createFullSampleClusterResources
+}
+
 /*sourceSets {
     main {
         java {

migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/clusters/FullSampleCluster.java

@@ -17,10 +17,19 @@
 
 package org.apache.ignite.migrationtools.tests.clusters;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.nio.file.Path;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.ignite.migrationtools.tests.containers.Ignite2ClusterContainer;
 import org.apache.ignite.migrationtools.tests.containers.Ignite2ClusterWithSamples;
+import org.jetbrains.annotations.Nullable;
+import org.testcontainers.utility.MountableFile;
 
 /** Cluster with all the samples from all the caches. */
 public class FullSampleCluster extends Ignite2ClusterWithSamples {
@@ -42,10 +51,32 @@ public FullSampleCluster() {
 
     @Override
     protected Ignite2ClusterContainer createClusterContainers() {
-        return new Ignite2ClusterContainer(
+        var cluster = new Ignite2ClusterContainer(
                 CLUSTER_CFG_PATH,
                 TEST_CLUSTER_PATH,
                 clusterNodeIds
         );
+
+        List<String> dependencies;
+        @Nullable InputStream rs = FullSampleCluster.class.getResourceAsStream("/fullsamplecluster");
+        if (rs == null) {
+            throw new IllegalStateException("Could not find required resource for loading dependencies.");
+        }
+
+        try (
+                rs;
+                InputStreamReader irs = new InputStreamReader(rs, UTF_8);
+                BufferedReader r = new BufferedReader(irs)
+        ) {
+            dependencies = r.lines().map(String::trim).collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        for (var path : dependencies) {
+            cluster.withFileInClasspath(MountableFile.forHostPath(path));
+        }
+
+        return cluster;
     }
 }

migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterContainer.java

@@ -40,6 +40,7 @@
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.lifecycle.Startable;
 import org.testcontainers.lifecycle.Startables;
+import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.MountableFile;
 
 /** Container of an Ignite 2 cluster. */
@@ -53,6 +54,8 @@ public class Ignite2ClusterContainer implements Startable {
 
     private final boolean storagePathMappedToExternal;
 
+    private final String igniteHome;
+
     /**
      * Port on host which binds container's 10800.
      */
@@ -80,11 +83,18 @@ public Ignite2ClusterContainer(Network network, Path cfgFilePath, @Nullable Path
         this.containers = new ArrayList<>(nodeIds.size());
         this.storagePathMappedToExternal = storagePathOnHost != null;
 
+        String dockerImageName = System.getProperty("ignite2.docker.image");
+        assert dockerImageName != null : "ignite2.docker.image must be defined";
+        DockerImageName dockerImage = DockerImageName.parse(dockerImageName);
+
+        this.igniteHome = "/opt/ignite/apache-ignite";
+
         for (int i = 0; i < nodeIds.size(); i++) {
             String hostname = "node" + (1 + i);
             String nodeId = nodeIds.get(i);
 
             var nodeContainer = createIgnite2Container(
+                    dockerImage,
                     hostname,
                     nodeId,
                     cfgFilePath,
@@ -103,17 +113,16 @@ public Network getNetwork() {
     }
 
     private GenericContainer<?> createIgnite2Container(
+            DockerImageName dockerImage,
             String hostName,
             String nodeId,
             Path cfgFilePath,
             @Nullable Path storagePathOnHost
     ) {
         Consumer<OutputFrame> logConsumer = new CheckpointerLogConsumer();
         String heapSize = System.getProperty("ai2.sampleCluster.Xmx", "10g");
-        String ignite2DockerImage = System.getProperty("ignite2.docker.image");
-        assert ignite2DockerImage != null : "ignite2.docker.image must be defined";
 
-        GenericContainer<?> container = new GenericContainer<>(ignite2DockerImage);
+        GenericContainer<?> container = new GenericContainer<>(dockerImage);
 
         if (storagePathMappedToExternal) {
             container.withFileSystemBind(storagePathOnHost.toString(), "/storage", BindMode.READ_WRITE)
@@ -251,6 +260,17 @@ public String dockerHost() {
         return dockerHost;
     }
 
+    /**
+     * Copies the supplied file to all the containers classpath.
+     *
+     * @param fileToCopy File to copy.
+     */
+    public void withFileInClasspath(MountableFile fileToCopy) {
+        for (var container : this.containers) {
+            container.withCopyFileToContainer(fileToCopy, this.igniteHome + "/libs/");
+        }
+    }
+
     private static class CheckpointerLogConsumer implements Consumer<OutputFrame> {
 
         private List<Runnable> listeners = new CopyOnWriteArrayList<>();

migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterWithSamples.java

@@ -28,11 +28,9 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.extension.BeforeAllCallback;
 import org.junit.jupiter.api.extension.ExtensionContext;
-import org.testcontainers.containers.BindMode;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.Network;
 import org.testcontainers.containers.output.OutputFrame;
-import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.utility.MountableFile;
 
 /** Ignite2ClusterWithSamples. */
@@ -59,22 +57,6 @@ public static boolean waitForCondition(BooleanSupplier cond, long sleepMillis, l
         return false;
     }
 
-    private static GenericContainer createIgnite2Container(String nodeId, String nodeName, Network network,
-            Consumer<OutputFrame> logConsumer) {
-        return new GenericContainer<>("apacheignite/ignite:2.15.0-jdk11")
-                .withLabel("ai2.sample-cluster.node", nodeName)
-                .withNetwork(network)
-                .withNetworkAliases(nodeName)
-                .withCopyFileToContainer(MountableFile.forHostPath(FullSampleCluster.CLUSTER_CFG_PATH), "/config-file.xml")
-                .withFileSystemBind(FullSampleCluster.TEST_CLUSTER_PATH.toString(), "/storage", BindMode.READ_WRITE)
-                .withEnv("CONFIG_URI", "/config-file.xml")
-                .withEnv("IGNITE_WORK_DIR", "/storage")
-                .withEnv("IGNITE_QUIET", "false")
-                .withEnv("IGNITE_NODE_NAME", nodeId)
-                .withLogConsumer(logConsumer)
-                .waitingFor(Wait.forLogMessage(".*Node started .*", 1));
-    }
-
     protected abstract Ignite2ClusterContainer createClusterContainers();
 
     private void recreateClusterFolder() throws InterruptedException, IOException {

migration-tools/modules/migration-tools-persistence/build.gradle

@@ -32,6 +32,7 @@ dependencies {
     implementation libs.commons.collections4
     implementation libs.jackson.databind
     implementation libs.slf4j.api
+    implementation libs.bytebuddy
     compileOnly libs.spotbugs.annotations
 
     testImplementation project(":migration-tools-commons-tests")

migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/MarshallerTest.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.CacheType;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.FieldSource;
+
+@ExtendWith(FullSampleCluster.class)
+class MarshallerTest {
+    private static final List<String> CACHE_NAMES = List.of(
+            "MySimpleMap",
+            "MyPersonPojoCache",
+            "MyOrganizations",
+            "MyIntArrCache",
+            "MyListArrCache",
+            "MyBinaryPersonPojoCache",
+            "MyBinaryOrganizationCache",
+            "MyBinaryTestCache"
+    );
+
+    @ExtendWith(BasePersistentTestContext.class)
+    private List<MigrationKernalContext> nodeContexts;
+
+    private MigrationKernalContext nodeCtx;
+
+    @BeforeEach
+    void beforeEach() throws IgniteCheckedException {
+        nodeCtx = nodeContexts.get(0);
+        nodeCtx.start();
+    }
+
+    @Test
+    void loadAllTest() throws IgniteCheckedException {
+        ((MigrationCacheProcessor) nodeCtx.cache()).loadAllDescriptors();
+
+        Collection<DynamicCacheDescriptor> descriptors = nodeCtx.cache().persistentCaches().stream()
+                .filter(c -> c.cacheType() == CacheType.USER)
+                .collect(Collectors.toList());
+
+        assertThat(descriptors).hasSize(8);
+
+        assertThat(descriptors)
+                .map(DynamicCacheDescriptor::cacheConfiguration)
+                .doesNotContainNull();
+    }
+
+    @ParameterizedTest
+    @FieldSource("CACHE_NAMES")
+    void loadEachTest(String cacheName) {
+        var cacheDescriptor = nodeCtx.cache().cacheDescriptor(cacheName);
+        assertThat(cacheDescriptor).isNotNull();
+        assertThat(cacheDescriptor.cacheConfiguration()).isNotNull();
+    }
+}

migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/MigrationKernalContext.java

@@ -63,6 +63,7 @@
 import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.migrationtools.persistence.marshallers.ForeignJdkMarshaller;
 import org.apache.ignite.spi.deployment.local.LocalDeploymentSpi;
 import org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi;
 import org.apache.ignite.spi.eventstorage.NoopEventStorageSpi;
@@ -82,6 +83,8 @@ public class MigrationKernalContext extends GridKernalContextImpl {
 
     private static final Field MARSH_CTX_FIELD;
 
+    private static final Field JDK_MARSHALLER_FIELD;
+
     static {
         try {
             CFG_FIELD = GridKernalContextImpl.class.getDeclaredField("cfg");
@@ -92,6 +95,9 @@ public class MigrationKernalContext extends GridKernalContextImpl {
 
             MARSH_CTX_FIELD = GridKernalContextImpl.class.getDeclaredField("marshCtx");
             MARSH_CTX_FIELD.setAccessible(true);
+
+            JDK_MARSHALLER_FIELD = MarshallerContextImpl.class.getDeclaredField("jdkMarsh");
+            JDK_MARSHALLER_FIELD.setAccessible(true);
         } catch (NoSuchFieldException e) {
             throw new RuntimeException(e);
         }
@@ -127,6 +133,7 @@ public MigrationKernalContext(IgniteConfiguration cfg, File nodeFolder, Serializ
 
         try {
             CFG_FIELD.set(this, adaptedConfiguration);
+            JDK_MARSHALLER_FIELD.set(marshCtx, new ForeignJdkMarshaller());
             MARSH_CTX_FIELD.set(this, marshCtx);
 
             // Unnecessarily required by CacheObjectBinaryProcessorImpl & by GridCacheDefaultAffinityKeyMapper#ignite

migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/marshallers/ForeignJdkMarshaller.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence.marshallers;
+
+import java.io.InputStream;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.jetbrains.annotations.Nullable;
+
+/** {@link JdkMarshaller} implementation which uses {@link ForeignObjectInputStream}. */
+public class ForeignJdkMarshaller extends JdkMarshaller {
+    @Override
+    protected <T> T unmarshal0(InputStream in, @Nullable ClassLoader clsLdr) throws IgniteCheckedException {
+        // Essentially the same impl as JdkMarshaller but with a different underlying ObjectInputStream.
+        assert in != null;
+
+        ClassLoader localClassLoader = (clsLdr != null) ? clsLdr : getClass().getClassLoader();
+
+        try (var objIn = new ForeignObjectInputStream(in, localClassLoader)) {
+            return (T) objIn.readObject();
+        } catch (ClassNotFoundException e) {
+            throw new IgniteCheckedException("Failed to find class with given class loader for unmarshalling "
+                    + "[clsLdr=" + localClassLoader + ", cls=" + e.getMessage() + "]", e);
+        } catch (Exception e) {
+            throw new IgniteCheckedException("Failed to deserialize object with given class loader: " + localClassLoader, e);
+        }
+    }
+}