Deduplicate template staging and release from dependent modules (#2283)

Author: Abacn

plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/TemplateDefinitionsParser.java

@@ -18,39 +18,58 @@
 import com.google.cloud.teleport.metadata.MultiTemplate;
 import com.google.cloud.teleport.metadata.Template;
 import com.google.cloud.teleport.plugin.model.TemplateDefinitions;
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 import org.reflections.Reflections;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Utility class that will be used to scan for {@link Template} or {@link MultiTemplate} classes in
  * the classpath.
  */
 public final class TemplateDefinitionsParser {
+  private static final Logger LOG = LoggerFactory.getLogger(TemplateDefinitionsParser.class);
 
   private TemplateDefinitionsParser() {}
 
   /**
    * Given a ClassLoader, this method will scan look for every class that is annotated with {@link
-   * Template} or {@link MultiTemplate}, by using {@link Reflections}. It then wraps all the
-   * annotations and class name in a {@link TemplateDefinitions} instance, puts them in a list and
-   * returns to the caller.
+   * Template} or {@link MultiTemplate} from {@code outputDir}, by using {@link Reflections}. It
+   * then wraps all the annotations and class name in a {@link TemplateDefinitions} instance, puts
+   * them in a list and returns to the caller.
    *
    * @param classLoader ClassLoader that should be used to scan for the annotations.
+   * @param outputDir dir of target class.
    * @return Listed with all definitions that could be found in the classpath.
    */
-  public static List<TemplateDefinitions> scanDefinitions(ClassLoader classLoader) {
-
+  public static List<TemplateDefinitions> scanDefinitions(ClassLoader classLoader, File outputDir) {
     List<TemplateDefinitions> allDefinitions = new ArrayList<>();
 
     // Scan every @Template class
     Set<Class<?>> templates = new Reflections(classLoader).getTypesAnnotatedWith(Template.class);
     for (Class<?> templateClass : templates) {
       Template templateAnnotation = templateClass.getAnnotation(Template.class);
-      if (!templateAnnotation.testOnly()) {
-        allDefinitions.add(new TemplateDefinitions(templateClass, templateAnnotation));
+      if (templateAnnotation.testOnly()) {
+        continue;
+      }
+
+      URI loc;
+      try {
+        loc = templateClass.getProtectionDomain().getCodeSource().getLocation().toURI();
+      } catch (URISyntaxException e) {
+        throw new RuntimeException(e);
       }
+      if (!loc.getPath().startsWith(outputDir.getPath())) {
+        LOG.info("Skip " + templateClass.getName() + " from dependency module " + loc);
+        continue;
+      }
+
+      allDefinitions.add(new TemplateDefinitions(templateClass, templateAnnotation));
     }
 
     // Scan every @MultiTemplate class

plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/TemplateSpecsGenerator.java

@@ -43,10 +43,11 @@ public class TemplateSpecsGenerator {
    * Scan the classloader for all Template classes, and then builds spec + saves the metadata for
    * every Template.
    */
-  public void generateSpecs(ClassLoader classLoader, File baseDirectory, File targetDirectory) {
+  public void generateSpecs(
+      ClassLoader classLoader, File outputDirectory, File baseDirectory, File targetDirectory) {
 
     List<TemplateDefinitions> templateDefinitions =
-        TemplateDefinitionsParser.scanDefinitions(classLoader);
+        TemplateDefinitionsParser.scanDefinitions(classLoader, outputDirectory);
     for (TemplateDefinitions definition : templateDefinitions) {
       LOG.info("Generating template " + definition.getTemplateAnnotation().name() + "...");
       // Skip generating docs for template annotations that are stage-only

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesBaseMojo.java

@@ -64,13 +64,17 @@ protected URLClassLoader buildClassloader()
     List<String> classpathElements = project.getCompileClasspathElements();
 
     List<URL> projectClasspathList = new ArrayList<>();
+    // put thin jar (if exists) in front of classPath for class loader to override fat jar.
+    // This is later used for filtering out templates from dependency modules when staging
+    // templates of a module
     projectClasspathList.add(
         new File(
                 outputDirectory,
                 "original-" + project.getArtifactId() + "-" + project.getVersion() + ".jar")
             .toURI()
             .toURL());
 
+    // add remaining classes in compileClasspath
     for (String element : classpathElements) {
       try {
         projectClasspathList.add(new File(element).toURI().toURL());

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesReleaseMojo.java

@@ -156,7 +156,7 @@ public void execute() throws MojoExecutionException {
       LOG.info("Releasing Templates to bucket '{}'...", bucketNameOnly(bucketName));
 
       List<TemplateDefinitions> templateDefinitions =
-          TemplateDefinitionsParser.scanDefinitions(loader);
+          TemplateDefinitionsParser.scanDefinitions(loader, outputDirectory);
 
       if (templateName != null && !templateName.isEmpty()) {
         templateDefinitions =

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesRunMojo.java

@@ -168,7 +168,7 @@ public void execute() throws MojoExecutionException {
       LOG.info("Staging Templates to bucket '{}'...", bucketNameOnly(bucketName));
 
       List<TemplateDefinitions> templateDefinitions =
-          TemplateDefinitionsParser.scanDefinitions(loader);
+          TemplateDefinitionsParser.scanDefinitions(loader, outputDirectory);
 
       Optional<TemplateDefinitions> definitionsOptional =
           templateDefinitions.stream()

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesSpecMojo.java

@@ -50,7 +50,7 @@ public void execute() throws MojoExecutionException {
       }
 
       TemplateSpecsGenerator generator = new TemplateSpecsGenerator();
-      generator.generateSpecs(loader, baseDirectory, targetDirectory);
+      generator.generateSpecs(loader, outputDirectory, baseDirectory, targetDirectory);
 
     } catch (DependencyResolutionRequiredException e) {
       throw new MojoExecutionException("Dependency resolution failed", e);

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java

@@ -264,7 +264,7 @@ public void execute() throws MojoExecutionException {
       LOG.info("Staging Templates to bucket '{}'...", bucketNameOnly(bucketName));
 
       List<TemplateDefinitions> templateDefinitions =
-          TemplateDefinitionsParser.scanDefinitions(loader);
+          TemplateDefinitionsParser.scanDefinitions(loader, outputDirectory);
       for (TemplateDefinitions definition : templateDefinitions) {
 
         ImageSpec imageSpec = definition.buildSpecModel(false);

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesTerraformMojo.java

@@ -92,7 +92,7 @@ private List<TemplateDefinitions> loadDefinitions()
 
     LOG.info("Generating Template Specs, saving at target: {}", targetDirectory);
 
-    return TemplateDefinitionsParser.scanDefinitions(loader);
+    return TemplateDefinitionsParser.scanDefinitions(loader, outputDirectory);
   }
 
   private File modulePath(TemplateDefinitions definition) {

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesValidateMojo.java

@@ -50,7 +50,7 @@ public void execute() throws MojoExecutionException {
       LOG.info("Validating Templates...");
 
       List<TemplateDefinitions> templateDefinitions =
-          TemplateDefinitionsParser.scanDefinitions(loader);
+          TemplateDefinitionsParser.scanDefinitions(loader, outputDirectory);
       for (TemplateDefinitions definition : templateDefinitions) {
         ImageSpec imageSpec = definition.buildSpecModel(true);
         imageSpec.validate();