feat: Improve class file mapping #2021

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

scanpipe/pipelines/deploy_to_develop.py

@@ -83,6 +83,15 @@ def steps(cls):
             cls.find_grammar_packages,
             cls.map_grammar_to_class,
             cls.map_jar_to_grammar_source,
+            cls.find_groovy_packages,
+            cls.map_groovy_to_class,
+            cls.map_jar_to_groovy_source,
+            cls.find_aspectj_packages,
+            cls.map_aspectj_to_class,
+            cls.map_jar_to_aspectj_source,
+            cls.find_clojure_packages,
+            cls.map_clojure_to_class,
+            cls.map_jar_to_clojure_source,
             cls.find_xtend_packages,
             cls.map_xtend_to_class,
             cls.map_javascript,
@@ -208,7 +217,7 @@ def find_scala_packages(self):
 
     @optional_step("Scala")
     def map_scala_to_class(self):
-        """Map a .class compiled file to its .java source."""
+        """Map a .class compiled file to its .scala source."""
         d2d.map_jvm_to_class(
             project=self.project, jvm_lang=jvm.ScalaLanguage, logger=self.log
         )
@@ -222,14 +231,14 @@ def map_jar_to_scala_source(self):
 
     @optional_step("Kotlin")
     def find_kotlin_packages(self):
-        """Find the java package of the .java source files."""
+        """Find the java package of the kotlin source files."""
         d2d.find_jvm_packages(
             project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log
         )
 
     @optional_step("Kotlin")
     def map_kotlin_to_class(self):
-        """Map a .class compiled file to its .java source."""
+        """Map a .class compiled file to its kotlin source."""
         d2d.map_jvm_to_class(
             project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log
         )
@@ -262,6 +271,69 @@ def map_jar_to_grammar_source(self):
             project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log
         )
 
+    @optional_step("Groovy")
+    def find_groovy_packages(self):
+        """Find the package of the .groovy source files."""
+        d2d.find_jvm_packages(
+            project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log
+        )
+
+    @optional_step("Groovy")
+    def map_groovy_to_class(self):
+        """Map a .class compiled file to its .groovy source."""
+        d2d.map_jvm_to_class(
+            project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log
+        )
+
+    @optional_step("Groovy")
+    def map_jar_to_groovy_source(self):
+        """Map .jar files to their related source directory."""
+        d2d.map_jar_to_jvm_source(
+            project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log
+        )
+
+    @optional_step("AspectJ")
+    def find_aspectj_packages(self):
+        """Find the package of the .aj source files."""
+        d2d.find_jvm_packages(
+            project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log
+        )
+
+    @optional_step("AspectJ")
+    def map_aspectj_to_class(self):
+        """Map a .class compiled file to its .aj source."""
+        d2d.map_jvm_to_class(
+            project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log
+        )
+
+    @optional_step("AspectJ")
+    def map_jar_to_aspectj_source(self):
+        """Map .jar files to their related source directory."""
+        d2d.map_jar_to_jvm_source(
+            project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log
+        )
+
+    @optional_step("Clojure")
+    def find_clojure_packages(self):
+        """Find the package of the .clj source files."""
+        d2d.find_jvm_packages(
+            project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log
+        )
+
+    @optional_step("Clojure")
+    def map_clojure_to_class(self):
+        """Map a .class compiled file to its .clj source."""
+        d2d.map_jvm_to_class(
+            project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log
+        )
+
+    @optional_step("Clojure")
+    def map_jar_to_clojure_source(self):
+        """Map .jar files to their related source directory."""
+        d2d.map_jar_to_jvm_source(
+            project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log
+        )
+
     @optional_step("Xtend")
     def find_xtend_packages(self):
         """Find the java package of the xtend source files."""

scanpipe/pipes/d2d.py

@@ -239,8 +239,8 @@ def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None):
 
     if logger:
         logger(
-            f"Mapping {to_resource_count:,d} .class resources to "
-            f"{from_resource_count:,d} {jvm_lang.source_extensions}"
+            f"Mapping {to_resource_count:,d} .class (or other deployed file) "
+            f"resources to {from_resource_count:,d} {jvm_lang.source_extensions}"
         )
 
     # build an index using from-side fully qualified class file names

scanpipe/pipes/d2d_config.py

@@ -112,6 +112,14 @@ class EcosystemConfig:
             "*kotlin-project-structure-metadata.json",
         ],
     ),
+    "Groovy": EcosystemConfig(
+        ecosystem_option="Groovy",
+        matchable_package_extensions=[".jar", ".war"],
+        matchable_resource_extensions=[".class"],
+        deployed_resource_path_exclusions=[
+            "*META-INF/*",
+        ],
+    ),
     "JavaScript": EcosystemConfig(
         ecosystem_option="JavaScript",
         matchable_resource_extensions=[

scanpipe/pipes/jvm.py

@@ -192,12 +192,39 @@ class JavaLanguage(JvmLanguage):
 class ScalaLanguage(JvmLanguage):
     name = "scala"
     source_extensions = (".scala",)
-    binary_extensions = (".class",)
+    binary_extensions = (".class", ".tasty")
     source_package_attribute_name = "scala_package"
     package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
     binary_map_type = "scala_to_class"
 
 
+class GroovyLanguage(JvmLanguage):
+    name = "groovy"
+    source_extensions = (".groovy",)
+    binary_extensions = (".class",)
+    source_package_attribute_name = "groovy_package"
+    package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
+    binary_map_type = "groovy_to_class"
+
+
+class AspectJLanguage(JvmLanguage):
+    name = "aspectj"
+    source_extensions = (".aj",)
+    binary_extensions = (".class",)
+    source_package_attribute_name = "aspectj_package"
+    package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
+    binary_map_type = "aspectj_to_class"
+
+
+class ClojureLanguage(JvmLanguage):
+    name = "clojure"
+    source_extensions = (".clj",)
+    binary_extensions = (".class",)
+    source_package_attribute_name = "clojure_package"
+    package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
+    binary_map_type = "clojure_to_class"
+
+
 class KotlinLanguage(JvmLanguage):
     name = "kotlin"
     source_extensions = (".kt", ".kts")

scanpipe/tests/pipes/test_d2d.py

@@ -372,7 +372,7 @@ def test_scanpipe_pipes_d2d_map_java_to_class(self):
             self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage
         )
 
-        expected = "Mapping 3 .class resources to 2 ('.java',)"
+        expected = "Mapping 3 .class (or other deployed file) resources to 2 ('.java',)"
         self.assertIn(expected, buffer.getvalue())
 
         self.assertEqual(2, self.project1.codebaserelations.count())
@@ -432,7 +432,7 @@ def test_scanpipe_pipes_d2d_map_java_to_class_with_java_in_deploy(self):
         d2d.map_jvm_to_class(
             self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage
         )
-        expected = "Mapping 1 .class resources to 1 ('.java',)"
+        expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.java',)"
         self.assertIn(expected, buffer.getvalue())
 
     def test_scanpipe_pipes_d2d_map_grammar_to_class(self):
@@ -452,7 +452,9 @@ def test_scanpipe_pipes_d2d_map_grammar_to_class(self):
             self.project1, logger=buffer.write, jvm_lang=jvm.GrammarLanguage
         )
 
-        expected = "Mapping 1 .class resources to 1 ('.g', '.g4')"
+        expected = (
+            "Mapping 1 .class (or other deployed file) resources to 1 ('.g', '.g4')"
+        )
         self.assertIn(expected, buffer.getvalue())
         self.assertEqual(1, self.project1.codebaserelations.count())
 
@@ -480,7 +482,9 @@ def test_scanpipe_pipes_d2d_map_xtend_to_class(self):
             self.project1, logger=buffer.write, jvm_lang=jvm.XtendLanguage
         )
 
-        expected = "Mapping 1 .class resources to 1 ('.xtend',)"
+        expected = (
+            "Mapping 1 .class (or other deployed file) resources to 1 ('.xtend',)"
+        )
         self.assertIn(expected, buffer.getvalue())
         self.assertEqual(1, self.project1.codebaserelations.count())
 
@@ -569,6 +573,124 @@ def test_scanpipe_pipes_d2d_map_jar_to_java_source(self):
         self.assertEqual(from2, relation.from_resource)
         self.assertEqual(to_jar, relation.to_resource)
 
+    def test_scanpipe_pipes_d2d_map_groovy_to_class(self):
+        from1 = make_resource_file(
+            self.project1,
+            path="from/project/test.groovy",
+            extra_data={"groovy_package": "project"},
+        )
+
+        to1 = make_resource_file(
+            self.project1,
+            path="to/project/test.class",
+        )
+
+        buffer = io.StringIO()
+        d2d.map_jvm_to_class(
+            self.project1, logger=buffer.write, jvm_lang=jvm.GroovyLanguage
+        )
+
+        expected = (
+            "Mapping 1 .class (or other deployed file) resources to 1 ('.groovy',)"
+        )
+        self.assertIn(expected, buffer.getvalue())
+        self.assertEqual(1, self.project1.codebaserelations.count())
+
+        r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1)
+        self.assertEqual("groovy_to_class", r1.map_type)
+        expected = {"from_source_root": "from/"}
+        self.assertEqual(expected, r1.extra_data)
+
+    def test_scanpipe_pipes_d2d_map_aspectj_to_class(self):
+        from1 = make_resource_file(
+            self.project1,
+            path="from/project/test.aj",
+            extra_data={"aspectj_package": "project"},
+        )
+
+        to1 = make_resource_file(
+            self.project1,
+            path="to/project/test.class",
+        )
+
+        buffer = io.StringIO()
+        d2d.map_jvm_to_class(
+            self.project1, logger=buffer.write, jvm_lang=jvm.AspectJLanguage
+        )
+
+        expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.aj',)"
+        self.assertIn(expected, buffer.getvalue())
+        self.assertEqual(1, self.project1.codebaserelations.count())
+
+        r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1)
+        self.assertEqual("aspectj_to_class", r1.map_type)
+        expected = {"from_source_root": "from/"}
+        self.assertEqual(expected, r1.extra_data)
+
+    def test_scanpipe_pipes_d2d_map_clojure_to_class(self):
+        from1 = make_resource_file(
+            self.project1,
+            path="from/project/test.clj",
+            extra_data={"clojure_package": "project"},
+        )
+
+        to1 = make_resource_file(
+            self.project1,
+            path="to/project/test.class",
+        )
+
+        buffer = io.StringIO()
+        d2d.map_jvm_to_class(
+            self.project1, logger=buffer.write, jvm_lang=jvm.ClojureLanguage
+        )
+
+        expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.clj',)"
+        self.assertIn(expected, buffer.getvalue())
+        self.assertEqual(1, self.project1.codebaserelations.count())
+
+        r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1)
+        self.assertEqual("clojure_to_class", r1.map_type)
+        expected = {"from_source_root": "from/"}
+        self.assertEqual(expected, r1.extra_data)
+
+    def test_scanpipe_pipes_d2d_map_scala_to_class(self):
+        from1 = make_resource_file(
+            self.project1,
+            path="from/tastyquery/Annotations.scala",
+            extra_data={"scala_package": "tastyquery"},
+        )
+
+        to1 = make_resource_file(
+            self.project1,
+            path="to/tastyquery/Annotations.tasty",
+        )
+
+        to2 = make_resource_file(
+            self.project1,
+            path="to/tastyquery/Annotations.class",
+        )
+
+        buffer = io.StringIO()
+        d2d.map_jvm_to_class(
+            self.project1, logger=buffer.write, jvm_lang=jvm.ScalaLanguage
+        )
+
+        expected = (
+            "Mapping 2 .class (or other deployed file) resources to 1 ('.scala',)"
+        )
+        self.assertIn(expected, buffer.getvalue())
+        self.assertEqual(2, self.project1.codebaserelations.count())
+
+        r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1)
+        self.assertEqual("scala_to_class", r1.map_type)
+        expected = {"from_source_root": "from/"}
+        self.assertEqual(expected, r1.extra_data)
+
+        r2 = self.project1.codebaserelations.get(to_resource=to2, from_resource=from1)
+        self.assertEqual("scala_to_class", r2.map_type)
+        expected = {"from_source_root": "from/"}
+        self.assertEqual(expected, r2.extra_data)
+
     def test_scanpipe_pipes_d2d_map_jar_to_scala_source(self):
         from1 = make_resource_file(
             self.project1,