Update Java D2D Pipeline to Include Checksum Mapped Sources for Accurate Java Mapping (#1870)

* Update Java D2D Pipeline to Include Checksum Mapped Sources for Accurate Java Mapping #1854

Signed-off-by: Chin Yeung Li <[email protected]>

* Add test for checksum and java mapping in the same run #1854

Signed-off-by: Chin Yeung Li <[email protected]>

* Update test to use 'map_jvm_to_class' #1854

Signed-off-by: Chin Yeung Li <[email protected]>

---------

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

scanpipe/pipes/d2d.py

@@ -197,9 +197,15 @@ def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None):
     Map to/ compiled Jvm's binary files to from/ using Jvm language's fully
     qualified paths and indexing from/ Jvm lang's source files.
     """
-    project_files = project.codebaseresources.files().no_status()
+    project_files = project.codebaseresources.files()
+    # Collect all files from "from_codebase", even if they already have a
+    # status or are mapped. This is necessary because the deploy codebase
+    # may contain sources that match "from_codebase" via checksum. If those
+    # checksum-matched files are excluded from mapping, it can result in
+    # .class files failing to resolve. See
+    # https://github.com/aboutcode-org/scancode.io/issues/1854#issuecomment-3273472895
     from_resources = project_files.from_codebase()
-    to_resources = project_files.to_codebase().has_no_relation()
+    to_resources = project_files.to_codebase().no_status().has_no_relation()
 
     has_source_pkg_attr_name = {
         f"extra_data__{jvm_lang.source_package_attribute_name}__isnull": False

scanpipe/tests/pipes/test_d2d.py

@@ -393,6 +393,52 @@ def test_scanpipe_pipes_d2d_map_java_to_class(self):
         to3.refresh_from_db()
         self.assertEqual("", to3.status)
 
+    def test_scanpipe_pipes_d2d_map_java_to_class_with_java_in_deploy(self):
+        sha1 = "abcde"
+        from1 = make_resource_file(
+            self.project1,
+            path="from/flume-ng-node-1.9.0-sources.jar-extract/org/apache/flume/node/"
+            "AbstractConfigurationProvider.java",
+            extra_data={"java_package": "org.apache.flume.node"},
+            sha1=sha1,
+        )
+        to1 = make_resource_file(
+            self.project1,
+            path="to/flume-ng-node-1.9.0.jar-extract/org/apache/flume/node/"
+            "AbstractConfigurationProvider.java",
+            sha1=sha1,
+        )
+        to2 = make_resource_file(
+            self.project1,
+            path="to/flume-ng-node-1.9.0.jar-extract/org/apache/flume/node/"
+            "AbstractConfigurationProvider.class",
+        )
+
+        buffer = io.StringIO()
+
+        # The pipeline will run map_checksum first
+        d2d.map_checksum(self.project1, "sha1", logger=buffer.write)
+        expected = "Mapping 1 to/ resources using sha1 against from/ codebase"
+        self.assertEqual(expected, buffer.getvalue())
+        self.assertEqual(1, to1.related_from.count())
+        relation1 = to1.related_from.get()
+        self.assertEqual("sha1", relation1.map_type)
+        self.assertEqual(from1, relation1.from_resource)
+
+        # Now run map_java_to_class
+        d2d.map_jvm_to_class(
+            self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage
+        )
+        expected = "Mapping 1 .class resources to 1 ('.java',)"
+        self.assertIn(expected, buffer.getvalue())
+        self.assertEqual(2, self.project1.codebaserelations.count())
+        relation2 = self.project1.codebaserelations.get(
+            to_resource=to2, from_resource=from1
+        )
+        self.assertEqual("java_to_class", relation2.map_type)
+        expected = {"from_source_root": "from/flume-ng-node-1.9.0-sources.jar-extract/"}
+        self.assertEqual(expected, relation2.extra_data)
+
     def test_scanpipe_pipes_d2d_map_java_to_class_no_java(self):
         make_resource_file(self.project1, path="to/Abstract.class")
         buffer = io.StringIO()