Added "META-INF" and "module-info.class" to ignore list (#1943)

* Added "META-INF" and "module-info.class" to ignore list #1876

- Everything under META-INF/ is generally metadata or configuration files that can be ignored.
- module-info.class is typically a generated file - it's the compiled version of a Java module descriptor that can be ignored as well.

Signed-off-by: Chin Yeung Li <[email protected]>

* Fixed failing tests #1876

- Added scan_ignored_to_files()
- Updated tests

Signed-off-by: Chin Yeung Li <[email protected]>

* Update test function to be consistent with the existing #1876

Signed-off-by: Chin Yeung Li <[email protected]>

---------

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

scanpipe/pipelines/deploy_to_develop.py

@@ -104,6 +104,7 @@ def steps(cls):
             cls.perform_house_keeping_tasks,
             cls.match_purldb_resources_post_process,
             cls.remove_packages_without_resources,
+            cls.scan_ignored_to_files,
             cls.scan_unmapped_to_files,
             cls.scan_mapped_from_for_files,
             cls.collect_and_create_license_detections,
@@ -411,6 +412,16 @@ def remove_packages_without_resources(self):
         )
         package_without_resources.delete()
 
+    def scan_ignored_to_files(self):
+        """
+        Scan status="ignored-from-config" ``to/`` files for copyrights,
+        licenses, emails, and urls. These files are ignored based on
+        ecosystem specific configurations. These files are not used for the
+        D2D purpose, but scanning them may provide useful information about
+        the deployed codebase.
+        """
+        d2d.scan_ignored_to_files(project=self.project, logger=self.log)
+
     def scan_unmapped_to_files(self):
         """
         Scan unmapped/matched ``to/`` files for copyrights, licenses,

scanpipe/pipes/d2d.py

@@ -1507,6 +1507,23 @@ def flag_undeployed_resources(project):
     from_unmapped.update(status=flag.NOT_DEPLOYED)
 
 
+def scan_ignored_to_files(project, logger=None):
+    """
+    Scan status="ignored-from-config" ``to/`` files for copyrights, licenses,
+    emails, and urls.
+    """
+    scan_files = (
+        project.codebaseresources.files()
+        .to_codebase()
+        .filter(status=flag.IGNORED_FROM_CONFIG)
+    )
+    scancode.scan_for_files(project, scan_files, progress_logger=logger)
+
+    project.codebaseresources.files().to_codebase().filter(status=flag.SCANNED).update(
+        status=flag.IGNORED_FROM_CONFIG
+    )
+
+
 def scan_unmapped_to_files(project, logger=None):
     """
     Scan unmapped/matched ``to/`` files for copyrights, licenses,

scanpipe/pipes/d2d_config.py

@@ -85,6 +85,7 @@ class EcosystemConfig:
         ecosystem_option="Java",
         matchable_package_extensions=[".jar", ".war"],
         matchable_resource_extensions=[".class"],
+        deployed_resource_path_exclusions=["*META-INF/*", "*/module-info.class"],
     ),
     "Scala": EcosystemConfig(
         ecosystem_option="Scala",

scanpipe/tests/data/d2d/config/ecosystem_config.json

@@ -3,8 +3,8 @@
     "matchable_package_extensions": [".jar", ".war", ".gem", ".zip", ".tar.gz", ".tar.xz"],
     "matchable_resource_extensions": [".map", ".js", ".mjs", ".ts", ".d.ts", ".jsx", ".tsx", ".css", ".scss", ".less", ".sass", ".soy",".class", ".rb"],
     "doc_extensions": [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".tex", ".odt", ".odp"],
-    "deployed_resource_path_exclusions": ["*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig"],
+    "deployed_resource_path_exclusions": ["*META-INF/*", "*/module-info.class", "*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig"],
     "devel_resource_path_exclusions": ["*/tests/*"],
     "standard_symbols_to_exclude": [],
     "source_symbol_extensions": []
-}
+}

scanpipe/tests/data/d2d/flume-ng-node-d2d.json

@@ -1545,7 +1545,7 @@
       "path": "to/flume-ng-node-1.9.0.jar-extract/META-INF/DEPENDENCIES",
       "type": "file",
       "name": "DEPENDENCIES",
-      "status": "requires-review",
+      "status": "ignored-from-config",
       "for_packages": [],
       "tag": "to",
       "extension": "",
@@ -2885,7 +2885,7 @@
       "path": "to/flume-ng-node-1.9.0.jar-extract/META-INF/MANIFEST.MF",
       "type": "file",
       "name": "MANIFEST.MF",
-      "status": "requires-review",
+      "status": "ignored-from-config",
       "for_packages": [],
       "tag": "to",
       "extension": ".MF",
@@ -4014,4 +4014,4 @@
       "from_resource": "from/flume-ng-node-1.9.0-sources.jar-extract/org/apache/flume/node/StaticZooKeeperConfigurationProvider.java"
     }
   ]
-}
+}

scanpipe/tests/pipes/test_d2d.py

@@ -470,6 +470,22 @@ def test_scanpipe_pipes_d2d_map_java_to_class_no_java(self):
         expected = "No ('.java',) resources to map."
         self.assertIn(expected, buffer.getvalue())
 
+    def test_scanpipe_pipes_d2d_java_ignore_pattern(self):
+        make_resource_file(self.project1, path="to/module-info.class")
+        make_resource_file(self.project1, path="to/META-INF/MANIFEST.MF")
+        make_resource_file(self.project1, path="to/test.class")
+        make_resource_file(self.project1, path="to/META-INF/others.txt")
+        buffer = io.StringIO()
+
+        java_config = d2d_config.get_ecosystem_config(ecosystem="Java")
+        d2d.ignore_unmapped_resources_from_config(
+            project=self.project1,
+            patterns_to_ignore=java_config.deployed_resource_path_exclusions,
+            logger=buffer.write,
+        )
+        expected = "Ignoring 3 to/ resources with ecosystem specific configurations."
+        self.assertIn(expected, buffer.getvalue())
+
     def test_scanpipe_pipes_d2d_map_jar_to_java_source(self):
         from1 = make_resource_file(
             self.project1,
@@ -1311,6 +1327,30 @@ def test_flag_undeployed_resources(self):
 
         self.assertEqual(1, expected)
 
+    def test_scanpipe_pipes_d2d_scan_ignored_to_files(self):
+        to_dir = (
+            self.project1.codebase_path / "to/project.tar.zst-extract/META-INF/foo-bar"
+        )
+        to_input_location = self.data / "d2d/find_java_packages/Foo.java"
+        to_dir.mkdir(parents=True)
+        copy_input(to_input_location, to_dir)
+
+        pipes.collect_and_create_codebase_resources(self.project1)
+
+        foo_java = self.project1.codebaseresources.get(
+            path=("to/project.tar.zst-extract/META-INF/foo-bar/Foo.java")
+        )
+        foo_java.update(status=flag.IGNORED_FROM_CONFIG)
+
+        d2d.scan_ignored_to_files(self.project1)
+        foo_java.refresh_from_db()
+
+        expected = self.project1.codebaseresources.filter(
+            status=flag.IGNORED_FROM_CONFIG
+        ).count()
+
+        self.assertEqual(1, expected)
+
     def test_scan_unmapped_to_files(self):
         to_dir = (
             self.project1.codebase_path / "to/project.tar.zst-extract/osgi/marketplace/"

scanpipe/tests/test_pipelines.py

@@ -693,9 +693,8 @@ def assertPipelineResultEqual(
         expected_json = self._normalize_package_uids(expected_json)
         expected_data = self._without_keys(expected_json, self.exclude_from_diff)
         if sort_dependencies:
-            result_data = self._sort_dependencies(result_data)
+            expected_data = self._sort_dependencies(expected_data)
         expected_data = sort_for_os_compatibility(expected_data)
-
         self.assertEqual(expected_data, result_data)
 
     @skipIf(from_docker_image, "Random failure in the Docker context.")