Skip to content

Commit aff08bf

Browse files
authored
Added "META-INF" and "module-info.class" to ignore list (#1943)
* Added "META-INF" and "module-info.class" to ignore list #1876 - Everything under META-INF/ is generally metadata or configuration files that can be ignored. - module-info.class is typically a generated file - it's the compiled version of a Java module descriptor that can be ignored as well. Signed-off-by: Chin Yeung Li <[email protected]> * Fixed failing tests #1876 - Added scan_ignored_to_files() - Updated tests Signed-off-by: Chin Yeung Li <[email protected]> * Update test function to be consistent with the existing #1876 Signed-off-by: Chin Yeung Li <[email protected]> --------- Signed-off-by: Chin Yeung Li <[email protected]>
1 parent cb0c6e6 commit aff08bf

File tree

7 files changed

+75
-7
lines changed

7 files changed

+75
-7
lines changed

scanpipe/pipelines/deploy_to_develop.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ def steps(cls):
104104
cls.perform_house_keeping_tasks,
105105
cls.match_purldb_resources_post_process,
106106
cls.remove_packages_without_resources,
107+
cls.scan_ignored_to_files,
107108
cls.scan_unmapped_to_files,
108109
cls.scan_mapped_from_for_files,
109110
cls.collect_and_create_license_detections,
@@ -411,6 +412,16 @@ def remove_packages_without_resources(self):
411412
)
412413
package_without_resources.delete()
413414

415+
def scan_ignored_to_files(self):
416+
"""
417+
Scan status="ignored-from-config" ``to/`` files for copyrights,
418+
licenses, emails, and urls. These files are ignored based on
419+
ecosystem specific configurations. These files are not used for the
420+
D2D purpose, but scanning them may provide useful information about
421+
the deployed codebase.
422+
"""
423+
d2d.scan_ignored_to_files(project=self.project, logger=self.log)
424+
414425
def scan_unmapped_to_files(self):
415426
"""
416427
Scan unmapped/matched ``to/`` files for copyrights, licenses,

scanpipe/pipes/d2d.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1507,6 +1507,23 @@ def flag_undeployed_resources(project):
15071507
from_unmapped.update(status=flag.NOT_DEPLOYED)
15081508

15091509

1510+
def scan_ignored_to_files(project, logger=None):
1511+
"""
1512+
Scan status="ignored-from-config" ``to/`` files for copyrights, licenses,
1513+
emails, and urls.
1514+
"""
1515+
scan_files = (
1516+
project.codebaseresources.files()
1517+
.to_codebase()
1518+
.filter(status=flag.IGNORED_FROM_CONFIG)
1519+
)
1520+
scancode.scan_for_files(project, scan_files, progress_logger=logger)
1521+
1522+
project.codebaseresources.files().to_codebase().filter(status=flag.SCANNED).update(
1523+
status=flag.IGNORED_FROM_CONFIG
1524+
)
1525+
1526+
15101527
def scan_unmapped_to_files(project, logger=None):
15111528
"""
15121529
Scan unmapped/matched ``to/`` files for copyrights, licenses,

scanpipe/pipes/d2d_config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ class EcosystemConfig:
8585
ecosystem_option="Java",
8686
matchable_package_extensions=[".jar", ".war"],
8787
matchable_resource_extensions=[".class"],
88+
deployed_resource_path_exclusions=["*META-INF/*", "*/module-info.class"],
8889
),
8990
"Scala": EcosystemConfig(
9091
ecosystem_option="Scala",

scanpipe/tests/data/d2d/config/ecosystem_config.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
"matchable_package_extensions": [".jar", ".war", ".gem", ".zip", ".tar.gz", ".tar.xz"],
44
"matchable_resource_extensions": [".map", ".js", ".mjs", ".ts", ".d.ts", ".jsx", ".tsx", ".css", ".scss", ".less", ".sass", ".soy",".class", ".rb"],
55
"doc_extensions": [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".tex", ".odt", ".odp"],
6-
"deployed_resource_path_exclusions": ["*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig"],
6+
"deployed_resource_path_exclusions": ["*META-INF/*", "*/module-info.class", "*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig"],
77
"devel_resource_path_exclusions": ["*/tests/*"],
88
"standard_symbols_to_exclude": [],
99
"source_symbol_extensions": []
10-
}
10+
}

scanpipe/tests/data/d2d/flume-ng-node-d2d.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1545,7 +1545,7 @@
15451545
"path": "to/flume-ng-node-1.9.0.jar-extract/META-INF/DEPENDENCIES",
15461546
"type": "file",
15471547
"name": "DEPENDENCIES",
1548-
"status": "requires-review",
1548+
"status": "ignored-from-config",
15491549
"for_packages": [],
15501550
"tag": "to",
15511551
"extension": "",
@@ -2885,7 +2885,7 @@
28852885
"path": "to/flume-ng-node-1.9.0.jar-extract/META-INF/MANIFEST.MF",
28862886
"type": "file",
28872887
"name": "MANIFEST.MF",
2888-
"status": "requires-review",
2888+
"status": "ignored-from-config",
28892889
"for_packages": [],
28902890
"tag": "to",
28912891
"extension": ".MF",
@@ -4014,4 +4014,4 @@
40144014
"from_resource": "from/flume-ng-node-1.9.0-sources.jar-extract/org/apache/flume/node/StaticZooKeeperConfigurationProvider.java"
40154015
}
40164016
]
4017-
}
4017+
}

scanpipe/tests/pipes/test_d2d.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,22 @@ def test_scanpipe_pipes_d2d_map_java_to_class_no_java(self):
470470
expected = "No ('.java',) resources to map."
471471
self.assertIn(expected, buffer.getvalue())
472472

473+
def test_scanpipe_pipes_d2d_java_ignore_pattern(self):
474+
make_resource_file(self.project1, path="to/module-info.class")
475+
make_resource_file(self.project1, path="to/META-INF/MANIFEST.MF")
476+
make_resource_file(self.project1, path="to/test.class")
477+
make_resource_file(self.project1, path="to/META-INF/others.txt")
478+
buffer = io.StringIO()
479+
480+
java_config = d2d_config.get_ecosystem_config(ecosystem="Java")
481+
d2d.ignore_unmapped_resources_from_config(
482+
project=self.project1,
483+
patterns_to_ignore=java_config.deployed_resource_path_exclusions,
484+
logger=buffer.write,
485+
)
486+
expected = "Ignoring 3 to/ resources with ecosystem specific configurations."
487+
self.assertIn(expected, buffer.getvalue())
488+
473489
def test_scanpipe_pipes_d2d_map_jar_to_java_source(self):
474490
from1 = make_resource_file(
475491
self.project1,
@@ -1311,6 +1327,30 @@ def test_flag_undeployed_resources(self):
13111327

13121328
self.assertEqual(1, expected)
13131329

1330+
def test_scanpipe_pipes_d2d_scan_ignored_to_files(self):
1331+
to_dir = (
1332+
self.project1.codebase_path / "to/project.tar.zst-extract/META-INF/foo-bar"
1333+
)
1334+
to_input_location = self.data / "d2d/find_java_packages/Foo.java"
1335+
to_dir.mkdir(parents=True)
1336+
copy_input(to_input_location, to_dir)
1337+
1338+
pipes.collect_and_create_codebase_resources(self.project1)
1339+
1340+
foo_java = self.project1.codebaseresources.get(
1341+
path=("to/project.tar.zst-extract/META-INF/foo-bar/Foo.java")
1342+
)
1343+
foo_java.update(status=flag.IGNORED_FROM_CONFIG)
1344+
1345+
d2d.scan_ignored_to_files(self.project1)
1346+
foo_java.refresh_from_db()
1347+
1348+
expected = self.project1.codebaseresources.filter(
1349+
status=flag.IGNORED_FROM_CONFIG
1350+
).count()
1351+
1352+
self.assertEqual(1, expected)
1353+
13141354
def test_scan_unmapped_to_files(self):
13151355
to_dir = (
13161356
self.project1.codebase_path / "to/project.tar.zst-extract/osgi/marketplace/"

scanpipe/tests/test_pipelines.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -693,9 +693,8 @@ def assertPipelineResultEqual(
693693
expected_json = self._normalize_package_uids(expected_json)
694694
expected_data = self._without_keys(expected_json, self.exclude_from_diff)
695695
if sort_dependencies:
696-
result_data = self._sort_dependencies(result_data)
696+
expected_data = self._sort_dependencies(expected_data)
697697
expected_data = sort_for_os_compatibility(expected_data)
698-
699698
self.assertEqual(expected_data, result_data)
700699

701700
@skipIf(from_docker_image, "Random failure in the Docker context.")

0 commit comments

Comments
 (0)