Merge pull request #3315 from nexB/package_key_files

Add `get_top_level_resources()` to `DatafileHandler` class

Author: JonoYang

CHANGELOG.rst

@@ -69,6 +69,11 @@ Package detection:
 
   https://github.com/nexB/scancode-toolkit/issues/3290
 
+- DatafileHandlers now have a classmethod named ``get_top_level_resources()``,
+  which is supposed to yield the top-level Resources of a Package codebase,
+  relative to a Package manifest file. ``maven.MavenPomXmlHandler`` is the first
+  DatafileHandler that has this method implemented.
+
 
 License detection:
 ~~~~~~~~~~~~~~~~~~~
@@ -161,10 +166,10 @@ License detection:
 
   - We can now detect licenses using custom license texts and license rules
     stored in a directory or packaged as a plugin for consistent reuse and deployment.
-  
+
   - There is an ``--additional-directory`` option with the ``scancode-reindex-licenses``
     command to add the licenses from a directory.
-    
+
   - There is also a ``--only-builtin`` option to use ony builtin licenses
     ignoring any additional license plugins.
 
@@ -218,7 +223,7 @@ v31.2.5 - 2023-01-09
 
 This is a minor fix backport release.
 
-This adds license rule changes and was requested here: 
+This adds license rule changes and was requested here:
 https://github.com/nexB/scancode-toolkit/issues/3310
 This was originally merged in #3218 and included in
 the latest release v32.x, and is also being backported
@@ -612,6 +617,10 @@ Changes:
     - `other_holders`
     - `other_languages`
 
+- A new field ``run_order`` has been added to ``BasePlugin`` and set on all
+  ScanCode plugins. Plugin run order and output order are now set independently
+  of one another.
+
 
 Documentation Update
 ~~~~~~~~~~~~~~~~~~~~~~~~

requirements.txt

@@ -48,7 +48,7 @@ pefile==2022.5.30
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
 pluggy==1.0.0
-plugincode==31.0.0
+plugincode==32.0.0
 ply==3.11
 publicsuffix2==2.20191221
 pyahocorasick==2.0.0

setup.cfg

@@ -97,7 +97,7 @@ install_requires =
     pkginfo2 >= 30.0.0
     pip-requirements-parser >= 32.0.1
     pluggy >= 1.0.0
-    plugincode >= 31.0.0
+    plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
     pygmars >= 0.7.0
@@ -165,7 +165,6 @@ console_scripts =
 scancode_pre_scan =
     ignore = scancode.plugin_ignore:ProcessIgnore
     facet = summarycode.facet:AddFacet
-    classify = summarycode.classify_plugin:FileClassifier
 
 
 # scancode_scan is the entry point for scan plugins that run a scan after the
@@ -196,6 +195,7 @@ scancode_post_scan =
     filter-clues = cluecode.plugin_filter_clues:RedundantCluesFilter
     consolidate = summarycode.plugin_consolidate:Consolidator
     license-references = licensedcode.licenses_reference:LicenseReference
+    classify = summarycode.classify_plugin:FileClassifier
 
 
 # scancode_output_filter is the entry point for filter plugins executed after

src/cluecode/plugin_copyright.py

@@ -28,6 +28,7 @@ class CopyrightScanner(ScanPlugin):
         ('authors', attr.ib(default=attr.Factory(list))),
     ])
 
+    run_order = 6
     sort_order = 6
 
     options = [

src/cluecode/plugin_email.py

@@ -25,6 +25,7 @@ class EmailScanner(ScanPlugin):
     """
     resource_attributes = dict(emails=attr.ib(default=attr.Factory(list)))
 
+    run_order = 7
     sort_order = 7
 
     options = [

src/cluecode/plugin_filter_clues.py

@@ -41,6 +41,7 @@ class RedundantCluesFilter(PostScanPlugin):
     Filter redundant clues (copyrights, authors, emails, and urls) that are
     already contained in a matched license text.
     """
+    run_order = 1
     sort_order = 1
 
     options = [

src/cluecode/plugin_url.py

@@ -26,6 +26,7 @@ class UrlScanner(ScanPlugin):
 
     resource_attributes = dict(urls=attr.ib(default=attr.Factory(list)))
 
+    run_order = 8
     sort_order = 8
 
     options = [

src/licensedcode/licenses_reference.py

@@ -48,6 +48,7 @@ class LicenseReference(PostScanPlugin):
     )
 
     # TODO: send to the tail of the scan, after files
+    run_order = 1000
     sort_order = 1000
 
     options = [

src/licensedcode/plugin_license.py

@@ -71,6 +71,7 @@ class LicenseScanner(ScanPlugin):
         license_detections=attr.ib(default=attr.Factory(list)),
     )
 
+    run_order = 4
     sort_order = 4
 
     options = [

src/licensedcode/plugin_license_policy.py

@@ -51,6 +51,7 @@ class LicensePolicy(PostScanPlugin):
 
     resource_attributes = dict(license_policy=attr.ib(default=attr.Factory(list)))
 
+    run_order = 9
     sort_order = 9
 
     options = [
@@ -100,8 +101,8 @@ def process_codebase(self, codebase, license_policy, **kwargs):
                     if key == policy.get('license_key'):
                         # Apply the policy to the Resource
                         license_policies.append(policy)
-            
-            resource.license_policy = sorted(license_policies, key=lambda d: d['license_key']) 
+
+            resource.license_policy = sorted(license_policies, key=lambda d: d['license_key'])
             codebase.save_resource(resource)