Split packages from package_manifest #1027

The packages scan return now a single package_manifest key (not a list)
And a post_scan plugin (responding to the same --package) option perform
a roll-up of the manifest informationat the proper level for a package
type as the "packages" attribute (which is still a list). For instance
a package.json "package_manifest" will end up having a "packages" entry
in its parent directory.

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

etc/scripts/testdata/livescan/expected.csv

@@ -1,20 +1,19 @@
-Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,files_count,dirs_count,size_count,scan_errors,license__key,license__score,license__short_name,license__category,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_choice,matched_rule__licenses,copyright,copyright_holder,email,url,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__code_type,package__description,package__size,package__release_date,package__parties,package__homepage_url,package__download_url,package__bug_tracking_url,package__vcs_repository,package__copyright,package__declared_licensing,package__notice_text
-/json2csv.rb,file,json2csv.rb,json2csv,.rb,1599,2018-04-11,6cfb0bd0fb0b784f57164d15bdfca2b734ad87a6,f18e519b77bc7f3e4213215033db3857,text/x-python,"Python script, ASCII text executable",Ruby,False,True,False,False,True,True,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,apache-2.0,98.45,Apache 2.0,Permissive,Apache Software Foundation,http://www.apache.org/licenses/,http://www.apache.org/licenses/LICENSE-2.0,https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0,Apache-2.0,https://spdx.org/licenses/Apache-2.0,5,24,apache-2.0_scancode.RULE,False,"[u'apache-2.0', u'scancode-acknowledgment']",,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,scancode-acknowledgment,98.45,ScanCode acknowledgment,Permissive,nexB,https://github.com/nexB/scancode-toolkit/,,https://enterprise.dejacode.com/urn/urn:dje:license:scancode-acknowledgment,,,5,24,apache-2.0_scancode.RULE,False,"[u'apache-2.0', u'scancode-acknowledgment']",,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,Copyright (c) 2017 nexB Inc. and others.,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,,nexB Inc. and others.,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,http://nexb.com/,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,https://github.com/nexB/scancode-toolkit/,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10,,,,,,,http://apache.org/licenses/LICENSE-2.0,,,,,,,,,,,,,,,,,,,
-/license,file,license,license,,679,2018-04-11,75c5490a718ddd45e40e0cc7ce0c756abc373123,b965a762efb9421cf1bf4405f336e278,text/plain,ASCII text,,False,True,False,False,False,False,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/license,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,100.00,GPL 2.0 or later,Copyleft,Free Software Foundation (FSF),http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0-plus,GPL-2.0+,https://spdx.org/licenses/GPL-2.0,1,12,gpl-2.0-plus.LICENSE,False,[u'gpl-2.0-plus'],,,,,,,,,,,,,,,,,,,,,,,
-/package.json,file,package.json,package,.json,2200,2018-04-11,918376afce796ef90eeda1d6695f2289c90491ac,1f66239a9b850c5e60a9382dbe2162d2,text/plain,"ASCII text, with very long lines",JSON,False,True,False,False,True,False,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,mit,15.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit_27.RULE,False,[u'mit'],,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,mit,100.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit.LICENSE,False,[u'mit'],,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,23,26,,,,Copyright (c) 2012 LearnBoost <[email protected]>,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,23,26,,,,,LearnBoost,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,12,,,,,,[email protected],,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,16,16,,,,,,,https://github.com/visionmedia/node-cookie-signature.git,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,27,27,,,,,,,https://github.com/visionmedia/node-cookie-signature/issues,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,TJ Holowaychuk,,,,,,,
+Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,package_manifest,files_count,dirs_count,size_count,scan_errors,license__key,license__score,license__short_name,license__category,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_choice,matched_rule__licenses,copyright,copyright_holder,email,url
+/json2csv.rb,file,json2csv.rb,json2csv,.rb,1599,2018-04-11,6cfb0bd0fb0b784f57164d15bdfca2b734ad87a6,f18e519b77bc7f3e4213215033db3857,text/x-python,"Python script, ASCII text executable",Ruby,False,True,False,False,True,True,,0,0,0,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,apache-2.0,98.45,Apache 2.0,Permissive,Apache Software Foundation,http://www.apache.org/licenses/,http://www.apache.org/licenses/LICENSE-2.0,https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0,Apache-2.0,https://spdx.org/licenses/Apache-2.0,5,24,apache-2.0_scancode.RULE,False,"[u'apache-2.0', u'scancode-acknowledgment']",,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,scancode-acknowledgment,98.45,ScanCode acknowledgment,Permissive,nexB,https://github.com/nexB/scancode-toolkit/,,https://enterprise.dejacode.com/urn/urn:dje:license:scancode-acknowledgment,,,5,24,apache-2.0_scancode.RULE,False,"[u'apache-2.0', u'scancode-acknowledgment']",,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,Copyright (c) 2017 nexB Inc. and others.,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,,nexB Inc. and others.,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,http://nexb.com/
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,https://github.com/nexB/scancode-toolkit/
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10,,,,,,,http://apache.org/licenses/LICENSE-2.0
+/license,file,license,license,,679,2018-04-11,75c5490a718ddd45e40e0cc7ce0c756abc373123,b965a762efb9421cf1bf4405f336e278,text/plain,ASCII text,,False,True,False,False,False,False,,0,0,0,,,,,,,,,,,,,,,,,,,,
+/license,,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,100.00,GPL 2.0 or later,Copyleft,Free Software Foundation (FSF),http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0-plus,GPL-2.0+,https://spdx.org/licenses/GPL-2.0,1,12,gpl-2.0-plus.LICENSE,False,[u'gpl-2.0-plus'],,,,
+/package.json,file,package.json,package,.json,2200,2018-04-11,918376afce796ef90eeda1d6695f2289c90491ac,1f66239a9b850c5e60a9382dbe2162d2,text/plain,"ASCII text, with very long lines",JSON,False,True,False,False,True,False,"OrderedDict([(u'type', u'npm'), (u'namespace', None), (u'name', u'cookie-signature'), (u'version', u'1.0.3'), (u'qualifiers', None), (u'subpath', None), (u'primary_language', u'JavaScript'), (u'code_type', None), (u'description', u'Sign and unsign cookies'), (u'size', None), (u'release_date', None), (u'parties', [OrderedDict([(u'type', u'person'), (u'role', u'author'), (u'name', u'TJ Holowaychuk'), (u'email', u'[email protected]'), (u'url', None)])]), (u'keywords', [u'cookie', u'sign', u'unsign']), (u'homepage_url', None), (u'download_url', u'https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz'), (u'download_checksums', []), (u'bug_tracking_url', u'https://github.com/visionmedia/node-cookie-signature/issues'), (u'code_view_url', None), (u'vcs_tool', u'git'), (u'vcs_repository', u'https://github.com/visionmedia/node-cookie-signature.git'), (u'vcs_revision', None), (u'copyright', None), (u'license_expression', None), (u'declared_licensing', None), (u'notice_text', None), (u'dependencies', [OrderedDict([(u'purl', u'pkg:npm/mocha'), (u'requirement', u'*'), (u'scope', u'devDependencies'), (u'is_runtime', False), (u'is_optional', True), (u'is_resolved', False)]), OrderedDict([(u'purl', u'pkg:npm/should'), (u'requirement', u'*'), (u'scope', u'devDependencies'), (u'is_runtime', False), (u'is_optional', True), (u'is_resolved', False)])]), (u'related_packages', [])])",0,0,0,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,15.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit_27.RULE,False,[u'mit'],,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,100.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit.LICENSE,False,[u'mit'],,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,23,26,,,,Copyright (c) 2012 LearnBoost <[email protected]>,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,23,26,,,,,LearnBoost,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,12,,,,,,[email protected],
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,16,16,,,,,,,https://github.com/visionmedia/node-cookie-signature.git
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,27,27,,,,,,,https://github.com/visionmedia/node-cookie-signature/issues

setup.py

@@ -16,10 +16,8 @@
 from setuptools import find_packages
 from setuptools import setup
 
-
 version = '2.9.1'
 
-
 #### Small hack to force using a plain version number if the option
 #### --plain-version is passed to setup.py
 
@@ -31,6 +29,7 @@
     pass
 ####
 
+
 def get_version(default=version, template='{tag}.{distance}.{commit}{dirty}',
                 use_default=USE_DEFAULT_VERSION):
     """
@@ -230,7 +229,7 @@ def read(*names, **kwargs):
             'info = scancode.plugin_info:InfoScanner',
             'licenses = scancode.plugin_license:LicenseScanner',
             'copyrights = scancode.plugin_copyright:CopyrightScanner',
-            'packages = scancode.plugin_package:PackageScanner',
+            'packages = scancode.plugin_package:PackageManifestScanner',
             'emails = scancode.plugin_email:EmailScanner',
             'urls = scancode.plugin_url:UrlScanner',
         ],
@@ -247,7 +246,9 @@ def read(*names, **kwargs):
             'mark-source = scancode.plugin_mark_source:MarkSource',
             'copyrights-summary = scancode.plugin_copyrights_summary:CopyrightSummary',
             'license-policy = scancode.plugin_license_policy:LicensePolicy',
+            'package-root = scancode.plugin_package:PackageRootSummarizer',
         ],
+        
 
         # scancode_output_filter is the entry point for filter plugins executed
         # after the post-scan plugins and used by the output plugins to

src/packagedcode/__init__.py

@@ -73,3 +73,31 @@
     models.PlainZipPackage,
 ]
 
+PACKAGES_BY_TYPE = {cls.type.default: cls for cls in PACKAGE_TYPES}
+
+
+def get_package_class(scan_data):
+    """
+    Return the Package subclass that corresponds to the package type in a
+    mapping of package `scan_data`.
+
+    For example:
+    >>> data = {'type': 'cpan'}
+    >>> assert models.CpanModule == get_package_class(data)
+    >>> data = {'type': 'some stuff'}
+    >>> assert models.Package == get_package_class(data)
+    >>> data = {'type': None}
+    >>> assert models.Package == get_package_class(data)
+    >>> data = {}
+    >>> assert models.Package == get_package_class(data)
+    >>> data = []
+    >>> assert models.Package == get_package_class(data)
+    >>> data = None
+    >>> assert models.Package == get_package_class(data)
+    """
+    ptype = scan_data and scan_data.get('type') or None
+    if not ptype:
+        # basic type for unknown package types
+        return models.Package
+    ptype_class = PACKAGES_BY_TYPE.get(ptype)
+    return ptype_class or models.Package

src/packagedcode/maven.py

@@ -74,6 +74,13 @@ class MavenPomPackage(models.Package):
     def recognize(cls, location):
         return parse(location)
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        if manifest_resource.name.endswith('pom.xml'):
+            return manifest_resource.parent(codebase)
+        # FIXME: this is NOT correct
+        return manifest_resource
+
 
 class ParentPom(artifact.Artifact):
     """

src/packagedcode/models.py

@@ -613,7 +613,26 @@ def recognize(cls, location):
         """
         return cls(location)
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        """
+        Return the Resource for the package root given a `manifest_resource`
+        Resource object that represents a manifest in the `codebase` Codebase.
+
+        Each package type and instance have different conventions on how a
+        package manifest realtes to the toor of a package.
+
+        For instance, given a "package.json" file, the root of an npm is the
+        parent directory. The same applies with a Maven "pom.xml". In the case
+        of a "xyz.pom" file found inside a JAR META-INF/ directory, the root is
+        the JAR itself which may not be the direct parent
+
+        Each package type should subclass as needed. This deafult to return the
+        same path.
+        """
+        return manifest_resource
 #
+
 # Package types
 # NOTE: this is somewhat redundant with extractcode archive handlers
 # yet the purpose and semantics are rather different here
@@ -691,12 +710,20 @@ class BowerPackage(Package):
     type = StringType(default='bower')
     primary_language = StringType(default='JavaScript')
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
 
 class MeteorPackage(Package):
     metafiles = ('package.js',)
     type = StringType(default='meteor')
     primary_language = StringType(default='JavaScript')
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
 
 class CpanModule(Package):
     metafiles = ('*.pod', '*.pm', 'MANIFEST', 'Makefile.PL', 'META.yml', 'META.json', '*.meta', 'dist.ini')
@@ -707,11 +734,16 @@ class CpanModule(Package):
 
 
 # TODO: refine me: Go packages are a mess but something is emerging
+# TODO: move to and use godeps.py
 class Godep(Package):
     metafiles = ('Godeps',)
     type = StringType(default='go')
     primary_language = StringType(default='Go')
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
 
 class RubyGem(Package):
     metafiles = ('*.control', '*.gemspec', 'Gemfile', 'Gemfile.lock',)
@@ -721,6 +753,10 @@ class RubyGem(Package):
     type = StringType(default='gem')
     primary_language = StringType(default='gem')
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
 
 class AndroidApp(Package):
     filetypes = ('zip archive',)

src/packagedcode/npm.py

@@ -74,6 +74,10 @@ class NpmPackage(models.Package):
     def recognize(cls, location):
         return parse(location)
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
     def repository_homepage_url(self, baseurl=default_web_baseurl):
         return npm_homepage_url(self.namespace, self.name, registry=baseurl)
 

src/packagedcode/nuget.py

@@ -51,6 +51,14 @@ class NugetPackage(models.Package):
     def recognize(cls, location):
         return parse(location)
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        if manifest_resource.name.endswith('.nupkg'):
+            return manifest_resource
+        if manifest_resource.name.endswith(cls.metafiles):
+            return manifest_resource.parent(codebase)
+        return manifest_resource
+
 
 nuspec_tags = [
     'id',

src/packagedcode/phpcomposer.py

@@ -75,6 +75,10 @@ class PHPComposerPackage(models.Package):
     def recognize(cls, location):
         return parse(location)
 
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
 
 def is_phpcomposer_json(location):
     return (filetype.is_file(location)

src/scancode/api.py

@@ -187,18 +187,43 @@ def get_licenses(location, min_score=0, include_text=False, diag=False,
 
 def get_package_info(location, **kwargs):
     """
-    mappings for package information detected in the file at `location`.
+    Return a mapping of package manifest information detected in the
+    file at `location`.
+
+    Note that all exceptions are caught if there are any errors while parsing a
+    package manifest.
+    """
+    from packagedcode.recognize import recognize_package
+    manifest = recognize_package(location)
+    if manifest:
+        return  dict(package_manifest=manifest.to_dict())
+    return dict(package_manifest=None)
+
+
+def get_package_info2(location, **kwargs):
+    """
+    Return a mapping of package manifest information detected in the
+    file at `location`.
+
+    Note that all exceptions are caught if there are any errors while parsing a
+    package manifest.
     """
     from packagedcode.recognize import recognize_package
-    package = recognize_package(location)
-    if package:
-        return  dict(packages=[package.to_dict()])
-    return dict(packages=[])
+    try:
+        manifest = recognize_package(location)
+        if manifest:
+            return  dict(package_manifest=manifest.to_dict())
+    except Exception:
+        # FIXME: this should be logged somehow, but for now we avoid useless
+        # errors per #983
+        pass
+    return dict(package_manifest=None)
+
 
 
 def get_file_info(location, **kwargs):
     """
-    Return a mappings of file information collected for the file at `location`.
+    Return a mapping of file information collected for the file at `location`.
     """
     result = OrderedDict()