Merge remote-tracking branch 'upstream/develop' into misc-copyrights

Author: pombredanne

.gitignore

@@ -108,7 +108,6 @@ TAGS
 Procfile
 local.cfg
 geckodriver.log
-var
 .metaflow
 selenium
 /dist/

src/packagedcode/__init__.py

@@ -216,7 +216,10 @@
 
     debian.DebianInstalledFilelistHandler,
     debian.DebianInstalledMd5sumFilelistHandler,
-    debian.DebianInstalledStatusDatabaseHandler
+    debian.DebianInstalledStatusDatabaseHandler,
+
+    rpm.RpmLicenseFilesHandler,
+    rpm.RpmMarinerContainerManifestHandler
 ]
 
 if on_linux:

src/packagedcode/debian.py

@@ -127,7 +127,7 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
 class DebianControlFileInExtractedDebHandler(models.DatafileHandler):
     datasource_id = 'debian_control_extracted_deb'
     default_package_type = 'deb'
-    path_patterns = ('*/control.tar.gz-extract/control',)
+    path_patterns = ('*/control.tar.gz-extract/control','*/control.tar.xz-extract/control')
     description = 'Debian control file - extracted layout'
     documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'
 
@@ -590,7 +590,7 @@ def parse_debian_files_list(location, datasource_id, package_type):
     else:
         name = None
         # For DebianMd5sumFilelistInPackageHandler we cannot infer name
-        if not name == "md5sums":
+        if not filename == "md5sums":
             name = filename
 
     file_references = []
@@ -663,6 +663,19 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
         party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email)
         parties.append(party)
 
+    uploaders = debian_data.get('uploaders')
+    if uploaders:
+        for uploader in uploaders.split(", "):
+            uploader_name, uploader_email = parse_debian_maintainers(uploader)
+            party = models.Party(role='uploader', name=uploader_name, email=uploader_email)
+            parties.append(party)
+
+    vcs_url = debian_data.get('vcs-git')
+    if vcs_url and ' ' in vcs_url:
+        vcs_url = vcs_url.split(' ')[0]
+
+    code_view_url = debian_data.get('vcs-browser')
+
     keywords = []
     keyword = debian_data.get('section')
     if keyword:
@@ -712,6 +725,8 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
         qualifiers=qualifiers,
         description=description,
         homepage_url=homepage_url,
+        vcs_url=vcs_url,
+        code_view_url=code_view_url,
         size=size,
         source_packages=source_packages,
         keywords=keywords,

src/packagedcode/licensing.py

@@ -60,6 +60,13 @@ def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 
+RESOURCE_TO_PACKAGE_LICENSE_FIELDS = {
+    'detected_license_expression': 'declared_license_expression',
+    'detected_license_expression_spdx': 'declared_license_expression_spdx',
+    'license_detections': 'license_detections',
+}
+
+
 def add_referenced_license_matches_for_package(resource, codebase):
     """
     Return an updated ``resource`` saving it in place, after adding new license

src/packagedcode/models.py

@@ -28,6 +28,9 @@
 from commoncode.datautils import String
 from commoncode.fileutils import as_posixpath
 from commoncode.resource import Resource
+from license_expression import combine_expressions
+from license_expression import Licensing
+
 try:
     from typecode import contenttype
 except ImportError:
@@ -118,11 +121,8 @@
 - IdentifiablePackageData: a base class for a Package-like class with a Package URL.
 """
 
-SCANCODE_DEBUG_PACKAGE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)
-SCANCODE_DEBUG_PACKAGE_ASSEMBLY = os.environ.get('SCANCODE_DEBUG_PACKAGE_ASSEMBLY', False)
-
-TRACE = SCANCODE_DEBUG_PACKAGE
-TRACE_UPDATE = SCANCODE_DEBUG_PACKAGE_ASSEMBLY
+TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)
+TRACE_UPDATE = os.environ.get('SCANCODE_DEBUG_PACKAGE_ASSEMBLY', False)
 
 
 def logger_debug(*args):
@@ -1618,6 +1618,8 @@ def update(
         include_qualifiers=False,
         include_subpath=False,
         ignore_name_check=False,
+        default_relation='AND',
+        licensing=Licensing(),
     ):
         """
         Update this Package with data from the ``package_data`` PackageData.
@@ -1670,16 +1672,16 @@ def update(
             'file_references',
         ])
 
+        license_modified = False
         for name, value in existing.items():
             new_value = new_package_data.get(name)
+            if not new_value:
+                if TRACE_UPDATE: logger_debug(f'  No new value: {name!r}: skipping')
+                continue
 
             if TRACE_UPDATE:
                 logger_debug(f'update: {name!r}={value!r} with new_value: {new_value!r}')
 
-            if not new_value:
-                if TRACE_UPDATE: logger_debug('  No new value: skipping')
-                continue
-
             if not value:
                 if TRACE_UPDATE: logger_debug('  set existing value to new')
                 setattr(self, name, new_value)
@@ -1694,6 +1696,18 @@ def update(
             if name == 'extra_data':
                 value.update(new_value)
 
+            if 'license_detections' in name:
+                license_modified = True
+                license_keys = licensing.license_keys(
+                    expression=new_package_data.get("declared_license_expression"),
+                    unique=True,
+                )
+                if name == 'license_detections' and len(license_keys) > 1:
+                    setattr(self, 'other_license_detections', new_value)
+                else:
+                    merged = value + new_value
+                    setattr(self, name, merged)
+
             if name in list_fields:
                 if TRACE_UPDATE: logger_debug('  merge lists of values')
                 merged = merge_sequences(list1=value, list2=new_value)
@@ -1702,8 +1716,36 @@ def update(
             elif TRACE_UPDATE and value != new_value:
                 if TRACE_UPDATE: logger_debug('  skipping update: no replace')
 
+        if license_modified:
+            self.refresh_license_expressions(default_relation=default_relation)
+
         return True
 
+    def refresh_license_expressions(self, default_relation='AND'):
+        if self.license_detections:
+            self.declared_license_expression = str(combine_expressions(
+                    expressions=[
+                        detection["license_expression"]
+                        for detection in self.license_detections
+                    ],
+                    relation=default_relation,
+            ))
+            self.declared_license_expression_spdx = get_declared_license_expression_spdx(
+                declared_license_expression=self.declared_license_expression,
+            )
+        
+        if self.other_license_detections:
+            self.other_license_expression = str(combine_expressions(
+                expressions=[
+                    detection["license_expression"]
+                    for detection in self.other_license_detections
+                ],
+                relation=default_relation,
+            ))
+            self.other_license_expression_spdx = get_declared_license_expression_spdx(
+                declared_license_expression=self.other_license_expression,
+            )
+
     def get_packages_files(self, codebase):
         """
         Yield all the Resource of this package found in codebase.

src/packagedcode/npm.py

@@ -12,6 +12,7 @@
 import logging
 import json
 import re
+import sys
 import urllib.parse
 from functools import partial
 from itertools import islice
@@ -35,8 +36,10 @@
 
 
 SCANCODE_DEBUG_PACKAGE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)
+SCANCODE_DEBUG_PACKAGE_NPM = os.environ.get('SCANCODE_DEBUG_PACKAGE_NPM', False)
 
 TRACE = SCANCODE_DEBUG_PACKAGE
+TRACE_NPM = SCANCODE_DEBUG_PACKAGE_NPM
 
 
 def logger_debug(*args):
@@ -45,8 +48,7 @@ def logger_debug(*args):
 
 logger = logging.getLogger(__name__)
 
-if TRACE:
-    import sys
+if TRACE or TRACE_NPM:
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
@@ -643,11 +645,16 @@ def parse(cls, location, package_only=False):
                         # <alias-package>@npm:<package>
                         if "@npm:" in ns:
                             ns = ns.split(':')[1]
+                        if "@npm:" in name:
+                            name = name.split(':')[1]
                         top_requirements.append((ns, name, constraint,))
 
                 else:
                     raise Exception('Inconsistent content')
 
+            if TRACE_NPM:
+                logger_debug(f'YarnLockV1Handler: parse: top_requirements: {top_requirements}')
+
             # top_requirements should be all for the same package
             ns_names = set([(ns, name) for ns, name, _constraint in top_requirements])
             assert len(ns_names) == 1, f'Different names for same dependency is not supported: {ns_names!r}'