Merge branch '660-rpms' into develop

Author: steven-esser

src/packagedcode/models.py

@@ -892,7 +892,7 @@ def recognize(cls, location):
 
         Sub-classes should override to implement their own package recognition.
         """
-        return
+        return cls()
 
     @property
     def component_version(self):

src/packagedcode/recognize.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -26,17 +26,29 @@
 from __future__ import print_function
 
 import logging
+import sys
 
 from commoncode import filetype
-from typecode import contenttype
-
-from packagedcode import PACKAGE_TYPES
 from commoncode.system import on_linux
 from commoncode.fileutils import path_to_bytes
+from packagedcode import PACKAGE_TYPES
+from typecode import contenttype
 
 
+TRACE = False
+
+def logger_debug(*args):
+    pass
+
 logger = logging.getLogger(__name__)
 
+if TRACE:
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args))
+
 
 """
 Recognize packages in files or directories.
@@ -56,24 +68,25 @@ def recognize_package(location):
     mtype = T.mimetype_file
 
 
-    for package in PACKAGE_TYPES:
+    for package_type in PACKAGE_TYPES:
         # Note: default to True if there is nothing to match against
-        metafiles = package.metafiles
+        metafiles = package_type.metafiles
         if on_linux:
             metafiles = (path_to_bytes(m) for m in metafiles)
         if location.endswith(tuple(metafiles)):
-            return package.recognize(location)
+            logger_debug('metafile matching: package_type is of type:', package_type)
+            return package_type.recognize(location)
 
-        if package.filetypes:
-            type_matched = any(t in ftype for t in package.filetypes)
+        if package_type.filetypes:
+            type_matched = any(t in ftype for t in package_type.filetypes)
         else:
             type_matched = False
-        if package.mimetypes:
-            mime_matched = any(m in mtype for m in package.mimetypes)
+        if package_type.mimetypes:
+            mime_matched = any(m in mtype for m in package_type.mimetypes)
         else:
             mime_matched = False
 
-        extensions = package.extensions
+        extensions = package_type.extensions
         if extensions:
             if on_linux:
                 extensions = tuple(path_to_bytes(e) for e in extensions)
@@ -83,4 +96,9 @@ def recognize_package(location):
 
         if type_matched and mime_matched and extension_matched:
             # we return the first match in the order of PACKAGE_TYPES
-            return package(location=location)
+            logger_debug('all matching: package is of type:', package_type)
+            recognized = package_type.recognize(location)
+            logger_debug('all matching: recognized as:', repr(recognized))
+            return recognized
+
+        logger_debug('no match: package is not of known type:', package_type)

src/packagedcode/rpm.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -26,14 +26,33 @@
 from __future__ import print_function
 
 from collections import namedtuple
+import logging
 import string
+import sys
 
-import typecode.contenttype
 
 from packagedcode import models
 from packagedcode import nevra
 from packagedcode.pyrpm.rpm import RPM
 
+import typecode.contenttype
+
+
+TRACE = False
+
+def logger_debug(*args):
+    pass
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args))
+
+
 # TODO: retrieve dependencies
 
 # TODO: parse spec files see:
@@ -99,6 +118,7 @@ def info(location, include_desc=False):
     the long RPM description value if include_desc is True.
     """
     tgs = tags(location, include_desc)
+    print(tgs)
     return tgs and RPMInfo(**tgs) or None
 
 
@@ -169,7 +189,7 @@ class RpmPackage(models.Package):
     related_packages = models.ListType(models.ModelType(RPMRelatedPackage))
 
     @classmethod
-    def recognize(location):
+    def recognize(cls, location):
         return parse(location)
 
 
@@ -179,6 +199,7 @@ def parse(location):
     not an RPM.
     """
     infos = info(location, include_desc=True)
+    logger_debug('parse: infos', infos)
     if not infos:
         return
 
@@ -192,9 +213,8 @@ def parse(location):
     if infos.source_rpm:
         epoch, name, version, release, _arch = nevra.from_name(infos.source_rpm)
         evr = EVR(version, release, epoch)
-        related_packages = [RPMRelatedPackage(name=name,
-                                              version=evr,
-                                              payload_type=models.payload_src)]
+        related_packages = [
+            RPMRelatedPackage(name=name, version=evr, payload_type=models.payload_src)]
 
     package = RpmPackage(
         summary=infos.summary,
@@ -204,9 +224,7 @@ def parse(location):
         homepage_url=infos.url,
         distributors=[models.Party(name=infos.distribution)],
         vendors=[models.Party(name=infos.vendor)],
-
         asserted_licenses=asserted_licenses,
-
         related_packages=related_packages
     )
     return package

tests/packagedcode/test_recognize.py

@@ -85,10 +85,10 @@ def test_recognize_package_tarball(self):
         package = recognize_package(test_file)
         assert isinstance(package, packagedcode.models.TarPackage)
 
-    def test_recognize_cpan_manifest_is_not_yet_supported(self):
+    def test_recognize_cpan_manifest_as_plain_package(self):
         test_file = self.get_test_loc('cpan/MANIFEST')
         package = recognize_package(test_file)
-        assert not package
+        assert isinstance(package, packagedcode.models.CpanModule)
 
     def test_recognize_maven_dot_pom(self):
         test_file = self.get_test_loc('m2/aspectj/aspectjrt/1.5.3/aspectjrt-1.5.3.pom')

tests/scancode/data/rpm/fping-2.4-0.b2.rhfc1.dag.i386.rpm

@@ -0,0 +1,89 @@
+{
+  "scancode_notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
+  "scancode_options": {
+    "--package": true,
+    "--license-score": 0,
+    "--format": "json"
+  },
+  "files_count": 1,
+  "files": [
+    {
+      "path": "rpm/fping-2.4-0.b2.rhfc1.dag.i386.rpm",
+      "scan_errors": [],
+      "packages": [
+        {
+          "type": "RPM",
+          "name": "fping",
+          "version": "2.4-0.b2.rhfc1.dag",
+          "primary_language": null,
+          "packaging": "archive",
+          "summary": "A utility to ping multiple hosts at once.",
+          "description": "fping is a ping-like program which uses the Internet Control Message\nProtocol (ICMP) echo request to determine if a target host is responding.\n\nfping is different from ping in that you can specify any number of hosts\non the command line, or specify a file containing the lists of hosts to\nping. Instead of trying one host until it timeouts or replies, fping will\nsend out a ping packet and move on to the next host in a round-robin fashion.\nIf a host replies, it is noted and removed from the list of hosts to check.\nIf a host does not respond within a certain time limit and/or retry limit it\nwill be considered unreachable.",
+          "payload_type": null,
+          "size": null,
+          "release_date": null,
+          "authors": [],
+          "maintainers": [],
+          "contributors": [],
+          "owners": [],
+          "packagers": [],
+          "distributors": [
+            {
+              "type": null,
+              "name": "",
+              "email": null,
+              "url": null
+            }
+          ],
+          "vendors": [
+            {
+              "type": null,
+              "name": "Dag Apt Repository, http://dag.wieers.com/apt/",
+              "email": null,
+              "url": null
+            }
+          ],
+          "keywords": [],
+          "keywords_doc_url": null,
+          "metafile_locations": [],
+          "metafile_urls": [],
+          "homepage_url": "http://www.fping.com/",
+          "notes": null,
+          "download_urls": [],
+          "download_sha1": null,
+          "download_sha256": null,
+          "download_md5": null,
+          "bug_tracking_url": null,
+          "support_contacts": [],
+          "code_view_url": null,
+          "vcs_tool": null,
+          "vcs_repository": null,
+          "vcs_revision": null,
+          "copyright_top_level": null,
+          "copyrights": [],
+          "asserted_licenses": [
+            {
+              "license": "distributable",
+              "url": null,
+              "text": null,
+              "notice": null
+            }
+          ],
+          "legal_file_locations": [],
+          "license_expression": null,
+          "license_texts": [],
+          "notice_texts": [],
+          "dependencies": {},
+          "related_packages": [
+            {
+              "type": "RPM",
+              "name": "fping",
+              "version": "2.4-0.b2.rhfc1.dag",
+              "payload_type": "source"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

tests/scancode/data/rpm/fping-2.4-0.b2.rhfc1.dag.i386.rpm.expected.json

@@ -667,6 +667,17 @@ def test_scan_does_scan_php_composer():
     check_json_scan(expected_file, result_file)
 
 
+def test_scan_does_scan_rpm():
+    test_file = test_env.get_test_loc('rpm/fping-2.4-0.b2.rhfc1.dag.i386.rpm')
+    expected_file = test_env.get_test_loc('rpm/fping-2.4-0.b2.rhfc1.dag.i386.rpm.expected.json')
+    result_file = test_env.get_temp_file('results.json')
+
+    result = run_scan_click(['--package', test_file, result_file])
+    assert result.exit_code == 0
+    assert 'Scanning done' in result.output
+    check_json_scan(expected_file, result_file, regen=False)
+
+
 class TestFixedWidthFilename(TestCase):
 
     def test_fixed_width_file_name_with_file_name_larger_than_max_length_is_shortened(self):