#253 Recognize python wheels as packages

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: rakesh balusa

src/packagedcode/pypi.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
+# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -25,12 +25,16 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import json
 import logging
+import os
 import re
 
+from commoncode import fileutils
 from packagedcode.models import AssertedLicense
 from packagedcode.models import PythonPackage
 
+
 """
 Handle Python PyPi packages
 """
@@ -54,9 +58,8 @@ def get_attribute(setup_location, attribute):
     'requests' is returned for the attribute 'name'
     """
     setup_text = open(setup_location, 'rb').read()
-    setup_text = setup_text.replace('\n', '')
     # FIXME Use a valid parser for parsing 'setup.py'
-    values = re.findall('setup\(.*?'+attribute+'=[\"\']{1}.*?\',', setup_text)
+    values = re.findall('setup\(.*?'+attribute+'=[\"\']{1}.*?\',', setup_text.replace('\n', ''))
     if len(values) > 1:
         return
     else:
@@ -68,18 +71,47 @@ def get_attribute(setup_location, attribute):
             return output
 
 
+def parse_metadata(location):
+    parent_dir = fileutils.parent_directory(location)
+    if os.path.exists(os.path.join(parent_dir, 'METADATA')) and os.path.exists(os.path.join(parent_dir, 'DESCRIPTION.rst')):
+        infos = json.loads(open(location, 'rb').read())
+        homepage_url = None
+        authors = []
+        if infos['extensions']:
+            try:
+                homepage_url = infos['extensions']['python.details']['project_urls']['Home']
+            except:
+                pass
+            try:
+                for contact in infos['extensions']['python.details']['contacts']:
+                    authors.append(contact['name'])
+            except:
+                pass
+        package = PythonPackage(
+            name=infos.get('name'),
+            version=infos.get('version'),
+            summary=infos.get('summary'),
+            asserted_licenses=[AssertedLicense(license=infos.get('license'))],
+            homepage_url=homepage_url,
+            authors=authors,
+        )
+        return package
+
+
 def parse(location):
     """
     Parse a 'setup.py' and return a PythonPackage object.
     """
-    if not location.endswith('setup.py'):
-        return
-    package = PythonPackage(
-        name=get_attribute(location, 'name'),
-        homepage_url=get_attribute(location, 'url'),
-        description=get_attribute(location, 'description'),
-        version=get_attribute(location, 'version'),
-        authors=[get_attribute(location, 'author')],
-        asserted_licenses=[AssertedLicense(license=get_attribute(location, 'license'))],
-    )
-    return package
+    file_name = fileutils.file_name(location)
+    if file_name == 'setup.py':
+        package = PythonPackage(
+            name=get_attribute(location, 'name'),
+            homepage_url=get_attribute(location, 'url'),
+            description=get_attribute(location, 'description'),
+            version=get_attribute(location, 'version'),
+            authors=[get_attribute(location, 'author')],
+            asserted_licenses=[AssertedLicense(license=get_attribute(location, 'license'))],
+        )
+        return package
+    if file_name == 'metadata.json':
+        parse_metadata(location)

tests/packagedcode/data/pypi/DESCRIPTION.rst

@@ -0,0 +1,18 @@
+Six is a Python 2 and 3 compatibility library.  It provides utility functions
+for smoothing over the differences between the Python versions with the goal of
+writing Python code that is compatible on both Python versions.  See the
+documentation for more information on what is provided.
+
+Six supports every Python version since 2.6.  It is contained in only one Python
+file, so it can be easily copied into your project. (The copyright and license
+notice must be retained.)
+
+Online documentation is at https://pythonhosted.org/six/.
+
+Bugs can be reported to https://bitbucket.org/gutworth/six.  The code can also
+be found there.
+
+For questions about six or porting in general, email the python-porting mailing
+list: https://mail.python.org/mailman/listinfo/python-porting
+
+

tests/packagedcode/data/pypi/METADATA

@@ -0,0 +1,34 @@
+Metadata-Version: 2.0
+Name: six
+Version: 1.10.0
+Summary: Python 2 and 3 compatibility utilities
+Home-page: http://pypi.python.org/pypi/six/
+Author: Benjamin Peterson
+Author-email: [email protected]
+License: MIT
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 3
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+
+Six is a Python 2 and 3 compatibility library.  It provides utility functions
+for smoothing over the differences between the Python versions with the goal of
+writing Python code that is compatible on both Python versions.  See the
+documentation for more information on what is provided.
+
+Six supports every Python version since 2.6.  It is contained in only one Python
+file, so it can be easily copied into your project. (The copyright and license
+notice must be retained.)
+
+Online documentation is at https://pythonhosted.org/six/.
+
+Bugs can be reported to https://bitbucket.org/gutworth/six.  The code can also
+be found there.
+
+For questions about six or porting in general, email the python-porting mailing
+list: https://mail.python.org/mailman/listinfo/python-porting
+
+

tests/packagedcode/data/pypi/metadata.json

@@ -0,0 +1,33 @@
+{
+  "generator": "bdist_wheel (0.26.0)",
+  "summary": "Python 2 and 3 compatibility utilities",
+  "classifiers": [
+    "Programming Language :: Python :: 2",
+    "Programming Language :: Python :: 3",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Utilities"
+  ],
+  "extensions": {
+    "python.details": {
+      "project_urls": {
+        "Home": "http://pypi.python.org/pypi/six/"
+      },
+      "contacts": [
+        {
+          "email": "[email protected]",
+          "name": "Benjamin Peterson",
+          "role": "author"
+        }
+      ],
+      "document_names": {
+        "description": "DESCRIPTION.rst"
+      }
+    }
+  },
+  "license": "MIT",
+  "metadata_version": "2.0",
+  "name": "six",
+  "version": "1.10.0"
+}

tests/packagedcode/test_pypi.py

@@ -49,3 +49,13 @@ def test_get_attribute(self):
         assert 'scancode-toolkit' == pypi.get_attribute(test_file, 'name')
         assert '1.5.0' == pypi.get_attribute(test_file, 'version')
         assert 'ScanCode' == pypi.get_attribute(test_file, 'author')
+
+    def test_parse_metadata(self):
+        test_file = self.get_test_loc('pypi/metadata.json')
+        package = pypi.parse_metadata(test_file)
+        assert 'six' == package.name
+        assert '1.10.0' == package.version
+        assert 'Python 2 and 3 compatibility utilities' == package.summary
+        assert 'MIT' == package.asserted_licenses[0].license
+        assert ['Benjamin Peterson'] == package.authors
+        assert 'http://pypi.python.org/pypi/six/' == package.homepage_url