Improve Python3 portability #280

 * we still have unicode and CSV issues but only a few

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/attributecode/__init__.py

@@ -14,14 +14,18 @@
 #  limitations under the License.
 # ============================================================================
 
-
 from __future__ import absolute_import
 from __future__ import print_function
 from __future__ import unicode_literals
 
 from collections import namedtuple
 import logging
 
+try:
+    basestring  # Python 2
+except NameError:
+    basestring = str  # Python 3
+
 
 __version__ = '3.0.0.dev5'
 
@@ -68,16 +72,16 @@ def clean_string(s):
     if not s:
         return s
     if s.startswith(('u"', "u'")):
-        msg = s.lstrip('u')
-    msg = s.replace('[u"', '["')
-    msg = s.replace("[u'", "['")
-    msg = s.replace("(u'", "('")
-    msg = s.replace("(u'", "('")
-    msg = s.replace("{u'", "{'")
-    msg = s.replace("{u'", "{'")
-    msg = s.replace(" u'", " '")
-    msg = s.replace(" u'", " '")
-    return msg
+        s = s.lstrip('u')
+    s = s.replace('[u"', '["')
+    s = s.replace("[u'", "['")
+    s = s.replace("(u'", "('")
+    s = s.replace("(u'", "('")
+    s = s.replace("{u'", "{'")
+    s = s.replace("{u'", "{'")
+    s = s.replace(" u'", " '")
+    s = s.replace(" u'", " '")
+    return s
 
 # modeled after the logging levels
 CRITICAL = 50

src/attributecode/api.py

@@ -20,7 +20,13 @@
 
 from collections import namedtuple
 import json
-import urllib
+
+try:
+    from urllib import urlencode
+    from urllib import quote
+except ImportError:
+    from urllib.parse import urlencode
+    from urllib.parse import quote
 
 try:
     import httplib  # Python 2
@@ -49,12 +55,12 @@ def build_api_url(url, api_username, api_key, license_key):
     payload = {'username': api_username,
                'api_key': api_key,
                'format': 'json'}
-    encoded_payload = urllib.urlencode(payload)
+    encoded_payload = urlencode(payload)
 
     api_url = '%(url)s/%(license_key)s/?%(encoded_payload)s' % locals()
 
     # handle special characters in URL such as space etc.
-    api_url = urllib.quote(api_url, safe="%/:=&?~#+!$,;'@()*[]")
+    api_url = quote(api_url, safe="%/:=&?~#+!$,;'@()*[]")
     return api_url
 
 
@@ -128,10 +134,10 @@ def request_license_data(url, api_key, license_key):
     }
 
     url = url.rstrip('/')
-    encoded_payload = urllib.urlencode(payload)
+    encoded_payload = urlencode(payload)
     full_url = '%(url)s/?%(encoded_payload)s' % locals()
     # handle special characters in URL such as space etc.
-    full_url = urllib.quote(full_url, safe="%/:=&?~#+!$,;'@()*[]")
+    full_url = quote(full_url, safe="%/:=&?~#+!$,;'@()*[]")
     headers = {'Authorization': 'Token %s' % api_key}
     license_data = {}
     errors = []

src/attributecode/cmd.py

@@ -200,15 +200,13 @@ def attrib(location, output, template, mapping, inventory, quiet):
     if mapping:
         attributecode.util.have_mapping = True
 
-    err, abouts = model.collect_inventory(location)
-    no_match_errors = attributecode.attrib.generate_and_save(abouts, output, mapping,
-                                                          template_loc=template,
-                                                          inventory_location=inventory)
-    errors = []
-    for e in err:
-        errors.append(e)
+    errors, abouts = model.collect_inventory(location)
+    no_match_errors = attributecode.attrib.generate_and_save(
+        abouts, output, mapping, template_loc=template, inventory_location=inventory)
+
     for no_match_error in no_match_errors:
         errors.append(no_match_error)
+
     log_errors(errors, quiet, os.path.dirname(output))
     click.echo('Finished.')
 

src/attributecode/gen.py

@@ -182,8 +182,8 @@ def generate(location, base_dir, mapping, license_notice_text_location, fetch_li
     # Check if the fetch_license contains valid argument
     if fetch_license:
         # Strip the ' and " for api_url, and api_key from input
-        api_url = fetch_license[0].strip("'").strip("\"")
-        api_key = fetch_license[1].strip("'").strip("\"")
+        api_url = fetch_license[0].strip("'").strip('"')
+        api_key = fetch_license[1].strip("'").strip('"')
         gen_license = True
 
     bdir = to_posix(base_dir)

src/attributecode/model.py

@@ -1447,9 +1447,10 @@ def parse_license_expression(lic_expression):
     return special_char, lic_list
 
 def special_char_in_license_expresion(lic_expression):
-    not_support_char = ['!', '@', '#', '$', '%', '^', '&', '*', '(', ')',
-                    '+', '=', '{', '}', '|', '[', ']', '\\',
-                    ':', ';', '<', '>', '?', ',', '/']
+    not_support_char = [
+        '!', '@', '#', '$', '%', '^', '&', '*', '(', ')',
+        '+', '=', '{', '}', '|', '[', ']', '\\',
+        ':', ';', '<', '>', '?', ',', '/']
     special_character = []
     for char in not_support_char:
         if char in lic_expression:

src/attributecode/saneyaml.py

@@ -35,6 +35,11 @@
 except NameError:
     unicode = str  # Python 3
 
+try:
+    basestring  # Python 2
+except NameError:
+    basestring = str  # Python 3
+
 
 """
 Wrapper around PyYAML to provide sane defaults ensuring that dump/load does

src/attributecode/util.py

@@ -136,9 +136,6 @@ def get_locations(location):
     a file or a directory tree containing ABOUT files.
     File locations are normalized using posix path separators.
     """
-    # See https://bugs.python.org/issue4071
-    if on_windows:
-        location = unicode(location)
     location = add_unc(location)
     location = get_absolute(location)
     assert os.path.exists(location)

tests/test_attrib.py

@@ -56,6 +56,6 @@ def test_generate_from_file_with_default_template(self):
         _errors, abouts = model.collect_inventory(test_file)
         result = attrib.generate_from_file(abouts)
         with open(get_test_loc('attrib_gen/expected_default_attrib.html')) as exp:
-            expected = unicode(exp.read())
+            expected = exp.read()
         self.assertEqual([x.rstrip() for x in expected.splitlines()],
                          [x.rstrip() for x in  result.splitlines()])

tests/test_model.py

@@ -89,17 +89,16 @@ def test_PathField_check_missing_location(self):
         assert None == result
 
     def test_TextField_loads_file(self):
-        field = model.FileTextField(name='f', value='license.LICENSE',
-                                present=True)
+        field = model.FileTextField(
+            name='f', value='license.LICENSE', present=True)
 
 
         base_dir = get_test_loc('fields')
         errors = field.validate(base_dir=base_dir)
         assert [] == errors
 
-        expected = [('license.LICENSE', u'some license text')]
-        result = field.value.items()
-        assert expected == result
+        expected = {'license.LICENSE': u'some license text'}
+        assert expected == field.value
 
     def test_UrlField_is_valid_url(self):
         assert model.UrlField.is_valid_url('http://www.google.com')
@@ -262,11 +261,11 @@ def test_parse_error_for_invalid_field_name(self):
         assert expected == result
 
         expected_errors = [
-            Error(CRITICAL, "Invalid line: 0: u'invalid space:value\\n'"),
-            Error(CRITICAL, "Invalid line: 1: u'other-field: value\\n'"),
-            Error(CRITICAL, "Invalid line: 4: u'_invalid_dash: value\\n'"),
-            Error(CRITICAL, "Invalid line: 5: u'3invalid_number: value\\n'"),
-            Error(CRITICAL, "Invalid line: 6: u'invalid.dot: value'")
+            Error(CRITICAL, "Invalid line: 0: 'invalid space:value\\n'"),
+            Error(CRITICAL, "Invalid line: 1: 'other-field: value\\n'"),
+            Error(CRITICAL, "Invalid line: 4: '_invalid_dash: value\\n'"),
+            Error(CRITICAL, "Invalid line: 5: '3invalid_number: value\\n'"),
+            Error(CRITICAL, "Invalid line: 6: 'invalid.dot: value'")
             ]
         assert expected_errors == errors
 
@@ -289,11 +288,11 @@ def test_parse_rejects_non_ascii_names_and_accepts_unicode_values(self):
         errors, result = model.parse(test)
         expected = [(u'name', u'name'),
                     (u'about_resource', u'.'),
-                    (u'owner', u'Mat\xedas Aguirre')]
+                    (u'owner', 'Matías Aguirre')]
         assert expected == result
 
         expected_errors = [
-            Error(CRITICAL, "Invalid line: 3: u'Mat\\xedas: unicode field name\\n'")]
+            Error(CRITICAL, "Invalid line: 3: 'Matías: unicode field name\\n'")]
         assert expected_errors == errors
 
     def test_parse_handles_blank_lines_and_spaces_in_field_names(self):
@@ -481,10 +480,8 @@ def test_About_custom_fields_are_collected_correctly_as_multiline_scalar(self):
     def test_About_has_errors_for_illegal_custom_field_name(self):
         test_file = get_test_loc('parse/illegal_custom_field.about')
         a = model.About(test_file)
-        result = a.custom_fields.items()
-        expected = [
-            ]
-        assert expected == result
+        result = a.custom_fields
+        assert {} == result
 
     def test_About_file_fields_are_empty_if_present_and_path_missing(self):
         test_file = get_test_loc('parse/missing_notice_license_files.ABOUT')
@@ -496,19 +493,12 @@ def test_About_file_fields_are_empty_if_present_and_path_missing(self):
         err_msg1 = u'Field license_file: Path %s not found' % file_path1
         err_msg2 = u'Field notice_file: Path %s not found' % file_path2
 
-        expected_errors = [
-            err_msg1, err_msg2
-            ]
-
+        expected_errors = [err_msg1, err_msg2]
         errors = model.check_file_field_exist(a, test_file)
         assert expected_errors == errors
-        expected = [(u'test.LICENSE', None)]
-        result = a.license_file.value.items()
-        assert expected == result
 
-        expected = [(u'test.NOTICE', None)]
-        result = a.notice_file.value.items()
-        assert expected == result
+        assert {'test.LICENSE': None} == a.license_file.value
+        assert {'test.NOTICE': None} == a.notice_file.value
 
     def test_About_notice_and_license_text_are_loaded_from_file(self):
         test_file = get_test_loc('parse/license_file_notice_file.ABOUT')