Use backports.csv rather than unicodecsv #280

 * and fix tests accordingly

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

etc/conf/base.txt

@@ -16,7 +16,7 @@ six
 
 pyyaml==3.12
 
-unicodecsv==0.14.1
+backports.csv==1.0.5
 
 # license expression support
 license_expression

src/attributecode/cmd.py

@@ -25,7 +25,6 @@
 import sys
 
 import click
-import unicodecsv
 
 import attributecode
 from attributecode import CRITICAL

src/attributecode/gen.py

@@ -22,12 +22,8 @@
 from collections import OrderedDict
 import logging
 import posixpath
-import sys
 
-if sys.version_info >= (3, 0):
-    from unicodecsv.py3 import UnicodeReader
-else:
-    from unicodecsv.py2 import UnicodeReader
+import backports.csv as csv
 
 from attributecode import ERROR
 from attributecode import CRITICAL
@@ -55,7 +51,7 @@ def check_duplicated_columns(location):
     """
     location = add_unc(location)
     with codecs.open(location, 'rb', encoding='utf-8', errors='ignore') as csvfile:
-        reader = UnicodeReader(csvfile)
+        reader = csv.reader(csvfile)
         columns = reader.next()
         columns = [col for col in columns]
 

src/attributecode/model.py

@@ -36,8 +36,8 @@
 from posixpath import dirname
 import re
 
+import backports.csv as csv
 from license_expression import Licensing
-import unicodecsv
 
 try:
     import urllib2  # Python 2
@@ -1294,7 +1294,7 @@ def write_output(abouts, location, format, with_absent=False, with_empty=True):
     with codecs.open(location, mode='wb', encoding='utf-8') as output_file:
         if format == 'csv':
             fieldnames = field_names(abouts)
-            writer = unicodecsv.DictWriter(output_file, fieldnames)
+            writer = csv.DictWriter(output_file, fieldnames)
             writer.writeheader()
             for row in about_dictionary_list:
                 # See https://github.com/dejacode/about-code-tool/issues/167

src/attributecode/util.py

@@ -32,7 +32,7 @@
 import string
 import sys
 
-import unicodecsv
+import backports.csv as csv
 
 try:
     import httplib  # Python 2
@@ -230,12 +230,12 @@ def resource_name(path):
     return right.strip()
 
 
-class OrderedDictReader(unicodecsv.DictReader):
+class OrderedDictReader(csv.DictReader):
     """
     A DictReader that return OrderedDicts
     """
     def next(self):
-        row_dict = unicodecsv.DictReader.next(self)
+        row_dict = csv.DictReader.next(self)
         result = OrderedDict()
         # reorder based on fieldnames order
         for name in self.fieldnames:
@@ -253,7 +253,7 @@ def get_mappings(location=None):
         location = abspath(dirname(__file__))
     mappings = {}
     try:
-        with open(join(location, 'mapping.config'), 'rU') as mapping_file:
+        with open(join(location, 'mapping.config')) as mapping_file:
             for line in mapping_file:
                 if not line or not line.strip() or line.strip().startswith('#'):
                     continue
@@ -282,7 +282,7 @@ def apply_mappings(abouts, mappings=None):
     mappings = mappings or get_mappings()
     mapped_abouts = []
     for about in abouts:
-        mapped_about = {}
+        mapped_about = OrderedDict()
         for key in about:
             mapped = []
             for mapping_keys, input_keys in mappings.items():
@@ -321,12 +321,12 @@ def load_csv(mapping, location):
     results = []
     with codecs.open(location, mode='rb', encoding='utf-8', errors='ignore') as csvfile:
         for row in OrderedDictReader(csvfile):
-            input_row = {}
-            # convert all the column keys to lower case as the same behavior as
-            # when user use the --mapping
-            for key in row.keys():
-                input_row[key.lower()] = row[key]
-            results.append(input_row)
+            # convert all the column keys to lower case as the same
+            # behavior as when user use the --mapping
+            updated_row = OrderedDict(
+                [(key.lower(), value) for key, value in row.items()]
+            )
+            results.append(updated_row)
     # user has the mapping option set
     if mapping:
         results = apply_mappings(results)

tests/test_model.py

@@ -19,6 +19,7 @@
 from __future__ import unicode_literals
 
 from collections import OrderedDict
+import json
 import posixpath
 import unittest
 from unittest.case import expectedFailure
@@ -41,6 +42,27 @@
 from attributecode.util import load_csv
 
 
+def check_csvs(expected, result):
+    """
+    Assert that the contents of two CSV files are equal.
+    """
+    mapping = None
+    expected = sorted([d.items() for d in load_csv(mapping, expected)])
+    result = sorted([d.items() for d in load_csv(mapping, result)])
+    assert expected == result
+
+
+def check_json(expected, result):
+    """
+    Assert that the contents of two JSON files are equal.
+    """
+    with open(expected) as e:
+        expected = json.load(e, object_pairs_hook=OrderedDict)
+    with open(result) as r:
+        result = json.load(r, object_pairs_hook=OrderedDict)
+    assert expected == result
+
+
 class FieldTest(unittest.TestCase):
     def test_Field_init(self):
         model.Field()
@@ -1016,19 +1038,6 @@ def test_load_dict_handles_field_validation_correctly(self):
                             with_empty=True)
         assert test == dict(as_dict)
 
-    def check_csvs(self, expected, result):
-        """
-        Assert that the content of two CSV file locations are equal.
-        """
-        mapping = None
-        expected = [d.items() for d in load_csv(mapping, expected)]
-        result = [d.items() for d in load_csv(mapping, result)]
-        for ie, expect in enumerate(expected):
-            res = result[ie]
-            for ii, exp in enumerate(expect):
-                r = res[ii]
-                assert exp == r
-
     def test_write_output_csv(self):
         path = 'load/this.ABOUT'
         test_file = get_test_loc(path)
@@ -1038,7 +1047,7 @@ def test_write_output_csv(self):
         model.write_output([a], tmp_file, format='csv')
 
         expected = get_test_loc('load/expected.csv')
-        self.check_csvs(expected, tmp_file)
+        check_csvs(expected, tmp_file)
 
     def test_write_output_json(self):
         path = 'load/this.ABOUT'
@@ -1049,7 +1058,8 @@ def test_write_output_json(self):
         model.write_output([a], tmp_file, format='json')
 
         expected = get_test_loc('load/expected.json')
-        self.check_csvs(expected, tmp_file)
+        check_json(expected, tmp_file)
+
 
 class CollectorTest(unittest.TestCase):
 

tests/test_util.py

@@ -21,6 +21,7 @@
 from collections import OrderedDict
 import string
 import unittest
+from unittest.case import expectedFailure
 
 from testing_utils import extract_test_loc
 from testing_utils import get_test_loc
@@ -304,7 +305,8 @@ def test_get_about_file_path_from_json(self):
 
     # The column names should be converted to lowercase as the same behavior as
     # when user use the mapping.config
-    """def test_load_csv_does_not_convert_column_names_to_lowercase(self):
+    @expectedFailure
+    def test_load_csv_does_not_convert_column_names_to_lowercase(self):
         mapping = None
         test_file = get_test_loc('util/about_key_with_upper_case.csv')
         expected = [OrderedDict(
@@ -314,7 +316,7 @@ def test_get_about_file_path_from_json(self):
                      ('Version', '0.8.1')])
                     ]
         result = util.load_csv(mapping, test_file)
-        assert expected == result"""
+        assert expected == result
 
     def test_get_locations_with_very_long_path(self):
         longpath = (

thirdparty/prod/PSF.LICENSE

@@ -0,0 +1,48 @@
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights
+Reserved" are retained in Python alone or in any derivative version prepared by
+Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.

thirdparty/prod/backports.csv-1.0.5-py2.py3-none-any.whl

@@ -0,0 +1,13 @@
+about_resource: backports.csv-1.0.5-py2.py3-none-any.whl
+version: 1.0.5
+download_url: https://pypi.python.org/packages/23/01/0b1fce3fd8199fe32338dc66747baad52c7183b7f587128bc77cefde0620/backports.csv-1.0.5-py2.py3-none-any.whl#md5=2b3dc8b5ca0a6f50ed50595a074ed870
+
+name: backports.csv
+homepage: https://github.com/ryanhiebert/backports.csv
+owner: Ryan Hiebert
+
+license: python
+license_file: PSF.LICENSE
+
+vcs_tool: git
+vcs_repository: https://github.com/ryanhiebert/backports.csv.git