Release 2.0.2

* Handle input's encoding issues
* Better error handling
* Writing to and reading from Windows OS with paths > 255 chars

Author: chinyeungli

.gitignore

@@ -31,4 +31,4 @@ nosetests.xml
 # PyCharm project
 .idea/
 
-/*.egg_info
+/*.egg-info/

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 
 python:
-  - "2.6"
   - "2.7"
 install:
     - source configure etc/conf

README.rst

@@ -71,14 +71,14 @@ And on Windows::
 
 For instance on Linux the whole installation would be like this::
 
-   $ wget https://github.com/dejacode/about-code-tool/archive/v2.0.1.zip
-   $ unzip  v2.0.1.zip
-   $ cd about-code-tool-2.0.1/
+   $ wget https://github.com/dejacode/about-code-tool/archive/v2.0.2.zip
+   $ unzip  v2.0.2.zip
+   $ cd about-code-tool-2.0.2/
    $ source configure
 
 On Windows, the whole installation would be like this:
 
- * Download and extract https://github.com/dejacode/about-code-tool/archive/v2.0.1.zip
+ * Download and extract https://github.com/dejacode/about-code-tool/archive/v2.0.2.zip
  * open a command prompt and cd to the directory where the zip extraction directory
  * run configure
 

about_code_tool/about.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -32,28 +32,31 @@
 import csv
 from datetime import datetime
 from email.parser import HeaderParser
-from os.path import basename, dirname, join, normpath, realpath
 import errno
 import httplib
 import logging
+import ntpath
 import optparse
 import os
+from os.path import basename, dirname, join, normpath, realpath
 import posixpath
 import socket
 import string
 import sys
 import urlparse
-import ntpath
 
 
-__version__ = '2.0.1'
+on_windows = 'win32' in sys.platform
+UNC_PREFIX = u'\\\\?\\'
+
+__version__ = '2.0.2'
 
 # See http://dejacode.org
 __about_spec_version__ = '1.0'
 
 
 __copyright__ = """
-Copyright (c) 2013-2014 nexB Inc. All rights reserved.
+Copyright (c) 2013-2015 nexB Inc. All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -554,7 +557,14 @@ class AboutFile(object):
     """
     def __init__(self, location=None):
         self.about_resource = None
-        self.location = location
+        # The os.path.abspath(None) will cause error in linux system.
+        # See https://bugs.python.org/issue22587
+        # Note that the os.path.abspath is needed for windows when there
+        # is long path/filename.
+        if on_windows:
+            self.location = os.path.abspath(location)
+        else:
+            self.location = location
 
         self.parsed = None
         self.parsed_fields = None
@@ -1082,7 +1092,10 @@ def get_dje_license_name(self):
         """
         Return the about object's dje_license_name.
         """
-        return self.parsed.get('dje_license_name', '')
+        try:
+            return self.parsed.get('dje_license_name', '')
+        except:
+            return ''
 
 def check_invalid_chars(field_name, line):
     """
@@ -1104,6 +1117,8 @@ def check_invalid_chars(field_name, line):
         warnings = Warn(IGNORED, field_name, line, msg)
     return invalid_chars, warnings
 
+def posix_unc_prefix():
+    return posix_path(u'\\\\?\\')
 
 class Collector(object):
     """
@@ -1144,6 +1159,23 @@ def collect(location):
         """
         # FIXME: we should not accept both a file and dir location as input
         paths = []
+
+        if on_windows:
+            location = unicode(location)
+        """
+        Convert a location to an absolute Window UNC path to support long paths
+        on Windows. Return the location unchanged if not on Windows.
+        See https://msdn.microsoft.com/en-us/library/aa365247.aspx
+        """
+        if on_windows and not location.startswith(UNC_PREFIX):
+            location = UNC_PREFIX + os.path.abspath(location)
+        location = os.path.expanduser(location)
+        location = os.path.expandvars(location)
+        location = os.path.normpath(location)
+        location = os.path.abspath(location)
+
+        assert os.path.exists(location)
+
         if location:
             if os.path.isfile(location) and is_about_file(location):
                 paths.append(location)
@@ -1197,6 +1229,10 @@ def get_relative_path(self, location):
         """
         user_loc = normpath(self.location)
         if os.path.isdir(self.normalized_location):
+            # Making sure both are in posix path format before
+            # doing any string partition.
+            location = posix_path(location)
+            user_loc = posix_path(user_loc)
             parent_name = basename(user_loc)
             subpath = '/' + parent_name + location.partition(user_loc)[2]
             if user_loc[-1] == '/':
@@ -1241,6 +1277,10 @@ def get_about_context(self, about_object):
         if '\n' in about_object.get_dje_license_name():
             msg = ('Multiple licenses is not supported. '
                    'Skipping License generation.')
+            if on_windows:
+                if (about_object.location.startswith(posix_unc_prefix())
+                    or about_object.location.startswith(UNC_PREFIX)):
+                    about_object.location = about_object.location.strip(posix_unc_prefix()).strip(UNC_PREFIX)
             err = Error(GENATTRIB, 'dje_license',
                         about_object.location, msg)
             self.genattrib_errors.append(err)
@@ -1258,6 +1298,10 @@ def get_about_context(self, about_object):
             and not '\n' in about_object.get_dje_license_name():
             msg = ('No license_text found. '
                    'Skipping License generation.')
+            if on_windows:
+                if (about_object.location.startswith(posix_unc_prefix())
+                    or about_object.location.startswith(UNC_PREFIX)):
+                    about_object.location = about_object.location.strip(posix_unc_prefix()).strip(UNC_PREFIX)
             err = Error(GENATTRIB, 'license_text_file',
                         about_object.location, msg)
             self.genattrib_errors.append(err)
@@ -1314,6 +1358,9 @@ def generate_attribution(self, template_path=None, limit_to=None, verification=N
                         break
 
                 if not component_exist:
+                    if on_windows:
+                        if self.location.startswith(posix_unc_prefix()):
+                            self.location = self.location.strip(posix_unc_prefix())
                     loc = self.location + component
                     msg = ('The requested ABOUT file: %r does not exist. '
                            'No attribution generated for this file.' % loc)
@@ -1364,6 +1411,9 @@ def check_paths(self, paths):
         for path in paths:
             path = posix_path(path)
             afp = join(self.location, path)
+            if on_windows:
+                if afp.startswith(posix_unc_prefix()):
+                    afp = afp.strip(posix_unc_prefix())
             msg = ('The requested ABOUT file: %(afp)r does not exist. '
                    'No attribution generated for this file.' % locals())
             err = Error(GENATTRIB, 'about_file', path, msg)

about_code_tool/genabout.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -13,7 +13,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 # ============================================================================
-
 """
 This is a tool to generate ABOUT files based on the input file.
 The input file should be a csv format which contains information about the
@@ -22,29 +21,29 @@
 
 from __future__ import print_function
 
+from collections import namedtuple
 import copy
 import csv
 import errno
 import json
 import logging
 import optparse
+from os import makedirs
 import os
+from os.path import exists, dirname, join, abspath, isdir, normpath, basename, expanduser
 import shutil
 import sys
 import urllib
 import urllib2
-
-from collections import namedtuple
 from urlparse import urljoin, urlparse
-from os import makedirs
-from os.path import exists, dirname, join, abspath, isdir, normpath, basename, expanduser
 
 import about
 
-__version__ = '2.0.0'
+
+__version__ = '2.0.2'
 
 __copyright__ = """
-Copyright (c) 2013-2014 nexB Inc. All rights reserved.
+Copyright (c) 2013-2015 nexB Inc. All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -135,7 +134,7 @@ def get_input_list(input_file):
         for row in csvfile:
             row_dict = {}
             for key in row:
-                row_dict[key.lower()] = row[key]
+                row_dict[key.lower()] = row[key].rstrip()
             input_list.append(row_dict)
         return input_list
 
@@ -455,11 +454,12 @@ def pre_process_and_dje_license_dict(self, input_list, api_url, api_username, ap
                     if not lic in license_dict:
                         detail_list = []
                         detail = self.get_license_details_from_api(api_url, api_username, api_key, lic)
-                        license_dict[lic] = detail[0]
-                        line['dje_license_name'] = detail[0]
                         dje_key = detail[1]
+                        line['dje_license_key'] = dje_key
+                        license_dict[dje_key] = detail[0]
+                        line['dje_license_name'] = detail[0]
                         license_context = detail [2]
-                        line['dje_license_url'] = dje_lic_urn + lic
+                        line['dje_license_url'] = dje_lic_urn + dje_key
                         detail_list.append(dje_key)
                         detail_list.append(license_context)
                         key_text_dict[detail[0]] = detail_list
@@ -515,7 +515,14 @@ def pre_generation(self, gen_location, input_list, action_num):
             about_file_location = join(gen_location, file_location)
             about_file_dir = dirname(about_file_location)
             if not os.path.exists(about_file_dir):
-                makedirs(about_file_dir)
+                # Check for invalid file path
+                try:
+                    makedirs(about_file_dir)
+                except:
+                    msg = 'Invalid ABOUT file path.'
+                    self.errors.append(Error(VALUE, 'about_file_path',
+                                             about_file_dir, msg))
+                    continue
             about_file_exist = _exists(about_file_location)
             if about_file_exist:
                 if action_num == ACTION_DO_NOTHING_IF_ABOUT_FILE_EXIST:
@@ -604,6 +611,11 @@ def format_output(input_list):
                     value = about_dict_list[item].replace('\n', '\n ')
                     if (value or item in about.MANDATORY_FIELDS) and not item\
                         in about.ERROR_WARN_FIELDS and not item == 'about_resource':
+                        # It will cause error if value has different coding
+                        try:
+                            value = unicode(value, errors='ignore')
+                        except:
+                            pass
                         context += item + ': ' + value + '\n'
 
             component.append(about_file_location)
@@ -614,6 +626,8 @@ def format_output(input_list):
     @staticmethod
     def write_output(output):
         for about_file_location, context in output:
+            if about.on_windows:
+                about_file_location = about.UNC_PREFIX + os.path.abspath(about_file_location)
             if _exists(about_file_location):
                 os.remove(about_file_location)
             with open(about_file_location, 'wb') as output_file:
@@ -855,6 +869,10 @@ def main(parser, options, args):
                 sys.exit(errno.EINVAL)
 
     if gen_license:
+        # Strip the ' and " for api_url, api_username and api_key from input
+        api_url = api_url.strip("'").strip("\"")
+        api_username = api_username.strip("'").strip("\"")
+        api_key = api_key.strip("'").strip("\"")
         dje_license_dict = gen.pre_process_and_dje_license_dict(input_list,
                                                                 api_url,
                                                                 api_username,

about_code_tool/genattrib.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -43,12 +43,12 @@
 logger.addHandler(handler)
 file_logger = logging.getLogger(__name__ + '_file')
 
-__version__ = '2.0.0'
+__version__ = '2.0.2'
 
 __about_spec_version__ = '1.0.0'  # See http://dejacode.org
 
 __copyright__ = """
-Copyright (c) 2013-2014 nexB Inc. All rights reserved.
+Copyright (c) 2013-2015 nexB Inc. All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.