Handle encoding issue

 * Adding `errors=replace` for all the io.open functions

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

src/attributecode/attrib.py

@@ -221,7 +221,7 @@ def generate_from_file(abouts, is_about_input, license_dict, scancode, min_licen
             template_loc = add_unc(DEFAULT_TEMPLATE_FILE)
     else:
         template_loc = add_unc(template_loc)
-    with io.open(template_loc, encoding='utf-8') as tplf:
+    with io.open(template_loc, encoding='utf-8', errors='replace') as tplf:
         tpls = tplf.read()
     return generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=tpls, variables=variables)
 
@@ -259,7 +259,7 @@ def generate_and_save(abouts, is_about_input, license_dict, output_location, sca
 
     if rendered:
         output_location = add_unc(output_location)
-        with io.open(output_location, 'w', encoding='utf-8') as of:
+        with io.open(output_location, 'w', encoding='utf-8', errors='replace') as of:
             of.write(rendered)
 
     return errors, rendered

src/attributecode/cmd.py

@@ -285,7 +285,7 @@ def validate_template(ctx, param, value):
     if not value:
         return None
 
-    with io.open(value, encoding='utf-8') as templatef:
+    with io.open(value, encoding='utf-8', errors='replace') as templatef:
         template_error = check_template(templatef.read())
 
     if template_error:
@@ -728,7 +728,7 @@ def report_errors(errors, quiet, verbose, log_file_loc=None):
         click.echo(msg)
     if log_file_loc and errors:
         log_msgs, _ = get_error_messages(errors, quiet=False, verbose=True)
-        with io.open(log_file_loc, 'w', encoding='utf-8') as lf:
+        with io.open(log_file_loc, 'w', encoding='utf-8', errors='replace') as lf:
             lf.write('\n'.join(log_msgs))
     return severe_errors_count
 

src/attributecode/model.py

@@ -584,7 +584,7 @@ def _validate(self, *args, **kwargs):
             try:
                 # TODO: we have lots the location by replacing it with a text
                 location = add_unc(location)
-                with io.open(location, encoding='utf-8') as txt:
+                with io.open(location, encoding='utf-8', errors='replace') as txt:
                     text = txt.read()
                 self.value[path] = text
             except Exception as e:
@@ -1207,7 +1207,7 @@ def dump(self, location, lic_dict=None):
         if on_windows:
             about_file_path = add_unc(about_file_path)
 
-        with io.open(about_file_path, mode='w', encoding='utf-8') as dumped:
+        with io.open(about_file_path, mode='w', encoding='utf-8', errors='replace') as dumped:
             dumped.write(genereated_tk_version)
             dumped.write(self.dumps(lic_dict))
 
@@ -1218,7 +1218,7 @@ def dump_android_notice(self, path, context):
         if on_windows:
             path = add_unc(path)
 
-        with io.open(path, mode='w', encoding='utf-8') as dumped:
+        with io.open(path, mode='w', encoding='utf-8', errors='replace') as dumped:
             dumped.write(context)
 
     def android_module_license(self, about_parent_path):
@@ -1285,7 +1285,7 @@ def dump_lic(self, location, license_dict):
                             license_name, license_filename, license_context, license_url = license_dict[lic_key]
                             license_info = (lic_key, license_name, license_filename, license_context, license_url)
                             license_key_name_context_url.append(license_info)
-                            with io.open(license_path, mode='w', encoding='utf-8', newline='\n') as lic:
+                            with io.open(license_path, mode='w', encoding='utf-8', newline='\n', errors='replace') as lic:
                                 lic.write(license_context)
                     except Exception as e:
                         # TODO: it should return error if exception caught
@@ -1526,7 +1526,7 @@ def save_as_json(location, about_dicts):
         output_file.write(json.dumps(data, indent=2))
 
 def save_as_csv(location, about_dicts, field_names):
-    with io.open(location, mode='w', encoding='utf-8', newline='') as output_file:
+    with io.open(location, mode='w', encoding='utf-8', newline='', errors='replace') as output_file:
         writer = csv.DictWriter(output_file, field_names)
         writer.writeheader()
         csv_formatted_list = util.format_about_dict_output(about_dicts)

src/attributecode/transform.py

@@ -282,7 +282,7 @@ def from_file(cls, location):
         Load and return a Transformer instance from a YAML configuration file at
         `location`.
         """
-        with io.open(location, encoding='utf-8') as conf:
+        with io.open(location, encoding='utf-8', errors='replace') as conf:
             data = saneyaml.load(replace_tab_with_spaces(conf.read()))
         return cls(
             field_renamings=data.get('field_renamings', {}),
@@ -400,7 +400,7 @@ def write_csv(location, data, field_names):  # NOQA
     Write a CSV file at `location` the `data` list of ordered dicts using the
     `field_names`.
     """
-    with io.open(location, 'w', encoding='utf-8', newline='\n') as csvfile:
+    with io.open(location, 'w', encoding='utf-8', newline='\n', errors='replace') as csvfile:
         writer = csv.DictWriter(csvfile, fieldnames=field_names)
         writer.writeheader()
         writer.writerows(data)

tests/test_attrib.py

@@ -63,7 +63,7 @@ def test_check_template_all_builtin_templates_are_valid(self):
         builtin_templates_dir = os.path.dirname(attrib.DEFAULT_TEMPLATE_FILE)
         for template in os.listdir(builtin_templates_dir):
             template_loc = os.path.join(builtin_templates_dir, template)
-            with io.open(template_loc, 'r', encoding='utf-8') as tmpl:
+            with io.open(template_loc, 'r', encoding='utf-8', errors='replace') as tmpl:
                 template = tmpl.read()
             try:
                 assert None == attrib.check_template(template)

tests/test_cmd.py

@@ -154,7 +154,7 @@ def test_report_errors_can_write_to_logfile():
     result_file = get_temp_file()
     _ec = cmd.report_errors(errors, quiet=False, verbose=True,
                            log_file_loc=result_file)
-    with io.open(result_file, 'r', encoding='utf-8') as rf:
+    with io.open(result_file, 'r', encoding='utf-8', errors='replace') as rf:
         result = rf.read()
     expected = [
         'Command completed with 3 errors or warnings.',

tests/test_model.py

@@ -93,7 +93,7 @@ def get_unicode_content(location):
     """
     Read file at location and return a unicode string.
     """
-    with io.open(location, encoding='utf-8') as doc:
+    with io.open(location, encoding='utf-8', errors='replace') as doc:
         return doc.read()