CLI/Display: use utf-8 (or non-ascii) encoding (#400)

We were hardcoding the use of 'ascii' encoding in CLI/Display.py which
was not a good idea.

Default string encoding in Python 2 is 'ascii', while it is 'utf-8' in
Python 3, so we can't just use str.encode() without argument. We don't
want to use 'ascii' encoding, so let's force to 'utf-8' encoding unless
the system provides a default to some other encoding. In Python 3, it
will default to 'utf-8' which is what we want anyway.

This patch also fixes the scope of ValueError exception handling in
clubak() for Display() object initializer. UnicodeEncodingError actually
inherits from ValueError so this error appeared as an option mismatch to
the user but this was definitively not. Any other ValueError like
UnicodeEncodingError is now catched in Clubak.main() instead.

Fixes #400 (clubak --diff fails on non-ascii text)

Change-Id: I9ea1e49df9c39bfa03f35f88f8b20a65ad6c98b1

Author: thiell

lib/ClusterShell/CLI/Clubak.py

@@ -161,15 +161,17 @@ def clubak():
     # Display results
     try:
         disp = Display(options)
-        if options.debug:
-            std_group_resolver().set_verbosity(1)
-            print("clubak: line_mode=%s gather=%s tree_depth=%d"
-                  % (bool(options.line_mode), bool(disp.gather), tree._depth()),
-                  file=sys.stderr)
-        display(tree, disp, disp.gather or disp.regroup, \
-                options.trace_mode, enable_nodeset_key is not False)
     except ValueError as exc:
         parser.error("option mismatch (%s)" % exc)
+        return
+
+    if options.debug:
+        std_group_resolver().set_verbosity(1)
+        print("clubak: line_mode=%s gather=%s tree_depth=%d"
+              % (bool(options.line_mode), bool(disp.gather), tree._depth()),
+              file=sys.stderr)
+    display(tree, disp, disp.gather or disp.regroup, \
+            options.trace_mode, enable_nodeset_key is not False)
 
 def main():
     """main script function"""

lib/ClusterShell/CLI/Display.py

@@ -36,6 +36,12 @@
 THREE_CHOICES = ["never", "always", "auto"]
 WHENCOLOR_CHOICES = THREE_CHOICES   # deprecated; use THREE_CHOICES
 
+if sys.getdefaultencoding() == 'ascii':
+    STRING_ENCODING = 'utf-8'  # enforce UTF-8 with Python 2
+else:
+    STRING_ENCODING = sys.getdefaultencoding()
+
+
 # Python 3 compat: wrappers for standard streams
 def sys_stdin():
     return getattr(sys.stdin, 'buffer', sys.stdin)
@@ -166,21 +172,21 @@ def format_header(self, nodeset, indent=0):
             (indstr, self.SEP,
              indstr, self._format_nodeset(nodeset), nodecntstr,
              indstr, self.SEP))
-        return hdr.encode('ascii') + b'\n'
+        return hdr.encode(STRING_ENCODING) + b'\n'
 
     def print_line(self, nodeset, line):
         """Display a line with optional label."""
         if self.label:
             prefix = self.color_stdout_fmt % ("%s: " % nodeset)
-            self.out.write(prefix.encode('ascii') + line + b'\n')
+            self.out.write(prefix.encode(STRING_ENCODING) + line + b'\n')
         else:
             self.out.write(line + b'\n')
 
     def print_line_error(self, nodeset, line):
         """Display an error line with optional label."""
         if self.label:
             prefix = self.color_stderr_fmt % ("%s: " % nodeset)
-            self.err.write(prefix.encode('ascii') + line + b'\n')
+            self.err.write(prefix.encode(STRING_ENCODING) + line + b'\n')
         else:
             self.err.write(line + b'\n')
 
@@ -234,7 +240,7 @@ def _print_diff(self, nodeset, content):
                 else:
                     output += line
                 output += '\n'
-            self.out.write(output.encode('ascii'))
+            self.out.write(output.encode(STRING_ENCODING))
 
     def _print_lines(self, nodeset, msg):
         """Display a MsgTree buffer by line with prefixed header."""
@@ -243,7 +249,7 @@ def _print_lines(self, nodeset, msg):
             header = self.color_stdout_fmt % \
                         ("%s: " % self._format_nodeset(nodeset))
             for line in msg:
-                out.write(header.encode('ascii') + line + b'\n')
+                out.write(header.encode(STRING_ENCODING) + line + b'\n')
         else:
             for line in msg:
                 out.write(line + b'\n')

tests/CLIClubakTest.py

@@ -169,6 +169,9 @@ def test_009_diff(self):
                        b"line_mode=False gather=True tree_depth=1\n")
         self._clubak_t(["--diff", "-L"], b"foo1: bar\nfoo2: bar", b'', 2,
                        b"error: option mismatch (diff not supported in line_mode)\n")
+        # GH #400
+        self._clubak_t(["--diff"], b"host1: \xc3\xa5\nhost2: a\nhost2: b\nhost1: b\n",
+                       b"--- host1\n+++ host2\n@@ -1,2 +1,2 @@\n- \xc3\xa5\n+ a\n  b\n")
 
 
 class CLIClubakTestGroupsConf(CLIClubakTest):