WIP: Changes encoding. Fixes running chinese path in python2.7 (#1880)

Changes encoding. Fixes running chinese path in python2.7 under Windows

(fixes Windows, breaks Linux :( )
Users are advised to move to Python 3.6 or higher.

Author: HelioGuilherme66

src/robotide/contrib/testrunner/TestRunnerAgent.py

@@ -94,10 +94,10 @@
     from robot.utils.encoding import SYSTEM_ENCODING
 except ImportError:
     encoding = None
-    # print("TestRunnerAgent: Maybe you did not
-    # installed RIDE under this Python?")  # DEBUG
+    # print("TestRunnerAgent: Maybe you did not installed RIDE under this Python?")  # DEBUG
     raise     # DEBUG
 
+# print("DEBUG: console %s system %s" % (encoding.CONSOLE_ENCODING, encoding.SYSTEM_ENCODING))
 
 if sys.hexversion > 0x2060000:
     import json

src/robotide/contrib/testrunner/runprofiles.py

@@ -37,9 +37,12 @@
 from robotide.utils import (overrides, SYSTEM_ENCODING, ArgumentParser,
                             is_unicode, PY3)
 from robotide.contrib.testrunner.usages import USAGE
+from sys import getfilesystemencoding
+
 if PY3:
     from robotide.utils import unicode
 
+OUTPUT_ENCODING = getfilesystemencoding()
 
 class BaseProfile(object):
     """Base class for all test runner profiles
@@ -254,10 +257,18 @@ def TagsPanel(self, parent):
                                            "Only run tests with these tags")
         exclude_cb = self._create_checkbox(panel, self.apply_exclude_tags,
                                            "Skip tests with these tags")
-        self._include_tags = wx.TextCtrl(panel, wx.ID_ANY, size=(150, -1),
-                                         value=self.include_tags)
-        self._exclude_tags = wx.TextCtrl(panel, wx.ID_ANY, size=(150, -1),
-                                         value=self.exclude_tags)
+        try:
+            self._include_tags = wx.TextCtrl(panel, wx.ID_ANY, size=(150, -1),
+                                             value=self.include_tags)
+        except UnicodeError:
+            self._include_tags = wx.TextCtrl(panel, wx.ID_ANY, size=(150, -1),
+                                             value="unicode_error")
+        try:
+            self._exclude_tags = wx.TextCtrl(panel, wx.ID_ANY, size=(150, -1),
+                                             value=self.exclude_tags)
+        except UnicodeError:
+            self._exclude_tags = wx.TextCtrl(panel, wx.ID_ANY, size=(150, -1),
+                                             value="unicode_error")
 
         panel.Bind(wx.EVT_CHECKBOX, self.OnIncludeCheckbox, include_cb)
         panel.Bind(wx.EVT_CHECKBOX, self.OnExcludeCheckbox, exclude_cb)
@@ -308,11 +319,12 @@ def MySetToolTip(self, obj, tip):
     def _get_invalid_message(self, args):
         invalid = None
         try:
-            # print("DEBUG: runprofiles get inv msg: %s\n
+            # print("DEBUG: runprofiles get inv msg: %s\n" % args)
             # raw: %s\n" % (bytes(args), args) )
-            if PY3:
-                args = args.encode(SYSTEM_ENCODING)  # DEBUG SYSTEM_ENCODING
+            #if PY3:
+            #    args = args.encode(SYSTEM_ENCODING)  # DEBUG SYSTEM_ENCODING
             _, invalid = ArgumentParser(USAGE).parse_args(args.split())
+            # print("DEBUG: runprofiles get inv msg: %s\n" % args)
         except Information:
             return 'Does not execute - help or version option given'
         except (DataError, Exception) as e:  # DEBUG not caught DataError?

src/robotide/contrib/testrunner/testrunner.py

@@ -52,28 +52,39 @@
 from robotide.context import IS_WINDOWS
 from robotide.contrib.testrunner import TestRunnerAgent
 from robotide.controller.testexecutionresults import TestExecutionResults
-from robotide.utils import PY2, is_unicode # , unicode
+from robotide.utils import PY2, is_unicode
 try:
     from robotide.lib.robot.utils import encoding
 except ImportError:
-    encoding = None
+    from robotide.lib.robot.utils.encodingsniffer import (get_console_encoding,
+                                                          get_system_encoding)
+    CONSOLE_ENCODING = get_console_encoding()
+    SYSTEM_ENCODING = get_system_encoding()
+    encoding = {'CONSOLE': CONSOLE_ENCODING,
+                'SYSTEM': SYSTEM_ENCODING}
+
 # DEBUG we are forcing UTF-8
-if encoding:
+if IS_WINDOWS:
+    encoding.OUTPUT_ENCODING = 'mbcs'
+else:
     encoding.OUTPUT_ENCODING = sys.getfilesystemencoding()  # 'UTF-8'
+# print("DEBUG: console %s system %s output %s" % (encoding.CONSOLE_ENCODING, encoding.SYSTEM_ENCODING, encoding.OUTPUT_ENCODING))
 
 ATEXIT_LOCK = threading.RLock()
 
-# Solution from https://stackoverflow.com/questions/10009753/python-dealing-with-mixed-encoding-files
-def mixed_decoder(unicodeError):
-    errStr = unicodeError[1]
-    errLen = unicodeError.end - unicodeError.start
-    nextPosition = unicodeError.start + errLen
-    errHex = errStr[unicodeError.start:unicodeError.end].encode('hex')
-    # return u'?', nextPosition
-    return u'%s' % errHex, nextPosition  # Comment this line out to get a question mark
+
+# Solution from https://stackoverflow.com/questions/10009753/
+# python-dealing-with-mixed-encoding-files
+def mixed_decoder(unicode_error):
+    err_str = unicode_error[1]
+    err_len = unicode_error.end - unicode_error.start
+    next_position = unicode_error.start + err_len
+    err_hex = err_str[unicode_error.start:unicode_error.end].encode('hex')
+    # Alternative, return u'?', next_position
+    return u'%s' % err_hex, next_position  # Comment this line out to get a ?
 
 
-codecs.register_error("mixed", mixed_decoder)
+# codecs.register_error("mixed", mixed_decoder)
 
 
 class TestRunner(object):
@@ -212,10 +223,7 @@ def send_step_over_signal(self):
 
     def run_command(self, command, cwd):
         self._pid_to_kill = None
-        if IS_WINDOWS:
-            self._process = Process(cwd)  # .encode(encoding.SYSTEM_ENCODING))
-        else:
-            self._process = Process(cwd.encode(encoding.OUTPUT_ENCODING))
+        self._process = Process(cwd)  # .encode(encoding.SYSTEM_ENCODING))
         # print("DEBUG: run_command command: %s\nCWD: %s\n" % (command, cwd))
         self._process.run_command(command)
 
@@ -291,14 +299,23 @@ def _add_pythonpath_if_in_settings_and_not_given_by_user(
 
     @staticmethod
     def _write_argfile(argfile, args):
-        if PY2:
-            f = codecs.open(argfile, "wb")
+        if PY2 or IS_WINDOWS:
+            f = codecs.open(argfile, "wb") #, encoding=encoding.OUTPUT_ENCODING)
             for item in args:
                 if is_unicode(item):
-                    enc_arg = item.encode(encoding.CONSOLE_ENCODING)  # DEBUG .OUTPUT_ENCODING)
+                    enc_arg = item.encode('utf-8') # encoding.OUTPUT_ENCODING)  # DEBUG CONSOLE_ENCODING
                 else:
                     enc_arg = item
-                f.write(enc_arg+"\n")
+                try:
+                    f.write(enc_arg) #.encode(encoding.OUTPUT_ENCODING))
+                    f.write("\n".encode(encoding.OUTPUT_ENCODING))
+                except UnicodeError:
+                    if PY2:
+                        f.write(bytes(item))
+                    else:
+                        f.write(bytes(item, 'utf-8'))
+                    f.write(b"\n")
+                    # print("DEBUG: unicodedecodeerror when writting arg file")
         else:
             f = codecs.open(argfile, "w", "utf-8")
             f.write("\n".join(args))
@@ -348,10 +365,7 @@ def run_command(self, command):
         else:
             subprocess_args['preexec_fn'] = os.setsid
             subprocess_args['shell'] = True
-        if IS_WINDOWS:
-            self._process = subprocess.Popen(command, **subprocess_args)
-        else:
-            self._process = subprocess.Popen(command.encode(encoding.OUTPUT_ENCODING), **subprocess_args)
+        self._process = subprocess.Popen(command, **subprocess_args)
         self._process.stdin.close()
         self._output_stream = StreamReaderThread(self._process.stdout)
         self._error_stream = StreamReaderThread(self._process.stderr)
@@ -406,7 +420,9 @@ def _send_socket(self, data):
             if PY2:
                 sock.send(data)
             else:
-                sock.send(bytes(data, encoding.SYSTEM_ENCODING))
+                sock.send(bytes(data, encoding.OUTPUT_ENCODING)) #DEBUG SYSTEM_ENCODING
+        # except Exception:
+        #     print(r"DEBUG: Exception at send socket %s" % data)
         finally:
             sock.close()
 

src/robotide/contrib/testrunner/testrunnerplugin.py

@@ -67,19 +67,21 @@
 from robotide.pluginapi import Plugin, ActionInfo
 from robotide.widgets import Label, ImageProvider
 from robotide.robotapi import LOG_LEVELS
-from robotide.utils import robottime, is_unicode, PY2, PY3
+from robotide.utils import robottime, is_unicode, PY2
 from sys import getfilesystemencoding
-try:
-    from robotide.lib.robot.utils import encoding
-except ImportError:
-    encoding = None
-# if encoding:
-#     encoding.CONSOLE_ENCODING = getfilesystemencoding()
-
-if PY3:
-    from robotide.utils import unicode
-
-# print("DEBUG: TestRunnerPlugin encoding=%s" % encoding)
+from robotide.lib.robot.utils.encodingsniffer import (get_console_encoding,
+                                                      get_system_encoding)
+CONSOLE_ENCODING = get_console_encoding()
+if PY2 and IS_WINDOWS:
+    SYSTEM_ENCODING = 'mbcs'
+else:
+    SYSTEM_ENCODING = get_system_encoding()
+OUTPUT_ENCODING = getfilesystemencoding()
+encoding = {'CONSOLE': CONSOLE_ENCODING,
+            'SYSTEM': SYSTEM_ENCODING,
+            'OUTPUT': OUTPUT_ENCODING}
+
+#print("DEBUG: TestRunnerPlugin encoding=%s" % encoding)
 
 ID_RUN = wx.NewId()
 ID_RUNDEBUG = wx.NewId()
@@ -300,12 +302,16 @@ def OnRun(self, event):
                 return
         self._initialize_ui_for_running()
         command = self._create_command()
-        self._output("command: %s\n" % command)  # DEBUG encode
+        if PY2:
+            self._output("command: %s\n" % command)  # DEBUG encode
+        else:
+            self._output("command: %s\n" % command, enc=False)  # DEBUG on Py3 it not shows correct if tags with latin chars
         try:
-            if PY2 and IS_WINDOWS:
+            if PY2:  # and IS_WINDOWS:
                 cwd = self._get_current_working_dir()  # DEBUG It fails if a directory has chinese or latin symbols
-                cwd = cwd.encode(encoding.SYSTEM_ENCODING)
-                self._test_runner.run_command(command, cwd)
+                cwd = cwd.encode(encoding['OUTPUT']) # DEBUG SYSTEM_ENCODING
+                # print("DEBUG: encoded cwd: %s" % cwd)
+                self._test_runner.run_command(command.encode(encoding['OUTPUT']), cwd)  # --include Áçãoµ
             else:
                 self._test_runner.run_command(command, self._get_current_working_dir())
             # self._output("DEBUG: Passed test_runner.run_command\n")
@@ -489,7 +495,7 @@ def _format_command(self, argv):
         result = []
         for arg in argv:
             if PY2 and is_unicode(arg):
-                arg = arg.encode(encoding.CONSOLE_ENCODING)  # DEBUG "utf-8")
+                arg = arg.encode(encoding['SYSTEM'])  # DEBUG "utf-8") CONSOLE_ENCODING
                 # print("DEBUG: PY2 unicode args %s" % arg)
             if "'" in arg or " " in arg or "&" in arg:
                 # for windows, if there are spaces we need to use
@@ -508,7 +514,7 @@ def _show_notebook_tab(self):
             self._reload_model()
         self.show_tab(self.panel)
 
-    def _AppendText(self, textctrl, string, source="stdout"):
+    def _AppendText(self, textctrl, string, source="stdout", enc=True):
         if not self.panel or not textctrl:
             return
         textctrl.update_scroll_width(string)
@@ -520,15 +526,21 @@ def _AppendText(self, textctrl, string, source="stdout"):
 
         textctrl.SetReadOnly(False)
         try:
-            if PY2:
-                # textctrl.AppendText(string.encode(encoding.OUTPUT_ENCODING)) # DEBUG encoding.CONSOLE_ENCODING))  # DEBUG 'utf-8'))
-                textctrl.AppendText(string.encode(encoding.SYSTEM_ENCODING))  # encoding.SYSTEM_ENCODING)) 'utf-8'
+            if enc:
+                textctrl.AppendText(string.encode(encoding['SYSTEM']))
             else:
-                textctrl.AppendText(str(string))  # DEBUG
+                textctrl.AppendText(string)
+            # print("DEBUG _AppendText Printed OK")
         except UnicodeDecodeError as e:
             # I'm not sure why I sometimes get this, and I don't know what I
             # can do other than to ignore it.
-            textctrl.AppendTextRaw(bytes(string))  # DEBUG .encode('utf-8'))
+            if PY2:
+                if is_unicode(string):
+                    textctrl.AppendTextRaw(bytes(string.encode('utf-8')))
+                else:
+                    textctrl.AppendTextRaw(string)
+            else:
+                textctrl.AppendTextRaw(bytes(string, encoding['SYSTEM']))  # DEBUG .encode('utf-8'))
             # print(r"DEBUG UnicodeDecodeError appendtext string=%s\n" % string)
             pass
         except UnicodeEncodeError as e:
@@ -580,9 +592,9 @@ def _build_config_panel(self, parent):
         self.config_panel = panel
         return panel
 
-    def _output(self, string, source="stdout"):
+    def _output(self, string, source="stdout", enc=True):
         """Put output to the text control"""
-        self._AppendText(self.out, string, source)
+        self._AppendText(self.out, string, source, enc)
 
     def _build_runner_toolbar(self):
         toolbar = wx.ToolBar(self.panel, wx.ID_ANY,
@@ -778,7 +790,7 @@ def _create_output_textctrl(self):
         face = font.GetFaceName()
         size = font.GetPointSize()
         textctrl.SetFont(font)
-        # textctrl.StyleSetFontEncoding(wx.stc.STC_STYLE_DEFAULT, wx.FONTENCODING_SYSTEM)  # DEBUG Chinese wx.FONTENCODING_CP936)
+        # textctrl.StyleSetFontEncoding(wx.stc.STC_STYLE_DEFAULT, wx.FONTENCODING_CP936)  # DEBUG Chinese wx.) wx.FONTENCODING_SYSTEM
         textctrl.StyleSetSpec(wx.stc.STC_STYLE_DEFAULT, "face:%s,size:%d" %
                               (face, size))
         textctrl.StyleSetSpec(STYLE_STDERR, "fore:#b22222")  # firebrick