Merge pull request #199 from bfredl/unicode_errors

change default unicode error mode to "surrogateescape"

Author: bfredl

neovim/api/common.py

@@ -1,6 +1,8 @@
 """Code shared between the API classes."""
 import functools
 
+from ..compat import unicode_errors_default
+
 
 class Remote(object):
 
@@ -156,7 +158,7 @@ def _identity(obj, session, method, kind):
 def decode_if_bytes(obj, mode=True):
     """Decode obj if it is bytes."""
     if mode is True:
-        mode = "strict"
+        mode = unicode_errors_default
     if isinstance(obj, bytes):
         return obj.decode("utf-8", errors=mode)
     return obj

neovim/compat.py

@@ -30,7 +30,9 @@ def find_module(fullname, path):
 
     # There is no 'long' type in Python3 just int
     long = int
+    unicode_errors_default = 'surrogateescape'
 else:
     find_module = original_find_module
+    unicode_errors_default = 'strict'
 
 NUM_TYPES = (int, long, float)

neovim/msgpack_rpc/msgpack_stream.py

@@ -3,6 +3,7 @@
 
 from msgpack import Packer, Unpacker
 
+from ..compat import unicode_errors_default
 
 logger = logging.getLogger(__name__)
 debug, info, warn = (logger.debug, logger.info, logger.warning,)
@@ -19,7 +20,7 @@ class MsgpackStream(object):
     def __init__(self, event_loop):
         """Wrap `event_loop` on a msgpack-aware interface."""
         self._event_loop = event_loop
-        self._packer = Packer(use_bin_type=True)
+        self._packer = Packer(unicode_errors=unicode_errors_default)
         self._unpacker = Unpacker()
         self._message_cb = None
 

neovim/plugin/decorators.py

@@ -3,7 +3,7 @@
 import inspect
 import logging
 
-from ..compat import IS_PYTHON3
+from ..compat import IS_PYTHON3, unicode_errors_default
 
 logger = logging.getLogger(__name__)
 debug, info, warn = (logger.debug, logger.info, logger.warning,)
@@ -141,7 +141,7 @@ def shutdown_hook(f):
     return f
 
 
-def decode(mode='strict'):
+def decode(mode=unicode_errors_default):
     """Configure automatic encoding/decoding of strings."""
     def dec(f):
         f._nvim_decode = mode

test/test_buffer.py

@@ -2,6 +2,8 @@
 from nose.tools import with_setup, eq_ as eq, ok_ as ok
 from test_common import vim, cleanup
 
+from neovim.compat import IS_PYTHON3
+
 
 @with_setup(setup=cleanup)
 def test_get_length():
@@ -147,6 +149,15 @@ def test_mark():
     vim.command('mark V')
     eq(vim.current.buffer.mark('V'), [3, 0])
 
+@with_setup(setup=cleanup)
+def test_invalid_utf8():
+    vim.command('normal "=printf("%c", 0xFF)\np')
+    eq(vim.eval("char2nr(getline(1))"), 0xFF)
+
+    eq(vim.current.buffer[:], ['\udcff'] if IS_PYTHON3 else ['\xff'])
+    vim.current.line += 'x'
+    eq(vim.eval("getline(1)", decode=False), b'\xFFx')
+    eq(vim.current.buffer[:], ['\udcffx'] if IS_PYTHON3 else ['\xffx'])
 
 @with_setup(setup=cleanup)
 def test_get_exceptions():