add _post_init hook

- fix io overrides (reuse the _pyio impls)
- fix builtin codecs definitions
- fix strcount

Author: cosminbasca

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -153,6 +153,8 @@
  * {@link PythonContext} has its own core.
  */
 public final class Python3Core implements PythonCore {
+    private static final String POST_INIT_MODULE_NAME = "_post_init";
+
     // Order matters!
     private static final String[] CORE_FILES = new String[]{
                     "_descriptor",
@@ -355,6 +357,10 @@ public void initialize() {
         exportCInterface(getContext());
         currentException = null;
         initialized = true;
+
+        // apply the patches after initialization is done (we need to lookup modules that are mostly
+        // in stdlib)
+        loadFile(POST_INIT_MODULE_NAME, coreHome);
     }
 
     public Object duplicate(Map<Object, Object> replacements, Object value) {

graalpython/lib-graalpython/_codecs.py

@@ -136,162 +136,202 @@ def __codec_registry_init__():
         raise RuntimeError("can't initialize codec registry")
 
 
-# TODO implement the methods
-def escape_encode(data, errors):
+# TODO implement the encode / decode methods
+@staticmethod
+def escape_encode(data, errors=None):
     raise NotImplementedError()
 
 
-def escape_decode(data, errors):
+@staticmethod
+def escape_decode(data, errors=None):
     raise NotImplementedError()
 
 
-def utf_8_encode(string, errors):
+@staticmethod
+def utf_8_encode(string, errors=None):
     return __truffle_encode(string, "utf-8", errors)
 
 
-def utf_8_decode(string, errors, final):
+@staticmethod
+def utf_8_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-8", errors)
 
 
-def utf_7_encode(string, errors):
+@staticmethod
+def utf_7_encode(string, errors=None):
     return __truffle_encode(string, "utf-7", errors)
 
 
-def utf_7_decode(string, errors, final):
+@staticmethod
+def utf_7_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-7", errors)
 
 
-def utf_16_encode(string, errors, byteorder):
+@staticmethod
+def utf_16_encode(string, errors=None, byteorder=0):
     return __truffle_encode(string, "utf-16", errors)
 
 
-def utf_16_decode(string, errors, final):
+@staticmethod
+def utf_16_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-16", errors)
 
 
-def utf_16_le_encode(string, errors):
+@staticmethod
+def utf_16_le_encode(string, errors=None):
     return __truffle_encode(string, "utf-16-le", errors)
 
 
-def utf_16_le_decode(string, errors, final):
+@staticmethod
+def utf_16_le_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-16-le", errors)
 
 
-def utf_16_be_encode(string, errors):
+@staticmethod
+def utf_16_be_encode(string, errors=None):
     return __truffle_encode(string, "utf-16-be", errors)
 
 
-def utf_16_be_decode(string, errors, final):
+@staticmethod
+def utf_16_be_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-16-be", errors)
 
 
-def utf_16_ex_decode(data, errors, byteorder, final):
+@staticmethod
+def utf_16_ex_decode(data, errors=None, byteorder=0, final=False):
     raise NotImplementedError()
 
 
-def utf_32_encode(string, errors, byteorder):
+@staticmethod
+def utf_32_encode(string, errors=None, byteorder=0):
     return __truffle_encode(string, "utf-32", errors)
 
 
-def utf_32_decode(string, errors, final):
+@staticmethod
+def utf_32_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-32", errors)
 
 
-def utf_32_le_encode(string, errors):
+@staticmethod
+def utf_32_le_encode(string, errors=None):
     return __truffle_encode(string, "utf-32-le", errors)
 
 
-def utf_32_le_decode(string, errors, final):
+@staticmethod
+def utf_32_le_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-32-le", errors)
 
 
-def utf_32_be_encode(string, errors):
+@staticmethod
+def utf_32_be_encode(string, errors=None):
     return __truffle_encode(string, "utf-32-be", errors)
 
 
-def utf_32_be_decode(string, errors, final):
+@staticmethod
+def utf_32_be_decode(string, errors=None, final=False):
     return __truffle_decode(string, "utf-32-be", errors)
 
 
-def utf_32_ex_decode(data, errors, byteorder, final):
+@staticmethod
+def utf_32_ex_decode(data, errors=None, byteorder=0, final=False):
     raise NotImplementedError()
 
 
-def unicode_escape_encode(string, errors):
+@staticmethod
+def unicode_escape_encode(string, errors=None):
     raise NotImplementedError()
 
 
-def unicode_escape_decode(string, errors):
+@staticmethod
+def unicode_escape_decode(string, errors=None):
     raise NotImplementedError()
 
 
-def unicode_internal_encode(obj, errors):
+@staticmethod
+def unicode_internal_encode(obj, errors=None):
     raise NotImplementedError()
 
 
-def unicode_internal_decode(obj, errors):
+@staticmethod
+def unicode_internal_decode(obj, errors=None):
     raise NotImplementedError()
 
 
-def raw_unicode_escape_encode(string, errors):
+@staticmethod
+def raw_unicode_escape_encode(string, errors=None):
     raise NotImplementedError()
 
 
-def raw_unicode_escape_decode(string, errors):
+@staticmethod
+def raw_unicode_escape_decode(string, errors=None):
     raise NotImplementedError()
 
 
-def latin_1_encode(string, errors):
+@staticmethod
+def latin_1_encode(string, errors=None):
     return __truffle_encode(string, "latin-1", errors)
 
 
-def latin_1_decode(string, errors):
+@staticmethod
+def latin_1_decode(string, errors=None):
     return __truffle_decode(string, "latin-1", errors)
 
 
-def ascii_encode(string, errors):
+@staticmethod
+def ascii_encode(string, errors=None):
     return __truffle_encode(string, "ascii", errors)
 
 
-def ascii_decode(string, errors):
+@staticmethod
+def ascii_decode(string, errors=None):
     return __truffle_decode(string, "ascii", errors)
 
 
-def charmap_encode(string, errors, mapping):
+@staticmethod
+def charmap_encode(string, errors=None, mapping=None):
     raise NotImplementedError()
 
 
-def charmap_decode(string, errors, mapping):
+@staticmethod
+def charmap_decode(string, errors=None, mapping=None):
     raise NotImplementedError()
 
 
+@staticmethod
 def charmap_build(mapping):
     raise NotImplementedError()
 
 
-def readbuffer_encode(data, errors):
+@staticmethod
+def readbuffer_encode(data, errors=None):
     raise NotImplementedError()
 
 
-def mbcs_encode(string, errors):
+@staticmethod
+def mbcs_encode(string, errors=None):
     raise NotImplementedError()
 
 
-def mbcs_decode(string, errors, final):
+@staticmethod
+def mbcs_decode(string, errors=None, final=False):
     raise NotImplementedError()
 
 
+@staticmethod
 def oem_encode(string, errors):
     raise NotImplementedError()
 
 
-def oem_decode(string, errors, final):
+@staticmethod
+def oem_decode(string, errors=None, final=False):
     raise NotImplementedError()
 
 
-def code_page_encode(code_page, string, errors):
+@staticmethod
+def code_page_encode(code_page, string, errors=None):
     raise NotImplementedError()
 
 
-def code_page_decode(code_page, string, errors, final):
+@staticmethod
+def code_page_decode(code_page, string, errors=None, final=False):
     raise NotImplementedError()

graalpython/lib-graalpython/_io.py

@@ -671,11 +671,8 @@ class TextIOWrapper(_TextIOBase):
 
 
 def open(*args, **kwargs):
-    import _pyio
-    import builtins
-    setattr(builtins, 'open', _pyio.open)
-    globals()['open'] = _pyio.open
-    return _pyio.open(*args, **kwargs)
+    # this method will be overwritten in _patches
+    pass
 
 
 # set the builtins open method

graalpython/lib-graalpython/_post_init.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2018, Oracle and/or its affiliates.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or data
+# (collectively the "Software"), free of charge and under any and all copyright
+# rights in the Software, and any and all patent rights owned or freely
+# licensable by each licensor hereunder covering either (i) the unmodified
+# Software as contributed to or provided by such licensor, or (ii) the Larger
+# Works (as defined below), to deal in both
+#
+# (a) the Software, and
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+#     one is included with the Software (each a "Larger Work" to which the
+#     Software is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import sys
+import builtins
+
+# ----------------------------------------------------------------------------------------------------------------------
+#
+# patch _io
+#
+# ----------------------------------------------------------------------------------------------------------------------
+
+import _io
+import _pyio
+import io
+
+for module in [_io, io]:
+    setattr(module, 'open', _pyio.open)
+    setattr(module, 'TextIOWrapper', _pyio.TextIOWrapper)
+    setattr(module, 'IncrementalNewlineDecoder', _pyio.IncrementalNewlineDecoder)
+    setattr(module, 'BufferedRandom', _pyio.BufferedRandom)
+    setattr(module, 'BufferedRWPair', _pyio.BufferedRWPair)
+    setattr(module, 'BufferedWriter', _pyio.BufferedWriter)
+    setattr(module, 'BufferedReader', _pyio.BufferedReader)
+    setattr(module, 'StringIO', _pyio.StringIO)
+    setattr(module, '_IOBase', _pyio.IOBase)
+    setattr(module, 'RawIOBase', _pyio.RawIOBase)
+    setattr(module, 'BytesIO', _pyio.BytesIO)
+    setattr(module, '_TextIOBase', _pyio.TextIOBase)
+
+setattr(builtins, 'open', _pyio.open)
+globals()['open'] = _pyio.open

graalpython/lib-graalpython/staticmethod.py

@@ -41,3 +41,6 @@ def __init__(self, func):
 
     def __get__(self, instance, owner=None):
         return self.__func__
+
+    def __call__(self, *args, **kwargs):
+        return self.__func__(*args, **kwargs)

graalpython/lib-graalpython/str.py

@@ -316,6 +316,8 @@ def __iter__(self):
 
 
 def strcount(self, sub, start=0, end=-1):
+    if len(self) == 0:
+        return 0
     if end < 0:
         end = (len(self) + end) % len(self)
     cnt = 0

graalpython/lib-python/3/codecs.py

@@ -318,6 +318,7 @@ def _buffer_decode(self, input, errors, final):
     def decode(self, input, final=False):
         # decode input (taking the buffer into account)
         data = self.buffer + input
+        __breakpoint__()
         (result, consumed) = self._buffer_decode(data, self.errors, final)
         # keep undecoded input until the next call
         self.buffer = data[consumed:]