[GR-37249] Fix pickle / struct issues.

PullRequest: graalpython/2199

Author: cosminbasca

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "61d85a232c44f3e9131549f92a042d168bb65a07" }
+{ "overlay": "2fb56d2621f3b269485587d232a94182b1aeb795" }

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -40,9 +40,8 @@
 import glob
 import os
 import subprocess
-import test
-
 import sys
+import test
 
 if os.environ.get("ENABLE_CPYTHON_TAGGED_UNITTESTS") == "true" or __name__ == "__main__":
     TAGS_DIR = os.path.join(os.path.dirname(__file__), "unittest_tags")
@@ -139,7 +138,18 @@ class TestTaggedUnittests(unittest.TestCase):
         total = 1
     assert selected < total
 
-    working_tests = collect_working_tests()[selected::total]
+    working_tests = collect_working_tests()
+    selection = os.environ.get('TAGGED_UNITTEST_SELECTION')
+    if not selection:
+        working_tests = working_tests[selected::total]
+    else:
+        selection = set(s.strip() for s in selection.split(","))
+        working_tests = [x for x in working_tests if x[0] in selection]
+    print("-----------------------------------------------------------------------------------------------------------")
+    print("working tests: ")
+    print([x[0] for x in working_tests])
+    print("-----------------------------------------------------------------------------------------------------------")
+
     for idx, working_test in enumerate(working_tests):
         fn = make_test_function(working_test)
         fn.__name__ = "%s[%d/%d]" % (fn.__name__, idx + 1, len(working_tests))

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_pickle.txt

@@ -21,7 +21,6 @@
 *graalpython.lib-python.3.test.test_pickle.InMemoryPickleTests.test_binunicode8
 *graalpython.lib-python.3.test.test_pickle.InMemoryPickleTests.test_buffer_callback_error
 *graalpython.lib-python.3.test.test_pickle.InMemoryPickleTests.test_buffers_error
-*graalpython.lib-python.3.test.test_pickle.InMemoryPickleTests.test_buffers_numpy
 *graalpython.lib-python.3.test.test_pickle.InMemoryPickleTests.test_bytearray
 *graalpython.lib-python.3.test.test_pickle.InMemoryPickleTests.test_bytearray8
 *graalpython.lib-python.3.test.test_pickle.InMemoryPickleTests.test_bytes
@@ -170,7 +169,6 @@
 *graalpython.lib-python.3.test.test_pickle.PyPicklerTests.test_bad_getattr
 *graalpython.lib-python.3.test.test_pickle.PyPicklerTests.test_buffer_callback_error
 *graalpython.lib-python.3.test.test_pickle.PyPicklerTests.test_buffers_error
-*graalpython.lib-python.3.test.test_pickle.PyPicklerTests.test_buffers_numpy
 *graalpython.lib-python.3.test.test_pickle.PyPicklerTests.test_bytearray
 *graalpython.lib-python.3.test.test_pickle.PyPicklerTests.test_bytes
 *graalpython.lib-python.3.test.test_pickle.PyPicklerTests.test_c_methods
@@ -315,7 +313,6 @@
 *test.pickletester.AbstractPickleTests.test_bad_getattr
 *test.pickletester.AbstractPickleTests.test_buffer_callback_error
 *test.pickletester.AbstractPickleTests.test_buffers_error
-*test.pickletester.AbstractPickleTests.test_buffers_numpy
 *test.pickletester.AbstractPickleTests.test_in_band_buffers
 *test.pickletester.AbstractPickleTests.test_local_lookup_error
 *test.pickletester.AbstractPickleTests.test_oob_buffers

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/EmptySequenceStorage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -27,11 +27,15 @@
 
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
 
+import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary;
 import com.oracle.graal.python.nodes.ErrorMessages;
 import com.oracle.graal.python.nodes.PRaiseNode;
 import com.oracle.graal.python.util.PythonUtils;
 import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.library.ExportLibrary;
+import com.oracle.truffle.api.library.ExportMessage;
 
+@ExportLibrary(PythonBufferAccessLibrary.class)
 public final class EmptySequenceStorage extends SequenceStorage {
 
     public static final EmptySequenceStorage INSTANCE = new EmptySequenceStorage();
@@ -149,4 +153,45 @@ public Object getInternalArrayObject() {
     public ListStorageType getElementType() {
         return ListStorageType.Empty;
     }
+
+    @ExportMessage
+    @SuppressWarnings("static-method")
+    boolean isBuffer() {
+        return true;
+    }
+
+    @ExportMessage
+    int getBufferLength() {
+        return 0;
+    }
+
+    @ExportMessage
+    byte readByte(@SuppressWarnings("unused") int byteOffset) throws IndexOutOfBoundsException {
+        throw new IndexOutOfBoundsException("EmptySequenceStorage is always empty!");
+    }
+
+    @ExportMessage
+    short readShort(@SuppressWarnings("unused") int byteOffset) throws IndexOutOfBoundsException {
+        throw new IndexOutOfBoundsException("EmptySequenceStorage is always empty!");
+    }
+
+    @ExportMessage
+    int readInt(@SuppressWarnings("unused") int byteOffset) throws IndexOutOfBoundsException {
+        throw new IndexOutOfBoundsException("EmptySequenceStorage is always empty!");
+    }
+
+    @ExportMessage
+    long readLong(@SuppressWarnings("unused") int byteOffset) throws IndexOutOfBoundsException {
+        throw new IndexOutOfBoundsException("EmptySequenceStorage is always empty!");
+    }
+
+    @ExportMessage
+    float readFloat(@SuppressWarnings("unused") int byteOffset) throws IndexOutOfBoundsException {
+        throw new IndexOutOfBoundsException("EmptySequenceStorage is always empty!");
+    }
+
+    @ExportMessage
+    double readDouble(@SuppressWarnings("unused") int byteOffset) throws IndexOutOfBoundsException {
+        throw new IndexOutOfBoundsException("EmptySequenceStorage is always empty!");
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/NumericSupport.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -315,7 +315,7 @@ public void putBigInteger(byte[] buffer, int index, BigInteger value, int numByt
         final byte[] src = value.toByteArray();
         int srcIndex = 0;
         int srcBytes = src.length;
-        if (src.length > numBytes) {
+        if (srcBytes > numBytes) {
             for (int i = 0; i < src.length; i++) {
                 srcIndex = i;
                 if (src[i] != 0) {
@@ -326,6 +326,13 @@ public void putBigInteger(byte[] buffer, int index, BigInteger value, int numByt
             if (srcBytes > numBytes) {
                 throw OverflowException.INSTANCE;
             }
+        } else if (srcBytes < numBytes) {
+            // perform sign extension
+            if (value.signum() < 0) {
+                for (int i = 0; i < numBytes - srcBytes; i++) {
+                    buffer[i] = -1;
+                }
+            }
         }
         int dstIndex = index + (numBytes - srcBytes);
         PythonUtils.arraycopy(src, srcIndex, buffer, dstIndex, srcBytes);

mx.graalpython/mx_graalpython.py

@@ -465,6 +465,7 @@ class GraalPythonTags(object):
     unittest_hpy_sandboxed = 'python-unittest-hpy-sandboxed'
     unittest_posix = 'python-unittest-posix'
     tagged = 'python-tagged-unittest'
+    tagged_sandboxed = 'python-tagged-unittest-sandboxed'
     svmunit = 'python-svm-unittest'
     svmunit_sandboxed = 'python-svm-unittest-sandboxed'
     shared_object = 'python-so'
@@ -563,6 +564,11 @@ def python_managed_gvm(_=None):
     mx.log(launcher)
     return launcher
 
+def python_enterprise_gvm(_=None):
+    home = _graalvm_home(envfile="graalpython-managed-bash-launcher")
+    launcher = _join_bin(home, "graalpython")
+    mx.log(launcher)
+    return launcher
 
 def python_gvm_with_assertions():
     launcher = python_gvm()
@@ -668,7 +674,7 @@ def is_included(path):
     return testfiles
 
 
-def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=False, exclude=None, env=None, use_pytest=False):
+def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=False, exclude=None, env=None, use_pytest=False, cwd=None):
     # ensure that the test distribution is up-to-date
     mx.command_function("build")(["--dep", "com.oracle.graal.python.test"])
 
@@ -714,7 +720,7 @@ def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=Fa
                 # jacoco only dumps the data on exit, and when we run all our unittests
                 # at once it generates so much data we run out of heap space
                 for testfile in testfiles:
-                    mx.run([launcher_path] + args + [testfile], nonZeroIsFatal=False, env=env)
+                    mx.run([launcher_path] + args + [testfile], nonZeroIsFatal=False, env=env, cwd=cwd)
             finally:
                 shutil.move(launcher_path_bak, launcher_path)
         else:
@@ -727,11 +733,11 @@ def graalvm_vm_arg(java_arg):
             # jacoco only dumps the data on exit, and when we run all our unittests
             # at once it generates so much data we run out of heap space
             for testfile in testfiles:
-                mx.run([python_binary, "--jvm", agent_args] + args + [testfile], nonZeroIsFatal=False, env=env)
+                mx.run([python_binary, "--jvm", agent_args] + args + [testfile], nonZeroIsFatal=False, env=env, cwd=cwd)
     else:
         args += testfiles
         mx.logv(" ".join([python_binary] + args))
-        return mx.run([python_binary] + args, nonZeroIsFatal=True, env=env)
+        return mx.run([python_binary] + args, nonZeroIsFatal=True, env=env, cwd=cwd)
 
 
 def is_bash_launcher(launcher_path):
@@ -764,7 +770,7 @@ def run_hpy_unittests(python_binary, args=None):
         return run_python_unittests(python_binary, args=args, paths=[_hpy_test_root()], env=env, use_pytest=True)
 
 
-def run_tagged_unittests(python_binary, env=None):
+def run_tagged_unittests(python_binary, env=None, cwd=None):
     if env is None:
         env = os.environ
     sub_env = dict(
@@ -778,6 +784,7 @@ def run_tagged_unittests(python_binary, env=None):
         args=["-v"],
         paths=["test_tagged_unittests.py"],
         env=sub_env,
+        cwd=cwd
     )
 
 

mx.graalpython/suite.py

@@ -44,23 +44,23 @@
             },
             {
                 "name": "tools",
-                "version": "63f1a04fe54b7986537d04199359699f1c079713",
+                "version": "53208652115456e57e02126b8dae044f7d84d9ed",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},
                 ],
             },
             {
                 "name": "sulong",
-                "version": "63f1a04fe54b7986537d04199359699f1c079713",
+                "version": "53208652115456e57e02126b8dae044f7d84d9ed",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},
                 ]
             },
             {
                 "name": "regex",
-                "version": "63f1a04fe54b7986537d04199359699f1c079713",
+                "version": "53208652115456e57e02126b8dae044f7d84d9ed",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},