chore: split pytest files

Merge pull request #66 from Distributive-Network/Xmader/chore/pytest

Author: Xmader

tests/python/conftest.py

@@ -0,0 +1,13 @@
+import pytest
+import pythonmonkey as pm
+import gc
+
+# This is run at the end of each test function
+@pytest.fixture(scope="function", autouse=True)
+def teardown_function():
+    """
+    Forcing garbage collection (twice) whenever a test function finishes, 
+    to locate GC-related errors
+    """
+    gc.collect(), pm.collect()
+    gc.collect(), pm.collect()

tests/python/test_bigints.py

@@ -0,0 +1,156 @@
+import pytest
+import pythonmonkey as pm
+import random
+
+def test_eval_numbers_bigints():
+    def test_bigint(py_number: int):
+        js_number = pm.eval(f'{repr(py_number)}n')
+        assert py_number == js_number
+
+    test_bigint(0)
+    test_bigint(1)
+    test_bigint(-1)
+
+    # CPython would reuse the objects for small ints in range [-5, 256] 
+    # Making sure we don't do any changes on them
+    def test_cached_int_object(py_number):
+        # type is still int
+        assert type(py_number) == int
+        assert type(py_number) != pm.bigint
+        test_bigint(py_number)
+        assert type(py_number) == int
+        assert type(py_number) != pm.bigint
+        # the value doesn't change
+        # TODO (Tom Tang): Find a way to create a NEW int object with the same value, because int literals also reuse the cached int objects
+    for _ in range(2):
+        test_cached_int_object(0) # _PyLong_FromByteArray reuses the int 0 object,
+                                  # see https://github.com/python/cpython/blob/3.9/Objects/longobject.c#L862
+        for i in range(10):
+            test_cached_int_object(random.randint(-5, 256))
+
+    test_bigint(18014398509481984)      #  2**54
+    test_bigint(-18014398509481984)     # -2**54
+    test_bigint(18446744073709551615)   #  2**64-1
+    test_bigint(18446744073709551616)   #  2**64
+    test_bigint(-18446744073709551617)  # -2**64-1
+
+    limit = 2037035976334486086268445688409378161051468393665936250636140449354381299763336706183397376
+    #     = 2**300
+    for i in range(10):
+        py_number = random.randint(-limit, limit)
+        test_bigint(py_number)
+
+    # TODO (Tom Tang): test -0 (negative zero)
+    # There's no -0 in both Python int and JS BigInt, 
+    # but this could be possible in JS BigInt's internal representation as it uses a sign bit flag.
+    # On the other hand, Python int uses `ob_size` 0 for 0, >0 for positive values, <0 for negative values
+
+def test_eval_boxed_numbers_bigints():
+    def test_boxed_bigint(py_number: int):
+        # `BigInt()` can only be called without `new`
+        #   https://tc39.es/ecma262/#sec-bigint-constructor
+        js_number = pm.eval(f'new Object({repr(py_number)}n)')
+        assert py_number == js_number
+
+    test_boxed_bigint(0)
+    test_boxed_bigint(1)
+    test_boxed_bigint(-1)
+
+    limit = 2037035976334486086268445688409378161051468393665936250636140449354381299763336706183397376
+    #     = 2**300
+    for i in range(10):
+        py_number = random.randint(-limit, limit)
+        test_boxed_bigint(py_number)
+
+def test_eval_functions_bigints():
+    ident = pm.eval("(a) => { return a }")
+    add = pm.eval("(a, b) => { return a + b }")
+
+    int1 = random.randint(-1000000,1000000)
+    bigint1 = pm.bigint(int1)
+    assert int1 == bigint1
+
+    # should return pm.bigint
+    assert type(ident(bigint1)) == pm.bigint
+    assert ident(bigint1) is not bigint1
+    # should return float (because JS number is float64)
+    assert type(ident(int1)) == float
+    assert ident(int1) == ident(bigint1)
+
+    # should raise exception on ints > (2^53-1), or < -(2^53-1)
+    def not_raise(num):
+        ident(num)
+    def should_raise(num):
+        with pytest.raises(OverflowError, match="Use pythonmonkey.bigint instead"):
+            ident(num)
+    not_raise(9007199254740991)     #   2**53-1, 0x433_FFFFFFFFFFFFF in float64
+    should_raise(9007199254740992)  #   2**53,   0x434_0000000000000 in float64
+    should_raise(9007199254740993)  #   2**53+1, NOT 0x434_0000000000001 (2**53+2)
+    not_raise(-9007199254740991)    # -(2**53-1)
+    should_raise(-9007199254740992) # -(2**53)
+    should_raise(-9007199254740993) # -(2**53+1)
+
+    # should also raise exception on large integers (>=2**53) that can be exactly represented by a float64
+    #   in our current implementation
+    should_raise(9007199254740994)  #   2**53+2, 0x434_0000000000001 in float64
+    should_raise(2**61+2**9)        #            0x43C_0000000000001 in float64
+
+    # should raise "Use pythonmonkey.bigint" instead of `PyLong_AsLongLong`'s "OverflowError: int too big to convert" on ints larger than 64bits
+    should_raise(2**65)
+    should_raise(-2**65)
+    not_raise(pm.bigint(2**65))
+    not_raise(pm.bigint(-2**65))
+
+    # should raise JS error when mixing a BigInt with a number in arithmetic operations
+    def should_js_error(a, b):
+        with pytest.raises(pm.SpiderMonkeyError, match="can't convert BigInt to number"):
+            add(a, b)
+    should_js_error(pm.bigint(0), 0)
+    should_js_error(pm.bigint(1), 2)
+    should_js_error(3, pm.bigint(4))
+    should_js_error(-5, pm.bigint(6))
+
+    assert add(pm.bigint(0), pm.bigint(0)) == 0
+    assert add(pm.bigint(1), pm.bigint(0)) == 1
+    assert add(pm.bigint(1), pm.bigint(2)) == 3
+    assert add(pm.bigint(-1), pm.bigint(1)) == 0
+    assert add(pm.bigint(2**60), pm.bigint(0)) == 1152921504606846976
+    assert add(pm.bigint(2**65), pm.bigint(-2**65-1)) == -1
+
+    # fuzztest
+    limit = 2037035976334486086268445688409378161051468393665936250636140449354381299763336706183397376 # 2**300
+    for i in range(10):
+        num1 = random.randint(-limit, limit)
+        num2 = random.randint(-limit, limit)
+        assert add(pm.bigint(num1), pm.bigint(num2)) == num1+num2
+
+def test_eval_functions_bigint_factorial():
+    factorial = pm.eval("(num) => {let r = 1n; for(let i = 0n; i<num; i++){r *= num - i}; return r}")
+    assert factorial(pm.bigint(1)) == 1
+    assert factorial(pm.bigint(18)) == 6402373705728000
+    assert factorial(pm.bigint(19)) == 121645100408832000 # > Number.MAX_SAFE_INTEGER
+    assert factorial(pm.bigint(21)) == 51090942171709440000 # > 64 bit int
+    assert factorial(pm.bigint(35)) == 10333147966386144929666651337523200000000 # > 128 bit
+
+def test_eval_functions_bigint_crc32():
+    crc_table_at = pm.eval("""
+    // translated from https://rosettacode.org/wiki/CRC-32#Python
+    const crc_table = (function create_table() {
+        const a = []
+        for (let i = 0n; i < 256n; i++) {
+            let k = i
+            for (let j = 0n; j < 8n; j++) {
+                // must use bigint here as js number is trimmed to int32 in bitwise operations
+                if (k & 1n) k ^= 0x1db710640n
+                k >>= 1n
+            }
+            a.push(k)
+        }
+        return a
+    })();
+    (n) => crc_table[n]
+    """)
+    assert type(crc_table_at(1)) == pm.bigint
+    assert crc_table_at(0) == 0
+    assert crc_table_at(1) == 1996959894
+    assert crc_table_at(255) == 755167117 # last item

tests/python/test_buffer_typed_array.py

@@ -0,0 +1,198 @@
+import pytest
+import pythonmonkey as pm
+import gc
+import numpy, array, struct
+
+def test_py_buffer_to_js_typed_array():
+    # JS TypedArray/ArrayBuffer should coerce to Python memoryview type
+    def assert_js_to_py_memoryview(buf: memoryview):
+        assert type(buf) is memoryview
+        assert None == buf.obj # https://docs.python.org/3.9/c-api/buffer.html#c.Py_buffer.obj
+        assert 2 * 4 == buf.nbytes # 2 elements * sizeof(int32_t)
+        assert "02000000ffffffff" == buf.hex() # native (little) endian
+    buf1 = pm.eval("new Int32Array([2,-1])")
+    buf2 = pm.eval("new Int32Array([2,-1]).buffer")
+    assert_js_to_py_memoryview(buf1)
+    assert_js_to_py_memoryview(buf2)
+    assert [2, -1] == buf1.tolist()
+    assert [2, 0, 0, 0, 255, 255, 255, 255] == buf2.tolist()
+    assert -1 == buf1[1]
+    assert 255 == buf2[7]
+    with pytest.raises(IndexError, match="index out of bounds on dimension 1"):
+        buf1[2]
+    with pytest.raises(IndexError, match="index out of bounds on dimension 1"):
+        buf2[8]
+    del buf1, buf2
+
+    # test element value ranges
+    buf3 = pm.eval("new Uint8Array(1)")
+    with pytest.raises(ValueError, match="memoryview: invalid value for format 'B'"):
+        buf3[0] = 256
+    with pytest.raises(ValueError, match="memoryview: invalid value for format 'B'"):
+        buf3[0] = -1
+    with pytest.raises(IndexError, match="index out of bounds on dimension 1"): # no automatic resize
+        buf3[1] = 0
+    del buf3
+
+    # Python buffers should coerce to JS TypedArray
+    # and the typecode maps to TypedArray subtype (Uint8Array, Float64Array, ...)
+    assert True == pm.eval("(arr)=>arr instanceof Uint8Array")( bytearray([1,2,3]) )
+    assert True == pm.eval("(arr)=>arr instanceof Uint8Array")( numpy.array([1], dtype=numpy.uint8) )
+    assert True == pm.eval("(arr)=>arr instanceof Uint16Array")( numpy.array([1], dtype=numpy.uint16) )
+    assert True == pm.eval("(arr)=>arr instanceof Uint32Array")( numpy.array([1], dtype=numpy.uint32) )
+    assert True == pm.eval("(arr)=>arr instanceof BigUint64Array")( numpy.array([1], dtype=numpy.uint64) )
+    assert True == pm.eval("(arr)=>arr instanceof Int8Array")( numpy.array([1], dtype=numpy.int8) )
+    assert True == pm.eval("(arr)=>arr instanceof Int16Array")( numpy.array([1], dtype=numpy.int16) )
+    assert True == pm.eval("(arr)=>arr instanceof Int32Array")( numpy.array([1], dtype=numpy.int32) )
+    assert True == pm.eval("(arr)=>arr instanceof BigInt64Array")( numpy.array([1], dtype=numpy.int64) )
+    assert True == pm.eval("(arr)=>arr instanceof Float32Array")( numpy.array([1], dtype=numpy.float32) )
+    assert True == pm.eval("(arr)=>arr instanceof Float64Array")( numpy.array([1], dtype=numpy.float64) )
+    assert pm.eval("new Uint8Array([1])").format == "B"
+    assert pm.eval("new Uint16Array([1])").format == "H"
+    assert pm.eval("new Uint32Array([1])").format == "I" # FIXME (Tom Tang): this is "L" on 32-bit systems
+    assert pm.eval("new BigUint64Array([1n])").format == "Q"
+    assert pm.eval("new Int8Array([1])").format == "b"
+    assert pm.eval("new Int16Array([1])").format == "h"
+    assert pm.eval("new Int32Array([1])").format == "i"
+    assert pm.eval("new BigInt64Array([1n])").format == "q"
+    assert pm.eval("new Float32Array([1])").format == "f"
+    assert pm.eval("new Float64Array([1])").format == "d"
+
+    # not enough bytes to populate an element of the TypedArray
+    with pytest.raises(pm.SpiderMonkeyError, match="RangeError: buffer length for BigInt64Array should be a multiple of 8"):
+        pm.eval("(arr) => new BigInt64Array(arr.buffer)")(array.array('i', [-11111111]))
+
+    # TypedArray with `byteOffset` and `length`
+    arr1 = array.array('i', [-11111111, 22222222, -33333333, 44444444])
+    with pytest.raises(pm.SpiderMonkeyError, match="RangeError: invalid or out-of-range index"):
+        pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ -4)")(arr1)
+    with pytest.raises(pm.SpiderMonkeyError, match="RangeError: start offset of Int32Array should be a multiple of 4"):
+        pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 1)")(arr1)
+    with pytest.raises(pm.SpiderMonkeyError, match="RangeError: size of buffer is too small for Int32Array with byteOffset"):
+        pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 20)")(arr1)
+    with pytest.raises(pm.SpiderMonkeyError, match="RangeError: invalid or out-of-range index"):
+        pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 4, /*length*/ -1)")(arr1)
+    with pytest.raises(pm.SpiderMonkeyError, match="RangeError: attempting to construct out-of-bounds Int32Array on ArrayBuffer"):
+        pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 4, /*length*/ 4)")(arr1)
+    arr2 = pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 4, /*length*/ 2)")(arr1)
+    assert 2 * 4 == arr2.nbytes # 2 elements * sizeof(int32_t)
+    assert [22222222, -33333333] == arr2.tolist()
+    assert "8e155301ab5f03fe" == arr2.hex() # native (little) endian
+    assert 22222222 == arr2[0] # offset 1 int32
+    with pytest.raises(IndexError, match="index out of bounds on dimension 1"):
+        arr2[2]
+    arr3 = pm.eval("(arr) => new Int32Array(arr.buffer, 16 /* byteOffset */)")(arr1) # empty Int32Array
+    assert 0 == arr3.nbytes
+    del arr3
+
+    # test GC
+    del arr1
+    gc.collect(), pm.collect()
+    gc.collect(), pm.collect()
+    # TODO (Tom Tang): the 0th element in the underlying buffer is still accessible after GC, even is not referenced by the JS TypedArray with byteOffset
+    del arr2
+
+    # mutation
+    mut_arr_original = bytearray(4)
+    pm.eval("""
+    (/* @type Uint8Array */ arr) => {
+        // 2.25 in float32 little endian
+        arr[2] = 0x10
+        arr[3] = 0x40
+    }
+    """)(mut_arr_original)
+    assert 0x10 == mut_arr_original[2]
+    assert 0x40 == mut_arr_original[3]
+    # mutation to a different TypedArray accessing the same underlying data block will also change the original buffer
+    def do_mutation(mut_arr_js):
+        assert 2.25 == mut_arr_js[0]
+        mut_arr_js[0] = 225.50048828125 # float32 little endian: 0x 20 80 61 43 
+        assert "20806143" == mut_arr_original.hex()
+        assert 225.50048828125 == array.array("f", mut_arr_original)[0]
+    mut_arr_new = pm.eval("""
+    (/* @type Uint8Array */ arr, do_mutation) => {
+        const mut_arr_js = new Float32Array(arr.buffer)
+        do_mutation(mut_arr_js)
+        return arr
+    }
+    """)(mut_arr_original, do_mutation)
+    assert [0x20, 0x80, 0x61, 0x43] == mut_arr_new.tolist()
+
+    # simple 1-D numpy array should just work as well
+    numpy_int16_array = numpy.array([0, 1, 2, 3], dtype=numpy.int16)
+    assert "0,1,2,3" == pm.eval("(typedArray) => typedArray.toString()")(numpy_int16_array)
+    assert 3.0 == pm.eval("(typedArray) => typedArray[3]")(numpy_int16_array)
+    assert True == pm.eval("(typedArray) => typedArray instanceof Int16Array")(numpy_int16_array)
+    numpy_memoryview = pm.eval("(typedArray) => typedArray")(numpy_int16_array)
+    assert 2 == numpy_memoryview[2]
+    assert 4 * 2 == numpy_memoryview.nbytes # 4 elements * sizeof(int16_t)
+    assert "h" == numpy_memoryview.format # the type code for int16 is 'h', see https://docs.python.org/3.9/library/array.html
+    with pytest.raises(IndexError, match="index out of bounds on dimension 1"):
+        numpy_memoryview[4]
+
+    # can work for empty Python buffer
+    def assert_empty_py_buffer(buf, type: str):
+        assert 0 == pm.eval("(typedArray) => typedArray.length")(buf)
+        assert None == pm.eval("(typedArray) => typedArray[0]")(buf) # `undefined`
+        assert True == pm.eval("(typedArray) => typedArray instanceof "+type)(buf)
+    assert_empty_py_buffer(bytearray(b''), "Uint8Array")
+    assert_empty_py_buffer(numpy.array([], dtype=numpy.uint64), "BigUint64Array")
+    assert_empty_py_buffer(array.array('d', []), "Float64Array")
+
+    # can work for empty TypedArray
+    def assert_empty_typedarray(buf: memoryview, typecode: str):
+        assert typecode == buf.format
+        assert struct.calcsize(typecode) == buf.itemsize
+        assert 0 == buf.nbytes
+        assert "" == buf.hex()
+        assert b"" == buf.tobytes()
+        assert [] == buf.tolist()
+        buf.release()
+    assert_empty_typedarray(pm.eval("new BigInt64Array()"), "q")
+    assert_empty_typedarray(pm.eval("new Float32Array(new ArrayBuffer(4), 4 /*byteOffset*/)"), "f")
+    assert_empty_typedarray(pm.eval("(arr)=>arr")( bytearray([]) ), "B")
+    assert_empty_typedarray(pm.eval("(arr)=>arr")( numpy.array([], dtype=numpy.uint16) ),"H")
+    assert_empty_typedarray(pm.eval("(arr)=>arr")( array.array("d", []) ),"d")
+
+    # can work for empty ArrayBuffer
+    def assert_empty_arraybuffer(buf):
+        assert "B" == buf.format
+        assert 1 == buf.itemsize
+        assert 0 == buf.nbytes
+        assert "" == buf.hex()
+        assert b"" == buf.tobytes()
+        assert [] == buf.tolist()
+        buf.release()
+    assert_empty_arraybuffer(pm.eval("new ArrayBuffer()"))
+    assert_empty_arraybuffer(pm.eval("new Uint8Array().buffer"))
+    assert_empty_arraybuffer(pm.eval("new Float64Array().buffer"))
+    assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( bytearray([]) ))
+    assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( pm.eval("(arr)=>arr.buffer")(bytearray()) ))
+    assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( numpy.array([], dtype=numpy.uint64) ))
+    assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( array.array("d", []) ))
+
+    # TODO (Tom Tang): shared ArrayBuffer should be disallowed
+    # pm.eval("new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }).buffer")
+
+    # TODO (Tom Tang): once a JS ArrayBuffer is transferred to a worker thread, it should be invalidated in Python-land as well
+
+    # TODO (Tom Tang): error for detached ArrayBuffer, or should it be considered as empty?
+
+    # should error on immutable Python buffers 
+    # Note: Python `bytes` type must be converted to a (mutable) `bytearray` because there's no such a concept of read-only ArrayBuffer in JS
+    with pytest.raises(BufferError, match="Object is not writable."):
+        pm.eval("(typedArray) => {}")(b'')
+    immutable_numpy_array = numpy.arange(10)
+    immutable_numpy_array.setflags(write=False)
+    with pytest.raises(ValueError, match="buffer source array is read-only"):
+        pm.eval("(typedArray) => {}")(immutable_numpy_array)
+
+    # buffer should be in C order (row major)
+    fortran_order_arr = numpy.array([[1, 2], [3, 4]], order="F") # 1-D array is always considered C-contiguous because it doesn't matter if it's row or column major in 1-D
+    with pytest.raises(ValueError, match="ndarray is not C-contiguous"):
+        pm.eval("(typedArray) => {}")(fortran_order_arr)
+
+    # disallow multidimensional array
+    numpy_2d_array = numpy.array([[1, 2], [3, 4]], order="C")
+    with pytest.raises(BufferError, match="multidimensional arrays are not allowed"):
+        pm.eval("(typedArray) => {}")(numpy_2d_array)