Testing for Safe Buffers (#171)

* reduce test cases for codegen

* regression testing

* ruff format

* formatting stuff

* fix

* Update test output

* Support windows platform reference files

---------

Co-authored-by: Willow Ahrens <[email protected]>
Co-authored-by: Mateusz Sokół <[email protected]>
Co-authored-by: Mateusz Sokół <[email protected]>

Author: 3 people

src/finchlite/codegen/c.py

@@ -202,6 +202,7 @@ def construct_from_c(fmt, c_obj):
     )
     # construction from c is just the identity.
     register_property(t, "construct_from_c", "__attr__", lambda fmt, c_value: c_value)
+    register_property(t, "numba_type", "__attr__", lambda t: t)
 
 register_property(
     np.generic,
@@ -529,24 +530,25 @@ def c_type(t):
     ctypes.c_wchar: ("wchar_t", ["wchar.h"]),
     ctypes.c_byte: ("char", []),
     ctypes.c_ubyte: ("unsigned char", []),
-    ctypes.c_short: ("short", []),
-    ctypes.c_ushort: ("unsigned short", []),
-    ctypes.c_int: ("int", []),
     ctypes.c_int8: ("int8_t", ["stdint.h"]),
     ctypes.c_int16: ("int16_t", ["stdint.h"]),
     ctypes.c_int32: ("int32_t", ["stdint.h"]),
     ctypes.c_int64: ("int64_t", ["stdint.h"]),
-    ctypes.c_uint: ("unsigned int", []),
     ctypes.c_uint8: ("uint8_t", ["stdint.h"]),
     ctypes.c_uint16: ("uint16_t", ["stdint.h"]),
     ctypes.c_uint32: ("uint32_t", ["stdint.h"]),
     ctypes.c_uint64: ("uint64_t", ["stdint.h"]),
-    ctypes.c_long: ("long", []),
-    ctypes.c_ulong: ("unsigned long", []),
-    ctypes.c_longlong: ("long long", []),
-    ctypes.c_ulonglong: ("unsigned long long", []),
-    ctypes.c_size_t: ("size_t", ["stddef.h"]),
-    ctypes.c_ssize_t: ("ssize_t", ["unistd.h"]),
+    # use standard types instead of aliases
+    # ctypes.c_short: ("short", []),
+    # ctypes.c_ushort: ("unsigned short", []),
+    # ctypes.c_int: ("int", []),
+    # ctypes.c_uint: ("unsigned int", []),
+    # ctypes.c_long: ("long", []),
+    # ctypes.c_ulong: ("unsigned long", []),
+    # ctypes.c_longlong: ("long long", []),
+    # ctypes.c_ulonglong: ("unsigned long long", []),
+    # ctypes.c_size_t: ("size_t", ["stddef.h"]),
+    # ctypes.c_ssize_t: ("ssize_t", ["unistd.h"]),
     ctypes.c_float: ("float", []),
     ctypes.c_double: ("double", []),
     ctypes.c_char_p: ("char*", []),

src/finchlite/codegen/numpy_buffer.py

@@ -154,6 +154,7 @@ def c_unpack(self, ctx, var_n, val):
         data = ctx.freshen(var_n, "data")
         length = ctx.freshen(var_n, "length")
         t = ctx.ctype_name(c_type(self._dtype))
+        ctx.add_header("#include <stddef.h>")
         ctx.exec(
             f"{ctx.feed}{t}* {data} = ({t}*){ctx(val)}->data;\n"
             f"{ctx.feed}size_t {length} = {ctx(val)}->length;"

src/finchlite/codegen/safe_buffer.py

@@ -65,14 +65,14 @@ def c_data(self, *args, **kwargs):
     def _c_check(self, ctx, buf, idx):
         ctx.add_header("#include <stdio.h>")
         ctx.add_header("#include <stdlib.h>")
+        ctx.add_header("#include <stddef.h>")
         idx_n = ctx.freshen("computed")
         ctx.exec(
             f"{ctx.feed}size_t {idx_n} = ({ctx(idx)});\n"
-            f"{ctx.feed}if ({idx_n} < 0 || {idx_n} >= ({self.c_length(ctx, buf)}))"
-            "{"
-            f'fprintf(stderr, "Encountered an index out of bounds error!");\n'
-            f"exit(1);\n"
-            "}"
+            f"{ctx.feed}if ({idx_n} < 0 || {idx_n} >= ({self.c_length(ctx, buf)})) {{\n"
+            f'{ctx.feed}    fprintf(stderr, "Index out of bounds error!");\n'
+            f"{ctx.feed}    exit(1);\n"
+            f"{ctx.feed}}}"
         )
         return asm.Variable(idx_n, ctypes.c_size_t)
 

tests/reference/test_dot_product_regression_compiler0__c_NumpyBuffer_.c

@@ -1,12 +1,12 @@
-#include <stddef.h>
-typedef void* (*fptr)( void**, size_t );
+#include <stdint.h>
+typedef void* (*fptr)( void**, uint64_t );
 struct CNumpyBuffer {
     void* arr;
     void* data;
-    size_t length;
+    uint64_t length;
     fptr resize;
 };
-#include <unistd.h>
+#include <stddef.h>
 double dot_product(struct CNumpyBuffer* a, struct CNumpyBuffer* b) {
     double c = (double)0.0;
     struct CNumpyBuffer* a_ = a;
@@ -15,7 +15,7 @@ double dot_product(struct CNumpyBuffer* a, struct CNumpyBuffer* b) {
     struct CNumpyBuffer* b_ = b;
     double* b__data = (double*)b_->data;
     size_t b__length = b_->length;
-    for (ssize_t i = (ssize_t)0; i < a__length; i++) {
+    for (int64_t i = (int64_t)0; i < a__length; i++) {
         c = c + (a__data)[i] * (b__data)[i];
     }
     a_->data = (void*)a__data;

tests/reference/test_safe_loadstore_regression_compiler0__c_any_.c

@@ -0,0 +1,41 @@
+#include <stdint.h>
+typedef void* (*fptr)( void**, uint64_t );
+struct CNumpyBuffer {
+    void* arr;
+    void* data;
+    uint64_t length;
+    fptr resize;
+};
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+int64_t finch_access(struct CNumpyBuffer* a, uint64_t idx) {
+    struct CNumpyBuffer* a_ = a;
+    int64_t* a__data = (int64_t*)a_->data;
+    size_t a__length = a_->length;
+    size_t computed = (idx);
+    if (computed < 0 || computed >= (a__length)) {
+        fprintf(stderr, "Index out of bounds error!");
+        exit(1);
+    }
+    int64_t val = (a__data)[computed];
+    size_t computed_2 = (idx);
+    if (computed_2 < 0 || computed_2 >= (a__length)) {
+        fprintf(stderr, "Index out of bounds error!");
+        exit(1);
+    }
+    int64_t val2 = (a__data)[computed_2];
+    return val;
+}
+int64_t finch_change(struct CNumpyBuffer* a, uint64_t idx, int64_t val) {
+    struct CNumpyBuffer* a_ = a;
+    int64_t* a__data_2 = (int64_t*)a_->data;
+    size_t a__length_2 = a_->length;
+    size_t computed_3 = (idx);
+    if (computed_3 < 0 || computed_3 >= (a__length_2)) {
+        fprintf(stderr, "Index out of bounds error!");
+        exit(1);
+    }
+    (a__data_2)[computed_3] = val;
+    return (int64_t)0;
+}

tests/reference/test_safe_loadstore_regression_compiler1__py_any_.py

@@ -0,0 +1,29 @@
+import _operator, builtins
+from numba import njit
+import numpy
+from numpy import int64, float64
+
+
+@njit
+def finch_access(a: builtins.list, idx: ctypes.c_ulong) -> int64:
+    a_ = a
+    a__arr = a_[0]
+    computed = (idx)
+    if computed < 0 or computed >= (len(a__arr)):
+        raise IndexError()
+    val: int64 = a__arr[computed]
+    computed_2 = (idx)
+    if computed_2 < 0 or computed_2 >= (len(a__arr)):
+        raise IndexError()
+    val2: int64 = a__arr[computed_2]
+    return val
+
+@njit
+def finch_change(a: builtins.list, idx: ctypes.c_ulong, val: ctypes.c_long) -> int64:
+    a_ = a
+    a__arr_2 = a_[0]
+    computed_3 = (idx)
+    if computed_3 < 0 or computed_3 >= (len(a__arr_2)):
+        raise IndexError()
+    a__arr_2[computed_3] = val
+    return c_long(0)

tests/reference/test_safe_loadstore_regression_compiler1__py_win_.py

@@ -0,0 +1,29 @@
+import _operator, builtins
+from numba import njit
+import numpy
+from numpy import int64, float64
+
+
+@njit
+def finch_access(a: builtins.list, idx: ctypes.c_ulonglong) -> int64:
+    a_ = a
+    a__arr = a_[0]
+    computed = (idx)
+    if computed < 0 or computed >= (len(a__arr)):
+        raise IndexError()
+    val: int64 = a__arr[computed]
+    computed_2 = (idx)
+    if computed_2 < 0 or computed_2 >= (len(a__arr)):
+        raise IndexError()
+    val2: int64 = a__arr[computed_2]
+    return val
+
+@njit
+def finch_change(a: builtins.list, idx: ctypes.c_ulonglong, val: ctypes.c_longlong) -> int64:
+    a_ = a
+    a__arr_2 = a_[0]
+    computed_3 = (idx)
+    if computed_3 < 0 or computed_3 >= (len(a__arr_2)):
+        raise IndexError()
+    a__arr_2[computed_3] = val
+    return c_longlong(0)

tests/test_codegen.py

@@ -441,6 +441,67 @@ def test_simple_struct(compiler):
     assert result == np.float64(9.0)
 
 
+@pytest.mark.parametrize(
+    ["compiler", "extension", "platform"],
+    [
+        (CGenerator(), ".c", "any"),
+        (NumbaGenerator(), ".py", "win" if sys.platform == "win32" else "any"),
+    ],
+)
+def test_safe_loadstore_regression(compiler, extension, platform, file_regression):
+    a = np.array(range(3), dtype=ctypes.c_int64)
+    ab = NumpyBuffer(a)
+    ab_safe = SafeBuffer(ab)
+    ab_v = asm.Variable("a", ab_safe.ftype)
+    ab_slt = asm.Slot("a_", ab_safe.ftype)
+    idx = asm.Variable("idx", ctypes.c_size_t)
+    val = asm.Variable("val", ctypes.c_int64)
+
+    res_var = asm.Variable("val", ab_safe.ftype.element_type)
+    res_var2 = asm.Variable("val2", ab_safe.ftype.element_type)
+    mod = asm.Module(
+        (
+            asm.Function(
+                asm.Variable("finch_access", ab_safe.ftype.element_type),
+                (ab_v, idx),
+                asm.Block(
+                    (
+                        asm.Unpack(ab_slt, ab_v),
+                        # we assign twice like this; this is intentional and
+                        # designed to check correct refreshing.
+                        asm.Assign(
+                            res_var,
+                            asm.Load(ab_slt, idx),
+                        ),
+                        asm.Assign(
+                            res_var2,
+                            asm.Load(ab_slt, idx),
+                        ),
+                        asm.Return(res_var),
+                    )
+                ),
+            ),
+            asm.Function(
+                asm.Variable("finch_change", ab_safe.ftype.element_type),
+                (ab_v, idx, val),
+                asm.Block(
+                    (
+                        asm.Unpack(ab_slt, ab_v),
+                        asm.Store(
+                            ab_slt,
+                            idx,
+                            val,
+                        ),
+                        asm.Return(asm.Literal(ctypes.c_int64(0))),
+                    )
+                ),
+            ),
+        )
+    )
+    output = compiler(mod)
+    file_regression.check(output, extension=extension)
+
+
 @pytest.mark.parametrize(
     "size,idx",
     [(size, idx) for size in range(1, 4) for idx in range(-1, 4)],
@@ -472,34 +533,10 @@ def test_c_load_safebuffer(size, idx):
     [
         (*params, compiler)
         for params in [
-            (
-                -1,
-                2,
-            ),
-            (
-                -1,
-                3,
-            ),
-            (
-                0,
-                2,
-            ),
-            (
-                1,
-                2,
-            ),
-            (
-                2,
-                3,
-            ),
-            (
-                2,
-                2,
-            ),
-            (
-                3,
-                2,
-            ),
+            (-1, 2),
+            (1, 2),
+            (2, 3),
+            (2, 2),
         ]
         for compiler in [asm.AssemblyInterpreter(), NumbaCompiler()]
     ],
@@ -548,12 +585,9 @@ def test_numba_load_safebuffer(size, idx, compiler):
         (*params, compiler)
         for params in [
             (-1, 2, 3),
-            (-1, 3, 1434),
-            (0, 2, 3),
-            (1, 2, 3),
-            (2, 3, 3),
+            (1, 2, 1434),
+            (2, 3, 1434),
             (2, 2, 3),
-            (3, 2, 3),
         ]
         for compiler in [NumbaCompiler(), asm.AssemblyInterpreter()]
     ],
@@ -599,34 +633,10 @@ def test_numba_store_safebuffer(size, idx, value, compiler):
     [
         (*params, value)
         for params in [
-            (
-                -1,
-                2,
-            ),
-            (
-                -1,
-                3,
-            ),
-            (
-                0,
-                2,
-            ),
-            (
-                1,
-                2,
-            ),
-            (
-                2,
-                3,
-            ),
-            (
-                2,
-                2,
-            ),
-            (
-                3,
-                2,
-            ),
+            (-1, 2),
+            (1, 2),
+            (2, 3),
+            (2, 2),
         ]
         for value in [-1, 1434]
     ],
@@ -663,9 +673,7 @@ def test_c_store_safebuffer(size, idx, value):
     "value,np_type,c_type",
     [
         (3, np.int64, ctypes.c_int64),
-        (2, np.int32, ctypes.c_int32),
         (1, np.float32, ctypes.c_float),
-        (1.0, np.float64, ctypes.c_double),
         (1.2, np.float64, ctypes.c_double),
     ],
 )
@@ -682,9 +690,7 @@ def test_np_c_serialization(value, np_type, c_type):
     "value,c_type",
     [
         (3, ctypes.c_int64),
-        (2, ctypes.c_int32),
         (1, ctypes.c_float),
-        (1.0, ctypes.c_double),
         (1.2, ctypes.c_double),
     ],
 )
@@ -702,9 +708,7 @@ def test_ctypes_c_serialization(value, c_type):
     "value,np_type",
     [
         (3, np.int64),
-        (2, np.int32),
         (1, np.float32),
-        (1.0, np.float64),
         (1.2, np.float64),
     ],
 )