Integer Tuple to Integer Tuple Hash Tables

.github/workflows/ci.yml

@@ -19,6 +19,8 @@ jobs:
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
@@ -36,7 +38,7 @@ jobs:
           poetry install --extras test
       - name: Run tests
         run: |
-          poetry run pytest --junit-xml=test-${{ matrix.os }}-Python-${{ matrix.python }}.xml
+          poetry run pytest -s --junit-xml=test-${{ matrix.os }}-Python-${{ matrix.python }}.xml
       - uses: codecov/codecov-action@v5
 
 on:

.github/workflows/publish.yml

@@ -7,6 +7,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Set up Python
         uses: actions/setup-python@v2
         with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/finchlite/codegen/stc"]
+	path = src/finchlite/codegen/stc
+	url = https://github.com/stclib/stc

pyproject.toml

@@ -58,4 +58,4 @@ section-order = [
 
 [tool.mypy]
 ignore_missing_imports = true
-exclude = ["tests/reference"]
+exclude = ["tests/reference", "src/finchlite/codegen/stc"]

src/finchlite/codegen/c.py

@@ -6,6 +6,7 @@
 import tempfile
 from abc import ABC, abstractmethod
 from collections import namedtuple
+from collections.abc import Callable, Hashable
 from functools import lru_cache
 from pathlib import Path
 from types import NoneType
@@ -15,7 +16,14 @@
 
 from .. import finch_assembly as asm
 from ..algebra import query_property, register_property
-from ..finch_assembly import AssemblyStructFType, BufferFType, TupleFType
+from ..finch_assembly import (
+    AssemblyStructFType,
+    BufferFType,
+    DictFType,
+    ImmutableStructFType,
+    MutableStructFType,
+    TupleFType,
+)
 from ..symbolic import Context, Namespace, ScopedDict, fisinstance, ftype
 from ..util import config
 from ..util.cache import file_cache
@@ -140,7 +148,7 @@ def construct_from_c(fmt, c_obj):
         return fmt.construct_from_c(c_obj)
     try:
         return query_property(fmt, "construct_from_c", "__attr__", c_obj)
-    except NotImplementedError:
+    except AttributeError:
         return fmt(c_obj)
 
 
@@ -204,12 +212,14 @@ def construct_from_c(fmt, c_obj):
     register_property(t, "construct_from_c", "__attr__", lambda fmt, c_value: c_value)
     register_property(t, "numba_type", "__attr__", lambda t: t)
 
+
 register_property(
     np.generic,
     "serialize_to_c",
     "__attr__",
-    lambda fmt, obj: np.ctypeslib.as_ctypes(obj),
+    lambda fmt, obj: np.ctypeslib.as_ctypes(np.array(obj)),
 )
+
 # pass by value -> no op
 register_property(
     np.generic,
@@ -254,11 +264,9 @@ def __call__(self, *args):
             self.argtypes, args, serial_args, strict=False
         ):
             deserialize_from_c(type_, arg, serial_arg)
-        if hasattr(self.ret_type, "construct_from_c"):
-            return construct_from_c(res.ftype, res)
         if self.ret_type is type(None):
             return None
-        return self.ret_type(res)
+        return construct_from_c(self.ret_type, res)
 
 
 class CModule:
@@ -315,7 +323,6 @@ def __call__(self, prgm):
         for func in prgm.funcs:
             match func:
                 case asm.Function(asm.Variable(func_name, return_t), args, _):
-                    return_t = c_type(return_t)
                     arg_ts = [arg.result_format for arg in args]
                     kern = CKernel(getattr(lib, func_name), return_t, arg_ts)
                     kernels[func_name] = kern
@@ -569,6 +576,9 @@ def __call__(self, prgm: asm.AssemblyNode):
 class CContext(Context):
     """
     A class to represent a C environment.
+
+    The context has functionality to track which datastructure definitions need
+    to get declared via the stc library.
     """
 
     def __init__(
@@ -597,12 +607,24 @@ def __init__(
         self.fptr = fptr
         self.types = types
         self.slots = slots
+        self.datastructures: dict[Hashable, Any] = {}
 
     def add_header(self, header):
         if header not in self._headerset:
             self.headers.append(header)
             self._headerset.add(header)
 
+    def add_datastructure(self, key: Hashable, handler: "Callable[[CContext], Any]"):
+        """
+        Code to add a datastructure declaration.
+        This is the minimum required to prevent redundancy.
+        """
+        if key in self.datastructures:
+            return
+        # at least mark something is there.
+        self.datastructures[key] = None
+        handler(self)
+
     def emit_global(self):
         """
         Emit the headers for the C code.
@@ -792,6 +814,15 @@ def __call__(self, prgm: asm.AssemblyNode):
             case asm.Length(buf):
                 buf = self.resolve(buf)
                 return buf.result_format.c_length(self, buf)
+            case asm.LoadDict(map, idx):
+                map = self.resolve(map)
+                return map.result_format.c_loaddict(self, map, idx)
+            case asm.ExistsDict(map, idx):
+                map = self.resolve(map)
+                return map.result_format.c_existsdict(self, map, idx)
+            case asm.StoreDict(map, idx, val):
+                map = self.resolve(map)
+                return map.result_format.c_storedict(self, map, idx, val)
             case asm.Block(bodies):
                 ctx_2 = self.block()
                 for body in bodies:
@@ -936,6 +967,34 @@ def construct_from_c(self, res):
         """
 
 
+class CDictFType(DictFType, CArgumentFType, ABC):
+    """
+    Abstract base class for the ftype of dictionaries. The ftype defines how
+    the data in a Map is organized and accessed.
+    """
+
+    @abstractmethod
+    def c_existsdict(self, ctx, map, idx):
+        """
+        Return C code which checks whether a given key exists in a map.
+        """
+        ...
+
+    @abstractmethod
+    def c_loaddict(self, ctx, map, idx):
+        """
+        Return C code which gets a value corresponding to a certain key.
+        """
+        ...
+
+    @abstractmethod
+    def c_storedict(self, ctx, buffer, idx, value):
+        """
+        Return C code which stores a certain value given a certain integer tuple key.
+        """
+        ...
+
+
 class CBufferFType(BufferFType, CArgumentFType, ABC):
     """
     Abstract base class for the ftype of datastructures. The ftype defines how
@@ -1035,36 +1094,55 @@ def struct_c_type(fmt: AssemblyStructFType):
     return new_struct
 
 
+"""
+Note: When serializing any struct to C, it will get serialized to a struct with
+no indirection.
+
+When you pass a struct into a kernel that expects a struct pointer, ctypes can
+intelligently infer whether we are working with a pointer arg type (pass by
+reference) or a non-pointer type (in which case it will immediately apply
+indirection)
+"""
+
 register_property(
-    AssemblyStructFType,
+    MutableStructFType,
     "c_type",
     "__attr__",
     lambda fmt: ctypes.POINTER(struct_c_type(fmt)),
 )
 
+register_property(
+    ImmutableStructFType, "c_type", "__attr__", lambda fmt: struct_c_type(fmt)
+)
 
-def struct_c_getattr(fmt: AssemblyStructFType, ctx, obj, attr):
-    return f"{obj}->{attr}"
 
+register_property(
+    MutableStructFType,
+    "c_getattr",
+    "__attr__",
+    lambda fmt, ctx, obj, attr: f"{obj}->{attr}",
+)
 
 register_property(
-    AssemblyStructFType,
+    ImmutableStructFType,
     "c_getattr",
     "__attr__",
-    struct_c_getattr,
+    lambda fmt, ctx, obj, attr: f"{obj}.{attr}",
 )
 
 
-def struct_c_setattr(fmt: AssemblyStructFType, ctx, obj, attr, val):
+def struct_mutable_setattr(fmt: AssemblyStructFType, ctx, obj, attr, val):
     ctx.emit(f"{ctx.feed}{obj}->{attr} = {val};")
-    return
 
 
+# the equivalent for immutable is f"{ctx.feed}{obj}.{attr} = {val};"
+# but we will not include that because it's bad.
+
 register_property(
-    AssemblyStructFType,
+    MutableStructFType,
     "c_setattr",
     "__attr__",
-    struct_c_setattr,
+    struct_mutable_setattr,
 )
 
 
@@ -1092,18 +1170,23 @@ def serialize_tuple_to_c(fmt, obj):
     "__attr__",
     serialize_tuple_to_c,
 )
+
+
+def tuple_construct_from_c(fmt: TupleFType, c_struct):
+    args = [getattr(c_struct, name) for name in fmt.struct_fieldnames]
+    return tuple(args)
+
+
 register_property(
     TupleFType,
     "construct_from_c",
     "__attr__",
-    lambda fmt, obj, c_tuple: tuple(c_tuple),
+    tuple_construct_from_c,
 )
 
 register_property(
     TupleFType,
     "c_type",
     "__attr__",
-    lambda fmt: ctypes.POINTER(
-        struct_c_type(asm.NamedTupleFType("CTuple", fmt.struct_fields))
-    ),
+    lambda fmt: struct_c_type(asm.NamedTupleFType("CTuple", fmt.struct_fields)),
 )