Clean up the requested vs allocated size mess

This commit adds a new `rsize` field+accessor for the requested size,
and uses it where appropriate, to avoid headaches with the true
allocated size not matching the requested one.

Author: ctrueden

src/appose/types.py

@@ -45,8 +45,24 @@ class which can be used with a `with` statement. When the program flow
     `unlink_on_dispose` flag.
     """
 
-    def __init__(self, name: str = None, create: bool = False, size: int = 0):
-        super().__init__(name=name, create=create, size=size)
+    def __init__(self, name: str = None, create: bool = False, rsize: int = 0):
+        """
+        Create a new shared memory block, or attach to an existing one.
+
+        :param name:
+            The unique name for the requested shared memory, specified as a
+            string. If create is True (i.e. a new shared memory block) and
+            no name is given, a novel name will be generated.
+        :param create:
+            Whether a new shared memory block is created (True)
+            or an existing one is attached to (False).
+        :param rsize:
+            Requested size in bytes. The true allocated size will be at least
+            this much, but may be rounded up to the next block size multiple,
+            depending on the running platform.
+        """
+        super().__init__(name=name, create=create, size=rsize)
+        self.rsize = rsize
         self._unlink_on_dispose = create
         if _is_worker:
             # HACK: Remove this shared memory block from the resource_tracker,
@@ -146,7 +162,7 @@ def __init__(self, dtype: str, shape: Sequence[int], shm: SharedMemory = None):
         self.shape = shape
         self.shm = (
             SharedMemory(
-                create=True, size=ceil(prod(shape) * _bytes_per_element(dtype))
+                create=True, rsize=ceil(prod(shape) * _bytes_per_element(dtype))
             )
             if shm is None
             else shm
@@ -157,7 +173,7 @@ def __str__(self):
             f"NDArray("
             f"dtype='{self.dtype}', "
             f"shape={self.shape}, "
-            f"shm='{self.shm.name}' ({self.shm.size}))"
+            f"shm='{self.shm.name}' ({self.shm.rsize}))"
         )
 
     def ndarray(self):
@@ -188,7 +204,7 @@ def default(self, obj):
             return {
                 "appose_type": "shm",
                 "name": obj.name,
-                "size": obj.size,
+                "rsize": obj.rsize,
             }
         if isinstance(obj, NDArray):
             return {
@@ -204,7 +220,7 @@ def _appose_object_hook(obj: Dict):
     atype = obj.get("appose_type")
     if atype == "shm":
         # Attach to existing shared memory block.
-        return SharedMemory(name=(obj["name"]), size=(obj["size"]))
+        return SharedMemory(name=(obj["name"]), rsize=(obj["rsize"]))
     elif atype == "ndarray":
         return NDArray(obj["dtype"], obj["shape"], obj["shm"])
     else:

tests/test_shm.py

@@ -31,6 +31,7 @@
 from appose.service import TaskStatus
 
 ndarray_inspect = """
+task.outputs["rsize"] = data.shm.rsize
 task.outputs["size"] = data.shm.size
 task.outputs["dtype"] = data.dtype
 task.outputs["shape"] = data.shape
@@ -41,7 +42,7 @@
 def test_ndarray():
     env = appose.system()
     with env.python() as service:
-        with appose.SharedMemory(create=True, size=2 * 2 * 20 * 25) as shm:
+        with appose.SharedMemory(create=True, rsize=2 * 2 * 20 * 25) as shm:
             # Construct the data.
             shm.buf[0] = 123
             shm.buf[456] = 78
@@ -54,7 +55,10 @@ def test_ndarray():
 
             # Validate the execution result.
             assert TaskStatus.COMPLETE == task.status
-            assert 2 * 20 * 25 * 2 == task.outputs["size"]
+            # The requested size is 2*20*25*2=2000, but actual allocated
+            # shm size varies by platform; e.g. on macOS it is 16384.
+            assert 2 * 20 * 25 * 2 == task.outputs["rsize"]
+            assert task.outputs["size"] >= task.outputs["rsize"]
             assert "uint16" == task.outputs["dtype"]
             assert [2, 20, 25] == task.outputs["shape"]
             assert 123 + 78 + 210 == task.outputs["sum"]

tests/test_types.py

@@ -56,7 +56,7 @@ class TypesTest(unittest.TestCase):
             '"shm":{'                   # noqa: E131
                 '"appose_type":"shm",'  # noqa: E131
                 '"name":"SHM_NAME",'    # noqa: E131
-                '"size":4000'           # noqa: E131
+                '"rsize":4000'          # noqa: E131
             "}"                         # noqa: E131
         "}"
         # fmt: on
@@ -103,7 +103,7 @@ def test_encode(self):
             self.assertEqual(expected, json_str)
 
     def test_decode(self):
-        with appose.SharedMemory(create=True, size=4000) as shm:
+        with appose.SharedMemory(create=True, rsize=4000) as shm:
             shm_name = shm.name
             data = appose.types.decode(self.JSON.replace("SHM_NAME", shm_name))
             self.assertIsNotNone(data)