make chunks a little smaller

Author: tasansal

src/mdio/core/utils_write.py

@@ -9,7 +9,7 @@
     from numpy.typing import DTypeLike
 
 
-MAX_SIZE_LIVE_MASK = 256 * 1024**2
+MAX_SIZE_LIVE_MASK = 128 * 1024**2
 MAX_COORDINATES_BYTES = 16 * 1024**2
 
 

tests/unit/test_auto_chunking.py

@@ -48,9 +48,9 @@ class TestAutoChunkLiveMask:
         [
             ((100,), (100,)),  # small 1d
             ((100, 100), (100, 100)),  # small 2d
-            ((50000, 50000), (25000, 25000)),  # large 2d
-            ((1500, 1500, 1500), (750, 750, 750)),  # large 3d
-            ((1000, 1000, 100, 36), (334, 334, 100, 36)),  # large 4d
+            ((50000, 50000), (12500, 12500)),  # large 2d
+            ((1500, 1500, 1500), (500, 500, 500)),  # large 3d
+            ((1000, 1000, 100, 36), (200, 200, 100, 36)),  # large 4d
         ],
     )
     def test_auto_chunk_live_mask(
@@ -65,7 +65,7 @@ def test_auto_chunk_live_mask(
     @pytest.mark.parametrize(
         "shape",
         [
-            # Below are >500MiB. Smaller ones tested above
+            # Below are >250MiB. Smaller ones tested above
             (32768, 32768),
             (46341, 46341),
             (86341, 96341),
@@ -74,7 +74,7 @@ def test_auto_chunk_live_mask(
             (1024, 1024, 1024),
             (215, 215, 215, 215),
             (512, 216, 512, 400),
-            (74, 74, 74, 74, 74),
+            (64, 128, 64, 32, 64),
             (512, 17, 43, 200, 50),
         ],
     )
@@ -84,6 +84,7 @@ def test_auto_chunk_live_mask_nbytes(self, shape: tuple[int, ...]) -> None:
         result = get_live_mask_chunksize(shape)
         chunk_elements = np.prod(result)
 
-        # We want them to be 500MB +/- 25%
+        # We want them to be 250MB +/- 50%
+        print(result)
         assert chunk_elements > MAX_SIZE_LIVE_MASK * 0.75
         assert chunk_elements < MAX_SIZE_LIVE_MASK * 1.25