File tree Expand file tree Collapse file tree 1 file changed +7
-1
lines changed Expand file tree Collapse file tree 1 file changed +7
-1
lines changed Original file line number Diff line number Diff line change @@ -39,6 +39,7 @@ class Grid:
3939 live_mask : ZarrArray | None = None
4040
4141 _TARGET_MEMORY_PER_BATCH = 1 * 1024 ** 3 # 1GB limit for
42+ _INTERNAL_CHUNK_SIZE_TARGET = 10 * 1024 ** 2 # 10MB target for internal chunks
4243
4344 def __post_init__ (self ):
4445 """Initialize convenience properties."""
@@ -100,7 +101,12 @@ def build_map(self, index_headers: HeaderArray) -> None:
100101
101102 # Initialize Zarr arrays for the map and live mask
102103 live_shape = self .shape [:- 1 ]
103- chunks = get_constrained_chunksize (live_shape , map_dtype , 10 * 1024 ** 2 )
104+ chunks = get_constrained_chunksize (
105+ shape = live_shape ,
106+ dtype = map_dtype ,
107+ max_bytes = self ._INTERNAL_CHUNK_SIZE_TARGET ,
108+ )
109+ # Temporary zarrs for ingestion.
104110 self .map = zarr .full (live_shape , fill_value , dtype = map_dtype , chunks = chunks )
105111 self .live_mask = zarr .zeros (live_shape , dtype = "bool" , chunks = chunks )
106112
You can’t perform that action at this time.
0 commit comments