45 code UI update for new chunking memory approach

documentation/docs/guide/backproject.md

@@ -45,7 +45,8 @@ The offset of the minimum bounding box is stored in the output tiff's descriptio
 - `Output Min Bounding Box` - Save only the minimum volume needed to contain the backprojected slices. The offset will be stored in the `-configuration.json` file under `backprojection_offset`. This value is the (x_min, y_min, z_min).
 - `Binary Backprojection` - Whether or not to binarize all the values of the backprojection. Enable this to backproject a segmentation.
 - `Offset in Filename` - Whether or not to include the (x_min, y_min, z_min) offset for min bounding box in the output file name. Only applies if `Output Min Bounding Box` is true.
-- `Max RAM (GB)` - 0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM.
+- `Process Count` - Number of parallel processes to use during the backprojection process.  Reducing count will slow process but decrease memory usage; at default chunk size each process tends to approximate 0.5 GB in usage.
+- `Chunk Size` - Size of each dimension (x/y/z) of the processing chunk.  Increasing will increase memory usage; performance is a more complex tradeoff and making it too large will negatively affect performance.
 
 ### How Does Backprojection Work?
 

documentation/docs/guide/slicing.md

@@ -36,7 +36,6 @@ Slicing is one of the primary features of Ouroboros, available in the CLI and th
 - `Bounding Box Parameters`
     - `Max Depth` - The maximum depth for binary space partitioning. It is not recommended to change this option unless you encounter RAM issues.
     - `Target Slices Per Box` - If you are running on a low-RAM system, or you are taking very large slices, you may want to decrease this.
-- `Max RAM (GB)` - 0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM.
 
 ### How Does Slicing Work?
 

python/Dockerfile

@@ -3,13 +3,16 @@ FROM thehale/python-poetry:2.1.3-py3.11-slim AS builder
 ENV POETRY_NO_INTERACTION=1 \
     POETRY_VIRTUALENVS_IN_PROJECT=1 \
     POETRY_VIRTUALENVS_CREATE=1 \
-    POETRY_CACHE_DIR=/tmp/poetry_cache
-
-RUN apt-get update -y
-RUN apt-get install \
-	gcc \
-	libpq-dev \
-	cargo
+    POETRY_CACHE_DIR=/tmp/poetry_cache \
+    DCONF_DISABLE_ASYNC=1
+
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends \
+    gcc \
+    libpq-dev \
+    cargo \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 

python/Dockerfile-prod

@@ -2,13 +2,17 @@
 
 # Assumes that dist/*.whl has been built
 
-FROM thehale/python-poetry:2.1.3-py3.11-slim as python-base
+FROM thehale/python-poetry:2.1.3-py3.11-slim AS python-base
 
-RUN apt-get update -y
-RUN apt-get install -y \
-	gcc \
-	libpq-dev \
-	cargo
+ENV DCONF_DISABLE_ASYNC=1
+
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends \
+    gcc \
+    libpq-dev \
+    cargo \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY ./dist/*.whl ./
 

python/ouroboros/helpers/memory_usage.py

@@ -3,22 +3,6 @@
 GIGABYTE = 1024**3
 
 
-def calculate_gigabytes_in_array(array: np.ndarray) -> float:
-    """
-    Calculate the number of gigabytes in a numpy array.
-
-    Parameters:
-    ----------
-        array (numpy.ndarray): The numpy array.
-
-    Returns:
-    -------
-        float: The number of gigabytes in the numpy array.
-    """
-
-    return array.nbytes / GIGABYTE
-
-
 def calculate_gigabytes_from_dimensions(shape: tuple[int], dtype: np.dtype) -> float:
     """
     Calculate the number of gigabytes in an array with the given shape and data type.
@@ -43,38 +27,3 @@ def calculate_gigabytes_from_dimensions(shape: tuple[int], dtype: np.dtype) -> f
     num_bytes = np.multiply(num_elements, dtype_size, dtype=np.uint64, casting='unsafe')
 
     return num_bytes / GIGABYTE
-
-
-def calculate_chunk_size(shape: tuple, dtype: np.dtype, max_ram_gb: int = 0, axis=0) -> int:
-    """
-    Calculate the chunk size based on the shape and dtype of the volume.
-
-    Parameters
-    ----------
-    shape : tuple
-        The shape of the volume.
-    dtype : numpy.dtype
-        The dtype of the volume.
-    max_ram_gb : int
-        The maximum amount of RAM to use in GB.
-    axis : int, optional
-        The axis along which to calculate the chunk size.
-        The default is 0.
-
-    Returns
-    -------
-    int
-        The chunk size.
-    """
-    # Calculate the memory usage of the volume
-    total_gb = calculate_gigabytes_from_dimensions(shape, dtype)
-
-    # Determine the length of the axis
-    axis_length = shape[axis]
-
-    # Calculate the memory usage along the axis
-    axis_gb = total_gb / axis_length
-
-    # Calculate the chunk size
-    # Note: The max function is used to ensure that the chunk size is at least 1
-    return max(int(max_ram_gb / axis_gb), 1)

python/ouroboros/helpers/options.py

@@ -1,3 +1,5 @@
+from os import cpu_count
+
 from pydantic import BaseModel, field_serializer, field_validator
 
 from ouroboros.helpers.bounding_boxes import BoundingBoxParams
@@ -11,7 +13,6 @@ class CommonOptions(BaseModel):
     output_file_name: str  # Name of the output file
     flush_cache: bool = False  # Whether to flush the cache after processing
     make_single_file: bool = True  # Whether to save the output to a single file
-    max_ram_gb: int = 0  # Maximum amount of RAM to use in GB (0 means no limit)
     output_mip_level: int = 0  # MIP level for the output image layer
 
 
@@ -68,6 +69,8 @@ class BackprojectOptions(CommonOptions):
     )
     upsample_order: int = 2  # Order of the interpolation for upsampling
     offset_in_name: bool = True  # Whether to include the offset in the output file name
+    process_count: int = cpu_count() 	# Number of parallel process during the backprojection step.
+    chunk_size: int = 160 	# Size in each dimension of processing chunk.
 
 
 DEFAULT_SLICE_OPTIONS = SliceOptions(

python/ouroboros/pipeline/backproject_pipeline.py

@@ -1,7 +1,6 @@
 import concurrent.futures
 from dataclasses import astuple
 from functools import partial
-from multiprocessing import cpu_count
 from multiprocessing.pool import ThreadPool
 import os
 from pathlib import Path
@@ -26,6 +25,7 @@
 from ouroboros.helpers.volume_cache import VolumeCache, get_mip_volume_sizes, update_writable_rects
 from ouroboros.helpers.bounding_boxes import BoundingBox
 from .pipeline import PipelineStep
+from .pipeline_input import PipelineInput
 from ouroboros.helpers.options import BackprojectOptions
 from ouroboros.helpers.files import (
     format_backproject_resave_volume,
@@ -38,12 +38,8 @@
 from ouroboros.helpers.shapes import DataRange, ImgSliceC
 
 
-DEFAULT_CHUNK_SIZE = 160
-AXIS = 0
-
-
 class BackprojectPipelineStep(PipelineStep):
-    def __init__(self, processes=cpu_count()) -> None:
+    def __init__(self) -> None:
         super().__init__(
             inputs=(
                 "backproject_options",
@@ -52,9 +48,9 @@ def __init__(self, processes=cpu_count()) -> None:
             )
         )
 
-        self.num_processes = processes
-
-    def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]:
+    def _process(self,
+                 input_data: tuple[BackprojectOptions, VolumeCache, np.ndarray, PipelineInput]
+                 ) -> tuple[any, None] | tuple[None, any]:
         config, volume_cache, slice_rects, pipeline_input = input_data
 
         # Verify that a config object is provided
@@ -169,8 +165,8 @@ def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]:
 
         # Allocate procs equally between BP math and writing if we're rescaling, otherwise 3-1 favoring
         # the BP calculation.
-        exec_procs = self.num_processes // 4 * (2 if scaling_factors is not None else 3)
-        write_procs = self.num_processes // 4 * (2 if scaling_factors is not None else 1)
+        exec_procs = config.process_count // 4 * (2 if scaling_factors is not None else 3)
+        write_procs = config.process_count // 4 * (2 if scaling_factors is not None else 1)
 
         # Process each bounding box in parallel, writing the results to the backprojected volume
         try:
@@ -179,7 +175,7 @@ def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]:
                 bp_futures = []
                 write_futures = []
 
-                chunk_range = DataRange(FPShape.make_with(0), FPShape, FPShape.make_with(DEFAULT_CHUNK_SIZE))
+                chunk_range = DataRange(FPShape.make_with(0), FPShape, FPShape.make_with(config.chunk_size))
                 chunk_iter = partial(BackProjectIter, shape=FPShape, slice_rects=np.array(slice_rects))
                 processed = np.zeros(astuple(chunk_range.length))
                 z_sources = np.zeros((write_shape[0], ) + astuple(chunk_range.length), dtype=bool)
@@ -204,8 +200,8 @@ def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]:
                 def note_written(write_future):
                     nonlocal pages_written
                     pages_written += 1
-                    self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / self.num_processes)
-                                         + (pages_written / num_pages) * (write_procs / self.num_processes))
+                    self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / config.process_count)
+                                         + (pages_written / num_pages) * (write_procs / config.process_count))
                     for key, value in write_future.result().items():
                         self.add_timing(key, value)
 
@@ -220,10 +216,10 @@ def note_written(write_future):
 
                     # Update the progress bar
                     processed[index] = 1
-                    self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / self.num_processes)
-                                         + (pages_written / num_pages) * (write_procs / self.num_processes))
+                    self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / config.process_count)
+                                         + (pages_written / num_pages) * (write_procs / config.process_count))
 
-                    update_writable_rects(processed, slice_rects, min_dim, writeable, DEFAULT_CHUNK_SIZE)
+                    update_writable_rects(processed, slice_rects, min_dim, writeable, config.chunk_size)
 
                     if np.any(writeable == 1):
                         write = np.flatnonzero(writeable == 1)

python/pyproject.toml

@@ -28,7 +28,7 @@ uuid = "^1.30"
 sse-starlette = "^2.3.2"
 typing-extensions = "^4.14.0"
 uvicorn = "^0.29.0"
-opencv-python = "^4.0.0"
+opencv-python-headless = "^4.0.0"
 filelock = "^3.20.0"
 
 [tool.poetry.group.dev.dependencies]

python/test/helpers/test_memory_usage.py

@@ -1,22 +1,6 @@
-from ouroboros.helpers.memory_usage import (
-    GIGABYTE,
-    calculate_chunk_size,
-    calculate_gigabytes_in_array,
-    calculate_gigabytes_from_dimensions,
-)
-
 import numpy as np
 
-
-def test_calculate_gigabytes_in_array():
-    # Create a numpy array with 1 GB size
-    array = np.zeros(int((GIGABYTE) / np.dtype(np.float64).itemsize), dtype=np.float64)
-
-    # Calculate the expected result
-    expected_result = 1.0
-
-    # Call the function and check the result
-    assert calculate_gigabytes_in_array(array) == expected_result
+from ouroboros.helpers.memory_usage import calculate_gigabytes_from_dimensions
 
 
 def test_calculate_gigabytes_from_dimensions():
@@ -29,17 +13,3 @@ def test_calculate_gigabytes_from_dimensions():
 
     # Call the function and check the result
     assert calculate_gigabytes_from_dimensions(shape, dtype) == expected_result
-
-
-def test_calculate_chunk_size():
-    shape = (1024, 1024, 1024)
-    dtype = np.float64
-    max_ram_gb = 8
-    expected_chunk_size = 1024  # Expected result based on the function logic
-    assert calculate_chunk_size(shape, dtype, max_ram_gb) == expected_chunk_size
-
-    shape = (1024, 1024, 1024)
-    dtype = np.float64
-    max_ram_gb = 0
-    expected_chunk_size = 1  # Expected result based on the function logic
-    assert calculate_chunk_size(shape, dtype, max_ram_gb) == expected_chunk_size

src/renderer/src/interfaces/options.tsx

@@ -254,10 +254,7 @@ export class SliceOptionsFile extends CompoundEntry {
 				).withDescription(
 					'If you are running on a low-RAM system, or you are taking very large slices, you may want to decrease this.'
 				)
-			]),
-			new Entry('max_ram_gb', 'Max RAM (GB) (0 = no limit)', 0, 'number').withDescription(
-				'0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM.'
-			)
+			])
 		])
 
 		this.setValue(values)
@@ -289,12 +286,7 @@ export class BackprojectOptionsFile extends CompoundEntry {
 			new Entry('output_mip_level', 'Output MIP Level', 0, 'number').withDescription(
 				'The MIP level to output the backprojection in (essentially an upsample option). Use this if you downsampled in the slicing step.  MIP levels that would downsample are ignored currently.'
 			),
-			new Entry(
-				'upsample_order',
-				'Upsample Order (2 = Cubic)',
-				2,
-				'number'
-			).withDescription(
+			new Entry('upsample_order', 'Upsample Order (2 = Cubic)', 2, 'number').withDescription(
 				'The interpolation order Ouroboros uses to interpolate values from a lower MIP level (matches opencv interpolation parameter). If you check the binary option, feel free to set this to 0.'
 			),
 			new Entry(
@@ -328,8 +320,16 @@ export class BackprojectOptionsFile extends CompoundEntry {
 				'Whether or not to include the (x_min, y_min, z_min) offset for min bounding box in the output file name. Only applies if `Output Min Bounding Box` is true.'
 			),
 			new Entry('flush_cache', 'Flush CloudVolume Cache', false, 'boolean').withHidden(),
-			new Entry('max_ram_gb', 'Max RAM (GB) (0 = no limit)', 0, 'number').withDescription(
-				'0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM.'
+			new Entry(
+				'process_count',
+				'Process Count',
+				navigator.hardwareConcurrency,
+				'number'
+			).withDescription(
+				'Number of parallel processes to use during the backprojection process.'
+			),
+			new Entry('chunk_size', 'Chunk Size', 160, 'number').withDescription(
+				'Size of each dimension (x/y/z) of the processing chunk.  Larger chunks will use more memory and (after a certain point) slow down calculation.'
 			)
 		])