diff --git a/documentation/docs/guide/backproject.md b/documentation/docs/guide/backproject.md index 08d2ecd..f186530 100644 --- a/documentation/docs/guide/backproject.md +++ b/documentation/docs/guide/backproject.md @@ -45,7 +45,8 @@ The offset of the minimum bounding box is stored in the output tiff's descriptio - `Output Min Bounding Box` - Save only the minimum volume needed to contain the backprojected slices. The offset will be stored in the `-configuration.json` file under `backprojection_offset`. This value is the (x_min, y_min, z_min). - `Binary Backprojection` - Whether or not to binarize all the values of the backprojection. Enable this to backproject a segmentation. - `Offset in Filename` - Whether or not to include the (x_min, y_min, z_min) offset for min bounding box in the output file name. Only applies if `Output Min Bounding Box` is true. -- `Max RAM (GB)` - 0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM. +- `Process Count` - Number of parallel processes to use during the backprojection process. Reducing count will slow process but decrease memory usage; at default chunk size each process tends to approximate 0.5 GB in usage. +- `Chunk Size` - Size of each dimension (x/y/z) of the processing chunk. Increasing will increase memory usage; performance is a more complex tradeoff and making it too large will negatively affect performance. ### How Does Backprojection Work? diff --git a/documentation/docs/guide/slicing.md b/documentation/docs/guide/slicing.md index b69ec9d..4319d7b 100644 --- a/documentation/docs/guide/slicing.md +++ b/documentation/docs/guide/slicing.md @@ -36,7 +36,6 @@ Slicing is one of the primary features of Ouroboros, available in the CLI and th - `Bounding Box Parameters` - `Max Depth` - The maximum depth for binary space partitioning. It is not recommended to change this option unless you encounter RAM issues. - `Target Slices Per Box` - If you are running on a low-RAM system, or you are taking very large slices, you may want to decrease this. -- `Max RAM (GB)` - 0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM. ### How Does Slicing Work? diff --git a/package-lock.json b/package-lock.json index f976c92..3f0b588 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4894,6 +4894,7 @@ "integrity": "sha512-rcJUkMfnJpfCboZoOOPf4L29TRtEieHNOeAbYPWPxlaBw/Z1RKrRA86dOI9rwaI4tQSc/RD82zTNHprfUHXsoQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "app-builder-lib": "24.13.3", "builder-util": "24.13.1", @@ -10961,7 +10962,6 @@ "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", "dev": true, "license": "MIT", - "peer": true, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } diff --git a/python/Dockerfile b/python/Dockerfile index c409c5a..48306ce 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -3,13 +3,16 @@ FROM thehale/python-poetry:2.1.3-py3.11-slim AS builder ENV POETRY_NO_INTERACTION=1 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ POETRY_VIRTUALENVS_CREATE=1 \ - POETRY_CACHE_DIR=/tmp/poetry_cache - -RUN apt-get update -y -RUN apt-get install \ - gcc \ - libpq-dev \ - cargo + POETRY_CACHE_DIR=/tmp/poetry_cache \ + DCONF_DISABLE_ASYNC=1 + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + gcc \ + libpq-dev \ + cargo \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/python/Dockerfile-prod b/python/Dockerfile-prod index 0e55fd7..04dceb6 100644 --- a/python/Dockerfile-prod +++ b/python/Dockerfile-prod @@ -2,13 +2,17 @@ # Assumes that dist/*.whl has been built -FROM thehale/python-poetry:2.1.3-py3.11-slim as python-base +FROM thehale/python-poetry:2.1.3-py3.11-slim AS python-base -RUN apt-get update -y -RUN apt-get install -y \ - gcc \ - libpq-dev \ - cargo +ENV DCONF_DISABLE_ASYNC=1 + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + gcc \ + libpq-dev \ + cargo \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY ./dist/*.whl ./ diff --git a/python/ouroboros/helpers/memory_usage.py b/python/ouroboros/helpers/memory_usage.py index d18accf..b80f8f5 100644 --- a/python/ouroboros/helpers/memory_usage.py +++ b/python/ouroboros/helpers/memory_usage.py @@ -3,22 +3,6 @@ GIGABYTE = 1024**3 -def calculate_gigabytes_in_array(array: np.ndarray) -> float: - """ - Calculate the number of gigabytes in a numpy array. - - Parameters: - ---------- - array (numpy.ndarray): The numpy array. - - Returns: - ------- - float: The number of gigabytes in the numpy array. - """ - - return array.nbytes / GIGABYTE - - def calculate_gigabytes_from_dimensions(shape: tuple[int], dtype: np.dtype) -> float: """ Calculate the number of gigabytes in an array with the given shape and data type. @@ -43,38 +27,3 @@ def calculate_gigabytes_from_dimensions(shape: tuple[int], dtype: np.dtype) -> f num_bytes = np.multiply(num_elements, dtype_size, dtype=np.uint64, casting='unsafe') return num_bytes / GIGABYTE - - -def calculate_chunk_size(shape: tuple, dtype: np.dtype, max_ram_gb: int = 0, axis=0) -> int: - """ - Calculate the chunk size based on the shape and dtype of the volume. - - Parameters - ---------- - shape : tuple - The shape of the volume. - dtype : numpy.dtype - The dtype of the volume. - max_ram_gb : int - The maximum amount of RAM to use in GB. - axis : int, optional - The axis along which to calculate the chunk size. - The default is 0. - - Returns - ------- - int - The chunk size. - """ - # Calculate the memory usage of the volume - total_gb = calculate_gigabytes_from_dimensions(shape, dtype) - - # Determine the length of the axis - axis_length = shape[axis] - - # Calculate the memory usage along the axis - axis_gb = total_gb / axis_length - - # Calculate the chunk size - # Note: The max function is used to ensure that the chunk size is at least 1 - return max(int(max_ram_gb / axis_gb), 1) diff --git a/python/ouroboros/helpers/options.py b/python/ouroboros/helpers/options.py index d3b9a66..55a466c 100644 --- a/python/ouroboros/helpers/options.py +++ b/python/ouroboros/helpers/options.py @@ -1,3 +1,5 @@ +from os import cpu_count + from pydantic import BaseModel, field_serializer, field_validator from ouroboros.helpers.bounding_boxes import BoundingBoxParams @@ -11,7 +13,6 @@ class CommonOptions(BaseModel): output_file_name: str # Name of the output file flush_cache: bool = False # Whether to flush the cache after processing make_single_file: bool = True # Whether to save the output to a single file - max_ram_gb: int = 0 # Maximum amount of RAM to use in GB (0 means no limit) output_mip_level: int = 0 # MIP level for the output image layer @@ -68,6 +69,8 @@ class BackprojectOptions(CommonOptions): ) upsample_order: int = 2 # Order of the interpolation for upsampling offset_in_name: bool = True # Whether to include the offset in the output file name + process_count: int = cpu_count() # Number of parallel process during the backprojection step. + chunk_size: int = 160 # Size in each dimension of processing chunk. DEFAULT_SLICE_OPTIONS = SliceOptions( diff --git a/python/ouroboros/pipeline/backproject_pipeline.py b/python/ouroboros/pipeline/backproject_pipeline.py index 5cef5be..1690fc5 100644 --- a/python/ouroboros/pipeline/backproject_pipeline.py +++ b/python/ouroboros/pipeline/backproject_pipeline.py @@ -1,7 +1,6 @@ import concurrent.futures from dataclasses import astuple from functools import partial -from multiprocessing import cpu_count from multiprocessing.pool import ThreadPool import os from pathlib import Path @@ -26,6 +25,7 @@ from ouroboros.helpers.volume_cache import VolumeCache, get_mip_volume_sizes, update_writable_rects from ouroboros.helpers.bounding_boxes import BoundingBox from .pipeline import PipelineStep +from .pipeline_input import PipelineInput from ouroboros.helpers.options import BackprojectOptions from ouroboros.helpers.files import ( format_backproject_resave_volume, @@ -38,12 +38,8 @@ from ouroboros.helpers.shapes import DataRange, ImgSliceC -DEFAULT_CHUNK_SIZE = 160 -AXIS = 0 - - class BackprojectPipelineStep(PipelineStep): - def __init__(self, processes=cpu_count()) -> None: + def __init__(self) -> None: super().__init__( inputs=( "backproject_options", @@ -52,9 +48,9 @@ def __init__(self, processes=cpu_count()) -> None: ) ) - self.num_processes = processes - - def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]: + def _process(self, + input_data: tuple[BackprojectOptions, VolumeCache, np.ndarray, PipelineInput] + ) -> tuple[any, None] | tuple[None, any]: config, volume_cache, slice_rects, pipeline_input = input_data # Verify that a config object is provided @@ -169,8 +165,8 @@ def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]: # Allocate procs equally between BP math and writing if we're rescaling, otherwise 3-1 favoring # the BP calculation. - exec_procs = self.num_processes // 4 * (2 if scaling_factors is not None else 3) - write_procs = self.num_processes // 4 * (2 if scaling_factors is not None else 1) + exec_procs = config.process_count // 4 * (2 if scaling_factors is not None else 3) + write_procs = config.process_count // 4 * (2 if scaling_factors is not None else 1) # Process each bounding box in parallel, writing the results to the backprojected volume try: @@ -179,7 +175,7 @@ def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]: bp_futures = [] write_futures = [] - chunk_range = DataRange(FPShape.make_with(0), FPShape, FPShape.make_with(DEFAULT_CHUNK_SIZE)) + chunk_range = DataRange(FPShape.make_with(0), FPShape, FPShape.make_with(config.chunk_size)) chunk_iter = partial(BackProjectIter, shape=FPShape, slice_rects=np.array(slice_rects)) processed = np.zeros(astuple(chunk_range.length)) z_sources = np.zeros((write_shape[0], ) + astuple(chunk_range.length), dtype=bool) @@ -204,8 +200,8 @@ def _process(self, input_data: any) -> tuple[any, None] | tuple[None, any]: def note_written(write_future): nonlocal pages_written pages_written += 1 - self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / self.num_processes) - + (pages_written / num_pages) * (write_procs / self.num_processes)) + self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / config.process_count) + + (pages_written / num_pages) * (write_procs / config.process_count)) for key, value in write_future.result().items(): self.add_timing(key, value) @@ -220,10 +216,10 @@ def note_written(write_future): # Update the progress bar processed[index] = 1 - self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / self.num_processes) - + (pages_written / num_pages) * (write_procs / self.num_processes)) + self.update_progress((np.sum(processed) / len(chunk_range)) * (exec_procs / config.process_count) + + (pages_written / num_pages) * (write_procs / config.process_count)) - update_writable_rects(processed, slice_rects, min_dim, writeable, DEFAULT_CHUNK_SIZE) + update_writable_rects(processed, slice_rects, min_dim, writeable, config.chunk_size) if np.any(writeable == 1): write = np.flatnonzero(writeable == 1) diff --git a/python/poetry.lock b/python/poetry.lock index 489bcc2..d0c64dc 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -2696,20 +2696,20 @@ files = [ ] [[package]] -name = "opencv-python" +name = "opencv-python-headless" version = "4.11.0.86" description = "Wrapper package for OpenCV python bindings." optional = false python-versions = ">=3.6" groups = ["main"] files = [ - {file = "opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4"}, - {file = "opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:432f67c223f1dc2824f5e73cdfcd9db0efc8710647d4e813012195dc9122a52a"}, - {file = "opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:9d05ef13d23fe97f575153558653e2d6e87103995d54e6a35db3f282fe1f9c66"}, - {file = "opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b92ae2c8852208817e6776ba1ea0d6b1e0a1b5431e971a2a0ddd2a8cc398202"}, - {file = "opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b02611523803495003bd87362db3e1d2a0454a6a63025dc6658a9830570aa0d"}, - {file = "opencv_python-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:810549cb2a4aedaa84ad9a1c92fbfdfc14090e2749cedf2c1589ad8359aa169b"}, - {file = "opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec"}, + {file = "opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:48128188ade4a7e517237c8e1e11a9cdf5c282761473383e77beb875bb1e61ca"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6efabcaa9df731f29e5ea9051776715b1bdd1845d7c9530065c7951d2a2899eb"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca"}, ] [package.dependencies] @@ -4810,4 +4810,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "1bf3f1761d13853aa420300e55e88f9778a98fd0b0c65a334c074770513ea13c" +content-hash = "09f311bc53ad4638cd70a34e87e7edac84acda29812a35454bd688c49282cb0c" diff --git a/python/pyproject.toml b/python/pyproject.toml index f609da6..fa5bb1b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,7 +28,7 @@ uuid = "^1.30" sse-starlette = "^2.3.2" typing-extensions = "^4.14.0" uvicorn = "^0.29.0" -opencv-python = "^4.0.0" +opencv-python-headless = "^4.0.0" filelock = "^3.20.0" [tool.poetry.group.dev.dependencies] diff --git a/python/test/helpers/test_memory_usage.py b/python/test/helpers/test_memory_usage.py index 826c0ce..ab1524b 100644 --- a/python/test/helpers/test_memory_usage.py +++ b/python/test/helpers/test_memory_usage.py @@ -1,22 +1,6 @@ -from ouroboros.helpers.memory_usage import ( - GIGABYTE, - calculate_chunk_size, - calculate_gigabytes_in_array, - calculate_gigabytes_from_dimensions, -) - import numpy as np - -def test_calculate_gigabytes_in_array(): - # Create a numpy array with 1 GB size - array = np.zeros(int((GIGABYTE) / np.dtype(np.float64).itemsize), dtype=np.float64) - - # Calculate the expected result - expected_result = 1.0 - - # Call the function and check the result - assert calculate_gigabytes_in_array(array) == expected_result +from ouroboros.helpers.memory_usage import calculate_gigabytes_from_dimensions def test_calculate_gigabytes_from_dimensions(): @@ -29,17 +13,3 @@ def test_calculate_gigabytes_from_dimensions(): # Call the function and check the result assert calculate_gigabytes_from_dimensions(shape, dtype) == expected_result - - -def test_calculate_chunk_size(): - shape = (1024, 1024, 1024) - dtype = np.float64 - max_ram_gb = 8 - expected_chunk_size = 1024 # Expected result based on the function logic - assert calculate_chunk_size(shape, dtype, max_ram_gb) == expected_chunk_size - - shape = (1024, 1024, 1024) - dtype = np.float64 - max_ram_gb = 0 - expected_chunk_size = 1 # Expected result based on the function logic - assert calculate_chunk_size(shape, dtype, max_ram_gb) == expected_chunk_size diff --git a/src/renderer/src/interfaces/options.tsx b/src/renderer/src/interfaces/options.tsx index b9106e1..4d17958 100644 --- a/src/renderer/src/interfaces/options.tsx +++ b/src/renderer/src/interfaces/options.tsx @@ -254,10 +254,7 @@ export class SliceOptionsFile extends CompoundEntry { ).withDescription( 'If you are running on a low-RAM system, or you are taking very large slices, you may want to decrease this.' ) - ]), - new Entry('max_ram_gb', 'Max RAM (GB) (0 = no limit)', 0, 'number').withDescription( - '0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM.' - ) + ]) ]) this.setValue(values) @@ -289,12 +286,7 @@ export class BackprojectOptionsFile extends CompoundEntry { new Entry('output_mip_level', 'Output MIP Level', 0, 'number').withDescription( 'The MIP level to output the backprojection in (essentially an upsample option). Use this if you downsampled in the slicing step. MIP levels that would downsample are ignored currently.' ), - new Entry( - 'upsample_order', - 'Upsample Order (2 = Cubic)', - 2, - 'number' - ).withDescription( + new Entry('upsample_order', 'Upsample Order (2 = Cubic)', 2, 'number').withDescription( 'The interpolation order Ouroboros uses to interpolate values from a lower MIP level (matches opencv interpolation parameter). If you check the binary option, feel free to set this to 0.' ), new Entry( @@ -328,8 +320,16 @@ export class BackprojectOptionsFile extends CompoundEntry { 'Whether or not to include the (x_min, y_min, z_min) offset for min bounding box in the output file name. Only applies if `Output Min Bounding Box` is true.' ), new Entry('flush_cache', 'Flush CloudVolume Cache', false, 'boolean').withHidden(), - new Entry('max_ram_gb', 'Max RAM (GB) (0 = no limit)', 0, 'number').withDescription( - '0 indicates no RAM limit. Setting a RAM limit allows Ouroboros to optimize performance and avoid overusing RAM.' + new Entry( + 'process_count', + 'Process Count', + navigator.hardwareConcurrency, + 'number' + ).withDescription( + 'Number of parallel processes to use during the backprojection process.' + ), + new Entry('chunk_size', 'Chunk Size', 160, 'number').withDescription( + 'Size of each dimension (x/y/z) of the processing chunk. Larger chunks will use more memory and (after a certain point) slow down calculation.' ) ])