Raise error if max slices smaller than one slice plus padding slices

See the following PR comments for a discussion on what could be done in
the future to try and salvage the pipeline run instead of raising an
error: #680.

Author: yousefmoazzam

httomo/runner/task_runner.py

@@ -459,6 +459,20 @@ def determine_max_slices(self, section: Section, slicing_dim: int):
             max_slices_methods[idx] = min(max_slices, slices_estimated)
             non_slice_dims_shape = output_dims
 
+        if (
+            min(max_slices_methods)
+            < 1 + self.source.padding[0] + self.source.padding[1]
+        ):
+            padded_method = next(
+                method.method_name for method in section.methods if method.padding
+            )
+            err_str = (
+                "Unable to process data due to GPU memory limitations.\n"
+                f"Please remove method '{padded_method}' from the pipeline, or run on a "
+                "machine with more GPU memory."
+            )
+            raise ValueError(err_str)
+
         section.max_slices = min(max_slices_methods)
 
     def _pass_min_block_length_to_intermediate_data_wrapper(self, section: Section):

tests/runner/test_task_runner.py

@@ -228,6 +228,48 @@ def test_can_determine_max_slices_with_cpu_large(
     assert s[0].max_slices == 64
 
 
+def test_determine_max_slices_raises_error_if_padded_data_cannot_fit(
+    mocker: MockerFixture, tmp_path: PathLike
+):
+    METHOD_NAME = "method_requiring_padding"
+    PADDING = (5, 5)
+    MAX_SLICES_LESS_THAN_CORE_PLUS_PADDING = 9
+    data = np.ones((10, 10, 10), dtype=np.float32)
+    aux = AuxiliaryData(angles=np.ones(data.shape[0], dtype=np.float32))
+    block = DataSetBlock(data, aux)
+    loader = make_test_loader(mocker, block=block, padding=PADDING)
+    padded_gpu_method = make_test_method(
+        mocker,
+        method_name=METHOD_NAME,
+        gpu=True,
+        padding=True,
+        memory_gpu=GpuMemoryRequirement(multiplier=2.0, method="direct"),
+    )
+    mocker.patch.object(
+        padded_gpu_method,
+        "calculate_max_slices",
+        return_value=(MAX_SLICES_LESS_THAN_CORE_PLUS_PADDING, 0),
+    )
+    mocker.patch.object(
+        padded_gpu_method,
+        "calculate_output_dims",
+        return_value=(data.shape[1], data.shape[2]),
+    )
+
+    p = Pipeline(loader=loader, methods=[padded_gpu_method])
+    t = TaskRunner(p, reslice_dir=tmp_path, comm=MPI.COMM_WORLD)
+    t._prepare()
+    s = sectionize(p)
+
+    err_str = (
+        "Unable to process data due to GPU memory limitations.\n"
+        f"Please remove method '{METHOD_NAME}' from the pipeline, or run on a machine with "
+        "more GPU memory."
+    )
+    with pytest.raises(ValueError, match=err_str):
+        t.determine_max_slices(s[0], 0)
+
+
 def test_append_side_outputs(mocker: MockerFixture, tmp_path: PathLike):
     p = Pipeline(make_test_loader(mocker), methods=[])
     t = TaskRunner(p, reslice_dir=tmp_path, comm=MPI.COMM_WORLD)

tests/testing_utils.py

@@ -55,6 +55,7 @@ def make_test_loader(
     block: Optional[DataSetBlock] = None,
     pattern: Pattern = Pattern.all,
     method_name="testloader",
+    padding: tuple[int, int] = (0, 0),
 ) -> LoaderInterface:
     interface: LoaderInterface = mocker.create_autospec(
         LoaderInterface,
@@ -66,9 +67,6 @@ def make_test_loader(
     )
     if block is not None:
 
-        # NOTE: Even though the `padding` parameter is unused, this is needed in order to
-        # replicate the signature of the `make_data_source()` method defined on the
-        # `LoaderInterface` protocol
         def mock_make_data_source(padding) -> DataSetSource:
             ret = mocker.create_autospec(
                 DataSetSource,
@@ -79,7 +77,7 @@ def mock_make_data_source(padding) -> DataSetSource:
                 chunk_index=block.chunk_index,
                 slicing_dim=1 if interface.pattern == Pattern.sinogram else 0,
                 aux_data=block.aux_data,
-                padding=(0, 0),
+                padding=padding,
             )
             slicing_dim: Literal[0, 1, 2] = (
                 1 if interface.pattern == Pattern.sinogram else 0
@@ -103,7 +101,7 @@ def mock_make_data_source(padding) -> DataSetSource:
         mocker.patch.object(
             interface,
             "make_data_source",
-            side_effect=mock_make_data_source,
+            return_value=mock_make_data_source(padding),
         )
     return interface