Merge branch 'main' into cupy-update-radway-65

Author: dkazanc

docs/source/backends/templates.rst

@@ -21,6 +21,8 @@ These are archived YAML templates that can be used with already released and tag
    :download:`HTTomo version 2.5 templates <../templates_archive/httomo_ver2_5_yaml_templates.zip>`
 
    :download:`HTTomo version 2.6 templates <../templates_archive/httomo_ver2_6_yaml_templates.zip>`
+   
+   :download:`HTTomo version 3.0 templates <../templates_archive/httomo_ver3_0_yaml_templates.zip>`
 
 .. _latest_templates:
 

docs/source/howto/run_httomo.rst

@@ -235,14 +235,15 @@ directory created by HTTomo would be
 Options/flags
 #############
 
-The :code:`run` command has 16 options/flags:
+The :code:`run` command has 17 options/flags:
 
 - :code:`--output-folder-name`
 - :code:`--save-all`
 - :code:`--gpu-id`
 - :code:`--reslice-dir`
 - :code:`--max-cpu-slices`
 - :code:`--max-memory`
+- :code:`--save-snapshots`
 - :code:`--monitor`
 - :code:`--monitor-output`
 - :code:`--intermediate-format`
@@ -364,6 +365,22 @@ The :code:`--max-memory` flag is for telling HTTomo how much RAM the machine has
 so then it can switch to using a file during execution of the pipeline if
 necessary.
 
+:code:`--save-snapshots`
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+When this flag is enabled, the pipeline saves image snapshots at specific execution points.
+These snapshots are captured during selected methods - typically when a section boundary 
+is reached and data is transferred to the CPU. At which time a slice of the data is saved for 
+inspection.
+
+This feature is particularly useful for complex pipelines (e.g. 360 degrees with stitching and phase contrast),
+where intermediate processing steps involved in reconstruction may unintentionally alter
+the data. By reviewing these snapshot images (JPEGs), users can more easily pinpoint
+where issues are introduced in the pipeline. 
+
+Enabling snapshots incurs almost no additional computational cost, unlike the :code:`--save-all` 
+flag, which requires saving the entire dataset into a file for each method.
+
 :code:`--monitor`
 ~~~~~~~~~~~~~~~~~
 

docs/source/pipelines/yaml.rst

@@ -22,6 +22,8 @@ These are archived full YAML pipelines that can be used with already released an
    :download:`HTTomo version 2.5 full YAML pipelines <../templates_archive/httomo_ver2_5_full_yaml_pipelines.zip>`
 
    :download:`HTTomo version 2.6 full YAML pipelines <../templates_archive/httomo_ver2_6_full_yaml_pipelines.zip>`
+   
+   :download:`HTTomo version 3.0 full YAML pipelines <../templates_archive/httomo_ver3_0_full_yaml_pipelines.zip>`
 
 
 .. warning:: At DLS, the templates below should work with the :code:`httomo/latest` module, however, for production please use :ref:`full_pipelines_archived`. 
@@ -109,4 +111,4 @@ One can build CPU-only pipelines by using mostly TomoPy methods.
 .. dropdown:: CPU pipeline using auto-centering and the gridrec reconstruction method from TomoPy.
 
     .. literalinclude:: ../pipelines_full/tomopy_gridrec.yaml
-        :language: yaml
+        :language: yaml

docs/source/templates_archive/httomo_ver3_0_full_yaml_pipelines.zip

@@ -92,6 +92,8 @@ other fields directly under the :code:`parameters` field.
 * We check methods exist for the given module path.
 * We check that the parameters for each method are valid. For example, :code:`find_center_vo` method from :code:`tomopy.recon.rotation` takes :code:`ratio` as a parameter with a float value. If you pass a string instead, it will raise an error. Again the trick is to refer the documentation always.
 * We check the required parameters for each method are present.
+* We check parameters that are omitted are not required (in particular, that omitted parameters
+  have a default value that can be assumed)
 * If you pass :code:`IN_DATA` (path to the data) along with the yaml config, as:
 
 .. code-block:: console

docs/source/templates_archive/httomo_ver3_0_yaml_templates.zip

@@ -21,7 +21,6 @@
 from httomo.runner.task_runner import TaskRunner
 from httomo.ui_layer import UiLayer, PipelineFormat
 
-
 try:
     from . import __version__
 except:
@@ -137,6 +136,11 @@ def check(pipeline: Union[Path, str], in_data_file: Optional[Path] = None):
     default="0",
     help="Limit the amount of memory used by the pipeline to the given memory (supports strings like 3.2G or bytes)",
 )
+@click.option(
+    "--save-snapshots",
+    is_flag=True,
+    help="Save intermediate images (snapshots) from some methods in the pipeline.",
+)
 @click.option(
     "--monitor",
     type=click.STRING,
@@ -212,6 +216,7 @@ def run(
     reslice_dir: Union[Path, None],
     max_cpu_slices: int,
     max_memory: str,
+    save_snapshots: bool,
     monitor: List[str],
     pipeline_format: str,
     monitor_output: TextIO,
@@ -265,6 +270,7 @@ def run(
             monitor,
             monitor_output,
             reslice_dir,
+            save_snapshots,
         )
     else:
         execute_sweep_run(pipeline, global_comm)
@@ -398,6 +404,7 @@ def execute_high_throughput_run(
     monitor: List[str],
     monitor_output: TextIO,
     reslice_dir: Union[Path, None],
+    save_snapshots: bool,
 ) -> None:
     # we use half the memory for blocks since we typically have inputs/output
     memory_limit = transform_limit_str_to_bytes(max_memory) // 2
@@ -416,6 +423,7 @@ def execute_high_throughput_run(
             global_comm,
             monitor=mon,
             memory_limit_bytes=memory_limit,
+            save_snapshots=save_snapshots,
         )
         runner.execute()
         if mon is not None:

docs/source/utilities/yaml_checker.rst

@@ -117,7 +117,6 @@ def aux_data(self) -> AuxiliaryData:
     def write_block(self, block: DataSetBlock):
         if self._readonly:
             raise ValueError("Cannot write after creating a reader")
-        block.to_cpu()
         start = max(block.chunk_index_unpadded)
         if self._data is None:
             # if non-slice dims in block are different, update the shapes here
@@ -284,7 +283,19 @@ def __init__(
             self._slicing_dim = source.slicing_dim
             self._data = source_data
         else:
+            start = time.perf_counter()
             self._data = self._reslice(source.slicing_dim, slicing_dim, source_data)
+            end = time.perf_counter()
+            if slicing_dim == 1:
+                log_once(
+                    f"Slicing axis change (reslice) from projection to sinogram took {(end - start):.9f}s.",
+                    level=logging.INFO,
+                )
+            else:
+                log_once(
+                    f"Slicing axis change (reslice) from sinogram to projection took {(end - start):.9f}s.",
+                    level=logging.INFO,
+                )
             self._slicing_dim = slicing_dim
 
         self._padding = (0, 0) if padding is None else padding
@@ -343,7 +354,7 @@ def _reslice(
                 array, newdim, startidx = reslice(
                     data, old_slicing_dim + 1, new_slicing_dim + 1, self._comm
                 )
-                self._chunk_shape = array.shape  #  type: ignore
+                self._chunk_shape = array.shape  # type: ignore
                 assert newdim == new_slicing_dim + 1
                 idx = [0, 0, 0]
                 idx[new_slicing_dim] = startidx

httomo/cli.py

@@ -7,7 +7,6 @@
 
 from httomo.utils import log_once
 
-
 __all__ = ["alltoall", "alltoall_ring"]
 
 

httomo/data/dataset_store.py

@@ -5,7 +5,6 @@
 
 from httomo.preview import PreviewConfig
 
-
 DIMS = [0, 1, 2]