Merge pull request #604 from neutrons/EWM7798-DiffractionFocussing-FullBinsOnly

Use `DiffractionFocussing` with 'FullBinsOnly' in `FocusSpectraAlgorithm`

Author: ekapadi

src/snapred/backend/dao/RunMetadata.py

@@ -49,8 +49,8 @@ class RunMetadata(BaseModel, Mapping):
 
     runTitle: str
 
-    startTime: datetime
-    endTime: datetime
+    startTime: datetime.datetime
+    endTime: datetime.datetime
 
     protonCharge: float
 

src/snapred/backend/dao/ingredients/ReductionGroupProcessingIngredients.py

@@ -5,6 +5,7 @@
 
 class ReductionGroupProcessingIngredients(BaseModel):
     pixelGroup: PixelGroup
+    preserveEvents: bool
 
     model_config = ConfigDict(
         extra="forbid",

src/snapred/backend/dao/ingredients/ReductionIngredients.py

@@ -51,7 +51,7 @@ def getDetectorPeaks(self, groupingIndex: int) -> List[GroupPeakList]:
         return self.detectorPeaksMany[groupingIndex]
 
     def groupProcessing(self, groupingIndex: int) -> ReductionGroupProcessingIngredients:
-        return ReductionGroupProcessingIngredients(pixelGroup=self.pixelGroups[groupingIndex])
+        return ReductionGroupProcessingIngredients(pixelGroup=self.pixelGroups[groupingIndex], preserveEvents=False)
 
     def generateFocussedVanadium(self, groupingIndex: int) -> GenerateFocussedVanadiumIngredients:
         return GenerateFocussedVanadiumIngredients(

src/snapred/backend/dao/request/FocusSpectraRequest.py

@@ -9,6 +9,7 @@ class FocusSpectraRequest(BaseModel):
     runNumber: str
     useLiteMode: bool
     focusGroup: FocusGroup
+    preserveEvents: bool
 
     inputWorkspace: str
     groupingWorkspace: str

src/snapred/backend/data/GroceryService.py

@@ -6,6 +6,7 @@
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 
 import numpy as np
+from mantid.api import FileLoaderRegistry
 from mantid.dataobjects import MaskWorkspace
 from pydantic import validate_call
 
@@ -282,11 +283,7 @@ def _createDiffCalTableFilepath(
 
     @validate_call
     def _lookupNormalizationWorkspaceFilename(
-        self,
-        normcalRunNumber: str,
-        useLiteMode: bool,
-        version: Optional[int],
-        state: str,
+        self, normcalRunNumber: str, useLiteMode: bool, version: Optional[int], state: str
     ) -> str:
         # NOTE: The normCal Record currently does NOT store workspace type -> workspace name mappings.
         #       It is just a list of workspace names. So we need to infer.
@@ -852,7 +849,14 @@ def _fetchNeutronDataSingleUse(self, item: GroceryListItem, missingDataHandler:
             self.getCloneOfWorkspace(rawWorkspaceName, workspaceName)
         elif bool(filePath) and filePath.exists() and item.liveDataArgs is None:
             workspaceName = self._createNeutronWorkspaceName(runNumber, useLiteMode)
-            data = self.grocer.executeRecipe(str(filePath), workspaceName, loader)
+            loaderArgs = "{}"
+            if not bool(loader) and Config["nexus.dataFormat.event"]:
+                # If the loader hasn't been specified, check if this is original input data in event format.
+                # In this case, specifying only a single bin allows a much faster load.
+                loader = FileLoaderRegistry.Instance().chooseLoader(str(filePath)).name()
+                if loader == "LoadEventNexus":
+                    loaderArgs = '{"NumberOfBins": 1}'
+            data = self.grocer.executeRecipe(str(filePath), workspaceName, loader, loaderArgs=loaderArgs)
         else:
             data = missingDataHandler()
             workspaceName = data["workspace"]
@@ -1242,11 +1246,12 @@ def fetchNormalizationWorkspace(self, item: GroceryListItem) -> Dict[str, Any]:
         :rtype: Dict[str, Any]
         """
 
-        runNumber, useLiteMode, version, state, hidden = (
+        runNumber, useLiteMode, version, state, loader, hidden = (
             item.runNumber,
             item.useLiteMode,
             item.normCalVersion,
             item.state,
+            item.loader,
             item.hidden,
         )
         normcalRunNumber = self._lookupNormcalRunNumber(runNumber, useLiteMode, version, state)
@@ -1262,14 +1267,14 @@ def fetchNormalizationWorkspace(self, item: GroceryListItem) -> Dict[str, Any]:
             self._clearNormalizationCache()
 
             # then load the new one
-            filename = self._lookupNormalizationWorkspaceFilename(normcalRunNumber, useLiteMode, version, state)
+            filePath = self._lookupNormalizationWorkspaceFilename(normcalRunNumber, useLiteMode, version, state)
+
+            # Unfortunately, `LoadNexusProcessed` does not support the `NumberOfBins=1` optimization for loading
+            # event data.  As a work-around: event-format normalization data will be saved with only one bin
+            # to speed up reload.  (See also:  "nexus.dataFormat.event" flag in "application.yml".)
 
             # Note: 'LoadNexusProcessed' neither requires nor makes use of an instrument donor.
-            data = self.grocer.executeRecipe(
-                filename=filename,
-                workspace=workspaceName,
-                loader="LoadNexusProcessed",
-            )
+            data = self.grocer.executeRecipe(filename=filePath, workspace=workspaceName, loader=loader)
             self._processNeutronDataCopy(item, workspaceName)
             self.normalizationCache.add(workspaceName)
         return data
@@ -1620,7 +1625,7 @@ def getResidentWorkspaces(self, excludeCache: bool):
         return list(workspaces)
 
     def _filterEvents(self, runNumber, workspaceName):
-        # NOTE: We always want to generate the instruemnt state, to get the latest SNAPInstPRm parameters.
+        # NOTE: We always want to generate the instrument state, to get the latest SNAPInstPRm parameters.
         instrumentState = self.dataService.generateInstrumentState(runNumber)
 
         self.mantidSnapper.CropWorkspace(

src/snapred/backend/data/LocalDataService.py

@@ -15,7 +15,8 @@
 from urllib.parse import urlparse
 
 import h5py
-from mantid.api import Run
+import numpy as np
+from mantid.api import IEventWorkspace, Run
 from mantid.dataobjects import MaskWorkspace
 from mantid.kernel import ConfigService, PhysicalConstants
 from pydantic import validate_call
@@ -676,12 +677,31 @@ def writeNormalizationWorkspaces(self, record: NormalizationRecord):
         Record must be set with correct version and workspace names must be finalized.
         -- assumes that `writeNormalizationRecord` has already been called, and that the version folder exists
         """
-        state, _ = self.generateStateId(record.runNumber)
-        indexer = self.normalizationIndexer(record.useLiteMode, state)
+        instrumentState = self.generateInstrumentState(record.runNumber)
+        indexer = self.normalizationIndexer(record.useLiteMode, instrumentState.id.hex)
         normalizationDataPath: Path = indexer.versionPath(record.version)
         if not normalizationDataPath.exists():
             normalizationDataPath.mkdir(parents=True, exist_ok=True)
+
         for workspace in record.workspaceNames:
+            ws = self.mantidSnapper.mtd[workspace]
+            if Config["nexus.dataFormat.event"] and isinstance(ws, IEventWorkspace):
+                # Remove binning from any event workspaces in order to speed-up reload:
+                #    (XMin, XMax, Delta) == (NaN, NaN, <outer bound>) will create a single bin
+                #    [<actual minimum>, <actual maximum>]
+                TOFOuterBound = instrumentState.particleBounds.tof.maximum * 10.0
+                numberOfSpectra = ws.getNumberHistograms()
+                self.mantidSnapper.RebinRagged(
+                    "strip binning info, before saving normalization workspace",
+                    OutputWorkspace=workspace,
+                    InputWorkspace=workspace,
+                    XMin=list((np.nan,) * numberOfSpectra),
+                    XMax=list((np.nan,) * numberOfSpectra),
+                    Delta=list((TOFOuterBound,) * numberOfSpectra),
+                    PreserveEvents=True,
+                )
+                self.mantidSnapper.executeQueue()
+
             filename = Path(workspace + ".nxs")
             self.writeWorkspace(normalizationDataPath, filename, workspace)
 

src/snapred/backend/recipe/FetchGroceriesRecipe.py

@@ -23,7 +23,7 @@ def executeRecipe(
         instrumentPropertySource=None,
         instrumentSource="",
         *,
-        loaderArgs: str = "",
+        loaderArgs: str = "{}",
     ) -> Dict[str, Any]:
         """
         Wraps the fetch groceries algorithm

src/snapred/backend/recipe/GenerateFocussedVanadiumRecipe.py

@@ -2,7 +2,6 @@
 
 from snapred.backend.dao.ingredients import GenerateFocussedVanadiumIngredients as Ingredients
 from snapred.backend.log.logger import snapredLogger
-from snapred.backend.recipe.RebinFocussedGroupDataRecipe import RebinFocussedGroupDataRecipe
 from snapred.backend.recipe.Recipe import Recipe
 
 logger = snapredLogger.getLogger(__name__)
@@ -61,20 +60,13 @@ def queueNaturalNormalization(self):
             SmoothingParameter=self.smoothingParameter,
         )
 
-    def _rebinInputWorkspace(self):
-        """
-        Rebins the input workspace to the pixel group.
-        """
-        rebinRecipe = RebinFocussedGroupDataRecipe(self.utensils)
-        rebinIngredients = RebinFocussedGroupDataRecipe.Ingredients(pixelGroup=self.pixelGroup)
-        rebinRecipe.cook(rebinIngredients, {"inputWorkspace": self.inputWS})
-
     def queueAlgos(self):
         """
-        Queues up the procesing algorithms for the recipe.
+        Queues up the processing algorithms for the recipe.
         Requires: unbagged groceries.
         """
-        self._rebinInputWorkspace()
+        # Note: rebinning of the input workspace is no longer required here.
+        #   After `ReductionGroupProcessing`, the workspace will already have optimal ragged binning.
 
         if self.artificialNormalizationIngredients is not None:
             self.queueArtificialNormalization()

src/snapred/backend/recipe/GroupDiffCalRecipe.py

@@ -81,9 +81,7 @@ def chopIngredients(self, ingredients: Ingredients) -> None:
         self.runNumber: str = ingredients.runConfig.runNumber
 
         # from grouping parameters, read the overall min/max d-spacings
-        self.dMin = ingredients.pixelGroup.dMin()
-        self.dMax = ingredients.pixelGroup.dMax()
-        self.dBin = ingredients.pixelGroup.dBin()
+        self.pixelGroup = ingredients.pixelGroup
 
         self.grouping = ingredients.pixelGroup.focusGroup.name
 
@@ -325,24 +323,14 @@ def convertAndFocusAndReturn(self, inputWS: str, outputWS: str, note: str, units
             Target="dSpacing",
         )
 
-        # Diffraction-focus the d-spacing data
-        self.mantidSnapper.DiffractionFocussing(
+        # Diffraction focus, and rebin the d-spacing data.
+        self.mantidSnapper.FocusSpectraAlgorithm(
             "Diffraction-focus the d-spacing data",
             InputWorkspace=tmpWSdsp,
             GroupingWorkspace=self.focusWS,
             OutputWorkspace=tmpWSdsp,
-        )
-
-        # Ragged rebin the diffraction-focussed data
-        self.mantidSnapper.RebinRagged(
-            "Ragged rebin the diffraction-focussed data",
-            InputWorkspace=tmpWSdsp,
-            OutputWorkspace=tmpWSdsp,
-            XMin=self.dMin,
-            XMax=self.dMax,
-            Delta=self.dBin,
-            preserveEvents=keepEvents,
-            FullBinsOnly=True,
+            PixelGroup=self.pixelGroup.model_dump_json(),
+            PreserveEvents=keepEvents,
         )
 
         if units == "TOF":

src/snapred/backend/recipe/RebinFocussedGroupDataRecipe.py

@@ -56,7 +56,7 @@ def unbagGroceries(self, groceries: Dict[str, WorkspaceName]):
 
     def queueAlgos(self):
         """
-        Queues up the procesing algorithms for the recipe.
+        Queues up the processing algorithms for the recipe.
         Requires: unbagged groceries and chopped ingredients.
         """
         self.mantidSnapper.RebinRagged(