Merge pull request #623 from neutrons/EWM12331_notify_default_diffcal

ContinueWarning for DEFAULT_DIFFRACTION_CALIBRATION

Author: ekapadi

pixi.lock

@@ -11,17 +11,17 @@ class DiffcalStateMetadata(StrEnum):
     """Class to hold tags related to a workspace"""
 
     UNSET: str = UNSET  # the default condition before any settings
-    DEFAULT: str = "default"  # the default condition before any settings
-    EXISTS: str = "exists"  # the state exists and the corresponding .h5 file located
+    EXISTS: str = "exists"  # the state exists and the corresponding .h5 file has been located
     ALTERNATE: str = "alternate"  # the user supplied an alternate .h5 that they believe is usable.
-    NONE: str = "none"  # proceed using the defaul (IDF) geometry.
+    # Note: `MISSING_DIFFRACTION_CALIBRATION` is now the same as the former `DEFAULT_DIFFRACTION_CALIBRATION`.
+    NONE: str = "none"  # proceed using the default (IDF + parameter values) geometry.
 
 
 class NormalizationStateMetadata(StrEnum):
     """Class to hold tags related to a workspace"""
 
     UNSET: str = UNSET  # the default condition before any settings
-    EXISTS: str = "exists"  # the state exists and the corresponding .h5 file located
+    EXISTS: str = "exists"  # the state exists and the corresponding .h5 file has been located
     ALTERNATE: str = "alternate"  # the user supplied an alternate .h5 that they believe is usable
     NONE: str = "none"  # proceed without applying any normalization
     FAKE: str = "fake"  # proceed by creating a "fake vanadium"
@@ -31,7 +31,7 @@ class ParticleNormalizationMethod(StrEnum):
     """Class to hold tags related to a workspace"""
 
     UNSET: str = UNSET  # the default condition before any settings
-    PROTON_CHARGE: str = "proton_charge"  # the state exists and the corresponding .h5 file located
+    PROTON_CHARGE: str = "proton_charge"  # the state exists and the corresponding .h5 file has been located
     MONITOR_COUNTS: str = "monitor_counts"  # the user supplied an alternate .h5 that they believe is usable
     NONE: str = "none"  # proceed without applying any normalization
 

src/snapred/backend/dao/WorkspaceMetadata.py

@@ -2,8 +2,12 @@
 from snapred.meta.Enum import StrEnum
 
 
-# NOTE: This should probably not be reconfigurable at runtime.
-#       This would be liable to only cause indexing issues.
+# NOTES:
+#   * This should probably not be reconfigurable at runtime.
+#     This would be liable to only cause indexing issues.
+#   * `VERSION_START` is used as the DEFAULT version number.
+#     For example, when diffraction calibration would otherwise
+#     be missing and the default diffraction calibration is used.
 def VERSION_START():
     return Config["version.start"]
 
@@ -16,7 +20,9 @@ class VersionState(StrEnum):
     LEGACY_INST_PRM = "1.4"
 
 
+""" TODO: remove this.
 # I'm not sure why ci is failing without this, it doesn't seem to be used anywhere
 VERSION_DEFAULT = VersionState.DEFAULT
+"""
 
 Version = int | VersionState

src/snapred/backend/dao/indexing/Versioning.py

@@ -1090,6 +1090,8 @@ def _writeDefaultDiffCalTable(self, runNumber: str, useLiteMode: bool):
         grocer.deleteWorkspaceUnconditional(outWS)
 
     def generateInstrumentState(self, runId: str):
+        # Generate a default instrument state.
+
         # Read the detector state from the PV data file,
         #   and generate the stateID SHA.
         stateId, detectorState = self.generateStateId(runId)

src/snapred/backend/data/LocalDataService.py

@@ -13,9 +13,9 @@ class ContinueWarning(Exception):
 
     class Type(Flag):
         UNSET = 0
+        # Note: `MISSING_DIFFRACTION_CALIBRATION` is now the same as the former `DEFAULT_DIFFRACTION_CALIBRATION`.
         MISSING_DIFFRACTION_CALIBRATION = auto()
         ALTERNATE_DIFFRACTION_CALIBRATION = auto()
-        DEFAULT_DIFFRACTION_CALIBRATION = auto()
         MISSING_NORMALIZATION = auto()
         LOW_PEAK_COUNT = auto()
         NO_WRITE_PERMISSIONS = auto()

src/snapred/backend/error/ContinueWarning.py

@@ -149,19 +149,17 @@ def PyExec(self):
             )
 
         self.mantidSnapper.executeQueue()
-        self.outputWorkspace = self.mantidSnapper.mtd[self.outputWorkspaceName]
 
-        # Apply peak clipping to each histogram in the workspace
-        for i in range(self.outputWorkspace.getNumberHistograms()):
-            dataY = self.outputWorkspace.readY(i)
+        for i in range(self.mantidSnapper.mtd[self.outputWorkspaceName].getNumberHistograms()):
+            dataY = self.mantidSnapper.mtd[self.outputWorkspaceName].readY(i)
             clippedData = self.peakClip(
                 data=dataY,
                 winSize=self.peakWindowClippingSize,
                 decrese=self.decreaseParameter,
                 LLS=self.LSS,
                 smoothing=self.smoothingParameter,
             )
-            self.outputWorkspace.setY(i, clippedData)
+            self.mantidSnapper.mtd[self.outputWorkspaceName].setY(i, clippedData)
 
         # Set the output workspace property
         self.setProperty("OutputWorkspace", self.outputWorkspaceName)

src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py

@@ -3,6 +3,7 @@
 
 from snapred.backend.dao import Limit
 from snapred.backend.dao.indexing.IndexEntry import IndexEntry
+from snapred.backend.dao.indexing.Versioning import VERSION_START
 from snapred.backend.dao.ingredients import (
     GroceryListItem,
 )
@@ -235,11 +236,10 @@ def normalization(self, request: NormalizationRequest):
         ).dict()
 
     def _markWorkspaceMetadata(self, request: NormalizationRequest, workspace: WorkspaceName):
-        calibrationState = (
-            DiffcalStateMetadata.DEFAULT
-            if ContinueWarning.Type.DEFAULT_DIFFRACTION_CALIBRATION in request.continueFlags
-            else DiffcalStateMetadata.EXISTS
-        )
+        if ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION in request.continueFlags:
+            calibrationState = DiffcalStateMetadata.NONE
+        else:
+            calibrationState = DiffcalStateMetadata.EXISTS
         metadata = WorkspaceMetadata(diffcalState=calibrationState, normalizationState=NormalizationStateMetadata.UNSET)
         self.groceryService.writeWorkspaceMetadataAsTags(workspace, metadata)
 
@@ -265,22 +265,34 @@ def _validateDiffractionCalibrationExists(self, request: NormalizationRequest):
 
         self.sousChef.verifyCalibrationExists(request.runNumber, request.useLiteMode)
         state, _ = self.dataFactoryService.constructStateId(request.runNumber)
+
+        # Notes:
+        #   * At this point, the state will be initialized.
+        #   * In an initialized state, `calVersion` will never be `None`.
+        #   * `MISSING_DIFFRACTION_CALIBRATION` is now the same as the former `DEFAULT_DIFFRACTION_CALIBRATION`.
+        #   * The default calibration uses `VERSION_START`.
         calVersion = self.dataFactoryService.getLatestApplicableCalibrationVersion(
             request.runNumber, request.useLiteMode, state
         )
         if calVersion is None:
-            continueFlags = continueFlags | ContinueWarning.Type.DEFAULT_DIFFRACTION_CALIBRATION
+            raise RuntimeError(
+                "Usage error: for an initialized state, "
+                "diffraction-calibration version should always be at least the default version (VERSION_START)."
+            )
+        if calVersion == VERSION_START():
+            continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION
 
+        # remove any continue flags that were already present in the request
         if request.continueFlags:
-            continueFlags = continueFlags ^ (request.continueFlags & continueFlags)
+            continueFlags ^= request.continueFlags & continueFlags
 
         if continueFlags:
-            raise ContinueWarning(
-                "Only the default Diffraction Calibration data is available for this run.\n"
-                "Normalizations may not be accurate to true state of the instrument, and will be marked as such.\n"
-                "Continue anyway?",
-                continueFlags,
+            message = (
+                "<p><b>Diffraction calibration is missing.</b></p>"
+                "<p>Default calibration will be used in place of actual calibration.</p>"
+                "<p>Would you like to continue anyway?</p>"
             )
+            raise ContinueWarning(message, continueFlags)
 
     @Register("validateWritePermissions")
     def validateWritePermissions(self, request: CalibrationWritePermissionsRequest):

src/snapred/backend/service/NormalizationService.py

@@ -4,7 +4,7 @@
 from typing import Any, Dict, List, Optional
 
 from snapred.backend.dao import RunMetadata
-from snapred.backend.dao.indexing.Versioning import VersionState
+from snapred.backend.dao.indexing.Versioning import VERSION_START, VersionState
 from snapred.backend.dao.ingredients import (
     GroceryListItem,
     ReductionIngredients,
@@ -129,46 +129,61 @@ def validateReduction(self, request: ReductionRequest):
 
         # Check if a normalization is present
         normalizationExists = self.dataFactoryService.normalizationExists(request.runNumber, request.useLiteMode, state)
+        if not normalizationExists:
+            continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION
 
-        calibrationExists = (
-            self.dataFactoryService.calibrationExists(request.runNumber, request.useLiteMode, state)
-            or request.alternativeCalibrationFilePath is not None
-        )
+        # Notes:
+        #  * At this point, the state will be initialized.
+        #  * In an initialized state, `calVersion` will never be `None`.
+        #  * `MISSING_DIFFRACTION_CALIBRATION` is now the same as the former `DEFAULT_DIFFRACTION_CALIBRATION`.
+        #  * The default calibration uses `VERSION_START`.
+        calibrationExists = True
+        if request.alternativeCalibrationFilePath is None:
+            calVersion = self.dataFactoryService.getLatestApplicableCalibrationVersion(
+                request.runNumber, request.useLiteMode, state
+            )
+            if calVersion is None:
+                raise RuntimeError(
+                    "Usage error: for an initialized state, "
+                    "diffraction-calibration version should always be at least the default version (VERSION_START)."
+                )
+            if calVersion == VERSION_START():
+                calibrationExists = False
+                continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION
 
         # Determine the action based on missing components
         if not calibrationExists and normalizationExists:
             # Case: No calibration but normalization exists
-            continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION
             message = (
-                "Warning: diffraction calibration is missing.If you continue, default instrument geometry will be used."
+                "<p><b>Diffraction calibration is missing.</b></p>"
+                "<p>Default calibration will be used in place of actual calibration.</p>"
+                "<p>Would you like to continue anyway?</p>"
             )
         elif calibrationExists and not normalizationExists:
             # Case: Calibration exists but normalization is missing
-            continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION
             message = (
-                "Warning: Reduction is missing normalization data. "
-                "Artificial normalization will be created in place of actual normalization. "
-                "Would you like to continue?"
+                "<p><b>Normalization calibration is missing.</b></p>"
+                "<p>Artificial normalization will be created in place of actual normalization.</p>"
+                "<p>Would you like to continue anyway?</p>"
             )
         elif not calibrationExists and not normalizationExists:
             # Case: No calibration and no normalization
-            continueFlags |= (
-                ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION
-            )
             message = (
-                "Warning: Reduction is missing both normalization and calibration data. "
-                "If you continue, default instrument geometry will be used and data will be artificially normalized. "
+                "<p><b>Both normalization and diffraction calibrations are missing.</b></p>"
+                "<p>Default calibration will be used in place of actual calibration.</p>"
+                "<p>Artificial normalization will be created in place of actual normalization.</p>"
+                "<p>Would you like to continue anyway?</p>"
             )
 
-        # Remove any continue flags that are present in the request by XOR-ing with the flags
+        # Remove any continue flags that are also present in the request by XOR-ing with the request flags
         if request.continueFlags:
             continueFlags ^= request.continueFlags & continueFlags
 
         # If there are any continue flags set, raise a ContinueWarning with the appropriate message
         if continueFlags and message:
             raise ContinueWarning(message, continueFlags)
 
-        # Ensure separate continue warnings for permission check
+        # Reinitialized continue flags for the upcoming permissions check
         continueFlags = ContinueWarning.Type.UNSET
 
         # Check that the user has write permissions to the save directory
@@ -248,12 +263,14 @@ def _createReductionRecord(
             state, _ = self.dataFactoryService.constructStateId(request.runNumber)
 
         if request.continueFlags is not None:
-            if ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION not in request.continueFlags:
-                # If a diffraction calibration exists,
-                #   its version will have been filled in by `fetchReductionGroceries`.
-                calibration = self.dataFactoryService.getCalibrationRecord(
-                    request.runNumber, request.useLiteMode, request.versions.calibration, state
-                )
+            # Notes:
+            #   * If a diffraction calibration exists,
+            #     its version will have been filled in by `fetchReductionGroceries`.
+            #   * `MISSING_DIFFRACTION_CALIBRATION` now means that the default diffraction calibration
+            #     with `VERSION_START` is being applied.
+            calibration = self.dataFactoryService.getCalibrationRecord(
+                request.runNumber, request.useLiteMode, request.versions.calibration, state
+            )
             if ContinueWarning.Type.MISSING_NORMALIZATION not in request.continueFlags:
                 normalization = self.dataFactoryService.getNormalizationRecord(
                     request.runNumber, request.useLiteMode, state, request.versions.normalization
@@ -355,21 +372,23 @@ def prepCombinedMask(self, request: ReductionRequest) -> WorkspaceName:
 
         # if there is a mask associated with the diffcal file, load it here
         calVersion = request.versions.calibration
-        if (
-            ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION not in request.continueFlags
-            and calVersion is VersionState.LATEST
-        ):
+        if calVersion is VersionState.LATEST:
             calVersion = self.dataFactoryService.getLatestApplicableCalibrationVersion(
                 runNumber, useLiteMode, state=state
             )
 
-        if calVersion is not None:
-            self.groceryClerk.name("diffcalMaskWorkspace").diffcal_mask(state, calVersion, runNumber).useLiteMode(
-                useLiteMode
+        if calVersion is None:
+            raise RuntimeError(
+                "Usage error: for an initialized state, "
+                "diffraction-calibration version should always be at least the default version (VERSION_START)."
             )
-            if request.alternativeCalibrationFilePath is not None:
-                self.groceryClerk.diffCalFilePath(request.alternativeCalibrationFilePath)
-            self.groceryClerk.add()
+
+        self.groceryClerk.name("diffcalMaskWorkspace").diffcal_mask(state, calVersion, runNumber).useLiteMode(
+            useLiteMode
+        )
+        if request.alternativeCalibrationFilePath is not None:
+            self.groceryClerk.diffCalFilePath(request.alternativeCalibrationFilePath)
+        self.groceryClerk.add()
 
         # if the user specified masks to use, also pull those
         residentMasks = {}
@@ -476,10 +495,15 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]:
             # If no alternativeState state is provided, use the sample's state.
             state, _ = self.dataFactoryService.constructStateId(request.runNumber)
 
-        if ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION not in request.continueFlags:
-            calVersion = self.dataFactoryService.getLatestApplicableCalibrationVersion(
-                request.runNumber, request.useLiteMode, state
+        calVersion = self.dataFactoryService.getLatestApplicableCalibrationVersion(
+            request.runNumber, request.useLiteMode, state
+        )
+        if calVersion is None:
+            raise RuntimeError(
+                "Usage error: for an initialized state, "
+                "diffraction-calibration version should always be at least the default version (VERSION_START)."
             )
+
         if ContinueWarning.Type.MISSING_NORMALIZATION not in request.continueFlags:
             normVersion = self.dataFactoryService.getLatestApplicableNormalizationVersion(
                 request.runNumber, request.useLiteMode, state
@@ -551,7 +575,8 @@ def _markWorkspaceMetadata(self, request: ReductionRequest, workspace: Workspace
             altDiffCalFilePath = str(request.alternativeCalibrationFilePath)
         else:
             calibrationState = DiffcalStateMetadata.EXISTS
-        # The reduction workflow will automatically create a "fake" vanadium, so it shouldnt ever be None?
+
+        # The reduction workflow will automatically create a "fake" vanadium, so it shouldn't ever be None?
         normalizationState = (
             NormalizationStateMetadata.FAKE
             if ContinueWarning.Type.MISSING_NORMALIZATION in request.continueFlags

src/snapred/backend/service/ReductionService.py

@@ -9,7 +9,6 @@
 from pydantic.functional_validators import BeforeValidator
 from typing_extensions import Annotated, Self
 
-# *** DEBUG *** : CIRCULAR IMPORT?!
 from snapred.backend.dao.indexing.Versioning import VERSION_START, VersionState
 from snapred.meta.Config import Config
 from snapred.meta.decorators.classproperty import classproperty
@@ -224,9 +223,6 @@ def formatVersion(cls, version: Optional[int], fmt=ValueFormat.versionFormat.WOR
         # handle two special cases of unassigned or default version
         # in those cases, format will be a user-specified string
 
-        # *** DEBUG *** : CIRCULAR IMPORT?!
-        from snapred.backend.dao.indexing.Versioning import VersionState
-
         formattedVersion = ""
         if version == VersionState.DEFAULT:
             formattedVersion = f"v{VERSION_START()}"