Ewm 6274 skip pixel calibration (#455)

* Added ability to skip pixel calibration on Tweak Peaks tab,
Warning message now shows up when masking workspace has too many masked
pixels, updated relevant unit tests

* Added unit test, updated functionality to match acceptance criteria

* Fixed unit test

* Added threshold value to config, removed flag from FarmFreshIngredients

* Fixed lite mode toggle on Tweak Peaks tab

* Addressed PR comments

src/snapred/backend/dao/ingredients/DiffractionCalibrationIngredients.py

@@ -28,6 +28,7 @@ class DiffractionCalibrationIngredients(BaseModel):
     peakFunction: SymmetricPeakEnum = SymmetricPeakEnum[Config["calibration.diffraction.peakFunction"]]
     maxOffset: float = Config["calibration.diffraction.maximumOffset"]
     maxChiSq: float = Config["constants.GroupDiffractionCalibration.MaxChiSq"]
+    skipPixelCalibration: bool = False
     # NOTE: removeBackground == True means that the background IS NOT removed
     # NOTE: removeBackgroud == False means that the background IS removed
     removeBackground: bool = True

src/snapred/backend/dao/request/DiffractionCalibrationRequest.py

@@ -35,6 +35,7 @@ class DiffractionCalibrationRequest(BaseModel, extra="forbid"):
     maximumOffset: float = Config["calibration.diffraction.maximumOffset"]
     fwhmMultipliers: Pair[float] = Pair.model_validate(Config["calibration.parameters.default.FWHMMultiplier"])
     maxChiSq: float = Config["constants.GroupDiffractionCalibration.MaxChiSq"]
+    skipPixelCalibration: bool = False
     removeBackground: Optional[bool]
 
     continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET

src/snapred/backend/recipe/DiffractionCalibrationRecipe.py

@@ -35,6 +35,7 @@ def chopIngredients(self, ingredients: Ingredients):
         self.runNumber = ingredients.runConfig.runNumber
         self.threshold = ingredients.convergenceThreshold
         self.maxIterations = Config["calibration.diffraction.maximumIterations"]
+        self.skipPixelCalibration = ingredients.skipPixelCalibration
         self.removeBackground = ingredients.removeBackground
 
     def unbagGroceries(self, groceries: Dict[str, Any]):
@@ -64,42 +65,43 @@ def executeRecipe(self, ingredients: Ingredients, groceries: Dict[str, str]) ->
         dataSteps: List[Dict[str, Any]] = []
         medianOffsets: List[float] = []
 
-        logger.info("Calibrating by cross-correlation and adjusting offsets...")
-        pixelAlgo = PixelDiffractionCalibration()
-        pixelAlgo.initialize()
-        pixelAlgo.setProperty("InputWorkspace", self.rawInput)
-        pixelAlgo.setProperty("GroupingWorkspace", self.groupingWS)
-        pixelAlgo.setProperty("Ingredients", ingredients.json())
-        pixelAlgo.setProperty("CalibrationTable", self.calTable)
-        pixelAlgo.setProperty("MaskWorkspace", self.maskWS)
-        try:
-            pixelAlgo.execute()
-            dataSteps.append(json.loads(pixelAlgo.getPropertyValue("data")))
-            medianOffsets.append(dataSteps[-1]["medianOffset"])
-        except RuntimeError as e:
-            errorString = str(e)
-            raise RuntimeError(errorString) from e
-        counter = 1
-        while abs(medianOffsets[-1]) > self.threshold and counter < self.maxIterations:
-            counter = counter + 1
-            logger.info(f"... converging to answer; step {counter}, {medianOffsets[-1]} > {self.threshold}")
+        if self.skipPixelCalibration is not True:
+            logger.info("Calibrating by cross-correlation and adjusting offsets...")
+            pixelAlgo = PixelDiffractionCalibration()
+            pixelAlgo.initialize()
+            pixelAlgo.setProperty("InputWorkspace", self.rawInput)
+            pixelAlgo.setProperty("GroupingWorkspace", self.groupingWS)
+            pixelAlgo.setProperty("Ingredients", ingredients.json())
+            pixelAlgo.setProperty("CalibrationTable", self.calTable)
+            pixelAlgo.setProperty("MaskWorkspace", self.maskWS)
             try:
                 pixelAlgo.execute()
-                newDataStep = json.loads(pixelAlgo.getPropertyValue("data"))
+                dataSteps.append(json.loads(pixelAlgo.getPropertyValue("data")))
+                medianOffsets.append(dataSteps[-1]["medianOffset"])
             except RuntimeError as e:
                 errorString = str(e)
                 raise RuntimeError(errorString) from e
-            # ensure monotonic decrease in the median offset
-            if newDataStep["medianOffset"] < dataSteps[-1]["medianOffset"]:
-                dataSteps.append(newDataStep)
-                medianOffsets.append(newDataStep["medianOffset"])
-            else:
-                logger.warning("Offsets failed to converge monotonically")
-                break
-        if counter >= self.maxIterations:
-            logger.warning("Offset convergence reached max iterations without convergning")
-        data["steps"] = dataSteps
-        logger.info(f"Initial calibration converged.  Offsets: {medianOffsets}")
+            counter = 1
+            while abs(medianOffsets[-1]) > self.threshold and counter < self.maxIterations:
+                counter = counter + 1
+                logger.info(f"... converging to answer; step {counter}, {medianOffsets[-1]} > {self.threshold}")
+                try:
+                    pixelAlgo.execute()
+                    newDataStep = json.loads(pixelAlgo.getPropertyValue("data"))
+                except RuntimeError as e:
+                    errorString = str(e)
+                    raise RuntimeError(errorString) from e
+                # ensure monotonic decrease in the median offset
+                if newDataStep["medianOffset"] < dataSteps[-1]["medianOffset"]:
+                    dataSteps.append(newDataStep)
+                    medianOffsets.append(newDataStep["medianOffset"])
+                else:
+                    logger.warning("Offsets failed to converge monotonically")
+                    break
+            if counter >= self.maxIterations:
+                logger.warning("Offset convergence reached max iterations without convergning")
+            data["steps"] = dataSteps
+            logger.info(f"Initial calibration converged.  Offsets: {medianOffsets}")
 
         logger.info("Beginning group-by-group fitting calibration")
         groupedAlgo = GroupDiffractionCalibration()

src/snapred/backend/service/CalibrationService.py

@@ -2,6 +2,7 @@
 from typing import Dict, List, Optional
 
 import pydantic
+from mantid.api import mtd
 
 from snapred.backend.dao import Limit, RunConfig
 from snapred.backend.dao.calibration import (
@@ -154,7 +155,23 @@ def diffractionCalibration(self, request: DiffractionCalibrationRequest):
         groceries = self.fetchDiffractionCalibrationGroceries(request)
 
         # now have all ingredients and groceries, run recipe
-        return DiffractionCalibrationRecipe().executeRecipe(ingredients, groceries)
+        res = DiffractionCalibrationRecipe().executeRecipe(ingredients, groceries)
+
+        if request.skipPixelCalibration is False:
+            maskWS = groceries.get("maskWorkspace", "")
+            percentMasked = mtd[maskWS].getNumberMasked() / mtd[maskWS].getNumberHistograms()
+            threshold = Config["constants.maskedPixelThreshold"]
+            if percentMasked > threshold:
+                raise Exception(
+                    (
+                        f"WARNING: More than {threshold*100}% of pixels failed calibration. Please check your input "
+                        "data. If input data has poor statistics, you may get better results by disabling Cross "
+                        "Correlation. You can also improve statistics by activating Lite mode if this is not "
+                        "already activated."
+                    ),
+                )
+
+        return res
 
     def validateRequest(self, request: DiffractionCalibrationRequest):
         """

src/snapred/resources/application.yml

@@ -189,6 +189,7 @@ constants:
   millisecondsPerSecond: 1000
   PeakIntensityFractionThreshold: 0.05
   m2cm: 10000.0 # conversion factor for m^2 to cm^2
+  maskedPixelThreshold: 0.15
 
   CrystallographicInfo:
     crystalDMin: 0.4

src/snapred/ui/view/DiffCalTweakPeakView.py

@@ -94,6 +94,9 @@ def __init__(self, samples=[], groups=[], parent=None):
         self.recalculationButton = QPushButton("Recalculate")
         self.recalculationButton.clicked.connect(self.emitValueChange)
 
+        # skip pixel calibration button
+        self.skipPixelCalToggle = self._labeledField("Skip Pixel Calibration", Toggle(parent=self, state=False))
+
         # add all elements to the grid layout
         self.layout.addWidget(self.runNumberField, 0, 0)
         self.layout.addWidget(self.litemodeToggle, 0, 1)
@@ -104,6 +107,7 @@ def __init__(self, samples=[], groups=[], parent=None):
         self.layout.addWidget(self.groupingFileDropdown, 4, 1)
         self.layout.addWidget(self.peakFunctionDropdown, 4, 2)
         self.layout.addWidget(self.recalculationButton, 5, 0, 1, 2)
+        self.layout.addWidget(self.skipPixelCalToggle, 3, 2, 1, 2)
 
         self.layout.setRowStretch(2, 10)
 

src/snapred/ui/workflow/DiffCalWorkflow.py

@@ -160,6 +160,13 @@ def _populateGroupingDropdown(self):
     def _switchLiteNativeGroups(self):
         # when the run number is updated, freeze the drop down to populate it
         useLiteMode = self._requestView.litemodeToggle.field.getState()
+        self._tweakPeakView.litemodeToggle.field.setState(useLiteMode)
+
+        # Enable pixel calibration skip only if not using lite mode
+        if useLiteMode is False:
+            self._tweakPeakView.skipPixelCalToggle.field.setState(True)
+        else:
+            self._tweakPeakView.skipPixelCalToggle.field.setState(False)
 
         self._requestView.groupingFileDropdown.setEnabled(False)
         # TODO: Use threads, account for fail cases
@@ -218,6 +225,7 @@ def _specifyRun(self, workflowPresenter):
             nBinsAcrossPeakWidth=self.nBinsAcrossPeakWidth,
             fwhmMultipliers=self.prevFWHM,
             maxChiSq=self.maxChiSq,
+            skipPixelCalibration=self._tweakPeakView.skipPixelCalToggle.field.getState(),
             removeBackground=self.removeBackground,
         )
 
@@ -301,6 +309,7 @@ def _renewIngredients(self, xtalDMin, xtalDMax, peakFunction, fwhm, maxChiSq):
             crystalDMax=xtalDMax,
             fwhmMultipliers=fwhm,
             maxChiSq=maxChiSq,
+            skipPixelCalibration=self._tweakPeakView.skipPixelCalToggle.field.getState(),
             removeBackground=self.removeBackground,
         )
         response = self.request(path="calibration/ingredients", payload=payload.json())
@@ -352,6 +361,7 @@ def _triggerDiffractionCalibration(self, workflowPresenter):
             nBinsAcrossPeakWidth=self.nBinsAcrossPeakWidth,
             fwhmMultipliers=self.prevFWHM,
             maxChiSq=self.maxChiSq,
+            skipPixelCalibration=self._tweakPeakView.skipPixelCalToggle.field.getState(),
             removeBackground=self.removeBackground,
         )
 

tests/resources/application.yml

@@ -207,6 +207,7 @@ constants:
   millisecondsPerSecond: 1000
   PeakIntensityFractionThreshold: 0.05
   m2cm: 10000.0 # conversion factor for m^2 to cm^2
+  maskedPixelThreshold: 0.15
 
   CrystallographicInfo:
     crystalDMin: 0.4

tests/unit/backend/service/test_CalibrationService.py

@@ -719,7 +719,7 @@ def test_diffractionCalibration(
         DiffractionCalibrationRecipe().executeRecipe.return_value = {"calibrationTable": "fake"}
 
         self.instance.groceryClerk = mock.Mock()
-        self.instance.groceryService.fetchGroceryDict = mock.Mock(return_value={"grocery1": "orange"})
+        self.instance.groceryService.fetchGroceryDict = mock.Mock(return_value={"maskWorkspace": self.sampleMaskWS})
 
         # Call the method with the provided parameters
         request = DiffractionCalibrationRequest(
@@ -802,6 +802,41 @@ def test_validateWritePermissions_no_permissions(self):
         with pytest.raises(RuntimeError, match=r".*you don't have permissions to write.*"):
             self.instance.validateWritePermissions(permissionsRequest)
 
+    @mock.patch(thisService + "FarmFreshIngredients", spec_set=FarmFreshIngredients)
+    @mock.patch(thisService + "DiffractionCalibrationRecipe", spec_set=DiffractionCalibrationRecipe)
+    def test_diffractionCalibration_with_bad_masking(
+        self,
+        DiffractionCalibrationRecipe,
+        FarmFreshIngredients,
+    ):
+        FarmFreshIngredients.return_value = mock.Mock(runNumber="123")
+        self.instance.dataFactoryService.getCifFilePath = mock.Mock(return_value="bundt/cake.egg")
+        self.instance.dataExportService.getCalibrationStateRoot = mock.Mock(return_value="lah/dee/dah")
+        self.instance.dataExportService.checkWritePermissions = mock.Mock(return_value=True)
+        self.instance.sousChef = SculleryBoy()
+
+        DiffractionCalibrationRecipe().executeRecipe.return_value = {"calibrationTable": "fake"}
+
+        numHistograms = mtd[self.sampleMaskWS].getNumberHistograms()
+        for pixel in range(numHistograms):
+            mtd[self.sampleMaskWS].setY(pixel, [1.0])
+        self.instance.groceryClerk = mock.Mock()
+        self.instance.groceryService.fetchGroceryDict = mock.Mock(return_value={"maskWorkspace": self.sampleMaskWS})
+
+        # Call the method with the provided parameters
+        # Call the method with the provided parameters
+        request = DiffractionCalibrationRequest(
+            runNumber="123",
+            useLiteMode=True,
+            calibrantSamplePath="bundt/cake_egg.py",
+            removeBackground=True,
+            focusGroup=FocusGroup(name="all", definition="path/to/all"),
+            continueFlags=ContinueWarning.Type.UNSET,
+            skipPixelCalibration=False,
+        )
+        with pytest.raises(Exception, match=r".*pixels failed calibration*"):
+            self.instance.diffractionCalibration(request)
+
     @mock.patch(thisService + "FarmFreshIngredients", spec_set=FarmFreshIngredients)
     @mock.patch(thisService + "FocusSpectraRecipe")
     def test_focusSpectra_not_exist(

tests/util/diffraction_calibration_synthetic_data.py

@@ -111,6 +111,7 @@ def __init__(self, workspaceType: str = "Histogram", scale: float = 1000.0):
             maxOffset=100.0,  # bins: '100.0' seems to work
             pixelGroup=self.fakePixelGroup,
             maxChiSq=100.0,
+            skipPixelCalibration=False,
         )
 
     @staticmethod