diff --git a/src/snapred/backend/dao/ingredients/DiffractionCalibrationIngredients.py b/src/snapred/backend/dao/ingredients/DiffractionCalibrationIngredients.py index 8a7ec5872..ed5c24c0b 100644 --- a/src/snapred/backend/dao/ingredients/DiffractionCalibrationIngredients.py +++ b/src/snapred/backend/dao/ingredients/DiffractionCalibrationIngredients.py @@ -28,6 +28,7 @@ class DiffractionCalibrationIngredients(BaseModel): peakFunction: SymmetricPeakEnum = SymmetricPeakEnum[Config["calibration.diffraction.peakFunction"]] maxOffset: float = Config["calibration.diffraction.maximumOffset"] maxChiSq: float = Config["constants.GroupDiffractionCalibration.MaxChiSq"] + skipPixelCalibration: bool = False # NOTE: removeBackground == True means that the background IS NOT removed # NOTE: removeBackgroud == False means that the background IS removed removeBackground: bool = True diff --git a/src/snapred/backend/dao/request/DiffractionCalibrationRequest.py b/src/snapred/backend/dao/request/DiffractionCalibrationRequest.py index ed28d7c0a..5fa81c7bf 100644 --- a/src/snapred/backend/dao/request/DiffractionCalibrationRequest.py +++ b/src/snapred/backend/dao/request/DiffractionCalibrationRequest.py @@ -35,6 +35,7 @@ class DiffractionCalibrationRequest(BaseModel, extra="forbid"): maximumOffset: float = Config["calibration.diffraction.maximumOffset"] fwhmMultipliers: Pair[float] = Pair.model_validate(Config["calibration.parameters.default.FWHMMultiplier"]) maxChiSq: float = Config["constants.GroupDiffractionCalibration.MaxChiSq"] + skipPixelCalibration: bool = False removeBackground: Optional[bool] continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET diff --git a/src/snapred/backend/recipe/DiffractionCalibrationRecipe.py b/src/snapred/backend/recipe/DiffractionCalibrationRecipe.py index 256f40632..4a48640ae 100644 --- a/src/snapred/backend/recipe/DiffractionCalibrationRecipe.py +++ b/src/snapred/backend/recipe/DiffractionCalibrationRecipe.py @@ -35,6 +35,7 @@ def chopIngredients(self, ingredients: Ingredients): self.runNumber = ingredients.runConfig.runNumber self.threshold = ingredients.convergenceThreshold self.maxIterations = Config["calibration.diffraction.maximumIterations"] + self.skipPixelCalibration = ingredients.skipPixelCalibration self.removeBackground = ingredients.removeBackground def unbagGroceries(self, groceries: Dict[str, Any]): @@ -64,42 +65,43 @@ def executeRecipe(self, ingredients: Ingredients, groceries: Dict[str, str]) -> dataSteps: List[Dict[str, Any]] = [] medianOffsets: List[float] = [] - logger.info("Calibrating by cross-correlation and adjusting offsets...") - pixelAlgo = PixelDiffractionCalibration() - pixelAlgo.initialize() - pixelAlgo.setProperty("InputWorkspace", self.rawInput) - pixelAlgo.setProperty("GroupingWorkspace", self.groupingWS) - pixelAlgo.setProperty("Ingredients", ingredients.json()) - pixelAlgo.setProperty("CalibrationTable", self.calTable) - pixelAlgo.setProperty("MaskWorkspace", self.maskWS) - try: - pixelAlgo.execute() - dataSteps.append(json.loads(pixelAlgo.getPropertyValue("data"))) - medianOffsets.append(dataSteps[-1]["medianOffset"]) - except RuntimeError as e: - errorString = str(e) - raise RuntimeError(errorString) from e - counter = 1 - while abs(medianOffsets[-1]) > self.threshold and counter < self.maxIterations: - counter = counter + 1 - logger.info(f"... converging to answer; step {counter}, {medianOffsets[-1]} > {self.threshold}") + if self.skipPixelCalibration is not True: + logger.info("Calibrating by cross-correlation and adjusting offsets...") + pixelAlgo = PixelDiffractionCalibration() + pixelAlgo.initialize() + pixelAlgo.setProperty("InputWorkspace", self.rawInput) + pixelAlgo.setProperty("GroupingWorkspace", self.groupingWS) + pixelAlgo.setProperty("Ingredients", ingredients.json()) + pixelAlgo.setProperty("CalibrationTable", self.calTable) + pixelAlgo.setProperty("MaskWorkspace", self.maskWS) try: pixelAlgo.execute() - newDataStep = json.loads(pixelAlgo.getPropertyValue("data")) + dataSteps.append(json.loads(pixelAlgo.getPropertyValue("data"))) + medianOffsets.append(dataSteps[-1]["medianOffset"]) except RuntimeError as e: errorString = str(e) raise RuntimeError(errorString) from e - # ensure monotonic decrease in the median offset - if newDataStep["medianOffset"] < dataSteps[-1]["medianOffset"]: - dataSteps.append(newDataStep) - medianOffsets.append(newDataStep["medianOffset"]) - else: - logger.warning("Offsets failed to converge monotonically") - break - if counter >= self.maxIterations: - logger.warning("Offset convergence reached max iterations without convergning") - data["steps"] = dataSteps - logger.info(f"Initial calibration converged. Offsets: {medianOffsets}") + counter = 1 + while abs(medianOffsets[-1]) > self.threshold and counter < self.maxIterations: + counter = counter + 1 + logger.info(f"... converging to answer; step {counter}, {medianOffsets[-1]} > {self.threshold}") + try: + pixelAlgo.execute() + newDataStep = json.loads(pixelAlgo.getPropertyValue("data")) + except RuntimeError as e: + errorString = str(e) + raise RuntimeError(errorString) from e + # ensure monotonic decrease in the median offset + if newDataStep["medianOffset"] < dataSteps[-1]["medianOffset"]: + dataSteps.append(newDataStep) + medianOffsets.append(newDataStep["medianOffset"]) + else: + logger.warning("Offsets failed to converge monotonically") + break + if counter >= self.maxIterations: + logger.warning("Offset convergence reached max iterations without convergning") + data["steps"] = dataSteps + logger.info(f"Initial calibration converged. Offsets: {medianOffsets}") logger.info("Beginning group-by-group fitting calibration") groupedAlgo = GroupDiffractionCalibration() diff --git a/src/snapred/backend/service/CalibrationService.py b/src/snapred/backend/service/CalibrationService.py index 3a6565786..0586980a9 100644 --- a/src/snapred/backend/service/CalibrationService.py +++ b/src/snapred/backend/service/CalibrationService.py @@ -2,6 +2,7 @@ from typing import Dict, List, Optional import pydantic +from mantid.api import mtd from snapred.backend.dao import Limit, RunConfig from snapred.backend.dao.calibration import ( @@ -154,7 +155,23 @@ def diffractionCalibration(self, request: DiffractionCalibrationRequest): groceries = self.fetchDiffractionCalibrationGroceries(request) # now have all ingredients and groceries, run recipe - return DiffractionCalibrationRecipe().executeRecipe(ingredients, groceries) + res = DiffractionCalibrationRecipe().executeRecipe(ingredients, groceries) + + if request.skipPixelCalibration is False: + maskWS = groceries.get("maskWorkspace", "") + percentMasked = mtd[maskWS].getNumberMasked() / mtd[maskWS].getNumberHistograms() + threshold = Config["constants.maskedPixelThreshold"] + if percentMasked > threshold: + raise Exception( + ( + f"WARNING: More than {threshold*100}% of pixels failed calibration. Please check your input " + "data. If input data has poor statistics, you may get better results by disabling Cross " + "Correlation. You can also improve statistics by activating Lite mode if this is not " + "already activated." + ), + ) + + return res def validateRequest(self, request: DiffractionCalibrationRequest): """ diff --git a/src/snapred/resources/application.yml b/src/snapred/resources/application.yml index 6eec01d2c..b539440d5 100644 --- a/src/snapred/resources/application.yml +++ b/src/snapred/resources/application.yml @@ -189,6 +189,7 @@ constants: millisecondsPerSecond: 1000 PeakIntensityFractionThreshold: 0.05 m2cm: 10000.0 # conversion factor for m^2 to cm^2 + maskedPixelThreshold: 0.15 CrystallographicInfo: crystalDMin: 0.4 diff --git a/src/snapred/ui/view/DiffCalTweakPeakView.py b/src/snapred/ui/view/DiffCalTweakPeakView.py index 78ff9baeb..77f535f14 100644 --- a/src/snapred/ui/view/DiffCalTweakPeakView.py +++ b/src/snapred/ui/view/DiffCalTweakPeakView.py @@ -94,6 +94,9 @@ def __init__(self, samples=[], groups=[], parent=None): self.recalculationButton = QPushButton("Recalculate") self.recalculationButton.clicked.connect(self.emitValueChange) + # skip pixel calibration button + self.skipPixelCalToggle = self._labeledField("Skip Pixel Calibration", Toggle(parent=self, state=False)) + # add all elements to the grid layout self.layout.addWidget(self.runNumberField, 0, 0) self.layout.addWidget(self.litemodeToggle, 0, 1) @@ -104,6 +107,7 @@ def __init__(self, samples=[], groups=[], parent=None): self.layout.addWidget(self.groupingFileDropdown, 4, 1) self.layout.addWidget(self.peakFunctionDropdown, 4, 2) self.layout.addWidget(self.recalculationButton, 5, 0, 1, 2) + self.layout.addWidget(self.skipPixelCalToggle, 3, 2, 1, 2) self.layout.setRowStretch(2, 10) diff --git a/src/snapred/ui/workflow/DiffCalWorkflow.py b/src/snapred/ui/workflow/DiffCalWorkflow.py index 7d9c8dcd4..4b373c16f 100644 --- a/src/snapred/ui/workflow/DiffCalWorkflow.py +++ b/src/snapred/ui/workflow/DiffCalWorkflow.py @@ -160,6 +160,13 @@ def _populateGroupingDropdown(self): def _switchLiteNativeGroups(self): # when the run number is updated, freeze the drop down to populate it useLiteMode = self._requestView.litemodeToggle.field.getState() + self._tweakPeakView.litemodeToggle.field.setState(useLiteMode) + + # Enable pixel calibration skip only if not using lite mode + if useLiteMode is False: + self._tweakPeakView.skipPixelCalToggle.field.setState(True) + else: + self._tweakPeakView.skipPixelCalToggle.field.setState(False) self._requestView.groupingFileDropdown.setEnabled(False) # TODO: Use threads, account for fail cases @@ -218,6 +225,7 @@ def _specifyRun(self, workflowPresenter): nBinsAcrossPeakWidth=self.nBinsAcrossPeakWidth, fwhmMultipliers=self.prevFWHM, maxChiSq=self.maxChiSq, + skipPixelCalibration=self._tweakPeakView.skipPixelCalToggle.field.getState(), removeBackground=self.removeBackground, ) @@ -301,6 +309,7 @@ def _renewIngredients(self, xtalDMin, xtalDMax, peakFunction, fwhm, maxChiSq): crystalDMax=xtalDMax, fwhmMultipliers=fwhm, maxChiSq=maxChiSq, + skipPixelCalibration=self._tweakPeakView.skipPixelCalToggle.field.getState(), removeBackground=self.removeBackground, ) response = self.request(path="calibration/ingredients", payload=payload.json()) @@ -352,6 +361,7 @@ def _triggerDiffractionCalibration(self, workflowPresenter): nBinsAcrossPeakWidth=self.nBinsAcrossPeakWidth, fwhmMultipliers=self.prevFWHM, maxChiSq=self.maxChiSq, + skipPixelCalibration=self._tweakPeakView.skipPixelCalToggle.field.getState(), removeBackground=self.removeBackground, ) diff --git a/tests/resources/application.yml b/tests/resources/application.yml index e31bffccf..bfe92de2b 100644 --- a/tests/resources/application.yml +++ b/tests/resources/application.yml @@ -207,6 +207,7 @@ constants: millisecondsPerSecond: 1000 PeakIntensityFractionThreshold: 0.05 m2cm: 10000.0 # conversion factor for m^2 to cm^2 + maskedPixelThreshold: 0.15 CrystallographicInfo: crystalDMin: 0.4 diff --git a/tests/unit/backend/service/test_CalibrationService.py b/tests/unit/backend/service/test_CalibrationService.py index c1af82628..2ce5c5853 100644 --- a/tests/unit/backend/service/test_CalibrationService.py +++ b/tests/unit/backend/service/test_CalibrationService.py @@ -719,7 +719,7 @@ def test_diffractionCalibration( DiffractionCalibrationRecipe().executeRecipe.return_value = {"calibrationTable": "fake"} self.instance.groceryClerk = mock.Mock() - self.instance.groceryService.fetchGroceryDict = mock.Mock(return_value={"grocery1": "orange"}) + self.instance.groceryService.fetchGroceryDict = mock.Mock(return_value={"maskWorkspace": self.sampleMaskWS}) # Call the method with the provided parameters request = DiffractionCalibrationRequest( @@ -802,6 +802,41 @@ def test_validateWritePermissions_no_permissions(self): with pytest.raises(RuntimeError, match=r".*you don't have permissions to write.*"): self.instance.validateWritePermissions(permissionsRequest) + @mock.patch(thisService + "FarmFreshIngredients", spec_set=FarmFreshIngredients) + @mock.patch(thisService + "DiffractionCalibrationRecipe", spec_set=DiffractionCalibrationRecipe) + def test_diffractionCalibration_with_bad_masking( + self, + DiffractionCalibrationRecipe, + FarmFreshIngredients, + ): + FarmFreshIngredients.return_value = mock.Mock(runNumber="123") + self.instance.dataFactoryService.getCifFilePath = mock.Mock(return_value="bundt/cake.egg") + self.instance.dataExportService.getCalibrationStateRoot = mock.Mock(return_value="lah/dee/dah") + self.instance.dataExportService.checkWritePermissions = mock.Mock(return_value=True) + self.instance.sousChef = SculleryBoy() + + DiffractionCalibrationRecipe().executeRecipe.return_value = {"calibrationTable": "fake"} + + numHistograms = mtd[self.sampleMaskWS].getNumberHistograms() + for pixel in range(numHistograms): + mtd[self.sampleMaskWS].setY(pixel, [1.0]) + self.instance.groceryClerk = mock.Mock() + self.instance.groceryService.fetchGroceryDict = mock.Mock(return_value={"maskWorkspace": self.sampleMaskWS}) + + # Call the method with the provided parameters + # Call the method with the provided parameters + request = DiffractionCalibrationRequest( + runNumber="123", + useLiteMode=True, + calibrantSamplePath="bundt/cake_egg.py", + removeBackground=True, + focusGroup=FocusGroup(name="all", definition="path/to/all"), + continueFlags=ContinueWarning.Type.UNSET, + skipPixelCalibration=False, + ) + with pytest.raises(Exception, match=r".*pixels failed calibration*"): + self.instance.diffractionCalibration(request) + @mock.patch(thisService + "FarmFreshIngredients", spec_set=FarmFreshIngredients) @mock.patch(thisService + "FocusSpectraRecipe") def test_focusSpectra_not_exist( diff --git a/tests/util/diffraction_calibration_synthetic_data.py b/tests/util/diffraction_calibration_synthetic_data.py index b3b27e96c..edf4ce95c 100644 --- a/tests/util/diffraction_calibration_synthetic_data.py +++ b/tests/util/diffraction_calibration_synthetic_data.py @@ -111,6 +111,7 @@ def __init__(self, workspaceType: str = "Histogram", scale: float = 1000.0): maxOffset=100.0, # bins: '100.0' seems to work pixelGroup=self.fakePixelGroup, maxChiSq=100.0, + skipPixelCalibration=False, ) @staticmethod