purge peaks (#480)

docs/source/api/snapred.ui.view.rst

@@ -52,7 +52,7 @@ snapred.ui.view.DiffCalTweakPeakView module
 .. automodule:: snapred.ui.view.DiffCalTweakPeakView
    :members:
    :undoc-members:
-   :exclude-members: signalRunNumberUpdate, signalValueChanged, signalUpdateRecalculationButton, signalPeakThresholdUpdate, signalMaxChiSqUpdate, signalContinueAnyway
+   :exclude-members: signalRunNumberUpdate, signalValueChanged, signalUpdateRecalculationButton, signalPeakThresholdUpdate, signalMaxChiSqUpdate, signalContinueAnyway, signalPurgeBadPeaks
    :show-inheritance:
 
 

src/snapred/backend/dao/request/SimpleDiffCalRequest.py

@@ -0,0 +1,11 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+from snapred.backend.dao.ingredients import DiffractionCalibrationIngredients
+
+
+class SimpleDiffCalRequest(BaseModel):
+    ingredients: DiffractionCalibrationIngredients
+    groceries: Dict[str, str]
+    skipPixelCalibration: bool

src/snapred/backend/service/CalibrationService.py

@@ -28,6 +28,7 @@
     FocusSpectraRequest,
     HasStateRequest,
     InitializeStateRequest,
+    SimpleDiffCalRequest,
 )
 from snapred.backend.dao.response.CalibrationAssessmentResponse import CalibrationAssessmentResponse
 from snapred.backend.data.DataExportService import DataExportService
@@ -91,6 +92,7 @@ def __init__(self):
         self.registerPath("loadQualityAssessment", self.loadQualityAssessment)
         self.registerPath("index", self.getCalibrationIndex)
         self.registerPath("diffraction", self.diffractionCalibration)
+        self.registerPath("calibrate", self.calibrate)
         self.registerPath("validateWritePermissions", self.validateWritePermissions)
         return
 
@@ -118,7 +120,9 @@ def prepDiffractionCalibrationIngredients(
             fwhmMultipliers=request.fwhmMultipliers,
             maxChiSq=request.maxChiSq,
         )
-        return self.sousChef.prepDiffractionCalibrationIngredients(farmFresh)
+        ingredients = self.sousChef.prepDiffractionCalibrationIngredients(farmFresh)
+        ingredients.removeBackground = request.removeBackground
+        return ingredients
 
     @FromString
     def fetchDiffractionCalibrationGroceries(self, request: DiffractionCalibrationRequest) -> Dict[str, str]:
@@ -147,18 +151,20 @@ def fetchDiffractionCalibrationGroceries(self, request: DiffractionCalibrationRe
     @FromString
     def diffractionCalibration(self, request: DiffractionCalibrationRequest):
         self.validateRequest(request)
+        payload = SimpleDiffCalRequest(
+            ingredients=self.prepDiffractionCalibrationIngredients(request),
+            groceries=self.fetchDiffractionCalibrationGroceries(request),
+            skipPixelCalibration=request.skipPixelCalibration,
+        )
+        return self.calibrate(payload.json())
 
-        # ingredients
-        ingredients = self.prepDiffractionCalibrationIngredients(request)
-        ingredients.removeBackground = request.removeBackground
-        # groceries
-        groceries = self.fetchDiffractionCalibrationGroceries(request)
-
+    @FromString
+    def calibrate(self, request: SimpleDiffCalRequest):
         # now have all ingredients and groceries, run recipe
-        res = DiffractionCalibrationRecipe().executeRecipe(ingredients, groceries)
+        res = DiffractionCalibrationRecipe().executeRecipe(request.ingredients, request.groceries)
 
         if request.skipPixelCalibration is False:
-            maskWS = groceries.get("maskWorkspace", "")
+            maskWS = request.groceries.get("maskWorkspace", "")
             percentMasked = mtd[maskWS].getNumberMasked() / mtd[maskWS].getNumberHistograms()
             threshold = Config["constants.maskedPixelThreshold"]
             if percentMasked > threshold:

src/snapred/ui/view/DiffCalTweakPeakView.py

@@ -50,6 +50,7 @@ class DiffCalTweakPeakView(BackendRequestView):
     signalUpdateRecalculationButton = Signal(bool)
     signalMaxChiSqUpdate = Signal(float)
     signalContinueAnyway = Signal(bool)
+    signalPurgeBadPeaks = Signal(float)
 
     def __init__(self, samples=[], groups=[], parent=None):
         super().__init__(parent=parent)
@@ -94,6 +95,10 @@ def __init__(self, samples=[], groups=[], parent=None):
         self.recalculationButton = QPushButton("Recalculate")
         self.recalculationButton.clicked.connect(self.emitValueChange)
 
+        # a little ol purge bad peaks button
+        self.purgePeaksButton = QPushButton("Purge Bad Peaks")
+        self.purgePeaksButton.clicked.connect(self.emitPurge)
+
         # skip pixel calibration button
         self.skipPixelCalToggle = self._labeledField("Skip Pixel Calibration", Toggle(parent=self, state=False))
 
@@ -103,11 +108,12 @@ def __init__(self, samples=[], groups=[], parent=None):
         self.layout.addWidget(self.navigationBar, 1, 0)
         self.layout.addWidget(self.canvas, 2, 0, 1, -1)
         self.layout.addLayout(peakControlLayout, 3, 0, 1, 2)
+        self.layout.addWidget(self.skipPixelCalToggle, 3, 2, 1, 2)
         self.layout.addWidget(self.sampleDropdown, 4, 0)
         self.layout.addWidget(self.groupingFileDropdown, 4, 1)
         self.layout.addWidget(self.peakFunctionDropdown, 4, 2)
-        self.layout.addWidget(self.recalculationButton, 5, 0, 1, 2)
-        self.layout.addWidget(self.skipPixelCalToggle, 3, 2, 1, 2)
+        self.layout.addWidget(self.purgePeaksButton, 4, 3)
+        self.layout.addWidget(self.recalculationButton, 5, 0, 1, 4)
 
         self.layout.setRowStretch(2, 10)
 
@@ -183,6 +189,20 @@ def emitValueChange(self):
             return
         self.signalValueChanged.emit(groupingIndex, xtalDMin, xtalDMax, peakFunction, fwhm, maxChiSq)
 
+    @Slot()
+    def emitPurge(self):
+        try:
+            maxChiSq = float(self.maxChiSqField.text())
+        except ValueError:
+            QMessageBox.warning(
+                self,
+                "Invalid Chi-Squared Maximum",
+                "Enter a valid value for the maximum chi-squared before purging.",
+                QMessageBox.Ok,
+            )
+            return
+        self.signalPurgeBadPeaks.emit(maxChiSq)
+
     def updateGraphs(self, workspace, peaks, diagnostic):
         # get the updated workspaces and optimal graph grid
         self.peaks = pydantic.TypeAdapter(List[GroupPeakList]).validate_python(peaks)

src/snapred/ui/workflow/DiffCalWorkflow.py

@@ -1,4 +1,8 @@
+from mantid.simpleapi import mtd
 from qtpy.QtCore import Slot
+from qtpy.QtWidgets import (
+    QMessageBox,
+)
 
 from snapred.backend.dao import RunConfig
 from snapred.backend.dao.indexing.IndexEntry import IndexEntry
@@ -14,10 +18,12 @@
     FitMultiplePeaksRequest,
     FocusSpectraRequest,
     HasStateRequest,
+    SimpleDiffCalRequest,
 )
 from snapred.backend.dao.SNAPResponse import SNAPResponse
 from snapred.backend.error.ContinueWarning import ContinueWarning
 from snapred.backend.log.logger import snapredLogger
+from snapred.backend.recipe.algorithm.FitMultiplePeaksAlgorithm import FitOutputEnum
 from snapred.meta.Config import Config
 from snapred.meta.decorators.ExceptionToErrLog import ExceptionToErrLog
 from snapred.meta.mantid.AllowedPeakTypes import SymmetricPeakEnum
@@ -87,11 +93,14 @@ def __init__(self, parent=None):
         self._requestView.litemodeToggle.field.connectUpdate(self._switchLiteNativeGroups)
         self._requestView.runNumberField.editingFinished.connect(self._populateGroupingDropdown)
         self._tweakPeakView.signalValueChanged.connect(self.onValueChange)
+        self._tweakPeakView.signalPurgeBadPeaks.connect(self.purgeBadPeaks)
 
         self.prevFWHM = DiffCalTweakPeakView.FWHM
         self.prevXtalDMin = DiffCalTweakPeakView.XTAL_DMIN
         self.prevXtalDMax = DiffCalTweakPeakView.XTAL_DMAX
 
+        self.peaksWerePurged = False
+
         # 1. input run number and other basic parameters
         # 2. display peak graphs, allow adjustments to view
         # 3. perform diffraction calibration after user approves peaks then move on
@@ -272,9 +281,17 @@ def onValueChange(self, groupingIndex, xtalDMin, xtalDMax, peakFunction, fwhm, m
             peakFunctionChanged = peakFunction != self.peakFunction
             fwhmChanged = fwhm != self.prevFWHM
             maxChiSqChanged = maxChiSq != self.maxChiSq
-            if xtalDMinValueChanged or xtalDMaxValueChanged or peakFunctionChanged or fwhmChanged or maxChiSqChanged:
+            if (
+                xtalDMinValueChanged
+                or xtalDMaxValueChanged
+                or peakFunctionChanged
+                or fwhmChanged
+                or maxChiSqChanged
+                or self.peaksWerePurged
+            ):
                 self._renewIngredients(xtalDMin, xtalDMax, peakFunction, fwhm, maxChiSq)
                 self._renewFitPeaks(peakFunction)
+                self.peaksWerePurged = False
 
             # if the grouping file changes, load new grouping and refocus
             if groupingIndex != self.prevGroupingIndex:
@@ -344,6 +361,54 @@ def _renewFitPeaks(self, peakFunction):
         )
         return self.request(path="calibration/fitpeaks", payload=payload.json())
 
+    @ExceptionToErrLog
+    @Slot(float)
+    def purgeBadPeaks(self, maxChiSq):
+        self._tweakPeakView.disableRecalculateButton()
+        try:
+            # update the max chi sq
+            self.maxChiSq = maxChiSq
+            allPeaks = self.ingredients.groupedPeakLists
+            param_table = mtd[self.fitPeaksDiagnostic].getItem(FitOutputEnum.Parameters.value).toDict()
+            index = param_table["wsindex"]
+            allChi2 = param_table["chi2"]
+            goodPeaks = []
+            for wkspIndex, groupPeaks in enumerate(allPeaks):
+                peaks = groupPeaks.peaks
+                # collect the fit chi-sq parameters for this spectrum, and the fits
+                chi2 = [x2 for i, x2 in zip(index, allChi2) if i == wkspIndex]
+                goodPeaks.append([peak for x2, peak in zip(chi2, peaks) if x2 < maxChiSq])
+            too_fews = [goodPeak for goodPeak in goodPeaks if len(goodPeak) < 2]
+            if too_fews != []:
+                QMessageBox.critical(
+                    self._tweakPeakView,
+                    "Too Few Peaks",
+                    "Purging would result in fewer than the required 2 peaks for calibration.  "
+                    "The current set of peaks will be retained.",
+                    QMessageBox.Ok,
+                )
+            else:
+                for wkspIndex, groupPeaks in enumerate(allPeaks):
+                    groupPeaks.peaks = goodPeaks[wkspIndex]
+                self.peaksWerePurged = True
+
+            # renew the fits to the peaks
+            self._renewFitPeaks(self.peakFunction)
+
+            # update graph with reduced peak list
+            self._tweakPeakView.updateGraphs(
+                self.focusedWorkspace,
+                self.ingredients.groupedPeakLists,
+                self.fitPeaksDiagnostic,
+            )
+
+        except Exception as e:  # noqa BLE001
+            # NOTE this has the same issue as onValueChange
+            print(e)
+
+        # renable button when graph is updated
+        self._tweakPeakView.enableRecalculateButton()
+
     @Slot(WorkflowPresenter, result=SNAPResponse)
     def _triggerDiffractionCalibration(self, workflowPresenter):
         view = workflowPresenter.widget.tabView
@@ -352,24 +417,13 @@ def _triggerDiffractionCalibration(self, workflowPresenter):
         self._saveView.updateRunNumber(self.runNumber)
         self.focusGroupPath = view.groupingFileDropdown.currentText()
 
-        payload = DiffractionCalibrationRequest(
-            runNumber=self.runNumber,
-            calibrantSamplePath=self.calibrantSamplePath,
-            focusGroup=self.focusGroups[self.focusGroupPath],
-            useLiteMode=self.useLiteMode,
-            # fiddly bits
-            peakFunction=self.peakFunction,
-            crystalDMin=self.prevXtalDMin,
-            crystalDMax=self.prevXtalDMax,
-            convergenceThreshold=self.convergenceThreshold,
-            nBinsAcrossPeakWidth=self.nBinsAcrossPeakWidth,
-            fwhmMultipliers=self.prevFWHM,
-            maxChiSq=self.maxChiSq,
+        payload = SimpleDiffCalRequest(
+            ingredients=self.ingredients,
+            groceries=self.groceries,
             skipPixelCalibration=self._tweakPeakView.skipPixelCalToggle.field.getState(),
-            removeBackground=self.removeBackground,
         )
 
-        response = self.request(path="calibration/diffraction", payload=payload.json())
+        response = self.request(path="calibration/calibrate", payload=payload.json())
 
         payload = CalibrationAssessmentRequest(
             run=RunConfig(runNumber=self.runNumber),