From c5f898626b9c16f16c1932b097a8f07f052d9abb Mon Sep 17 00:00:00 2001 From: Darsh Date: Fri, 25 Oct 2024 11:20:23 -0400 Subject: [PATCH 01/11] Refactor of previous implementation of ArtificialNormalization --- .../CreateArtificialNormalizationRequest.py | 17 ++ .../backend/dao/request/ReductionRequest.py | 1 + src/snapred/backend/recipe/GenericRecipe.py | 5 + .../backend/service/ReductionService.py | 98 ++++++++-- src/snapred/ui/view/BackendRequestView.py | 4 + .../reduction/ArtificialNormalizationView.py | 172 ++++++++++++++++++ .../ui/view/reduction/ReductionRequestView.py | 2 + src/snapred/ui/widget/TrueFalseDropDown.py | 33 ++++ src/snapred/ui/workflow/ReductionWorkflow.py | 129 ++++++++++--- 9 files changed, 415 insertions(+), 46 deletions(-) create mode 100644 src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py create mode 100644 src/snapred/ui/view/reduction/ArtificialNormalizationView.py create mode 100644 src/snapred/ui/widget/TrueFalseDropDown.py diff --git a/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py new file mode 100644 index 000000000..7c94d89de --- /dev/null +++ b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel + +from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName + + +class CreateArtificialNormalizationRequest(BaseModel): + runNumber: str + useLiteMode: bool + peakWindowClippingSize: int + smoothingParameter: float + decreaseParameter: bool = True + lss: bool = True + diffractionWorkspace: WorkspaceName + + class Config: + arbitrary_types_allowed = True # Allow arbitrary types like WorkspaceName + extra = "forbid" # Forbid extra fields diff --git a/src/snapred/backend/dao/request/ReductionRequest.py b/src/snapred/backend/dao/request/ReductionRequest.py index dd2c1099d..cb82e272b 100644 --- a/src/snapred/backend/dao/request/ReductionRequest.py +++ b/src/snapred/backend/dao/request/ReductionRequest.py @@ -23,6 +23,7 @@ class ReductionRequest(BaseModel): versions: Versions = Versions(None, None) pixelMasks: List[WorkspaceName] = [] + artificialNormalization: Optional[WorkspaceName] = None # TODO: Move to SNAPRequest continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET diff --git a/src/snapred/backend/recipe/GenericRecipe.py b/src/snapred/backend/recipe/GenericRecipe.py index 437ba3ebf..22284c4ce 100644 --- a/src/snapred/backend/recipe/GenericRecipe.py +++ b/src/snapred/backend/recipe/GenericRecipe.py @@ -7,6 +7,7 @@ from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.algorithm.BufferMissingColumnsAlgo import BufferMissingColumnsAlgo from snapred.backend.recipe.algorithm.CalibrationMetricExtractionAlgorithm import CalibrationMetricExtractionAlgorithm +from snapred.backend.recipe.algorithm.CreateArtificialNormalizationAlgo import CreateArtificialNormalizationAlgo from snapred.backend.recipe.algorithm.DetectorPeakPredictor import DetectorPeakPredictor from snapred.backend.recipe.algorithm.FitMultiplePeaksAlgorithm import FitMultiplePeaksAlgorithm from snapred.backend.recipe.algorithm.FocusSpectraAlgorithm import FocusSpectraAlgorithm @@ -104,3 +105,7 @@ class ConvertTableToMatrixWorkspaceRecipe(GenericRecipe[ConvertTableToMatrixWork class BufferMissingColumnsRecipe(GenericRecipe[BufferMissingColumnsAlgo]): pass + + +class ArtificialNormalizationRecipe(GenericRecipe[CreateArtificialNormalizationAlgo]): + pass diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 84d09cdbf..9aac9b68f 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -3,9 +3,14 @@ from pathlib import Path from typing import Any, Dict, List -from snapred.backend.dao.ingredients import GroceryListItem, ReductionIngredients +from snapred.backend.dao.ingredients import ( + ArtificialNormalizationIngredients, + GroceryListItem, + ReductionIngredients, +) from snapred.backend.dao.reduction.ReductionRecord import ReductionRecord from snapred.backend.dao.request import ( + CreateArtificialNormalizationRequest, FarmFreshIngredients, ReductionExportRequest, ReductionRequest, @@ -20,6 +25,7 @@ from snapred.backend.error.StateValidationException import StateValidationException from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.algorithm.MantidSnapper import MantidSnapper +from snapred.backend.recipe.GenericRecipe import ArtificialNormalizationRecipe from snapred.backend.recipe.ReductionRecipe import ReductionRecipe from snapred.backend.service.Service import Service from snapred.backend.service.SousChef import SousChef @@ -72,6 +78,9 @@ def __init__(self): self.registerPath("checkWritePermissions", self.checkWritePermissions) self.registerPath("getSavePath", self.getSavePath) self.registerPath("getStateIds", self.getStateIds) + self.registerPath("validateReduction", self.validateReduction) + self.registerPath("artificialNormalization", self.artificialNormalization) + self.registerPath("grabDiffractionWorkspaceforArtificialNorm", self.grabDiffractionWorkspaceforArtificialNorm) return @staticmethod @@ -80,47 +89,68 @@ def name(): def validateReduction(self, request: ReductionRequest): """ - Validate the reduction request. + Validate the reduction request, providing specific messages if normalization + or calibration data is missing. Notify the user if artificial normalization + will be created when normalization is absent. :param request: a reduction request :type request: ReductionRequest """ continueFlags = ContinueWarning.Type.UNSET - # check if a normalization is present - if not self.dataFactoryService.normalizationExists(request.runNumber, request.useLiteMode): + useArtificialNorm = False + message = "" + + # Check if a normalization is present + normalizationExists = self.dataFactoryService.normalizationExists(request.runNumber, request.useLiteMode) + # Check if a diffraction calibration is present + calibrationExists = self.dataFactoryService.calibrationExists(request.runNumber, request.useLiteMode) + + # Determine the action based on missing components + if not calibrationExists and not normalizationExists: + # Case: No calibration and no normalization + continueFlags |= ContinueWarning.Type.MISSING_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION + message = ( + "Reduction is missing both normalization and calibration data. " + "Would you like to continue in uncalibrated mode?" + ) + elif calibrationExists and not normalizationExists: + # Case: Calibration exists but normalization is missing continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION - # check if a diffraction calibration is present - if not self.dataFactoryService.calibrationExists(request.runNumber, request.useLiteMode): - continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION + useArtificialNorm = True + message = ( + "Reduction is missing normalization data. " + "Artificial normalization will be created in place of actual normalization. " + "Would you like to continue?" + ) - # remove any continue flags that are present in the request by xor-ing with the flags + # Remove any continue flags that are present in the request by XOR-ing with the flags if request.continueFlags: - continueFlags = continueFlags ^ (request.continueFlags & continueFlags) + continueFlags ^= request.continueFlags & continueFlags - if continueFlags: - raise ContinueWarning( - "Reduction is missing calibration data, continue in uncalibrated mode?", continueFlags - ) + # If there are any continue flags set, raise a ContinueWarning with the appropriate message + if continueFlags and message: + raise ContinueWarning(message, continueFlags) - # ... ensure separate continue warnings ... + # Ensure separate continue warnings for permission check continueFlags = ContinueWarning.Type.UNSET - # check that the user has write permissions to the save directory + # Check that the user has write permissions to the save directory if not self.checkWritePermissions(request.runNumber): continueFlags |= ContinueWarning.Type.NO_WRITE_PERMISSIONS - # remove any continue flags that are present in the request by xor-ing with the flags + # Remove any continue flags that are present in the request by XOR-ing with the flags if request.continueFlags: - continueFlags = continueFlags ^ (request.continueFlags & continueFlags) + continueFlags ^= request.continueFlags & continueFlags if continueFlags: raise ContinueWarning( f"

It looks like you don't have permissions to write to " f"
{self.getSavePath(request.runNumber)},
" - + "but you can still save using the workbench tools.

" - + "

Would you like to continue anyway?

", + "but you can still save using the workbench tools.

" + "

Would you like to continue anyway?

", continueFlags, ) + return useArtificialNorm @FromString def reduction(self, request: ReductionRequest): @@ -130,7 +160,6 @@ def reduction(self, request: ReductionRequest): :param request: a ReductionRequest object holding needed information :type request: ReductionRequest """ - self.validateReduction(request) groupingResults = self.fetchReductionGroupings(request) request.focusGroups = groupingResults["focusGroups"] @@ -424,3 +453,32 @@ def _groupByVanadiumVersion(self, requests: List[SNAPRequest]): def getCompatibleMasks(self, request: ReductionRequest) -> List[WorkspaceName]: runNumber, useLiteMode = request.runNumber, request.useLiteMode return self.dataFactoryService.getCompatibleReductionMasks(runNumber, useLiteMode) + + def artificialNormalization(self, request: CreateArtificialNormalizationRequest): + ingredients = ArtificialNormalizationIngredients( + peakWindowClippingSize=request.peakWindowClippingSize, + smoothingParameter=request.smoothingParameter, + decreaseParameter=request.decreaseParameter, + lss=request.lss, + ) + artificialNormWorkspace = ArtificialNormalizationRecipe().executeRecipe( + InputWorkspace=request.diffractionWorkspace, + Ingredients=ingredients, + ) + return artificialNormWorkspace + + def grabDiffractionWorkspaceforArtificialNorm(self, request: ReductionRequest): + calVersion = None + calVersion = self.dataFactoryService.getThisOrLatestCalibrationVersion(request.runNumber, request.useLiteMode) + groceryList = ( + self.groceryClerk.name("diffractionWorkspace") + .diffcal_output(request.runNumber, calVersion) + .useLiteMode(request.useLiteMode) + .unit(wng.Units.DSP) + .group("column") + .buildDict() + ) + + groceries = self.groceryService.fetchGroceryDict(groceryList) + diffractionWorkspace = groceries.get("diffractionWorkspace") + return diffractionWorkspace diff --git a/src/snapred/ui/view/BackendRequestView.py b/src/snapred/ui/view/BackendRequestView.py index 6def779e2..1644b17f0 100644 --- a/src/snapred/ui/view/BackendRequestView.py +++ b/src/snapred/ui/view/BackendRequestView.py @@ -6,6 +6,7 @@ from snapred.ui.widget.LabeledField import LabeledField from snapred.ui.widget.MultiSelectDropDown import MultiSelectDropDown from snapred.ui.widget.SampleDropDown import SampleDropDown +from snapred.ui.widget.TrueFalseDropDown import TrueFalseDropDown class BackendRequestView(QWidget): @@ -33,6 +34,9 @@ def _labeledCheckBox(self, label): def _sampleDropDown(self, label, items=[]): return SampleDropDown(label, items, self) + def _trueFalseDropDown(self, label): + return TrueFalseDropDown(label, self) + def _multiSelectDropDown(self, label, items=[]): return MultiSelectDropDown(label, items, self) diff --git a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py new file mode 100644 index 000000000..7f5ad4ff3 --- /dev/null +++ b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py @@ -0,0 +1,172 @@ +import matplotlib.pyplot as plt +from mantid.plots.datafunctions import get_spectrum +from mantid.simpleapi import mtd +from qtpy.QtCore import Signal, Slot +from qtpy.QtWidgets import ( + QHBoxLayout, + QLineEdit, + QMessageBox, + QPushButton, +) +from snapred.meta.Config import Config +from snapred.meta.decorators.Resettable import Resettable +from snapred.ui.view.BackendRequestView import BackendRequestView +from snapred.ui.widget.SmoothingSlider import SmoothingSlider +from workbench.plotting.figuremanager import MantidFigureCanvas +from workbench.plotting.toolbar import WorkbenchNavigationToolbar + + +@Resettable +class ArtificialNormalizationView(BackendRequestView): + signalRunNumberUpdate = Signal(str) + signalValueChanged = Signal(float, bool, bool, int) + signalUpdateRecalculationButton = Signal(bool) + signalUpdateFields = Signal(float, bool, bool) + + def __init__(self, parent=None): + super().__init__(parent=parent) + + # create the run number fields + self.fieldRunNumber = self._labeledField("Run Number", QLineEdit()) + + # create the graph elements + self.figure = plt.figure(constrained_layout=True) + self.canvas = MantidFigureCanvas(self.figure) + self.navigationBar = WorkbenchNavigationToolbar(self.canvas, self) + + # create the other specification elements + self.lssDropdown = self._trueFalseDropDown("LSS") + self.decreaseParameterDropdown = self._trueFalseDropDown("Decrease Parameter") + + # disable run number + for x in [self.fieldRunNumber]: + x.setEnabled(False) + + # create the adjustment controls + self.smoothingSlider = self._labeledField("Smoothing", SmoothingSlider()) + self.peakWindowClippingSize = self._labeledField( + "Peak Window Clipping Size", + QLineEdit(str(Config["constants.ArtificialNormalization.peakWindowClippingSize"])), + ) + + peakControlLayout = QHBoxLayout() + peakControlLayout.addWidget(self.smoothingSlider, 2) + peakControlLayout.addWidget(self.peakWindowClippingSize) + + # a big ol recalculate button + self.recalculationButton = QPushButton("Recalculate") + self.recalculationButton.clicked.connect(self.emitValueChange) + + # add all elements to the grid layout + self.layout.addWidget(self.fieldRunNumber, 0, 0) + self.layout.addWidget(self.navigationBar, 1, 0) + self.layout.addWidget(self.canvas, 2, 0, 1, -1) + self.layout.addLayout(peakControlLayout, 3, 0, 1, 2) + self.layout.addWidget(self.lssDropdown, 4, 0) + self.layout.addWidget(self.decreaseParameterDropdown, 4, 1) + self.layout.addWidget(self.recalculationButton, 5, 0, 1, 2) + + self.layout.setRowStretch(2, 10) + + # store the initial layout without graphs + self.initialLayoutHeight = self.size().height() + + self.signalUpdateRecalculationButton.connect(self.setEnableRecalculateButton) + self.signalUpdateFields.connect(self._updateFields) + + @Slot(str) + def _updateRunNumber(self, runNumber): + self.fieldRunNumber.setText(runNumber) + + def updateRunNumber(self, runNumber): + self.signalRunNumberUpdate.emit(runNumber) + + @Slot(float, bool, bool) + def _updateFields(self, smoothingParameter, lss, decreaseParameter): + self.smoothingSlider.field.setValue(smoothingParameter) + self.lssDropdown.setCurrentIndex(lss) + self.decreaseParameterDropdown.setCurrentIndex(decreaseParameter) + + def updateFields(self, smoothingParameter, lss, decreaseParameter): + self.signalUpdateFields.emit(smoothingParameter, lss, decreaseParameter) + + @Slot() + def emitValueChange(self): + # verify the fields before recalculation + try: + smoothingValue = self.smoothingSlider.field.value() + lss = self.lssDropdown.currentIndex() == "True" + decreaseParameter = self.decreaseParameterDropdown.currentIndex == "True" + peakWindowClippingSize = int(self.peakWindowClippingSize.field.text()) + except ValueError as e: + QMessageBox.warning( + self, + "Invalid Peak Parameters", + f"Smoothing or peak window clipping size is invalid: {str(e)}", + QMessageBox.Ok, + ) + return + self.signalValueChanged.emit(smoothingValue, lss, decreaseParameter, peakWindowClippingSize) + + def updateWorkspaces(self, diffractionWorkspace, artificialNormWorkspace): + self.diffractionWorkspace = diffractionWorkspace + self.artificialNormWorkspace = artificialNormWorkspace + self._updateGraphs() + + def _updateGraphs(self): + # get the updated workspaces and optimal graph grid + diffractionWorkspace = mtd[self.diffractionWorkspace] + artificialNormWorkspace = mtd[self.artificialNormWorkspace] + numGraphs = diffractionWorkspace.getNumberHistograms() + nrows, ncols = self._optimizeRowsAndCols(numGraphs) + + # now re-draw the figure + self.figure.clear() + for i in range(numGraphs): + ax = self.figure.add_subplot(nrows, ncols, i + 1, projection="mantid") + ax.plot(diffractionWorkspace, wkspIndex=i, label="Diffcal Data", normalize_by_bin_width=True) + ax.plot( + artificialNormWorkspace, + wkspIndex=i, + label="Artificial Normalization Data", + normalize_by_bin_width=True, + linestyle="--", + ) + ax.legend() + ax.tick_params(direction="in") + ax.set_title(f"Group ID: {i + 1}") + # fill in the discovered peaks for easier viewing + x, y, _, _ = get_spectrum(diffractionWorkspace, i, normalize_by_bin_width=True) + # for each detected peak in this group, shade in the peak region + + # resize window and redraw + self.setMinimumHeight(self.initialLayoutHeight + int(self.figure.get_size_inches()[1] * self.figure.dpi)) + self.canvas.draw() + + def _optimizeRowsAndCols(self, numGraphs): + # Get best size for layout + sqrtSize = int(numGraphs**0.5) + if sqrtSize == numGraphs**0.5: + rowSize = sqrtSize + colSize = sqrtSize + elif numGraphs <= ((sqrtSize + 1) * sqrtSize): + rowSize = sqrtSize + colSize = sqrtSize + 1 + else: + rowSize = sqrtSize + 1 + colSize = sqrtSize + 1 + return rowSize, colSize + + @Slot(bool) + def setEnableRecalculateButton(self, enable): + self.recalculationButton.setEnabled(enable) + + def disableRecalculateButton(self): + self.signalUpdateRecalculationButton.emit(False) + + def enableRecalculateButton(self): + self.signalUpdateRecalculationButton.emit(True) + + def verify(self): + # TODO what needs to be verified? + return True diff --git a/src/snapred/ui/view/reduction/ReductionRequestView.py b/src/snapred/ui/view/reduction/ReductionRequestView.py index be181e2b4..58c1c66cc 100644 --- a/src/snapred/ui/view/reduction/ReductionRequestView.py +++ b/src/snapred/ui/view/reduction/ReductionRequestView.py @@ -136,6 +136,8 @@ def clearRunNumbers(self): def verify(self): currentText = self.runNumberDisplay.toPlainText() runNumbers = [num.strip() for num in currentText.split("\n") if num.strip()] + if not runNumbers: + raise ValueError("Please enter at least one run number.") for runNumber in runNumbers: if not runNumber.isdigit(): raise ValueError( diff --git a/src/snapred/ui/widget/TrueFalseDropDown.py b/src/snapred/ui/widget/TrueFalseDropDown.py new file mode 100644 index 000000000..9a0fe6594 --- /dev/null +++ b/src/snapred/ui/widget/TrueFalseDropDown.py @@ -0,0 +1,33 @@ +from qtpy.QtWidgets import QComboBox, QVBoxLayout, QWidget + + +class TrueFalseDropDown(QWidget): + def __init__(self, label, parent=None): + super(TrueFalseDropDown, self).__init__(parent) + self.setStyleSheet("background-color: #F5E9E2;") + self._label = label + + self.dropDown = QComboBox() + self._initItems() + + layout = QVBoxLayout() + layout.addWidget(self.dropDown) + self.setLayout(layout) + + def _initItems(self): + self.dropDown.clear() + self.dropDown.addItem(self._label) + self.dropDown.addItems(["True", "False"]) + self.dropDown.model().item(0).setEnabled(False) + self.dropDown.setCurrentIndex(1) + + def currentIndex(self): + # Subtract 1 because the label is considered an index + return self.dropDown.currentIndex() - 1 + + def setCurrentIndex(self, index): + # Add 1 to skip the label + self.dropDown.setCurrentIndex(index + 1) + + def currentText(self): + return self.dropDown.currentText() diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index ebc9edb06..269c5dc93 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -3,6 +3,7 @@ from qtpy.QtCore import Slot from snapred.backend.dao.request import ( + CreateArtificialNormalizationRequest, ReductionExportRequest, ReductionRequest, ) @@ -11,6 +12,7 @@ from snapred.backend.log.logger import snapredLogger from snapred.meta.decorators.ExceptionToErrLog import ExceptionToErrLog from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName +from snapred.ui.view.reduction.ArtificialNormalizationView import ArtificialNormalizationView from snapred.ui.view.reduction.ReductionRequestView import ReductionRequestView from snapred.ui.view.reduction.ReductionSaveView import ReductionSaveView from snapred.ui.workflow.WorkflowBuilder import WorkflowBuilder @@ -33,9 +35,8 @@ def __init__(self, parent=None): self._reductionRequestView.enterRunNumberButton.clicked.connect(lambda: self._populatePixelMaskDropdown()) self._reductionRequestView.pixelMaskDropdown.dropDown.view().pressed.connect(self._onPixelMaskSelection) - self._reductionSaveView = ReductionSaveView( - parent=parent, - ) + self._artificialNormalizationView = ArtificialNormalizationView(parent=parent) + self._reductionSaveView = ReductionSaveView(parent=parent) self.workflow = ( WorkflowBuilder( @@ -51,10 +52,16 @@ def __init__(self, parent=None): "Reduction", continueAnywayHandler=self._continueAnywayHandler, ) + .addNode( + self._continueWithNormalization, + self._artificialNormalizationView, + "Artificial Normalization", + ) .build() ) self._reductionRequestView.retainUnfocusedDataCheckbox.checkedChanged.connect(self._enableConvertToUnits) + self._artificialNormalizationView.signalValueChanged.connect(self.onArtificialNormalizationValueChange) def _enableConvertToUnits(self): state = self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked() @@ -130,11 +137,8 @@ def _validateRunNumbers(self, runNumbers: List[str]): stateIds = self.request(path="reduction/getStateIds", payload=runNumbers).data except Exception as e: # noqa: BLE001 raise ValueError(f"Unable to get instrument state for {runNumbers}: {e}") - if len(stateIds) > 1: - stateId = stateIds[0] - for id_ in stateIds[1:]: - if id_ != stateId: - raise ValueError("all run numbers must be from the same state") + if len(stateIds) > 1 and len(set(stateIds)) > 1: + raise ValueError("All run numbers must be from the same state") def _reconstructPixelMaskNames(self, pixelMasks: List[str]) -> List[WorkspaceName]: return [self._compatibleMasks[name] for name in pixelMasks] @@ -169,26 +173,101 @@ def _triggerReduction(self, workflowPresenter): convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), ) - response = self.request(path="reduction/", payload=request_) - if response.code == ResponseCode.OK: - record, unfocusedData = response.data.record, response.data.unfocusedData - - # .. update "save" panel message: - self.savePath = self.request(path="reduction/getSavePath", payload=record.runNumber).data + # Validate reduction; if artificial normalization is needed, handle it + response = self.request(path="reduction/validateReduction", payload=request_) + if response.data: + response = self.request(path="reduction/grabDiffractionWorkspaceforArtificialNorm", payload=request_) + + # Prepare artificial normalization request + request = CreateArtificialNormalizationRequest( # noqa: F841 + runNumber=str(runNumber), + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + peakWindowClippingSize=int(self._artificialNormalizationView.peakWindowClippingSize.field.text()), + smoothingParameter=self._artificialNormalizationView.smoothingSlider.field.value(), + diffractionWorkspace=request.data, # noqa: F821 + ) + response = self.request(path="reduction/artificialNormalization", payload=request_) + self._continueWithNormalization(workflowPresenter) + else: + # Proceed with reduction if artificial normalization is not needed + response = self.request(path="reduction/", payload=request_) + if response.code == ResponseCode.OK: + record, unfocusedData = response.data.record, response.data.unfocusedData + self._finalizeReduction(record, unfocusedData) + + def _artificialNormalization(self, workflowPresenter, responseData, runNumber): + """Handles artificial normalization for the workflow.""" + view = workflowPresenter.widget.tabView # noqa: F841 + request_ = CreateArtificialNormalizationRequest( + runNumber=runNumber, + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + peakWindowClippingSize=int(self._artificialNormalizationView.peakWindowClippingSize.field.text()), + smoothingParameter=self._artificialNormalizationView.smoothingSlider.field.value(), + decreaseParameter=self._artificialNormalizationView.decreaseParameterDropdown.currentIndex() == 1, + lss=self._artificialNormalizationView.lssDropdown.currentIndex() == 1, + diffractionWorkspace=responseData.diffractionWorkspace, + ) + response = self.request(path="reduction/artificialNormalization", payload=request_) + # Update artificial normalization view with the response + if response.code == ResponseCode.OK: + self._artificialNormalizationView.updateWorkspaces(responseData.diffractionWorkspace, response.data) + + @Slot(float, bool, bool, int) + def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreaseParameter, peakWindowClippingSize): + """Updates artificial normalization based on user input.""" + self._artificialNormalizationView.disableRecalculateButton() + runNumber = self._artificialNormalizationView.fieldRunNumber.text() + diffractionWorkspace = self._artificialNormalizationView.diffractionWorkspace + + request_ = CreateArtificialNormalizationRequest( + runNumber=runNumber, + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + peakWindowClippingSize=peakWindowClippingSize, + smoothingParameter=smoothingValue, + decreaseParameter=decreaseParameter, + lss=lss, + diffractionWorkspace=diffractionWorkspace, + ) - # Save the reduced data. (This is automatic: it happens before the "save" panel opens.) - if ContinueWarning.Type.NO_WRITE_PERMISSIONS not in self.continueAnywayFlags: - self.request(path="reduction/save", payload=ReductionExportRequest(record=record)) + response = self.request(path="reduction/artificialNormalization", payload=request_) + self._artificialNormalizationView.updateWorkspaces(diffractionWorkspace, response.data) + self._artificialNormalizationView.enableRecalculateButton() - # Retain the output workspaces after the workflow is complete. - self.outputs.extend(record.workspaceNames) + def _continueWithNormalization(self, workflowPresenter): # noqa: ARG002 + """Continues the workflow using the artificial normalization workspace.""" + artificialNormWorkspace = self._artificialNormalizationView.artificialNormWorkspace + pixelMasks = self._reconstructPixelMaskNames(self._reductionRequestView.getPixelMasks()) + timestamp = self.request(path="reduction/getUniqueTimestamp").data - # Also retain the unfocused data after the workflow is complete (if the box was checked), - # but do not actually save it as part of the reduction-data file. - # The unfocused data does not get added to the response.workspaces list. - if unfocusedData is not None: - self.outputs.append(unfocusedData) + request_ = ReductionRequest( + runNumber=str(self._artificialNormalizationView.fieldRunNumber.text()), + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + timestamp=timestamp, + continueFlags=self.continueAnywayFlags, + pixelMasks=pixelMasks, + keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(), + convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), + normalizationWorkspace=artificialNormWorkspace, + ) + response = self.request(path="reduction/", payload=request_) + if response.code == ResponseCode.OK: + record, unfocusedData = response.data.record, response.data.unfocusedData + self._finalizeReduction(record, unfocusedData) + + def _finalizeReduction(self, record, unfocusedData): + """Handles post-reduction tasks, including saving and workspace management.""" + self.savePath = self.request(path="reduction/getSavePath", payload=record.runNumber).data + # Save the reduced data. (This is automatic: it happens before the "save" panel opens.) + if ContinueWarning.Type.NO_WRITE_PERMISSIONS not in self.continueAnywayFlags: + self.request(path="reduction/save", payload=ReductionExportRequest(record=record)) + # Retain the output workspaces after the workflow is complete. + self.outputs.extend(record.workspaceNames) + # Also retain the unfocused data after the workflow is complete (if the box was checked), + # but do not actually save it as part of the reduction-data file. + # The unfocused data does not get added to the response.workspaces list. + if unfocusedData: + self.outputs.append(unfocusedData) # Note that the run number is deliberately not deleted from the run numbers list. # Almost certainly it should be moved to a "completed run numbers" list. @@ -197,8 +276,6 @@ def _triggerReduction(self, workflowPresenter): # TODO: make '_clearWorkspaces' a public method (i.e make this combination a special `cleanup` method). self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True) - return self.responses[-1] - @property def widget(self): return self.workflow.presenter.widget From 7e2fa8834c3c8d41613bb97b9bbd6a1fd27a09c1 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 28 Oct 2024 09:15:55 -0400 Subject: [PATCH 02/11] Some more updates --- .../CreateArtificialNormalizationRequest.py | 10 +++++++++- src/snapred/ui/workflow/ReductionWorkflow.py | 18 +++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py index 7c94d89de..f8792ea94 100644 --- a/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py +++ b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, root_validator from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName @@ -11,7 +11,15 @@ class CreateArtificialNormalizationRequest(BaseModel): decreaseParameter: bool = True lss: bool = True diffractionWorkspace: WorkspaceName + outputWorkspace: WorkspaceName = None + + @root_validator(pre=True) + def set_output_workspace(cls, values): + if values.get("diffractionWorkspace") and not values.get("outputWorkspace"): + values["outputWorkspace"] = WorkspaceName(f"{values['diffractionWorkspace']}_artificialNorm") + return values class Config: arbitrary_types_allowed = True # Allow arbitrary types like WorkspaceName extra = "forbid" # Forbid extra fields + validate_assignment = True # Enable dynamic validation diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index 269c5dc93..51fc3cf8d 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -177,16 +177,7 @@ def _triggerReduction(self, workflowPresenter): response = self.request(path="reduction/validateReduction", payload=request_) if response.data: response = self.request(path="reduction/grabDiffractionWorkspaceforArtificialNorm", payload=request_) - - # Prepare artificial normalization request - request = CreateArtificialNormalizationRequest( # noqa: F841 - runNumber=str(runNumber), - useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), - peakWindowClippingSize=int(self._artificialNormalizationView.peakWindowClippingSize.field.text()), - smoothingParameter=self._artificialNormalizationView.smoothingSlider.field.value(), - diffractionWorkspace=request.data, # noqa: F821 - ) - response = self.request(path="reduction/artificialNormalization", payload=request_) + self._artificialNormalization(workflowPresenter, response.data, runNumber) self._continueWithNormalization(workflowPresenter) else: # Proceed with reduction if artificial normalization is not needed @@ -194,6 +185,7 @@ def _triggerReduction(self, workflowPresenter): if response.code == ResponseCode.OK: record, unfocusedData = response.data.record, response.data.unfocusedData self._finalizeReduction(record, unfocusedData) + return self.responses[-1] def _artificialNormalization(self, workflowPresenter, responseData, runNumber): """Handles artificial normalization for the workflow.""" @@ -205,12 +197,14 @@ def _artificialNormalization(self, workflowPresenter, responseData, runNumber): smoothingParameter=self._artificialNormalizationView.smoothingSlider.field.value(), decreaseParameter=self._artificialNormalizationView.decreaseParameterDropdown.currentIndex() == 1, lss=self._artificialNormalizationView.lssDropdown.currentIndex() == 1, - diffractionWorkspace=responseData.diffractionWorkspace, + diffractionWorkspace=responseData, ) response = self.request(path="reduction/artificialNormalization", payload=request_) # Update artificial normalization view with the response if response.code == ResponseCode.OK: self._artificialNormalizationView.updateWorkspaces(responseData.diffractionWorkspace, response.data) + else: + raise RuntimeError("Failed to run artificial normalization.") @Slot(float, bool, bool, int) def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreaseParameter, peakWindowClippingSize): @@ -255,6 +249,8 @@ def _continueWithNormalization(self, workflowPresenter): # noqa: ARG002 record, unfocusedData = response.data.record, response.data.unfocusedData self._finalizeReduction(record, unfocusedData) + return self.responses[-1] + def _finalizeReduction(self, record, unfocusedData): """Handles post-reduction tasks, including saving and workspace management.""" self.savePath = self.request(path="reduction/getSavePath", payload=record.runNumber).data From fec2e3cd7b88c525a3f1f4ff7d250f9fdf2e4425 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 28 Oct 2024 09:16:22 -0400 Subject: [PATCH 03/11] Whoops --- src/snapred/backend/service/ReductionService.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 9aac9b68f..9c5ae9652 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -464,6 +464,7 @@ def artificialNormalization(self, request: CreateArtificialNormalizationRequest) artificialNormWorkspace = ArtificialNormalizationRecipe().executeRecipe( InputWorkspace=request.diffractionWorkspace, Ingredients=ingredients, + OutputWorkspace=request.outputWorkspace, ) return artificialNormWorkspace From e5f7fae29680bf007fa0e618d92177ad9b6970c9 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 28 Oct 2024 11:04:40 -0400 Subject: [PATCH 04/11] Fixes within workflow view --- .../backend/dao/request/ReductionRequest.py | 2 +- src/snapred/ui/presenter/WorkflowPresenter.py | 16 ++++++++++------ .../reduction/ArtificialNormalizationView.py | 1 + src/snapred/ui/workflow/ReductionWorkflow.py | 9 ++++++--- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/snapred/backend/dao/request/ReductionRequest.py b/src/snapred/backend/dao/request/ReductionRequest.py index cb82e272b..2b77e8480 100644 --- a/src/snapred/backend/dao/request/ReductionRequest.py +++ b/src/snapred/backend/dao/request/ReductionRequest.py @@ -23,7 +23,7 @@ class ReductionRequest(BaseModel): versions: Versions = Versions(None, None) pixelMasks: List[WorkspaceName] = [] - artificialNormalization: Optional[WorkspaceName] = None + artificialNormalization: Optional[str] = None # TODO: Move to SNAPRequest continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET diff --git a/src/snapred/ui/presenter/WorkflowPresenter.py b/src/snapred/ui/presenter/WorkflowPresenter.py index 2a6d64321..2b543331d 100644 --- a/src/snapred/ui/presenter/WorkflowPresenter.py +++ b/src/snapred/ui/presenter/WorkflowPresenter.py @@ -141,12 +141,7 @@ def handleSkipButtonClicked(self): def advanceWorkflow(self): if self.view.currentTab >= self.view.totalNodes - 1: - QMessageBox.information( - self.view, - "‧₊Workflow Complete‧₊", - self.completionMessageLambda(), - ) - self.reset() + self.completeWorkflow() else: self.view.advanceWorkflow() @@ -196,3 +191,12 @@ def continueAnyway(self, continueInfo: ContinueWarning.Model): else: raise NotImplementedError(f"Continue anyway handler not implemented: {self.view.tabModel}") self.handleContinueButtonClicked(self.view.tabModel) + + def completeWorkflow(self): + # Directly show the completion message and reset the workflow + QMessageBox.information( + self.view, + "‧₊Workflow Complete‧₊", + self.completionMessageLambda(), + ) + self.reset() diff --git a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py index 7f5ad4ff3..9a9e637df 100644 --- a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py +++ b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py @@ -73,6 +73,7 @@ def __init__(self, parent=None): self.signalUpdateRecalculationButton.connect(self.setEnableRecalculateButton) self.signalUpdateFields.connect(self._updateFields) + self.signalRunNumberUpdate.connect(self._updateRunNumber) @Slot(str) def _updateRunNumber(self, runNumber): diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index 51fc3cf8d..17e40a090 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -176,15 +176,16 @@ def _triggerReduction(self, workflowPresenter): # Validate reduction; if artificial normalization is needed, handle it response = self.request(path="reduction/validateReduction", payload=request_) if response.data: + self._artificialNormalizationView.updateRunNumber(runNumber) response = self.request(path="reduction/grabDiffractionWorkspaceforArtificialNorm", payload=request_) self._artificialNormalization(workflowPresenter, response.data, runNumber) - self._continueWithNormalization(workflowPresenter) else: # Proceed with reduction if artificial normalization is not needed response = self.request(path="reduction/", payload=request_) if response.code == ResponseCode.OK: record, unfocusedData = response.data.record, response.data.unfocusedData self._finalizeReduction(record, unfocusedData) + self.workflowPresenter._completeWorkflow() return self.responses[-1] def _artificialNormalization(self, workflowPresenter, responseData, runNumber): @@ -202,10 +203,12 @@ def _artificialNormalization(self, workflowPresenter, responseData, runNumber): response = self.request(path="reduction/artificialNormalization", payload=request_) # Update artificial normalization view with the response if response.code == ResponseCode.OK: - self._artificialNormalizationView.updateWorkspaces(responseData.diffractionWorkspace, response.data) + self._artificialNormalizationView.updateWorkspaces(responseData, response.data) else: raise RuntimeError("Failed to run artificial normalization.") + return self.responses[-1] + @Slot(float, bool, bool, int) def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreaseParameter, peakWindowClippingSize): """Updates artificial normalization based on user input.""" @@ -241,7 +244,7 @@ def _continueWithNormalization(self, workflowPresenter): # noqa: ARG002 pixelMasks=pixelMasks, keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(), convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), - normalizationWorkspace=artificialNormWorkspace, + artificialNormalization=artificialNormWorkspace, ) response = self.request(path="reduction/", payload=request_) From 597ebb4e13c1dca86a51f20fe693d30468f99a5f Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 28 Oct 2024 15:16:33 -0400 Subject: [PATCH 05/11] Final fixes to the workflow. --- .../reduction/ArtificialNormalizationView.py | 21 ++++++++++++++++++- src/snapred/ui/workflow/ReductionWorkflow.py | 4 +++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py index 9a9e637df..f08c5a5b2 100644 --- a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py +++ b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py @@ -1,9 +1,10 @@ import matplotlib.pyplot as plt from mantid.plots.datafunctions import get_spectrum from mantid.simpleapi import mtd -from qtpy.QtCore import Signal, Slot +from qtpy.QtCore import Qt, Signal, Slot from qtpy.QtWidgets import ( QHBoxLayout, + QLabel, QLineEdit, QMessageBox, QPushButton, @@ -75,6 +76,12 @@ def __init__(self, parent=None): self.signalUpdateFields.connect(self._updateFields) self.signalRunNumberUpdate.connect(self._updateRunNumber) + self.messageLabel = QLabel("") + self.messageLabel.setStyleSheet("font-size: 24px; font-weight: bold; color: black;") + self.messageLabel.setAlignment(Qt.AlignCenter) + self.layout.addWidget(self.messageLabel, 0, 0, 1, 2) + self.messageLabel.hide() + @Slot(str) def _updateRunNumber(self, runNumber): self.fieldRunNumber.setText(runNumber) @@ -171,3 +178,15 @@ def enableRecalculateButton(self): def verify(self): # TODO what needs to be verified? return True + + def showMessage(self, message: str): + self.clearView() + self.messageLabel.setText(message) + self.messageLabel.show() + + def clearView(self): + # Remove all existing widgets except the layout + for i in reversed(range(self.layout.count())): + widget = self.layout.itemAt(i).widget() + if widget is not None and widget != self.messageLabel: + widget.deleteLater() # Delete the widget diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index 17e40a090..a827fdb42 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -185,7 +185,9 @@ def _triggerReduction(self, workflowPresenter): if response.code == ResponseCode.OK: record, unfocusedData = response.data.record, response.data.unfocusedData self._finalizeReduction(record, unfocusedData) - self.workflowPresenter._completeWorkflow() + self._artificialNormalizationView.updateRunNumber(runNumber) + self._artificialNormalizationView.showMessage("Artificial Normalization not Needed") + workflowPresenter.advanceWorkflow() return self.responses[-1] def _artificialNormalization(self, workflowPresenter, responseData, runNumber): From 240e00d6895687aa7a55871a0cbd9cc74111f1be Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 28 Oct 2024 16:43:06 -0400 Subject: [PATCH 06/11] Update tests --- .../backend/service/ReductionService.py | 21 ++++++++++++------- .../backend/service/test_ReductionService.py | 11 ++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 9c5ae9652..ac322cc7e 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -106,13 +106,10 @@ def validateReduction(self, request: ReductionRequest): calibrationExists = self.dataFactoryService.calibrationExists(request.runNumber, request.useLiteMode) # Determine the action based on missing components - if not calibrationExists and not normalizationExists: - # Case: No calibration and no normalization - continueFlags |= ContinueWarning.Type.MISSING_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION - message = ( - "Reduction is missing both normalization and calibration data. " - "Would you like to continue in uncalibrated mode?" - ) + if not calibrationExists and normalizationExists: + # Case: No calibration but normalization exists + continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION + message = "Reduction is missing calibration data. " "Would you like to continue in uncalibrated mode?" elif calibrationExists and not normalizationExists: # Case: Calibration exists but normalization is missing continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION @@ -122,6 +119,15 @@ def validateReduction(self, request: ReductionRequest): "Artificial normalization will be created in place of actual normalization. " "Would you like to continue?" ) + elif not calibrationExists and not normalizationExists: + # Case: No calibration and no normalization + continueFlags |= ( + ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION + ) + message = ( + "Reduction is missing both normalization and calibration data. " + "Would you like to continue in uncalibrated mode?" + ) # Remove any continue flags that are present in the request by XOR-ing with the flags if request.continueFlags: @@ -150,6 +156,7 @@ def validateReduction(self, request: ReductionRequest): "

Would you like to continue anyway?

", continueFlags, ) + return useArtificialNorm @FromString diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index c8a157f9c..d0717756f 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -300,17 +300,16 @@ def test_validateReduction_no_permissions_and_no_calibrations(self): fakeDataService.calibrationExists.return_value = False fakeDataService.normalizationExists.return_value = False self.instance.dataFactoryService = fakeDataService + fakeExportService = mock.Mock() fakeExportService.checkWritePermissions.return_value = False self.instance.dataExportService = fakeExportService + with pytest.raises(ContinueWarning) as excInfo: self.instance.validateReduction(self.request) - # Note: this tests the _first_ continue-anyway check, - # which _only_ deals with the calibrations. - assert ( - excInfo.value.model.flags - == ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION + assert excInfo.value.model.flags == ( + ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION ) def test_validateReduction_no_permissions_and_no_calibrations_first_reentry(self): @@ -328,8 +327,6 @@ def test_validateReduction_no_permissions_and_no_calibrations_first_reentry(self with pytest.raises(ContinueWarning) as excInfo: self.instance.validateReduction(self.request) - # Note: this tests re-entry for the _first_ continue-anyway check, - # but with no re-entry for the second continue-anyway check. assert excInfo.value.model.flags == ContinueWarning.Type.NO_WRITE_PERMISSIONS def test_validateReduction_no_permissions_and_no_calibrations_second_reentry(self): From dc643ad3aa5665ecf531e82e0834d9547718606a Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 28 Oct 2024 17:21:09 -0400 Subject: [PATCH 07/11] Code cov --- .../backend/service/test_ReductionService.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index d0717756f..240e0aa23 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -14,6 +14,7 @@ from snapred.backend.dao.ingredients.ReductionIngredients import ReductionIngredients from snapred.backend.dao.reduction.ReductionRecord import ReductionRecord from snapred.backend.dao.request import ( + CreateArtificialNormalizationRequest, ReductionExportRequest, ReductionRequest, ) @@ -347,6 +348,58 @@ def test_validateReduction_no_permissions_and_no_calibrations_second_reentry(sel # and in addition, re-entry for the second continue-anyway check. self.instance.validateReduction(self.request) + @mock.patch(thisService + "ArtificialNormalizationRecipe") + def test_artificialNormalization(self, mockArtificialNormalizationRecipe): + mockArtificialNormalizationRecipe.return_value = mock.Mock() + mockResult = mock.Mock() + mockArtificialNormalizationRecipe.return_value.executeRecipe.return_value = mockResult + + request = CreateArtificialNormalizationRequest( + runNumber="123", + useLiteMode=False, + peakWindowClippingSize=5, + smoothingParameter=0.1, + decreaseParameter=True, + lss=True, + diffractionWorkspace="mock_diffraction_workspace", + outputWorkspace="mock_output_workspace", + ) + + result = self.instance.artificialNormalization(request) + + mockArtificialNormalizationRecipe.return_value.executeRecipe.assert_called_once_with( + InputWorkspace=request.diffractionWorkspace, + Ingredients=mock.ANY, + OutputWorkspace=request.outputWorkspace, + ) + assert result == mockResult + + @mock.patch(thisService + "GroceryService") + def test_grabDiffractionWorkspaceforArtificialNorm(self, mockGroceryService): + self.instance.groceryService = mockGroceryService + + request = ReductionRequest( + runNumber="123", + useLiteMode=False, + timestamp=self.instance.getUniqueTimestamp(), + versions=(1, 2), + pixelMasks=[], + focusGroups=[FocusGroup(name="apple", definition="path/to/grouping")], + ) + + mockCalVersion = 1 + self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=mockCalVersion) + + groceryList = { + "diffractionWorkspace": "mock_diffraction_workspace", + } + mockGroceryService.fetchGroceryDict.return_value = groceryList + + result = self.instance.grabDiffractionWorkspaceforArtificialNorm(request) + + mockGroceryService.fetchGroceryDict.assert_called_once() + assert result == "mock_diffraction_workspace" + class TestReductionServiceMasks: @pytest.fixture(autouse=True, scope="class") From 96ffb6af2fed45d1ee9415cc83a2105fef068c47 Mon Sep 17 00:00:00 2001 From: Darsh Date: Tue, 29 Oct 2024 07:34:31 -0400 Subject: [PATCH 08/11] Updates to warning messages in ReductionService.py for Defect 7756 --- src/snapred/backend/service/ReductionService.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index ac322cc7e..1bc99141d 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -109,13 +109,16 @@ def validateReduction(self, request: ReductionRequest): if not calibrationExists and normalizationExists: # Case: No calibration but normalization exists continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION - message = "Reduction is missing calibration data. " "Would you like to continue in uncalibrated mode?" + message = ( + "Warning: diffraction calibration is missing." + "If you continue, default instrument geometry will be used." + ) elif calibrationExists and not normalizationExists: # Case: Calibration exists but normalization is missing continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION useArtificialNorm = True message = ( - "Reduction is missing normalization data. " + "Warning: Reduction is missing normalization data. " "Artificial normalization will be created in place of actual normalization. " "Would you like to continue?" ) @@ -125,8 +128,8 @@ def validateReduction(self, request: ReductionRequest): ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION ) message = ( - "Reduction is missing both normalization and calibration data. " - "Would you like to continue in uncalibrated mode?" + "Warning: Reduction is missing both normalization and calibration data. " + "If you continue, default instrument geometry will be used and data will not be normalized." ) # Remove any continue flags that are present in the request by XOR-ing with the flags From 19b1437120455033dd1433716fdc0ef64043592f Mon Sep 17 00:00:00 2001 From: Darsh Date: Tue, 29 Oct 2024 16:36:10 -0400 Subject: [PATCH 09/11] Updates based on Michael's comments --- src/snapred/backend/service/ReductionService.py | 4 ---- src/snapred/ui/workflow/ReductionWorkflow.py | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 1bc99141d..dd04084d8 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -97,7 +97,6 @@ def validateReduction(self, request: ReductionRequest): :type request: ReductionRequest """ continueFlags = ContinueWarning.Type.UNSET - useArtificialNorm = False message = "" # Check if a normalization is present @@ -116,7 +115,6 @@ def validateReduction(self, request: ReductionRequest): elif calibrationExists and not normalizationExists: # Case: Calibration exists but normalization is missing continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION - useArtificialNorm = True message = ( "Warning: Reduction is missing normalization data. " "Artificial normalization will be created in place of actual normalization. " @@ -160,8 +158,6 @@ def validateReduction(self, request: ReductionRequest): continueFlags, ) - return useArtificialNorm - @FromString def reduction(self, request: ReductionRequest): """ diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index a827fdb42..f3d4a3b51 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -14,7 +14,6 @@ from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName from snapred.ui.view.reduction.ArtificialNormalizationView import ArtificialNormalizationView from snapred.ui.view.reduction.ReductionRequestView import ReductionRequestView -from snapred.ui.view.reduction.ReductionSaveView import ReductionSaveView from snapred.ui.workflow.WorkflowBuilder import WorkflowBuilder from snapred.ui.workflow.WorkflowImplementer import WorkflowImplementer @@ -36,7 +35,6 @@ def __init__(self, parent=None): self._reductionRequestView.pixelMaskDropdown.dropDown.view().pressed.connect(self._onPixelMaskSelection) self._artificialNormalizationView = ArtificialNormalizationView(parent=parent) - self._reductionSaveView = ReductionSaveView(parent=parent) self.workflow = ( WorkflowBuilder( @@ -175,7 +173,7 @@ def _triggerReduction(self, workflowPresenter): # Validate reduction; if artificial normalization is needed, handle it response = self.request(path="reduction/validateReduction", payload=request_) - if response.data: + if ContinueWarning.Type.MISSING_NORMALIZATION in self.continueAnywayFlags: self._artificialNormalizationView.updateRunNumber(runNumber) response = self.request(path="reduction/grabDiffractionWorkspaceforArtificialNorm", payload=request_) self._artificialNormalization(workflowPresenter, response.data, runNumber) From 302de0b80532c6214cb497deb03ed9746b71d55a Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 30 Oct 2024 13:58:54 -0400 Subject: [PATCH 10/11] More updates based on comments --- .../backend/service/ReductionService.py | 18 +++++++-------- src/snapred/ui/widget/TrueFalseDropDown.py | 18 +++++++-------- .../backend/service/test_ReductionService.py | 23 +++++++++++++------ 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index dd04084d8..9b75492a6 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -128,6 +128,8 @@ def validateReduction(self, request: ReductionRequest): message = ( "Warning: Reduction is missing both normalization and calibration data. " "If you continue, default instrument geometry will be used and data will not be normalized." + "Artificial normalization cannot currently be made for uncalibrated data as we are missing peak positions." # noqa: E501 + "We are working on a solution to this problem." ) # Remove any continue flags that are present in the request by XOR-ing with the flags @@ -477,15 +479,11 @@ def artificialNormalization(self, request: CreateArtificialNormalizationRequest) def grabDiffractionWorkspaceforArtificialNorm(self, request: ReductionRequest): calVersion = None calVersion = self.dataFactoryService.getThisOrLatestCalibrationVersion(request.runNumber, request.useLiteMode) - groceryList = ( - self.groceryClerk.name("diffractionWorkspace") - .diffcal_output(request.runNumber, calVersion) - .useLiteMode(request.useLiteMode) - .unit(wng.Units.DSP) - .group("column") - .buildDict() - ) + calRecord = self.dataFactoryService.getCalibrationRecord(request.runNumber, request.useLiteMode, calVersion) + filePath = self.dataFactoryService.getCalibrationDataPath(request.runNumber, request.useLiteMode, calVersion) + diffCalOutput = calRecord.workspaces[wngt.DIFFCAL_OUTPUT][0] + diffcalOutputFilePath = str(filePath) + "/" + str(diffCalOutput) + ".nxs.h5" - groceries = self.groceryService.fetchGroceryDict(groceryList) - diffractionWorkspace = groceries.get("diffractionWorkspace") + groceries = self.groceryService.fetchWorkspace(diffcalOutputFilePath, "diffractionWorkspace") + diffractionWorkspace = groceries.get("workspace") return diffractionWorkspace diff --git a/src/snapred/ui/widget/TrueFalseDropDown.py b/src/snapred/ui/widget/TrueFalseDropDown.py index 9a0fe6594..bb41951f2 100644 --- a/src/snapred/ui/widget/TrueFalseDropDown.py +++ b/src/snapred/ui/widget/TrueFalseDropDown.py @@ -1,33 +1,31 @@ -from qtpy.QtWidgets import QComboBox, QVBoxLayout, QWidget +from qtpy.QtWidgets import QComboBox, QHBoxLayout, QLabel, QWidget class TrueFalseDropDown(QWidget): def __init__(self, label, parent=None): super(TrueFalseDropDown, self).__init__(parent) self.setStyleSheet("background-color: #F5E9E2;") - self._label = label + self._label = QLabel(label + ":", self) self.dropDown = QComboBox() self._initItems() - layout = QVBoxLayout() + layout = QHBoxLayout() + layout.addWidget(self._label) layout.addWidget(self.dropDown) + layout.setContentsMargins(5, 5, 5, 5) self.setLayout(layout) def _initItems(self): self.dropDown.clear() - self.dropDown.addItem(self._label) self.dropDown.addItems(["True", "False"]) - self.dropDown.model().item(0).setEnabled(False) - self.dropDown.setCurrentIndex(1) + self.dropDown.setCurrentIndex(0) def currentIndex(self): - # Subtract 1 because the label is considered an index - return self.dropDown.currentIndex() - 1 + return self.dropDown.currentIndex() def setCurrentIndex(self, index): - # Add 1 to skip the label - self.dropDown.setCurrentIndex(index + 1) + self.dropDown.setCurrentIndex(index) def currentText(self): return self.dropDown.currentText() diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index 240e0aa23..4790097a3 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -375,8 +375,10 @@ def test_artificialNormalization(self, mockArtificialNormalizationRecipe): assert result == mockResult @mock.patch(thisService + "GroceryService") - def test_grabDiffractionWorkspaceforArtificialNorm(self, mockGroceryService): + @mock.patch(thisService + "DataFactoryService") + def test_grabDiffractionWorkspaceforArtificialNorm(self, mockDataFactoryService, mockGroceryService): self.instance.groceryService = mockGroceryService + self.instance.dataFactoryService = mockDataFactoryService request = ReductionRequest( runNumber="123", @@ -388,16 +390,23 @@ def test_grabDiffractionWorkspaceforArtificialNorm(self, mockGroceryService): ) mockCalVersion = 1 - self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=mockCalVersion) + mockDataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=mockCalVersion) - groceryList = { - "diffractionWorkspace": "mock_diffraction_workspace", - } - mockGroceryService.fetchGroceryDict.return_value = groceryList + mockCalRecord = mock.Mock() + mockCalRecord.workspaces = {"diffCalOutput": ["mock_diffraction_workspace"]} + + mockDataFactoryService.getCalibrationRecord = mock.Mock(return_value=mockCalRecord) + + mockDataFactoryService.getCalibrationDataPath = mock.Mock(return_value="mock/path/to/calibration") + + mockGroceryService.fetchWorkspace = mock.Mock(return_value={"workspace": "mock_diffraction_workspace"}) result = self.instance.grabDiffractionWorkspaceforArtificialNorm(request) - mockGroceryService.fetchGroceryDict.assert_called_once() + expected_file_path = "mock/path/to/calibration/mock_diffraction_workspace.nxs.h5" + mockGroceryService.fetchWorkspace.assert_called_once_with(expected_file_path, "diffractionWorkspace") + + # Verify the result assert result == "mock_diffraction_workspace" From 53d2f010c0b99a8fb0c52dabfd1f6a985ba7c5dd Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 30 Oct 2024 15:21:02 -0400 Subject: [PATCH 11/11] Last updates. --- .../backend/service/ReductionService.py | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 9b75492a6..a44280b5f 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -127,9 +127,7 @@ def validateReduction(self, request: ReductionRequest): ) message = ( "Warning: Reduction is missing both normalization and calibration data. " - "If you continue, default instrument geometry will be used and data will not be normalized." - "Artificial normalization cannot currently be made for uncalibrated data as we are missing peak positions." # noqa: E501 - "We are working on a solution to this problem." + "If you continue, default instrument geometry will be used and data will be artificially normalized. " ) # Remove any continue flags that are present in the request by XOR-ing with the flags @@ -477,13 +475,26 @@ def artificialNormalization(self, request: CreateArtificialNormalizationRequest) return artificialNormWorkspace def grabDiffractionWorkspaceforArtificialNorm(self, request: ReductionRequest): - calVersion = None - calVersion = self.dataFactoryService.getThisOrLatestCalibrationVersion(request.runNumber, request.useLiteMode) - calRecord = self.dataFactoryService.getCalibrationRecord(request.runNumber, request.useLiteMode, calVersion) - filePath = self.dataFactoryService.getCalibrationDataPath(request.runNumber, request.useLiteMode, calVersion) - diffCalOutput = calRecord.workspaces[wngt.DIFFCAL_OUTPUT][0] - diffcalOutputFilePath = str(filePath) + "/" + str(diffCalOutput) + ".nxs.h5" - - groceries = self.groceryService.fetchWorkspace(diffcalOutputFilePath, "diffractionWorkspace") - diffractionWorkspace = groceries.get("workspace") + try: + calVersion = None + calVersion = self.dataFactoryService.getThisOrLatestCalibrationVersion( + request.runNumber, request.useLiteMode + ) + calRecord = self.dataFactoryService.getCalibrationRecord(request.runNumber, request.useLiteMode, calVersion) + filePath = self.dataFactoryService.getCalibrationDataPath( + request.runNumber, request.useLiteMode, calVersion + ) + diffCalOutput = calRecord.workspaces[wngt.DIFFCAL_OUTPUT][0] + diffcalOutputFilePath = str(filePath) + "/" + str(diffCalOutput) + ".nxs.h5" + + groceries = self.groceryService.fetchWorkspace(diffcalOutputFilePath, "diffractionWorkspace") + diffractionWorkspace = groceries.get("workspace") + except: # noqa: E722 + raise RuntimeError( + "This feature is not yet implemented. " + "Artificial normalization cannot currently be made for uncalibrated data as we are missing peak positions. " # noqa: E501 + "We are working on a solution to this problem.\n\n " + f"No calibration record found for run number: {request.runNumber}.\n" + "Please create calibration data for this run number and try again." + ) return diffractionWorkspace