Ewm6378 fix full masking in reduction (#479)

* Fix for testing on analysis.

* Fixes for test on analysis now that I've used pdb to debug.

* Test fixes.

* Update code cov

* Fix diffcal_masking_script.py

* Revert

* Revert

src/snapred/backend/recipe/ReductionRecipe.py

@@ -143,6 +143,21 @@ def _prepGroupingWorkspaces(self, groupingIndex: int):
             self.groceries["normalizationWorkspace"] = normalizationClone
         return sampleClone, normalizationClone
 
+    def _isGroupFullyMasked(self, groupingWorkspace: str) -> bool:
+        maskWorkspace = self.mantidSnapper.mtd[self.maskWs]
+        groupWorkspace = self.mantidSnapper.mtd[groupingWorkspace]
+
+        totalMaskedPixels = 0
+        totalGroupPixels = 0
+
+        for i in range(groupWorkspace.getNumberHistograms()):
+            group_spectra = groupWorkspace.readY(i)
+            for spectrumIndex in group_spectra:
+                if maskWorkspace.readY(int(spectrumIndex))[0] == 1:
+                    totalMaskedPixels += 1
+                totalGroupPixels += 1
+        return totalMaskedPixels == totalGroupPixels
+
     def queueAlgos(self):
         pass
 
@@ -172,7 +187,15 @@ def execute(self):
         for groupingIndex, groupingWs in enumerate(self.groupingWorkspaces):
             self.groceries["groupingWorkspace"] = groupingWs
 
-            # Clone
+            if self.maskWs and self._isGroupFullyMasked(groupingWs):
+                # Notify the user of a fully masked group, but continue with the workflow
+                self.logger().warning(
+                    f"\nAll pixels masked within {groupingWs} schema.\n"
+                    + "Skipping all algorithm execution for this group.\n"
+                    + "This will affect future reductions."
+                )
+                continue
+
             sampleClone, normalizationClone = self._prepGroupingWorkspaces(groupingIndex)
 
             # 2. ReductionGroupProcessingRecipe

tests/cis_tests/diffcal_masking_script.py

@@ -71,12 +71,11 @@
     maskSpectra,
     setGroupSpectraToZero,
     maskGroups,
-    pause
 )
 from util.IPTS_override import datasearch_directories
 
 ## If required: override the IPTS search directories: ##
-instrumentHome = "/mnt/R5_data1/data1/workspaces/ORNL-work/SNAPRed/SNS_root/SNAP"
+instrumentHome = "/SNS/SNAP"
 ConfigService.Instance().setDataSearchDirs(datasearch_directories(instrumentHome))
 Config._config["instrument"]["home"] = instrumentHome + os.sep
 ########################################################
@@ -105,7 +104,6 @@
     focusGroups=[{"name": groupingScheme, "definition": ""}],
     cifPath=cifPath,
     calibrantSamplePath=calibrantSamplePath,
-    peakIntensityThresold=peakThreshold,
     convergenceThreshold=offsetConvergenceLimit,
     maxOffset=100.0,
 )
@@ -127,10 +125,12 @@
 
 ### Here any specific spectra or isolated detectors can be masked in the input, if required for testing...
 # ---
-# maskWS = mtd[maskWSName]
-# inputWS = mtd[inputWSName]
-# groupingWS = mtd[groupingWSName]
+maskWS = mtd[maskWSName]
+inputWS = mtd[inputWSName]
+groupingWS = mtd[groupingWSName]
 
+allSpectra = list(range(inputWS.getNumberHistograms()))
+maskSpectra(maskWS, inputWS, allSpectra)
 # # mask all detectors contributing to spectra 10, 20, and 30:
 # spectraToMask = (10, 20, 30)
 # maskSpectra(maskWS, inputWS, spectraToMask)

tests/unit/backend/recipe/test_ReductionRecipe.py

@@ -130,11 +130,25 @@ def test_cloneAndConvertWorkspace(self):
         with pytest.raises(ValueError, match=r"cannot convert to unit.*"):
             recipe._cloneAndConvertWorkspace(workspace, units)
 
-    def test_keepUnfocusedData(self):
-        # Prepare recipe for testing
+    @mock.patch("mantid.simpleapi.mtd", create=True)
+    def test_keepUnfocusedData(self, mockMtd):
+        mockMantidSnapper = mock.Mock()
+
+        mockMaskWorkspace = mock.Mock()
+        mockGroupWorkspace = mock.Mock()
+
+        mockGroupWorkspace.getNumberHistograms.return_value = 10
+        mockGroupWorkspace.readY.return_value = [0] * 10
+        mockMaskWorkspace.readY.return_value = [0] * 10
+
+        # Mock mtd to return mask and group workspaces
+        mockMtd.__getitem__.side_effect = lambda ws_name: mockMaskWorkspace if ws_name == "mask" else mockGroupWorkspace
         recipe = ReductionRecipe()
-        recipe.groceries = {}
+        recipe.mantidSnapper = mockMantidSnapper
+        recipe.mantidSnapper.mtd = mockMtd
 
+        # Set up ingredients and other variables for the recipe
+        recipe.groceries = {}
         recipe.ingredients = mock.Mock()
         recipe.ingredients.groupProcessing = mock.Mock(
             return_value=lambda groupingIndex: f"groupProcessing_{groupingIndex}"
@@ -146,22 +160,26 @@ def test_keepUnfocusedData(self):
             return_value=lambda groupingIndex: f"applyNormalization_{groupingIndex}"
         )
 
+        # Mock internal methods of recipe
         recipe._applyRecipe = mock.Mock()
         recipe._cloneIntermediateWorkspace = mock.Mock()
         recipe._deleteWorkspace = mock.Mock()
         recipe._cloneAndConvertWorkspace = mock.Mock()
         recipe._prepGroupingWorkspaces = mock.Mock()
         recipe._prepGroupingWorkspaces.return_value = ("sample_grouped", "norm_grouped")
+
+        # Set up other recipe variables
         recipe.sampleWs = "sample"
         recipe.maskWs = "mask"
         recipe.normalizationWs = "norm"
         recipe.groupingWorkspaces = ["group1", "group2"]
         recipe.keepUnfocused = True
-
-        # Test keeping unfocused data in dSpacing units
         recipe.convertUnitsTo = "dSpacing"
+
+        # Execute the recipe
         result = recipe.execute()
 
+        # Assertions
         recipe._cloneAndConvertWorkspace.assert_called_once_with("sample", "dSpacing")
         assert recipe._deleteWorkspace.call_count == len(recipe._prepGroupingWorkspaces.return_value)
         recipe._deleteWorkspace.assert_called_with("norm_grouped")
@@ -289,12 +307,26 @@ def test_cloneIntermediateWorkspace(self):
             mock.ANY, InputWorkspace="input", OutputWorkspace="output"
         )
 
-    def test_execute(self):
+    @mock.patch("mantid.simpleapi.mtd", create=True)
+    def test_execute(self, mockMtd):
+        mockMantidSnapper = mock.Mock()
+
+        mockMaskworkspace = mock.Mock()
+        mockGroupWorkspace = mock.Mock()
+
+        mockGroupWorkspace.getNumberHistograms.return_value = 10
+        mockGroupWorkspace.readY.return_value = [0] * 10
+        mockMaskworkspace.readY.return_value = [0] * 10
+
+        mockMtd.__getitem__.side_effect = lambda ws_name: mockMaskworkspace if ws_name == "mask" else mockGroupWorkspace
+
         recipe = ReductionRecipe()
-        recipe.groceries = {}
+        recipe.mantidSnapper = mockMantidSnapper
+        recipe.mantidSnapper.mtd = mockMtd
 
+        # Set up ingredients and other variables for the recipe
+        recipe.groceries = {}
         recipe.ingredients = mock.Mock()
-        # recipe.ingredients.preprocess = mock.Mock()
         recipe.ingredients.groupProcessing = mock.Mock(
             return_value=lambda groupingIndex: f"groupProcessing_{groupingIndex}"
         )
@@ -305,21 +337,26 @@ def test_execute(self):
             return_value=lambda groupingIndex: f"applyNormalization_{groupingIndex}"
         )
 
+        # Mock internal methods of recipe
         recipe._applyRecipe = mock.Mock()
         recipe._cloneIntermediateWorkspace = mock.Mock()
         recipe._deleteWorkspace = mock.Mock()
         recipe._cloneAndConvertWorkspace = mock.Mock()
         recipe._prepGroupingWorkspaces = mock.Mock()
         recipe._prepGroupingWorkspaces.return_value = ("sample_grouped", "norm_grouped")
+
+        # Set up other recipe variables
         recipe.sampleWs = "sample"
         recipe.maskWs = "mask"
         recipe.normalizationWs = "norm"
         recipe.groupingWorkspaces = ["group1", "group2"]
         recipe.keepUnfocused = True
         recipe.convertUnitsTo = "TOF"
 
+        # Execute the recipe
         result = recipe.execute()
 
+        # Perform assertions
         recipe._applyRecipe.assert_any_call(
             PreprocessReductionRecipe,
             recipe.ingredients.preprocess(),
@@ -368,6 +405,125 @@ def test_execute(self):
         assert recipe._deleteWorkspace.call_count == len(recipe._prepGroupingWorkspaces.return_value)
         assert result["outputs"][0] == "sample_grouped"
 
+    @mock.patch("mantid.simpleapi.mtd", create=True)
+    def test_isGroupFullyMasked(self, mockMtd):
+        mockMantidSnapper = mock.Mock()
+
+        # Mock the group and mask workspaces
+        mockMaskWorkspace = mock.Mock()
+        mockGroupWorkspace = mock.Mock()
+
+        # Case 1: All pixels are masked
+        mockGroupWorkspace.getNumberHistograms.return_value = 10
+        mockGroupWorkspace.readY.side_effect = lambda i: [i]  # Assume each group has a single index per spectrum
+        mockMaskWorkspace.readY.side_effect = lambda i: [1]  # Assume every pixel is masked # noqa: ARG005
+
+        # Mock mtd to return the group and mask workspaces
+        mockMtd.__getitem__.side_effect = lambda ws_name: mockMaskWorkspace if ws_name == "mask" else mockGroupWorkspace
+
+        # Attach mocked mantidSnapper to recipe and assign mocked mtd
+        recipe = ReductionRecipe()
+        recipe.mantidSnapper = mockMantidSnapper
+        recipe.mantidSnapper.mtd = mockMtd
+        recipe.maskWs = "mask"
+
+        # Test when all pixels are masked
+        result = recipe._isGroupFullyMasked("groupWorkspace")
+        assert result is True, "Expected _isGroupFullyMasked to return True when all pixels are masked."
+
+        # Case 2: Not all pixels are masked
+        mockMaskWorkspace.readY.side_effect = lambda i: [0] if i % 2 == 0 else [1]  # Only half the pixels are masked
+
+        # Test when not all pixels are masked
+        result = recipe._isGroupFullyMasked("groupWorkspace")
+        assert result is False, "Expected _isGroupFullyMasked to return False when not all pixels are masked."
+
+    @mock.patch("mantid.simpleapi.mtd", create=True)
+    def test_execute_with_fully_masked_group(self, mockMtd):
+        mock_mantid_snapper = mock.Mock()
+
+        # Mock the mask and group workspaces
+        mockMaskWorkspace = mock.Mock()
+        mockGroupWorkspace = mock.Mock()
+
+        # Mock groupWorkspace to have all pixels masked
+        mockGroupWorkspace.getNumberHistograms.return_value = 10
+        mockGroupWorkspace.readY.side_effect = lambda i: [i]  # Spectrum index per spectrum
+        mockMaskWorkspace.readY.side_effect = lambda i: [1]  # All pixels are masked # noqa: ARG005
+
+        # Mock mtd to return the group and mask workspaces
+        mockMtd.__getitem__.side_effect = lambda ws_name: mockMaskWorkspace if ws_name == "mask" else mockGroupWorkspace
+
+        # Attach mocked mantidSnapper to recipe and assign mocked mtd
+        recipe = ReductionRecipe()
+        recipe.mantidSnapper = mock_mantid_snapper
+        recipe.mantidSnapper.mtd = mockMtd
+        recipe.maskWs = "mask"
+
+        # Set up logger to capture warnings
+        recipe.logger = mock.Mock()
+
+        # Set up ingredients and other variables for the recipe
+        recipe.groceries = {}
+        recipe.ingredients = mock.Mock()
+        recipe.ingredients.groupProcessing = mock.Mock(
+            return_value=lambda groupingIndex: f"groupProcessing_{groupingIndex}"
+        )
+        recipe.ingredients.generateFocussedVanadium = mock.Mock(
+            return_value=lambda groupingIndex: f"generateFocussedVanadium_{groupingIndex}"
+        )
+        recipe.ingredients.applyNormalization = mock.Mock(
+            return_value=lambda groupingIndex: f"applyNormalization_{groupingIndex}"
+        )
+
+        # Mock internal methods of recipe
+        recipe._applyRecipe = mock.Mock()
+        recipe._cloneIntermediateWorkspace = mock.Mock()
+        recipe._deleteWorkspace = mock.Mock()
+        recipe._cloneAndConvertWorkspace = mock.Mock()
+        recipe._prepGroupingWorkspaces = mock.Mock()
+        recipe._prepGroupingWorkspaces.return_value = ("sample_grouped", "norm_grouped")
+
+        # Set up other recipe variables
+        recipe.sampleWs = "sample"
+        recipe.normalizationWs = "norm"
+        recipe.groupingWorkspaces = ["group1", "group2"]
+        recipe.keepUnfocused = True
+        recipe.convertUnitsTo = "TOF"
+
+        # Execute the recipe
+        result = recipe.execute()
+
+        # Assertions for both groups being fully masked
+        expected_warning_message_group1 = (
+            "\nAll pixels masked within group1 schema.\n"
+            "Skipping all algorithm execution for this group.\n"
+            "This will affect future reductions."
+        )
+
+        expected_warning_message_group2 = (
+            "\nAll pixels masked within group2 schema.\n"
+            "Skipping all algorithm execution for this group.\n"
+            "This will affect future reductions."
+        )
+
+        # Check that the warnings were logged for both groups
+        recipe.logger().warning.assert_any_call(expected_warning_message_group1)
+        recipe.logger().warning.assert_any_call(expected_warning_message_group2)
+
+        # Ensure the warning was called twice (once per group)
+        assert (
+            recipe.logger().warning.call_count == 2
+        ), "Expected warning to be logged twice for the fully masked groups."
+
+        # Ensure no algorithms were applied for the fully masked groups
+        assert (
+            recipe._applyRecipe.call_count == 2
+        ), "Expected _applyRecipe to not be called for the fully masked groups."
+
+        # Check the output result contains the mask workspace
+        assert result["outputs"][0] == "mask", "Expected the mask workspace to be included in the outputs."
+
     def test_cook(self):
         recipe = ReductionRecipe()
         recipe.prep = mock.Mock()