From 4a5cc2faa7488b538da05fb190289429d118f2c8 Mon Sep 17 00:00:00 2001
From: Casey McGinley <cmcginley@splunk.com>
Date: Tue, 29 Oct 2024 15:40:07 -0700
Subject: [PATCH] adding error filtering

---
 .../objects/content_versioning_service.py     | 18 ++++++++++-----
 contentctl/objects/correlation_search.py      | 22 ++++++++++++++++---
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py
index 6ebc064c..fc9eb08e 100644
--- a/contentctl/objects/content_versioning_service.py
+++ b/contentctl/objects/content_versioning_service.py
@@ -19,10 +19,12 @@
 # TODO (cmcginley):
 #   - [x] version naming scheme seems to have changed from X - X to X.X
 #   - [x] sourcetype no longer holds detection name but instead is stash_common_detection_model
-#   - [ ] action.escu.full_search_name no longer available
-#   - [ ] check to see if we can get "name"
+#   - [x] action.escu.full_search_name no longer available
+#   - [x] check to see if we can get "name"
 #   - [ ] move strings to enums
-#   - [ ] additionally, timeout for cms_parser seems to need more time
+#   - [x] additionally, timeout for cms_parser seems to need more time
+#   - [ ] validate multi-line fields -> search, description, action.notable.param.rule_description,
+#         action.notable.param.drilldown_searches
 
 # TODO (cmcginley): suppress logging
 # Suppress logging by default; enable for local testing
@@ -366,14 +368,20 @@ def validate_content_against_cms(self) -> None:
         remaining_detections = {x.name: x for x in self.detections}
         matched_detections: dict[str, Detection] = {}
 
+        # Create a filter for a specific memory error we're ok ignoring
+        sub_second_order_pattern = re.compile(
+            r".*Events might not be returned in sub-second order due to search memory limits.*"
+        )
+
         # Iterate over the results until we've gone through them all
         while offset < result_count:
             iterator = ResultIterator(
-                job.results(                                                                        # type: ignore
+                response_reader=job.results(                                                        # type: ignore
                     output_mode="json",
                     count=count,
                     offset=offset
-                )
+                ),
+                error_filters=[sub_second_order_pattern]
             )
 
             # Iterate over the currently fetched results
diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py
index 8fa3d534..8a27ebf3 100644
--- a/contentctl/objects/correlation_search.py
+++ b/contentctl/objects/correlation_search.py
@@ -1,6 +1,7 @@
 import logging
 import time
 import json
+import re
 from typing import Any
 from enum import Enum
 from functools import cached_property
@@ -34,7 +35,7 @@
 
 
 # Suppress logging by default; enable for local testing
-ENABLE_LOGGING = False
+ENABLE_LOGGING = True
 LOG_LEVEL = logging.DEBUG
 LOG_PATH = "correlation_search.log"
 
@@ -93,15 +94,25 @@ class ResultIterator:
     Given a ResponseReader, constructs a JSONResultsReader and iterates over it; when Message instances are encountered,
     they are logged if the message is anything other than "error", in which case an error is raised. Regular results are
     returned as expected
+
     :param response_reader: a ResponseReader object
-    :param logger: a Logger object
+    :type response_reader: :class:`splunklib.binding.ResponseReader`
+    :param error_filters: set of re Patterns used to filter out errors we're ok ignoring
+    :type error_filters: list[:class:`re.Pattern[str]`]
     """
-    def __init__(self, response_reader: ResponseReader) -> None:
+    def __init__(
+            self,
+            response_reader: ResponseReader,
+            error_filters: list[re.Pattern[str]] = []
+    ) -> None:
         # init the results reader
         self.results_reader: JSONResultsReader = JSONResultsReader(
             response_reader
         )
 
+        # the list of patterns for errors to ignore
+        self.error_filters: list[re.Pattern[str]] = error_filters
+
         # get logger
         self.logger: logging.Logger = Utils.get_logger(
             __name__,
@@ -127,6 +138,11 @@ def __next__(self) -> dict[str, Any]:
                 message = f"SPLUNK: {result.message}"
                 self.logger.log(level, message)
                 if level == logging.ERROR:
+                    # if the error matches any of the filters, just continue on
+                    for filter in self.error_filters:
+                        if filter.match(message):
+                            continue
+                    # if no filter was matched, raise
                     raise ServerError(message)
 
             # if dict, just return