From bd1cb6ba7c20cfd07a11c502d8ddc6629b4c4548 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Thu, 17 Oct 2024 17:38:39 -0400 Subject: [PATCH 1/4] feat: original copy of code_owner code This is the code_owner code as found in edx-django-utils. --- .../datadog_monitoring/code_owner/__init__.py | 6 + .../code_owner/middleware.py | 175 ++++++++++ .../datadog_monitoring/code_owner/utils.py | 305 +++++++++++++++++ ..._code_owner_custom_attribute_to_an_ida.rst | 85 +++++ ..._monitoring_for_squad_or_theme_changes.rst | 44 +++ .../tests/code_owner/__init__.py | 0 .../tests/code_owner/mock_views.py | 12 + .../tests/code_owner/test_middleware.py | 321 ++++++++++++++++++ .../tests/code_owner/test_utils.py | 146 ++++++++ 9 files changed, 1094 insertions(+) create mode 100644 edx_arch_experiments/datadog_monitoring/code_owner/__init__.py create mode 100644 edx_arch_experiments/datadog_monitoring/code_owner/middleware.py create mode 100644 edx_arch_experiments/datadog_monitoring/code_owner/utils.py create mode 100644 edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst create mode 100644 edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst create mode 100644 edx_arch_experiments/datadog_monitoring/tests/code_owner/__init__.py create mode 100644 edx_arch_experiments/datadog_monitoring/tests/code_owner/mock_views.py create mode 100644 edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py create mode 100644 edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py diff --git a/edx_arch_experiments/datadog_monitoring/code_owner/__init__.py b/edx_arch_experiments/datadog_monitoring/code_owner/__init__.py new file mode 100644 index 0000000..952966e --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/code_owner/__init__.py @@ -0,0 +1,6 @@ +""" +This directory should only be used internally. + +Its public API is exposed in the top-level monitoring __init__.py. +See its README.rst for details. +""" diff --git a/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py b/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py new file mode 100644 index 0000000..1af5583 --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py @@ -0,0 +1,175 @@ +""" +Middleware for code_owner custom attribute +""" +import logging + +from django.urls import resolve +from django.urls.exceptions import Resolver404 + +from ..utils import set_custom_attribute +from .utils import ( + _get_catch_all_code_owner, + get_code_owner_from_module, + is_code_owner_mappings_configured, + set_code_owner_custom_attributes +) + +try: + import newrelic.agent +except ImportError: + newrelic = None # pylint: disable=invalid-name + +log = logging.getLogger(__name__) + + +class MonitoringTransaction(): + """ + Represents a monitoring transaction (likely the current transaction). + """ + def __init__(self, transaction): + self.transaction = transaction + + @property + def name(self): + """ + The name of the transaction. + + For NewRelic, the name may look like: + openedx.core.djangoapps.contentserver.middleware:StaticContentServer + + """ + if self.transaction and hasattr(self.transaction, 'name'): + return self.transaction.name + return None + + +def get_current_transaction(): + """ + Returns the current transaction. This is only used internally and won't + be ported over to the backends framework, because transactions will be + very different based on the backend. + """ + current_transaction = None + if newrelic: + current_transaction = newrelic.agent.current_transaction() + + return MonitoringTransaction(current_transaction) + + +class CodeOwnerMonitoringMiddleware: + """ + Django middleware object to set custom attributes for the owner of each view. + + For instructions on usage, see: + https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst + + Custom attributes set: + - code_owner: The owning team mapped to the current view. + - code_owner_module: The module found from the request or current transaction. + - code_owner_path_error: The error mapping by path, if code_owner isn't found in other ways. + - code_owner_transaction_error: The error mapping by transaction, if code_owner isn't found in other ways. + - code_owner_transaction_name: The current transaction name used to try to map to code_owner. + This can be used to find missing mappings. + + """ + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + self._set_code_owner_attribute(request) + return response + + def process_exception(self, request, exception): # pylint: disable=W0613 + self._set_code_owner_attribute(request) + + def _set_code_owner_attribute(self, request): + """ + Sets the code_owner custom attribute for the request. + """ + code_owner = None + module = self._get_module_from_request(request) + if module: + code_owner = get_code_owner_from_module(module) + if not code_owner: + code_owner = _get_catch_all_code_owner() + + if code_owner: + set_code_owner_custom_attributes(code_owner) + + def _get_module_from_request(self, request): + """ + Get the module from the request path or the current transaction. + + Side-effects: + Sets code_owner_module custom attribute, used to determine code_owner. + If module was not found, may set code_owner_path_error and/or + code_owner_transaction_error custom attributes if applicable. + + Returns: + str: module name or None if not found + + """ + if not is_code_owner_mappings_configured(): + return None + + module, path_error = self._get_module_from_request_path(request) + if module: + set_custom_attribute('code_owner_module', module) + return module + + module, transaction_error = self._get_module_from_current_transaction() + if module: + set_custom_attribute('code_owner_module', module) + return module + + # monitor errors if module was not found + if path_error: + set_custom_attribute('code_owner_path_error', path_error) + if transaction_error: + set_custom_attribute('code_owner_transaction_error', transaction_error) + return None + + def _get_module_from_request_path(self, request): + """ + Uses the request path to get the view_func module. + + Returns: + (str, str): (module, error_message), where at least one of these should be None + + """ + try: + view_func, _, _ = resolve(request.path) + module = view_func.__module__ + return module, None + # TODO: Replace ImportError with ModuleNotFoundError when Python 3.5 support is dropped. + except (ImportError, Resolver404) as e: + return None, str(e) + except Exception as e: # pragma: no cover + # will remove broad exceptions after ensuring all proper cases are covered + set_custom_attribute('deprecated_broad_except__get_module_from_request_path', e.__class__) + return None, str(e) + + def _get_module_from_current_transaction(self): + """ + Uses the current transaction to get the module. + + Side-effects: + Sets code_owner_transaction_name custom attribute, used to determine code_owner + + Returns: + (str, str): (module, error_message), where at least one of these should be None + + """ + try: + # Example: openedx.core.djangoapps.contentserver.middleware:StaticContentServer + transaction_name = get_current_transaction().name + if not transaction_name: + return None, 'No current transaction name found.' + module = transaction_name.split(':')[0] + set_custom_attribute('code_owner_transaction_name', transaction_name) + return module, None + except Exception as e: + # will remove broad exceptions after ensuring all proper cases are covered + set_custom_attribute('deprecated_broad_except___get_module_from_current_transaction', e.__class__) + return None, str(e) diff --git a/edx_arch_experiments/datadog_monitoring/code_owner/utils.py b/edx_arch_experiments/datadog_monitoring/code_owner/utils.py new file mode 100644 index 0000000..b8a108c --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/code_owner/utils.py @@ -0,0 +1,305 @@ +""" +Utilities for monitoring code_owner +""" +import logging +import re +from functools import wraps + +from django.conf import settings + +from ..utils import set_custom_attribute + +log = logging.getLogger(__name__) + + +def get_code_owner_from_module(module): + """ + Attempts lookup of code_owner based on a code module, + finding the most specific match. If no match, returns None. + + For example, if the module were 'openedx.features.discounts.views', + this lookup would match on 'openedx.features.discounts' before + 'openedx.features', because the former is more specific. + + See how to: + https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst + + """ + if not module: + return None + + code_owner_mappings = get_code_owner_mappings() + if not code_owner_mappings: + return None + + module_parts = module.split('.') + # To make the most specific match, start with the max number of parts + for number_of_parts in range(len(module_parts), 0, -1): + partial_path = '.'.join(module_parts[0:number_of_parts]) + if partial_path in code_owner_mappings: + code_owner = code_owner_mappings[partial_path] + return code_owner + return None + + +def is_code_owner_mappings_configured(): + """ + Returns True if code owner mappings were configured, and False otherwise. + """ + return isinstance(get_code_owner_mappings(), dict) + + +# cached lookup table for code owner given a module path. +# do not access this directly, but instead use get_code_owner_mappings. +_PATH_TO_CODE_OWNER_MAPPINGS = None + + +def get_code_owner_mappings(): + """ + Returns the contents of the CODE_OWNER_MAPPINGS Django Setting, processed + for efficient lookup by path. + + Returns: + (dict): dict mapping modules to code owners, or None if there are no + configured mappings, or an empty dict if there is an error processing + the setting. + + Example return value:: + + { + 'xblock_django': 'team-red', + 'openedx.core.djangoapps.xblock': 'team-red', + 'badges': 'team-blue', + } + + """ + global _PATH_TO_CODE_OWNER_MAPPINGS + + # Return cached processed mappings if already processed + if _PATH_TO_CODE_OWNER_MAPPINGS is not None: + return _PATH_TO_CODE_OWNER_MAPPINGS + + # Uses temporary variable to build mappings to avoid multi-threading issue with a partially + # processed map. Worst case, it is processed more than once at start-up. + path_to_code_owner_mapping = {} + + # .. setting_name: CODE_OWNER_MAPPINGS + # .. setting_default: None + # .. setting_description: Used for monitoring and reporting of ownership. Use a + # dict with keys of code owner name and value as a list of dotted path + # module names owned by the code owner. + code_owner_mappings = getattr(settings, 'CODE_OWNER_MAPPINGS', {}) + + try: + for code_owner in code_owner_mappings: + path_list = code_owner_mappings[code_owner] + for path in path_list: + path_to_code_owner_mapping[path] = code_owner + optional_module_prefix_match = _OPTIONAL_MODULE_PREFIX_PATTERN.match(path) + # if path has an optional prefix, also add the module name without the prefix + if optional_module_prefix_match: + path_without_prefix = path[optional_module_prefix_match.end():] + path_to_code_owner_mapping[path_without_prefix] = code_owner + except TypeError as e: + log.exception( + 'Error processing CODE_OWNER_MAPPINGS. {}'.format(e) # pylint: disable=logging-format-interpolation + ) + raise e + + _PATH_TO_CODE_OWNER_MAPPINGS = path_to_code_owner_mapping + return _PATH_TO_CODE_OWNER_MAPPINGS + + +def _get_catch_all_code_owner(): + """ + If the catch-all module "*" is configured, return the code_owner. + + Returns: + (str): code_owner or None if no catch-all configured. + + """ + try: + code_owner = get_code_owner_from_module('*') + return code_owner + except Exception as e: # pragma: no cover + # will remove broad exceptions after ensuring all proper cases are covered + set_custom_attribute('deprecated_broad_except___get_module_from_current_transaction', e.__class__) + return None + + +def set_code_owner_attribute_from_module(module): + """ + Updates the code_owner and code_owner_module custom attributes. + + Celery tasks or other non-web functions do not use middleware, so we need + an alternative way to set the code_owner custom attribute. + + Note: These settings will be overridden by the CodeOwnerMonitoringMiddleware. + This method can't be used to override web functions at this time. + + Usage:: + + set_code_owner_attribute_from_module(__name__) + + """ + set_custom_attribute('code_owner_module', module) + code_owner = get_code_owner_from_module(module) + if not code_owner: + code_owner = _get_catch_all_code_owner() + + if code_owner: + set_code_owner_custom_attributes(code_owner) + + +def set_code_owner_custom_attributes(code_owner): + """ + Sets custom metrics for code_owner, code_owner_theme, and code_owner_squad + """ + if not code_owner: # pragma: no cover + return + set_custom_attribute('code_owner', code_owner) + theme = _get_theme_from_code_owner(code_owner) + if theme: + set_custom_attribute('code_owner_theme', theme) + squad = _get_squad_from_code_owner(code_owner) + if squad: + set_custom_attribute('code_owner_squad', squad) + + +def set_code_owner_attribute(wrapped_function): + """ + Decorator to set the code_owner and code_owner_module custom attributes. + + Celery tasks or other non-web functions do not use middleware, so we need + an alternative way to set the code_owner custom attribute. + + Usage:: + + @task() + @set_code_owner_attribute + def example_task(): + ... + + Note: If the decorator can't be used for some reason, call + ``set_code_owner_attribute_from_module`` directly. + + An untested potential alternative is documented in the ADR covering this decision: + docs/decisions/0003-code-owner-for-celery-tasks.rst + + """ + @wraps(wrapped_function) + def new_function(*args, **kwargs): + set_code_owner_attribute_from_module(wrapped_function.__module__) + return wrapped_function(*args, **kwargs) + return new_function + + +def clear_cached_mappings(): + """ + Clears the cached code owner mappings. Useful for testing. + """ + global _PATH_TO_CODE_OWNER_MAPPINGS + _PATH_TO_CODE_OWNER_MAPPINGS = None + global _CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS + _CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS = None + + +# TODO: Retire this once edx-platform import_shims is no longer used. +# Note: This should be ready for removal because import_shims has been removed. +# See https://github.com/openedx/edx-platform/tree/854502b560bda74ef898501bb2a95ce238cf794c/import_shims +_OPTIONAL_MODULE_PREFIX_PATTERN = re.compile(r'^(lms|common|openedx\.core)\.djangoapps\.') + + +# Cached lookup table for code owner theme and squad given a code owner. +# - Although code owner is "theme-squad", a hyphen may also be in the theme or squad name, so this ensures we get both +# correctly from config. +# Do not access this directly, but instead use get_code_owner_theme_squad_mappings. +_CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS = None + + +def get_code_owner_theme_squad_mappings(): + """ + Returns the contents of the CODE_OWNER_THEMES Django Setting, processed + for efficient lookup by path. + + Returns: + (dict): dict mapping code owners to a dict containing the squad and theme, or + an empty dict if there are no configured mappings. + + Example return value:: + + { + 'theme-x-team-red': { + 'theme': 'theme-x', + 'squad': 'team-red', + }, + 'theme-x-team-blue': { + 'theme': 'theme-x', + 'squad': 'team-blue', + }, + } + + """ + global _CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS + + # Return cached processed mappings if already processed + if _CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS is not None: + return _CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS + + # Uses temporary variable to build mappings to avoid multi-threading issue with a partially + # processed map. Worst case, it is processed more than once at start-up. + code_owner_to_theme_and_squad_mapping = {} + + # .. setting_name: CODE_OWNER_THEMES + # .. setting_default: None + # .. setting_description: Used for monitoring and reporting of ownership. Use a + # dict with keys of code owner themes and values as a list of code owner names + # including theme and squad, separated with a hyphen. + code_owner_themes = getattr(settings, 'CODE_OWNER_THEMES', {}) + + try: + for theme in code_owner_themes: + code_owner_list = code_owner_themes[theme] + for code_owner in code_owner_list: + squad = code_owner.split(theme + '-', 1)[1] + code_owner_details = { + 'theme': theme, + 'squad': squad, + } + code_owner_to_theme_and_squad_mapping[code_owner] = code_owner_details + except TypeError as e: + log.exception( + 'Error processing CODE_OWNER_THEMES setting. {}'.format(e) # pylint: disable=logging-format-interpolation + ) + raise e + + _CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS = code_owner_to_theme_and_squad_mapping + return _CODE_OWNER_TO_THEME_AND_SQUAD_MAPPINGS + + +def _get_theme_from_code_owner(code_owner): + """ + Returns theme for a code_owner (e.g. 'theme-my-squad' => 'theme') + """ + mappings = get_code_owner_theme_squad_mappings() + if mappings is None: # pragma: no cover + return None + + if code_owner in mappings: + return mappings[code_owner]['theme'] + + return None + + +def _get_squad_from_code_owner(code_owner): + """ + Returns squad for a code_owner (e.g. 'theme-my-squad' => 'my-squad') + """ + mappings = get_code_owner_theme_squad_mappings() + if mappings is None: # pragma: no cover + return None + + if code_owner in mappings: + return mappings[code_owner]['squad'] + + return None diff --git a/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst b/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst new file mode 100644 index 0000000..b1a50b5 --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst @@ -0,0 +1,85 @@ +Add Code_Owner Custom Attributes to an IDA +========================================== + +.. contents:: + :local: + :depth: 2 + +What are the code owner custom attributes? +------------------------------------------ + +The code owner custom attributes can be used to create custom dashboards and alerts for monitoring the things that you own. It was originally introduced for the LMS, as is described in this `ADR on monitoring by code owner`_. + +The code owner custom attributes consist of: + +* code_owner: The owner name. When themes and squads are used, this will be the theme and squad names joined by a hyphen. +* code_owner_theme: The theme name of the owner. +* code_owner_squad: The squad name of the owner. Use this to avoid issues when theme name changes. + +You can now easily add this same attribute to any IDA so that your dashboards and alerts can work across multiple IDAs at once. + +If you want to know about custom attributes in general, see :doc:`using_custom_attributes`. + +.. _ADR on monitoring by code owner: https://github.com/openedx/edx-platform/blob/master/lms/djangoapps/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst + +Setting up the Middleware +------------------------- + +You simply need to add ``edx_django_utils.monitoring.CodeOwnerMonitoringMiddleware`` as described in the README to make this functionality available. Then it is ready to be configured. + +Handling celery tasks +--------------------- + +Celery tasks require use of a special decorator to set the ``code_owner`` custom attributes because no middleware will be run. + +Here is an example:: + + @task() + @set_code_owner_attribute + def example_task(): + ... + +If the task is not compatible with additional decorators, you can use the following alternative:: + + @task() + def example_task(): + set_code_owner_attribute_from_module(__name__) + ... + +An untested potential alternative to the decorator is documented in the `Code Owner for Celery Tasks ADR`_, should we run into maintenance issues using the decorator. + +.. _Code Owner for Celery Tasks ADR: https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst +Configuring your app settings +----------------------------- + +Once the Middleware is made available, simply set the Django Settings ``CODE_OWNER_MAPPINGS`` and ``CODE_OWNER_THEMES`` appropriately. + +The following example shows how you can include an optional config for a catch-all using ``'*'``. Although you might expect this example to use Python, it is intentionally illustrated in YAML because the catch-all requires special care in YAML. + +:: + + # YAML format of example CODE_OWNER_MAPPINGS + CODE_OWNER_MAPPINGS: + theme-x-team-red: + - xblock_django + - openedx.core.djangoapps.xblock + theme-x-team-blue: + - '*' # IMPORTANT: you must surround * with quotes in yml + + # YAML format of example CODE_OWNER_THEMES + CODE_OWNER_THEMES: + theme-x: + - theme-x-team-red + - theme-x-team-blue + +How to find and fix code_owner mappings +--------------------------------------- + +If you are missing the ``code_owner`` custom attributes on a particular Transaction or Error, or if ``code_owner`` is matching the catch-all, but you want to add a more specific mapping, you can use the other `code_owner supporting attributes`_ to determine what the appropriate mappings should be. + +.. _code_owner supporting attributes: https://github.com/openedx/edx-django-utils/blob/c022565/edx_django_utils/monitoring/internal/code_owner/middleware.py#L30-L34 + +Updating New Relic monitoring +----------------------------- + +To update monitoring in the event of a squad or theme name change, see :doc:`update_monitoring_for_squad_or_theme_changes`. diff --git a/edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst b/edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst new file mode 100644 index 0000000..c00011a --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst @@ -0,0 +1,44 @@ +Update Monitoring for Squad or Theme Changes +============================================ + +.. contents:: + :local: + :depth: 2 + +Understanding code owner custom attributes +------------------------------------------ + +If you first need some background on the ``code_owner_squad`` and ``code_owner_theme`` custom attributes, see :doc:`add_code_owner_custom_attribute_to_an_ida`. + +Expand and contract name changes +-------------------------------- + +NRQL (New Relic Query Language) statements that use the ``code_owner_squad`` or ``code_owner_theme`` (or ``code_owner``) custom attributes may be found in alert conditions or dashboards. + +To change a squad or theme name, you should *expand* the NRQL before the change, and *contract* the NRQL after the change. + +.. note:: + + For edx.org, it is useful to wait a month before implementing the contract phase of the monitoring update. + +Example expand phase NRQL:: + + code_owner_squad IN ('old-squad-name', 'new-squad-name') + code_owner_theme IN ('old-theme-name', 'new-theme-name') + +Example contract phase NRQL:: + + code_owner_squad = 'new-squad-name' + code_owner_theme = 'new-theme-name' + +To find the relevant NRQL to update, see `Searching New Relic NRQL`_. + +Searching New Relic NRQL +------------------------ + +See :doc:`search_new_relic` for general information about the ``new_relic_search.py`` script. + +This script can be especially useful for helping with the expand/contract phase when changing squad or theme names. For example, you could use the following:: + + new_relic_search.py --regex old-squad-name + new_relic_search.py --regex new-squad-name diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/__init__.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/mock_views.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/mock_views.py new file mode 100644 index 0000000..fddc5b4 --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/tests/code_owner/mock_views.py @@ -0,0 +1,12 @@ +""" +Mock views with a different module to enable testing of mapping +code_owner to modules. Trying to mock __module__ on a view was +getting too complex. +""" +from django.views.generic import View + + +class MockViewTest(View): + """ + Mock view for use in testing. + """ diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py new file mode 100644 index 0000000..aad9e2a --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py @@ -0,0 +1,321 @@ +""" +Tests for the code_owner monitoring middleware +""" +from unittest import TestCase +from unittest.mock import ANY, MagicMock, Mock, call, patch + +import ddt +from django.test import RequestFactory, override_settings +from django.urls import re_path +from django.views.generic import View + +from edx_django_utils.monitoring import CodeOwnerMonitoringMiddleware +from edx_django_utils.monitoring.internal.code_owner.utils import clear_cached_mappings + +from .mock_views import MockViewTest + + +class MockMiddlewareViewTest(View): + pass + + +urlpatterns = [ + re_path(r'^middleware-test/$', MockMiddlewareViewTest.as_view()), + re_path(r'^test/$', MockViewTest.as_view()), +] + +SET_CUSTOM_ATTRIBUTE_MOCK = MagicMock() + + +# Enables the same mock to be used from different modules, using +# patch with new_callable=get_set_custom_attribute_mock +def get_set_custom_attribute_mock(): + return SET_CUSTOM_ATTRIBUTE_MOCK + + +@ddt.ddt +class CodeOwnerMetricMiddlewareTests(TestCase): + """ + Tests for the code_owner monitoring utility functions + """ + urls = 'lms.djangoapps.monitoring.tests.test_middleware.test_urls' + + def setUp(self): + super().setUp() + clear_cached_mappings() + SET_CUSTOM_ATTRIBUTE_MOCK.reset_mock() + self.mock_get_response = Mock() + self.middleware = CodeOwnerMonitoringMiddleware(self.mock_get_response) + + def test_init(self): + self.assertEqual(self.middleware.get_response, self.mock_get_response) + + def test_request_call(self): + self.mock_get_response.return_value = 'test-response' + request = Mock() + self.assertEqual(self.middleware(request), 'test-response') + + _REQUEST_PATH_TO_MODULE_PATH = { + '/middleware-test/': 'edx_django_utils.monitoring.tests.code_owner.test_middleware', + '/test/': 'edx_django_utils.monitoring.tests.code_owner.mock_views', + } + + @override_settings( + CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views']}, + CODE_OWNER_THEMES={'team': ['team-red']}, + ROOT_URLCONF=__name__, + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @ddt.data( + ('/middleware-test/', None), + ('/test/', 'team-red'), + ) + @ddt.unpack + def test_code_owner_path_mapping_hits_and_misses( + self, request_path, expected_owner, mock_set_custom_attribute, _ + ): + request = RequestFactory().get(request_path) + self.middleware(request) + expected_path_module = self._REQUEST_PATH_TO_MODULE_PATH[request_path] + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=expected_path_module, + check_theme_and_squad=True + ) + + mock_set_custom_attribute.reset_mock() + self.middleware.process_exception(request, None) + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=expected_path_module, + check_theme_and_squad=True + ) + + @override_settings( + CODE_OWNER_MAPPINGS={ + 'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views'], + 'team-blue': ['*'], + }, + ROOT_URLCONF=__name__, + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @ddt.data( + ('/middleware-test/', 'team-blue'), + ('/test/', 'team-red'), + ) + @ddt.unpack + def test_code_owner_path_mapping_with_catch_all( + self, request_path, expected_owner, mock_set_custom_attribute, _ + ): + request = RequestFactory().get(request_path) + self.middleware(request) + expected_path_module = self._REQUEST_PATH_TO_MODULE_PATH[request_path] + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=expected_path_module + ) + + @override_settings( + CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views']}, + ROOT_URLCONF=__name__, + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch('newrelic.agent') + @ddt.data( + ( + 'edx_django_utils.monitoring.tests.code_owner.test_middleware', + 'edx_django_utils.monitoring.tests.code_owner.test_middleware:MockMiddlewareViewTest', + None + ), + ( + 'edx_django_utils.monitoring.tests.code_owner.mock_views', + 'edx_django_utils.monitoring.tests.code_owner.mock_views:MockViewTest', + 'team-red' + ), + ) + @ddt.unpack + def test_code_owner_transaction_mapping_hits_and_misses( # pylint: disable=too-many-positional-arguments + self, path_module, transaction_name, expected_owner, mock_newrelic_agent, mock_set_custom_attribute, _ + ): + mock_newrelic_agent.current_transaction().name = transaction_name + request = RequestFactory().get('/bad/path/') + self.middleware(request) + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=path_module, + transaction_name=transaction_name + ) + + mock_set_custom_attribute.reset_mock() + self.middleware.process_exception(request, None) + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=path_module, + transaction_name=transaction_name + ) + + @override_settings( + CODE_OWNER_MAPPINGS={ + 'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views'], + 'team-blue': ['*'], + }, + ROOT_URLCONF=__name__, + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch('newrelic.agent') + @ddt.data( + ( + 'edx_django_utils.monitoring.tests.code_owner.test_middleware', + 'edx_django_utils.monitoring.tests.code_owner.test_middleware:MockMiddlewareViewTest', + 'team-blue' + ), + ( + 'edx_django_utils.monitoring.tests.code_owner.mock_views', + 'edx_django_utils.monitoring.tests.code_owner.mock_views:MockViewTest', + 'team-red' + ), + ) + @ddt.unpack + def test_code_owner_transaction_mapping_with_catch_all( # pylint: disable=too-many-positional-arguments + self, path_module, transaction_name, expected_owner, mock_newrelic_agent, mock_set_custom_attribute, _ + ): + mock_newrelic_agent.current_transaction().name = transaction_name + request = RequestFactory().get('/bad/path/') + self.middleware(request) + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=path_module, + transaction_name=transaction_name + ) + + @override_settings( + CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views']}, + ROOT_URLCONF=__name__, + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch('newrelic.agent') + def test_code_owner_transaction_mapping_error(self, mock_newrelic_agent, mock_set_custom_attribute, _): + mock_newrelic_agent.current_transaction = Mock(side_effect=Exception('forced exception')) + request = RequestFactory().get('/bad/path/') + self.middleware(request) + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, has_path_error=True, has_transaction_error=True + ) + + @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + def test_code_owner_no_mappings(self, mock_set_custom_attribute): + request = RequestFactory().get('/test/') + self.middleware(request) + mock_set_custom_attribute.assert_not_called() + + @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + def test_code_owner_transaction_no_mappings(self, mock_set_custom_attribute): + request = RequestFactory().get('/bad/path/') + self.middleware(request) + mock_set_custom_attribute.assert_not_called() + + @override_settings( + CODE_OWNER_MAPPINGS={'team-red': ['lms.djangoapps.monitoring.tests.mock_views']}, + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + def test_no_resolver_for_path_and_no_transaction(self, mock_set_custom_attribute, _): + request = RequestFactory().get('/bad/path/') + self.middleware(request) + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, has_path_error=True, has_transaction_error=True + ) + + @override_settings( + CODE_OWNER_MAPPINGS={'team-red': ['*']}, + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + @patch( + 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + new_callable=get_set_custom_attribute_mock + ) + def test_catch_all_with_errors(self, mock_set_custom_attribute, _): + request = RequestFactory().get('/bad/path/') + self.middleware(request) + self._assert_code_owner_custom_attributes( + mock_set_custom_attribute, has_path_error=True, has_transaction_error=True, expected_code_owner='team-red' + ) + + @override_settings( + CODE_OWNER_MAPPINGS=['invalid_setting_as_list'], + ROOT_URLCONF=__name__, + ) + def test_load_config_with_invalid_dict(self): + request = RequestFactory().get('/test/') + with self.assertRaises(TypeError): + self.middleware(request) + + def _assert_code_owner_custom_attributes( # pylint: disable=too-many-positional-arguments + self, mock_set_custom_attribute, expected_code_owner=None, + path_module=None, has_path_error=False, + transaction_name=None, has_transaction_error=False, + check_theme_and_squad=False): + """ Performs a set of assertions around having set the proper custom attributes. """ + call_list = [] + if expected_code_owner: + call_list.append(call('code_owner', expected_code_owner)) + if check_theme_and_squad: + call_list.append(call('code_owner_theme', expected_code_owner.split('-')[0])) + call_list.append(call('code_owner_squad', expected_code_owner.split('-')[1])) + if path_module: + call_list.append(call('code_owner_module', path_module)) + if has_path_error: + call_list.append(call('code_owner_path_error', ANY)) + if transaction_name: + call_list.append(call('code_owner_transaction_name', transaction_name)) + if has_transaction_error: + call_list.append(call('code_owner_transaction_error', ANY)) + # TODO: Remove this list filtering once the ``deprecated_broad_except_XXX`` custom attributes have been removed. + actual_filtered_call_list = [ + mock_call + for mock_call in mock_set_custom_attribute.call_args_list + if not mock_call[0][0].startswith('deprecated_') + ] + mock_set_custom_attribute.assert_has_calls(call_list, any_order=True) + self.assertEqual( + len(actual_filtered_call_list), len(call_list), + f'Expected calls {call_list} vs actual calls {actual_filtered_call_list}' + ) diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py new file mode 100644 index 0000000..6507d84 --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py @@ -0,0 +1,146 @@ +""" +Tests for the code_owner monitoring middleware +""" +import timeit +from unittest import TestCase +from unittest.mock import call, patch + +import ddt +from django.test import override_settings + +from edx_django_utils.monitoring import ( + get_code_owner_from_module, + set_code_owner_attribute, + set_code_owner_attribute_from_module +) +from edx_django_utils.monitoring.internal.code_owner.utils import clear_cached_mappings + + +@set_code_owner_attribute +def decorated_function(pass_through): + """ + For testing the set_code_owner_attribute decorator. + """ + return pass_through + + +@ddt.ddt +class MonitoringUtilsTests(TestCase): + """ + Tests for the code_owner monitoring utility functions + """ + def setUp(self): + super().setUp() + clear_cached_mappings() + + @override_settings(CODE_OWNER_MAPPINGS={ + 'team-red': [ + 'openedx.core.djangoapps.xblock', + 'lms.djangoapps.grades', + ], + 'team-blue': [ + 'common.djangoapps.xblock_django', + ], + }) + @ddt.data( + ('xbl', None), + ('xblock_2', None), + ('xblock', 'team-red'), + ('openedx.core.djangoapps', None), + ('openedx.core.djangoapps.xblock', 'team-red'), + ('openedx.core.djangoapps.xblock.views', 'team-red'), + ('grades', 'team-red'), + ('lms.djangoapps.grades', 'team-red'), + ('xblock_django', 'team-blue'), + ('common.djangoapps.xblock_django', 'team-blue'), + ) + @ddt.unpack + def test_code_owner_mapping_hits_and_misses(self, module, expected_owner): + actual_owner = get_code_owner_from_module(module) + self.assertEqual(expected_owner, actual_owner) + + @override_settings(CODE_OWNER_MAPPINGS=['invalid_setting_as_list']) + @patch('edx_django_utils.monitoring.internal.code_owner.utils.log') + def test_code_owner_mapping_with_invalid_dict(self, mock_logger): + with self.assertRaises(TypeError): + get_code_owner_from_module('xblock') + + mock_logger.exception.assert_called_with( + 'Error processing CODE_OWNER_MAPPINGS. list indices must be integers or slices, not str', + ) + + def test_code_owner_mapping_with_no_settings(self): + self.assertIsNone(get_code_owner_from_module('xblock')) + + def test_code_owner_mapping_with_no_module(self): + self.assertIsNone(get_code_owner_from_module(None)) + + def test_mapping_performance(self): + code_owner_mappings = { + 'team-red': [] + } + # create a long list of mappings that are nearly identical + for n in range(1, 200): + path = f'openedx.core.djangoapps.{n}' + code_owner_mappings['team-red'].append(path) + with override_settings(CODE_OWNER_MAPPINGS=code_owner_mappings): + call_iterations = 100 + time = timeit.timeit( + # test a module name that matches nearly to the end, but doesn't actually match + lambda: get_code_owner_from_module('openedx.core.djangoapps.XXX.views'), number=call_iterations + ) + average_time = time / call_iterations + self.assertLess(average_time, 0.0005, f'Mapping takes {average_time}s which is too slow.') + + @override_settings( + CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils']}, + CODE_OWNER_THEMES={'team': ['team-red']}, + ) + @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + def test_set_code_owner_attribute_success(self, mock_set_custom_attribute): + self.assertEqual(decorated_function('test'), 'test') + self._assert_set_custom_attribute( + mock_set_custom_attribute, code_owner='team-red', module=__name__, check_theme_and_squad=True + ) + + @override_settings( + CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils']}, + CODE_OWNER_THEMES='invalid-setting', + ) + def test_set_code_owner_attribute_with_invalid_setting(self): + with self.assertRaises(TypeError): + decorated_function('test') + + @override_settings(CODE_OWNER_MAPPINGS={ + 'team-red': ['*'] + }) + @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + def test_set_code_owner_attribute_catch_all(self, mock_set_custom_attribute): + self.assertEqual(decorated_function('test'), 'test') + self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner='team-red', module=__name__) + + @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + def test_set_code_owner_attribute_no_mappings(self, mock_set_custom_attribute): + self.assertEqual(decorated_function('test'), 'test') + self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner=None, module=__name__) + + @override_settings(CODE_OWNER_MAPPINGS={ + 'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils'] + }) + @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + def test_set_code_owner_attribute_from_module_success(self, mock_set_custom_attribute): + set_code_owner_attribute_from_module(__name__) + self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner='team-red', module=__name__) + + def _assert_set_custom_attribute(self, mock_set_custom_attribute, code_owner, module, check_theme_and_squad=False): + """ + Helper to assert that the proper set_custom_metric calls were made. + """ + call_list = [] + if code_owner: + call_list.append(call('code_owner', code_owner)) + if check_theme_and_squad: + call_list.append(call('code_owner_theme', code_owner.split('-')[0])) + call_list.append(call('code_owner_squad', code_owner.split('-')[1])) + call_list.append(call('code_owner_module', module)) + mock_set_custom_attribute.assert_has_calls(call_list, any_order=True) From d9566de884499b4c2864409ffb9102c4ed4f62f6 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Wed, 23 Oct 2024 13:04:47 -0400 Subject: [PATCH 2/4] feat: refactor code_owner code from edx-dajango-utils Initial rollout of moving code_owner monitoring code from edx-django-utils to this plugin. - Adds near duplicate of code owner middleware from edx-django-utils. - Adds code owner for celery using Datadog span processing of celery.run spans. - Uses temporary span tags names using `_2`, like `code_owner_2`, for rollout and comparison with the original span tags. See https://github.com/edx/edx-arch-experiments/issues/784 --- CHANGELOG.rst | 10 + edx_arch_experiments/__init__.py | 2 +- .../datadog_monitoring/README.rst | 6 + .../datadog_monitoring/__init__.py | 0 .../datadog_monitoring/apps.py | 53 +++++ .../code_owner/middleware.py | 107 ++------- .../datadog_monitoring/code_owner/utils.py | 49 ++--- ..._code_owner_custom_attribute_to_an_ida.rst | 50 ++--- ..._monitoring_for_squad_or_theme_changes.rst | 35 ++- .../datadog_monitoring/tests/__init__.py | 0 .../tests/code_owner/test_middleware.py | 207 ++---------------- .../tests/code_owner/test_utils.py | 34 ++- .../datadog_monitoring/tests/test_app.py | 62 ++++++ requirements/base.txt | 24 +- requirements/ci.txt | 8 +- requirements/dev.txt | 100 +++++---- requirements/doc.txt | 66 ++++-- requirements/pip-tools.txt | 8 +- requirements/pip.txt | 4 +- requirements/quality.txt | 88 +++++--- requirements/scripts.txt | 28 +-- requirements/test.in | 1 + requirements/test.txt | 49 +++-- 23 files changed, 465 insertions(+), 526 deletions(-) create mode 100644 edx_arch_experiments/datadog_monitoring/README.rst create mode 100644 edx_arch_experiments/datadog_monitoring/__init__.py create mode 100644 edx_arch_experiments/datadog_monitoring/apps.py create mode 100644 edx_arch_experiments/datadog_monitoring/tests/__init__.py create mode 100644 edx_arch_experiments/datadog_monitoring/tests/test_app.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1cf8e8d..bf5d34a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,16 @@ Change Log Unreleased ~~~~~~~~~~ +[5.1.0] - 2024-10-23 +~~~~~~~~~~~~~~~~~~~~ +Added +----- +* Added Datadog monitoring app which adds code owner monitoring. This is the first step in moving code owner code from edx-django-utils to this plugin. + + * Adds near duplicate of code owner middleware from edx-django-utils. + * Adds code owner for celery using Datadog span processing of celery.run spans. + * Uses temporary span tags names using ``_2``, like ``code_owner_2``, for rollout and comparison with the original span tags. + [5.0.0] - 2024-10-22 ~~~~~~~~~~~~~~~~~~~~ Removed diff --git a/edx_arch_experiments/__init__.py b/edx_arch_experiments/__init__.py index 068150b..5a61fdb 100644 --- a/edx_arch_experiments/__init__.py +++ b/edx_arch_experiments/__init__.py @@ -2,4 +2,4 @@ A plugin to include applications under development by the architecture team at 2U. """ -__version__ = '5.0.0' +__version__ = '5.1.0' diff --git a/edx_arch_experiments/datadog_monitoring/README.rst b/edx_arch_experiments/datadog_monitoring/README.rst new file mode 100644 index 0000000..5498d75 --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/README.rst @@ -0,0 +1,6 @@ +Datadog Monitoring +################### + +When installed in the LMS as a plugin app, the ``datadog_monitoring`` app adds additional monitoring. + +This is where our code_owner_2 monitoring code lives, for example. diff --git a/edx_arch_experiments/datadog_monitoring/__init__.py b/edx_arch_experiments/datadog_monitoring/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edx_arch_experiments/datadog_monitoring/apps.py b/edx_arch_experiments/datadog_monitoring/apps.py new file mode 100644 index 0000000..6aafb76 --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/apps.py @@ -0,0 +1,53 @@ +""" +App for 2U-specific edx-platform Datadog monitoring. +""" + +import logging + +from django.apps import AppConfig + +from .code_owner.utils import get_code_owner_from_module + +log = logging.getLogger(__name__) + + +class DatadogMonitoringSpanProcessor: + """Datadog span processor that adds custom monitoring (e.g. code owner tags).""" + + def on_span_start(self, span): + if not span or not getattr(span, 'name') or not getattr(span, 'resource'): + return + + if span.name == 'celery.run': + # We can use this for celery spans, because the resource name is more predictable + # and available from the start. For django requests, we'll instead continue to use + # django middleware for setting code owner. + get_code_owner_from_module(span.resource) + + def on_span_finish(self, span): + pass + + def shutdown(self, _timeout): + pass + + +class DatadogMonitoring(AppConfig): + """ + Django application to handle 2U-specific Datadog monitoring. + """ + name = 'edx_arch_experiments.datadog_monitoring' + + # Mark this as a plugin app + plugin_app = {} + + def ready(self): + try: + from ddtrace import tracer # pylint: disable=import-outside-toplevel + # QUESTION: Do we want to publish a base constraint that avoids DD major changes without first testing them? + tracer._span_processors.append(DatadogMonitoringSpanProcessor()) # pylint: disable=protected-access + log.info("Attached DatadogMonitoringSpanProcessor") + except ImportError: + log.warning( + "Unable to attach DatadogMonitoringSpanProcessor" + " -- ddtrace module not found." + ) diff --git a/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py b/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py index 1af5583..f06b64d 100644 --- a/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py +++ b/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py @@ -1,75 +1,32 @@ """ -Middleware for code_owner custom attribute +Middleware for code_owner_2 custom attribute """ import logging from django.urls import resolve -from django.urls.exceptions import Resolver404 -from ..utils import set_custom_attribute +from edx_django_utils.monitoring import set_custom_attribute + from .utils import ( - _get_catch_all_code_owner, get_code_owner_from_module, is_code_owner_mappings_configured, set_code_owner_custom_attributes ) -try: - import newrelic.agent -except ImportError: - newrelic = None # pylint: disable=invalid-name - log = logging.getLogger(__name__) -class MonitoringTransaction(): - """ - Represents a monitoring transaction (likely the current transaction). - """ - def __init__(self, transaction): - self.transaction = transaction - - @property - def name(self): - """ - The name of the transaction. - - For NewRelic, the name may look like: - openedx.core.djangoapps.contentserver.middleware:StaticContentServer - - """ - if self.transaction and hasattr(self.transaction, 'name'): - return self.transaction.name - return None - - -def get_current_transaction(): - """ - Returns the current transaction. This is only used internally and won't - be ported over to the backends framework, because transactions will be - very different based on the backend. - """ - current_transaction = None - if newrelic: - current_transaction = newrelic.agent.current_transaction() - - return MonitoringTransaction(current_transaction) - - class CodeOwnerMonitoringMiddleware: """ Django middleware object to set custom attributes for the owner of each view. For instructions on usage, see: - https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst + https://github.com/edx/edx-arch-experiments/blob/master/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst Custom attributes set: - - code_owner: The owning team mapped to the current view. - - code_owner_module: The module found from the request or current transaction. - - code_owner_path_error: The error mapping by path, if code_owner isn't found in other ways. - - code_owner_transaction_error: The error mapping by transaction, if code_owner isn't found in other ways. - - code_owner_transaction_name: The current transaction name used to try to map to code_owner. - This can be used to find missing mappings. + - code_owner_2: The owning team mapped to the current view. + - code_owner_2_module: The module found from the request or current transaction. + - code_owner_2_path_error: The error mapping by path, if code_owner_2 isn't found in other ways. """ def __init__(self, get_response): @@ -85,14 +42,12 @@ def process_exception(self, request, exception): # pylint: disable=W0613 def _set_code_owner_attribute(self, request): """ - Sets the code_owner custom attribute for the request. + Sets the code_owner_2 custom attribute for the request. """ code_owner = None module = self._get_module_from_request(request) if module: code_owner = get_code_owner_from_module(module) - if not code_owner: - code_owner = _get_catch_all_code_owner() if code_owner: set_code_owner_custom_attributes(code_owner) @@ -102,9 +57,9 @@ def _get_module_from_request(self, request): Get the module from the request path or the current transaction. Side-effects: - Sets code_owner_module custom attribute, used to determine code_owner. - If module was not found, may set code_owner_path_error and/or - code_owner_transaction_error custom attributes if applicable. + Sets code_owner_2_module custom attribute, used to determine code_owner_2. + If module was not found, may set code_owner_2_path_error custom attribute + if applicable. Returns: str: module name or None if not found @@ -115,19 +70,12 @@ def _get_module_from_request(self, request): module, path_error = self._get_module_from_request_path(request) if module: - set_custom_attribute('code_owner_module', module) - return module - - module, transaction_error = self._get_module_from_current_transaction() - if module: - set_custom_attribute('code_owner_module', module) + set_custom_attribute('code_owner_2_module', module) return module # monitor errors if module was not found if path_error: - set_custom_attribute('code_owner_path_error', path_error) - if transaction_error: - set_custom_attribute('code_owner_transaction_error', transaction_error) + set_custom_attribute('code_owner_2_path_error', path_error) return None def _get_module_from_request_path(self, request): @@ -142,34 +90,5 @@ def _get_module_from_request_path(self, request): view_func, _, _ = resolve(request.path) module = view_func.__module__ return module, None - # TODO: Replace ImportError with ModuleNotFoundError when Python 3.5 support is dropped. - except (ImportError, Resolver404) as e: - return None, str(e) except Exception as e: # pragma: no cover - # will remove broad exceptions after ensuring all proper cases are covered - set_custom_attribute('deprecated_broad_except__get_module_from_request_path', e.__class__) - return None, str(e) - - def _get_module_from_current_transaction(self): - """ - Uses the current transaction to get the module. - - Side-effects: - Sets code_owner_transaction_name custom attribute, used to determine code_owner - - Returns: - (str, str): (module, error_message), where at least one of these should be None - - """ - try: - # Example: openedx.core.djangoapps.contentserver.middleware:StaticContentServer - transaction_name = get_current_transaction().name - if not transaction_name: - return None, 'No current transaction name found.' - module = transaction_name.split(':')[0] - set_custom_attribute('code_owner_transaction_name', transaction_name) - return module, None - except Exception as e: - # will remove broad exceptions after ensuring all proper cases are covered - set_custom_attribute('deprecated_broad_except___get_module_from_current_transaction', e.__class__) return None, str(e) diff --git a/edx_arch_experiments/datadog_monitoring/code_owner/utils.py b/edx_arch_experiments/datadog_monitoring/code_owner/utils.py index b8a108c..bd71206 100644 --- a/edx_arch_experiments/datadog_monitoring/code_owner/utils.py +++ b/edx_arch_experiments/datadog_monitoring/code_owner/utils.py @@ -1,5 +1,5 @@ """ -Utilities for monitoring code_owner +Utilities for monitoring code_owner_2 """ import logging import re @@ -7,7 +7,7 @@ from django.conf import settings -from ..utils import set_custom_attribute +from edx_django_utils.monitoring import set_custom_attribute log = logging.getLogger(__name__) @@ -88,7 +88,9 @@ def get_code_owner_mappings(): # .. setting_description: Used for monitoring and reporting of ownership. Use a # dict with keys of code owner name and value as a list of dotted path # module names owned by the code owner. - code_owner_mappings = getattr(settings, 'CODE_OWNER_MAPPINGS', {}) + code_owner_mappings = getattr(settings, 'CODE_OWNER_MAPPINGS', None) + if code_owner_mappings is None: + return None try: for code_owner in code_owner_mappings: @@ -110,42 +112,23 @@ def get_code_owner_mappings(): return _PATH_TO_CODE_OWNER_MAPPINGS -def _get_catch_all_code_owner(): - """ - If the catch-all module "*" is configured, return the code_owner. - - Returns: - (str): code_owner or None if no catch-all configured. - - """ - try: - code_owner = get_code_owner_from_module('*') - return code_owner - except Exception as e: # pragma: no cover - # will remove broad exceptions after ensuring all proper cases are covered - set_custom_attribute('deprecated_broad_except___get_module_from_current_transaction', e.__class__) - return None - - def set_code_owner_attribute_from_module(module): """ - Updates the code_owner and code_owner_module custom attributes. + Updates the code_owner_2 and code_owner_2_module custom attributes. Celery tasks or other non-web functions do not use middleware, so we need - an alternative way to set the code_owner custom attribute. + an alternative way to set the code_owner_2 custom attribute. Note: These settings will be overridden by the CodeOwnerMonitoringMiddleware. This method can't be used to override web functions at this time. Usage:: - set_code_owner_attribute_from_module(__name__) + set_code_owner_2_attribute_from_module(__name__) """ - set_custom_attribute('code_owner_module', module) + set_custom_attribute('code_owner_2_module', module) code_owner = get_code_owner_from_module(module) - if not code_owner: - code_owner = _get_catch_all_code_owner() if code_owner: set_code_owner_custom_attributes(code_owner) @@ -153,30 +136,30 @@ def set_code_owner_attribute_from_module(module): def set_code_owner_custom_attributes(code_owner): """ - Sets custom metrics for code_owner, code_owner_theme, and code_owner_squad + Sets custom metrics for code_owner_2, code_owner_2_theme, and code_owner_2_squad """ if not code_owner: # pragma: no cover return - set_custom_attribute('code_owner', code_owner) + set_custom_attribute('code_owner_2', code_owner) theme = _get_theme_from_code_owner(code_owner) if theme: - set_custom_attribute('code_owner_theme', theme) + set_custom_attribute('code_owner_2_theme', theme) squad = _get_squad_from_code_owner(code_owner) if squad: - set_custom_attribute('code_owner_squad', squad) + set_custom_attribute('code_owner_2_squad', squad) def set_code_owner_attribute(wrapped_function): """ - Decorator to set the code_owner and code_owner_module custom attributes. + Decorator to set the code_owner_2 and code_owner_2_module custom attributes. Celery tasks or other non-web functions do not use middleware, so we need - an alternative way to set the code_owner custom attribute. + an alternative way to set the code_owner_2 custom attribute. Usage:: @task() - @set_code_owner_attribute + @set_code_owner_2_attribute def example_task(): ... diff --git a/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst b/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst index b1a50b5..7392e3c 100644 --- a/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst +++ b/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst @@ -1,54 +1,38 @@ -Add Code_Owner Custom Attributes to an IDA -========================================== +Using Code_Owner Custom Span Tags +================================= .. contents:: :local: :depth: 2 -What are the code owner custom attributes? +What are the code owner custom span tags? ------------------------------------------ -The code owner custom attributes can be used to create custom dashboards and alerts for monitoring the things that you own. It was originally introduced for the LMS, as is described in this `ADR on monitoring by code owner`_. +The code owner custom span tags can be used to create custom dashboards and alerts for monitoring the things that you own. It was originally introduced for the LMS, as is described in this `ADR on monitoring by code owner`_. However, it was first moved to edx-django-utils to be used in any IDA. It was later moved to this 2U-specific plugin because it is for 2U. The code owner custom attributes consist of: -* code_owner: The owner name. When themes and squads are used, this will be the theme and squad names joined by a hyphen. -* code_owner_theme: The theme name of the owner. -* code_owner_squad: The squad name of the owner. Use this to avoid issues when theme name changes. +* code_owner_2: The owner name. When themes and squads are used, this will be the theme and squad names joined by a hyphen. +* code_owner_2_theme: The theme name of the owner. +* code_owner_2_squad: The squad name of the owner. Use this to avoid issues when theme name changes. -You can now easily add this same attribute to any IDA so that your dashboards and alerts can work across multiple IDAs at once. +Note: The ``_2`` of the code_owner_2 naming is for initial rollout to compare with edx-django-utils span tags. Ultimately, we will use adjusted names, which may include dropping the theme. -If you want to know about custom attributes in general, see :doc:`using_custom_attributes`. +If you want to learn more about custom span tags in general, see `Enhanced Monitoring and Custom Attributes`_. .. _ADR on monitoring by code owner: https://github.com/openedx/edx-platform/blob/master/lms/djangoapps/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst +.. _Enhanced Monitoring and Custom Attributes: https://edx.readthedocs.io/projects/edx-django-utils/en/latest/monitoring/how_tos/using_custom_attributes.html Setting up the Middleware ------------------------- -You simply need to add ``edx_django_utils.monitoring.CodeOwnerMonitoringMiddleware`` as described in the README to make this functionality available. Then it is ready to be configured. +You simply need to add ``edx_arch_experiments/datadog_monitoring/code_owner/middleware.CodeOwnerMonitoringMiddleware`` to get code owner span tags on Django requests. Handling celery tasks --------------------- -Celery tasks require use of a special decorator to set the ``code_owner`` custom attributes because no middleware will be run. +For celery tasks, this plugin will automatically detect and add code owner span tags to any span with ``operation_name:celery.run``. -Here is an example:: - - @task() - @set_code_owner_attribute - def example_task(): - ... - -If the task is not compatible with additional decorators, you can use the following alternative:: - - @task() - def example_task(): - set_code_owner_attribute_from_module(__name__) - ... - -An untested potential alternative to the decorator is documented in the `Code Owner for Celery Tasks ADR`_, should we run into maintenance issues using the decorator. - -.. _Code Owner for Celery Tasks ADR: https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst Configuring your app settings ----------------------------- @@ -75,11 +59,11 @@ The following example shows how you can include an optional config for a catch-a How to find and fix code_owner mappings --------------------------------------- -If you are missing the ``code_owner`` custom attributes on a particular Transaction or Error, or if ``code_owner`` is matching the catch-all, but you want to add a more specific mapping, you can use the other `code_owner supporting attributes`_ to determine what the appropriate mappings should be. +If you are missing the ``code_owner_2`` custom attributes on a particular Transaction or Error, or if ``code_owner`` is matching the catch-all, but you want to add a more specific mapping, you can use the other supporting tags like ``code_owner_2_module`` and ``code_owner_2_path_error`` to determine what the appropriate mappings should be. -.. _code_owner supporting attributes: https://github.com/openedx/edx-django-utils/blob/c022565/edx_django_utils/monitoring/internal/code_owner/middleware.py#L30-L34 +Updating Datadog monitoring +--------------------------- -Updating New Relic monitoring ------------------------------ +To update monitoring in the event of a squad or theme name change, see `Update Monitoring for Squad or Theme Changes`_. -To update monitoring in the event of a squad or theme name change, see :doc:`update_monitoring_for_squad_or_theme_changes`. +.. _Update Monitoring for Squad or Theme Changes: diff --git a/edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst b/edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst index c00011a..8e79a4d 100644 --- a/edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst +++ b/edx_arch_experiments/datadog_monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst @@ -8,37 +8,30 @@ Update Monitoring for Squad or Theme Changes Understanding code owner custom attributes ------------------------------------------ -If you first need some background on the ``code_owner_squad`` and ``code_owner_theme`` custom attributes, see :doc:`add_code_owner_custom_attribute_to_an_ida`. +If you first need some background on the ``code_owner_2_squad`` and ``code_owner_2_theme`` custom attributes, see `Using Code_Owner Custom Span Tags`_. + +.. _Using Code_Owner Custom Span Tags: https://github.com/openedx/edx-arch-experiments/blob/master/edx_arch_experiments/datadog_monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst Expand and contract name changes -------------------------------- -NRQL (New Relic Query Language) statements that use the ``code_owner_squad`` or ``code_owner_theme`` (or ``code_owner``) custom attributes may be found in alert conditions or dashboards. - -To change a squad or theme name, you should *expand* the NRQL before the change, and *contract* the NRQL after the change. - -.. note:: +Datadog monitors or dashboards may use the ``code_owner_2_squad`` or ``code_owner_2_theme`` (or ``code_owner_2``) custom span tags. - For edx.org, it is useful to wait a month before implementing the contract phase of the monitoring update. +To change a squad or theme name, you should *expand* before the change, and *contract* after the change. -Example expand phase NRQL:: +Example expand phase:: - code_owner_squad IN ('old-squad-name', 'new-squad-name') - code_owner_theme IN ('old-theme-name', 'new-theme-name') + code_owner_2_squad:('old-squad-name', 'new-squad-name') + code_owner_2_theme:('old-theme-name', 'new-theme-name') Example contract phase NRQL:: - code_owner_squad = 'new-squad-name' - code_owner_theme = 'new-theme-name' - -To find the relevant NRQL to update, see `Searching New Relic NRQL`_. - -Searching New Relic NRQL ------------------------- + code_owner_2_squad:'new-squad-name' + code_owner_2_theme:'new-theme-name' -See :doc:`search_new_relic` for general information about the ``new_relic_search.py`` script. +To find relevant usage of these span tags, see `Searching Datadog monitors and dashboards`_. -This script can be especially useful for helping with the expand/contract phase when changing squad or theme names. For example, you could use the following:: +Searching Datadog monitors and dashboards +----------------------------------------- - new_relic_search.py --regex old-squad-name - new_relic_search.py --regex new-squad-name +TODO: This section needs to be updated as part of https://github.com/edx/edx-arch-experiments/issues/786, once the script has been migrated for use with Datadog. diff --git a/edx_arch_experiments/datadog_monitoring/tests/__init__.py b/edx_arch_experiments/datadog_monitoring/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py index aad9e2a..3a5e635 100644 --- a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py +++ b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py @@ -9,8 +9,8 @@ from django.urls import re_path from django.views.generic import View -from edx_django_utils.monitoring import CodeOwnerMonitoringMiddleware -from edx_django_utils.monitoring.internal.code_owner.utils import clear_cached_mappings +from edx_arch_experiments.datadog_monitoring.code_owner.middleware import CodeOwnerMonitoringMiddleware +from edx_arch_experiments.datadog_monitoring.code_owner.utils import clear_cached_mappings from .mock_views import MockViewTest @@ -56,21 +56,21 @@ def test_request_call(self): self.assertEqual(self.middleware(request), 'test-response') _REQUEST_PATH_TO_MODULE_PATH = { - '/middleware-test/': 'edx_django_utils.monitoring.tests.code_owner.test_middleware', - '/test/': 'edx_django_utils.monitoring.tests.code_owner.mock_views', + '/middleware-test/': 'edx_arch_experiments.datadog_monitoring.tests.code_owner.test_middleware', + '/test/': 'edx_arch_experiments.datadog_monitoring.tests.code_owner.mock_views', } @override_settings( - CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views']}, + CODE_OWNER_MAPPINGS={'team-red': ['edx_arch_experiments.datadog_monitoring.tests.code_owner.mock_views']}, CODE_OWNER_THEMES={'team': ['team-red']}, ROOT_URLCONF=__name__, ) @patch( - 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', + 'edx_arch_experiments.datadog_monitoring.code_owner.middleware.set_custom_attribute', new_callable=get_set_custom_attribute_mock ) @patch( - 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + 'edx_arch_experiments.datadog_monitoring.code_owner.utils.set_custom_attribute', new_callable=get_set_custom_attribute_mock ) @ddt.data( @@ -97,186 +97,26 @@ def test_code_owner_path_mapping_hits_and_misses( ) @override_settings( - CODE_OWNER_MAPPINGS={ - 'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views'], - 'team-blue': ['*'], - }, ROOT_URLCONF=__name__, ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @ddt.data( - ('/middleware-test/', 'team-blue'), - ('/test/', 'team-red'), - ) - @ddt.unpack - def test_code_owner_path_mapping_with_catch_all( - self, request_path, expected_owner, mock_set_custom_attribute, _ - ): - request = RequestFactory().get(request_path) - self.middleware(request) - expected_path_module = self._REQUEST_PATH_TO_MODULE_PATH[request_path] - self._assert_code_owner_custom_attributes( - mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=expected_path_module - ) - - @override_settings( - CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views']}, - ROOT_URLCONF=__name__, - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch('newrelic.agent') - @ddt.data( - ( - 'edx_django_utils.monitoring.tests.code_owner.test_middleware', - 'edx_django_utils.monitoring.tests.code_owner.test_middleware:MockMiddlewareViewTest', - None - ), - ( - 'edx_django_utils.monitoring.tests.code_owner.mock_views', - 'edx_django_utils.monitoring.tests.code_owner.mock_views:MockViewTest', - 'team-red' - ), - ) - @ddt.unpack - def test_code_owner_transaction_mapping_hits_and_misses( # pylint: disable=too-many-positional-arguments - self, path_module, transaction_name, expected_owner, mock_newrelic_agent, mock_set_custom_attribute, _ - ): - mock_newrelic_agent.current_transaction().name = transaction_name - request = RequestFactory().get('/bad/path/') - self.middleware(request) - self._assert_code_owner_custom_attributes( - mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=path_module, - transaction_name=transaction_name - ) - - mock_set_custom_attribute.reset_mock() - self.middleware.process_exception(request, None) - self._assert_code_owner_custom_attributes( - mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=path_module, - transaction_name=transaction_name - ) - - @override_settings( - CODE_OWNER_MAPPINGS={ - 'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views'], - 'team-blue': ['*'], - }, - ROOT_URLCONF=__name__, - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch('newrelic.agent') - @ddt.data( - ( - 'edx_django_utils.monitoring.tests.code_owner.test_middleware', - 'edx_django_utils.monitoring.tests.code_owner.test_middleware:MockMiddlewareViewTest', - 'team-blue' - ), - ( - 'edx_django_utils.monitoring.tests.code_owner.mock_views', - 'edx_django_utils.monitoring.tests.code_owner.mock_views:MockViewTest', - 'team-red' - ), - ) - @ddt.unpack - def test_code_owner_transaction_mapping_with_catch_all( # pylint: disable=too-many-positional-arguments - self, path_module, transaction_name, expected_owner, mock_newrelic_agent, mock_set_custom_attribute, _ - ): - mock_newrelic_agent.current_transaction().name = transaction_name - request = RequestFactory().get('/bad/path/') - self.middleware(request) - self._assert_code_owner_custom_attributes( - mock_set_custom_attribute, expected_code_owner=expected_owner, path_module=path_module, - transaction_name=transaction_name - ) - - @override_settings( - CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.mock_views']}, - ROOT_URLCONF=__name__, - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch('newrelic.agent') - def test_code_owner_transaction_mapping_error(self, mock_newrelic_agent, mock_set_custom_attribute, _): - mock_newrelic_agent.current_transaction = Mock(side_effect=Exception('forced exception')) - request = RequestFactory().get('/bad/path/') - self.middleware(request) - self._assert_code_owner_custom_attributes( - mock_set_custom_attribute, has_path_error=True, has_transaction_error=True - ) - - @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + @patch('edx_arch_experiments.datadog_monitoring.code_owner.middleware.set_custom_attribute') def test_code_owner_no_mappings(self, mock_set_custom_attribute): request = RequestFactory().get('/test/') self.middleware(request) mock_set_custom_attribute.assert_not_called() - @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') - def test_code_owner_transaction_no_mappings(self, mock_set_custom_attribute): - request = RequestFactory().get('/bad/path/') - self.middleware(request) - mock_set_custom_attribute.assert_not_called() - @override_settings( CODE_OWNER_MAPPINGS={'team-red': ['lms.djangoapps.monitoring.tests.mock_views']}, ) @patch( - 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - def test_no_resolver_for_path_and_no_transaction(self, mock_set_custom_attribute, _): - request = RequestFactory().get('/bad/path/') - self.middleware(request) - self._assert_code_owner_custom_attributes( - mock_set_custom_attribute, has_path_error=True, has_transaction_error=True - ) - - @override_settings( - CODE_OWNER_MAPPINGS={'team-red': ['*']}, - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute', - new_callable=get_set_custom_attribute_mock - ) - @patch( - 'edx_django_utils.monitoring.internal.code_owner.middleware.set_custom_attribute', + 'edx_arch_experiments.datadog_monitoring.code_owner.middleware.set_custom_attribute', new_callable=get_set_custom_attribute_mock ) - def test_catch_all_with_errors(self, mock_set_custom_attribute, _): + def test_no_resolver_for_path(self, mock_set_custom_attribute): request = RequestFactory().get('/bad/path/') self.middleware(request) self._assert_code_owner_custom_attributes( - mock_set_custom_attribute, has_path_error=True, has_transaction_error=True, expected_code_owner='team-red' + mock_set_custom_attribute, has_path_error=True ) @override_settings( @@ -291,31 +131,20 @@ def test_load_config_with_invalid_dict(self): def _assert_code_owner_custom_attributes( # pylint: disable=too-many-positional-arguments self, mock_set_custom_attribute, expected_code_owner=None, path_module=None, has_path_error=False, - transaction_name=None, has_transaction_error=False, check_theme_and_squad=False): """ Performs a set of assertions around having set the proper custom attributes. """ call_list = [] if expected_code_owner: - call_list.append(call('code_owner', expected_code_owner)) + call_list.append(call('code_owner_2', expected_code_owner)) if check_theme_and_squad: - call_list.append(call('code_owner_theme', expected_code_owner.split('-')[0])) - call_list.append(call('code_owner_squad', expected_code_owner.split('-')[1])) + call_list.append(call('code_owner_2_theme', expected_code_owner.split('-')[0])) + call_list.append(call('code_owner_2_squad', expected_code_owner.split('-')[1])) if path_module: - call_list.append(call('code_owner_module', path_module)) + call_list.append(call('code_owner_2_module', path_module)) if has_path_error: - call_list.append(call('code_owner_path_error', ANY)) - if transaction_name: - call_list.append(call('code_owner_transaction_name', transaction_name)) - if has_transaction_error: - call_list.append(call('code_owner_transaction_error', ANY)) - # TODO: Remove this list filtering once the ``deprecated_broad_except_XXX`` custom attributes have been removed. - actual_filtered_call_list = [ - mock_call - for mock_call in mock_set_custom_attribute.call_args_list - if not mock_call[0][0].startswith('deprecated_') - ] + call_list.append(call('code_owner_2_path_error', ANY)) mock_set_custom_attribute.assert_has_calls(call_list, any_order=True) self.assertEqual( - len(actual_filtered_call_list), len(call_list), - f'Expected calls {call_list} vs actual calls {actual_filtered_call_list}' + len(mock_set_custom_attribute.call_args_list), len(call_list), + f'Expected calls {call_list} vs actual calls {mock_set_custom_attribute.call_args_list}' ) diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py index 6507d84..ad7ac93 100644 --- a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py +++ b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py @@ -8,12 +8,12 @@ import ddt from django.test import override_settings -from edx_django_utils.monitoring import ( +from edx_arch_experiments.datadog_monitoring.code_owner.utils import ( + clear_cached_mappings, get_code_owner_from_module, set_code_owner_attribute, set_code_owner_attribute_from_module ) -from edx_django_utils.monitoring.internal.code_owner.utils import clear_cached_mappings @set_code_owner_attribute @@ -60,7 +60,7 @@ def test_code_owner_mapping_hits_and_misses(self, module, expected_owner): self.assertEqual(expected_owner, actual_owner) @override_settings(CODE_OWNER_MAPPINGS=['invalid_setting_as_list']) - @patch('edx_django_utils.monitoring.internal.code_owner.utils.log') + @patch('edx_arch_experiments.datadog_monitoring.code_owner.utils.log') def test_code_owner_mapping_with_invalid_dict(self, mock_logger): with self.assertRaises(TypeError): get_code_owner_from_module('xblock') @@ -93,10 +93,10 @@ def test_mapping_performance(self): self.assertLess(average_time, 0.0005, f'Mapping takes {average_time}s which is too slow.') @override_settings( - CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils']}, + CODE_OWNER_MAPPINGS={'team-red': ['edx_arch_experiments.datadog_monitoring.tests.code_owner.test_utils']}, CODE_OWNER_THEMES={'team': ['team-red']}, ) - @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + @patch('edx_arch_experiments.datadog_monitoring.code_owner.utils.set_custom_attribute') def test_set_code_owner_attribute_success(self, mock_set_custom_attribute): self.assertEqual(decorated_function('test'), 'test') self._assert_set_custom_attribute( @@ -104,30 +104,22 @@ def test_set_code_owner_attribute_success(self, mock_set_custom_attribute): ) @override_settings( - CODE_OWNER_MAPPINGS={'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils']}, + CODE_OWNER_MAPPINGS={'team-red': ['edx_arch_experiments.datadog_monitoring.tests.code_owner.test_utils']}, CODE_OWNER_THEMES='invalid-setting', ) def test_set_code_owner_attribute_with_invalid_setting(self): with self.assertRaises(TypeError): decorated_function('test') - @override_settings(CODE_OWNER_MAPPINGS={ - 'team-red': ['*'] - }) - @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') - def test_set_code_owner_attribute_catch_all(self, mock_set_custom_attribute): - self.assertEqual(decorated_function('test'), 'test') - self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner='team-red', module=__name__) - - @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + @patch('edx_arch_experiments.datadog_monitoring.code_owner.utils.set_custom_attribute') def test_set_code_owner_attribute_no_mappings(self, mock_set_custom_attribute): self.assertEqual(decorated_function('test'), 'test') self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner=None, module=__name__) @override_settings(CODE_OWNER_MAPPINGS={ - 'team-red': ['edx_django_utils.monitoring.tests.code_owner.test_utils'] + 'team-red': ['edx_arch_experiments.datadog_monitoring.tests.code_owner.test_utils'] }) - @patch('edx_django_utils.monitoring.internal.code_owner.utils.set_custom_attribute') + @patch('edx_arch_experiments.datadog_monitoring.code_owner.utils.set_custom_attribute') def test_set_code_owner_attribute_from_module_success(self, mock_set_custom_attribute): set_code_owner_attribute_from_module(__name__) self._assert_set_custom_attribute(mock_set_custom_attribute, code_owner='team-red', module=__name__) @@ -138,9 +130,9 @@ def _assert_set_custom_attribute(self, mock_set_custom_attribute, code_owner, mo """ call_list = [] if code_owner: - call_list.append(call('code_owner', code_owner)) + call_list.append(call('code_owner_2', code_owner)) if check_theme_and_squad: - call_list.append(call('code_owner_theme', code_owner.split('-')[0])) - call_list.append(call('code_owner_squad', code_owner.split('-')[1])) - call_list.append(call('code_owner_module', module)) + call_list.append(call('code_owner_2_theme', code_owner.split('-')[0])) + call_list.append(call('code_owner_2_squad', code_owner.split('-')[1])) + call_list.append(call('code_owner_2_module', module)) mock_set_custom_attribute.assert_has_calls(call_list, any_order=True) diff --git a/edx_arch_experiments/datadog_monitoring/tests/test_app.py b/edx_arch_experiments/datadog_monitoring/tests/test_app.py new file mode 100644 index 0000000..3688037 --- /dev/null +++ b/edx_arch_experiments/datadog_monitoring/tests/test_app.py @@ -0,0 +1,62 @@ +""" +Tests for plugin app. +""" +from unittest.mock import call, patch + +from ddtrace import tracer +from django.test import TestCase, override_settings + +from .. import apps + + +class FakeSpan: + """A fake Span instance with span name and resource.""" + def __init__(self, name, resource): + self.name = name + self.resource = resource + + +class TestDatadogMonitoringApp(TestCase): + """Tests for TestDatadogMonitoringApp.""" + + def setUp(self): + # Remove custom span processor from previous runs. + # pylint: disable=protected-access + tracer._span_processors = [sp for sp in tracer._span_processors if type(sp).__name__ != 'DatadogMonitoringSpanProcessor'] + + def test_add_processor(self): + def initialize(): + apps.DatadogMonitoring('edx_arch_experiments.datadog_monitoring', apps).ready() + + def get_processor_list(): + # pylint: disable=protected-access + return [type(sp).__name__ for sp in tracer._span_processors] + + initialize() + assert sorted(get_processor_list()) == [ + 'DatadogMonitoringSpanProcessor', 'EndpointCallCounterProcessor', 'TopLevelSpanProcessor', + ] + + +class TestDatadogMonitoringSpanProcessor(TestCase): + """Tests for DatadogMonitoringSpanProcessor.""" + + @patch('edx_arch_experiments.datadog_monitoring.apps.get_code_owner_from_module') + def test_celery_span(self, mock_get_code_owner): + """ Tests processor with a celery span. """ + proc = apps.DatadogMonitoringSpanProcessor() + celery_span = FakeSpan('celery.run', 'test.module.for.celery.task') + + proc.on_span_start(celery_span) + + mock_get_code_owner.assert_called_once_with('test.module.for.celery.task') + + @patch('edx_arch_experiments.datadog_monitoring.apps.get_code_owner_from_module') + def test_other_span(self, mock_get_code_owner): + """ Tests processor with a non-celery span. """ + proc = apps.DatadogMonitoringSpanProcessor() + celery_span = FakeSpan('other.span', 'test.resource.name') + + proc.on_span_start(celery_span) + + mock_get_code_owner.assert_not_called() diff --git a/requirements/base.txt b/requirements/base.txt index 17b4ae6..d4841cb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -16,7 +16,7 @@ cffi==1.17.1 # via # cryptography # pynacl -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -24,7 +24,7 @@ click==8.1.7 # edx-django-utils code-annotations==1.8.0 # via edx-toggles -cryptography==43.0.1 +cryptography==43.0.3 # via pyjwt django==4.2.16 # via @@ -52,13 +52,13 @@ djangorestframework==3.15.2 # -r requirements/base.in # drf-jwt # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via pymongo drf-jwt==1.19.2 # via edx-drf-extensions -edx-codejail==3.4.1 +edx-codejail==3.5.1 # via -r requirements/base.in -edx-django-utils==5.15.0 +edx-django-utils==7.0.0 # via # -r requirements/base.in # edx-drf-extensions @@ -75,15 +75,15 @@ jinja2==3.1.4 # via code-annotations jsonschema==4.23.0 # via -r requirements/base.in -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 -newrelic==9.13.0 +newrelic==10.2.0 # via edx-django-utils pbr==6.1.0 # via stevedore -psutil==6.0.0 +psutil==6.1.0 # via edx-django-utils pycparser==2.22 # via cffi @@ -91,7 +91,7 @@ pyjwt[crypto]==2.9.0 # via # drf-jwt # edx-drf-extensions -pymongo==4.9.1 +pymongo==4.10.1 # via edx-opaque-keys pynacl==1.5.0 # via edx-django-utils @@ -128,5 +128,5 @@ urllib3==2.2.3 # via requests # The following packages are considered to be unsafe in a requirements file: -setuptools==75.1.0 +setuptools==75.2.0 # via -r requirements/base.in diff --git a/requirements/ci.txt b/requirements/ci.txt index 3625d4c..854926a 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -10,7 +10,7 @@ chardet==5.2.0 # via tox colorama==0.4.6 # via tox -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via @@ -28,7 +28,7 @@ pluggy==1.5.0 # via tox pyproject-api==1.8.0 # via tox -tox==4.20.0 +tox==4.23.2 # via -r requirements/ci.in -virtualenv==20.26.5 +virtualenv==20.27.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 65ad589..7678ef8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -8,7 +8,7 @@ asgiref==3.8.1 # via # -r requirements/quality.txt # django -astroid==3.2.4 +astroid==3.3.5 # via # -r requirements/quality.txt # pylint @@ -18,14 +18,14 @@ attrs==24.2.0 # -r requirements/quality.txt # jsonschema # referencing -backports-tarfile==1.2.0 - # via - # -r requirements/quality.txt - # jaraco-context -build==1.2.2 +build==1.2.2.post1 # via # -r requirements/pip-tools.txt # pip-tools +bytecode==0.15.1 + # via + # -r requirements/quality.txt + # ddtrace cachetools==5.5.0 # via # -r requirements/ci.txt @@ -44,7 +44,7 @@ chardet==5.2.0 # -r requirements/ci.txt # diff-cover # tox -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/quality.txt # requests @@ -70,24 +70,29 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.6.1 +coverage[toml]==7.6.4 # via # -r requirements/quality.txt # pytest-cov -cryptography==43.0.1 +cryptography==43.0.3 # via # -r requirements/quality.txt # pyjwt - # secretstorage ddt==1.7.2 # via -r requirements/quality.txt +ddtrace==2.14.4 + # via -r requirements/quality.txt +deprecated==1.2.14 + # via + # -r requirements/quality.txt + # opentelemetry-api diff-cover==9.2.0 # via -r requirements/dev.in -dill==0.3.8 +dill==0.3.9 # via # -r requirements/quality.txt # pylint -distlib==0.3.8 +distlib==0.3.9 # via # -r requirements/ci.txt # virtualenv @@ -119,7 +124,7 @@ djangorestframework==3.15.2 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via # -r requirements/quality.txt # pymongo @@ -131,9 +136,9 @@ drf-jwt==1.19.2 # via # -r requirements/quality.txt # edx-drf-extensions -edx-codejail==3.4.1 +edx-codejail==3.5.1 # via -r requirements/quality.txt -edx-django-utils==5.15.0 +edx-django-utils==7.0.0 # via # -r requirements/quality.txt # edx-drf-extensions @@ -150,6 +155,10 @@ edx-opaque-keys==2.11.0 # edx-drf-extensions edx-toggles==5.2.0 # via -r requirements/quality.txt +envier==0.6.1 + # via + # -r requirements/quality.txt + # ddtrace filelock==3.16.1 # via # -r requirements/ci.txt @@ -162,7 +171,7 @@ idna==3.10 importlib-metadata==8.4.0 # via # -r requirements/quality.txt - # keyring + # opentelemetry-api # twine iniconfig==2.0.0 # via @@ -180,15 +189,10 @@ jaraco-context==6.0.1 # via # -r requirements/quality.txt # keyring -jaraco-functools==4.0.2 - # via - # -r requirements/quality.txt - # keyring -jeepney==0.8.0 +jaraco-functools==4.1.0 # via # -r requirements/quality.txt # keyring - # secretstorage jinja2==3.1.4 # via # -r requirements/quality.txt @@ -196,11 +200,11 @@ jinja2==3.1.4 # diff-cover jsonschema==4.23.0 # via -r requirements/quality.txt -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via # -r requirements/quality.txt # jsonschema -keyring==25.4.0 +keyring==25.4.1 # via # -r requirements/quality.txt # twine @@ -208,13 +212,13 @@ lxml[html-clean,html_clean]==5.3.0 # via # edx-i18n-tools # lxml-html-clean -lxml-html-clean==0.2.2 +lxml-html-clean==0.3.1 # via lxml markdown-it-py==3.0.0 # via # -r requirements/quality.txt # rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -r requirements/quality.txt # jinja2 @@ -231,7 +235,7 @@ more-itertools==10.5.0 # -r requirements/quality.txt # jaraco-classes # jaraco-functools -newrelic==9.13.0 +newrelic==10.2.0 # via # -r requirements/quality.txt # edx-django-utils @@ -239,6 +243,10 @@ nh3==0.2.18 # via # -r requirements/quality.txt # readme-renderer +opentelemetry-api==1.27.0 + # via + # -r requirements/quality.txt + # ddtrace packaging==24.1 # via # -r requirements/ci.txt @@ -276,7 +284,11 @@ pluggy==1.5.0 # tox polib==1.2.0 # via edx-i18n-tools -psutil==6.0.0 +protobuf==5.28.3 + # via + # -r requirements/quality.txt + # ddtrace +psutil==6.1.0 # via # -r requirements/quality.txt # edx-django-utils @@ -299,7 +311,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -pylint==3.2.7 +pylint==3.3.1 # via # -r requirements/quality.txt # edx-lint @@ -310,7 +322,7 @@ pylint-celery==0.3 # via # -r requirements/quality.txt # edx-lint -pylint-django==2.5.5 +pylint-django==2.6.1 # via # -r requirements/quality.txt # edx-lint @@ -319,7 +331,7 @@ pylint-plugin-utils==0.8.2 # -r requirements/quality.txt # pylint-celery # pylint-django -pymongo==4.9.1 +pymongo==4.10.1 # via # -r requirements/quality.txt # edx-opaque-keys @@ -331,7 +343,7 @@ pyproject-api==1.8.0 # via # -r requirements/ci.txt # tox -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # -r requirements/pip-tools.txt # build @@ -380,7 +392,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.8.1 +rich==13.9.3 # via # -r requirements/quality.txt # twine @@ -389,10 +401,6 @@ rpds-py==0.20.0 # -r requirements/quality.txt # jsonschema # referencing -secretstorage==3.3.3 - # via - # -r requirements/quality.txt - # keyring semantic-version==2.10.0 # via # -r requirements/quality.txt @@ -424,20 +432,21 @@ tomlkit==0.13.2 # via # -r requirements/quality.txt # pylint -tox==4.20.0 +tox==4.23.2 # via -r requirements/ci.txt twine==5.1.1 # via -r requirements/quality.txt typing-extensions==4.12.2 # via # -r requirements/quality.txt + # ddtrace # edx-opaque-keys urllib3==2.2.3 # via # -r requirements/quality.txt # requests # twine -virtualenv==20.26.5 +virtualenv==20.27.0 # via # -r requirements/ci.txt # tox @@ -445,6 +454,15 @@ wheel==0.44.0 # via # -r requirements/pip-tools.txt # pip-tools +wrapt==1.16.0 + # via + # -r requirements/quality.txt + # ddtrace + # deprecated +xmltodict==0.14.2 + # via + # -r requirements/quality.txt + # ddtrace zipp==3.20.2 # via # -r requirements/quality.txt @@ -455,7 +473,7 @@ pip==24.2 # via # -r requirements/pip-tools.txt # pip-tools -setuptools==75.1.0 +setuptools==75.2.0 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index d0bb9e7..f4b8a34 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -17,6 +17,10 @@ attrs==24.2.0 # referencing babel==2.16.0 # via sphinx +bytecode==0.15.1 + # via + # -r requirements/test.txt + # ddtrace certifi==2024.8.30 # via # -r requirements/test.txt @@ -26,7 +30,7 @@ cffi==1.17.1 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/test.txt # requests @@ -39,16 +43,22 @@ code-annotations==1.8.0 # via # -r requirements/test.txt # edx-toggles -coverage[toml]==7.6.1 +coverage[toml]==7.6.4 # via # -r requirements/test.txt # pytest-cov -cryptography==43.0.1 +cryptography==43.0.3 # via # -r requirements/test.txt # pyjwt ddt==1.7.2 # via -r requirements/test.txt +ddtrace==2.14.4 + # via -r requirements/test.txt +deprecated==1.2.14 + # via + # -r requirements/test.txt + # opentelemetry-api django==4.2.16 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -76,7 +86,7 @@ djangorestframework==3.15.2 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via # -r requirements/test.txt # pymongo @@ -92,9 +102,9 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-codejail==3.4.1 +edx-codejail==3.5.1 # via -r requirements/test.txt -edx-django-utils==5.15.0 +edx-django-utils==7.0.0 # via # -r requirements/test.txt # edx-drf-extensions @@ -109,12 +119,20 @@ edx-sphinx-theme==3.1.0 # via -r requirements/doc.in edx-toggles==5.2.0 # via -r requirements/test.txt +envier==0.6.1 + # via + # -r requirements/test.txt + # ddtrace idna==3.10 # via # -r requirements/test.txt # requests imagesize==1.4.1 # via sphinx +importlib-metadata==8.4.0 + # via + # -r requirements/test.txt + # opentelemetry-api iniconfig==2.0.0 # via # -r requirements/test.txt @@ -126,20 +144,24 @@ jinja2==3.1.4 # sphinx jsonschema==4.23.0 # via -r requirements/test.txt -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via # -r requirements/test.txt # jsonschema -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -r requirements/test.txt # jinja2 -newrelic==9.13.0 +newrelic==10.2.0 # via # -r requirements/test.txt # edx-django-utils nh3==0.2.18 # via readme-renderer +opentelemetry-api==1.27.0 + # via + # -r requirements/test.txt + # ddtrace packaging==24.1 # via # -r requirements/test.txt @@ -153,7 +175,11 @@ pluggy==1.5.0 # via # -r requirements/test.txt # pytest -psutil==6.0.0 +protobuf==5.28.3 + # via + # -r requirements/test.txt + # ddtrace +psutil==6.1.0 # via # -r requirements/test.txt # edx-django-utils @@ -171,7 +197,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -pymongo==4.9.1 +pymongo==4.10.1 # via # -r requirements/test.txt # edx-opaque-keys @@ -263,14 +289,28 @@ text-unidecode==1.3 typing-extensions==4.12.2 # via # -r requirements/test.txt + # ddtrace # edx-opaque-keys urllib3==2.2.3 # via # -r requirements/test.txt # requests +wrapt==1.16.0 + # via + # -r requirements/test.txt + # ddtrace + # deprecated +xmltodict==0.14.2 + # via + # -r requirements/test.txt + # ddtrace +zipp==3.20.2 + # via + # -r requirements/test.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==75.1.0 +setuptools==75.2.0 # via # -r requirements/test.txt # sphinx diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 49831ac..a268261 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade # -build==1.2.2 +build==1.2.2.post1 # via pip-tools click==8.1.7 # via pip-tools @@ -12,7 +12,7 @@ packaging==24.1 # via build pip-tools==7.4.1 # via -r requirements/pip-tools.in -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -22,5 +22,5 @@ wheel==0.44.0 # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via pip-tools -setuptools==75.1.0 +setuptools==75.2.0 # via pip-tools diff --git a/requirements/pip.txt b/requirements/pip.txt index 36c777e..173c476 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -10,5 +10,5 @@ wheel==0.44.0 # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via -r requirements/pip.in -setuptools==75.1.0 +setuptools==75.2.0 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index bc99982..0c3fe30 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -8,7 +8,7 @@ asgiref==3.8.1 # via # -r requirements/test.txt # django -astroid==3.2.4 +astroid==3.3.5 # via # pylint # pylint-celery @@ -17,8 +17,10 @@ attrs==24.2.0 # -r requirements/test.txt # jsonschema # referencing -backports-tarfile==1.2.0 - # via jaraco-context +bytecode==0.15.1 + # via + # -r requirements/test.txt + # ddtrace certifi==2024.8.30 # via # -r requirements/test.txt @@ -28,7 +30,7 @@ cffi==1.17.1 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/test.txt # requests @@ -46,18 +48,23 @@ code-annotations==1.8.0 # -r requirements/test.txt # edx-lint # edx-toggles -coverage[toml]==7.6.1 +coverage[toml]==7.6.4 # via # -r requirements/test.txt # pytest-cov -cryptography==43.0.1 +cryptography==43.0.3 # via # -r requirements/test.txt # pyjwt - # secretstorage ddt==1.7.2 # via -r requirements/test.txt -dill==0.3.8 +ddtrace==2.14.4 + # via -r requirements/test.txt +deprecated==1.2.14 + # via + # -r requirements/test.txt + # opentelemetry-api +dill==0.3.9 # via pylint django==4.2.16 # via @@ -86,7 +93,7 @@ djangorestframework==3.15.2 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via # -r requirements/test.txt # pymongo @@ -96,9 +103,9 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-codejail==3.4.1 +edx-codejail==3.5.1 # via -r requirements/test.txt -edx-django-utils==5.15.0 +edx-django-utils==7.0.0 # via # -r requirements/test.txt # edx-drf-extensions @@ -113,13 +120,18 @@ edx-opaque-keys==2.11.0 # edx-drf-extensions edx-toggles==5.2.0 # via -r requirements/test.txt +envier==0.6.1 + # via + # -r requirements/test.txt + # ddtrace idna==3.10 # via # -r requirements/test.txt # requests importlib-metadata==8.4.0 # via - # keyring + # -r requirements/test.txt + # opentelemetry-api # twine iniconfig==2.0.0 # via @@ -133,27 +145,23 @@ jaraco-classes==3.4.0 # via keyring jaraco-context==6.0.1 # via keyring -jaraco-functools==4.0.2 +jaraco-functools==4.1.0 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.4 # via # -r requirements/test.txt # code-annotations jsonschema==4.23.0 # via -r requirements/test.txt -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via # -r requirements/test.txt # jsonschema -keyring==25.4.0 +keyring==25.4.1 # via twine markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -r requirements/test.txt # jinja2 @@ -165,12 +173,16 @@ more-itertools==10.5.0 # via # jaraco-classes # jaraco-functools -newrelic==9.13.0 +newrelic==10.2.0 # via # -r requirements/test.txt # edx-django-utils nh3==0.2.18 # via readme-renderer +opentelemetry-api==1.27.0 + # via + # -r requirements/test.txt + # ddtrace packaging==24.1 # via # -r requirements/test.txt @@ -187,7 +199,11 @@ pluggy==1.5.0 # via # -r requirements/test.txt # pytest -psutil==6.0.0 +protobuf==5.28.3 + # via + # -r requirements/test.txt + # ddtrace +psutil==6.1.0 # via # -r requirements/test.txt # edx-django-utils @@ -208,7 +224,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -pylint==3.2.7 +pylint==3.3.1 # via # edx-lint # pylint-celery @@ -216,13 +232,13 @@ pylint==3.2.7 # pylint-plugin-utils pylint-celery==0.3 # via edx-lint -pylint-django==2.5.5 +pylint-django==2.6.1 # via edx-lint pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pymongo==4.9.1 +pymongo==4.10.1 # via # -r requirements/test.txt # edx-opaque-keys @@ -267,15 +283,13 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.8.1 +rich==13.9.3 # via twine rpds-py==0.20.0 # via # -r requirements/test.txt # jsonschema # referencing -secretstorage==3.3.3 - # via keyring semantic-version==2.10.0 # via # -r requirements/test.txt @@ -308,15 +322,27 @@ twine==5.1.1 typing-extensions==4.12.2 # via # -r requirements/test.txt + # ddtrace # edx-opaque-keys urllib3==2.2.3 # via # -r requirements/test.txt # requests # twine +wrapt==1.16.0 + # via + # -r requirements/test.txt + # ddtrace + # deprecated +xmltodict==0.14.2 + # via + # -r requirements/test.txt + # ddtrace zipp==3.20.2 - # via importlib-metadata + # via + # -r requirements/test.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==75.1.0 +setuptools==75.2.0 # via -r requirements/test.txt diff --git a/requirements/scripts.txt b/requirements/scripts.txt index e698b69..d7de5c4 100644 --- a/requirements/scripts.txt +++ b/requirements/scripts.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -25,7 +25,7 @@ cffi==1.17.1 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/base.txt # requests @@ -38,9 +38,9 @@ code-annotations==1.8.0 # via # -r requirements/base.txt # edx-toggles -confluent-kafka[avro]==2.5.3 +confluent-kafka[avro]==2.6.0 # via -r requirements/scripts.in -cryptography==43.0.1 +cryptography==43.0.3 # via # -r requirements/base.txt # pyjwt @@ -73,7 +73,7 @@ djangorestframework==3.15.2 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via # -r requirements/base.txt # pymongo @@ -83,9 +83,9 @@ drf-jwt==1.19.2 # edx-drf-extensions edx-ccx-keys==1.3.0 # via openedx-events -edx-codejail==3.4.1 +edx-codejail==3.5.1 # via -r requirements/base.txt -edx-django-utils==5.15.0 +edx-django-utils==7.0.0 # via # -r requirements/base.txt # edx-drf-extensions @@ -120,25 +120,25 @@ jinja2==3.1.4 # code-annotations jsonschema==4.23.0 # via -r requirements/base.txt -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via # -r requirements/base.txt # jsonschema -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -r requirements/base.txt # jinja2 -newrelic==9.13.0 +newrelic==10.2.0 # via # -r requirements/base.txt # edx-django-utils -openedx-events==9.14.1 +openedx-events==9.15.0 # via edx-event-bus-kafka pbr==6.1.0 # via # -r requirements/base.txt # stevedore -psutil==6.0.0 +psutil==6.1.0 # via # -r requirements/base.txt # edx-django-utils @@ -151,7 +151,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -pymongo==4.9.1 +pymongo==4.10.1 # via # -r requirements/base.txt # edx-opaque-keys @@ -215,5 +215,5 @@ urllib3==2.2.3 # requests # The following packages are considered to be unsafe in a requirements file: -setuptools==75.1.0 +setuptools==75.2.0 # via -r requirements/base.txt diff --git a/requirements/test.in b/requirements/test.in index 37a4119..c5dffdf 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -8,3 +8,4 @@ pytest-django # pytest extension for better Django support pytest-randomly # pytest extension for discovering order-sensitive tests code-annotations # provides commands used by the pii_check make target. ddt # data-driven tests +ddtrace # Required for testing datadog_monitoring app and middleware diff --git a/requirements/test.txt b/requirements/test.txt index 94c920d..9c67d9b 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # make upgrade @@ -13,6 +13,8 @@ attrs==24.2.0 # -r requirements/base.txt # jsonschema # referencing +bytecode==0.15.1 + # via ddtrace certifi==2024.8.30 # via # -r requirements/base.txt @@ -22,7 +24,7 @@ cffi==1.17.1 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/base.txt # requests @@ -36,14 +38,18 @@ code-annotations==1.8.0 # -r requirements/base.txt # -r requirements/test.in # edx-toggles -coverage[toml]==7.6.1 +coverage[toml]==7.6.4 # via pytest-cov -cryptography==43.0.1 +cryptography==43.0.3 # via # -r requirements/base.txt # pyjwt ddt==1.7.2 # via -r requirements/test.in +ddtrace==2.14.4 + # via -r requirements/test.in +deprecated==1.2.14 + # via opentelemetry-api # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt @@ -70,7 +76,7 @@ djangorestframework==3.15.2 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via # -r requirements/base.txt # pymongo @@ -78,9 +84,9 @@ drf-jwt==1.19.2 # via # -r requirements/base.txt # edx-drf-extensions -edx-codejail==3.4.1 +edx-codejail==3.5.1 # via -r requirements/base.txt -edx-django-utils==5.15.0 +edx-django-utils==7.0.0 # via # -r requirements/base.txt # edx-drf-extensions @@ -93,10 +99,14 @@ edx-opaque-keys==2.11.0 # edx-drf-extensions edx-toggles==5.2.0 # via -r requirements/base.txt +envier==0.6.1 + # via ddtrace idna==3.10 # via # -r requirements/base.txt # requests +importlib-metadata==8.4.0 + # via opentelemetry-api iniconfig==2.0.0 # via pytest jinja2==3.1.4 @@ -105,18 +115,20 @@ jinja2==3.1.4 # code-annotations jsonschema==4.23.0 # via -r requirements/base.txt -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via # -r requirements/base.txt # jsonschema -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -r requirements/base.txt # jinja2 -newrelic==9.13.0 +newrelic==10.2.0 # via # -r requirements/base.txt # edx-django-utils +opentelemetry-api==1.27.0 + # via ddtrace packaging==24.1 # via pytest pbr==6.1.0 @@ -125,7 +137,9 @@ pbr==6.1.0 # stevedore pluggy==1.5.0 # via pytest -psutil==6.0.0 +protobuf==5.28.3 + # via ddtrace +psutil==6.1.0 # via # -r requirements/base.txt # edx-django-utils @@ -138,7 +152,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -pymongo==4.9.1 +pymongo==4.10.1 # via # -r requirements/base.txt # edx-opaque-keys @@ -204,12 +218,21 @@ text-unidecode==1.3 typing-extensions==4.12.2 # via # -r requirements/base.txt + # ddtrace # edx-opaque-keys urllib3==2.2.3 # via # -r requirements/base.txt # requests +wrapt==1.16.0 + # via + # ddtrace + # deprecated +xmltodict==0.14.2 + # via ddtrace +zipp==3.20.2 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==75.1.0 +setuptools==75.2.0 # via -r requirements/base.txt From 4908ac802732d1fa56e37e0055fd6132671cdd59 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 24 Oct 2024 14:09:42 +0000 Subject: [PATCH 3/4] fixup! Re-compile requirements for 3.11 on Linux (without `--upgrade`) --- requirements/base.txt | 24 +++++++------- requirements/ci.txt | 8 ++--- requirements/dev.txt | 67 +++++++++++++++++++++++--------------- requirements/doc.txt | 26 +++++++-------- requirements/pip-tools.txt | 8 ++--- requirements/pip.txt | 4 +-- requirements/quality.txt | 50 ++++++++++++++++------------ requirements/scripts.txt | 28 ++++++++-------- requirements/test.txt | 26 +++++++-------- 9 files changed, 133 insertions(+), 108 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index d4841cb..17b4ae6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -16,7 +16,7 @@ cffi==1.17.1 # via # cryptography # pynacl -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -24,7 +24,7 @@ click==8.1.7 # edx-django-utils code-annotations==1.8.0 # via edx-toggles -cryptography==43.0.3 +cryptography==43.0.1 # via pyjwt django==4.2.16 # via @@ -52,13 +52,13 @@ djangorestframework==3.15.2 # -r requirements/base.in # drf-jwt # edx-drf-extensions -dnspython==2.7.0 +dnspython==2.6.1 # via pymongo drf-jwt==1.19.2 # via edx-drf-extensions -edx-codejail==3.5.1 +edx-codejail==3.4.1 # via -r requirements/base.in -edx-django-utils==7.0.0 +edx-django-utils==5.15.0 # via # -r requirements/base.in # edx-drf-extensions @@ -75,15 +75,15 @@ jinja2==3.1.4 # via code-annotations jsonschema==4.23.0 # via -r requirements/base.in -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2023.12.1 # via jsonschema -markupsafe==3.0.2 +markupsafe==2.1.5 # via jinja2 -newrelic==10.2.0 +newrelic==9.13.0 # via edx-django-utils pbr==6.1.0 # via stevedore -psutil==6.1.0 +psutil==6.0.0 # via edx-django-utils pycparser==2.22 # via cffi @@ -91,7 +91,7 @@ pyjwt[crypto]==2.9.0 # via # drf-jwt # edx-drf-extensions -pymongo==4.10.1 +pymongo==4.9.1 # via edx-opaque-keys pynacl==1.5.0 # via edx-django-utils @@ -128,5 +128,5 @@ urllib3==2.2.3 # via requests # The following packages are considered to be unsafe in a requirements file: -setuptools==75.2.0 +setuptools==75.1.0 # via -r requirements/base.in diff --git a/requirements/ci.txt b/requirements/ci.txt index 854926a..3625d4c 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -10,7 +10,7 @@ chardet==5.2.0 # via tox colorama==0.4.6 # via tox -distlib==0.3.9 +distlib==0.3.8 # via virtualenv filelock==3.16.1 # via @@ -28,7 +28,7 @@ pluggy==1.5.0 # via tox pyproject-api==1.8.0 # via tox -tox==4.23.2 +tox==4.20.0 # via -r requirements/ci.in -virtualenv==20.27.0 +virtualenv==20.26.5 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 7678ef8..4d98aa8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -8,7 +8,7 @@ asgiref==3.8.1 # via # -r requirements/quality.txt # django -astroid==3.3.5 +astroid==3.2.4 # via # -r requirements/quality.txt # pylint @@ -18,7 +18,11 @@ attrs==24.2.0 # -r requirements/quality.txt # jsonschema # referencing -build==1.2.2.post1 +backports-tarfile==1.2.0 + # via + # -r requirements/quality.txt + # jaraco-context +build==1.2.2 # via # -r requirements/pip-tools.txt # pip-tools @@ -44,7 +48,7 @@ chardet==5.2.0 # -r requirements/ci.txt # diff-cover # tox -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via # -r requirements/quality.txt # requests @@ -70,14 +74,15 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.6.4 +coverage[toml]==7.6.1 # via # -r requirements/quality.txt # pytest-cov -cryptography==43.0.3 +cryptography==43.0.1 # via # -r requirements/quality.txt # pyjwt + # secretstorage ddt==1.7.2 # via -r requirements/quality.txt ddtrace==2.14.4 @@ -88,11 +93,11 @@ deprecated==1.2.14 # opentelemetry-api diff-cover==9.2.0 # via -r requirements/dev.in -dill==0.3.9 +dill==0.3.8 # via # -r requirements/quality.txt # pylint -distlib==0.3.9 +distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv @@ -124,7 +129,7 @@ djangorestframework==3.15.2 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -dnspython==2.7.0 +dnspython==2.6.1 # via # -r requirements/quality.txt # pymongo @@ -136,9 +141,9 @@ drf-jwt==1.19.2 # via # -r requirements/quality.txt # edx-drf-extensions -edx-codejail==3.5.1 +edx-codejail==3.4.1 # via -r requirements/quality.txt -edx-django-utils==7.0.0 +edx-django-utils==5.15.0 # via # -r requirements/quality.txt # edx-drf-extensions @@ -171,6 +176,7 @@ idna==3.10 importlib-metadata==8.4.0 # via # -r requirements/quality.txt + # keyring # opentelemetry-api # twine iniconfig==2.0.0 @@ -189,10 +195,15 @@ jaraco-context==6.0.1 # via # -r requirements/quality.txt # keyring -jaraco-functools==4.1.0 +jaraco-functools==4.0.2 # via # -r requirements/quality.txt # keyring +jeepney==0.8.0 + # via + # -r requirements/quality.txt + # keyring + # secretstorage jinja2==3.1.4 # via # -r requirements/quality.txt @@ -200,11 +211,11 @@ jinja2==3.1.4 # diff-cover jsonschema==4.23.0 # via -r requirements/quality.txt -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2023.12.1 # via # -r requirements/quality.txt # jsonschema -keyring==25.4.1 +keyring==25.4.0 # via # -r requirements/quality.txt # twine @@ -212,13 +223,13 @@ lxml[html-clean,html_clean]==5.3.0 # via # edx-i18n-tools # lxml-html-clean -lxml-html-clean==0.3.1 +lxml-html-clean==0.2.2 # via lxml markdown-it-py==3.0.0 # via # -r requirements/quality.txt # rich -markupsafe==3.0.2 +markupsafe==2.1.5 # via # -r requirements/quality.txt # jinja2 @@ -235,7 +246,7 @@ more-itertools==10.5.0 # -r requirements/quality.txt # jaraco-classes # jaraco-functools -newrelic==10.2.0 +newrelic==9.13.0 # via # -r requirements/quality.txt # edx-django-utils @@ -288,7 +299,7 @@ protobuf==5.28.3 # via # -r requirements/quality.txt # ddtrace -psutil==6.1.0 +psutil==6.0.0 # via # -r requirements/quality.txt # edx-django-utils @@ -311,7 +322,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -pylint==3.3.1 +pylint==3.2.7 # via # -r requirements/quality.txt # edx-lint @@ -322,7 +333,7 @@ pylint-celery==0.3 # via # -r requirements/quality.txt # edx-lint -pylint-django==2.6.1 +pylint-django==2.5.5 # via # -r requirements/quality.txt # edx-lint @@ -331,7 +342,7 @@ pylint-plugin-utils==0.8.2 # -r requirements/quality.txt # pylint-celery # pylint-django -pymongo==4.10.1 +pymongo==4.9.1 # via # -r requirements/quality.txt # edx-opaque-keys @@ -343,7 +354,7 @@ pyproject-api==1.8.0 # via # -r requirements/ci.txt # tox -pyproject-hooks==1.2.0 +pyproject-hooks==1.1.0 # via # -r requirements/pip-tools.txt # build @@ -392,7 +403,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.9.3 +rich==13.8.1 # via # -r requirements/quality.txt # twine @@ -401,6 +412,10 @@ rpds-py==0.20.0 # -r requirements/quality.txt # jsonschema # referencing +secretstorage==3.3.3 + # via + # -r requirements/quality.txt + # keyring semantic-version==2.10.0 # via # -r requirements/quality.txt @@ -432,7 +447,7 @@ tomlkit==0.13.2 # via # -r requirements/quality.txt # pylint -tox==4.23.2 +tox==4.20.0 # via -r requirements/ci.txt twine==5.1.1 # via -r requirements/quality.txt @@ -446,7 +461,7 @@ urllib3==2.2.3 # -r requirements/quality.txt # requests # twine -virtualenv==20.27.0 +virtualenv==20.26.5 # via # -r requirements/ci.txt # tox @@ -473,7 +488,7 @@ pip==24.2 # via # -r requirements/pip-tools.txt # pip-tools -setuptools==75.2.0 +setuptools==75.1.0 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index f4b8a34..23c3cc5 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -30,7 +30,7 @@ cffi==1.17.1 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via # -r requirements/test.txt # requests @@ -43,11 +43,11 @@ code-annotations==1.8.0 # via # -r requirements/test.txt # edx-toggles -coverage[toml]==7.6.4 +coverage[toml]==7.6.1 # via # -r requirements/test.txt # pytest-cov -cryptography==43.0.3 +cryptography==43.0.1 # via # -r requirements/test.txt # pyjwt @@ -86,7 +86,7 @@ djangorestframework==3.15.2 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -dnspython==2.7.0 +dnspython==2.6.1 # via # -r requirements/test.txt # pymongo @@ -102,9 +102,9 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-codejail==3.5.1 +edx-codejail==3.4.1 # via -r requirements/test.txt -edx-django-utils==7.0.0 +edx-django-utils==5.15.0 # via # -r requirements/test.txt # edx-drf-extensions @@ -144,15 +144,15 @@ jinja2==3.1.4 # sphinx jsonschema==4.23.0 # via -r requirements/test.txt -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2023.12.1 # via # -r requirements/test.txt # jsonschema -markupsafe==3.0.2 +markupsafe==2.1.5 # via # -r requirements/test.txt # jinja2 -newrelic==10.2.0 +newrelic==9.13.0 # via # -r requirements/test.txt # edx-django-utils @@ -179,7 +179,7 @@ protobuf==5.28.3 # via # -r requirements/test.txt # ddtrace -psutil==6.1.0 +psutil==6.0.0 # via # -r requirements/test.txt # edx-django-utils @@ -197,7 +197,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -pymongo==4.10.1 +pymongo==4.9.1 # via # -r requirements/test.txt # edx-opaque-keys @@ -310,7 +310,7 @@ zipp==3.20.2 # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==75.2.0 +setuptools==75.1.0 # via # -r requirements/test.txt # sphinx diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index a268261..49831ac 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -build==1.2.2.post1 +build==1.2.2 # via pip-tools click==8.1.7 # via pip-tools @@ -12,7 +12,7 @@ packaging==24.1 # via build pip-tools==7.4.1 # via -r requirements/pip-tools.in -pyproject-hooks==1.2.0 +pyproject-hooks==1.1.0 # via # build # pip-tools @@ -22,5 +22,5 @@ wheel==0.44.0 # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via pip-tools -setuptools==75.2.0 +setuptools==75.1.0 # via pip-tools diff --git a/requirements/pip.txt b/requirements/pip.txt index 173c476..36c777e 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -10,5 +10,5 @@ wheel==0.44.0 # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via -r requirements/pip.in -setuptools==75.2.0 +setuptools==75.1.0 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index 0c3fe30..240bc6e 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -8,7 +8,7 @@ asgiref==3.8.1 # via # -r requirements/test.txt # django -astroid==3.3.5 +astroid==3.2.4 # via # pylint # pylint-celery @@ -17,6 +17,8 @@ attrs==24.2.0 # -r requirements/test.txt # jsonschema # referencing +backports-tarfile==1.2.0 + # via jaraco-context bytecode==0.15.1 # via # -r requirements/test.txt @@ -30,7 +32,7 @@ cffi==1.17.1 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via # -r requirements/test.txt # requests @@ -48,14 +50,15 @@ code-annotations==1.8.0 # -r requirements/test.txt # edx-lint # edx-toggles -coverage[toml]==7.6.4 +coverage[toml]==7.6.1 # via # -r requirements/test.txt # pytest-cov -cryptography==43.0.3 +cryptography==43.0.1 # via # -r requirements/test.txt # pyjwt + # secretstorage ddt==1.7.2 # via -r requirements/test.txt ddtrace==2.14.4 @@ -64,7 +67,7 @@ deprecated==1.2.14 # via # -r requirements/test.txt # opentelemetry-api -dill==0.3.9 +dill==0.3.8 # via pylint django==4.2.16 # via @@ -93,7 +96,7 @@ djangorestframework==3.15.2 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -dnspython==2.7.0 +dnspython==2.6.1 # via # -r requirements/test.txt # pymongo @@ -103,9 +106,9 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-codejail==3.5.1 +edx-codejail==3.4.1 # via -r requirements/test.txt -edx-django-utils==7.0.0 +edx-django-utils==5.15.0 # via # -r requirements/test.txt # edx-drf-extensions @@ -131,6 +134,7 @@ idna==3.10 importlib-metadata==8.4.0 # via # -r requirements/test.txt + # keyring # opentelemetry-api # twine iniconfig==2.0.0 @@ -145,23 +149,27 @@ jaraco-classes==3.4.0 # via keyring jaraco-context==6.0.1 # via keyring -jaraco-functools==4.1.0 +jaraco-functools==4.0.2 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.4 # via # -r requirements/test.txt # code-annotations jsonschema==4.23.0 # via -r requirements/test.txt -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2023.12.1 # via # -r requirements/test.txt # jsonschema -keyring==25.4.1 +keyring==25.4.0 # via twine markdown-it-py==3.0.0 # via rich -markupsafe==3.0.2 +markupsafe==2.1.5 # via # -r requirements/test.txt # jinja2 @@ -173,7 +181,7 @@ more-itertools==10.5.0 # via # jaraco-classes # jaraco-functools -newrelic==10.2.0 +newrelic==9.13.0 # via # -r requirements/test.txt # edx-django-utils @@ -203,7 +211,7 @@ protobuf==5.28.3 # via # -r requirements/test.txt # ddtrace -psutil==6.1.0 +psutil==6.0.0 # via # -r requirements/test.txt # edx-django-utils @@ -224,7 +232,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -pylint==3.3.1 +pylint==3.2.7 # via # edx-lint # pylint-celery @@ -232,13 +240,13 @@ pylint==3.3.1 # pylint-plugin-utils pylint-celery==0.3 # via edx-lint -pylint-django==2.6.1 +pylint-django==2.5.5 # via edx-lint pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pymongo==4.10.1 +pymongo==4.9.1 # via # -r requirements/test.txt # edx-opaque-keys @@ -283,13 +291,15 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.9.3 +rich==13.8.1 # via twine rpds-py==0.20.0 # via # -r requirements/test.txt # jsonschema # referencing +secretstorage==3.3.3 + # via keyring semantic-version==2.10.0 # via # -r requirements/test.txt @@ -344,5 +354,5 @@ zipp==3.20.2 # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==75.2.0 +setuptools==75.1.0 # via -r requirements/test.txt diff --git a/requirements/scripts.txt b/requirements/scripts.txt index d7de5c4..e698b69 100644 --- a/requirements/scripts.txt +++ b/requirements/scripts.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -25,7 +25,7 @@ cffi==1.17.1 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via # -r requirements/base.txt # requests @@ -38,9 +38,9 @@ code-annotations==1.8.0 # via # -r requirements/base.txt # edx-toggles -confluent-kafka[avro]==2.6.0 +confluent-kafka[avro]==2.5.3 # via -r requirements/scripts.in -cryptography==43.0.3 +cryptography==43.0.1 # via # -r requirements/base.txt # pyjwt @@ -73,7 +73,7 @@ djangorestframework==3.15.2 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -dnspython==2.7.0 +dnspython==2.6.1 # via # -r requirements/base.txt # pymongo @@ -83,9 +83,9 @@ drf-jwt==1.19.2 # edx-drf-extensions edx-ccx-keys==1.3.0 # via openedx-events -edx-codejail==3.5.1 +edx-codejail==3.4.1 # via -r requirements/base.txt -edx-django-utils==7.0.0 +edx-django-utils==5.15.0 # via # -r requirements/base.txt # edx-drf-extensions @@ -120,25 +120,25 @@ jinja2==3.1.4 # code-annotations jsonschema==4.23.0 # via -r requirements/base.txt -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2023.12.1 # via # -r requirements/base.txt # jsonschema -markupsafe==3.0.2 +markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 -newrelic==10.2.0 +newrelic==9.13.0 # via # -r requirements/base.txt # edx-django-utils -openedx-events==9.15.0 +openedx-events==9.14.1 # via edx-event-bus-kafka pbr==6.1.0 # via # -r requirements/base.txt # stevedore -psutil==6.1.0 +psutil==6.0.0 # via # -r requirements/base.txt # edx-django-utils @@ -151,7 +151,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -pymongo==4.10.1 +pymongo==4.9.1 # via # -r requirements/base.txt # edx-opaque-keys @@ -215,5 +215,5 @@ urllib3==2.2.3 # requests # The following packages are considered to be unsafe in a requirements file: -setuptools==75.2.0 +setuptools==75.1.0 # via -r requirements/base.txt diff --git a/requirements/test.txt b/requirements/test.txt index 9c67d9b..27d6f5e 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -24,7 +24,7 @@ cffi==1.17.1 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via # -r requirements/base.txt # requests @@ -38,9 +38,9 @@ code-annotations==1.8.0 # -r requirements/base.txt # -r requirements/test.in # edx-toggles -coverage[toml]==7.6.4 +coverage[toml]==7.6.1 # via pytest-cov -cryptography==43.0.3 +cryptography==43.0.1 # via # -r requirements/base.txt # pyjwt @@ -76,7 +76,7 @@ djangorestframework==3.15.2 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -dnspython==2.7.0 +dnspython==2.6.1 # via # -r requirements/base.txt # pymongo @@ -84,9 +84,9 @@ drf-jwt==1.19.2 # via # -r requirements/base.txt # edx-drf-extensions -edx-codejail==3.5.1 +edx-codejail==3.4.1 # via -r requirements/base.txt -edx-django-utils==7.0.0 +edx-django-utils==5.15.0 # via # -r requirements/base.txt # edx-drf-extensions @@ -115,15 +115,15 @@ jinja2==3.1.4 # code-annotations jsonschema==4.23.0 # via -r requirements/base.txt -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2023.12.1 # via # -r requirements/base.txt # jsonschema -markupsafe==3.0.2 +markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 -newrelic==10.2.0 +newrelic==9.13.0 # via # -r requirements/base.txt # edx-django-utils @@ -139,7 +139,7 @@ pluggy==1.5.0 # via pytest protobuf==5.28.3 # via ddtrace -psutil==6.1.0 +psutil==6.0.0 # via # -r requirements/base.txt # edx-django-utils @@ -152,7 +152,7 @@ pyjwt[crypto]==2.9.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -pymongo==4.10.1 +pymongo==4.9.1 # via # -r requirements/base.txt # edx-opaque-keys @@ -234,5 +234,5 @@ zipp==3.20.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==75.2.0 +setuptools==75.1.0 # via -r requirements/base.txt From 1d3af1ca69baa2d4cb37271fa3b065c017c8dd8f Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Thu, 24 Oct 2024 10:11:31 -0400 Subject: [PATCH 4/4] fixup! fix some quality failures --- .../datadog_monitoring/apps.py | 9 +++++++-- .../code_owner/middleware.py | 9 ++------- .../datadog_monitoring/code_owner/utils.py | 1 - .../tests/code_owner/test_middleware.py | 5 +++-- .../tests/code_owner/test_utils.py | 2 +- .../datadog_monitoring/tests/test_app.py | 18 +++++++++++++++--- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/edx_arch_experiments/datadog_monitoring/apps.py b/edx_arch_experiments/datadog_monitoring/apps.py index 6aafb76..725e645 100644 --- a/edx_arch_experiments/datadog_monitoring/apps.py +++ b/edx_arch_experiments/datadog_monitoring/apps.py @@ -15,7 +15,12 @@ class DatadogMonitoringSpanProcessor: """Datadog span processor that adds custom monitoring (e.g. code owner tags).""" def on_span_start(self, span): - if not span or not getattr(span, 'name') or not getattr(span, 'resource'): + """ + Adds custom monitoring at span creation time. + + Specifically, adds code owner span tag for celery run spans. + """ + if not span or not hasattr(span, 'name') or not hasattr(span, 'resource'): return if span.name == 'celery.run': @@ -43,7 +48,7 @@ class DatadogMonitoring(AppConfig): def ready(self): try: from ddtrace import tracer # pylint: disable=import-outside-toplevel - # QUESTION: Do we want to publish a base constraint that avoids DD major changes without first testing them? + tracer._span_processors.append(DatadogMonitoringSpanProcessor()) # pylint: disable=protected-access log.info("Attached DatadogMonitoringSpanProcessor") except ImportError: diff --git a/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py b/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py index f06b64d..f679959 100644 --- a/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py +++ b/edx_arch_experiments/datadog_monitoring/code_owner/middleware.py @@ -4,14 +4,9 @@ import logging from django.urls import resolve - from edx_django_utils.monitoring import set_custom_attribute -from .utils import ( - get_code_owner_from_module, - is_code_owner_mappings_configured, - set_code_owner_custom_attributes -) +from .utils import get_code_owner_from_module, is_code_owner_mappings_configured, set_code_owner_custom_attributes log = logging.getLogger(__name__) @@ -90,5 +85,5 @@ def _get_module_from_request_path(self, request): view_func, _, _ = resolve(request.path) module = view_func.__module__ return module, None - except Exception as e: # pragma: no cover + except Exception as e: # pragma: no cover, pylint: disable=broad-exception-caught return None, str(e) diff --git a/edx_arch_experiments/datadog_monitoring/code_owner/utils.py b/edx_arch_experiments/datadog_monitoring/code_owner/utils.py index bd71206..abb0086 100644 --- a/edx_arch_experiments/datadog_monitoring/code_owner/utils.py +++ b/edx_arch_experiments/datadog_monitoring/code_owner/utils.py @@ -6,7 +6,6 @@ from functools import wraps from django.conf import settings - from edx_django_utils.monitoring import set_custom_attribute log = logging.getLogger(__name__) diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py index 3a5e635..54c5830 100644 --- a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py +++ b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_middleware.py @@ -128,10 +128,11 @@ def test_load_config_with_invalid_dict(self): with self.assertRaises(TypeError): self.middleware(request) - def _assert_code_owner_custom_attributes( # pylint: disable=too-many-positional-arguments + def _assert_code_owner_custom_attributes( self, mock_set_custom_attribute, expected_code_owner=None, path_module=None, has_path_error=False, - check_theme_and_squad=False): + check_theme_and_squad=False + ): """ Performs a set of assertions around having set the proper custom attributes. """ call_list = [] if expected_code_owner: diff --git a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py index ad7ac93..d4ca5a0 100644 --- a/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py +++ b/edx_arch_experiments/datadog_monitoring/tests/code_owner/test_utils.py @@ -12,7 +12,7 @@ clear_cached_mappings, get_code_owner_from_module, set_code_owner_attribute, - set_code_owner_attribute_from_module + set_code_owner_attribute_from_module, ) diff --git a/edx_arch_experiments/datadog_monitoring/tests/test_app.py b/edx_arch_experiments/datadog_monitoring/tests/test_app.py index 3688037..ca539c8 100644 --- a/edx_arch_experiments/datadog_monitoring/tests/test_app.py +++ b/edx_arch_experiments/datadog_monitoring/tests/test_app.py @@ -1,10 +1,10 @@ """ Tests for plugin app. """ -from unittest.mock import call, patch +from unittest.mock import patch from ddtrace import tracer -from django.test import TestCase, override_settings +from django.test import TestCase from .. import apps @@ -22,7 +22,9 @@ class TestDatadogMonitoringApp(TestCase): def setUp(self): # Remove custom span processor from previous runs. # pylint: disable=protected-access - tracer._span_processors = [sp for sp in tracer._span_processors if type(sp).__name__ != 'DatadogMonitoringSpanProcessor'] + tracer._span_processors = [ + sp for sp in tracer._span_processors if type(sp).__name__ != 'DatadogMonitoringSpanProcessor' + ] def test_add_processor(self): def initialize(): @@ -60,3 +62,13 @@ def test_other_span(self, mock_get_code_owner): proc.on_span_start(celery_span) mock_get_code_owner.assert_not_called() + + @patch('edx_arch_experiments.datadog_monitoring.apps.get_code_owner_from_module') + def test_non_span(self, mock_get_code_owner): + """ Tests processor with an object that doesn't have span name or resource. """ + proc = apps.DatadogMonitoringSpanProcessor() + non_span = object() + + proc.on_span_start(non_span) + + mock_get_code_owner.assert_not_called()