From b6ea3751b6f73453d92657231313764a3ba7aba6 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Fri, 12 Apr 2024 16:32:29 +0530 Subject: [PATCH 1/5] Add test factory for Deployment --- apiserver/tests/factories/models/code.py | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/apiserver/tests/factories/models/code.py b/apiserver/tests/factories/models/code.py index ae383a4f..74da5079 100644 --- a/apiserver/tests/factories/models/code.py +++ b/apiserver/tests/factories/models/code.py @@ -1,5 +1,11 @@ from random import randint from uuid import uuid4 +from dora.service.deployments.models.models import ( + Deployment, + DeploymentStatus, + DeploymentType, +) +from dora.utils.string import uuid4_str from dora.store.models.code import ( PullRequestCommit, @@ -141,3 +147,30 @@ def get_repo_workflow_run( meta=meta, html_url=html_url, ) + + +def get_deployment( + repo_id=None, + entity_id=None, + actor=None, + head_branch=None, + status=None, + conducted_at=None, + meta=None, + duration=None, + html_url=None, + provider=None, +): + return Deployment( + deployment_type=DeploymentType.WORKFLOW, + repo_id=repo_id or "1234567", + entity_id=entity_id or uuid4_str(), + provider=provider or "github", + actor=actor or "samad-yar-khan", + head_branch=head_branch or "master", + conducted_at=conducted_at or time_now(), + duration=duration, + status=status or DeploymentStatus.SUCCESS, + html_url=html_url or "", + meta=meta or {}, + ) From b63c2fb77e279414de26f2b4d612c600e6b76edf Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Fri, 12 Apr 2024 16:34:04 +0530 Subject: [PATCH 2/5] Update __init__ for test factory --- apiserver/tests/factories/models/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/apiserver/tests/factories/models/__init__.py b/apiserver/tests/factories/models/__init__.py index 69062ee3..9d80b209 100644 --- a/apiserver/tests/factories/models/__init__.py +++ b/apiserver/tests/factories/models/__init__.py @@ -1,2 +1,12 @@ -from .code import get_repo_workflow_run -from .incidents import get_incident +from .code import ( + get_repo_workflow_run, + get_deployment, + get_pull_request, + get_pull_request_commit, + get_pull_request_event, +) +from .incidents import ( + get_incident, + get_change_failure_rate_metrics, + get_org_incident_service, +) From e13d44cf19beeafb6bc975bb7a118670678669d5 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Fri, 12 Apr 2024 16:35:03 +0530 Subject: [PATCH 3/5] Add ChangeFailureRateMetrics test factory --- apiserver/tests/factories/models/incidents.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/apiserver/tests/factories/models/incidents.py b/apiserver/tests/factories/models/incidents.py index 340bab00..bf0c3af4 100644 --- a/apiserver/tests/factories/models/incidents.py +++ b/apiserver/tests/factories/models/incidents.py @@ -1,7 +1,9 @@ from datetime import datetime -from typing import List +from typing import List, Set from voluptuous import default_factory +from dora.service.deployments.models.models import Deployment +from dora.service.incidents.models.mean_time_to_recovery import ChangeFailureRateMetrics from dora.store.models.incidents import IncidentType, OrgIncidentService from dora.store.models.incidents.incidents import ( @@ -82,3 +84,10 @@ def get_incident_org_incident_map( incident_id: str = uuid4_str(), service_id: str = uuid4_str() ): return IncidentOrgIncidentServiceMap(incident_id=incident_id, service_id=service_id) + + +def get_change_failure_rate_metrics( + failed_deployments: Set[Deployment] = None, + total_deployments: Set[Deployment] = None, +): + return ChangeFailureRateMetrics(failed_deployments, total_deployments) From 74c1fdaba0c168c02d43a417f1a3983a6de956c4 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Fri, 12 Apr 2024 16:35:25 +0530 Subject: [PATCH 4/5] Add tests for get_change_failure_rate_metrics --- .../Incidents/test_change_failure_rate.py | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 apiserver/tests/service/Incidents/test_change_failure_rate.py diff --git a/apiserver/tests/service/Incidents/test_change_failure_rate.py b/apiserver/tests/service/Incidents/test_change_failure_rate.py new file mode 100644 index 00000000..e0cde046 --- /dev/null +++ b/apiserver/tests/service/Incidents/test_change_failure_rate.py @@ -0,0 +1,200 @@ +from datetime import timedelta +from tests.factories.models.incidents import get_change_failure_rate_metrics +from dora.service.incidents.incidents import get_incident_service +from dora.utils.time import time_now + +from tests.factories.models import get_incident, get_deployment + + +# No incidents, no deployments +def test_get_change_failure_rate_for_no_incidents_no_deployments(): + incident_service = get_incident_service() + incidents = [] + deployments = [] + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics([], []) + assert change_failure_rate.change_failure_rate == 0 + + +# No incidents, some deployments +def test_get_change_failure_rate_for_no_incidents_and_some_deployments(): + incident_service = get_incident_service() + incidents = [] + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set(), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 0 + + +# Some incidents, no deployments +def test_get_deployment_incidents_count_map_returns_empty_dict_when_given_some_incidents_no_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] + deployments = [] + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics(set(), set()) + assert change_failure_rate.change_failure_rate == 0 + + +# One incident between two deployments +def test_get_change_failure_rate_for_one_incidents_bw_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=1))] + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set([deployment_1]), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 50 + + +# One incident before two deployments +def test_get_change_failure_rate_for_one_incidents_bef_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set([]), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 0 + + +# One incident after two deployments +def test_get_change_failure_rate_for_one_incidents_after_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(hours=1))] + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set([deployment_2]), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 50 + + +# Multiple incidents and deployments +def test_get_change_failure_rate_for_multi_incidents_multi_deployments(): + """ + Time Line: + + incident_0 + deployment1 + deployment_2 + incident_1 + deployment_3 + incident_2 + deployment_4 + incident_3 + deployment_5 + incident_4 + incident_5 + incident_6 + deployment_6 + + """ + + incident_service = get_incident_service() + + incident_0 = get_incident(creation_date=time_now() - timedelta(days=10)) + incident_1 = get_incident(creation_date=time_now() - timedelta(days=5)) + incident_2 = get_incident(creation_date=time_now() - timedelta(days=3)) + incident_3 = get_incident(creation_date=time_now() - timedelta(hours=20)) + incident_4 = get_incident(creation_date=time_now() - timedelta(hours=4)) + incident_5 = get_incident(creation_date=time_now() - timedelta(hours=2)) + incident_6 = get_incident(creation_date=time_now() - timedelta(hours=1)) + + incidents = [ + incident_0, + incident_1, + incident_2, + incident_3, + incident_4, + incident_5, + incident_6, + ] + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=7)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(days=6)) + deployment_3 = get_deployment(conducted_at=time_now() - timedelta(days=4)) + deployment_4 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_5 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + deployment_6 = get_deployment(conducted_at=time_now() - timedelta(minutes=30)) + + deployments = [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + + assert change_failure_rate == get_change_failure_rate_metrics( + set([deployment_2, deployment_3, deployment_4, deployment_5]), + set( + [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + ] + ), + ) + assert change_failure_rate.change_failure_rate == (4 / 6 * 100) From 9136f0ccd6826caef3c1209afd79ab04dce54dd5 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Fri, 12 Apr 2024 16:35:47 +0530 Subject: [PATCH 5/5] Update tests of get_deployment_incidents_map --- .../test_deployment_incident_mapper.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/apiserver/tests/service/Incidents/test_deployment_incident_mapper.py b/apiserver/tests/service/Incidents/test_deployment_incident_mapper.py index e17a6619..c4de193c 100644 --- a/apiserver/tests/service/Incidents/test_deployment_incident_mapper.py +++ b/apiserver/tests/service/Incidents/test_deployment_incident_mapper.py @@ -2,7 +2,7 @@ from dora.service.incidents.incidents import get_incident_service from dora.utils.time import time_now -from tests.factories.models import get_incident, get_repo_workflow_run +from tests.factories.models import get_incident, get_deployment # No incidents, no deployments @@ -22,8 +22,8 @@ def test_get_deployment_incidents_count_map_returns_deployment_incident_count_ma incident_service = get_incident_service() incidents = [] deployments = [ - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=2)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(hours=6)), + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), ] deployment_incidents_count_map = incident_service.get_deployment_incidents_map( deployments, @@ -48,8 +48,8 @@ def test_get_deployment_incidents_count_map_returns_deployment_incident_count_ma incident_service = get_incident_service() incidents = [get_incident(creation_date=time_now() - timedelta(days=1))] deployments = [ - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=2)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(hours=6)), + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), ] deployment_incidents_count_map = incident_service.get_deployment_incidents_map( deployments, incidents @@ -65,8 +65,8 @@ def test_get_deployment_incidents_count_map_returns_deployment_incident_count_ma incident_service = get_incident_service() incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] deployments = [ - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=2)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(hours=6)), + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), ] deployment_incidents_count_map = incident_service.get_deployment_incidents_map( deployments, incidents @@ -79,8 +79,8 @@ def test_get_deployment_incidents_count_map_returns_deployment_incident_count_ma incident_service = get_incident_service() incidents = [get_incident(creation_date=time_now() - timedelta(hours=1))] deployments = [ - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=2)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(hours=6)), + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), ] deployment_incidents_count_map = incident_service.get_deployment_incidents_map( deployments, incidents @@ -101,11 +101,11 @@ def test_get_deployment_incidents_count_map_returns_deployment_incident_count_ma get_incident(creation_date=time_now() - timedelta(hours=1)), ] deployments = [ - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=7)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=6)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=4)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(days=2)), - get_repo_workflow_run(conducted_at=time_now() - timedelta(hours=6)), + get_deployment(conducted_at=time_now() - timedelta(days=7)), + get_deployment(conducted_at=time_now() - timedelta(days=6)), + get_deployment(conducted_at=time_now() - timedelta(days=4)), + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), ] deployment_incidents_count_map = incident_service.get_deployment_incidents_map( deployments, incidents