From d64e5b4dc1433970c6e730a5426aa4923d24783c Mon Sep 17 00:00:00 2001 From: Jumana Bahrainwala Date: Thu, 19 Dec 2024 20:28:51 +0000 Subject: [PATCH 1/4] Remove date conversion --- app/dao/date_util.py | 2 +- app/dao/fact_notification_status_dao.py | 7 +++++-- app/service/rest.py | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index c53684d5e2..94cdcba978 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -103,7 +103,7 @@ def utc_midnight_n_days_ago(number_of_days): """ Returns utc midnight a number of days ago. """ - return get_midnight(datetime.utcnow() - timedelta(days=number_of_days)) + return get_midnight(datetime.now(timezone.utc) - timedelta(days=number_of_days)) def get_query_date_based_on_retention_period(retention_period): diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index 2ee37d6f73..80ed2700fc 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -247,6 +247,9 @@ def fetch_notification_stats_for_trial_services(): def fetch_notification_status_for_service_for_day(bst_day, service_id): + # Fetch data from bst_day 00:00:00 to bst_day 23:59:59 + # bst_dat is currently in UTC and the return is in UTC + bst_day = bst_day.replace(hour=0, minute=0, second=0) return ( db.session.query( # return current month as a datetime so the data has the same shape as the ft_notification_status query @@ -256,8 +259,8 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id): func.count().label("count"), ) .filter( - Notification.created_at >= get_local_timezone_midnight_in_utc(bst_day), - Notification.created_at < get_local_timezone_midnight_in_utc(bst_day + timedelta(days=1)), + Notification.created_at >= bst_day, + Notification.created_at < bst_day + timedelta(days=1), Notification.service_id == service_id, Notification.key_type != KEY_TYPE_TEST, ) diff --git a/app/service/rest.py b/app/service/rest.py index 0156490291..cb3075b9ed 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,5 @@ import itertools -from datetime import datetime +from datetime import datetime, timezone from flask import Blueprint, current_app, jsonify, request from notifications_utils.clients.redis import ( @@ -651,10 +651,10 @@ def get_monthly_notification_stats(service_id): stats = fetch_notification_status_for_service_by_month(start_date, end_date, service_id) statistics.add_monthly_notification_status_stats(data, stats) - now = datetime.utcnow() + now = datetime.now(timezone.utc) # TODO FF_ANNUAL_LIMIT removal if not current_app.config["FF_ANNUAL_LIMIT"] and end_date > now: - todays_deltas = fetch_notification_status_for_service_for_day(convert_utc_to_local_timezone(now), service_id=service_id) + todays_deltas = fetch_notification_status_for_service_for_day(now, service_id=service_id) statistics.add_monthly_notification_status_stats(data, todays_deltas) return jsonify(data=data) From 77f3e7218c9c956b0ddcdc85753444543a70aba4 Mon Sep 17 00:00:00 2001 From: Jumana Bahrainwala Date: Thu, 19 Dec 2024 20:33:56 +0000 Subject: [PATCH 2/4] additiona places it was left --- app/celery/process_pinpoint_receipts_tasks.py | 4 ++-- app/celery/process_ses_receipts_tasks.py | 4 ++-- app/celery/process_sns_receipts_tasks.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/celery/process_pinpoint_receipts_tasks.py b/app/celery/process_pinpoint_receipts_tasks.py index 5edd796169..54c6865385 100644 --- a/app/celery/process_pinpoint_receipts_tasks.py +++ b/app/celery/process_pinpoint_receipts_tasks.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from typing import Union from flask import current_app, json @@ -120,7 +120,7 @@ def process_pinpoint_results(self, response): if not annual_limit_client.was_seeded_today(service_id): annual_limit_client.set_seeded_at(service_id) notifications_to_seed = fetch_notification_status_for_service_for_day( - convert_utc_to_local_timezone(datetime.utcnow()), + datetime.now(timezone.utc), service_id=service_id, ) annual_limit_client.seed_annual_limit_notifications( diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 5d2c29b133..f10aa35b07 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from flask import current_app, json from notifications_utils.statsd_decorators import statsd @@ -97,7 +97,7 @@ def process_ses_results(self, response): # noqa: C901 if not annual_limit_client.was_seeded_today(service_id): annual_limit_client.set_seeded_at(service_id) notifications_to_seed = fetch_notification_status_for_service_for_day( - convert_utc_to_local_timezone(datetime.utcnow()), + convert_utc_to_local_timezone(datetime.now(timezone.utc)), service_id=service_id, ) annual_limit_client.seed_annual_limit_notifications( diff --git a/app/celery/process_sns_receipts_tasks.py b/app/celery/process_sns_receipts_tasks.py index 326784f3e3..2029db9ae5 100644 --- a/app/celery/process_sns_receipts_tasks.py +++ b/app/celery/process_sns_receipts_tasks.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from flask import current_app, json from notifications_utils.statsd_decorators import statsd @@ -77,7 +77,7 @@ def process_sns_results(self, response): if not annual_limit_client.was_seeded_today(service_id): annual_limit_client.set_seeded_at(service_id) notifications_to_seed = fetch_notification_status_for_service_for_day( - convert_utc_to_local_timezone(datetime.utcnow()), + convert_utc_to_local_timezone(datetime.now(timezone.utc)), service_id=service_id, ) annual_limit_client.seed_annual_limit_notifications( From c1cdbea4fe61a7830d9f56ac98648ff3d14e98a7 Mon Sep 17 00:00:00 2001 From: Jumana Bahrainwala Date: Thu, 19 Dec 2024 20:52:11 +0000 Subject: [PATCH 3/4] fix --- app/celery/process_pinpoint_receipts_tasks.py | 1 - app/celery/process_ses_receipts_tasks.py | 3 +-- app/celery/process_sns_receipts_tasks.py | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/app/celery/process_pinpoint_receipts_tasks.py b/app/celery/process_pinpoint_receipts_tasks.py index 54c6865385..82df455877 100644 --- a/app/celery/process_pinpoint_receipts_tasks.py +++ b/app/celery/process_pinpoint_receipts_tasks.py @@ -3,7 +3,6 @@ from flask import current_app, json from notifications_utils.statsd_decorators import statsd -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy.orm.exc import NoResultFound from app import annual_limit_client, notify_celery, statsd_client diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index f10aa35b07..e147a0c42f 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -2,7 +2,6 @@ from flask import current_app, json from notifications_utils.statsd_decorators import statsd -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy.orm.exc import NoResultFound from app import annual_limit_client, bounce_rate_client, notify_celery, statsd_client @@ -97,7 +96,7 @@ def process_ses_results(self, response): # noqa: C901 if not annual_limit_client.was_seeded_today(service_id): annual_limit_client.set_seeded_at(service_id) notifications_to_seed = fetch_notification_status_for_service_for_day( - convert_utc_to_local_timezone(datetime.now(timezone.utc)), + datetime.now(timezone.utc), service_id=service_id, ) annual_limit_client.seed_annual_limit_notifications( diff --git a/app/celery/process_sns_receipts_tasks.py b/app/celery/process_sns_receipts_tasks.py index 2029db9ae5..7b0fe36166 100644 --- a/app/celery/process_sns_receipts_tasks.py +++ b/app/celery/process_sns_receipts_tasks.py @@ -2,7 +2,6 @@ from flask import current_app, json from notifications_utils.statsd_decorators import statsd -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy.orm.exc import NoResultFound from app import annual_limit_client, notify_celery, statsd_client @@ -77,7 +76,7 @@ def process_sns_results(self, response): if not annual_limit_client.was_seeded_today(service_id): annual_limit_client.set_seeded_at(service_id) notifications_to_seed = fetch_notification_status_for_service_for_day( - convert_utc_to_local_timezone(datetime.now(timezone.utc)), + datetime.now(timezone.utc), service_id=service_id, ) annual_limit_client.seed_annual_limit_notifications( From 72feee5e184951890c53579db986cc1bb602a231 Mon Sep 17 00:00:00 2001 From: Jumana Bahrainwala Date: Thu, 19 Dec 2024 21:12:11 +0000 Subject: [PATCH 4/4] fix --- app/service/rest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/service/rest.py b/app/service/rest.py index cb3075b9ed..1934dabb7d 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -652,8 +652,10 @@ def get_monthly_notification_stats(service_id): statistics.add_monthly_notification_status_stats(data, stats) now = datetime.now(timezone.utc) + # end_date doesn't have tzinfo, so we need to remove it from now + end_date_now = now.replace(tzinfo=None) # TODO FF_ANNUAL_LIMIT removal - if not current_app.config["FF_ANNUAL_LIMIT"] and end_date > now: + if not current_app.config["FF_ANNUAL_LIMIT"] and end_date > end_date_now: todays_deltas = fetch_notification_status_for_service_for_day(now, service_id=service_id) statistics.add_monthly_notification_status_stats(data, todays_deltas)