From b991534aef9d5116b31c244d3eb24e6729981fb3 Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 14 Nov 2024 16:23:49 +0100 Subject: [PATCH 01/28] Changes to schedule all reports --- rocky/reports/forms.py | 33 ++-- .../partials/export_report_settings.html | 68 ++++---- .../partials/report_names_header.html | 52 +++--- rocky/reports/views/base.py | 148 ++++++------------ rocky/rocky/views/scheduler.py | 40 ++--- 5 files changed, 121 insertions(+), 220 deletions(-) diff --git a/rocky/reports/forms.py b/rocky/reports/forms.py index 265d0d15e27..a6726238f75 100644 --- a/rocky/reports/forms.py +++ b/rocky/reports/forms.py @@ -39,12 +39,11 @@ class ReportScheduleStartDateChoiceForm(BaseRockyForm): class ReportScheduleStartDateForm(BaseRockyForm): - start_date = forms.DateField( - label=_("Start date"), - widget=DateInput(format="%Y-%m-%d", attrs={"form": "generate_report"}), - initial=lambda: datetime.now(tz=timezone.utc).date(), + start_date = forms.DateTimeField( + label=_("Start date and time (UTC)"), + widget=forms.DateTimeInput(attrs={"form": "generate_report", "type": "datetime-local"}), required=True, - input_formats=["%Y-%m-%d"], + initial=lambda: datetime.now().strftime("%Y-%m-%d %H:%M"), ) @@ -63,7 +62,13 @@ class ReportScheduleRecurrenceForm(BaseRockyForm): label=_("Recurrence"), required=True, widget=forms.Select(attrs={"form": "generate_report"}), - choices=[("daily", _("Daily")), ("weekly", _("Weekly")), ("monthly", _("Monthly")), ("yearly", _("Yearly"))], + choices=[ + ("once", _("No recurrence, just once")), + ("daily", _("Daily")), + ("weekly", _("Weekly")), + ("monthly", _("Monthly")), + ("yearly", _("Yearly")), + ], ) @@ -103,19 +108,3 @@ class CustomReportScheduleForm(BaseRockyForm): end_date = forms.DateField( label=_(""), widget=forms.HiddenInput(), initial=lambda: datetime.now(tz=timezone.utc).date(), required=False ) - - -class ParentReportNameForm(BaseRockyForm): - parent_report_name = forms.CharField( - label=_("Report name format"), required=False, initial="${report_type} for ${oois_count} objects" - ) - - -class ChildReportNameForm(BaseRockyForm): - child_report_name = forms.CharField( - label=_("Subreports name format"), required=True, initial="${report_type} for ${ooi}" - ) - - -class ReportNameForm(ParentReportNameForm, ChildReportNameForm): - pass diff --git a/rocky/reports/templates/partials/export_report_settings.html b/rocky/reports/templates/partials/export_report_settings.html index 09e5384b0ae..7699b877d06 100644 --- a/rocky/reports/templates/partials/export_report_settings.html +++ b/rocky/reports/templates/partials/export_report_settings.html @@ -15,53 +15,43 @@

{% translate "Report schedule" %}

or monthly. If you need the report just for a single occasion, select the one-time option. {% endblocktranslate %}

-
- {% csrf_token %} - {% include "forms/report_form_fields.html" %} - {% include "partials/form/fieldset.html" with fields=report_schedule_form_recurrence_choice %} - - {% if is_scheduled_report %} -

- {% blocktranslate trimmed %} - The date you select will be the reference date for the data set for your report. - Please allow for up to 24 hours for your report to be ready. - {% endblocktranslate %} -

-
-
- {% include "partials/form/fieldset.html" with fields=report_schedule_form_start_date %} - -
-
- {% include "partials/form/fieldset.html" with fields=report_schedule_form_recurrence %} - -
-
- {% endif %} -
{% csrf_token %} {% include "forms/report_form_fields.html" %} - {% if not is_scheduled_report %} - {% include "partials/report_names_header.html" %} - {% include "partials/report_names_form.html" %} +
+
+ {% include "partials/form/fieldset.html" with fields=report_schedule_form_start_date %} - - {% else %} - {% include "partials/report_names_header.html" with recurrence=True %} - {% include "partials/form/fieldset.html" with fields=report_parent_name_form %} +
+
+ {% include "partials/form/fieldset.html" with fields=report_schedule_form_recurrence %} - {% if reports|length > 1 %} - {% include "partials/form/fieldset.html" with fields=report_child_name_form %} +
+
+ {% include "partials/report_names_header.html" with recurrence=True %} - {% endif %} - +
+
+ + +
+
+ {% if not is_single_report %} +
+
+ + +
+
{% endif %} +
{% else %} {% include "partials/return_button.html" with btn_text="Go back" %} diff --git a/rocky/reports/templates/partials/report_names_header.html b/rocky/reports/templates/partials/report_names_header.html index 1616fc02f25..35e56e8b103 100644 --- a/rocky/reports/templates/partials/report_names_header.html +++ b/rocky/reports/templates/partials/report_names_header.html @@ -2,34 +2,24 @@ {% load static %}

{% translate "Report name" %}

-{% if recurrence %} -

- {% blocktranslate trimmed %} - Define a custom name format for your report(s). This format will be applied to all generated - (sub)reports. - {% endblocktranslate %} -

-

- {% blocktranslate trimmed %} - To make the report names more descriptive, you can include placeholders for the - object name, the report type and/or the reference date. For subreports and reports over a single object, - use the placeholder "${ooi}" for the object name, "${report_type}" for the report type and use a - Python strftime code for the reference - date. For reports over multiple objects, use "${oois_count}" for the number of objects in the report. - {% endblocktranslate %} -

-

- {% blocktranslate trimmed %} - For example, the format "${report_type} for ${ooi} at %x" could generate: - "DNS Report for example.com at 01/01/25". - {% endblocktranslate %} -

-{% else %} -

- {% blocktranslate trimmed %} - Give your report(s) a custom name and optionally add the reports' reference date - to the name. To do so you can select a standard option or use a Python - strftime code in the report name. - {% endblocktranslate %} -

-{% endif %} +

+ {% blocktranslate trimmed %} + Define a custom name format for your report(s). This format will be applied to all generated + (sub)reports. + {% endblocktranslate %} +

+

+ {% blocktranslate trimmed %} + To make the report names more descriptive, you can include placeholders for the + object name, the report type and/or the reference date. For subreports and reports over a single object, + use the placeholder "${ooi}" for the object name, "${report_type}" for the report type and use a + Python strftime code for the reference + date. For reports over multiple objects, use "${oois_count}" for the number of objects in the report. + {% endblocktranslate %} +

+

+ {% blocktranslate trimmed %} + For example, the format "${report_type} for ${ooi} at %x" could generate: + "DNS Report for example.com at 01/01/25". + {% endblocktranslate %} +

diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index 2d8514a3f18..d019323366d 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -267,8 +267,15 @@ def is_scheduled_report(self) -> bool: recurrence_choice = self.request.POST.get("choose_recurrence", "once") return recurrence_choice == "repeat" + def is_single_report(self) -> bool: + return len(self.get_report_type_ids()) == 1 + def create_report_recipe( - self, report_name_format: str, subreport_name_format: str, parent_report_type: str | None, schedule: str + self, + report_name_format: str, + subreport_name_format: str | None, + parent_report_type: str | None, + schedule: str | None, ) -> ReportRecipe: report_recipe = ReportRecipe( recipe_id=uuid4(), @@ -302,6 +309,7 @@ def get_context_data(self, **kwargs): context["all_oois_selected"] = self.all_oois_selected() context["selected_oois"] = self.selected_oois context["selected_report_types"] = self.selected_report_types + context["is_single_report"] = self.is_single_report() return context @@ -449,132 +457,64 @@ def get_context_data(self, **kwargs): class ReportFinalSettingsView(BaseReportView, ReportBreadcrumbs, SchedulerView, TemplateView): report_type: type[BaseReport] | None = None task_type = "report" - is_a_scheduled_report = False def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: if not self.get_report_type_ids(): messages.error(request, self.NONE_REPORT_TYPE_SELECTION_MESSAGE) return PostRedirect(self.get_previous()) - - self.is_a_scheduled_report = self.is_scheduled_report() - return super().get(request, *args, **kwargs) - @staticmethod - def create_report_names(oois: list[OOI], report_types: list[type[BaseReport]]) -> dict[str, str]: - reports = {} - oois_count = len(oois) - report_types_count = len(report_types) - ooi = oois[0].human_readable - report_type = report_types[0].name - - # Create name for parent report - if not (report_types_count == 1 and oois_count == 1): - if report_types_count > 1 and oois_count > 1: - name = _("Concatenated Report for {oois_count} objects").format( - report_type=report_type, oois_count=oois_count - ) - elif report_types_count > 1 and oois_count == 1: - name = _("Concatenated Report for {ooi}").format(ooi=ooi) - elif report_types_count == 1 and oois_count > 1: - name = _("{report_type} for {oois_count} objects").format( - report_type=report_type, oois_count=oois_count - ) - reports[name] = "" - - # Create name for subreports or single reports - for ooi in oois: - for report_type_ in report_types: - name = _("{report_type} for {ooi}").format(report_type=report_type_.name, ooi=ooi.human_readable) - reports[name] = "" - - return reports - - def get_report_names(self) -> dict[str, str] | list[str]: - if self.report_type is not None and self.report_type == AggregateOrganisationReport: - return [_("Aggregate Report")] - if self.report_type is not None and self.report_type == MultiOrganizationReport: - return [_("Sector Report")] - - return self.create_report_names(self.get_oois(), self.get_report_types()) + def get_initial_report_name(self) -> str: + oois = self.get_total_oois() + if oois == 1: + return "${report_type} for ${ooi}" + if oois > 1: + return "${report_type} for ${oois_count} objects" + return "" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["reports"] = self.get_report_names() - + context["initial_report_name"] = self.get_initial_report_name() context["report_schedule_form_start_date"] = self.get_report_schedule_form_start_date() - context["report_schedule_form_recurrence_choice"] = self.get_report_schedule_form_recurrence_choice() context["report_schedule_form_recurrence"] = self.get_report_schedule_form_recurrence() - - context["report_parent_name_form"] = self.get_report_parent_name_form() - context["report_child_name_form"] = self.get_report_child_name_form() - - context["is_scheduled_report"] = self.is_a_scheduled_report - - context["created_at"] = datetime.now() return context class SaveReportView(BaseReportView, ReportBreadcrumbs, SchedulerView): task_type = "report" + def get_parent_report_type(self): + if self.report_type is not None: + return self.report_type.id + if not self.is_single_report(): + return ConcatenatedReport.id + return self.report_type + def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: - old_report_names = request.POST.getlist("old_report_name") - report_names = request.POST.getlist("report_name", []) - reference_dates = request.POST.getlist("reference_date") - - if not self.is_scheduled_report() and report_names: - final_report_names = list(zip(old_report_names, self.finalise_report_names(report_names, reference_dates))) - report_ooi = self.save_report(final_report_names) - - return redirect( - reverse("view_report", kwargs={"organization_code": self.organization.code}) - + "?" - + urlencode({"report_id": report_ooi.reference}) - ) - elif self.is_scheduled_report(): - report_name_format = request.POST.get("parent_report_name", "") - subreport_name_format = request.POST.get("child_report_name", "") - recurrence = request.POST.get("recurrence", "") - deadline_at = request.POST.get("start_date", datetime.now(timezone.utc).date()) - - parent_report_type = None - if self.report_type is not None: - parent_report_type = self.report_type.id - elif not self.report_type and subreport_name_format: - parent_report_type = ConcatenatedReport.id - - schedule = self.convert_recurrence_to_cron_expressions(recurrence) - - report_recipe = self.create_report_recipe( - report_name_format, subreport_name_format, parent_report_type, schedule - ) + deadline_at = request.POST.get("start_date") + start_date_time: datetime = ( + datetime.now(timezone.utc) if deadline_at is None else datetime.fromisoformat(deadline_at) + ) - self.create_report_schedule(report_recipe, deadline_at) + recurrence = request.POST.get("recurrence") + schedule = ( + self.convert_schedule_to_cron_expressions(start_date_time, recurrence) + if recurrence is not None and recurrence != "once" + else None + ) - return redirect(reverse("scheduled_reports", kwargs={"organization_code": self.organization.code})) + parent_report_type = self.get_parent_report_type() - messages.error(request, _("Empty name should not be possible.")) - return PostRedirect(self.get_previous()) + parent_report_name_format = request.POST.get("parent_report_name_format", "") + subreport_name_format = request.POST.get("subreport_name_format") - @staticmethod - def finalise_report_names(report_names: list[str], reference_dates: list[str]) -> list[str]: - final_report_names = [] - - if len(report_names) == len(reference_dates): - for index, report_name in enumerate(report_names): - date_format = "" - if reference_dates[index] and reference_dates[index] != "": - date_format = " - " - if reference_dates[index] == "week": - date_format += _("Week %W, %Y") - else: - date_format += reference_dates[index] - final_report_name = f"{report_name} {date_format}".strip() - final_report_names.append(final_report_name) - if not final_report_names: - return report_names - return final_report_names + report_recipe = self.create_report_recipe( + parent_report_name_format, subreport_name_format, parent_report_type, schedule + ) + + self.create_report_schedule(report_recipe, start_date_time.isoformat()) + + return redirect(reverse("scheduled_reports", kwargs={"organization_code": self.organization.code})) class ViewReportView(ObservedAtMixin, OrganizationView, TemplateView): diff --git a/rocky/rocky/views/scheduler.py b/rocky/rocky/views/scheduler.py index f2284980693..5021ba18d8b 100644 --- a/rocky/rocky/views/scheduler.py +++ b/rocky/rocky/views/scheduler.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime, timezone +from datetime import datetime from typing import Any from django.contrib import messages @@ -7,8 +7,6 @@ from django.utils.translation import gettext_lazy as _ from katalogus.client import Boefje, Normalizer from reports.forms import ( - ChildReportNameForm, - ParentReportNameForm, ReportRecurrenceChoiceForm, ReportScheduleRecurrenceForm, ReportScheduleStartDateChoiceForm, @@ -53,9 +51,6 @@ class SchedulerView(OctopoesView): report_schedule_form_recurrence_choice = ReportRecurrenceChoiceForm # once or repeat report_schedule_form_recurrence = ReportScheduleRecurrenceForm # select interval (daily, weekly, etc..) - report_parent_name_form = ParentReportNameForm # parent name format - report_child_name_form = ChildReportNameForm # child name format - def setup(self, request, *args, **kwargs): super().setup(request, *args, **kwargs) self.scheduler_client = scheduler_client(self.organization.code) @@ -105,12 +100,6 @@ def get_report_schedule_form_recurrence_choice(self): def get_report_schedule_form_recurrence(self): return self.report_schedule_form_recurrence() - def get_report_parent_name_form(self): - return self.report_parent_name_form() - - def get_report_child_name_form(self): - return self.report_child_name_form() - def get_task_details(self, task_id: str) -> Task | None: task = self.scheduler_client.get_task_details(task_id) @@ -262,28 +251,31 @@ def run_boefje_for_oois(self, boefje: Boefje, oois: list[OOI]) -> None: except SchedulerError as error: messages.error(self.request, error.message) - def convert_recurrence_to_cron_expressions(self, recurrence: str) -> str: + def convert_schedule_to_cron_expressions(self, start_date_time: datetime, recurrence: str) -> str: """ Because there is no time defined for the start date, we use midnight 00:00 for all expressions. """ - start_date = datetime.now(tz=timezone.utc).date() # for now, not set by user - - if start_date and recurrence: - day = start_date.day - month = start_date.month - week = start_date.strftime("%w").upper() # ex. 4 + if start_date_time and recurrence: + day = start_date_time.day + month = start_date_time.month + week = start_date_time.strftime("%w").upper() # ex. 4 + hour = start_date_time.hour + minute = start_date_time.minute cron_expr = { - "daily": "0 0 * * *", # Recurres every day at 00:00 - "weekly": f"0 0 * * {week}", # Recurres every week on the {week} at 00:00 - "yearly": f"0 0 {day} {month} *", # Recurres every year on the {day} of the {month} at 00:00 + "daily": f"{minute} {hour} * * *", # Recurres every day at the selected time + "weekly": f"{minute} {hour} * * {week}", # Recurres every week on the {week} at the selected time + "yearly": f"{minute} {hour} {day} {month} *", + # Recurres every year on the {day} of the {month} at the selected time } if 28 <= day <= 31: - cron_expr["monthly"] = "0 0 28-31 * *" + cron_expr["monthly"] = f"{minute} {hour} L * *" else: - cron_expr["monthly"] = f"0 0 {day} * *" # Recurres on the exact {day} of the month at 00:00 + cron_expr["monthly"] = ( + f"{minute} {hour} {day} * *" # Recurres on the exact {day} of the month at the selected time + ) return cron_expr.get(recurrence, "") return "" From 7351eda0df8a9cd67948ed2a4dcfd457c802610b Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 14 Nov 2024 16:43:08 +0100 Subject: [PATCH 02/28] remove form id --- rocky/reports/forms.py | 2 +- rocky/reports/templates/partials/export_report_settings.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rocky/reports/forms.py b/rocky/reports/forms.py index a6726238f75..861cd5e121b 100644 --- a/rocky/reports/forms.py +++ b/rocky/reports/forms.py @@ -41,7 +41,7 @@ class ReportScheduleStartDateChoiceForm(BaseRockyForm): class ReportScheduleStartDateForm(BaseRockyForm): start_date = forms.DateTimeField( label=_("Start date and time (UTC)"), - widget=forms.DateTimeInput(attrs={"form": "generate_report", "type": "datetime-local"}), + widget=forms.DateTimeInput(attrs={"type": "datetime-local"}), required=True, initial=lambda: datetime.now().strftime("%Y-%m-%d %H:%M"), ) diff --git a/rocky/reports/templates/partials/export_report_settings.html b/rocky/reports/templates/partials/export_report_settings.html index 7699b877d06..de93d568191 100644 --- a/rocky/reports/templates/partials/export_report_settings.html +++ b/rocky/reports/templates/partials/export_report_settings.html @@ -49,7 +49,7 @@

{% translate "Report schedule" %}

{% endif %} - From 3f5c77dc88b04e4a683490fdf60f0ac799bc1f05 Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 14 Nov 2024 21:25:51 +0100 Subject: [PATCH 03/28] fix lang --- rocky/rocky/locale/django.pot | 72 +++++++++-------------------------- 1 file changed, 17 insertions(+), 55 deletions(-) diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 245aa42c98a..0c82a83978d 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -9,7 +9,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-11-12 15:31+0000\n" +"POT-Creation-Date: 2024-11-14 20:25+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -2408,7 +2408,7 @@ msgid "Different date" msgstr "" #: reports/forms.py -msgid "Start date" +msgid "Start date and time (UTC)" msgstr "" #: reports/forms.py @@ -2424,6 +2424,10 @@ msgstr "" msgid "Recurrence" msgstr "" +#: reports/forms.py +msgid "No recurrence, just once" +msgstr "" + #: reports/forms.py msgid "Daily" msgstr "" @@ -2440,6 +2444,10 @@ msgstr "" msgid "Yearly" msgstr "" +#: reports/forms.py +msgid "Start date" +msgstr "" + #: reports/forms.py msgid "day" msgstr "" @@ -2468,14 +2476,6 @@ msgstr "" msgid "After" msgstr "" -#: reports/forms.py -msgid "Report name format" -msgstr "" - -#: reports/forms.py -msgid "Subreports name format" -msgstr "" - #: reports/report_types/aggregate_organisation_report/appendix.html #: reports/report_types/multi_organization_report/appendix.html #: reports/templates/partials/report_sidemenu.html @@ -3702,9 +3702,11 @@ msgid "" msgstr "" #: reports/templates/partials/export_report_settings.html -msgid "" -"The date you select will be the reference date for the data set for your " -"report. Please allow for up to 24 hours for your report to be ready." +msgid "Report name format" +msgstr "" + +#: reports/templates/partials/export_report_settings.html +msgid "Subreports name format" msgstr "" #: reports/templates/partials/export_report_settings.html @@ -3838,14 +3840,6 @@ msgid "" "\"DNS Report for example.com at 01/01/25\"." msgstr "" -#: reports/templates/partials/report_names_header.html -msgid "" -"Give your report(s) a custom name and optionally add the reports' reference " -"date to the name. To do so you can select a standard option or use a Python " -"strftime code in the report name." -msgstr "" - #: reports/templates/partials/report_navigation.html msgid "Report Navigation" msgstr "" @@ -3854,7 +3848,7 @@ msgstr "" msgid "Generate Report" msgstr "" -#: reports/templates/partials/report_navigation.html reports/views/base.py +#: reports/templates/partials/report_navigation.html msgid "Aggregate Report" msgstr "" @@ -4423,38 +4417,6 @@ msgstr "" msgid "Select at least one report type to proceed." msgstr "" -#: reports/views/base.py -#, python-brace-format -msgid "Concatenated Report for {oois_count} objects" -msgstr "" - -#: reports/views/base.py -#, python-brace-format -msgid "Concatenated Report for {ooi}" -msgstr "" - -#: reports/views/base.py -#, python-brace-format -msgid "{report_type} for {oois_count} objects" -msgstr "" - -#: reports/views/base.py -#, python-brace-format -msgid "{report_type} for {ooi}" -msgstr "" - -#: reports/views/base.py -msgid "Sector Report" -msgstr "" - -#: reports/views/base.py -msgid "Empty name should not be possible." -msgstr "" - -#: reports/views/base.py -msgid "Week %W, %Y" -msgstr "" - #: reports/views/mixins.py msgid "" "No data could be found for %(report_types). Object(s) did not exist on " @@ -4479,7 +4441,7 @@ msgstr "" #: reports/views/report_overview.py msgid "" -"Multiorganization report cannot go through a rerun. It consists of imported " +"Multi organization reports cannot be rescheduled. It consists of imported " "data from different organizations and not based on new generated data." msgstr "" From 7adc85f73b63a9651fec31176b6efbf65a64d5e5 Mon Sep 17 00:00:00 2001 From: Rieven Date: Mon, 18 Nov 2024 11:02:03 +0100 Subject: [PATCH 04/28] Update rocky/reports/templates/partials/export_report_settings.html Co-authored-by: Peter-Paul van Gemerden --- rocky/reports/templates/partials/export_report_settings.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rocky/reports/templates/partials/export_report_settings.html b/rocky/reports/templates/partials/export_report_settings.html index de93d568191..36ce6382aa8 100644 --- a/rocky/reports/templates/partials/export_report_settings.html +++ b/rocky/reports/templates/partials/export_report_settings.html @@ -33,8 +33,11 @@

{% translate "Report schedule" %}

- +
From bb7ecb395fd07e2008c96931292bc4a03d9c35ec Mon Sep 17 00:00:00 2001 From: Rieven Date: Mon, 18 Nov 2024 11:02:13 +0100 Subject: [PATCH 05/28] Update rocky/reports/templates/partials/export_report_settings.html Co-authored-by: Peter-Paul van Gemerden --- rocky/reports/templates/partials/export_report_settings.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rocky/reports/templates/partials/export_report_settings.html b/rocky/reports/templates/partials/export_report_settings.html index 36ce6382aa8..92b9f44fb04 100644 --- a/rocky/reports/templates/partials/export_report_settings.html +++ b/rocky/reports/templates/partials/export_report_settings.html @@ -45,8 +45,11 @@

{% translate "Report schedule" %}

{% if not is_single_report %}
- +
From f479f2f823148d3d43cf9eb5e729e083d5d2dce8 Mon Sep 17 00:00:00 2001 From: Rieven Date: Wed, 20 Nov 2024 13:49:39 +0100 Subject: [PATCH 06/28] set None atttrs --- octopoes/octopoes/models/ooi/reports.py | 2 +- .../report_overview/scheduled_reports_table.html | 8 +++++++- rocky/rocky/scheduler.py | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/octopoes/octopoes/models/ooi/reports.py b/octopoes/octopoes/models/ooi/reports.py index 525ad009b57..e13d45f34d8 100644 --- a/octopoes/octopoes/models/ooi/reports.py +++ b/octopoes/octopoes/models/ooi/reports.py @@ -61,6 +61,6 @@ class ReportRecipe(OOI): parent_report_type: str | None = None report_types: list[str] - cron_expression: str + cron_expression: str | None = None _natural_key_attrs = ["recipe_id"] diff --git a/rocky/reports/templates/report_overview/scheduled_reports_table.html b/rocky/reports/templates/report_overview/scheduled_reports_table.html index fc581e69913..04ef95c7218 100644 --- a/rocky/reports/templates/report_overview/scheduled_reports_table.html +++ b/rocky/reports/templates/report_overview/scheduled_reports_table.html @@ -42,7 +42,13 @@ {{ schedule.deadline_at }} - {{ schedule.cron }} + + {% if schedule.cron is None %} + {% translate "Once" %} + {% else %} + {{ schedule.cron }} + {% endif %} + - - {% endif %} + {% endif %} + {% if report.children_reports %} From 6d143f877527244bb2542cd0837c018a1b1f8491 Mon Sep 17 00:00:00 2001 From: JP Bruins Slot Date: Wed, 27 Nov 2024 16:40:53 +0100 Subject: [PATCH 14/28] Add additional check if task already run --- .../scheduler/schedulers/schedulers/report.py | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/mula/scheduler/schedulers/schedulers/report.py b/mula/scheduler/schedulers/schedulers/report.py index 099c21642d9..80a46f16718 100644 --- a/mula/scheduler/schedulers/schedulers/report.py +++ b/mula/scheduler/schedulers/schedulers/report.py @@ -7,7 +7,7 @@ from opentelemetry import trace from scheduler import context, storage -from scheduler.models import Organisation, ReportTask, Task +from scheduler.models import Organisation, ReportTask, Task, TaskStatus from scheduler.schedulers import Scheduler from scheduler.schedulers.queue import PriorityQueue, QueueFullError from scheduler.storage import filters @@ -87,6 +87,30 @@ def push_tasks_for_rescheduling(self): ) as executor: for schedule in schedules: report_task = ReportTask.model_validate(schedule.data) + + # When the schedule has no schedule, but a task is + # already executed we should not push the task again + task_db = None + try: + task_db = self.ctx.datastores.task_store.get_latest_task_by_hash(report_task.hash) + if task_db and schedule.schedule is None: + self.logger.debug( + "Schedule has no schedule, but task already executed", + schedule_id=schedule.id, + scheduler_id=self.scheduler_id, + organisation_id=self.organisation.id, + ) + continue + except storage.errors.StorageError as exc_db: + self.logger.error( + "Could not get latest task by hash %s", + report_task.hash, + scheduler_id=self.scheduler_id, + organisation_id=self.organisation.id, + exc_info=exc_db, + ) + continue + executor.submit(self.push_report_task, report_task, self.push_tasks_for_rescheduling.__name__) def push_report_task(self, report_task: ReportTask, caller: str = "") -> None: @@ -98,6 +122,16 @@ def push_report_task(self, report_task: ReportTask, caller: str = "") -> None: caller=caller, ) + if self.has_report_task_started_running(report_task): + self.logger.debug( + "Report task already running", + task_hash=report_task.hash, + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + caller=caller, + ) + return + if self.is_item_on_queue_by_hash(report_task.hash): self.logger.debug( "Report task already on queue", @@ -139,3 +173,28 @@ def push_report_task(self, report_task: ReportTask, caller: str = "") -> None: scheduler_id=self.scheduler_id, caller=caller, ) + + def has_report_task_started_running(self, task: ReportTask) -> bool: + task_db = None + try: + task_db = self.ctx.datastores.task_store.get_latest_task_by_hash(task.hash) + except storage.errors.StorageError as exc_db: + self.logger.error( + "Could not get latest task by hash %s", + task.hash, + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + exc_info=exc_db, + ) + raise exc_db + + if task_db is not None and task_db.status not in [TaskStatus.FAILED, TaskStatus.COMPLETED]: + self.logger.debug( + "Task is still running, according to the datastore", + task_id=task_db.id, + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + ) + return True + + return False From b727ec6a7eabc899247b51d0d13d3c4f06ad822b Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 28 Nov 2024 11:18:02 +0100 Subject: [PATCH 15/28] Fix report types pils --- .../report_overview/scheduled_reports_table.html | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/rocky/reports/templates/report_overview/scheduled_reports_table.html b/rocky/reports/templates/report_overview/scheduled_reports_table.html index ef0c9152875..af8beaf0ebe 100644 --- a/rocky/reports/templates/report_overview/scheduled_reports_table.html +++ b/rocky/reports/templates/report_overview/scheduled_reports_table.html @@ -27,7 +27,7 @@ {% if schedule.recipe %} {{ schedule.recipe.report_name_format }} - +
    {% if schedule.recipe.parent_report_type == "aggregate-organisation-report" %}
  • @@ -36,17 +36,21 @@ {% else %} {% for report_type in schedule.recipe.report_types %} {% if forloop.counter0 < 2 %} -
  • {{ report_type|get_report_type_name }}
    • +
  • + {{ report_type|get_report_type_name }} +
    • {% endif %} {% if forloop.counter0 == 2 %} -
  • + {{ schedule.recipe.report_types|length|add:"-2" }}
    • +
  • + + {{ schedule.recipe.report_types|length|add:"-2" }} +
    • {% endif %} {% endfor %} {% endif %}
{{ schedule.deadline_at }} - + {% if schedule.cron is None %} {% translate "Once" %} {% else %} From eca64aa9131f950f22d40ae8b9b68d97bfe934b5 Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 28 Nov 2024 16:09:38 +0100 Subject: [PATCH 16/28] Fix report names --- .../partials/export_report_settings.html | 6 +++--- rocky/reports/views/base.py | 18 +++++++++++------- rocky/rocky/views/scheduler.py | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/rocky/reports/templates/partials/export_report_settings.html b/rocky/reports/templates/partials/export_report_settings.html index 587bd440682..e52fd035526 100644 --- a/rocky/reports/templates/partials/export_report_settings.html +++ b/rocky/reports/templates/partials/export_report_settings.html @@ -50,17 +50,17 @@

{% translate "Report name" %}

+ value="{{ initial_report_names.0 }}">
- {% if not is_single_report %} + {% if initial_report_names.1 %}
+ value="{{ initial_report_names.1 }}">
{% endif %} diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index 9a1e5518d65..4d9600d10cb 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -504,17 +504,21 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: return PostRedirect(self.get_previous()) return super().get(request, *args, **kwargs) - def get_initial_report_name(self) -> str: + def get_initial_report_names(self) -> tuple[str, str]: oois = self.get_total_oois() - if oois == 1: - return "${report_type} for ${ooi}" + is_single_report = self.is_single_report() + + if oois == 1 and is_single_report: + return ("${report_type} for ${ooi}", "") + if oois == 1 and not is_single_report: + return ("${report_type} for ${ooi}", "${report_type} for ${ooi}") if oois > 1: - return "${report_type} for ${oois_count} objects" - return "" + return ("${report_type} for ${oois_count} objects", "${report_type} for ${ooi}") + return ("", "") def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["initial_report_name"] = self.get_initial_report_name() + context["initial_report_names"] = self.get_initial_report_names() context["report_schedule_form_start_date"] = self.get_report_schedule_form_start_date_time_recurrence() context["report_schedule_form_recurrence_choice"] = self.get_report_schedule_form_recurrence_choice() return context @@ -551,7 +555,7 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: parent_report_name_format, subreport_name_format, parent_report_type, schedule, self.get_query() ) - self.create_report_schedule(report_recipe, start_datetime.isoformat()) + self.create_report_schedule(report_recipe, start_datetime) return redirect(reverse("scheduled_reports", kwargs={"organization_code": self.organization.code})) return super().get(request, *args, **kwargs) diff --git a/rocky/rocky/views/scheduler.py b/rocky/rocky/views/scheduler.py index d968cb1ef76..41464e27bdb 100644 --- a/rocky/rocky/views/scheduler.py +++ b/rocky/rocky/views/scheduler.py @@ -101,7 +101,7 @@ def get_task_details(self, task_id: str) -> Task | None: except SchedulerTaskNotFound: raise Http404() - def create_report_schedule(self, report_recipe: ReportRecipe, deadline_at: str) -> ScheduleResponse | None: + def create_report_schedule(self, report_recipe: ReportRecipe, deadline_at: datetime) -> ScheduleResponse | None: try: report_task = ReportTask( organisation_id=self.organization.code, report_recipe_id=str(report_recipe.recipe_id) @@ -111,7 +111,7 @@ def create_report_schedule(self, report_recipe: ReportRecipe, deadline_at: str) scheduler_id=self.scheduler_id, data=report_task, schedule=report_recipe.cron_expression, - deadline_at=deadline_at, + deadline_at=deadline_at.isoformat(), ) submit_schedule = self.scheduler_client.post_schedule(schedule=schedule_request) From 496414df55a5ea436cde5cabefc7ea637ed21005 Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 28 Nov 2024 16:37:00 +0100 Subject: [PATCH 17/28] fix function input recipe --- rocky/reports/views/base.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index 36faa8f7176..20b63d2f744 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -268,14 +268,10 @@ def get_available_report_types(self) -> tuple[list[dict[str, str]] | dict[str, l def get_observed_at(self): return self.observed_at if self.observed_at < datetime.now(timezone.utc) else datetime.now(timezone.utc) - def is_scheduled_report(self) -> bool: - recurrence_choice = self.request.POST.get("choose_recurrence", "once") - return recurrence_choice == "repeat" - def is_single_report(self) -> bool: return len(self.get_report_type_ids()) == 1 - def get_query(self): + def get_input_recipe(self): object_selection = self.request.POST.get("object_selection", "") query = {} @@ -288,7 +284,11 @@ def get_query(self): "order_by": self.order_by, "asc_desc": self.sorting_order, } - return query + + if not query: + return {"input_oois": self.get_ooi_pks()} + + return {"query": query} def create_report_recipe( self, @@ -296,20 +296,12 @@ def create_report_recipe( subreport_name_format: str | None, parent_report_type: str | None, schedule: str | None, - query: dict[str, Any] | None, ) -> ReportRecipe: - input_recipe: dict[str, Any] = {} - - if query: - input_recipe = {"query": query} - else: - input_recipe = {"input_oois": self.get_ooi_pks()} - report_recipe = ReportRecipe( recipe_id=uuid4(), report_name_format=report_name_format, subreport_name_format=subreport_name_format, - input_recipe=input_recipe, + input_recipe=self.get_input_recipe(), parent_report_type=parent_report_type, report_types=self.get_report_type_ids(), cron_expression=schedule, @@ -553,7 +545,7 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: subreport_name_format = request.POST.get("subreport_name_format") report_recipe = self.create_report_recipe( - parent_report_name_format, subreport_name_format, parent_report_type, schedule, self.get_query() + parent_report_name_format, subreport_name_format, parent_report_type, schedule ) self.create_report_schedule(report_recipe, start_datetime) From ca7c88a5da1439b1a46ea10336e8fa10b350062c Mon Sep 17 00:00:00 2001 From: Rieven Date: Mon, 2 Dec 2024 09:46:21 +0100 Subject: [PATCH 18/28] fix test --- .../report_overview/scheduled_reports_table.html | 10 +++++----- rocky/tests/reports/test_aggregate_report_flow.py | 15 +++++++++------ rocky/tests/reports/test_generate_report_flow.py | 11 ++++++----- rocky/tests/reports/test_multi_report_flow.py | 8 +++++--- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/rocky/reports/templates/report_overview/scheduled_reports_table.html b/rocky/reports/templates/report_overview/scheduled_reports_table.html index af8beaf0ebe..4daef65318a 100644 --- a/rocky/reports/templates/report_overview/scheduled_reports_table.html +++ b/rocky/reports/templates/report_overview/scheduled_reports_table.html @@ -30,8 +30,8 @@
    {% if schedule.recipe.parent_report_type == "aggregate-organisation-report" %} -
  • - {{ schedule.recipe.parent_report_type|get_report_type_name }} +
  • + {{ schedule.recipe.parent_report_type|get_report_type_name }}
    • {% else %} {% for report_type in schedule.recipe.report_types %} @@ -49,8 +49,8 @@ {% endif %}
- {{ schedule.deadline_at }} - + {{ schedule.deadline_at }} + {% if schedule.cron is None %} {% translate "Once" %} {% else %} @@ -64,7 +64,7 @@ {% translate "Disabled" %} {% endif %} - + + {% translate "Skip onboarding" %} + + {% endblock content %} diff --git a/rocky/onboarding/views.py b/rocky/onboarding/views.py index 783fd0599e2..8480c13ef38 100644 --- a/rocky/onboarding/views.py +++ b/rocky/onboarding/views.py @@ -18,13 +18,12 @@ from katalogus.client import Plugin from reports.report_types.definitions import ReportPlugins from reports.report_types.dns_report.report import DNSReport -from reports.views.base import get_selection -from reports.views.generate_report import SaveGenerateReportMixin +from reports.views.base import BaseReportView, get_selection from tools.models import GROUP_ADMIN, GROUP_CLIENT, GROUP_REDTEAM, Organization, OrganizationMember from tools.ooi_helpers import get_or_create_ooi from tools.view_helpers import Breadcrumb -from octopoes.models import OOI +from octopoes.models import OOI, Reference from octopoes.models.ooi.dns.zone import Hostname from octopoes.models.ooi.network import Network from octopoes.models.ooi.web import URL @@ -42,6 +41,7 @@ from rocky.messaging import clearance_level_warning_dns_report from rocky.views.indemnification_add import IndemnificationAddView from rocky.views.ooi_view import SingleOOIMixin, SingleOOITreeMixin +from rocky.views.scheduler import SchedulerView User = get_user_model() @@ -310,9 +310,11 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]: class OnboardingSetupScanOOIDetailView( OrganizationPermissionRequiredMixin, - SingleOOITreeMixin, IntroductionStepsMixin, OnboardingBreadcrumbsMixin, + SingleOOIMixin, + BaseReportView, + SchedulerView, TemplateView, ): """ @@ -322,6 +324,33 @@ class OnboardingSetupScanOOIDetailView( template_name = "step_3c_setup_scan_ooi_detail.html" current_step = 3 permission_required = "tools.can_scan_organization" + task_type = "report" + + def setup(self, request, *args, **kwargs): + super().setup(request, *args, **kwargs) + self.selected_oois = self.get_ooi_pks() + self.selected_report_types = self.get_report_type_ids() + + def post(self, request, *args, **kwargs): + parent_report_name_format, subreport_name_format = self.get_initial_report_names() + parent_report_type = self.get_parent_report_type() + report_recipe = self.create_report_recipe( + parent_report_name_format, subreport_name_format, parent_report_type, None + ) + + self.create_report_schedule(report_recipe, datetime.now(timezone.utc)) + return redirect( + reverse("step_report", kwargs={"organization_code": self.organization.code}) + + get_selection(self.request, {"recipe_id": report_recipe.primary_key}) + ) + + def get_ooi_pks(self) -> list[str]: + ooi = self.get_ooi(self.request.GET.get("ooi", "")) + hostname_ooi = [Hostname(name=ooi.web_url.tokenized["netloc"]["name"], network=ooi.network)] + return [hostname_ooi[0].primary_key] + + def get_report_type_ids(self) -> list[str]: + return [self.request.GET.get("report_type", "")] def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -330,7 +359,7 @@ def get_context_data(self, **kwargs): class OnboardingReportView( - OrganizationPermissionRequiredMixin, SaveGenerateReportMixin, IntroductionStepsMixin, SingleOOIMixin, TemplateView + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, SingleOOIMixin, BaseReportView, TemplateView ): """ 10. The user already started the scan and is now waiting till scans are finished to generate the report. @@ -341,32 +370,25 @@ class OnboardingReportView( current_step = 4 permission_required = "tools.can_scan_organization" - def setup(self, request, *args, **kwargs): - super().setup(request, *args, **kwargs) - self.selected_oois = self.get_ooi_pks() - self.selected_report_types = self.get_report_type_ids() - - def get_ooi_pks(self) -> list[str]: - ooi = self.get_ooi(self.request.GET.get("ooi", "")) - hostname_ooi = [Hostname(name=ooi.web_url.tokenized["netloc"]["name"], network=ooi.network)] - return [hostname_ooi[0].primary_key] - - def get_report_type_ids(self) -> list[str]: - return [self.request.GET.get("report_type", "")] - def post(self, request, *args, **kwargs): self.set_member_onboarded() - report_ooi = self.save_report([("Onboarding Report", "Onboarding Report")]) + recipe_id = request.GET.get("recipe_id", "") - if report_ooi: - return redirect( - reverse("view_report", kwargs={"organization_code": self.organization.code}) - + "?" - + urlencode({"report_id": report_ooi.reference}) + if recipe_id: + reports = self.octopoes_api_connector.query( + "ReportRecipe. dict[str, Any]: } } + def get_initial_report_names(self) -> tuple[str, str]: + oois = self.get_total_oois() + is_single_report = self.is_single_report() + + if oois == 1 and is_single_report: + return ("${report_type} for ${ooi}", "") + if oois == 1 and not is_single_report: + return ("${report_type} for ${ooi}", "${report_type} for ${ooi}") + if oois > 1: + return ("${report_type} for ${oois_count} objects", "${report_type} for ${ooi}") + return ("", "") + + def get_parent_report_type(self): + if self.report_type is not None: + return self.report_type.id + if not self.is_single_report(): + return ConcatenatedReport.id + return self.report_type + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -500,18 +519,6 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: return PostRedirect(self.get_previous()) return super().get(request, *args, **kwargs) - def get_initial_report_names(self) -> tuple[str, str]: - oois = self.get_total_oois() - is_single_report = self.is_single_report() - - if oois == 1 and is_single_report: - return ("${report_type} for ${ooi}", "") - if oois == 1 and not is_single_report: - return ("${report_type} for ${ooi}", "${report_type} for ${ooi}") - if oois > 1: - return ("${report_type} for ${oois_count} objects", "${report_type} for ${ooi}") - return ("", "") - def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["initial_report_names"] = self.get_initial_report_names() @@ -524,13 +531,6 @@ class SaveReportView(BaseReportView, SchedulerView, FormView): task_type = "report" form_class = ReportScheduleStartDateForm - def get_parent_report_type(self): - if self.report_type is not None: - return self.report_type.id - if not self.is_single_report(): - return ConcatenatedReport.id - return self.report_type - def form_invalid(self, form): """ We need to overwrite this as FormView renders invalid forms with a get request, From 539be48385b1ffdbe1becfb3ed2fe845037fe5b2 Mon Sep 17 00:00:00 2001 From: Rieven Date: Fri, 10 Jan 2025 11:32:14 +0100 Subject: [PATCH 25/28] fix test --- rocky/onboarding/views.py | 5 ----- .../onboarding/test_onboarding_organization_steps.py | 8 +++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/rocky/onboarding/views.py b/rocky/onboarding/views.py index 8480c13ef38..a6c54a8905f 100644 --- a/rocky/onboarding/views.py +++ b/rocky/onboarding/views.py @@ -326,11 +326,6 @@ class OnboardingSetupScanOOIDetailView( permission_required = "tools.can_scan_organization" task_type = "report" - def setup(self, request, *args, **kwargs): - super().setup(request, *args, **kwargs) - self.selected_oois = self.get_ooi_pks() - self.selected_report_types = self.get_report_type_ids() - def post(self, request, *args, **kwargs): parent_report_name_format, subreport_name_format = self.get_initial_report_names() parent_report_type = self.get_parent_report_type() diff --git a/rocky/tests/onboarding/test_onboarding_organization_steps.py b/rocky/tests/onboarding/test_onboarding_organization_steps.py index bf55d591a2e..ddadb59fa12 100644 --- a/rocky/tests/onboarding/test_onboarding_organization_steps.py +++ b/rocky/tests/onboarding/test_onboarding_organization_steps.py @@ -279,9 +279,15 @@ def test_onboarding_select_plugins_perms(request, member, rf, url): @pytest.mark.parametrize("member", ["superuser_member", "admin_member", "redteam_member", "client_member"]) -def test_onboarding_ooi_detail_scan(request, member, rf, mock_organization_view_octopoes, url): +def test_onboarding_ooi_detail_scan( + request, mocker, member, mock_bytes_client, rf, mock_organization_view_octopoes, url +): member = request.getfixturevalue(member) + mocker.patch("account.mixins.OrganizationView.get_katalogus") + mock_organization_view_octopoes().get.return_value = url + mock_bytes_client().upload_raw.return_value = "raw_id" + response = OnboardingSetupScanOOIDetailView.as_view()( setup_request(rf.get("step_setup_scan_ooi_detail", {"ooi": url.primary_key}), member.user), organization_code=member.organization.code, From 59714b09fb2b5fc8c6e465cb98024f7a43c9be96 Mon Sep 17 00:00:00 2001 From: Rieven Date: Fri, 10 Jan 2025 13:36:06 +0100 Subject: [PATCH 26/28] add more test --- .../test_onboarding_organization_steps.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/rocky/tests/onboarding/test_onboarding_organization_steps.py b/rocky/tests/onboarding/test_onboarding_organization_steps.py index ddadb59fa12..f898b216d0f 100644 --- a/rocky/tests/onboarding/test_onboarding_organization_steps.py +++ b/rocky/tests/onboarding/test_onboarding_organization_steps.py @@ -303,6 +303,29 @@ def test_onboarding_ooi_detail_scan( assertContains(response, "Start scanning") +@pytest.mark.parametrize("member", ["superuser_member", "admin_member", "redteam_member", "client_member"]) +def test_onboarding_ooi_detail_scan_create_report_schedule( + request, mocker, member, mock_bytes_client, rf, mock_organization_view_octopoes, url +): + member = request.getfixturevalue(member) + + mocker.patch("account.mixins.OrganizationView.get_katalogus") + mock_organization_view_octopoes().get.return_value = url + mock_bytes_client().upload_raw.return_value = "raw_id" + + request_url = ( + reverse("step_setup_scan_ooi_detail", kwargs={"organization_code": member.organization.code}) + + f"?report_type=dns-report&ooi={url.primary_key}" + ) + + response = OnboardingSetupScanOOIDetailView.as_view()( + setup_request(rf.post(request_url), member.user), organization_code=member.organization.code + ) + + assert response.status_code == 302 + assert "recipe_id" in response.url + + @pytest.mark.parametrize("member", ["superuser_member", "admin_member", "redteam_member", "client_member"]) def test_onboarding_scanning_boefjes( request, member, rf, mock_organization_view_octopoes, url, mocker, mock_bytes_client From 6eebbff2aaeed92e68b63926aee152582def0228 Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 16 Jan 2025 16:57:36 +0100 Subject: [PATCH 27/28] add check for when scheduler is ready --- rocky/onboarding/views.py | 9 ++++++++- rocky/rocky/scheduler.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/rocky/onboarding/views.py b/rocky/onboarding/views.py index a6c54a8905f..a08b413612f 100644 --- a/rocky/onboarding/views.py +++ b/rocky/onboarding/views.py @@ -39,6 +39,7 @@ ) from rocky.exceptions import RockyError from rocky.messaging import clearance_level_warning_dns_report +from rocky.scheduler import scheduler_client from rocky.views.indemnification_add import IndemnificationAddView from rocky.views.ooi_view import SingleOOIMixin, SingleOOITreeMixin from rocky.views.scheduler import SchedulerView @@ -326,14 +327,20 @@ class OnboardingSetupScanOOIDetailView( permission_required = "tools.can_scan_organization" task_type = "report" + @staticmethod + def is_scheduler_enabled(organization: Organization) -> bool: + scheduler_id = f"report-{organization.code}" + return scheduler_client(organization.code).is_scheduler_ready(scheduler_id) + def post(self, request, *args, **kwargs): parent_report_name_format, subreport_name_format = self.get_initial_report_names() parent_report_type = self.get_parent_report_type() report_recipe = self.create_report_recipe( parent_report_name_format, subreport_name_format, parent_report_type, None ) + if self.is_scheduler_enabled(self.organization): + self.create_report_schedule(report_recipe, datetime.now(timezone.utc)) - self.create_report_schedule(report_recipe, datetime.now(timezone.utc)) return redirect( reverse("step_report", kwargs={"organization_code": self.organization.code}) + get_selection(self.request, {"recipe_id": report_recipe.primary_key}) diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index 3b1e619c832..96340601feb 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -3,6 +3,7 @@ import collections import datetime import logging +import time import uuid from enum import Enum from functools import cached_property @@ -187,6 +188,13 @@ class PaginatedSchedulesResponse(BaseModel): results: list[ScheduleResponse] +class SchedulerResponse(BaseModel): + id: str + enabled: bool + priority_queue: dict[str, Any] + last_activity: str | None + + class LazyTaskList: HARD_LIMIT = 500 @@ -295,6 +303,23 @@ def post_schedule_search(self, filters: dict[str, list[dict[str, str]]]) -> Pagi except ConnectError: raise SchedulerConnectError() + def is_scheduler_ready(self, scheduler_id: str) -> bool: + """Max trials is 100 seconds""" + trials = 0 + interval = 10 # in seconds + while trials < 10: + try: + res = self._client.get(f"/schedulers/{scheduler_id}") + res.raise_for_status() + break + except HTTPStatusError as http_error: + if http_error.response.status_code == codes.NOT_FOUND: + trials += 1 + time.sleep(interval) + continue + raise SchedulerHTTPError() + return SchedulerResponse.model_validate_json(res.content).enabled + def patch_schedule(self, schedule_id: str, params: dict[str, Any]) -> None: try: response = self._client.patch(f"/schedules/{schedule_id}", json=params) From 4fa7934ae8092dda6a3a2c9b13f93088fea2f2f7 Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 16 Jan 2025 18:28:31 +0100 Subject: [PATCH 28/28] fix test --- rocky/tests/onboarding/test_onboarding_organization_steps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rocky/tests/onboarding/test_onboarding_organization_steps.py b/rocky/tests/onboarding/test_onboarding_organization_steps.py index f898b216d0f..d46b8e18475 100644 --- a/rocky/tests/onboarding/test_onboarding_organization_steps.py +++ b/rocky/tests/onboarding/test_onboarding_organization_steps.py @@ -310,6 +310,7 @@ def test_onboarding_ooi_detail_scan_create_report_schedule( member = request.getfixturevalue(member) mocker.patch("account.mixins.OrganizationView.get_katalogus") + mocker.patch("onboarding.views.scheduler_client") mock_organization_view_octopoes().get.return_value = url mock_bytes_client().upload_raw.return_value = "raw_id"