From e8eea8e0e3fe50da46d3ce168e21e51d66f060b2 Mon Sep 17 00:00:00 2001 From: Rieven Date: Mon, 20 Jan 2025 12:14:57 +0100 Subject: [PATCH] Changes to schedule all reports, even for once (#3840) Co-authored-by: Peter-Paul van Gemerden Co-authored-by: Jan Klopper Co-authored-by: JP Bruins Slot Co-authored-by: ammar92 --- octopoes/octopoes/models/ooi/reports.py | 2 +- .../step_3c_setup_scan_ooi_detail.html | 14 +- rocky/onboarding/views.py | 74 ++++--- rocky/reports/forms.py | 26 +-- rocky/reports/runner/report_runner.py | 8 +- rocky/reports/runner/worker.py | 2 +- .../partials/export_report_settings.html | 73 ++++--- .../partials/report_names_header.html | 35 --- .../report_overview/report_history_table.html | 8 +- .../scheduled_reports_table.html | 24 ++- rocky/reports/views/base.py | 203 ++++++------------ rocky/rocky/locale/django.pot | 126 ++++------- rocky/rocky/scheduler.py | 27 ++- rocky/rocky/views/scheduler.py | 19 +- .../test_onboarding_organization_steps.py | 32 ++- .../reports/test_aggregate_report_flow.py | 15 +- .../reports/test_generate_report_flow.py | 11 +- rocky/tests/reports/test_multi_report_flow.py | 8 +- 18 files changed, 319 insertions(+), 388 deletions(-) delete mode 100644 rocky/reports/templates/partials/report_names_header.html diff --git a/octopoes/octopoes/models/ooi/reports.py b/octopoes/octopoes/models/ooi/reports.py index 525ad009b57..e13d45f34d8 100644 --- a/octopoes/octopoes/models/ooi/reports.py +++ b/octopoes/octopoes/models/ooi/reports.py @@ -61,6 +61,6 @@ class ReportRecipe(OOI): parent_report_type: str | None = None report_types: list[str] - cron_expression: str + cron_expression: str | None = None _natural_key_attrs = ["recipe_id"] diff --git a/rocky/onboarding/templates/step_3c_setup_scan_ooi_detail.html b/rocky/onboarding/templates/step_3c_setup_scan_ooi_detail.html index 4575db6899d..c0d589d748c 100644 --- a/rocky/onboarding/templates/step_3c_setup_scan_ooi_detail.html +++ b/rocky/onboarding/templates/step_3c_setup_scan_ooi_detail.html @@ -63,12 +63,14 @@

{% translate "Creating an object" %}

- +
+ {% csrf_token %} +
+ + {% translate "Skip onboarding" %} +
+
{% endblock content %} diff --git a/rocky/onboarding/views.py b/rocky/onboarding/views.py index 783fd0599e2..a08b413612f 100644 --- a/rocky/onboarding/views.py +++ b/rocky/onboarding/views.py @@ -18,13 +18,12 @@ from katalogus.client import Plugin from reports.report_types.definitions import ReportPlugins from reports.report_types.dns_report.report import DNSReport -from reports.views.base import get_selection -from reports.views.generate_report import SaveGenerateReportMixin +from reports.views.base import BaseReportView, get_selection from tools.models import GROUP_ADMIN, GROUP_CLIENT, GROUP_REDTEAM, Organization, OrganizationMember from tools.ooi_helpers import get_or_create_ooi from tools.view_helpers import Breadcrumb -from octopoes.models import OOI +from octopoes.models import OOI, Reference from octopoes.models.ooi.dns.zone import Hostname from octopoes.models.ooi.network import Network from octopoes.models.ooi.web import URL @@ -40,8 +39,10 @@ ) from rocky.exceptions import RockyError from rocky.messaging import clearance_level_warning_dns_report +from rocky.scheduler import scheduler_client from rocky.views.indemnification_add import IndemnificationAddView from rocky.views.ooi_view import SingleOOIMixin, SingleOOITreeMixin +from rocky.views.scheduler import SchedulerView User = get_user_model() @@ -310,9 +311,11 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]: class OnboardingSetupScanOOIDetailView( OrganizationPermissionRequiredMixin, - SingleOOITreeMixin, IntroductionStepsMixin, OnboardingBreadcrumbsMixin, + SingleOOIMixin, + BaseReportView, + SchedulerView, TemplateView, ): """ @@ -322,6 +325,34 @@ class OnboardingSetupScanOOIDetailView( template_name = "step_3c_setup_scan_ooi_detail.html" current_step = 3 permission_required = "tools.can_scan_organization" + task_type = "report" + + @staticmethod + def is_scheduler_enabled(organization: Organization) -> bool: + scheduler_id = f"report-{organization.code}" + return scheduler_client(organization.code).is_scheduler_ready(scheduler_id) + + def post(self, request, *args, **kwargs): + parent_report_name_format, subreport_name_format = self.get_initial_report_names() + parent_report_type = self.get_parent_report_type() + report_recipe = self.create_report_recipe( + parent_report_name_format, subreport_name_format, parent_report_type, None + ) + if self.is_scheduler_enabled(self.organization): + self.create_report_schedule(report_recipe, datetime.now(timezone.utc)) + + return redirect( + reverse("step_report", kwargs={"organization_code": self.organization.code}) + + get_selection(self.request, {"recipe_id": report_recipe.primary_key}) + ) + + def get_ooi_pks(self) -> list[str]: + ooi = self.get_ooi(self.request.GET.get("ooi", "")) + hostname_ooi = [Hostname(name=ooi.web_url.tokenized["netloc"]["name"], network=ooi.network)] + return [hostname_ooi[0].primary_key] + + def get_report_type_ids(self) -> list[str]: + return [self.request.GET.get("report_type", "")] def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -330,7 +361,7 @@ def get_context_data(self, **kwargs): class OnboardingReportView( - OrganizationPermissionRequiredMixin, SaveGenerateReportMixin, IntroductionStepsMixin, SingleOOIMixin, TemplateView + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, SingleOOIMixin, BaseReportView, TemplateView ): """ 10. The user already started the scan and is now waiting till scans are finished to generate the report. @@ -341,32 +372,25 @@ class OnboardingReportView( current_step = 4 permission_required = "tools.can_scan_organization" - def setup(self, request, *args, **kwargs): - super().setup(request, *args, **kwargs) - self.selected_oois = self.get_ooi_pks() - self.selected_report_types = self.get_report_type_ids() - - def get_ooi_pks(self) -> list[str]: - ooi = self.get_ooi(self.request.GET.get("ooi", "")) - hostname_ooi = [Hostname(name=ooi.web_url.tokenized["netloc"]["name"], network=ooi.network)] - return [hostname_ooi[0].primary_key] - - def get_report_type_ids(self) -> list[str]: - return [self.request.GET.get("report_type", "")] - def post(self, request, *args, **kwargs): self.set_member_onboarded() - report_ooi = self.save_report([("Onboarding Report", "Onboarding Report")]) + recipe_id = request.GET.get("recipe_id", "") - if report_ooi: - return redirect( - reverse("view_report", kwargs={"organization_code": self.organization.code}) - + "?" - + urlencode({"report_id": report_ooi.reference}) + if recipe_id: + reports = self.octopoes_api_connector.query( + "ReportRecipe. None: for ooi in data: ooi_human_readable = Reference.from_str(ooi).human_readable subreport_name = now.strftime( - Template(recipe.subreport_name_format).safe_substitute( - ooi=ooi_human_readable, report_type=str(report_type.name) - ) + Template( + "" if recipe.subreport_name_format is None else recipe.subreport_name_format + ).safe_substitute(ooi=ooi_human_readable, report_type=str(report_type.name)) ) subreport_names.append((subreport_name, subreport_name)) @@ -112,7 +112,7 @@ def run(self, report_task: ReportTask) -> None: ) if "${ooi}" in parent_report_name and oois_count == 1: - parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi[0].human_readable) + parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable) save_report_data( self.bytes_client, diff --git a/rocky/reports/runner/worker.py b/rocky/reports/runner/worker.py index 3a8181979b4..ecdff5d3261 100644 --- a/rocky/reports/runner/worker.py +++ b/rocky/reports/runner/worker.py @@ -243,7 +243,7 @@ def _start_working( finally: try: # The docker runner could have handled this already - if scheduler.get_task_details(p_item.id).status == TaskStatus.RUNNING: + if scheduler.get_task_details(str(p_item.id)).status == TaskStatus.RUNNING: scheduler.patch_task(p_item.id, status) # Note that implicitly, we have p_item.id == task_id logger.info("Set status to %s in the scheduler for task[id=%s]", status, p_item.id) except HTTPError: diff --git a/rocky/reports/templates/partials/export_report_settings.html b/rocky/reports/templates/partials/export_report_settings.html index cc509dfb794..cf43d37323b 100644 --- a/rocky/reports/templates/partials/export_report_settings.html +++ b/rocky/reports/templates/partials/export_report_settings.html @@ -15,53 +15,58 @@

{% translate "Report schedule" %}

or monthly. If you need the report just for a single occasion, select the one-time option. {% endblocktranslate %}

-
+ {% csrf_token %} {% include "forms/report_form_fields.html" %} - {% include "partials/form/fieldset.html" with fields=report_schedule_form_recurrence_choice %} + {% include "partials/form/fieldset.html" with fields=report_schedule_form_start_date fieldset_parent_class="column-2" %} - {% if is_scheduled_report %} +
+

{% translate "Report name" %}

{% blocktranslate trimmed %} - Please choose a start date, time and recurrence for scheduling your report(s). - If you select a date on the 28th-31st of the month, it will always be scheduled on the last day of the month. + Define a custom name format for your report(s). This format will be applied to all generated + (sub)reports. {% endblocktranslate %}

{% blocktranslate trimmed %} - The date you select will be the reference date for the data set for your report. - Please allow for up to 24 hours for your report to be ready. + To make the report names more descriptive, you can include placeholders for the + object name, the report type and/or the reference date. For subreports and reports over a single object, + use the placeholder "${ooi}" for the object name, "${report_type}" for the report type and use a + Python strftime code for the reference + date. For reports over multiple objects, use "${oois_count}" for the number of objects in the report. {% endblocktranslate %}

-
- {% include "partials/form/fieldset.html" with fields=report_schedule_form_start_date %} - +

+ {% blocktranslate trimmed %} + For example, the format "${report_type} for ${ooi} at %x" could generate: + "DNS Report for example.com at 01/01/25". + {% endblocktranslate %} +

+
+
+
+ +
+
+ {% if initial_report_names.1 %} +
+
+ + +
+
{% endif %} - -
- {% csrf_token %} - {% include "forms/report_form_fields.html" %} - - {% if not is_scheduled_report %} - {% include "partials/report_names_header.html" %} - {% include "partials/report_names_form.html" %} - - - {% else %} - {% include "partials/report_names_header.html" with recurrence=True %} - {% include "partials/form/fieldset.html" with fields=report_parent_name_form %} - - {% if reports|length > 1 %} - {% include "partials/form/fieldset.html" with fields=report_child_name_form %} - - {% endif %} - - {% endif %} +
{% else %} {% include "partials/return_button.html" with btn_text="Go back" %} diff --git a/rocky/reports/templates/partials/report_names_header.html b/rocky/reports/templates/partials/report_names_header.html deleted file mode 100644 index 1616fc02f25..00000000000 --- a/rocky/reports/templates/partials/report_names_header.html +++ /dev/null @@ -1,35 +0,0 @@ -{% load i18n %} -{% load static %} - -

{% translate "Report name" %}

-{% if recurrence %} -

- {% blocktranslate trimmed %} - Define a custom name format for your report(s). This format will be applied to all generated - (sub)reports. - {% endblocktranslate %} -

-

- {% blocktranslate trimmed %} - To make the report names more descriptive, you can include placeholders for the - object name, the report type and/or the reference date. For subreports and reports over a single object, - use the placeholder "${ooi}" for the object name, "${report_type}" for the report type and use a - Python strftime code for the reference - date. For reports over multiple objects, use "${oois_count}" for the number of objects in the report. - {% endblocktranslate %} -

-

- {% blocktranslate trimmed %} - For example, the format "${report_type} for ${ooi} at %x" could generate: - "DNS Report for example.com at 01/01/25". - {% endblocktranslate %} -

-{% else %} -

- {% blocktranslate trimmed %} - Give your report(s) a custom name and optionally add the reports' reference date - to the name. To do so you can select a standard option or use a Python - strftime code in the report name. - {% endblocktranslate %} -

-{% endif %} diff --git a/rocky/reports/templates/report_overview/report_history_table.html b/rocky/reports/templates/report_overview/report_history_table.html index 820bca9dd41..95825463d50 100644 --- a/rocky/reports/templates/report_overview/report_history_table.html +++ b/rocky/reports/templates/report_overview/report_history_table.html @@ -134,8 +134,8 @@ {{ report.parent_report.observed_at|date }} {{ report.parent_report.date_generated }} - {% if report.children_reports %} - + + {% if report.children_reports %} - - {% endif %} + {% endif %} + {% if report.children_reports %} diff --git a/rocky/reports/templates/report_overview/scheduled_reports_table.html b/rocky/reports/templates/report_overview/scheduled_reports_table.html index 8a241a53064..0adea129a2d 100644 --- a/rocky/reports/templates/report_overview/scheduled_reports_table.html +++ b/rocky/reports/templates/report_overview/scheduled_reports_table.html @@ -27,32 +27,36 @@ {% if schedule.recipe %} {{ schedule.recipe.report_name_format }} - +
    {% if schedule.recipe.parent_report_type == "aggregate-organisation-report" %} -
  • - {{ schedule.recipe.parent_report_type|get_report_type_name }} +
  • + {{ schedule.recipe.parent_report_type|get_report_type_name }}
  • {% else %} {% for report_type in schedule.recipe.report_types %} {% if forloop.counter0 < 2 %} -
  • {{ report_type|get_report_type_name }}
  • +
  • + {{ report_type|get_report_type_name }} +
  • {% endif %} {% if forloop.counter0 == 2 %} -
  • + {{ schedule.recipe.report_types|length|add:"-2" }}
  • +
  • + + {{ schedule.recipe.report_types|length|add:"-2" }} +
  • {% endif %} {% endfor %} {% endif %}
+ {{ schedule.deadline_at }} - {% if not schedule.enabled %} - - + {% if schedule.cron is None %} + {% translate "Once" %} {% else %} - {{ schedule.deadline_at }} + {{ schedule.cron|get_cron_description }} {% endif %} - {{ schedule.cron|get_cron_description }} {% if schedule.enabled %} {% translate "Enabled" %} @@ -60,7 +64,7 @@ {% translate "Disabled" %} {% endif %} - +