Merge branch 'notes-checklists' into 'main'

Import/export notes

See merge request reportcreator/reportcreator!428

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Next
+* Define initial note structure for projects in designs
+* Allow exporting and importing notes
+
+
 ## v2024.8 - 2024-01-23
 * Diff-view for version history
 * Set form fields readonly instead of disabled

api/src/reportcreator_api/archive/import_export/import_export.py

@@ -4,17 +4,18 @@
 import logging
 import tarfile
 from pathlib import Path
-from typing import Iterable, Type
+from typing import Iterable, Optional, Type, Union
 from django.conf import settings
-from django.utils import timezone
 from rest_framework import serializers
 from django.db import transaction
 from django.db.models import prefetch_related_objects, Prefetch
 from django.core.serializers.json import DjangoJSONEncoder
 
-from reportcreator_api.archive.import_export.serializers import PentestProjectExportImportSerializer, ProjectTypeExportImportSerializer, \
+from reportcreator_api.archive.import_export.serializers import NotesExportImportSerializer, PentestProjectExportImportSerializer, ProjectTypeExportImportSerializer, \
     FindingTemplateImportSerializerV1, FindingTemplateExportImportSerializerV2
-from reportcreator_api.pentests.models import FindingTemplate, ProjectNotebookPage, PentestFinding, PentestProject, ProjectMemberInfo, ProjectType, ReportSection
+from reportcreator_api.pentests.models import FindingTemplate, ProjectNotebookPage, PentestFinding, PentestProject, ProjectMemberInfo, ProjectType, ReportSection, NotebookPageMixin
+from reportcreator_api.pentests.models.notes import UserNotebookPage
+from reportcreator_api.users.models import PentestUser
 from reportcreator_api.utils.history import history_context
 
 
@@ -165,9 +166,13 @@ def import_archive(archive_file, serializer_classes: list[Type[serializers.Seria
                             error = ex
                 if error:
                     raise error
-                obj = serializer.perform_import()
-                log.info(f'Imported object {obj=} {obj.id=}')
-                imported_objects.append(obj)
+                imported_obj = serializer.perform_import()
+                for obj in imported_obj if isinstance(imported_obj, list) else [imported_obj]:
+                    log.info(f'Imported object {obj=} {obj.id}')
+                if isinstance(imported_obj, list):
+                    imported_objects.extend(imported_obj)
+                else:
+                    imported_objects.append(imported_obj)
 
             return imported_objects
     except Exception as ex:
@@ -209,6 +214,31 @@ def export_projects(data: Iterable[PentestProject], export_all=False):
         'add_design_notice_file': True,
     })
 
+def export_notes(project_or_user: PentestProject|PentestUser, notes: Optional[Iterable[ProjectNotebookPage|UserNotebookPage]] = None):
+    notes_qs = project_or_user.notes \
+        .select_related('parent')
+    if notes is not None:
+        # Only export sepcified notes and their children
+        def get_children_recursive(note, all_notes):
+            out = [note]
+            for n in all_notes:
+                if n.parent_id == note.id:
+                    out.extend(get_children_recursive(n, all_notes))
+            return out
+
+        all_notes = list(project_or_user.notes.all())
+        export_notes = []
+        for n in notes:
+            if (isinstance(project_or_user, PentestProject) and getattr(n, 'project', None) != project_or_user) or \
+                (isinstance(project_or_user, PentestUser) and getattr(n, 'user', None) != project_or_user):
+                raise serializers.ValidationError(f'Note {n.id} does not belong to {project_or_user}')
+            export_notes.extend(get_children_recursive(n, all_notes))
+        notes_qs = notes_qs \
+            .filter(id__in=map(lambda n: n.id, export_notes))
+
+    prefetch_related_objects([project_or_user], Prefetch('notes', queryset=notes_qs))
+    return export_archive_iter([project_or_user], serializer_class=NotesExportImportSerializer)
+
 
 def import_templates(archive_file):
     return import_archive(archive_file, serializer_classes=[FindingTemplateExportImportSerializerV2, FindingTemplateImportSerializerV1])
@@ -219,3 +249,8 @@ def import_project_types(archive_file):
 def import_projects(archive_file):
     return import_archive(archive_file, serializer_classes=[PentestProjectExportImportSerializer])
 
+def import_notes(archive_file, context):
+    if not context.get('project') and not context.get('user'):
+        raise ValueError('Either project or user must be provided')
+    return import_archive(archive_file, serializer_classes=[NotesExportImportSerializer], context=context)
+

api/src/reportcreator_api/archive/import_export/serializers.py

@@ -1,13 +1,13 @@
 from typing import Iterable
-from django.conf import settings
+from uuid import uuid4
 from django.core.files import File
 from django.core.exceptions import ObjectDoesNotExist
 from rest_framework import serializers
 from reportcreator_api.pentests.customfields.utils import HandleUndefinedFieldsOptions, ensure_defined_structure
 
 from reportcreator_api.pentests.models import FindingTemplate, ProjectNotebookPage, PentestFinding, PentestProject, ProjectType, ReportSection, \
     SourceEnum, UploadedAsset, UploadedImage, UploadedFileBase, ProjectMemberInfo, UploadedProjectFile, Language, ReviewStatus, \
-    FindingTemplateTranslation, UploadedTemplateImage, ProjectTypeStatus
+    FindingTemplateTranslation, UploadedTemplateImage, ProjectTypeStatus, UserNotebookPage, UploadedUserNotebookImage, UploadedUserNotebookFile
 from reportcreator_api.pentests.serializers import ProjectMemberInfoSerializer
 from reportcreator_api.users.models import PentestUser
 from reportcreator_api.users.serializers import RelatedUserSerializer
@@ -130,12 +130,12 @@ def extract_file(self, name):
         return self.context['archive'].extractfile(self.child.get_path_in_archive(name))
 
     def create(self, validated_data):
-        child_model_class = self.child.Meta.model
+        child_model_class = self.child.get_model_class()
         objs = [
             child_model_class(**attrs | {
                 'name_hash': UploadedFileBase.hash_name(attrs['name']),
                 'file': File(
-                    file=self.extract_file(attrs['name']), 
+                    file=self.extract_file(attrs.pop('name_internal', None) or attrs['name']), 
                     name=attrs['name']), 
                 'linked_object': self.child.get_linked_object(),
         }) for attrs in validated_data]
@@ -148,9 +148,15 @@ def create(self, validated_data):
 class FileExportImportSerializer(ExportImportSerializer):
     class Meta:
         fields = ['id', 'created', 'updated', 'name']
-        extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False, 'required': False}}
+        extra_kwargs = {
+            'id': {'read_only': True}, 
+            'created': {'read_only': False, 'required': False}
+        }
         list_serializer_class = FileListExportImportSerializer
 
+    def get_model_class(self):
+        return self.Meta.model
+
     def validate_name(self, name):
         if '/' in name or '\\' in name or '\x00' in name:
             raise serializers.ValidationError(f'Invalid filename: {name}')
@@ -405,35 +411,75 @@ def update(self, instance, validated_data):
         return out
 
 
-class ProjectNotebookPageExportImportSerializer(ExportImportSerializer):
+class NotebookPageExportImportSerializer(ExportImportSerializer):
     id = serializers.UUIDField(source='note_id')
     parent = serializers.UUIDField(source='parent.note_id', allow_null=True, required=False)
 
     class Meta:
         model = ProjectNotebookPage
         fields = [
             'id', 'created', 'updated',
-            'title', 'text', 'checked', 'icon_emoji', 'assignee',
+            'title', 'text', 'checked', 'icon_emoji',
             'order', 'parent',
         ]
         extra_kwargs = {
             'created': {'read_only': False, 'required': False},
             'icon_emoji': {'required': False},
+        }
+
+
+class ProjectNotebookPageExportImportSerializer(NotebookPageExportImportSerializer):
+    class Meta(NotebookPageExportImportSerializer.Meta):
+        fields = NotebookPageExportImportSerializer.Meta.fields + ['assignee']
+        extra_kwargs = NotebookPageExportImportSerializer.Meta.extra_kwargs | {
             'assignee': {'required': False}
         }
 
 
-class ProjectNotebookPageListExportImportSerializer(serializers.ListSerializer):
-    child = ProjectNotebookPageExportImportSerializer()
+class NotebookPageListExportImportSerializer(serializers.ListSerializer):
+    @property
+    def linked_object(self):
+        if project := self.context.get('project'):
+            return project
+        elif user := self.context.get('user'):
+            return user
+        else:
+            raise serializers.ValidationError('Missing project or user reference')
+
+    def create_instance(self, validated_data):
+        note_data = omit_keys(validated_data, ['parent'])
+        if isinstance(self.linked_object, PentestProject):
+            return ProjectNotebookPage(project=self.linked_object, **note_data)
+        else:
+            return UserNotebookPage(user=self.linked_object, **note_data)
 
     def create(self, validated_data):
-        instances = [ProjectNotebookPage(project=self.context['project'], **omit_keys(d, ['parent'])) for d in validated_data]
+        # Check for note ID collisions and update note_id on collision
+        existing_instances = list(self.linked_object.notes.all())
+        existing_ids = set(map(lambda n: n.note_id, existing_instances))
+        for n in validated_data:
+            if n['note_id'] in existing_ids:
+                old_id = n['note_id']
+                new_id = uuid4()
+                n['note_id'] = new_id
+                for cn in validated_data:
+                    if cn.get('parent', {}).get('note_id') == old_id:
+                        cn['parent']['note_id'] = new_id
+
+        # Create instances
+        instances = [self.create_instance(d) for d in validated_data]
         for i, d in zip(instances, validated_data):
             if d.get('parent'):
                 i.parent = next(filter(lambda e: e.note_id == d.get('parent', {}).get('note_id'), instances), None)
-
         ProjectNotebookPage.objects.check_parent_and_order(instances)
-        bulk_create_with_history(ProjectNotebookPage, instances)
+
+        # Update order to new top-level notes: append to end after existing notes
+        existing_toplevel_count = len([n for n in existing_instances if not n.parent])
+        for n in instances:
+            if not n.parent_id:
+                n.order += existing_toplevel_count
+
+        bulk_create_with_history(ProjectNotebookPage if isinstance(self.linked_object, PentestProject) else UserNotebookPage, instances)
         return instances
 
 
@@ -445,7 +491,7 @@ class PentestProjectExportImportSerializer(ExportImportSerializer):
     report_data = serializers.DictField(source='data_all')
     sections = ReportSectionExportImportSerializer(many=True)
     findings = PentestFindingExportImportSerializer(many=True)
-    notes = ProjectNotebookPageListExportImportSerializer(required=False)
+    notes = NotebookPageListExportImportSerializer(child=ProjectNotebookPageExportImportSerializer(), required=False)
     images = UploadedImageExportImportSerializer(many=True)
     files = UploadedProjectFileExportImportSerializer(many=True, required=False)
 
@@ -533,3 +579,116 @@ def create(self, validated_data):
 
         return project
 
+
+class NotesImageExportImportSerializer(FileExportImportSerializer):
+    class Meta(FileExportImportSerializer.Meta):
+        model = UploadedImage
+
+    def get_model_class(self):
+        return UploadedImage if isinstance(self.get_linked_object(), PentestProject) else UploadedUserNotebookImage
+
+    def get_linked_object(self):
+        if project := self.context.get('project'):
+            return project
+        elif user := self.context.get('user'):
+            return user
+        else:
+            raise serializers.ValidationError('Missing project or user reference')
+
+    def get_path_in_archive(self, name):
+        return str(self.context.get('import_id') or self.get_linked_object().id) + '-images/' + name
+
+    def is_file_referenced(self, f):
+        if isinstance(self.get_linked_object(), PentestProject):
+            return self.get_linked_object().is_file_referenced(f, findings=False, sections=False, notes=True)
+        else:
+            return self.get_linked_object().is_file_referenced(f)
+
+
+class NotesFileExportImportSerializer(FileExportImportSerializer):
+    class Meta(FileExportImportSerializer.Meta):
+        model = UploadedProjectFile
+
+    def get_model_class(self):
+        return UploadedProjectFile if isinstance(self.get_linked_object(), PentestProject) else UploadedUserNotebookFile
+
+    def get_linked_object(self):
+        if project := self.context.get('project'):
+            return project
+        elif user := self.context.get('user'):
+            return user
+        else:
+            raise serializers.ValidationError('Missing project or user reference')
+
+    def get_path_in_archive(self, name):
+        return str(self.context.get('import_id') or self.get_linked_object().id) + '-files/' + name
+
+
+class NotesExportImportSerializer(ExportImportSerializer):
+    format = FormatField('notes/v1')
+    id = serializers.UUIDField()
+    notes = NotebookPageListExportImportSerializer(child=NotebookPageExportImportSerializer())
+    images = FileListExportImportSerializer(child=NotesImageExportImportSerializer(), required=False)
+    files = FileListExportImportSerializer(child=NotesFileExportImportSerializer(), required=False)
+
+    class Meta:
+        fields = ['format', 'id', 'notes', 'images', 'files']
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if isinstance(self.instance, PentestProject):
+            self.context['project'] = self.instance
+        elif isinstance(self.instance, PentestUser):
+            self.context['user'] = self.instance
+        self.Meta.model = PentestProject if self.context.get('project') else PentestUser
+
+    def export(self):
+        out = super().export()
+        # Set parent_id = None for exported child-notes
+        exported_ids = set(map(lambda n: n['id'], out['notes']))
+        for n in out['notes']:
+            if n['parent'] and n['parent'] not in exported_ids:
+                n['parent'] = None
+        return out
+
+    def export_files(self) -> Iterable[tuple[str, File]]:
+        imgf = self.fields['images']
+        imgf.instance = list(imgf.get_attribute(self.instance).all())
+        yield from imgf.export_files()
+
+        ff = self.fields['files']
+        ff.instance = list(ff.get_attribute(self.instance).all())
+        yield from ff.export_files()
+
+    def create(self, validated_data):
+        # Check for file name collisions and rename files and update references
+        linked_object = self.context.get('project') or self.context.get('user')
+        existing_images = set(map(lambda i: i.name, linked_object.images.all()))
+        for ii in validated_data['images']:
+            i_name = ii['name']
+            while ii['name'] in existing_images:
+                ii['name'] = UploadedImage.objects.randomize_name(i_name)
+                ii['name_internal'] = i_name
+            if i_name != ii['name']:
+                for n in validated_data['notes']:
+                    n['text'] = n['text'].replace(f'/images/name/{i_name}', f'/images/name/{ii["name"]}')
+
+        existing_files = set(map(lambda f: f.name, linked_object.files.all()))
+        for fi in validated_data['files']:
+            f_name = fi['name']
+            while fi['name'] in existing_files:
+                fi['name'] = UploadedProjectFile.objects.randomize_name(f_name)
+                fi['name_internal'] = f_name
+            if f_name != fi['name']:
+                for n in validated_data['notes']:
+                    n['text'] = n['text'].replace(f'/files/name/{f_name}', f'/files/name/{fi["name"]}')
+
+        # Import notes
+        notes = self.fields['notes'].create(validated_data['notes'])
+
+        # Import images and files
+        self.context.update({'import_id': validated_data['id']})
+        self.fields['images'].create(validated_data.get('images', []))
+        self.fields['files'].create(validated_data.get('files', []))
+
+        return notes

api/src/reportcreator_api/pentests/models/__init__.py

@@ -2,7 +2,7 @@
 from .template import FindingTemplate, FindingTemplateTranslation
 from .project import ProjectType, ProjectTypeScope, ProjectTypeStatus, \
     PentestProject, ProjectMemberInfo, ProjectMemberRole, ReportSection, PentestFinding
-from .notes import ProjectNotebookPage, UserNotebookPage
+from .notes import ProjectNotebookPage, UserNotebookPage, NotebookPageMixin
 from .archive import ArchivedProject, UserPublicKey, ArchivedProjectKeyPart, ArchivedProjectPublicKeyEncryptedKeyPart
 from .files import UploadedFileBase, UploadedAsset, UploadedImage, UploadedProjectFile, UploadedTemplateImage, \
     UploadedUserNotebookImage, UploadedUserNotebookFile
@@ -12,7 +12,7 @@
     'FindingTemplate', 'FindingTemplateTranslation',
     'ProjectType', 'ProjectTypeScope', 'ProjectTypeStatus', 
     'PentestProject', 'ProjectMemberInfo', 'ProjectMemberRole', 'ReportSection', 'PentestFinding',
-    'ProjectNotebookPage', 'UserNotebookPage',
+    'ProjectNotebookPage', 'UserNotebookPage', 'NotebookPageMixin',
     'ArchivedProject', 'UserPublicKey', 'ArchivedProjectKeyPart', 'ArchivedProjectPublicKeyEncryptedKeyPart',
     'UploadedFileBase', 'UploadedAsset', 'UploadedImage', 'UploadedProjectFile', 'UploadedTemplateImage',
     'UploadedUserNotebookImage', 'UploadedUserNotebookFile',

api/src/reportcreator_api/pentests/models/notes.py

@@ -44,3 +44,4 @@ class UserNotebookPage(NotebookPageMixin, BaseModel):
 
     class Meta:
         unique_together = [('user', 'note_id')]
+

api/src/reportcreator_api/pentests/permissions.py

@@ -95,7 +95,7 @@ def has_object_permission(self, request, view, obj):
 
 class ProjectSubresourcePermissions(permissions.BasePermission):
     def has_permission(self, request, view):
-        if request.method in permissions.SAFE_METHODS or view.action in ['export_pdf']:
+        if request.method in permissions.SAFE_METHODS or view.action in ['export', 'export_all', 'export_pdf']:
             return True
         return not view.get_project().readonly