diff --git a/django/cantusdb_project/main_app/admin/source.py b/django/cantusdb_project/main_app/admin/source.py index 0bc8d6c4f..79818d5d2 100644 --- a/django/cantusdb_project/main_app/admin/source.py +++ b/django/cantusdb_project/main_app/admin/source.py @@ -3,7 +3,7 @@ from main_app.admin.base_admin import BaseModelAdmin, EXCLUDE, READ_ONLY from main_app.admin.filters import InputFilter from main_app.forms import AdminSourceForm -from main_app.models import Source +from main_app.models import Source, SourceIdentifier class SourceKeyFilter(InputFilter): @@ -15,10 +15,19 @@ def queryset(self, request, queryset): return queryset.filter(holding_institution__siglum__icontains=self.value()) +class IdentifiersInline(admin.TabularInline): + model = SourceIdentifier + extra = 0 + + def get_queryset(self, request): + return super().get_queryset(request).select_related("source__holding_institution") + + @admin.register(Source) class SourceAdmin(BaseModelAdmin): exclude = EXCLUDE + ("source_status",) raw_id_fields = ("holding_institution",) + inlines = (IdentifiersInline,) # These search fields are also available on the user-source inline relationship in the user admin page search_fields = ( @@ -28,6 +37,8 @@ class SourceAdmin(BaseModelAdmin): "holding_institution__migrated_identifier", "id", "provenance_notes", + "name", + "identifiers__identifier" ) readonly_fields = ( ("title", "siglum") diff --git a/django/cantusdb_project/main_app/forms.py b/django/cantusdb_project/main_app/forms.py index 81dce3c75..bea4213e8 100644 --- a/django/cantusdb_project/main_app/forms.py +++ b/django/cantusdb_project/main_app/forms.py @@ -1,5 +1,14 @@ from django import forms from django.contrib.auth.forms import ReadOnlyPasswordHashField +from django.contrib.auth import get_user_model +from django.db.models import Q +from django.contrib.admin.widgets import ( + FilteredSelectMultiple, +) +from django.forms.widgets import CheckboxSelectMultiple +from dal import autocomplete +from volpiano_display_utilities.cantus_text_syllabification import syllabify_text +from volpiano_display_utilities.latin_word_syllabification import LatinError from .models import ( Chant, Service, @@ -22,13 +31,6 @@ SelectWidget, CheckboxWidget, ) -from django.contrib.auth import get_user_model -from django.db.models import Q -from django.contrib.admin.widgets import ( - FilteredSelectMultiple, -) -from django.forms.widgets import CheckboxSelectMultiple -from dal import autocomplete # ModelForm allows to build a form directly from a model # see https://docs.djangoproject.com/en/3.0/topics/forms/modelforms/ @@ -71,6 +73,50 @@ def label_from_instance(self, obj): widget = CheckboxSelectMultiple() + +class CantusDBLatinField(forms.CharField): + """ + A custom CharField for chant text fields. Validates that the text + can be syllabified (essentially, that it does not have any improper + characters). + """ + + def validate(self, value): + super().validate(value) + if value: + try: + syllabify_text(value) + except LatinError as err: + raise forms.ValidationError(str(err)) + except ValueError as exc: + raise forms.ValidationError("Invalid characters in text.") from exc + + +class CantusDBSyllabifiedLatinField(forms.CharField): + """ + A custom CharField for chant syllabified text fields. Validates that the text + can be syllabified (essentially, that it does not have any improper + characters). + """ + + def validate(self, value): + super().validate(value) + if value: + try: + syllabify_text(value, text_presyllabified=True) + except ValueError as exc: + raise forms.ValidationError("Invalid characters in text.") from exc + +class StyledChoiceField(forms.ChoiceField): + """ + A custom ChoiceField that uses the custom SelectWidget defined in widgets.py + as its widget (for styling). + """ + + widget = SelectWidget() + + + class ChantCreateForm(forms.ModelForm): class Meta: model = Chant @@ -125,8 +171,8 @@ class Meta: "finalis": TextInputWidget(), "extra": TextInputWidget(), "chant_range": VolpianoInputWidget(), - # manuscript_full_text_std_spelling: defined below (required) - "manuscript_full_text": TextAreaWidget(), + # manuscript_full_text_std_spelling: defined below (required & special field) + # "manuscript_full_text": defined below (special field) "volpiano": VolpianoAreaWidget(), "image_link": TextInputWidget(), "melody_id": TextInputWidget(), @@ -153,14 +199,18 @@ class Meta: help_text="Each folio starts with '1'.", ) - manuscript_full_text_std_spelling = forms.CharField( + manuscript_full_text_std_spelling = CantusDBLatinField( + widget=TextAreaWidget, + help_text=Chant._meta.get_field("manuscript_full_text_std_spelling").help_text, + label="Full text as in Source (standardized spelling)", required=True, + ) + + manuscript_full_text = CantusDBLatinField( widget=TextAreaWidget, - help_text="Manuscript full text with standardized spelling. Enter the words " - "according to the manuscript but normalize their spellings following " - "Classical Latin forms. Use upper-case letters for proper nouns, " - 'the first word of each chant, and the first word after "Alleluia" for ' - "Mass Alleluias. Punctuation is omitted.", + label="Full text as in Source (source spelling)", + help_text=Chant._meta.get_field("manuscript_full_text").help_text, + required=False, ) project = SelectWidgetNameModelChoiceField( @@ -191,6 +241,7 @@ class Meta: # "siglum", "holding_institution", "shelfmark", + "name", "segment_m2m", "provenance", "provenance_notes", @@ -212,11 +263,15 @@ class Meta: "fragmentarium_id", "dact_id", "indexing_notes", + "production_method", + "source_completeness", ] widgets = { # "title": TextInputWidget(), # "siglum": TextInputWidget(), + "shelfmark": TextInputWidget(), "provenance": autocomplete.ModelSelect2(url="provenance-autocomplete"), + "name": TextInputWidget(), "provenance_notes": TextInputWidget(), "date": TextInputWidget(), "cursus": SelectWidget(), @@ -246,6 +301,8 @@ class Meta: "other_editors": autocomplete.ModelSelect2Multiple( url="all-users-autocomplete" ), + "production_method": SelectWidget(), + "source_completeness": SelectWidget(), } field_classes = { "segment_m2m": CheckboxNameModelMultipleChoiceField, @@ -253,32 +310,15 @@ class Meta: holding_institution = forms.ModelChoiceField( queryset=Institution.objects.all(), - required=True, widget=autocomplete.ModelSelect2(url="holding-autocomplete"), + required=False, ) - shelfmark = forms.CharField( - required=True, - widget=TextInputWidget, - ) - - TRUE_FALSE_CHOICES_SOURCE = ( - (True, "Full source"), - (False, "Fragment or Fragmented"), - ) - - full_source = forms.ChoiceField(choices=TRUE_FALSE_CHOICES_SOURCE, required=False) - full_source.widget.attrs.update( - {"class": "form-control custom-select custom-select-sm"} - ) TRUE_FALSE_CHOICES_INVEN = ((True, "Complete"), (False, "Incomplete")) - complete_inventory = forms.ChoiceField( + complete_inventory = StyledChoiceField( choices=TRUE_FALSE_CHOICES_INVEN, required=False ) - complete_inventory.widget.attrs.update( - {"class": "form-control custom-select custom-select-sm"} - ) class ChantEditForm(forms.ModelForm): @@ -319,8 +359,8 @@ class Meta: "rubrics", ] widgets = { - # manuscript_full_text_std_spelling: defined below (required) - "manuscript_full_text": TextAreaWidget(), + # manuscript_full_text_std_spelling: defined below (required) & special field + # manuscript_full_text: defined below (special field) "volpiano": VolpianoAreaWidget(), "marginalia": TextInputWidget(), # folio: defined below (required) @@ -354,14 +394,18 @@ class Meta: "rubrics": TextInputWidget(), } - manuscript_full_text_std_spelling = forms.CharField( + manuscript_full_text_std_spelling = CantusDBLatinField( + widget=TextAreaWidget, + help_text=Chant._meta.get_field("manuscript_full_text_std_spelling").help_text, + label="Full text as in Source (standardized spelling)", required=True, + ) + + manuscript_full_text = CantusDBLatinField( widget=TextAreaWidget, - help_text="Manuscript full text with standardized spelling. Enter the words " - "according to the manuscript but normalize their spellings following " - "Classical Latin forms. Use upper-case letters for proper nouns, " - 'the first word of each chant, and the first word after "Alleluia" for ' - "Mass Alleluias. Punctuation is omitted.", + label="Full text as in Source (source spelling)", + help_text=Chant._meta.get_field("manuscript_full_text").help_text, + required=False, ) folio = forms.CharField( @@ -391,6 +435,7 @@ class Meta: # "siglum", "holding_institution", "shelfmark", + "name", "segment_m2m", "provenance", "provenance_notes", @@ -413,12 +458,17 @@ class Meta: "full_text_entered_by", "proofreaders", "other_editors", + "production_method", + "source_completeness", ] widgets = { + "shelfmark": TextInputWidget(), "segment_m2m": CheckboxSelectMultiple(), + "name": TextInputWidget(), "provenance": autocomplete.ModelSelect2(url="provenance-autocomplete"), "provenance_notes": TextInputWidget(), "date": TextInputWidget(), + "cursus": SelectWidget(), "summary": TextAreaWidget(), "liturgical_occasions": TextAreaWidget(), "description": TextAreaWidget(), @@ -446,48 +496,24 @@ class Meta: "other_editors": autocomplete.ModelSelect2Multiple( url="all-users-autocomplete" ), + "production_method": SelectWidget(), + "source_completeness": SelectWidget(), } field_classes = { "segment_m2m": CheckboxNameModelMultipleChoiceField, } - shelfmark = forms.CharField( - required=True, - widget=TextInputWidget, - ) - holding_institution = forms.ModelChoiceField( queryset=Institution.objects.all(), - required=True, widget=autocomplete.ModelSelect2(url="holding-autocomplete"), + required=False, ) - CHOICES_FULL_SOURCE = ( - (None, "None"), - (True, "Full source"), - (False, "Fragment or Fragmented"), - ) - full_source = forms.ChoiceField(choices=CHOICES_FULL_SOURCE, required=False) - full_source.widget.attrs.update( - {"class": "form-control custom-select custom-select-sm"} - ) - - CHOICES_CURSUS = ( - (None, "None"), - ("Monastic", "Monastic"), - ("Secular", "Secular"), - ) - cursus = forms.ChoiceField(choices=CHOICES_CURSUS, required=False) - cursus.widget.attrs.update({"class": "form-control custom-select custom-select-sm"}) - CHOICES_COMPLETE_INV = ( (True, "complete inventory"), (False, "partial inventory"), ) - complete_inventory = forms.ChoiceField(choices=CHOICES_COMPLETE_INV, required=False) - complete_inventory.widget.attrs.update( - {"class": "form-control custom-select custom-select-sm"} - ) + complete_inventory = StyledChoiceField(choices=CHOICES_COMPLETE_INV, required=False) class SequenceEditForm(forms.ModelForm): @@ -550,10 +576,14 @@ class Meta: "manuscript_full_text", "manuscript_syllabized_full_text", ] - widgets = { - "manuscript_full_text": TextAreaWidget(), - "manuscript_syllabized_full_text": TextAreaWidget(), - } + + manuscript_full_text = CantusDBLatinField( + widget=TextAreaWidget, label="Full text as in Source (source spelling)" + ) + + manuscript_syllabized_full_text = CantusDBSyllabifiedLatinField( + widget=TextAreaWidget, label="Syllabized full text" + ) class AdminCenturyForm(forms.ModelForm): @@ -720,27 +750,31 @@ class Meta: model = Source fields = "__all__" - title = forms.CharField( - required=True, - widget=TextInputWidget, - help_text="Full Source Identification (City, Archive, Shelf-mark)", - ) - title.widget.attrs.update({"style": "width: 610px;"}) + # title = forms.CharField( + # required=True, + # widget=TextInputWidget, + # help_text="Full Source Identification (City, Archive, Shelf-mark)", + # ) + # title.widget.attrs.update({"style": "width: 610px;"}) + # + # siglum = forms.CharField( + # required=True, + # widget=TextInputWidget, + # help_text="RISM-style siglum + Shelf-mark (e.g. GB-Ob 202).", + # ) - siglum = forms.CharField( - required=True, - widget=TextInputWidget, - help_text="RISM-style siglum + Shelf-mark (e.g. GB-Ob 202).", - ) shelfmark = forms.CharField( required=True, widget=TextInputWidget, ) + name = forms.CharField(required=False, widget=TextInputWidget) + + holding_institution = forms.ModelChoiceField( - queryset=Institution.objects.all().order_by("name"), - required=True, + queryset=Institution.objects.all().order_by("city", "name"), + required=False, ) provenance = forms.ModelChoiceField( diff --git a/django/cantusdb_project/main_app/management/commands/map_cantus_ids.py b/django/cantusdb_project/main_app/management/commands/map_cantus_ids.py new file mode 100644 index 000000000..ad25ce5ae --- /dev/null +++ b/django/cantusdb_project/main_app/management/commands/map_cantus_ids.py @@ -0,0 +1,46 @@ +""" +This is a (potentially temporary) command to change one Cantus ID to another +Cantus ID. For the moment (October 2024), Cantus Index and Cantus Database are +working out some issues in the process of merging Cantus ID's, so this command +gives us a more fine-grained (but more general than changing them all manually) +approach to changing Cantus ID's. +""" + +from django.core.management.base import BaseCommand +import reversion # type: ignore[import-untyped] + +from main_app.models import Chant + + +class Command(BaseCommand): + help = "Change one Cantus ID to another Cantus ID." + + def add_arguments(self, parser): + parser.add_argument( + "old_cantus_id", + type=str, + help="The Cantus ID to change.", + ) + parser.add_argument( + "new_cantus_id", + type=str, + help="The Cantus ID to change to.", + ) + + def handle(self, *args, **options): + old_cantus_id = options["old_cantus_id"] + new_cantus_id = options["new_cantus_id"] + with reversion.create_revision(): + chants = Chant.objects.filter(cantus_id=old_cantus_id) + num_chants = chants.count() + for chant in chants.iterator(chunk_size=1_000): + chant.cantus_id = new_cantus_id + chant.save() + self.stdout.write( + self.style.SUCCESS( + f"Changed {old_cantus_id} to {new_cantus_id} in {num_chants} chants." + ) + ) + reversion.set_comment( + f"Changed Cantus ID: {old_cantus_id} to {new_cantus_id}" + ) diff --git a/django/cantusdb_project/main_app/management/commands/migrate_records.py b/django/cantusdb_project/main_app/management/commands/migrate_records.py index bfda4e0a6..bc3f26325 100644 --- a/django/cantusdb_project/main_app/management/commands/migrate_records.py +++ b/django/cantusdb_project/main_app/management/commands/migrate_records.py @@ -163,7 +163,7 @@ def handle(self, *args, **options): ) ) institution = print_inst - elif siglum in created_institutions: + elif siglum in created_institutions and source.id not in bad_siglum: print( self.style.SUCCESS( f"Re-using the pre-created institution for {siglum}" @@ -185,7 +185,7 @@ def handle(self, *args, **options): institution.alternate_names = "\n".join(list(deduped_names)) institution.save() - elif siglum not in created_institutions: + elif siglum not in created_institutions and source.id not in bad_siglum: print(self.style.SUCCESS(f"Creating institution record for {siglum}")) iobj = { @@ -229,6 +229,8 @@ def handle(self, *args, **options): created_institutions[siglum] = institution else: + source.shelfmark = shelfmark.strip() + source.save() print( self.style.ERROR( f"Could not determine the holding institution for {source}" diff --git a/django/cantusdb_project/main_app/management/commands/populate_source_completeness.py b/django/cantusdb_project/main_app/management/commands/populate_source_completeness.py new file mode 100644 index 000000000..fe7d9e279 --- /dev/null +++ b/django/cantusdb_project/main_app/management/commands/populate_source_completeness.py @@ -0,0 +1,22 @@ +""" +A temporary command to populate the source_completeness field in the Source model, +based on the full_source field. This command will be removed once the source_completeness +is initially populated. +""" + +from django.core.management.base import BaseCommand +from main_app.models import Source + + +class Command(BaseCommand): + def handle(self, *args, **options): + sources = Source.objects.all() + for source in sources: + if source.full_source: + source.source_completeness = ( + source.SourceCompletenessChoices.FULL_SOURCE + ) + else: + source.source_completeness = source.SourceCompletenessChoices.FRAGMENT + source.save() + self.stdout.write(self.style.SUCCESS("Source completeness populated")) diff --git a/django/cantusdb_project/main_app/management/commands/reformat_source_ids.py b/django/cantusdb_project/main_app/management/commands/reformat_source_ids.py new file mode 100644 index 000000000..b11f3941a --- /dev/null +++ b/django/cantusdb_project/main_app/management/commands/reformat_source_ids.py @@ -0,0 +1,49 @@ +""" +A command designed to do a one-time reformatting of DACT IDs and Fragment +IDs in the database. + +Fragment IDs should be of the form "F-XXXX" where XXXX is some alphanumeric. +Fragment IDs are currently assumed to be in the form "F-XXXX" or "XXXX". +DACT IDs should be of the form "D:0XXXX" where XXXX is the Fragment ID alphanumeric. +DACT IDs are currently assumed to be in the form "0XXXX" or "D-0XXXX". + +This command simply adds the prefix "F-" to all Fragment IDs and "D:" to all +DACT IDs where they are missing. +""" + +from django.core.management.base import BaseCommand + +from main_app.models import Source + + +class Command(BaseCommand): + help = "Reformat DACT IDs and Fragment IDs in the database." + + def handle(self, *args, **options): + sources = Source.objects.all() + for source in sources: + if source.dact_id: + if len(source.dact_id) == 5 and source.dact_id.startswith("0"): + source.dact_id = f"D:{source.dact_id}" + elif len(source.dact_id) == 7 and source.dact_id.startswith("D-0"): + source.dact_id = f"D:{source.dact_id[2:]}" + else: + self.stdout.write( + self.style.WARNING( + f"{source.id} | DACT ID {source.dact_id} is not in the correct format." + ) + ) + if source.fragmentarium_id: + if len(source.fragmentarium_id) == 4: + source.fragmentarium_id = f"F-{source.fragmentarium_id}" + elif len( + source.fragmentarium_id + ) == 6 and source.fragmentarium_id.startswith("F-"): + pass + else: + self.stdout.write( + self.style.WARNING( + f"{source.id} | Fragment ID {source.fragmentarium_id} is not in the correct format." + ) + ) + source.save() diff --git a/django/cantusdb_project/main_app/management/commands/remap_user_ids.py b/django/cantusdb_project/main_app/management/commands/remap_user_ids.py index a0a734ffc..51002d048 100644 --- a/django/cantusdb_project/main_app/management/commands/remap_user_ids.py +++ b/django/cantusdb_project/main_app/management/commands/remap_user_ids.py @@ -1,9 +1,9 @@ -from main_app.models import Source, Chant from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand -from sys import stdout from django.db.models.query import QuerySet -from typing import Optional +import reversion # type: ignore[import-untyped] + +from main_app.models import Source, Chant User = get_user_model() @@ -49,89 +49,66 @@ } -def reassign_sources() -> None: - CHUNK_SIZE = 1_000 - sources: QuerySet[Source] = Source.objects.all() - sources_count: int = sources.count() - start_index: int = 0 - while start_index <= sources_count: - stdout.write(f"processing chunk with {start_index=}\n") - chunk: QuerySet[Source] = sources[start_index : start_index + CHUNK_SIZE] - for source in chunk: - old_creator: Optional[User] = source.created_by - - updated_id: Optional[int] = None - try: - updated_id: int = USER_ID_MAPPING[old_creator.id] - except ( - KeyError, # old_creator.id not in USER_ID_MAPPING - AttributeError, # old_creator is None - ): - pass +class Command(BaseCommand): + def reassign_sources(self) -> None: + sources: QuerySet[Source] = Source.objects.filter( + created_by__in=USER_ID_MAPPING.keys() + ) + num_sources = sources.count() + self.stdout.write( + self.style.NOTICE(f"Reassigning {num_sources} sources to new users.") + ) + source_counter = 0 + for source in sources.iterator(chunk_size=1_000): + old_creator = source.created_by - if updated_id is None: - # user ID doesn't need to be remapped - continue + # We know the old_creator is in USER_ID_MAPPING.keys() because of the filter + # on the queryset. + updated_id = USER_ID_MAPPING[old_creator.id] # type: ignore[union-attr] - updated_creator: Optional[User] = None - try: - updated_creator = User.objects.get(id=updated_id) - except ( - User.DoesNotExist, - AttributeError, - ): - pass + updated_creator = User.objects.get(id=updated_id) source.created_by = updated_creator source.save() - start_index += CHUNK_SIZE - - -def reassign_chants() -> None: - CHUNK_SIZE = 1_000 - chants: QuerySet[Chant] = Chant.objects.all() - chants_count: int = chants.count() - start_index: int = 0 - while start_index <= chants_count: - stdout.write(f"processing chunk with {start_index=}\n") - chunk: QuerySet[Chant] = chants[start_index : start_index + CHUNK_SIZE] - for chant in chunk: - old_creator: Optional[User] = chant.created_by - - updated_id: Optional[int] = None - try: - updated_id: int = USER_ID_MAPPING[old_creator.id] - except ( - KeyError, # old_creator.id not in USER_ID_MAPPING - AttributeError, # old_creator is None - ): - pass - - if updated_id is None: - # user ID doesn't need to be remapped - continue - - updated_creator: Optional[User] = None - try: - updated_creator = User.objects.get(id=updated_id) - except User.DoesNotExist: - pass - + source_counter += 1 + if source_counter % 100 == 0: + self.stdout.write( + self.style.NOTICE(f"Reassigned {source_counter} sources.") + ) + + def reassign_chants(self) -> None: + chants: QuerySet[Chant] = Chant.objects.filter( + created_by__in=USER_ID_MAPPING.keys() + ) + num_chants = chants.count() + self.stdout.write( + self.style.NOTICE(f"Reassigning {num_chants} sources to new users.") + ) + chant_counter = 0 + for chant in chants.iterator(chunk_size=1_000): + old_creator = chant.created_by + # We know the old_creator is in USER_ID_MAPPING.keys() because of the filter + # on the queryset. + updated_id: int = USER_ID_MAPPING[old_creator.id] # type: ignore[union-attr] + updated_creator = User.objects.get(id=updated_id) chant.created_by = updated_creator chant.save() - start_index += CHUNK_SIZE - + chant_counter += 1 + if chant_counter % 100 == 0: + self.stdout.write( + self.style.NOTICE(f"Reassigned {chant_counter} chants.") + ) -class Command(BaseCommand): def handle(self, *args, **kwargs) -> None: - error_message = ( - "As of late November 2023, this command is not working. " - "It has been temporarily disabled until the bugs have been worked out." - ) - raise NotImplementedError(error_message) - stdout.write("\n\n==== Reassigning Sources ====\n") - reassign_sources() - stdout.write("\n== All sources successfully remapped! ==\n") - stdout.write("\n\n==== Reassigning Chants ====\n") - reassign_chants() - stdout.write("\n== All chants successfully remapped! ==\n") + with reversion.create_revision(): + self.stdout.write(self.style.NOTICE("==== Reassigning Sources ====")) + self.reassign_sources() + self.stdout.write( + self.style.SUCCESS("== All sources successfully remapped! ==") + ) + self.stdout.write(self.style.NOTICE("==== Reassigning Chants ====")) + self.reassign_chants() + self.stdout.write( + self.style.SUCCESS("== All chants successfully remapped! ==") + ) + reversion.set_comment("Command: remap user IDs") diff --git a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py index 7c0fb2ebc..820e02d2a 100644 --- a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py +++ b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py @@ -112,7 +112,10 @@ def make_chant_dict(chant: dict) -> dict: "incipit": chant["incipit"], "feast": chant["feast__name"], "genre": chant["genre__name"], - "service": chant["service__name"], + "office": chant[ + "service__name" + ], # We keep the office key for backwards compatibility + # with external applications (e.g. Cantus Index) using this export "position": chant["position"], "cantus_id": chant["cantus_id"], "image": chant["image_link"], diff --git a/django/cantusdb_project/main_app/migrations/0031_alter_source_holding_institution_and_more.py b/django/cantusdb_project/main_app/migrations/0031_alter_source_holding_institution_and_more.py deleted file mode 100644 index e793e3991..000000000 --- a/django/cantusdb_project/main_app/migrations/0031_alter_source_holding_institution_and_more.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 4.2.14 on 2024-08-30 13:58 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ("main_app", "0030_institution_is_private_collection"), - ] - - operations = [ - migrations.AlterField( - model_name="source", - name="holding_institution", - field=models.ForeignKey( - default=19, - on_delete=django.db.models.deletion.PROTECT, - to="main_app.institution", - ), - preserve_default=False, - ), - migrations.AlterField( - model_name="source", - name="shelfmark", - field=models.CharField( - default="XX-NN", - max_length=255, - ), - ), - ] diff --git a/django/cantusdb_project/main_app/migrations/0031_source_name_source_production_method_and_more.py b/django/cantusdb_project/main_app/migrations/0031_source_name_source_production_method_and_more.py new file mode 100644 index 000000000..b0440b7f8 --- /dev/null +++ b/django/cantusdb_project/main_app/migrations/0031_source_name_source_production_method_and_more.py @@ -0,0 +1,104 @@ +# Generated by Django 4.2.14 on 2024-10-02 17:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("main_app", "0030_institution_is_private_collection"), + ] + + operations = [ + migrations.AddField( + model_name="source", + name="name", + field=models.CharField( + blank=True, + help_text="A colloquial or commonly-used name for the source", + max_length=255, + null=True, + ), + ), + migrations.AddField( + model_name="source", + name="production_method", + field=models.IntegerField( + choices=[(1, "Manuscript"), (2, "Printed")], + default=1, + verbose_name="Manuscript/Printed", + ), + ), + migrations.AddField( + model_name="source", + name="source_completeness", + field=models.IntegerField( + choices=[ + (1, "Full source"), + (2, "Fragment/Fragmented"), + (3, "Reconstruction"), + ], + default=1, + verbose_name="Full Source/Fragment", + ), + ), + migrations.AlterField( + model_name="institution", + name="country", + field=models.CharField(default="[No Country]", max_length=64), + ), + migrations.AlterField( + model_name="institution", + name="name", + field=models.CharField(default="[No Name]", max_length=255), + ), + migrations.AlterField( + model_name="source", + name="shelfmark", + field=models.CharField( + default="[No Shelfmark]", + help_text="Primary Cantus Database identifier for the source (e.g. library shelfmark, DACT ID, etc.)", + max_length=255, + ), + ), + migrations.CreateModel( + name="SourceIdentifier", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("identifier", models.CharField(max_length=255)), + ( + "type", + models.IntegerField( + choices=[ + (1, "Other catalogues"), + (2, "olim (Former shelfmark)"), + (3, "Alternative names"), + (4, "RISM Online"), + ] + ), + ), + ("note", models.TextField(blank=True, null=True)), + ( + "source", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="identifiers", + to="main_app.source", + ), + ), + ], + options={ + "verbose_name": "Source Identifier", + "ordering": ("type",), + }, + ), + ] diff --git a/django/cantusdb_project/main_app/models/__init__.py b/django/cantusdb_project/main_app/models/__init__.py index ea1149eca..5ed2a4227 100644 --- a/django/cantusdb_project/main_app/models/__init__.py +++ b/django/cantusdb_project/main_app/models/__init__.py @@ -10,6 +10,7 @@ from main_app.models.segment import Segment from main_app.models.sequence import Sequence from main_app.models.source import Source +from main_app.models.source_identifier import SourceIdentifier from main_app.models.institution import Institution from main_app.models.institution_identifier import InstitutionIdentifier from main_app.models.project import Project diff --git a/django/cantusdb_project/main_app/models/institution.py b/django/cantusdb_project/main_app/models/institution.py index e38450d90..a5d966f1b 100644 --- a/django/cantusdb_project/main_app/models/institution.py +++ b/django/cantusdb_project/main_app/models/institution.py @@ -28,7 +28,7 @@ class Meta: ), ] - name = models.CharField(max_length=255, default="s.n.") + name = models.CharField(max_length=255, default="[No Name]") siglum = models.CharField( verbose_name="RISM Siglum", max_length=32, @@ -50,7 +50,7 @@ class Meta: region = models.CharField( max_length=64, blank=True, null=True, help_text=region_help_text ) - country = models.CharField(max_length=64, default="s.l.") + country = models.CharField(max_length=64, default="[No Country]") alternate_names = models.TextField( blank=True, null=True, help_text="Enter alternate names on separate lines." ) diff --git a/django/cantusdb_project/main_app/models/source.py b/django/cantusdb_project/main_app/models/source.py index 919fbac20..2887b95f6 100644 --- a/django/cantusdb_project/main_app/models/source.py +++ b/django/cantusdb_project/main_app/models/source.py @@ -42,13 +42,24 @@ class Source(BaseModel): holding_institution = models.ForeignKey( "Institution", on_delete=models.PROTECT, - null=False, - blank=False, + null=True, + blank=True, ) shelfmark = models.CharField( max_length=255, blank=False, null=False, + help_text=( + "Primary Cantus Database identifier for the source " + "(e.g. library shelfmark, DACT ID, etc.)" + ), + default="[No Shelfmark]", + ) + name = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A colloquial or commonly-used name for the source", ) provenance = models.ForeignKey( "Provenance", @@ -65,6 +76,18 @@ class Source(BaseModel): null=True, help_text="More exact indication of the provenance (if necessary)", ) + + class SourceCompletenessChoices(models.IntegerChoices): + FULL_SOURCE = 1, "Full source" + FRAGMENT = 2, "Fragment/Fragmented" + RECONSTRUCTION = 3, "Reconstruction" + + source_completeness = models.IntegerField( + choices=SourceCompletenessChoices.choices, + default=SourceCompletenessChoices.FULL_SOURCE, + verbose_name="Full Source/Fragment", + ) + full_source = models.BooleanField(blank=True, null=True) date = models.CharField( blank=True, @@ -134,6 +157,16 @@ class Source(BaseModel): blank=False, null=False, default=False ) + class ProductionMethodChoices(models.IntegerChoices): + MANUSCRIPT = 1, "Manuscript" + PRINTED = 2, "Printed" + + production_method = models.IntegerField( + default=ProductionMethodChoices.MANUSCRIPT, + choices=ProductionMethodChoices.choices, + verbose_name="Manuscript/Printed", + ) + # number_of_chants and number_of_melodies are used for rendering the source-list page (perhaps among other places) # they are automatically recalculated in main_app.signals.update_source_chant_count and # main_app.signals.update_source_melody_count every time a chant or sequence is saved or deleted @@ -157,10 +190,16 @@ def heading(self) -> str: city = f"{holdinst.city}," if holdinst.city else "" title.append(city) title.append(f"{holdinst.name},") + else: + title.append("Cantus") + + title.append(self.shelfmark) - tt = self.shelfmark if self.shelfmark else self.title + if self.source_completeness == self.SourceCompletenessChoices.FRAGMENT: + title.append("(fragment)") - title.append(tt) + if self.name: + title.append(f'("{self.name}")') return " ".join(title) @@ -170,9 +209,14 @@ def short_heading(self) -> str: if holdinst := self.holding_institution: if holdinst.siglum and holdinst.siglum != "XX-NN": title.append(f"{holdinst.siglum}") - elif holdinst.is_private_collector: - title.append("Private") + else: + title.append("Cantus") + else: + title.append("Cantus") + + title.append(self.shelfmark) + + if self.source_completeness == self.SourceCompletenessChoices.FRAGMENT: + title.append("(fragment)") - tt = self.shelfmark if self.shelfmark else self.title - title.append(tt) return " ".join(title) diff --git a/django/cantusdb_project/main_app/models/source_identifier.py b/django/cantusdb_project/main_app/models/source_identifier.py new file mode 100644 index 000000000..85ccb5f71 --- /dev/null +++ b/django/cantusdb_project/main_app/models/source_identifier.py @@ -0,0 +1,33 @@ +from django.db import models + +class SourceIdentifier(models.Model): + class Meta: + verbose_name = "Source Identifier" + ordering = ('type',) + + OTHER = 1 + OLIM = 2 + ALTN = 3 + RISM_ONLINE = 4 + + IDENTIFIER_TYPES = ( + (OTHER, 'Other catalogues'), + (OLIM, 'olim (Former shelfmark)'), + (ALTN, 'Alternative names'), + (RISM_ONLINE, "RISM Online") + ) + + identifier = models.CharField(max_length=255) + type = models.IntegerField(choices=IDENTIFIER_TYPES) + note = models.TextField(blank=True, null=True) + source = models.ForeignKey("Source", + related_name="identifiers", + on_delete=models.CASCADE) + + def __str__(self): + return f"{self.identifier}" + + @property + def identifier_type(self): + d = dict(self.IDENTIFIER_TYPES) + return d[self.type] diff --git a/django/cantusdb_project/main_app/templates/browse_chants.html b/django/cantusdb_project/main_app/templates/browse_chants.html index 8b9245520..771aceac0 100644 --- a/django/cantusdb_project/main_app/templates/browse_chants.html +++ b/django/cantusdb_project/main_app/templates/browse_chants.html @@ -88,7 +88,8 @@

Browse Chants

{{ chant.incipit|default:"" }}

{{ chant.manuscript_full_text_std_spelling|default:"" }}
- {% if chant.volpiano %} + + {% if chant.volpiano and chant.source.id != 680970 %} {{ chant.volpiano|default:"" }} {% endif %}

diff --git a/django/cantusdb_project/main_app/templates/chant_create.html b/django/cantusdb_project/main_app/templates/chant_create.html index 5aec5de03..4b29bb0bb 100644 --- a/django/cantusdb_project/main_app/templates/chant_create.html +++ b/django/cantusdb_project/main_app/templates/chant_create.html @@ -223,7 +223,7 @@

Create Chant

- + {{ form.manuscript_full_text_std_spelling }}

{{ form.manuscript_full_text_std_spelling.help_text }} @@ -241,8 +241,7 @@

Create Chant

- + {{ form.manuscript_full_text }}

{{ form.manuscript_full_text.help_text }} diff --git a/django/cantusdb_project/main_app/templates/chant_detail.html b/django/cantusdb_project/main_app/templates/chant_detail.html index 022f48101..a46a18156 100644 --- a/django/cantusdb_project/main_app/templates/chant_detail.html +++ b/django/cantusdb_project/main_app/templates/chant_detail.html @@ -240,8 +240,9 @@

{{ chant.incipit }}

Full text as in Source (source spelling)
{{ chant.manuscript_full_text }}
{% endif %} - - {% if chant.volpiano %} + + + {% if chant.volpiano and chant.source.id != 680970 %}
Volpiano

{{ chant.volpiano }}

@@ -253,7 +254,8 @@

{{ chant.incipit }}

{{ chant.indexing_notes }}
{% endif %} - {% if chant.volpiano %} + + {% if chant.volpiano and chant.source.id != 680970 %}
Melody with text
diff --git a/django/cantusdb_project/main_app/templates/chant_edit.html b/django/cantusdb_project/main_app/templates/chant_edit.html index 038eef616..47860c7d5 100644 --- a/django/cantusdb_project/main_app/templates/chant_edit.html +++ b/django/cantusdb_project/main_app/templates/chant_edit.html @@ -21,6 +21,7 @@ {% for message in messages %} {% endfor %} @@ -283,25 +284,33 @@

Syllabification is based on saved syllabized text.

{% endif %}
- {% for syl_text, syl_mel in syllabized_text_with_melody %} - -
{{ syl_mel }}
- - 
{{ syl_text }}
- - {% endfor %} + {% if syllabized_text_with_melody %} + {% for syl_text, syl_mel in syllabized_text_with_melody %} + +
{{ syl_mel }}
+ + 
{{ syl_text }}
+ + {% endfor %} + {% else %} +

Error aligning text and melody. Please check text for invalid characters.

+ {% endif %}
{% endif %} -
-
- - Edit syllabification (new window) - + + {% if syllabized_text_with_melody %} +
+
+ + Edit syllabification (new window) + +
-
+ {% endif %}
diff --git a/django/cantusdb_project/main_app/templates/chant_search.html b/django/cantusdb_project/main_app/templates/chant_search.html index 601bd380a..5ade5b0e1 100644 --- a/django/cantusdb_project/main_app/templates/chant_search.html +++ b/django/cantusdb_project/main_app/templates/chant_search.html @@ -240,6 +240,8 @@

Search Chants

{% for chant in chants %} + + {% if chant.source.id != 680970 or melodies != "true" %} {% if not source %} @@ -297,7 +299,8 @@

Search Chants

{% endif %} - {% if chant.volpiano %} + + {% if chant.volpiano and chant.source.id != 680970 %} ♫ {% endif %} @@ -307,6 +310,7 @@

Search Chants

{% endif %} + {% endif %} {% endfor %} diff --git a/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html b/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html index 446afd73a..65b49f09f 100644 --- a/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html +++ b/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html @@ -12,6 +12,7 @@ {% for message in messages %} {% endfor %} @@ -37,8 +38,8 @@

Edit Syllabification

-
@@ -46,8 +47,8 @@

Edit Syllabification

-
diff --git a/django/cantusdb_project/main_app/templates/source_create.html b/django/cantusdb_project/main_app/templates/source_create.html index 62a8d50a3..395a373fa 100644 --- a/django/cantusdb_project/main_app/templates/source_create.html +++ b/django/cantusdb_project/main_app/templates/source_create.html @@ -46,12 +46,9 @@

Create Source

{{ form.holding_institution }} -

- {{ form.holding_institution.help_text }} -

@@ -60,8 +57,17 @@

Create Source

Shelfmark:* {{ form.shelfmark }} -

- {{ form.shelfmark.help_text }} +

+ {{ form.shelfmark.help_text|safe }} +

+
+
+ + {{ form.name }} +

+ {{ form.name.help_text|safe }}

@@ -89,19 +95,20 @@

Create Source

- - {{ form.full_source }} -

{{ form.full_source.help_text }}

+ {{ form.source_completeness.label_tag }} + {{ form.source_completeness }}
-
+
{{ form.century.label_tag }} {{ form.century }}
+
+ {{ form.production_method.label_tag }} + {{ form.production_method }} +
diff --git a/django/cantusdb_project/main_app/templates/source_detail.html b/django/cantusdb_project/main_app/templates/source_detail.html index 357f5dc46..60e84a39d 100644 --- a/django/cantusdb_project/main_app/templates/source_detail.html +++ b/django/cantusdb_project/main_app/templates/source_detail.html @@ -34,15 +34,20 @@

{{ source.heading }}

{% endif %}
+
Cantus Siglum
+
{{ source.short_heading }}
{% if source.holding_institution %} -
Siglum
-
{{ source.short_heading }}
Holding Institution
{{ source.holding_institution }}
{% endif %} + {% if source.production_method %} +
Manuscript/Printed
+
{{ source.get_production_method_display }}
+ {% endif %} + {% if source.summary %}
Summary
{{ source.summary }}
@@ -112,10 +117,8 @@

{{ source.heading }}

{{ source.complete_inventory|yesno:"Complete Inventory,Partial Inventory" }}
{% endif %} - {% if source.full_source is not None %} -
Full Source/Fragment
-
{{ source.full_source|yesno:"Full Source,Fragment or Fragmented" }}
- {% endif %} +
Full Source/Fragment
+
{{ source.get_source_completeness_display }}
{% if user.is_authenticated %}
Source Status
diff --git a/django/cantusdb_project/main_app/templates/source_edit.html b/django/cantusdb_project/main_app/templates/source_edit.html index 89d876b07..385ebc1e9 100644 --- a/django/cantusdb_project/main_app/templates/source_edit.html +++ b/django/cantusdb_project/main_app/templates/source_edit.html @@ -48,11 +48,11 @@

{{ form.holding_institution }} -

- {{ form.holding_institution.help_text }} +

+ {{ form.holding_institution.help_text }}

@@ -62,8 +62,17 @@

Shelfmark:* {{ form.shelfmark }} -

- {{ form.shelfmark.help_text }} +

+ {{ form.shelfmark.help_text }} +

+

+
+ + {{ form.name }} +

+ {{ form.name.help_text|safe }}

@@ -90,18 +99,20 @@

More exact indication of the provenance (if necessary)

- - {{ form.full_source }} + {{ form.source_completeness.label_tag }} + {{ form.source_completeness }}
-
+
{{ form.century.label_tag }} {{ form.century }}
+
+ {{ form.production_method.label_tag }} + {{ form.production_method }} +
diff --git a/django/cantusdb_project/main_app/templates/source_list.html b/django/cantusdb_project/main_app/templates/source_list.html index a65800af0..330f48e82 100644 --- a/django/cantusdb_project/main_app/templates/source_list.html +++ b/django/cantusdb_project/main_app/templates/source_list.html @@ -88,8 +88,8 @@

Browse Sources

{% sortable_header request "country" %} - {% sortable_header request "city_institution" "City + Institution" %} - {% sortable_header request "source" "Source" %} + {% sortable_header request "city_institution" "City + Holding Institution" %} + {% sortable_header request "source" "Cantus Siglum" %} Summary Date/Origin Image Link @@ -103,9 +103,16 @@

Browse Sources

{{ source.holding_institution.country }} - - {{ source.holding_institution.city }}, {{ source.holding_institution.name }} - + {% if source.holding_institution %} + + + {% if source.holding_institution.city %} + {{ source.holding_institution.city }}, + {% endif %} + {{ source.holding_institution.name }} + + + {% endif %} @@ -113,6 +120,9 @@

Browse Sources

 + {% if source.name %} + "{{ source.name }}"
+ {% endif %} {{ source.summary|default:""|truncatechars_html:120 }} @@ -130,7 +140,8 @@

Browse Sources

{{ source.number_of_chants|default_if_none:"0" }} - {% if source.number_of_melodies %} + + {% if source.number_of_melodies and source.id != 680970 %}
/ {{ source.number_of_melodies }} {% endif %} diff --git a/django/cantusdb_project/main_app/tests/make_fakes.py b/django/cantusdb_project/main_app/tests/make_fakes.py index a2febbdbd..5991df946 100644 --- a/django/cantusdb_project/main_app/tests/make_fakes.py +++ b/django/cantusdb_project/main_app/tests/make_fakes.py @@ -362,15 +362,33 @@ def make_fake_institution( city: Optional[str] = None, region: Optional[str] = None, country: Optional[str] = None, + is_private_collector: Optional[bool] = None, ) -> Institution: + """ + Note that the siglum and is_private_collector fields + are mutually exclusive. If both are specified, an exception + will be raised. If neither are specified, the function will + randomly determine whether the institution is a private collector or + will be given a fake siglum. + """ name = name if name else faker.sentence() - siglum = siglum if siglum else faker.sentence(nb_words=1) city = city if city else faker.city() region = region if region else faker.country() country = country if country else faker.country() + if siglum and is_private_collector: + raise ValueError("Siglum and Private Collector cannot both be specified.") + is_private_collector = False if siglum else faker.boolean(chance_of_getting_true=20) + if not is_private_collector and not siglum: + siglum = faker.sentence(nb_words=1) + inst = Institution.objects.create( - name=name, siglum=siglum, city=city, region=region, country=country + name=name, + siglum=siglum, + city=city, + region=region, + country=country, + is_private_collector=is_private_collector, ) inst.save() @@ -468,4 +486,4 @@ def get_random_search_term(target): slice_start = random.randint(0, len(target) - 2) slice_end = random.randint(slice_start + 2, len(target)) search_term = target[slice_start:slice_end] - return search_term \ No newline at end of file + return search_term diff --git a/django/cantusdb_project/main_app/tests/test_functions.py b/django/cantusdb_project/main_app/tests/test_functions.py index a250d6cd6..0f89fb8d2 100644 --- a/django/cantusdb_project/main_app/tests/test_functions.py +++ b/django/cantusdb_project/main_app/tests/test_functions.py @@ -208,7 +208,7 @@ def test_concordances_structure(self): "incipit", "feast", "genre", - "service", + "office", "position", "cantus_id", "image", @@ -259,7 +259,7 @@ def test_concordances_values(self): ("incipit", chant.incipit), ("feast", chant.feast.name), ("genre", chant.genre.name), - ("service", chant.service.name), + ("office", chant.service.name), ("position", chant.position), ("cantus_id", chant.cantus_id), ("image", chant.image_link), diff --git a/django/cantusdb_project/main_app/tests/test_populate_source_completeness.py b/django/cantusdb_project/main_app/tests/test_populate_source_completeness.py new file mode 100644 index 000000000..065bbb2ea --- /dev/null +++ b/django/cantusdb_project/main_app/tests/test_populate_source_completeness.py @@ -0,0 +1,29 @@ +from django.test import TestCase +from django.core.management import call_command + +from main_app.models import Source +from main_app.tests.make_fakes import make_fake_source + + +class TestPopulateSourceCompleteness(TestCase): + def test_populate_source_completeness(self): + # make a few "Full Source" sources + for _ in range(5): + make_fake_source(full_source=True) + # make a few "Fragment" sources + for _ in range(3): + make_fake_source(full_source=False) + # run the command + call_command("populate_source_completeness") + sources = Source.objects.all() + for source in sources: + if source.full_source: + self.assertEqual( + source.source_completeness, + source.SourceCompletenessChoices.FULL_SOURCE, + ) + else: + self.assertEqual( + source.source_completeness, + source.SourceCompletenessChoices.FRAGMENT, + ) diff --git a/django/cantusdb_project/main_app/tests/test_reformat_source_ids.py b/django/cantusdb_project/main_app/tests/test_reformat_source_ids.py new file mode 100644 index 000000000..7148d786e --- /dev/null +++ b/django/cantusdb_project/main_app/tests/test_reformat_source_ids.py @@ -0,0 +1,56 @@ +from django.test import TestCase +from django.core.management import call_command + +from main_app.models import Source +from main_app.tests.make_fakes import make_fake_institution, make_fake_segment + + +class TestReformatSourceIDs(TestCase): + def test_command(self): + segment = make_fake_segment() + fake_inst = make_fake_institution() + correct_source_1 = Source.objects.create( + segment=segment, + shelfmark="Correct Source 1", + holding_institution=fake_inst, + dact_id="0a1b3", + fragmentarium_id="a1b3", + ) + correct_source_2 = Source.objects.create( + segment=segment, + shelfmark="Correct Source 2", + holding_institution=fake_inst, + dact_id="D-0a1b3", + fragmentarium_id="F-a1b3", + ) + source_with_no_ids = Source.objects.create( + segment=segment, + shelfmark="Source with no IDs", + holding_institution=fake_inst, + ) + source_with_incorrect_ids = Source.objects.create( + segment=segment, + shelfmark="Source with incorrect IDs", + holding_institution=fake_inst, + dact_id="a1b3", + fragmentarium_id="F-1b3", + ) + + call_command("reformat_source_ids") + self.assertEqual(Source.objects.get(pk=correct_source_1.pk).dact_id, "D:0a1b3") + self.assertEqual( + Source.objects.get(pk=correct_source_1.pk).fragmentarium_id, "F-a1b3" + ) + self.assertEqual(Source.objects.get(pk=correct_source_2.pk).dact_id, "D:0a1b3") + self.assertEqual( + Source.objects.get(pk=correct_source_2.pk).fragmentarium_id, "F-a1b3" + ) + self.assertIsNone(Source.objects.get(pk=source_with_no_ids.pk).dact_id) + self.assertIsNone(Source.objects.get(pk=source_with_no_ids.pk).fragmentarium_id) + self.assertEqual( + Source.objects.get(pk=source_with_incorrect_ids.pk).dact_id, "a1b3" + ) + self.assertEqual( + Source.objects.get(pk=source_with_incorrect_ids.pk).fragmentarium_id, + "F-1b3", + ) diff --git a/django/cantusdb_project/main_app/tests/test_views/test_api.py b/django/cantusdb_project/main_app/tests/test_views/test_api.py index 8991f3f44..05985e8f8 100644 --- a/django/cantusdb_project/main_app/tests/test_views/test_api.py +++ b/django/cantusdb_project/main_app/tests/test_views/test_api.py @@ -270,8 +270,7 @@ def test_json_melody_fields(self): "mid", "nid", "cid", - "holding_institution", - "shelfmark", + "siglum", "srcnid", "folio", "incipit", @@ -279,7 +278,7 @@ def test_json_melody_fields(self): "volpiano", "mode", "feast", - "service", + "office", "genre", "position", "chantlink", @@ -765,7 +764,7 @@ def test_structure(self): "incipit": "some string" "feast": "some string" "genre": "some string" - "service": "some string" + "office": "some string" "position": "some string" "cantus_id": "some string" "image": "some string" @@ -811,7 +810,7 @@ def test_structure(self): "incipit", "feast", "genre", - "service", + "office", "position", "cantus_id", "image", @@ -834,7 +833,7 @@ def test_values(self): "incipit": chant.incipit, "feast": chant.feast.name, "genre": chant.genre.name, - "service": chant.service.name, + "office": chant.service.name, "position": chant.position, "mode": chant.mode, "image": chant.image_link, diff --git a/django/cantusdb_project/main_app/tests/test_views/test_chant.py b/django/cantusdb_project/main_app/tests/test_views/test_chant.py index 706cfaba0..d91ef212c 100644 --- a/django/cantusdb_project/main_app/tests/test_views/test_chant.py +++ b/django/cantusdb_project/main_app/tests/test_views/test_chant.py @@ -4,6 +4,7 @@ from unittest.mock import patch import random +from typing import ClassVar from django.test import TestCase, Client from django.urls import reverse @@ -112,12 +113,11 @@ def test_chant_edit_link(self): ) # have to create project manager user - "View | Edit" toggle only visible for those with edit access for a chant's source - self.user = get_user_model().objects.create(email="test@test.com") - self.user.set_password("pass") - self.user.save() - self.client = Client() + pm_user = get_user_model().objects.create(email="test@test.com") + pm_user.set_password("pass") + pm_user.save() project_manager = Group.objects.get(name="project manager") - project_manager.user_set.add(self.user) + project_manager.user_set.add(pm_user) self.client.login(email="test@test.com", password="pass") response = self.client.get(reverse("chant-detail", args=[chant.id])) @@ -164,7 +164,6 @@ def setUp(self): self.user = get_user_model().objects.create(email="test@test.com") self.user.set_password("pass") self.user.save() - self.client = Client() project_manager = Group.objects.get(name="project manager") project_manager.user_set.add(self.user) self.client.login(email="test@test.com", password="pass") @@ -323,6 +322,44 @@ def test_proofread_chant(self): chant.refresh_from_db() self.assertIs(chant.manuscript_full_text_std_proofread, True) + def test_invalid_text(self) -> None: + """ + The user should not be able to create a chant with invalid text + (either invalid characters or unmatched brackets). + Instead, the user should be shown an error message. + """ + source = make_fake_source() + with self.subTest("Chant with invalid characters"): + response = self.client.post( + reverse("source-edit-chants", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a ch@nt t%xt with inv&lid ch!ra+ers", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Invalid characters in text.", + ) + with self.subTest("Chant with unmatched brackets"): + response = self.client.post( + reverse("source-edit-chants", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a chant with [ unmatched brackets", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Word [ contains non-alphabetic characters.", + ) + class ChantEditSyllabificationViewTest(TestCase): @classmethod @@ -363,7 +400,10 @@ def test_edit_syllabification(self): self.assertEqual(chant.manuscript_syllabized_full_text, "lorem ipsum") response = self.client.post( f"/edit-syllabification/{chant.id}", - {"manuscript_syllabized_full_text": "lore-m i-psum"}, + { + "manuscript_full_text": "lorem ipsum", + "manuscript_syllabized_full_text": "lore-m i-psum", + }, ) self.assertEqual(response.status_code, 302) # 302 Found chant.refresh_from_db() @@ -501,7 +541,8 @@ def test_filter_by_melody(self): source=source, volpiano=make_fake_volpiano(), ) - chant_without_melody = Chant.objects.create(source=source) + # Create a chant without a melody + Chant.objects.create(source=source) response = self.client.get(reverse("chant-search"), {"melodies": "true"}) # only chants with melodies should be in the result self.assertEqual(len(response.context["chants"]), 1) @@ -553,9 +594,11 @@ def test_search_bar_search(self): manuscript_full_text="Full text contains, but does not start with 'the'", cantus_id="123456", ) - chant_starting_with_a_number = make_fake_chant( + # Create a chant starting with a number that won't be found by either + # search term + make_fake_chant( manuscript_full_text=( - "1 is a number. " "How unusual, to find an arabic numeral in a chant!" + "1 is a number. How unusual, to find an arabic numeral in a chant!" ), cantus_id="234567", ) @@ -1370,7 +1413,7 @@ def test_column_header_links(self): # additional properties for which there are search fields feast = make_fake_feast() position = make_random_string(1) - chant = make_fake_chant( + make_fake_chant( manuscript_full_text_std_spelling=fulltext, service=service, genre=genre, @@ -1527,7 +1570,7 @@ def test_feast_column(self): url = feast.get_absolute_url() fulltext = "manuscript full text" search_term = "full" - chant = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling=fulltext, feast=feast, @@ -1551,7 +1594,7 @@ def test_service_column(self): url = service.get_absolute_url() fulltext = "manuscript full text" search_term = "full" - chant = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling=fulltext, service=service, @@ -1575,7 +1618,7 @@ def test_genre_column(self): url = genre.get_absolute_url() fulltext = "manuscript full text" search_term = "full" - chant = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling=fulltext, genre=genre, @@ -1818,7 +1861,8 @@ def test_filter_by_melody(self): source=source, volpiano=make_fake_volpiano, ) - chant_without_melody = Chant.objects.create(source=source) + # Create a chant without melody that won't be in the result + Chant.objects.create(source=source) response = self.client.get( reverse("chant-search-ms", args=[source.id]), {"melodies": "true"} ) @@ -1836,11 +1880,11 @@ def test_keyword_search_starts_with(self): source=source, manuscript_full_text_std_spelling="quick brown fox jumps over the lazy dog", ) - chant_2 = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling="brown fox jumps over the lazy dog", ) - chant_3 = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling="lazy brown fox jumps quick over the dog", ) @@ -1859,7 +1903,8 @@ def test_keyword_search_contains(self): source=source, manuscript_full_text_std_spelling="Quick brown fox jumps over the lazy dog", ) - chant_2 = make_fake_chant( + # Make a chant that won't be returned by the search term + make_fake_chant( source=source, manuscript_full_text_std_spelling="brown fox jumps over the lazy dog", ) @@ -1885,11 +1930,11 @@ def test_indexing_notes_search_starts_with(self): source=source, indexing_notes="quick brown fox jumps over the lazy dog", ) - chant_2 = make_fake_chant( + make_fake_chant( source=source, indexing_notes="brown fox jumps over the lazy dog", ) - chant_3 = make_fake_chant( + make_fake_chant( source=source, indexing_notes="lazy brown fox jumps quick over the dog", ) @@ -1908,7 +1953,8 @@ def test_indexing_notes_search_contains(self): source=source, indexing_notes="Quick brown fox jumps over the lazy dog", ) - chant_2 = make_fake_chant( + # Make a chant that won't be returned by the search term + make_fake_chant( source=source, indexing_notes="brown fox jumps over the lazy dog", ) @@ -1931,13 +1977,13 @@ def test_keyword_search_searching_all_fields(self): doesnt_include_search_term = "longevity is the soul of wit" source = make_fake_source() - chant_ms_spelling = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text=includes_search_term, # <== includes_search_term manuscript_full_text_std_spelling=doesnt_include_search_term, ) - chant_std_spelling = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text=doesnt_include_search_term, manuscript_full_text_std_spelling=includes_search_term, # <== @@ -1954,7 +2000,8 @@ def test_keyword_search_searching_all_fields(self): manuscript_full_text_std_spelling=None, ) - chant_without_search_term = make_fake_chant( + # This chant contains no search terms + make_fake_chant( source=source, manuscript_full_text=doesnt_include_search_term, manuscript_full_text_std_spelling=doesnt_include_search_term, @@ -2343,7 +2390,7 @@ def test_column_header_links(self): # additional properties for which there are search fields feast = make_fake_feast() position = make_random_string(1) - chant = make_fake_chant( + make_fake_chant( service=service, genre=genre, cantus_id=cantus_id, @@ -2449,9 +2496,7 @@ def test_source_link_column(self): url = source.get_absolute_url() fulltext = "manuscript full text" search_term = "full" - chant = make_fake_chant( - source=source, manuscript_full_text_std_spelling=fulltext - ) + make_fake_chant(source=source, manuscript_full_text_std_spelling=fulltext) response = self.client.get( reverse("chant-search-ms", args=[source.id]), {"keyword": search_term, "op": "contains"}, @@ -2487,7 +2532,7 @@ def test_feast_column(self): url = feast.get_absolute_url() fulltext = "manuscript full text" search_term = "full" - chant = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling=fulltext, feast=feast, @@ -2512,7 +2557,7 @@ def test_service_column(self): url = service.get_absolute_url() fulltext = "manuscript full text" search_term = "full" - chant = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling=fulltext, service=service, @@ -2537,7 +2582,7 @@ def test_genre_column(self): url = genre.get_absolute_url() fulltext = "manuscript full text" search_term = "full" - chant = make_fake_chant( + make_fake_chant( source=source, manuscript_full_text_std_spelling=fulltext, genre=genre, @@ -2699,37 +2744,62 @@ def test_image_link_column(self): self.assertIn(f'Image', html) +@patch("requests.get", mock_requests_get) class ChantCreateViewTest(TestCase): + source: ClassVar[Source] + @classmethod def setUpTestData(cls): - Group.objects.create(name="project manager") + # Create project manager and contributor users + prod_manager_group = Group.objects.create(name="project manager") + contributor_group = Group.objects.create(name="contributor") + user_model = get_user_model() + pm_usr = user_model.objects.create_user(email="pm@test.com", password="pass") + pm_usr.groups.set([prod_manager_group]) + pm_usr.save() + contributor_usr = user_model.objects.create_user( + email="contrib@test.com", password="pass" + ) + contributor_usr.groups.set([contributor_group]) + contributor_usr.save() + # Create a fake source that the contributor user can edit + cls.source = make_fake_source() + cls.source.current_editors.add(contributor_usr) + cls.source.save() def setUp(self): - self.user = get_user_model().objects.create(email="test@test.com") - self.user.set_password("pass") - self.user.save() - self.client = Client() - project_manager = Group.objects.get(name="project manager") - project_manager.user_set.add(self.user) - self.client.login(email="test@test.com", password="pass") - - def test_url_and_templates(self): - source = make_fake_source() + # Log in as a contributor before each test + self.client.login(email="contrib@test.com", password="pass") + + def test_permissions(self) -> None: + # The client starts logged in as a contributor + # with access to the source. Test that the client + # can access the ChantCreate view. + with self.subTest("Contributor can access ChantCreate view"): + response = self.client.get(reverse("chant-create", args=[self.source.id])) + self.assertEqual(response.status_code, 200) + with self.subTest("Project manager can access ChantCreate view"): + # Log in as a project manager + self.client.logout() + self.client.login(email="pm@test.com", password="pass") + response = self.client.get(reverse("chant-create", args=[self.source.id])) + self.assertEqual(response.status_code, 200) + with self.subTest("Unauthenticated user cannot access ChantCreate view"): + # Log out + self.client.logout() + response = self.client.get(reverse("chant-create", args=[self.source.id])) + self.assertEqual(response.status_code, 302) - with patch("requests.get", mock_requests_get): - response_1 = self.client.get(reverse("chant-create", args=[source.id])) - response_2 = self.client.get( - reverse("chant-create", args=[source.id + 100]) - ) + def test_url_and_templates(self) -> None: + source = self.source + response_1 = self.client.get(reverse("chant-create", args=[source.id])) self.assertEqual(response_1.status_code, 200) self.assertTemplateUsed(response_1, "chant_create.html") + self.assertTemplateUsed(response_1, "base.html") - self.assertEqual(response_2.status_code, 404) - self.assertTemplateUsed(response_2, "404.html") - - def test_create_chant(self): - source = make_fake_source() + def test_create_chant(self) -> None: + source = self.source response = self.client.post( reverse("chant-create", args=[source.id]), { @@ -2740,40 +2810,37 @@ def test_create_chant(self): ) self.assertEqual(response.status_code, 302) self.assertRedirects(response, reverse("chant-create", args=[source.id])) - chant = Chant.objects.first() + chant = Chant.objects.get(source=source) self.assertEqual(chant.manuscript_full_text_std_spelling, "initial") - def test_view_url_path(self): - source = make_fake_source() - with patch("requests.get", mock_requests_get): - response = self.client.get(f"/chant-create/{source.id}") + def test_view_url_path(self) -> None: + source = self.source + response = self.client.get(f"/chant-create/{source.id}") self.assertEqual(response.status_code, 200) - def test_context(self): - """some context variable passed to templates""" - source = make_fake_source() + def test_context(self) -> None: + """Test that correct source is in context""" + source = self.source url = reverse("chant-create", args=[source.id]) - with patch("requests.get", mock_requests_get): - response = self.client.get(url) - self.assertEqual(response.context["source"].title, source.title) + response = self.client.get(url) + self.assertEqual(response.context["source"].id, source.id) - def test_post_error(self): + def test_empty_fulltext(self) -> None: """post with correct source and empty full-text""" - source = make_fake_source() + source = self.source url = reverse("chant-create", args=[source.id]) response = self.client.post(url, data={"manuscript_full_text_std_spelling": ""}) self.assertFormError( - response, - "form", + response.context["form"], "manuscript_full_text_std_spelling", "This field is required.", ) - def test_nonexistent_source(self): + def test_nonexistent_source(self) -> None: """ users should not be able to access Chant Create page for a source that doesn't exist """ - nonexistent_source_id: str = faker.numerify( + nonexistent_source_id = faker.numerify( "#####" ) # there's not supposed to be 5-digits source id with patch("requests.get", mock_requests_get): @@ -2782,55 +2849,38 @@ def test_nonexistent_source(self): ) self.assertEqual(response.status_code, 404) - def test_repeated_seq(self): + def test_repeated_seq(self) -> None: """post with a folio and seq that already exists in the source""" - TEST_FOLIO = "001r" + test_folio = "001r" # create some chants in the test source - source = make_fake_source() + source = self.source for i in range(1, 5): Chant.objects.create( source=source, - manuscript_full_text=faker.text(10), - folio=TEST_FOLIO, + manuscript_full_text=" ".join(faker.words(faker.random_int(3, 10))), + folio=test_folio, c_sequence=i, ) # post a chant with the same folio and seq url = reverse("chant-create", args=[source.id]) - fake_text = faker.text(10) + fake_text = "this is also a fake but valid text" response = self.client.post( url, data={ "manuscript_full_text_std_spelling": fake_text, - "folio": TEST_FOLIO, + "folio": test_folio, "c_sequence": random.randint(1, 4), }, follow=True, ) self.assertFormError( - response, - "form", + response.context["form"], None, errors="Chant with the same sequence and folio already exists in this source.", ) - def test_view_url_reverse_name(self): - fake_sources = [make_fake_source() for _ in range(10)] - for source in fake_sources: - with patch("requests.get", mock_requests_get): - response = self.client.get(reverse("chant-create", args=[source.id])) - self.assertEqual(response.status_code, 200) - - def test_template_used(self): - fake_sources = [make_fake_source() for _ in range(10)] - for source in fake_sources: - with patch("requests.get", mock_requests_get): - response = self.client.get(reverse("chant-create", args=[source.id])) - self.assertEqual(response.status_code, 200) - self.assertTemplateUsed(response, "base.html") - self.assertTemplateUsed(response, "chant_create.html") - - def test_volpiano_signal(self): - source = make_fake_source() + def test_volpiano_signal(self) -> None: + source = self.source self.client.post( reverse("chant-create", args=[source.id]), { @@ -2844,10 +2894,9 @@ def test_volpiano_signal(self): # clefs, accidentals, etc., to be deleted }, ) - with patch("requests.get", mock_requests_get): - chant_1 = Chant.objects.get( - manuscript_full_text_std_spelling="ut queant lactose" - ) + chant_1 = Chant.objects.get( + manuscript_full_text_std_spelling="ut queant lactose" + ) self.assertEqual(chant_1.volpiano, "9abcdefg)A-B1C2D3E4F5G67?. yiz") self.assertEqual(chant_1.volpiano_notes, "9abcdefg9abcdefg") self.client.post( @@ -2859,16 +2908,13 @@ def test_volpiano_signal(self): "volpiano": "abacadaeafagahaja", }, ) - with patch("requests.get", mock_requests_get): - chant_2 = Chant.objects.get( - manuscript_full_text_std_spelling="resonare foobaz" - ) + chant_2 = Chant.objects.get(manuscript_full_text_std_spelling="resonare foobaz") self.assertEqual(chant_2.volpiano, "abacadaeafagahaja") self.assertEqual(chant_2.volpiano_intervals, "1-12-23-34-45-56-67-78-8") - def test_initial_values(self): + def test_initial_values(self) -> None: # create a chant with a known folio, feast, service, c_sequence and image_link - source: Source = make_fake_source() + source: Source = self.source folio: str = "001r" sequence: int = 1 feast: Feast = make_fake_feast() @@ -2885,12 +2931,11 @@ def test_initial_values(self): "image_link": image_link, }, ) - with patch("requests.get", mock_requests_get): - # when we request the Chant Create page, the same folio, feast, service and image_link should - # be preselected, and c_sequence should be incremented by 1. - response = self.client.get( - reverse("chant-create", args=[source.id]), - ) + # when we request the Chant Create page, the same folio, feast, service and image_link should + # be preselected, and c_sequence should be incremented by 1. + response = self.client.get( + reverse("chant-create", args=[source.id]), + ) observed_initial_folio: int = response.context["form"].initial["folio"] with self.subTest(subtest="test initial value of folio field"): @@ -2912,12 +2957,11 @@ def test_initial_values(self): with self.subTest(subtest="test initial value of image_link field"): self.assertEqual(observed_initial_image, image_link) - def test_suggested_chant_buttons(self): - source: Source = make_fake_source() - with patch("requests.get", mock_requests_get): - response_empty_source = self.client.get( - reverse("chant-create", args=[source.id]), - ) + def test_suggested_chant_buttons(self) -> None: + source: Source = self.source + response_empty_source = self.client.get( + reverse("chant-create", args=[source.id]), + ) with self.subTest( test="Ensure no suggestions displayed when there is no previous chant" ): @@ -2925,11 +2969,11 @@ def test_suggested_chant_buttons(self): response_empty_source, "Suggestions based on previous chant:" ) - previous_chant: Chant = make_fake_chant(cantus_id="001010", source=source) - with patch("requests.get", mock_requests_get): - response_after_previous_chant = self.client.get( - reverse("chant-create", args=[source.id]), - ) + # Make a chant to serve as the previous chant + make_fake_chant(cantus_id="001010", source=source) + response_after_previous_chant = self.client.get( + reverse("chant-create", args=[source.id]), + ) suggested_chants = response_after_previous_chant.context["suggested_chants"] with self.subTest( test="Ensure suggested chant suggestions present when previous chant exists" @@ -2940,11 +2984,11 @@ def test_suggested_chant_buttons(self): self.assertIsNotNone(suggested_chants) self.assertEqual(len(suggested_chants), 5) - rare_chant: Chant = make_fake_chant(cantus_id="a07763", source=source) - with patch("requests.get", mock_requests_get): - response_after_rare_chant = self.client.get( - reverse("chant-create", args=[source.id]), - ) + # Make a chant with a rare cantus_id to serve as the previous chant + make_fake_chant(cantus_id="a07763", source=source) + response_after_rare_chant = self.client.get( + reverse("chant-create", args=[source.id]), + ) with self.subTest( test="When previous chant has no suggested chants, ensure no suggestions are displayed" ): @@ -2956,6 +3000,45 @@ def test_suggested_chant_buttons(self): ) self.assertIsNone(response_after_rare_chant.context["suggested_chants"]) + def test_invalid_text(self) -> None: + """ + The user should not be able to create a chant with invalid text + (either invalid characters or unmatched brackets). + Instead, the user should be shown an error message. + """ + with self.subTest("Chant with invalid characters"): + source = self.source + response = self.client.post( + reverse("chant-create", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a ch@nt t%xt with inv&lid ch!ra+ers", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Invalid characters in text.", + ) + with self.subTest("Chant with unmatched brackets"): + source = self.source + response = self.client.post( + reverse("chant-create", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a chant with [ unmatched brackets", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Word [ contains non-alphabetic characters.", + ) + class CISearchViewTest(TestCase): diff --git a/django/cantusdb_project/main_app/tests/test_views/test_source.py b/django/cantusdb_project/main_app/tests/test_views/test_source.py index e7102d4ec..8c9e529a5 100644 --- a/django/cantusdb_project/main_app/tests/test_views/test_source.py +++ b/django/cantusdb_project/main_app/tests/test_views/test_source.py @@ -54,15 +54,14 @@ def test_url_and_templates(self): self.assertTemplateUsed(response, "source_create.html") def test_create_source(self): - hinst = make_fake_institution(siglum="FA-Ke") response = self.client.post( reverse("source-create"), { "shelfmark": "test-shelfmark", # shelfmark is a required field - "holding_institution": hinst.id, # holding institution is a required field + "source_completeness": "1", # required field + "production_method": "1", # required field }, ) - self.assertEqual(response.status_code, 302) created_source = Source.objects.get(shelfmark="test-shelfmark") self.assertRedirects( @@ -106,13 +105,13 @@ def test_url_and_templates(self): def test_edit_source(self): source = make_fake_source() - hinst = make_fake_institution(siglum="FA-Ke") response = self.client.post( reverse("source-edit", args=[source.id]), { "shelfmark": "test-shelfmark", # shelfmark is a required field, - "holding_institution": hinst.id, # holding institution is a required field + "source_completeness": "1", # required field + "production_method": "1", # required field }, ) @@ -981,10 +980,14 @@ def test_ordering(self) -> None: Order is currently available by country, city + institution name (parameter: "city_institution"), and siglum + shelfmark. Siglum + shelfmark is the default. """ - # Create a bunch of sources sources = [] + # Add a source from a private collector + private_collector = make_fake_institution(is_private_collector=True) + sources.append(make_fake_source(holding_institution=private_collector)) + # Create a bunch of other sources for _ in range(10): - sources.append(make_fake_source()) + inst = make_fake_institution(siglum=faker.word()) + sources.append(make_fake_source(holding_institution=inst)) # Default ordering is by siglum and shelfmark, ascending with self.subTest("Default ordering"): response = self.client.get(reverse("source-list")) @@ -992,7 +995,13 @@ def test_ordering(self) -> None: expected_source_order = sorted( sources, key=lambda source: ( - source.holding_institution.siglum, + 0 if source.holding_institution else 1, + 1 if source.holding_institution.is_private_collector else 0, + ( + source.holding_institution.siglum + if source.holding_institution.siglum + else "" + ), source.shelfmark, ), ) diff --git a/django/cantusdb_project/main_app/views/api.py b/django/cantusdb_project/main_app/views/api.py index 48f4da16e..f83cc6296 100644 --- a/django/cantusdb_project/main_app/views/api.py +++ b/django/cantusdb_project/main_app/views/api.py @@ -1,14 +1,13 @@ import csv -from typing import List -from typing import Optional +from typing import Optional, Union, Any from django.contrib.auth import get_user_model from django.contrib.flatpages.models import FlatPage from django.core.exceptions import PermissionDenied from django.db.models.query import QuerySet -from django.db.models import Model from django.http.response import JsonResponse -from django.http import HttpResponse, HttpResponseNotFound +from django.http import HttpResponse, HttpResponseNotFound, Http404, HttpRequest from django.urls.base import reverse +from django.shortcuts import get_object_or_404 from articles.models import Article from main_app.models import ( Chant, @@ -19,16 +18,9 @@ Source, ) from next_chants import next_chants -from django.http import Http404 -from django.core.exceptions import PermissionDenied -from django.urls import reverse -from django.contrib.auth import get_user_model -from typing import List -from django.contrib.flatpages.models import FlatPage -from django.shortcuts import get_object_or_404 -def ajax_melody_list(request, cantus_id) -> JsonResponse: +def ajax_melody_list(request: HttpRequest, cantus_id: str) -> JsonResponse: """ Function-based view responding to the AJAX call for melody list on the chant detail page, accessed with ``chants/``, click on "Display melodies connected with this chant" @@ -41,7 +33,9 @@ def ajax_melody_list(request, cantus_id) -> JsonResponse: """ chants: QuerySet[Chant] = ( Chant.objects.filter(cantus_id=cantus_id) - .select_related("source__holding_institution", "feast", "genre", "service") + .select_related( + "source", "source__holding_institution", "feast", "genre", "service" + ) .exclude(volpiano=None) .order_by("id") ) @@ -50,38 +44,29 @@ def ajax_melody_list(request, cantus_id) -> JsonResponse: if not display_unpublished: chants = chants.filter(source__published=True) - concordance_values: QuerySet[dict] = chants.values( - "source__holding_institution__siglum", - "source__shelfmark", - "folio", - "service__name", - "genre__name", - "position", - "feast__name", - "cantus_id", - "volpiano", - "mode", - # OldCantus seems to use whichever is present: ms spelling, std spelling, incipit (in that order) - "manuscript_full_text_std_spelling", - ) - - concordances: list[dict] = list(concordance_values) - for i, concordance in enumerate(concordances): - # we need to use each chant's _source_'s siglum, and not the - # legacy sigla that were attached to chants in OldCantus - holding_inst_sig = concordance.pop("source__holding_institution__siglum") - source_shelfmark = concordance.pop("source__shelfmark") - - concordance["siglum"] = f"{holding_inst_sig} {source_shelfmark}" - # for chants that do not have a source, do not attempt - # to return a source link - if chants[i].source: - concordance["source_link"] = chants[i].source.get_absolute_url() - concordance["ci_link"] = chants[i].get_ci_url() - concordance["chant_link"] = chants[i].get_absolute_url() - concordance["db"] = "CD" - - concordance_count: int = len(concordances) + concordances: list[dict[str, str]] = [] + for chant in chants: + concordance: dict[str, str] = { + "siglum": chant.source.short_heading, + "folio": chant.folio or "", + "service__name": chant.service.name if chant.service else "", + "genre__name": chant.genre.name if chant.genre else "", + "position": chant.position or "", + "feast__name": chant.feast.name if chant.feast else "", + "cantus_id": chant.cantus_id or "", + # Query above filters out chants with volpiano=None + "volpiano": chant.volpiano, # type: ignore[dict-item] + "mode": chant.mode or "", + "manuscript_full_text_std_spelling": chant.manuscript_full_text_std_spelling + or "", + "ci_link": chant.get_ci_url(), + "chant_link": chant.get_absolute_url(), + "source_link": chant.source.get_absolute_url(), + "db": "CD", + } + concordances.append(concordance) + + concordance_count = len(concordances) return JsonResponse( {"concordances": concordances, "concordance_count": concordance_count}, safe=True, @@ -154,7 +139,6 @@ def csv_export(request, source_id): feast = entry.feast.name if entry.feast else "" service = entry.service.name if entry.service else "" genre = entry.genre.name if entry.genre else "" - diff_db = entry.diff_db.id if entry.diff_db else "" writer.writerow( [ @@ -281,17 +265,21 @@ def ajax_melody_search(request): chants = chants.filter(feast__name__icontains=feast_name) if mode: chants = chants.filter(mode__icontains=mode) - - result_values = chants.order_by("id").values( - "id", - "source__holding_institution__siglum", - "source__shelfmark", - "folio", - "incipit", - "genre__name", - "feast__name", - "mode", - "volpiano", + # See #1635 re the following source exclusion. Temporarily disable volpiano display for this source. + result_values = ( + chants.exclude(source__id=680970) + .order_by("id") + .values( + "id", + "source__holding_institution__siglum", + "source__shelfmark", + "folio", + "incipit", + "genre__name", + "feast__name", + "mode", + "volpiano", + ) ) # convert queryset to a list of dicts because QuerySet is not JSON serializable # the above constructed queryset will be evaluated here @@ -357,88 +345,47 @@ def ajax_search_bar(request, search_term): return JsonResponse({"chants": returned_values}, safe=True) -def json_melody_export(request, cantus_id: str) -> JsonResponse: - chants = Chant.objects.filter( - cantus_id=cantus_id, volpiano__isnull=False, source__published=True - ) - - db_keys = [ - "melody_id", - "id", - "cantus_id", - "source__holding_institution", - "source__shelfmark", - "source__id", # don't fetch the entire Source object, just the id of - # the source. __id is removed in standardize_for_api below - "folio", - "incipit", - "manuscript_full_text", - "volpiano", - "mode", - "feast__id", - "service__id", - "genre__id", - "position", - ] - - chants_values = list(chants.values(*db_keys)) # a list of dictionaries. Each - # dictionary represents metadata on one chant - - standardized_chants_values = [ - standardize_dict_for_json_melody_export(cv, request) for cv in chants_values - ] - - return JsonResponse(standardized_chants_values, safe=False) - - -def standardize_dict_for_json_melody_export( - chant_values: List[dict], request -) -> List[dict]: - """Take a list of dictionaries, and in each dictionary, change several - of the keys to match their values in OldCantus - - Args: - chant_values (List[dict]): a list of dictionaries, each containing - information on a single chant in the database - request: passed when this is called in json_melody_export. Used to get the domain - while building the chant links - - Returns: - List[dict]: a list of dictionaries, with updated keys +def json_melody_export(request: HttpRequest, cantus_id: str) -> JsonResponse: """ - keymap = { # map attribute names from Chant model (i.e. db_keys - # in json_melody_export) to corresponding attribute names - # in old API, and remove artifacts of query process (i.e. __id suffixes) - "melody_id": "mid", # <- - "id": "nid", # <- - "cantus_id": "cid", # <- - "source__shelfmark": "shelfmark", - "source__holding_institution": "holding_institution", - "source__id": "srcnid", # <- - "folio": "folio", - "incipit": "incipit", - "manuscript_full_text": "fulltext", # <- - "volpiano": "volpiano", - "mode": "mode", - "feast__id": "feast", # <- - "service__id": "service", # <- - "genre__id": "genre", # <- - "position": "position", - } - - standardized_chant_values = {keymap[key]: chant_values[key] for key in chant_values} + Similar to the ajax_melody_list view, but designed for external use (for instance, + it returns absolute URLs for the chant and source detail pages), only returns + chants in published sources, and contains slightly different chant text fields. + """ + chants: QuerySet[Chant] = Chant.objects.filter( + cantus_id=cantus_id, volpiano__isnull=False, source__published=True + ).select_related("source") + + chants_export: list[dict[str, Optional[Union[str, int]]]] = [] + for chant in chants: + chant_values = { + "mid": chant.melody_id, + "nid": chant.id, + "cid": chant.cantus_id, + "siglum": chant.source.short_heading, + "srcnid": chant.source.id, + "folio": chant.folio, + "incipit": chant.incipit, + "fulltext": chant.manuscript_full_text_std_spelling, + "volpiano": chant.volpiano, + "mode": chant.mode, + "feast": chant.feast_id, + "office": chant.service_id, # We keep the office key for backwards compatibility + # with external applications + "genre": chant.genre_id, + "position": chant.position, + } + chant_uri = request.build_absolute_uri( + reverse("chant-detail", args=[chant_values["nid"]]) + ) + chant_values["chantlink"] = chant_uri + src_uri = request.build_absolute_uri( + reverse("source-detail", args=[chant_values["srcnid"]]) + ) + chant_values["srclink"] = src_uri - # manually build a couple of last fields that aren't represented in Chant object - chant_uri = request.build_absolute_uri( - reverse("chant-detail", args=[chant_values["id"]]) - ) - standardized_chant_values["chantlink"] = chant_uri - src_uri = request.build_absolute_uri( - reverse("source-detail", args=[chant_values["source__id"]]) - ) - standardized_chant_values["srclink"] = src_uri + chants_export.append(chant_values) - return standardized_chant_values + return JsonResponse(chants_export, safe=False) def json_sources_export(request) -> JsonResponse: @@ -501,7 +448,13 @@ def json_cid_export(request, cantus_id: str) -> JsonResponse: """ # the API in OldCantus appears to only return chants, and no sequences. - chants = Chant.objects.filter(cantus_id=cantus_id).filter(source__published=True) + chants = ( + Chant.objects.select_related( + "source", "source__holding_institution", "feast", "genre", "service" + ) + .filter(cantus_id=cantus_id) + .filter(source__published=True) + ) chant_dicts = [{"chant": build_json_cid_dictionary(c, request)} for c in chants] response = {"chants": chant_dicts} return JsonResponse(response) @@ -532,7 +485,9 @@ def build_json_cid_dictionary(chant, request) -> dict: "incipit": chant.incipit if chant.incipit else "", "feast": chant.feast.name if chant.feast else "", "genre": chant.genre.name if chant.genre else "", - "service": chant.service.name if chant.service else "", + "office": ( + chant.service.name if chant.service else "" + ), # We keep the office key for backwards compatibility with external applications "position": chant.position if chant.position else "", "cantus_id": chant.cantus_id if chant.cantus_id else "", "image": chant.image_link if chant.image_link else "", @@ -548,7 +503,7 @@ def build_json_cid_dictionary(chant, request) -> dict: return dictionary -def record_exists(rec_type: type[Model], pk: int) -> bool: +def record_exists(rec_type: Union[Chant, Source, Sequence, Article], pk: int) -> bool: """Determines whether record of specific type (chant, source, sequence, article) exists for a given pk Args: @@ -586,7 +541,9 @@ def get_user_id_from_old_indexer_id(pk: int) -> Optional[int]: return None -NODE_TYPES_AND_VIEWS = [ +NODE_TYPES_AND_VIEWS: list[ + tuple[Union[type[Chant], type[Source], type[Sequence], type[Article]], str] +] = [ (Chant, "chant-detail"), (Source, "source-detail"), (Sequence, "sequence-detail"), @@ -600,7 +557,7 @@ def get_user_id_from_old_indexer_id(pk: int) -> Optional[int]: NODE_ID_CUTOFF = 1_000_000 -def json_node_export(request, id: int) -> HttpResponse: +def json_node_export(request: HttpRequest, id: int) -> HttpResponse: """ returns all fields of the chant/sequence/source/indexer with the specified `id` """ @@ -612,12 +569,12 @@ def json_node_export(request, id: int) -> HttpResponse: raise Http404() user_id = get_user_id_from_old_indexer_id(id) - if get_user_id_from_old_indexer_id(id) is not None: + if user_id is not None: User = get_user_model() user = User.objects.filter(id=user_id) # in order to easily unpack the object's properties in `vals` below, `user` needs to be # a queryset rather than an individual object. - vals = dict(*user.values()) + vals: dict[str, Any] = dict(*user.values()) return JsonResponse(vals) # This seems to return the first object for which the node id matches. diff --git a/django/cantusdb_project/main_app/views/chant.py b/django/cantusdb_project/main_app/views/chant.py index 4bb38dc16..2d357da5f 100644 --- a/django/cantusdb_project/main_app/views/chant.py +++ b/django/cantusdb_project/main_app/views/chant.py @@ -7,6 +7,7 @@ from django.contrib.auth.mixins import UserPassesTestMixin from django.core.exceptions import PermissionDenied from django.db.models import Q, QuerySet +from django.forms import BaseModelForm from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.urls import reverse @@ -18,6 +19,7 @@ TemplateView, UpdateView, ) +from volpiano_display_utilities.latin_word_syllabification import LatinError from volpiano_display_utilities.cantus_text_syllabification import ( syllabify_text, flatten_syllabified_text, @@ -205,11 +207,14 @@ def get_context_data(self, **kwargs): # syllabification section if chant.volpiano: has_syl_text = bool(chant.manuscript_syllabized_full_text) - text_and_mel, _ = align_text_and_volpiano( - chant.get_best_text_for_syllabizing(), - chant.volpiano, - text_presyllabified=has_syl_text, - ) + try: + text_and_mel, _ = align_text_and_volpiano( + chant.get_best_text_for_syllabizing(), + chant.volpiano, + text_presyllabified=has_syl_text, + ) + except LatinError: + text_and_mel = None context["syllabized_text_with_melody"] = text_and_mel if project := chant.project: @@ -367,6 +372,9 @@ def get_context_data(self, **kwargs) -> dict: if search_position: search_parameters.append(f"position={search_position}") search_melodies: Optional[str] = self.request.GET.get("melodies") + # This was added to context so that we could implement #1635 and can be + # removed once that is undone. + context["melodies"] = search_melodies if search_melodies: search_parameters.append(f"melodies={search_melodies}") search_bar: Optional[str] = self.request.GET.get("search_bar") @@ -553,9 +561,9 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) # if searching in a specific source, pass the source into context if self.request.GET.get("source"): - context["source"] = Source.objects.get( - id=self.request.GET.get("source") - ).select_related("holding_institution", "feast", "service", "genre") + context["source"] = Source.objects.select_related( + "holding_institution" + ).get(id=self.request.GET.get("source")) return context @@ -653,6 +661,12 @@ def get_queryset(self) -> QuerySet: # If the "apply" button hasn't been clicked, return empty queryset if not self.request.GET: return Chant.objects.none() + # See #1635 re the following source exclusion. Temporarily disable volpiano display for this source. + if ( + self.request.GET.get("melodies") == "true" + and self.kwargs["source_pk"] == 680970 + ): + return Chant.objects.none() # Create a Q object to filter the QuerySet of Chants q_obj_filter = Q() @@ -760,13 +774,15 @@ class ChantCreateView(LoginRequiredMixin, UserPassesTestMixin, CreateView): template_name = "chant_create.html" form_class = ChantCreateForm pk_url_kwarg = "source_pk" + source: Source + latest_chant: Optional[Chant] def test_func(self): user = self.request.user source_id = self.kwargs.get(self.pk_url_kwarg) - source = get_object_or_404(Source, id=source_id) + self.source = get_object_or_404(Source, id=source_id) - return user_can_edit_chants_in_source(user, source) + return user_can_edit_chants_in_source(user, self.source) # if success_url and get_success_url not specified, will direct to chant detail page def get_success_url(self): @@ -784,8 +800,10 @@ def get_initial(self): """ try: latest_chant = self.source.chant_set.latest("date_updated") + self.latest_chant = latest_chant except Chant.DoesNotExist: # if there is no chant in source, start with folio 001r, and c_sequence 1 + self.latest_chant = None return { "folio": "001r", "feast": "", @@ -807,22 +825,12 @@ def get_initial(self): "image_link": latest_image, } - def dispatch(self, request, *args, **kwargs): - """Make sure the source specified in url exists before we display the form""" - self.source = get_object_or_404(Source, pk=kwargs["source_pk"]) - return super().dispatch(request, *args, **kwargs) - - def get_suggested_feasts(self): + def get_suggested_feasts(self, latest_chant: Chant) -> dict[Feast, int]: """based on the feast of the most recently edited chant, provide a list of suggested feasts that might follow the feast of that chant. Returns: a dictionary, with feast objects as keys and counts as values """ - try: - latest_chant = self.source.chant_set.latest("date_updated") - except Chant.DoesNotExist: - return None - current_feast = latest_chant.feast chants_that_end_current_feast = Chant.objects.filter( is_last_chant_in_feast=True, feast=current_feast @@ -843,31 +851,30 @@ def get_suggested_feasts(self): def get_context_data(self, **kwargs: Any) -> dict[Any, Any]: context = super().get_context_data(**kwargs) context["source"] = self.source - previous_chant: Optional[Chant] = None - try: - previous_chant = self.source.chant_set.latest("date_updated") - except Chant.DoesNotExist: - pass + previous_chant = self.latest_chant context["previous_chant"] = previous_chant - context["suggested_feasts"] = self.get_suggested_feasts() - - previous_cantus_id: Optional[str] = None + suggested_feasts = None + suggested_chants = None if previous_chant: + suggested_feasts = self.get_suggested_feasts(previous_chant) previous_cantus_id = previous_chant.cantus_id - - suggested_chants = None - if previous_cantus_id: - suggested_chants = get_suggested_chants(previous_cantus_id) + if previous_cantus_id: + suggested_chants = get_suggested_chants(previous_cantus_id) + context["suggested_feasts"] = suggested_feasts context["suggested_chants"] = suggested_chants return context def form_valid(self, form): - """compute source, incipit; folio/sequence (if left empty) - validate the form: add success/error message + """ + Validates the new chant. + + Custom validation steps are: + - Check if a chant with the same sequence and folio already exists in the source. + - Compute the chant incipit. + - Adds the "created_by" and "updated_by" fields to the chant. """ # compute source - form.instance.source = self.source # same effect as the next line - # form.instance.source = get_object_or_404(Source, pk=self.kwargs['source_pk']) + form.instance.source = self.source # compute incipit, within 30 charactors, keep words complete words = form.instance.manuscript_full_text_std_spelling.split(" ") @@ -903,8 +910,7 @@ def form_valid(self, form): "Chant '" + form.instance.incipit + "' created successfully!", ) return super().form_valid(form) - else: - return super().form_invalid(form) + return super().form_invalid(form) class ChantDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView): @@ -1123,11 +1129,18 @@ def get_context_data(self, **kwargs): has_syl_text = bool(chant.manuscript_syllabized_full_text) # Note: the second value returned is a flag indicating whether the alignment process # encountered errors. In future, this could be used to display a message to the user. - text_and_mel, _ = align_text_and_volpiano( - chant.get_best_text_for_syllabizing(), - chant.volpiano, - text_presyllabified=has_syl_text, - ) + try: + text_and_mel, _ = align_text_and_volpiano( + chant.get_best_text_for_syllabizing(), + chant.volpiano, + text_presyllabified=has_syl_text, + ) + except LatinError as err: + messages.error( + self.request, + "Error in aligning text and melody: " + str(err), + ) + text_and_mel = None context["syllabized_text_with_melody"] = text_and_mel user = self.request.user @@ -1234,12 +1247,20 @@ def get_initial(self): initial = super().get_initial() chant = self.get_object() has_syl_text = bool(chant.manuscript_syllabized_full_text) - syls_text, _ = syllabify_text( - text=chant.get_best_text_for_syllabizing(), - clean_text=True, - text_presyllabified=has_syl_text, - ) - self.flattened_syls_text = flatten_syllabified_text(syls_text) + try: + syls_text, _ = syllabify_text( + text=chant.get_best_text_for_syllabizing(), + clean_text=True, + text_presyllabified=has_syl_text, + ) + self.flattened_syls_text = flatten_syllabified_text(syls_text) + except LatinError as err: + messages.error( + self.request, + "Error in syllabifying text: " + str(err), + ) + syls_text = None + self.flattened_syls_text = "" initial["manuscript_syllabized_full_text"] = self.flattened_syls_text return initial