From 2dfc484145a43d4bba9e3c6d52f10fe36e2d0999 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 18 Jun 2024 09:07:05 -0400 Subject: [PATCH 01/13] feat!(models): add Project model Remove segment field from chants and sequences Add project field to chants and sequences --- ...emove_sequence_segment_project_and_more.py | 96 +++++++++++++++++++ .../main_app/models/__init__.py | 1 + .../main_app/models/base_chant.py | 9 +- .../main_app/models/project.py | 16 ++++ 4 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 django/cantusdb_project/main_app/migrations/0020_remove_chant_segment_remove_sequence_segment_project_and_more.py create mode 100644 django/cantusdb_project/main_app/models/project.py diff --git a/django/cantusdb_project/main_app/migrations/0020_remove_chant_segment_remove_sequence_segment_project_and_more.py b/django/cantusdb_project/main_app/migrations/0020_remove_chant_segment_remove_sequence_segment_project_and_more.py new file mode 100644 index 000000000..bedd0e86b --- /dev/null +++ b/django/cantusdb_project/main_app/migrations/0020_remove_chant_segment_remove_sequence_segment_project_and_more.py @@ -0,0 +1,96 @@ +# Generated by Django 4.2.11 on 2024-06-18 13:04 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("main_app", "0019_remove_source_rism_siglum_delete_rismsiglum"), + ] + + operations = [ + migrations.RemoveField( + model_name="chant", + name="segment", + ), + migrations.RemoveField( + model_name="sequence", + name="segment", + ), + migrations.CreateModel( + name="Project", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "date_created", + models.DateTimeField( + auto_now_add=True, help_text="The date this entry was created" + ), + ), + ( + "date_updated", + models.DateTimeField( + auto_now=True, help_text="The date this entry was updated" + ), + ), + ("name", models.CharField(max_length=63)), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_created_by_user", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "last_updated_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_last_updated_by_user", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.AddField( + model_name="chant", + name="project", + field=models.ForeignKey( + blank=True, + help_text="The project this chant belongs to. If left blank,this chant is considered part of the Cantus (default) project.", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="main_app.project", + ), + ), + migrations.AddField( + model_name="sequence", + name="project", + field=models.ForeignKey( + blank=True, + help_text="The project this chant belongs to. If left blank,this chant is considered part of the Cantus (default) project.", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="main_app.project", + ), + ), + ] diff --git a/django/cantusdb_project/main_app/models/__init__.py b/django/cantusdb_project/main_app/models/__init__.py index 35e764e0c..ef755c898 100644 --- a/django/cantusdb_project/main_app/models/__init__.py +++ b/django/cantusdb_project/main_app/models/__init__.py @@ -12,3 +12,4 @@ from main_app.models.source import Source from main_app.models.institution import Institution from main_app.models.institution_identifier import InstitutionIdentifier +from main_app.models.project import Project diff --git a/django/cantusdb_project/main_app/models/base_chant.py b/django/cantusdb_project/main_app/models/base_chant.py index 2419185e0..7885f538a 100644 --- a/django/cantusdb_project/main_app/models/base_chant.py +++ b/django/cantusdb_project/main_app/models/base_chant.py @@ -176,12 +176,15 @@ class Meta: # this field, populated by the populate_is_last_chant_in_feast script, exists in order to optimize .get_suggested_feasts() on the chant-create page is_last_chant_in_feast = models.BooleanField(blank=True, null=True) - segment = models.ForeignKey( - "Segment", + project = models.ForeignKey( + "Project", on_delete=models.PROTECT, null=True, blank=True, - help_text="The segment of the manuscript that contains this chant", + help_text=( + "The project this chant belongs to. If left blank," + "this chant is considered part of the Cantus (default) project." + ), ) # fragmentarium_id = models.CharField(blank=True, null=True, max_length=64) # # Digital Analysis of Chant Transmission diff --git a/django/cantusdb_project/main_app/models/project.py b/django/cantusdb_project/main_app/models/project.py new file mode 100644 index 000000000..bba9ca3e0 --- /dev/null +++ b/django/cantusdb_project/main_app/models/project.py @@ -0,0 +1,16 @@ +from main_app.models import BaseModel +from django.db import models + + +class Project(BaseModel): + """ + Chants can be tagged with the Project if their inventories are collected + as part of a particular project or initiative. Tagging a chant with a + project allows for the collection of project-specific chant data during + the inventory process and enables filtering by project during search. + """ + + name = models.CharField(max_length=63) + + def __str__(self): + return f"{self.name} ({self.id})" From 0ea44302f26833d06ece360fab9a82f282922554 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 18 Jun 2024 12:08:50 -0400 Subject: [PATCH 02/13] feat(commands): Create assign_sequences_to_bower_project command The assign_sequences_to_bower_project command fills in the new project field on sequences with the Clavis Sequentiarum project. Also, creates relevant test. --- .../commands/assign_chants_to_segments.py | 44 ------------------- .../assign_sequences_to_bower_project.py | 40 +++++++++++++++++ .../tests/test_assign_chants_to_segments.py | 37 ---------------- .../test_assign_sequences_to_bower_project.py | 30 +++++++++++++ 4 files changed, 70 insertions(+), 81 deletions(-) delete mode 100644 django/cantusdb_project/main_app/management/commands/assign_chants_to_segments.py create mode 100644 django/cantusdb_project/main_app/management/commands/assign_sequences_to_bower_project.py delete mode 100644 django/cantusdb_project/main_app/tests/test_assign_chants_to_segments.py create mode 100644 django/cantusdb_project/main_app/tests/test_assign_sequences_to_bower_project.py diff --git a/django/cantusdb_project/main_app/management/commands/assign_chants_to_segments.py b/django/cantusdb_project/main_app/management/commands/assign_chants_to_segments.py deleted file mode 100644 index e5e056329..000000000 --- a/django/cantusdb_project/main_app/management/commands/assign_chants_to_segments.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -This command is meant to be used one time toward solving issue -1420: Chants should belong to segments. - -This command iterates through all the sources in database and assigns -all chants in the database to the segment of the source they belong to. -""" - -from django.core.management.base import BaseCommand -from main_app.models import Source, Chant, Segment, Sequence - - -class Command(BaseCommand): - help = "Assigns all chants in the database to the segment of the source they belong to." - - def handle(self, *args, **options): - sources = Source.objects.all() - for source in sources: - segment = Segment.objects.get(id=source.segment_id) - chants = Chant.objects.filter(source=source) - sequences = Sequence.objects.filter(source=source) - chants_count = chants.count() - sequences_count = sequences.count() - if chants_count != 0 and sequences_count != 0: - self.stdout.write( - self.style.ERROR( - f"Source {source.id} has {chants_count} chants and {sequences_count} sequences." - ) - ) - continue - if chants_count > 0: - chants.update(segment=segment) - self.stdout.write( - self.style.SUCCESS( - f"Assigned {chants_count} chants in source {source.id} to segment {segment.id}." - ) - ) - else: - sequences.update(segment=segment) - self.stdout.write( - self.style.SUCCESS( - f"Assigned {sequences_count} sequences in source {source.id} to segment {segment.id}." - ) - ) diff --git a/django/cantusdb_project/main_app/management/commands/assign_sequences_to_bower_project.py b/django/cantusdb_project/main_app/management/commands/assign_sequences_to_bower_project.py new file mode 100644 index 000000000..dc59dc76c --- /dev/null +++ b/django/cantusdb_project/main_app/management/commands/assign_sequences_to_bower_project.py @@ -0,0 +1,40 @@ +""" +This command is meant to be used one time toward solving issue +1542, assigning chants and sequences, where necessary, to their appropriate +"project". + +This command assigns sequences to the Bower project. Chants +(in non-Bower sources) currently have +project. + +Note: This command can only be run *after* the Bower project has been created +in the database through the Admin interface. + +Note: This command is designed to be run once in order to complete the necessary +data migrations with the introduction of the Project model. It is not intended +for multiple runs. +""" + +from django.core.management.base import BaseCommand +from main_app.models import Project, Sequence + + +class Command(BaseCommand): + help = "Assigns all sequences to the Bower project." + + def handle(self, *args, **options): + sequences = Sequence.objects.all() + bower_project = Project.objects.get(name="Clavis Sequentiarum") + if not bower_project: + self.stdout.write( + self.style.ERROR( + "The Bower project does not exist. Please create it in the Admin interface." + ) + ) + return + sequences.update(project=bower_project) + self.stdout.write( + self.style.SUCCESS( + f"All sequences have been assigned to the {bower_project} project." + ) + ) diff --git a/django/cantusdb_project/main_app/tests/test_assign_chants_to_segments.py b/django/cantusdb_project/main_app/tests/test_assign_chants_to_segments.py deleted file mode 100644 index 6a66daa1e..000000000 --- a/django/cantusdb_project/main_app/tests/test_assign_chants_to_segments.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.test import TestCase -from django.core.management import call_command - -from main_app.models import Chant, Sequence - -from main_app.tests.make_fakes import make_fake_source, make_fake_segment - - -class AssignChantsToSegmentsTest(TestCase): - def test_assign_chants_to_segments(self): - segment_1 = make_fake_segment() - segment_2 = make_fake_segment() - source_1 = make_fake_source(segment=segment_1) - source_2 = make_fake_source(segment=segment_2) - sequence_source = make_fake_source(segment=segment_2) - for _ in range(5): - Chant.objects.create(source=source_1) - for _ in range(3): - Chant.objects.create(source=source_2) - for _ in range(4): - Sequence.objects.create(source=sequence_source) - all_chants = Chant.objects.all() - for chant in all_chants: - self.assertIsNone(chant.segment_id) - all_sequences = Sequence.objects.all() - for sequence in all_sequences: - self.assertIsNone(sequence.segment_id) - call_command("assign_chants_to_segments") - source_1_chants = Chant.objects.filter(source=source_1) - source_2_chants = Chant.objects.filter(source=source_2) - sequence_source_sequences = Sequence.objects.filter() - for chant in source_1_chants: - self.assertEqual(chant.segment_id, segment_1.id) - for chant in source_2_chants: - self.assertEqual(chant.segment_id, segment_2.id) - for sequence in sequence_source_sequences: - self.assertEqual(sequence.segment_id, segment_2.id) diff --git a/django/cantusdb_project/main_app/tests/test_assign_sequences_to_bower_project.py b/django/cantusdb_project/main_app/tests/test_assign_sequences_to_bower_project.py new file mode 100644 index 000000000..cb28960b8 --- /dev/null +++ b/django/cantusdb_project/main_app/tests/test_assign_sequences_to_bower_project.py @@ -0,0 +1,30 @@ +from django.test import TestCase +from django.core.management import call_command + +from main_app.models import Chant, Sequence, Project + +from main_app.tests.make_fakes import make_fake_source + + +class AssignSequencesToBowerProjectTest(TestCase): + def test_assign_sequences_to_bower_project(self): + project = Project.objects.create(name="Clavis Sequentiarum") + chant_source = make_fake_source() + sequence_source = make_fake_source() + for _ in range(5): + Chant.objects.create(source=chant_source) + for _ in range(4): + Sequence.objects.create(source=sequence_source) + all_chants = Chant.objects.all() + for chant in all_chants: + self.assertIsNone(chant.project_id) + all_sequences = Sequence.objects.all() + for sequence in all_sequences: + self.assertIsNone(sequence.project_id) + call_command("assign_sequences_to_bower_project") + all_chants = Chant.objects.all() + all_sequences = Sequence.objects.all() + for chant in all_chants: + self.assertIsNone(chant.project_id) + for sequence in all_sequences: + self.assertEqual(sequence.project_id, project.id) From 1216a2db8cf073c57d464eab2dcc3e9f4bdf1df0 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 18 Jun 2024 12:11:33 -0400 Subject: [PATCH 03/13] fix(templates): fix spelling of Clavis Sequentiarum in dropdown Refs #1544 --- django/cantusdb_project/templates/base.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/cantusdb_project/templates/base.html b/django/cantusdb_project/templates/base.html index 5a34c789e..e31558382 100644 --- a/django/cantusdb_project/templates/base.html +++ b/django/cantusdb_project/templates/base.html @@ -203,7 +203,7 @@
Cantus Sources - Clavis Sequentiarium (Sequence Database by Calvin + Clavis Sequentiarum (Sequence Database by Calvin Bower)
@@ -217,7 +217,7 @@
Chant Search Search by melody - Clavis Sequentiarium (Sequence Database by Calvin + Clavis Sequentiarum (Sequence Database by Calvin Bower)
From 64de5bc094efb05f8bc0e2d921003f5cf4c42043 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 18 Jun 2024 12:29:40 -0400 Subject: [PATCH 04/13] feat(models): Add Project model admin --- django/cantusdb_project/main_app/admin/__init__.py | 1 + django/cantusdb_project/main_app/admin/project.py | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 django/cantusdb_project/main_app/admin/project.py diff --git a/django/cantusdb_project/main_app/admin/__init__.py b/django/cantusdb_project/main_app/admin/__init__.py index 994faadaa..6c0fac619 100644 --- a/django/cantusdb_project/main_app/admin/__init__.py +++ b/django/cantusdb_project/main_app/admin/__init__.py @@ -11,3 +11,4 @@ from main_app.admin.source import SourceAdmin from main_app.admin.institution import InstitutionAdmin from main_app.admin.institution_identifier import InstitutionIdentifierAdmin +from main_app.admin.project import ProjectAdmin diff --git a/django/cantusdb_project/main_app/admin/project.py b/django/cantusdb_project/main_app/admin/project.py new file mode 100644 index 000000000..1bb98cb0b --- /dev/null +++ b/django/cantusdb_project/main_app/admin/project.py @@ -0,0 +1,9 @@ +from django.contrib import admin + +from main_app.admin.base_admin import BaseModelAdmin +from main_app.models import Project + + +@admin.register(Project) +class ProjectAdmin(BaseModelAdmin): + search_fields = ("name",) From 8084cfa3c252bde86bcea4a0605c9ecd73a563b7 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 18 Jun 2024 12:37:50 -0400 Subject: [PATCH 05/13] feat(chant-create;chant-edit): Add project field and benedicamus domino fields --- django/cantusdb_project/main_app/forms.py | 68 ++++++++----------- .../main_app/templates/chant_create.html | 22 +++--- .../main_app/templates/chant_edit.html | 21 +++--- .../static/js/chant_create.js | 16 ++--- .../cantusdb_project/static/js/chant_edit.js | 18 ++--- 5 files changed, 66 insertions(+), 79 deletions(-) diff --git a/django/cantusdb_project/main_app/forms.py b/django/cantusdb_project/main_app/forms.py index 4b7222c7a..2ae54e4d7 100644 --- a/django/cantusdb_project/main_app/forms.py +++ b/django/cantusdb_project/main_app/forms.py @@ -8,6 +8,7 @@ Feast, Source, Segment, + Project, Provenance, Century, Sequence, @@ -82,14 +83,13 @@ class Meta: "content_structure", "indexing_notes", "addendum", - # See issue #1521: Temporarily commenting out segment-related functions on Chant - # "segment", - # "liturgical_function", - # "polyphony", - # "cm_melody_id", - # "incipit_of_refrain", - # "later_addition", - # "rubrics", + "project", + "liturgical_function", + "polyphony", + "cm_melody_id", + "incipit_of_refrain", + "later_addition", + "rubrics", ] # the widgets dictionary is ignored for a model field with a non-empty # choices attribute. In this case, you must override the form field to @@ -148,14 +148,11 @@ class Meta: "Mass Alleluias. Punctuation is omitted.", ) - # See issue #1521: Temporarily commenting out segment-related functions on Chant - # segment = SelectWidgetNameModelChoiceField( - # queryset=Segment.objects.all().order_by("id"), - # required=True, - # initial=Segment.objects.get(id=4063), # Default to the "Cantus" segment - # help_text="Select the Database segment that the chant belongs to. " - # "In most cases, this will be the CANTUS segment.", - # ) + project = SelectWidgetNameModelChoiceField( + queryset=Project.objects.all().order_by("id"), + initial=None, + help_text="Select the project (if any) that the chant belongs to.", + ) # automatically computed fields # source and incipit are mandatory fields in model, @@ -280,14 +277,13 @@ class Meta: "manuscript_full_text_proofread", "volpiano_proofread", "proofread_by", - # See issue #1521: Temporarily commenting out segment-related functions on Chant - # "segment", - # "liturgical_function", - # "polyphony", - # "cm_melody_id", - # "incipit_of_refrain", - # "later_addition", - # "rubrics", + "project", + "liturgical_function", + "polyphony", + "cm_melody_id", + "incipit_of_refrain", + "later_addition", + "rubrics", ] widgets = { # manuscript_full_text_std_spelling: defined below (required) @@ -317,13 +313,12 @@ class Meta: "proofread_by": autocomplete.ModelSelect2Multiple( url="proofread-by-autocomplete" ), - # See issue #1521: Temporarily commenting out segment-related functions on Chant - # "polyphony": SelectWidget(), - # "liturgical_function": SelectWidget(), - # "cm_melody_id": TextInputWidget(), - # "incipit_of_refrain": TextInputWidget(), - # "later_addition": TextInputWidget(), - # "rubrics": TextInputWidget(), + "polyphony": SelectWidget(), + "liturgical_function": SelectWidget(), + "cm_melody_id": TextInputWidget(), + "incipit_of_refrain": TextInputWidget(), + "later_addition": TextInputWidget(), + "rubrics": TextInputWidget(), } manuscript_full_text_std_spelling = forms.CharField( @@ -348,13 +343,10 @@ class Meta: help_text="Each folio starts with '1'.", ) - # See issue #1521: Temporarily commenting out segment-related functions on Chant - # segment = SelectWidgetNameModelChoiceField( - # queryset=Segment.objects.all().order_by("id"), - # required=True, - # help_text="Select the Database segment that the chant belongs to. " - # "In most cases, this will be the CANTUS segment.", - # ) + project = SelectWidgetNameModelChoiceField( + queryset=Project.objects.all().order_by("id"), + help_text="Select the project (if any) that the chant belongs to.", + ) class SourceEditForm(forms.ModelForm): diff --git a/django/cantusdb_project/main_app/templates/chant_create.html b/django/cantusdb_project/main_app/templates/chant_create.html index 95f335c97..69082d3a4 100644 --- a/django/cantusdb_project/main_app/templates/chant_create.html +++ b/django/cantusdb_project/main_app/templates/chant_create.html @@ -90,16 +90,16 @@

Create Chant

{{ form.cantus_id }} - - +
@@ -147,11 +147,10 @@

Create Chant

{{ form.extra }}
- - +
@@ -187,9 +186,8 @@

Create Chant

- - - + --> +
diff --git a/django/cantusdb_project/main_app/templates/chant_edit.html b/django/cantusdb_project/main_app/templates/chant_edit.html index 8196d0d4b..97f6bd648 100644 --- a/django/cantusdb_project/main_app/templates/chant_edit.html +++ b/django/cantusdb_project/main_app/templates/chant_edit.html @@ -102,16 +102,15 @@ {{ form.melody_id }}
- - +
@@ -131,11 +130,10 @@ {{ form.extra.label_tag }} {{ form.extra }}
- - +
@@ -164,9 +162,8 @@ {% endif %}
- - - + --> +
{% if suggested_fulltext %}
diff --git a/django/cantusdb_project/static/js/chant_create.js b/django/cantusdb_project/static/js/chant_create.js index def18978e..fc3ee7027 100644 --- a/django/cantusdb_project/static/js/chant_create.js +++ b/django/cantusdb_project/static/js/chant_create.js @@ -7,18 +7,18 @@ window.addEventListener("load", function () { const standardText = document.getElementById('id_manuscript_full_text_std_spelling').value; document.getElementById('id_manuscript_full_text').value = standardText; } - // Add an event listener to the segment select field. + // Add an event listener to the segment project field. // If the user selects "Benedicamus Domino", show the additional fields - // in the "benedicamus-domino-segment-fields" div. By default, these + // in the "benedicamus-domino-project-fields" div. By default, these // are hidden. - const segmentSelectElem = document.getElementById("id_segment"); - segmentSelectElem.addEventListener("change", function () { - const benedicamusDominoSegmentFields = document.getElementById("benedicamus-domino-segment-fields"); - const selectedElemText = segmentSelectElem.options[segmentSelectElem.selectedIndex].text; + const projectSelectElem = document.getElementById("id_project"); + projectSelectElem.addEventListener("change", function () { + const benedicamusDominoProjectFields = document.getElementById("benedicamus-domino-project-fields"); + const selectedElemText = projectSelectElem.options[projectSelectElem.selectedIndex].text; if (selectedElemText === "Benedicamus Domino") { - benedicamusDominoSegmentFields.hidden = false; + benedicamusDominoProjectFields.hidden = false; } else { - benedicamusDominoSegmentFields.hidden = true; + benedicamusDominoProjectFields.hidden = true; } }); }) diff --git a/django/cantusdb_project/static/js/chant_edit.js b/django/cantusdb_project/static/js/chant_edit.js index b0687b043..0c355bbbe 100644 --- a/django/cantusdb_project/static/js/chant_edit.js +++ b/django/cantusdb_project/static/js/chant_edit.js @@ -47,21 +47,21 @@ window.addEventListener("load", function () { window.location.assign(url); } - // Add an event listener to the segment select field. + // Add an event listener to the project select field. // If the user selects "Benedicamus Domino", show the additional fields - // in the "benedicamus-domino-segment-fields" div. By default, these + // in the "benedicamus-domino-project-fields" div. By default, these // are hidden. - const segmentSelectElem = document.getElementById("id_segment"); - segmentSelectElem.addEventListener("change", function () { - const benedicamusDominoSegmentFields = document.getElementById("benedicamus-domino-segment-fields"); - const selectedElemText = segmentSelectElem.options[segmentSelectElem.selectedIndex].text; + const projectSelectElem = document.getElementById("id_project"); + projectSelectElem.addEventListener("change", function () { + const benedicamusDominoProjectFields = document.getElementById("benedicamus-domino-project-fields"); + const selectedElemText = projectSelectElem.options[projectSelectElem.selectedIndex].text; if (selectedElemText === "Benedicamus Domino") { - benedicamusDominoSegmentFields.hidden = false; + benedicamusDominoProjectFields.hidden = false; } else { - benedicamusDominoSegmentFields.hidden = true; + benedicamusDominoProjectFields.hidden = true; } }); - segmentSelectElem.dispatchEvent(new Event('change')); + projectSelectElem.dispatchEvent(new Event('change')); // ensures that the event listener is called on page load }) function autoFillSuggestedFullText(fullText) { From 6807731df35d4e8de5a1c909790cb694a7e541c9 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 18 Jun 2024 14:19:39 -0400 Subject: [PATCH 06/13] fix(tests): Add Project to make_fake_chant --- .../main_app/tests/make_fakes.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/django/cantusdb_project/main_app/tests/make_fakes.py b/django/cantusdb_project/main_app/tests/make_fakes.py index 931e347e8..15b9eb6eb 100644 --- a/django/cantusdb_project/main_app/tests/make_fakes.py +++ b/django/cantusdb_project/main_app/tests/make_fakes.py @@ -9,6 +9,7 @@ from main_app.models import Genre from main_app.models import Notation from main_app.models import Office +from main_app.models import Project from main_app.models import Provenance from main_app.models import Segment from main_app.models import Sequence @@ -151,7 +152,7 @@ def make_fake_chant( manuscript_syllabized_full_text=None, next_chant=None, differentia=None, - segment=None, + project=None, ) -> Chant: """Generates a fake Chant object.""" if source is None: @@ -189,8 +190,8 @@ def make_fake_chant( manuscript_syllabized_full_text = faker.sentence(20) if differentia is None: differentia = make_random_string(2) - if segment is None: - segment = make_fake_segment() + if project is None: + project = make_fake_project() chant = Chant.objects.create( source=source, @@ -225,7 +226,7 @@ def make_fake_chant( indexing_notes=faker.sentence(), json_info=None, next_chant=next_chant, - segment=segment, + project=project, ) chant.refresh_from_db() # several fields (e.g., incipit) are calculated automatically # upon chant save. By refreshing from db before returning, we ensure all the chant's fields @@ -303,6 +304,16 @@ def make_fake_segment(name: str = None, id: int = None) -> Segment: return segment +def make_fake_project(name: str = None, id: int = None) -> Project: + if name is None: + name = faker.sentence(nb_words=2) + if id is None: + project = Project.objects.create(name=name) + return project + project = Project.objects.create(name=name, id=id) + return project + + def make_fake_sequence(source=None, title=None, cantus_id=None) -> Sequence: """Generates a fake Sequence object.""" if source is None: From 8361d23a8ef16f86347d3b88089d330f5a8489af Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 18 Jun 2024 14:38:39 -0400 Subject: [PATCH 07/13] fix(tests): Remove segment from Chant view tests --- .../main_app/tests/test_views.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/django/cantusdb_project/main_app/tests/test_views.py b/django/cantusdb_project/main_app/tests/test_views.py index aca6039ac..02fda499a 100644 --- a/django/cantusdb_project/main_app/tests/test_views.py +++ b/django/cantusdb_project/main_app/tests/test_views.py @@ -3148,14 +3148,12 @@ def test_url_and_templates(self): def test_create_chant(self): source = make_fake_source() - segment = make_fake_segment() response = self.client.post( reverse("chant-create", args=[source.id]), { "manuscript_full_text_std_spelling": "initial", "folio": "001r", "c_sequence": "1", - "segment": segment.id, }, ) self.assertEqual(response.status_code, 302) @@ -3207,14 +3205,12 @@ def test_repeated_seq(self): TEST_FOLIO = "001r" # create some chants in the test source source = make_fake_source() - segment = make_fake_segment() for i in range(1, 5): Chant.objects.create( source=source, manuscript_full_text=faker.text(10), folio=TEST_FOLIO, c_sequence=i, - segment=segment, ) # post a chant with the same folio and seq url = reverse("chant-create", args=[source.id]) @@ -3225,7 +3221,6 @@ def test_repeated_seq(self): "manuscript_full_text_std_spelling": fake_text, "folio": TEST_FOLIO, "c_sequence": random.randint(1, 4), - "segment": segment.id, }, follow=True, ) @@ -3254,7 +3249,6 @@ def test_template_used(self): def test_volpiano_signal(self): source = make_fake_source() - segment = make_fake_segment() self.client.post( reverse("chant-create", args=[source.id]), { @@ -3266,7 +3260,6 @@ def test_volpiano_signal(self): "volpiano": "9abcdefg)A-B1C2D3E4F5G67?. yiz", # ^ ^ ^ ^ ^ ^ ^^^^^^^^ # clefs, accidentals, etc., to be deleted - "segment": segment.id, }, ) with patch("requests.get", mock_requests_get): @@ -3282,7 +3275,6 @@ def test_volpiano_signal(self): "folio": "001r", "c_sequence": "2", "volpiano": "abacadaeafagahaja", - "segment": segment.id, }, ) with patch("requests.get", mock_requests_get): @@ -3300,7 +3292,6 @@ def test_initial_values(self): feast: Feast = make_fake_feast() office: Office = make_fake_office() image_link: str = "https://www.youtube.com/watch?v=9bZkp7q19f0" - segment = make_fake_segment() self.client.post( reverse("chant-create", args=[source.id]), { @@ -3310,7 +3301,6 @@ def test_initial_values(self): "feast": feast.id, "office": office.id, "image_link": image_link, - "segment": segment.id, }, ) with patch("requests.get", mock_requests_get): @@ -3478,7 +3468,6 @@ def test_update_chant(self): folio = chant.folio c_sequence = chant.c_sequence - segment_id = chant.segment_id response = self.client.post( reverse("source-edit-chants", args=[source.id]), { @@ -3486,7 +3475,6 @@ def test_update_chant(self): "pk": chant.id, "folio": folio, "c_sequence": c_sequence, - "segment": segment_id, }, ) self.assertEqual(response.status_code, 302) @@ -3496,14 +3484,12 @@ def test_update_chant(self): self.assertEqual(chant.manuscript_full_text_std_spelling, "test") def test_volpiano_signal(self): - segment = make_fake_segment() source = make_fake_source() chant_1 = make_fake_chant( manuscript_full_text_std_spelling="ut queant lactose", source=source, folio="001r", c_sequence=1, - segment=segment, ) self.client.post( reverse("source-edit-chants", args=[source.id]), @@ -3516,7 +3502,6 @@ def test_volpiano_signal(self): "volpiano": "9abcdefg)A-B1C2D3E4F5G67?. yiz", # ^ ^ ^ ^ ^ ^ ^^^^^^^^ # clefs, accidentals, etc., to be deleted - "segment": segment.id, }, ) chant_1 = Chant.objects.get( @@ -3530,7 +3515,6 @@ def test_volpiano_signal(self): source=source, folio="001r", c_sequence=2, - segment=segment, ) expected_volpiano: str = "abacadaeafagahaja" expected_intervals: str = "1-12-23-34-45-56-67-78-8" @@ -3541,7 +3525,6 @@ def test_volpiano_signal(self): "folio": "001r", "c_sequence": "2", "volpiano": "abacadaeafagahaja", - "segment": segment.id, }, ) with patch("requests.get", mock_requests_get): @@ -3595,7 +3578,6 @@ def test_proofread_chant(self): "folio": folio, "c_sequence": c_sequence, "manuscript_full_text_std_spelling": ms_std, - "segment": chant.segment_id, }, ) self.assertEqual(response.status_code, 302) # 302 Found From 017a67f69d84d9d50d38d822dd598ac1d42ca941 Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 20 Jun 2024 14:36:21 +0000 Subject: [PATCH 08/13] test(cantusindex): add missing tests - tests and mock data for get_merged_cantus_ids and get_ci_text_search functions are added. - old cantus index domain is added as a global variable --- django/cantusdb_project/cantusindex.py | 8 +- .../main_app/tests/mock_cantusindex_data.py | 311 ++++++++++++++++++ .../main_app/tests/test_functions.py | 132 +++++++- 3 files changed, 444 insertions(+), 7 deletions(-) diff --git a/django/cantusdb_project/cantusindex.py b/django/cantusdb_project/cantusindex.py index 3bd7c6284..7980b2cc6 100644 --- a/django/cantusdb_project/cantusindex.py +++ b/django/cantusdb_project/cantusindex.py @@ -10,6 +10,7 @@ from requests.exceptions import SSLError, Timeout, HTTPError CANTUS_INDEX_DOMAIN: str = "https://cantusindex.uwaterloo.ca" +OLD_CANTUS_INDEX_DOMAIN: str = "https://cantusindex.org" DEFAULT_TIMEOUT: float = 2 # seconds NUMBER_OF_SUGGESTED_CHANTS: int = 3 # this number can't be too large, # since for each suggested chant, we make a request to Cantus Index. @@ -151,10 +152,10 @@ def get_merged_cantus_ids() -> Optional[list[Optional[dict]]]: # We have to use the old CI domain since the API is still not available on # cantusindex.uwaterloo.ca. Once it's available, we can use get_json_from_ci_api # json: Union[dict, list, None] = get_json_from_ci_api(endpoint_path) - uri: str = f"https://cantusindex.org{endpoint_path}" + uri: str = f"{OLD_CANTUS_INDEX_DOMAIN}{endpoint_path}" try: response: requests.Response = requests.get(uri, timeout=DEFAULT_TIMEOUT) - except requests.exceptions.Timeout: + except (SSLError, Timeout, HTTPError): return None if not response.status_code == 200: return None @@ -178,7 +179,8 @@ def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]: # We have to use the old CI domain since this API is still not available on # cantusindex.uwaterloo.ca. Once it's available, we can use get_json_from_ci_api # json: Union[dict, list, None] = get_json_from_ci_api(uri) - uri: str = f"https://cantusindex.org/json-text/{search_term}" + endpoint_path: str = f"/json-text/{search_term}" + uri: str = f"{OLD_CANTUS_INDEX_DOMAIN}{endpoint_path}" try: response: requests.Response = requests.get( uri, diff --git a/django/cantusdb_project/main_app/tests/mock_cantusindex_data.py b/django/cantusdb_project/main_app/tests/mock_cantusindex_data.py index 240ea0092..01cfa93a9 100644 --- a/django/cantusdb_project/main_app/tests/mock_cantusindex_data.py +++ b/django/cantusdb_project/main_app/tests/mock_cantusindex_data.py @@ -383,3 +383,314 @@ encoding="utf-8-sig", ) mock_json_cid_909030_json: dict = ujson.loads(mock_json_cid_909030_text) + + +########################################################################## +### mocking requests.get("https://cantusindex.org/json-merged-chants") ### +########################################################################## + +# Smaller sample of the full content. +mock_get_merged_cantus_ids_text: str = """ +[ + {"old": "g00831", "new": "920023", "date": "0000-00-00"}, + {"old": "920027a", "new": "920027", "date": "0000-00-00"}, + {"old": "g01393.1", "new": "g01393", "date": "0000-00-00"}, + {"old": "g00693a.1", "new": "g00693a", "date": "0000-00-00"}, + {"old": "g00838", "new": "008310", "date": "0000-00-00"}, + {"old": "g01132", "new": "503001", "date": "0000-00-00"}, + {"old": "g00681", "new": "g00678c", "date": "0000-00-00"}, + {"old": "g00682", "new": "g00678d", "date": "0000-00-00"}, + {"old": "g00683", "new": "g00678e", "date": "0000-00-00"}, + {"old": "g00684", "new": "g00678f", "date": "0000-00-00"}, + {"old": "g00685", "new": "g00678g", "date": "0000-00-00"}, + {"old": "g02384", "new": "g02374b", "date": "0000-00-00"}, + {"old": "g02494", "new": "g02373c", "date": "2016-08-25"}, + {"old": "g00980", "new": "001287", "date": "2016-08-25"}, + {"old": "g00241", "new": "g00240a", "date": "2016-12-29"}, + {"old": "g01340", "new": "g01339a", "date": "2016-12-30"}, + {"old": "g00476", "new": "g00475a", "date": "2016-12-30"}, + {"old": "g00047", "new": "g00046a", "date": "2016-12-30"}, + {"old": "g02487", "new": "g00029a", "date": "2016-12-30"}, + {"old": "g00413", "new": "g00412a", "date": "2017-01-02"} +] +""" +# We add the expected BOM in the response when using the old +# Cantus Index domain that we have to handle correctly. +utf8_bom: bytes = b"\xef\xbb\xbf\xef\xbb\xbf" +mock_get_merged_cantus_ids_content: bytes = utf8_bom + bytes( + mock_get_merged_cantus_ids_text, + encoding="utf-8", +) + + +######################################################################### +### mocking requests.get("https://cantusindex.org/json-text/qui+est") ### +######################################################################### +mock_get_ci_text_search_quiest_text: str = """ +[ + { + "cid": "001774", + "fulltext": "Caro et sanguis non revelavit tibi sed pater meus qui est in caelis", + "genre": "A" + }, + { + "cid": "002191", + "fulltext": "Dicebat Jesus turbis Judaeorum et principibus sacerdotum qui est ex deo verba dei audit responderunt Judaei et dixerunt ei nonne bene dicimus nos quia Samaritanus es tu et daemonium habes respondit Jesus ego daemonium non habeo sed honorifico patrem meum et vos inhonorastis me", + "genre": "A" + }, + { + "cid": "002303", + "fulltext": "Dixit Jesus turbis quis ex vobis arguet me de peccato si veritatem dico quare vos non creditis mihi qui est ex deo verba dei audit propterea vos non auditis quia ex deo non estis", + "genre": "A" + }, + { + "cid": "003257", + "fulltext": "In medio et in circuitu sedis dei quattuor animalia senas alas habentia oculis undique plena non cessant nocte ac die dicere sanctus sanctus sanctus dominus deus omnipotens qui erat et qui est et qui venturus est", + "genre": "A" + }, + { + "cid": "003857", + "fulltext": "Natus est nobis hodie salvator qui est Christus dominus in civitate David", + "genre": "A" + }, + { + "cid": "003870", + "fulltext": "Nemo ascendit in caelum nisi qui de caelo descendit filius hominis qui est in caelo alleluia", + "genre": "A" + }, + { + "cid": "004470", + "fulltext": "Qui est ex deo verba dei audit vos non auditis quia ex deo non estis", + "genre": "A" + }, + { + "cid": "004741", + "fulltext": "Sancti vero martyres Crispinus et Crispinianus audacter regi impio responderunt dicentes pro Christi amore nos venisse fatemur qui est deus verus in trinitate unus cui servimus in fide ac dilectione devoti", + "genre": "A" + }, + { + "cid": "004771", + "fulltext": "Sanctus autem Cucuphas ad praesidem dixit ego deum alium nescio praeter dominum qui est verus deus quem corde credo ore confiteor et omni studio praedico", + "genre": "A" + }, + { + "cid": "004796", + "fulltext": "Sanctus sanctus sanctus dominus deus omnipotens qui erat et qui est et qui venturus est", + "genre": "A" + }, + { + "cid": "004910", + "fulltext": "Si quis mihi ministraverit honorificabit eum pater meus qui est in caelis dicit dominus", + "genre": "A" + }, + { + "cid": "200554", + "fulltext": "Beatus es Simon Bar Jona quia caro et sanguis non revelavit tibi sed pater meus qui est in celsis dicit dominus", + "genre": "A" + }, + { + "cid": "201972", + "fulltext": "Gloriam deo in excelsis populi concinunt qui est semper mirabilis in suis sanctis et quos hodie ecclesia mater transmisit ad caelestia regna", + "genre": "A" + }, + { + "cid": "203255", + "fulltext": "Nil territa supplicio sic loquitur Fabricio tua ficta sculptilia sunt barathri daemonia caelos fecit solus deus qui est rex et sponsus meus", + "genre": "A" + }, + {"cid": "204438", "fulltext": "Sancti dei omnes qui estis (...)", "genre": "A"}, + { + "cid": "204637", + "fulltext": "Si veritatem dico quare non creditis mihi qui est ex deo verba dei audit", + "genre": "A" + }, + { + "cid": "206961", + "fulltext": "Sanctus pater Benedictus elevatis in aera oculis vidit spiritum germane sororis sancte videlicet Scholasticae virginis mirabiliter ascendentem ad deum qui est mirabilis dominus in altis", + "genre": "A" + }, + { + "cid": "a00438", + "fulltext": "Angeli eorum semper vident faciem patris mei qui est in caelis", + "genre": "A" + }, + { + "cid": "203255.1", + "fulltext": "Nil territa supplicio sic loquitur Fabricio deorum tuorum numina sunt barathri daemonia caelos deus fecit unus qui est rex et sponsus meus", + "genre": "A" + }, + { + "cid": "g00069.1", + "fulltext": "Responsum accepit Simeon pro eo qui Christum rogabat nigiter a spiritu sancto brandemis aetate dignitate quoque non visurum se mortem nisi videret Christum domini natum Maria virgine et cum inducerent puerum in templum parentes illius exsultant meo accepit eum in ulnas suas et benedixit deum et dixit cum laetitiam mensa nunc dimittis domine servum tuum in pace quia meruit conspicem Christum qui est rex regum et dominus in enim", + "genre": "A" + }, + { + "cid": "002303.1", + "fulltext": "Dixit dominus Jesus turbis judaeorum et principibus sacerdotum quis ex vobis arguet me de peccato si veritatem dico quare vos non creditis mihi qui est ex deo verba dei audit propterea vos non auditis quia ex deo non estis", + "genre": "A" + }, + { + "cid": "004485a", + "fulltext": "Si quis mihi ministraverit honorificabit eum pater meus qui est in caelis", + "genre": "AV" + }, + { + "cid": "g02235", + "fulltext": "Alleluia Beatus es Simon Petre quia caro et sanguis non revelavit tibi sed pater meus qui est in caelis", + "genre": "Al" + }, + { + "cid": "g02604", + "fulltext": "Alleluia Tu es Simon Bar Jona quia caro et sanguis non revelavit verbum patris sed ipse pater qui est in caelis", + "genre": "Al" + }, + { + "cid": "g02762", + "fulltext": "Alleluia Natus est nobis hodie salvator qui est Christus dominus in civitate David ", + "genre": "Al" + }, + { + "cid": "g00029a", + "fulltext": "Beatus es Simon Petre quia caro et sanguis non revelabit tibi sed pater meus qui est in caelis", + "genre": "AlV" + }, + { + "cid": "g01299e", + "fulltext": "Si quis mihi ministraverit honorificabit eum pater meus qui est in caelis", + "genre": "CmV" + }, + { + "cid": "g01293n", + "fulltext": "Si quis mihi ministraverit honorificabit eum pater meus qui est in caelis", + "genre": "CmV" + }, + { + "cid": "g00217", + "fulltext": "Si exprobramini in nomine Christi beati eritis quoniam quod est honoris gloriae et virtutis dei et qui est ejus spiritus super vos requiescet", + "genre": "GrV" + }, + { + "cid": "a00902", + "fulltext": "Cives caelestis patriae regi regum concinite qui est supremus opifex civitatis uranicae in cujus aedificio talis exstat fundatio", + "genre": "H" + }, + { + "cid": "a03400", + "fulltext": "Jam suae Christus hospitae dule parat convivium dum Marthae bene meritae caeleste confert gaudium | Quam ubertim inebriat de se pascit et satiat pro choro discumbentium sanctorum dans consortium | Divinitatis glorias Christi cernens quem paverat beatas agit gratias quod eam sic remunerat | O mira dei largitas quae centuplum repraemiat o ingens liberalitas quae mercedem sic ampliat | Christo Martha obsequitur intenta ministerio sed hanc Christus prosequitur aeternitatis bravio | Illi sit laus et gloria qui est sanctorum praemium qui Marthae per suffragium det nobis caeli solium", + "genre": "H" + }, + { + "cid": "ah20140", + "fulltext": "Mirabile mysterium Deus creator omnium per incorruptam virginem nostrum suscepit hominem et nata mater Patre est qui natus matre pater est | Credit Eva diabolo Maria credit angelo per illam mors introiit per istam vita rediit perdiderat hec condita hec restauravit perdita | Cedrus alta de Libano sub nostrae vallis ysopo cum visitavit Jericho cipressus fit ex platano cinnamomum ex balsamo benedicamus Domino | Usiae gigas geminae assumpto Deus homine alvo conceptus feminae non ex virili semine natus est rex ab homine Jesus est dictus nomine | De spinis uva legitur de stella lux exortitur de petra fons elicitur de virga flos egreditur de monte lapis lapsus est de lapide mons factus est | Qui fuit erit et qui est qui loquebatur praesens est nobiscum est rex Israel qui dicitur Emmanuel nos ergo multifarias Deo dicamus gratias", + "genre": "H" + }, + { + "cid": "100345", + "fulltext": "Venite cuncti qui estis vocati antiquam Simeonem imitantes qui desiderabat redemptorem suum in templo domini praesentari", + "genre": "I" + }, + { + "cid": "g01178", + "fulltext": "Dum clamarem ad dominum exaudivit vocem meam ab his qui appropinquant mihi et humiliavit eos qui est ante saecula et manet in aeternum jacta cogitatum tuum in domino et ipse te enutriet", + "genre": "In" + }, + { + "cid": "g02699", + "fulltext": "Viderunt ingressus tuos deus ingressus dei mei regis mei qui est in sancto", + "genre": "In" + }, + { + "cid": "g00488c", + "fulltext": "Haec enim dicit dominus spiritus meus qui est in te verba mea", + "genre": "InV" + }, + { + "cid": "g03280", + "fulltext": "Beatus es Simon Petre quia caro et sanguis non revelavit tibi sed pater meus qui est in caelis", + "genre": "Of" + }, + { + "cid": "g02676", + "fulltext": "Beatus es Simon Petre quia caro et sanguis non revelavit tibi sed pater meus qui est in caelis dicit dominus", + "genre": "OfV" + }, + { + "cid": "g00035a", + "fulltext": "Beatus es Simon Petre quia caro et sanguis non revelavit tibi sed pater meus qui est in caelis dicit dominus", + "genre": "OfV" + }, + { + "cid": "920054", + "fulltext": "In finem in carminibus intellectus David | Exaudi deus orationem meam et ne despexeris deprecationem meam | Intende mihi et exaudi me contristatus sum in exercitatione mea et conturbatus sum | A voce inimici et a tribulatione peccatoris quoniam declinaverunt in me iniquitates et in ira molesti erant mihi | Cor meum conturbatum est in me et formido mortis cecidit super me | Timor et tremor venerunt super me et contexerunt me tenebrae | Et dixi quis dabit mihi pennas sicut columbae et volabo et requiescam | Ecce elongavi fugiens et mansi in solitudine | Exspectabam eum qui salvum me fecit a pusillanimitate spiritus et tempestate | Praecipita domine divide linguas eorum quoniam vidi iniquitatem et contradictionem in civitate | Die ac nocte circumdabit eam super muros ejus iniquitas et labor in medio ejus | Et injustitia et non defecit de plateis ejus usura et dolus | Quoniam si inimicus meus maledixisset mihi sustinuissem utique et si is qui oderat me super me magna locutus fuisset abscondissem me forsitan ab eo | Tu vero homo unanimis dux meus et notus meus | Qui simul mecum dulces capiebas cibos in domo dei ambulavimus cum consensu | Veniat mors super illos et descendant in infernum viventes quoniam nequitiae in habitaculis eorum in medio eorum | Ego autem ad deum clamavi et dominus salvabit me | Vespere et mane et meridie narrabo et annuntiabo et exaudiet vocem meam | Redimet in pace animam meam ab his qui appropinquant mihi quoniam inter multos erant mecum | Exaudiet deus et humiliabit illos qui est ante saecula non enim est illis commutatio et non timuerunt deum | Extendit manum suam in retribuendo contaminaverunt testamentum ejus | Divisi sunt ab ira vultus ejus et appropinquavit cor illius molliti sunt sermones ejus super oleum et ipsi sunt jacula | Jacta super dominum curam tuam et ipse te enutriet non dabit in aeternum fluctuationem justo | Tu vero deus deduces eos in puteum interitus viri sanguinum et dolosi non dimidiabunt dies suos ego autem sperabo in te domine", + "genre": "PS" + }, + { + "cid": "920067", + "fulltext": "In finem psalmus cantici ipsi David | Exsurgat deus et dissipentur inimici ejus et fugiant qui oderunt eum a facie ejus | Sicut deficit fumus deficiant sicut fluit cera a facie ignis sic pereant peccatores a facie dei | Et justi epulentur et exsultent in conspectu dei et delectentur in laetitia | Cantate deo psalmum dicite nomini ejus iter facite ei qui ascendit super occasum dominus nomen illi exsultate in conspectu ejus turbabuntur a facie ejus | Patris orphanorum et judicis viduarum deus in loco sancto suo | Deus qui inhabitare facit unius moris in domo qui educit vinctos in fortitudine similiter eos qui exasperant qui habitant in sepulcris | Deus cum egredereris in conspectu populi tui cum pertransires in deserto | Terra mota est etenim caeli distillaverunt a facie dei Sinai a facie dei Israel | Pluviam voluntariam segregabis deus hereditati tuae et infirmata est tu vero perfecisti eam | Animalia tua habitabunt in ea parasti in dulcedine tua pauperi deus dominus dabit verbum evangelizantibus virtute multa | Rex virtutum dilecti dilecti et speciei domus dividere spolia | Si dormiatis inter medios cleros pennae columbae deargentatae et posteriora dorsi ejus in pallore auri | Dum discernit caelestis reges super eam nive dealbabuntur in Selmon | Mons dei mons pinguis mons coagulatus mons pinguis | Ut quid suspicamini montes coagulatos mons in quo beneplacitum est deo habitare in eo etenim dominus habitabit in finem | Currus dei decem milibus multiplex milia laetantium dominus in eis in Sina in sancto | Ascendisti in altum coepisti captivitatem accepisti dona in hominibus etenim non credentes inhabitare dominum deum | Benedictus dominus die cottidie prosperum iter faciet nobis deus salutarium nostrorum | Deus noster deus salvos faciendi et domini domini exitus mortis | Verumtamen deus confringet capita inimicorum suorum verticem capilli perambulantium in delictis suis | Dixit dominus ex Basan convertam convertam in profundum maris | Ut intingatur pes tuus in sanguine lingua canum tuorum ex inimicis ab ipso | Viderunt ingressus tuos deus ingressus dei mei regis mei qui est in sancto | Praevenerunt principes conjuncti psallentibus in medio juvencularum tympanistriarum | In ecclesiis benedicite deo domino de fontibus Israel | Ibi Benjamin adolescentulus in mentis excessu principes Juda duces eorum principes Zabulon principes Nephthali | Manda deus virtuti tuae confirma hoc deus quod operatus es in nobis | A templo tuo in Jerusalem tibi offerent reges munera | Increpa feras arundinis congregatio taurorum in vaccis populorum ut excludant eos qui probati sunt argento dissipa gentes quae bella volunt | Venient legati ex Aegypto Aethiopia praeveniet manus ejus deo | Regna terrae cantate deo psallite domino psallite deo | Qui ascendit super caelum caeli ad orientem ecce dabit voci suae vocem virtutis | Date gloriam deo super Israel magnificentia ejus et virtus ejus in nubibus | Mirabilis deus in sanctis suis deus Israel ipse dabit virtutem et fortitudinem plebi suae benedictus deus", + "genre": "PS" + }, + { + "cid": "006088", + "fulltext": "Angelus ad pastores ait annuntio vobis gaudium magnum quod erit omni populo quia natus est vobis salvator qui est Christus dominus in civitate David", + "genre": "R" + }, + { + "cid": "006206", + "fulltext": "Beatus es Simon Bar Jona quia caro et sanguis non revelavit tibi sed pater meus qui est in caelis dicit dominus", + "genre": "R" + }, + { + "cid": "006520", + "fulltext": "Dominus Jesus Christus qui est caput corporis ecclesiae per singula sua membra se colligit in caelum cottidie in quibus sanctum Findanum gloriae confessorum coaequavit quem de hac convalle lacrimarum hodie ad aeternae vitae gaudia sublevavit", + "genre": "R" + }, + { + "cid": "007311", + "fulltext": "Oculis caeci nati homo dei digitos imprimens dixit dominus Jesus Christus qui est vera lux quae illuminat credentes ipse te per invocationem sancti nominis sui illuminare dignetur", + "genre": "R" + }, + {"cid": "601691", "fulltext": "Omnes sancti qui estis (...)", "genre": "R"}, + { + "cid": "a00115", + "fulltext": "Dicebat dominus Jesus turbis Judaeorum et principibus sacerdotum quis ex vobis arguet me de peccato si veritatem dico quare non creditis mihi qui est ex deo verba dei audit ergo vos non auditis quia ex deo non estis ego autem a deo processi", + "genre": "R" + }, + { + "cid": "a00546", + "fulltext": "O vos angeli qui custoditis populos quorum forma fulget in facie vestra et o vos archangeli qui suscipitis animas justorum et vos virtutes potestates principatus dominationes et throni qui estis computati in quintum secretum numerum et o vos cherubin et seraphin sigillum secretorum dei sit laus vobis qui loculum antiqui cordis in fonte aspicitis", + "genre": "R" + }, + { + "cid": "a07762", + "fulltext": "Ab intus in fimbriis aureis circumamicta varietate haec turba virginum ei se ornari studuit qui est forma speciosus prae filiis hominum", + "genre": "R" + }, + { + "cid": "g02501", + "fulltext": "Columba aspexit per cancellos fenestrae ubi ante faciem ejus sudando sudavit balsamum de lucido Maximino calor solis exarsit et in tenebras resplenduit unde gemma surrexit in aedificatione templi purissimi cordis benevoli iste turris excelsa de ligno Libani et cupresso facta hyacintho et sardio ornata est urbs praecellens artes aliorum artificum ipse velox cervus cucurrit ad fontem purissimae aquae fluentis de fortissimo lapide qui dulcia aromata irrigavit o pigmentarii qui estis in suavissima viriditate ortorum regis ascendentes in altum quando sanctum sacrificium in arietibus perfecistis inter vos fulget hic artifex paries templi qui desideravit alas aquilae osculando nutricem sapientiam in gloriosa fecunditate ecclesiae o Maximine mons et vallis es et in utroque alta aedificatio appares ubi capricornus cum elephante exivit et sapientia in deliciis fuit tu es fortis et suavis in cerimoniis et in coruscatione altaris ascendens ut fumus aromatum ad columnam laudis ubi intercedis pro populo qui tendit ad speculum lucis cui laus est in altis", + "genre": "Sq" + } +] +""" + +# We add the expected BOM in the response when using the old +# Cantus Index domain that we have to handle correctly. +utf8_bom: bytes = b"\xef\xbb\xbf\xef\xbb\xbf" +mock_get_ci_text_search_quiest_content: bytes = utf8_bom + bytes( + mock_get_ci_text_search_quiest_text, + encoding="utf-8", +) + +######################################################################### +### mocking requests.get("https://cantusindex.org/json-text/123xyz") ### +######################################################################### + + +# We add the expected BOM in the response when using the old +# Cantus Index domain that we have to handle correctly. +mock_get_ci_text_search_123xyz_text: str = "" +utf8_bom: bytes = b"\xef\xbb\xbf\xef\xbb\xbf" +mock_get_ci_text_search_123xyz_content: bytes = utf8_bom + bytes( + mock_get_ci_text_search_123xyz_text, + encoding="utf-8", +) diff --git a/django/cantusdb_project/main_app/tests/test_functions.py b/django/cantusdb_project/main_app/tests/test_functions.py index c5eca19b8..de4774f1d 100644 --- a/django/cantusdb_project/main_app/tests/test_functions.py +++ b/django/cantusdb_project/main_app/tests/test_functions.py @@ -3,6 +3,7 @@ from django.test import TestCase from typing import Union, Optional from unittest.mock import patch +from requests.exceptions import SSLError, Timeout, HTTPError from main_app.models import ( Chant, Source, @@ -19,7 +20,10 @@ get_suggested_chants, get_json_from_ci_api, CANTUS_INDEX_DOMAIN, + OLD_CANTUS_INDEX_DOMAIN, get_suggested_fulltext, + get_merged_cantus_ids, + get_ci_text_search, ) # run with `python -Wa manage.py test main_app.tests.test_functions` @@ -68,7 +72,7 @@ def mock_requests_get(url: str, timeout: float) -> MockResponse: if timeout < 0.001: raise requests.exceptions.ConnectTimeout - if not (CANTUS_INDEX_DOMAIN in url): + if not (CANTUS_INDEX_DOMAIN or OLD_CANTUS_INDEX_DOMAIN in url): raise NotImplementedError( f"mock_requests_get is only set up to mock calls to Cantus Index. " f"The protocol and domain of url {url} do not correspond to those of Cantus Index." @@ -147,11 +151,41 @@ def mock_requests_get(url: str, timeout: float) -> MockResponse: text=None, json=None, ) + elif f"{OLD_CANTUS_INDEX_DOMAIN}/json-text/" in url: + if url.endswith("qui+est"): + return MockResponse( + status_code=200, + content=mock_cantusindex_data.mock_get_ci_text_search_quiest_content, + text=mock_cantusindex_data.mock_get_ci_text_search_quiest_text, + json=None, + ) + elif url.endswith("123xyz"): + return MockResponse( + status_code=200, + content=mock_cantusindex_data.mock_get_ci_text_search_123xyz_content, + text=mock_cantusindex_data.mock_get_ci_text_search_123xyz_text, + json=None, + ) + else: + return MockResponse( + status_code=500, + content=None, + text=None, + json=None, + ) + elif f"{OLD_CANTUS_INDEX_DOMAIN}/json-merged-chants" in url: + return MockResponse( + status_code=200, + content=mock_cantusindex_data.mock_get_merged_cantus_ids_content, + text=mock_cantusindex_data.mock_get_merged_cantus_ids_text, + json=None, + ) + else: raise NotImplementedError( - f"mock_requests_get is only set up to imitate only the /json-nextchants/ " - f"and /json-cid/ endpoints on Cantus Index. The path of the url {url} does " - f"not match either of these endpoints." + f"mock_requests_get is only set up to imitate only the /json-nextchants/, " + f"/json-cid/, and /json-text/ endpoints on Cantus Index. The path of the url " + f"{url} does not match either of these endpoints." ) @@ -418,3 +452,93 @@ def test_get_suggested_fulltext(self) -> None: with patch("requests.get", mock_requests_get): fulltext = get_suggested_fulltext("999999") self.assertIsNone(fulltext) + + def test_get_merged_cantus_ids(self) -> None: + with self.subTest("Test valid response"): + with patch("requests.get", mock_requests_get): + results = get_merged_cantus_ids() + self.assertIsInstance(results, list) + self.assertEqual(len(results), 20) + self.assertEqual(results[0]["old"], "g00831") + self.assertEqual(results[0]["new"], "920023") + self.assertEqual(results[0]["date"], "0000-00-00") + + with self.subTest("Test server error"): + mock_response = MockResponse( + status_code=500, + text=None, + json=None, + content=None, + ) + with patch("requests.get", return_value=mock_response): + results = get_merged_cantus_ids() + self.assertIsNone(results) + + with self.subTest("Test timeout"): + with patch("requests.get", side_effect=Timeout): + results = get_merged_cantus_ids() + self.assertRaises(Timeout) + self.assertIsNone(results) + + with self.subTest("Test SSLError"): + with patch("requests.get", side_effect=SSLError): + results = get_merged_cantus_ids() + self.assertRaises(SSLError) + self.assertIsNone(results) + + with self.subTest("Test HTTPError"): + with patch("requests.get", side_effect=HTTPError): + results = get_merged_cantus_ids() + self.assertRaises(HTTPError) + self.assertIsNone(results) + + def test_get_ci_text_search(self) -> None: + with self.subTest("Test valid search term"): + with patch("requests.get", mock_requests_get): + results = get_ci_text_search("qui+est") + self.assertIsInstance(results, list) + self.assertEqual(len(results), 50) + self.assertEqual(results[0]["cid"], "001774") + self.assertEqual( + results[0]["fulltext"], + "Caro et sanguis non revelavit tibi sed pater meus qui est in caelis", + ) + self.assertEqual(results[1]["cid"], "002191") + self.assertEqual( + results[1]["fulltext"], + "Dicebat Jesus turbis Judaeorum et principibus sacerdotum qui est ex deo verba dei audit responderunt Judaei et dixerunt ei nonne bene dicimus nos quia Samaritanus es tu et daemonium habes respondit Jesus ego daemonium non habeo sed honorifico patrem meum et vos inhonorastis me", + ) + + with self.subTest("Test invalid search term"): + with patch("requests.get", mock_requests_get): + results = get_ci_text_search("123xyz") + self.assertIsNone(results) + + with self.subTest("Test server error"): + mock_response = MockResponse( + status_code=500, + text=None, + json=None, + content=None, + ) + with patch("requests.get", return_value=mock_response): + results = get_ci_text_search("server_error") + self.assertIsNone(results) + + with self.subTest("Test SSLError"): + with patch("requests.get", side_effect=SSLError): + results = get_ci_text_search("SSLError") + self.assertRaises(SSLError) + self.assertIsNone(results) + + with self.subTest("Test Timeout"): + with patch("requests.get", side_effect=Timeout): + results = get_ci_text_search("Timeout") + self.assertRaises(Timeout) + self.assertIsNone(results) + + with self.subTest("Test HTTPError"): + with patch("requests.get", side_effect=HTTPError): + results = get_ci_text_search("HTTPError") + self.assertRaises(HTTPError) + self.assertIsNone(results) From 8ec1be5f9a20574a3435edda20c190b72f8006f5 Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 20 Jun 2024 16:28:42 +0000 Subject: [PATCH 09/13] test(cantusindex): add tests for CISearchView --- .../main_app/tests/test_views.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/django/cantusdb_project/main_app/tests/test_views.py b/django/cantusdb_project/main_app/tests/test_views.py index aca6039ac..701679ccc 100644 --- a/django/cantusdb_project/main_app/tests/test_views.py +++ b/django/cantusdb_project/main_app/tests/test_views.py @@ -3385,6 +3385,56 @@ def test_suggested_chant_buttons(self): self.assertIsNone(response_after_rare_chant.context["suggested_chants"]) +class CISearchViewTest(TestCase): + + def test_valid_search_term(self): + with patch("requests.get", mock_requests_get): + response = self.client.get(reverse("ci-search", args=["qui est"])) + + self.assertEqual(response.status_code, 200) + context = response.context + self.assertIn("results", context) + + results_zip = context["results"] + + self.assertEqual(len(results_zip), 50) + first_result = results_zip[0] + self.assertEqual(first_result[0], "001774") + self.assertEqual( + first_result[2], + "Caro et sanguis non revelavit tibi sed pater meus qui est in caelis", + ) + + second_result = results_zip[1] + self.assertEqual(second_result[0], "002191") + self.assertEqual( + second_result[2], + "Dicebat Jesus turbis Judaeorum et principibus sacerdotum qui est ex deo verba dei audit responderunt Judaei et dixerunt ei nonne bene dicimus nos quia Samaritanus es tu et daemonium habes respondit Jesus ego daemonium non habeo sed honorifico patrem meum et vos inhonorastis me", + ) + + def test_invalid_search_term(self): + with patch("requests.get", mock_requests_get): + response = self.client.get(reverse("ci-search", args=["123xyz"])) + + self.assertEqual(response.status_code, 200) + context = response.context + self.assertIn("results", context) + self.assertEqual( + context["results"], [["No results", "No results", "No results"]] + ) + + def test_server_error(self): + with patch("requests.get", mock_requests_get): + response = self.client.get(reverse("ci-search", args=["server_error"])) + + self.assertEqual(response.status_code, 200) + context = response.context + self.assertIn("results", context) + self.assertEqual( + list(context["results"]), [["No results", "No results", "No results"]] + ) + + class ChantDeleteViewTest(TestCase): @classmethod def setUpTestData(cls): From da68ff5509073891e5822d8f9b4628c73d78c9a1 Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 20 Jun 2024 16:30:00 +0000 Subject: [PATCH 10/13] fix(cantusindex): convert zip iterator to list in CISearchView context --- django/cantusdb_project/main_app/views/chant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/cantusdb_project/main_app/views/chant.py b/django/cantusdb_project/main_app/views/chant.py index 7ceee6250..3cab961be 100644 --- a/django/cantusdb_project/main_app/views/chant.py +++ b/django/cantusdb_project/main_app/views/chant.py @@ -906,7 +906,7 @@ def get_context_data(self, **kwargs): if len(cantus_id) == 0: context["results"] = [["No results", "No results", "No results"]] else: - context["results"] = zip(cantus_id, genre, full_text) + context["results"] = list(zip(cantus_id, genre, full_text)) return context From b9c594c9f6a14bf8d723b00e81e31b3884b6d3ba Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 9 Jul 2024 18:45:52 -0700 Subject: [PATCH 11/13] fix(forms): Make chant project field optional --- django/cantusdb_project/main_app/forms.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django/cantusdb_project/main_app/forms.py b/django/cantusdb_project/main_app/forms.py index 2ae54e4d7..3c366d35f 100644 --- a/django/cantusdb_project/main_app/forms.py +++ b/django/cantusdb_project/main_app/forms.py @@ -151,6 +151,7 @@ class Meta: project = SelectWidgetNameModelChoiceField( queryset=Project.objects.all().order_by("id"), initial=None, + required=False, help_text="Select the project (if any) that the chant belongs to.", ) @@ -346,6 +347,7 @@ class Meta: project = SelectWidgetNameModelChoiceField( queryset=Project.objects.all().order_by("id"), help_text="Select the project (if any) that the chant belongs to.", + required = False, ) From 97c7bd7a4d3886074eee3927180519e33bb6c7f2 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 9 Jul 2024 18:47:13 -0700 Subject: [PATCH 12/13] refactor: Remove unused import in widgets.py --- django/cantusdb_project/main_app/widgets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django/cantusdb_project/main_app/widgets.py b/django/cantusdb_project/main_app/widgets.py index 36df85145..152b66aaa 100644 --- a/django/cantusdb_project/main_app/widgets.py +++ b/django/cantusdb_project/main_app/widgets.py @@ -1,5 +1,4 @@ from django.forms.widgets import TextInput, Select, Textarea, CheckboxInput -from django.utils.safestring import mark_safe class TextInputWidget(TextInput): From 4d4e4a9fb4fa03a712bbffb8dd485b45883d265b Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 9 Jul 2024 18:58:13 -0700 Subject: [PATCH 13/13] fix(templates): Change links under Resources menu and add link to procedures manual --- django/cantusdb_project/templates/base.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/cantusdb_project/templates/base.html b/django/cantusdb_project/templates/base.html index e31558382..2632ab0a7 100644 --- a/django/cantusdb_project/templates/base.html +++ b/django/cantusdb_project/templates/base.html @@ -258,7 +258,8 @@
Tutorial for Cantus Contributors - Reference and Instructional Documents + Key Instructional Documents + Cantus Database Procedures Manual Citation Style References