From 192a1640cfa61bf21edcb880024fcdcc566edf3b Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Fri, 4 Oct 2024 11:59:42 -0400 Subject: [PATCH] feat(chant views): catch invalid text errors Modifies chant create, edit, and detail views to prevent and catch text syllabification errors. Modifies edit syllabification view for the same. Invalidates chant text fields if they error on syllabification. Catches errors for texts with error rather than propagating to a server error. --- django/cantusdb_project/main_app/forms.py | 107 +++++++++++++----- .../main_app/templates/chant_create.html | 5 +- .../main_app/templates/chant_edit.html | 35 +++--- .../templates/chant_syllabification_edit.html | 9 +- .../main_app/tests/test_views/test_chant.py | 87 +++++++++++++- .../cantusdb_project/main_app/views/chant.py | 68 +++++++---- 6 files changed, 233 insertions(+), 78 deletions(-) diff --git a/django/cantusdb_project/main_app/forms.py b/django/cantusdb_project/main_app/forms.py index 4f65fa883..997bc561b 100644 --- a/django/cantusdb_project/main_app/forms.py +++ b/django/cantusdb_project/main_app/forms.py @@ -1,5 +1,14 @@ from django import forms from django.contrib.auth.forms import ReadOnlyPasswordHashField +from django.contrib.auth import get_user_model +from django.db.models import Q +from django.contrib.admin.widgets import ( + FilteredSelectMultiple, +) +from django.forms.widgets import CheckboxSelectMultiple +from dal import autocomplete +from volpiano_display_utilities.cantus_text_syllabification import syllabify_text +from volpiano_display_utilities.latin_word_syllabification import LatinError from .models import ( Chant, Service, @@ -22,13 +31,6 @@ SelectWidget, CheckboxWidget, ) -from django.contrib.auth import get_user_model -from django.db.models import Q -from django.contrib.admin.widgets import ( - FilteredSelectMultiple, -) -from django.forms.widgets import CheckboxSelectMultiple -from dal import autocomplete # ModelForm allows to build a form directly from a model # see https://docs.djangoproject.com/en/3.0/topics/forms/modelforms/ @@ -71,6 +73,40 @@ def label_from_instance(self, obj): widget = CheckboxSelectMultiple() +class CantusDBLatinField(forms.CharField): + """ + A custom CharField for chant text fields. Validates that the text + can be syllabified (essentially, that it does not have any improper + characters). + """ + + def validate(self, value): + super().validate(value) + if value: + try: + syllabify_text(value) + except LatinError as err: + raise forms.ValidationError(str(err)) + except ValueError as exc: + raise forms.ValidationError("Invalid characters in text.") from exc + + +class CantusDBSyllabifiedLatinField(forms.CharField): + """ + A custom CharField for chant syllabified text fields. Validates that the text + can be syllabified (essentially, that it does not have any improper + characters). + """ + + def validate(self, value): + super().validate(value) + if value: + try: + syllabify_text(value, text_presyllabified=True) + except ValueError as exc: + raise forms.ValidationError("Invalid characters in text.") from exc + + class ChantCreateForm(forms.ModelForm): class Meta: model = Chant @@ -125,8 +161,8 @@ class Meta: "finalis": TextInputWidget(), "extra": TextInputWidget(), "chant_range": VolpianoInputWidget(), - # manuscript_full_text_std_spelling: defined below (required) - "manuscript_full_text": TextAreaWidget(), + # manuscript_full_text_std_spelling: defined below (required & special field) + # "manuscript_full_text": defined below (special field) "volpiano": VolpianoAreaWidget(), "image_link": TextInputWidget(), "melody_id": TextInputWidget(), @@ -153,14 +189,18 @@ class Meta: help_text="Each folio starts with '1'.", ) - manuscript_full_text_std_spelling = forms.CharField( + manuscript_full_text_std_spelling = CantusDBLatinField( + widget=TextAreaWidget, + help_text=Chant._meta.get_field("manuscript_full_text_std_spelling").help_text, + label="Full text as in Source (standardized spelling)", required=True, + ) + + manuscript_full_text = CantusDBLatinField( widget=TextAreaWidget, - help_text="Manuscript full text with standardized spelling. Enter the words " - "according to the manuscript but normalize their spellings following " - "Classical Latin forms. Use upper-case letters for proper nouns, " - 'the first word of each chant, and the first word after "Alleluia" for ' - "Mass Alleluias. Punctuation is omitted.", + label="Full text as in Source (source spelling)", + help_text=Chant._meta.get_field("manuscript_full_text").help_text, + required=False, ) project = SelectWidgetNameModelChoiceField( @@ -319,8 +359,8 @@ class Meta: "rubrics", ] widgets = { - # manuscript_full_text_std_spelling: defined below (required) - "manuscript_full_text": TextAreaWidget(), + # manuscript_full_text_std_spelling: defined below (required) & special field + # manuscript_full_text: defined below (special field) "volpiano": VolpianoAreaWidget(), "marginalia": TextInputWidget(), # folio: defined below (required) @@ -354,14 +394,18 @@ class Meta: "rubrics": TextInputWidget(), } - manuscript_full_text_std_spelling = forms.CharField( + manuscript_full_text_std_spelling = CantusDBLatinField( + widget=TextAreaWidget, + help_text=Chant._meta.get_field("manuscript_full_text_std_spelling").help_text, + label="Full text as in Source (standardized spelling)", required=True, + ) + + manuscript_full_text = CantusDBLatinField( widget=TextAreaWidget, - help_text="Manuscript full text with standardized spelling. Enter the words " - "according to the manuscript but normalize their spellings following " - "Classical Latin forms. Use upper-case letters for proper nouns, " - 'the first word of each chant, and the first word after "Alleluia" for ' - "Mass Alleluias. Punctuation is omitted.", + label="Full text as in Source (source spelling)", + help_text=Chant._meta.get_field("manuscript_full_text").help_text, + required=False, ) folio = forms.CharField( @@ -550,10 +594,14 @@ class Meta: "manuscript_full_text", "manuscript_syllabized_full_text", ] - widgets = { - "manuscript_full_text": TextAreaWidget(), - "manuscript_syllabized_full_text": TextAreaWidget(), - } + + manuscript_full_text = CantusDBLatinField( + widget=TextAreaWidget, label="Full text as in Source (source spelling)" + ) + + manuscript_syllabized_full_text = CantusDBSyllabifiedLatinField( + widget=TextAreaWidget, label="Syllabized full text" + ) class AdminCenturyForm(forms.ModelForm): @@ -738,10 +786,7 @@ class Meta: widget=TextInputWidget, ) - name = forms.CharField( - required=False, - widget=TextInputWidget - ) + name = forms.CharField(required=False, widget=TextInputWidget) holding_institution = forms.ModelChoiceField( queryset=Institution.objects.all().order_by("name"), diff --git a/django/cantusdb_project/main_app/templates/chant_create.html b/django/cantusdb_project/main_app/templates/chant_create.html index 5aec5de03..4b29bb0bb 100644 --- a/django/cantusdb_project/main_app/templates/chant_create.html +++ b/django/cantusdb_project/main_app/templates/chant_create.html @@ -223,7 +223,7 @@

Create Chant

- + {{ form.manuscript_full_text_std_spelling }}

{{ form.manuscript_full_text_std_spelling.help_text }} @@ -241,8 +241,7 @@

Create Chant

- + {{ form.manuscript_full_text }}

{{ form.manuscript_full_text.help_text }} diff --git a/django/cantusdb_project/main_app/templates/chant_edit.html b/django/cantusdb_project/main_app/templates/chant_edit.html index 038eef616..47860c7d5 100644 --- a/django/cantusdb_project/main_app/templates/chant_edit.html +++ b/django/cantusdb_project/main_app/templates/chant_edit.html @@ -21,6 +21,7 @@ {% for message in messages %}

{% endfor %} @@ -283,25 +284,33 @@

Syllabification is based on saved syllabized text.

{% endif %}
- {% for syl_text, syl_mel in syllabized_text_with_melody %} - -
{{ syl_mel }}
- - 
{{ syl_text }}
- - {% endfor %} + {% if syllabized_text_with_melody %} + {% for syl_text, syl_mel in syllabized_text_with_melody %} + +
{{ syl_mel }}
+ + 
{{ syl_text }}
+ + {% endfor %} + {% else %} +

Error aligning text and melody. Please check text for invalid characters.

+ {% endif %}
{% endif %} -
-
- - Edit syllabification (new window) - + + {% if syllabized_text_with_melody %} +
+
+ + Edit syllabification (new window) + +
-
+ {% endif %}
diff --git a/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html b/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html index 446afd73a..65b49f09f 100644 --- a/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html +++ b/django/cantusdb_project/main_app/templates/chant_syllabification_edit.html @@ -12,6 +12,7 @@ {% for message in messages %} {% endfor %} @@ -37,8 +38,8 @@

Edit Syllabification

-
@@ -46,8 +47,8 @@

Edit Syllabification

-
diff --git a/django/cantusdb_project/main_app/tests/test_views/test_chant.py b/django/cantusdb_project/main_app/tests/test_views/test_chant.py index 0d8374a47..d91ef212c 100644 --- a/django/cantusdb_project/main_app/tests/test_views/test_chant.py +++ b/django/cantusdb_project/main_app/tests/test_views/test_chant.py @@ -164,7 +164,6 @@ def setUp(self): self.user = get_user_model().objects.create(email="test@test.com") self.user.set_password("pass") self.user.save() - self.client = Client() project_manager = Group.objects.get(name="project manager") project_manager.user_set.add(self.user) self.client.login(email="test@test.com", password="pass") @@ -323,6 +322,44 @@ def test_proofread_chant(self): chant.refresh_from_db() self.assertIs(chant.manuscript_full_text_std_proofread, True) + def test_invalid_text(self) -> None: + """ + The user should not be able to create a chant with invalid text + (either invalid characters or unmatched brackets). + Instead, the user should be shown an error message. + """ + source = make_fake_source() + with self.subTest("Chant with invalid characters"): + response = self.client.post( + reverse("source-edit-chants", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a ch@nt t%xt with inv&lid ch!ra+ers", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Invalid characters in text.", + ) + with self.subTest("Chant with unmatched brackets"): + response = self.client.post( + reverse("source-edit-chants", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a chant with [ unmatched brackets", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Word [ contains non-alphabetic characters.", + ) + class ChantEditSyllabificationViewTest(TestCase): @classmethod @@ -363,7 +400,10 @@ def test_edit_syllabification(self): self.assertEqual(chant.manuscript_syllabized_full_text, "lorem ipsum") response = self.client.post( f"/edit-syllabification/{chant.id}", - {"manuscript_syllabized_full_text": "lore-m i-psum"}, + { + "manuscript_full_text": "lorem ipsum", + "manuscript_syllabized_full_text": "lore-m i-psum", + }, ) self.assertEqual(response.status_code, 302) # 302 Found chant.refresh_from_db() @@ -2817,13 +2857,13 @@ def test_repeated_seq(self) -> None: for i in range(1, 5): Chant.objects.create( source=source, - manuscript_full_text=faker.text(10), + manuscript_full_text=" ".join(faker.words(faker.random_int(3, 10))), folio=test_folio, c_sequence=i, ) # post a chant with the same folio and seq url = reverse("chant-create", args=[source.id]) - fake_text = faker.text(10) + fake_text = "this is also a fake but valid text" response = self.client.post( url, data={ @@ -2960,6 +3000,45 @@ def test_suggested_chant_buttons(self) -> None: ) self.assertIsNone(response_after_rare_chant.context["suggested_chants"]) + def test_invalid_text(self) -> None: + """ + The user should not be able to create a chant with invalid text + (either invalid characters or unmatched brackets). + Instead, the user should be shown an error message. + """ + with self.subTest("Chant with invalid characters"): + source = self.source + response = self.client.post( + reverse("chant-create", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a ch@nt t%xt with inv&lid ch!ra+ers", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Invalid characters in text.", + ) + with self.subTest("Chant with unmatched brackets"): + source = self.source + response = self.client.post( + reverse("chant-create", args=[source.id]), + { + "manuscript_full_text_std_spelling": "this is a chant with [ unmatched brackets", + "folio": "001r", + "c_sequence": "1", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertFormError( + response.context["form"], + "manuscript_full_text_std_spelling", + "Word [ contains non-alphabetic characters.", + ) + class CISearchViewTest(TestCase): diff --git a/django/cantusdb_project/main_app/views/chant.py b/django/cantusdb_project/main_app/views/chant.py index b0bbdfbc2..654fdf71f 100644 --- a/django/cantusdb_project/main_app/views/chant.py +++ b/django/cantusdb_project/main_app/views/chant.py @@ -7,6 +7,7 @@ from django.contrib.auth.mixins import UserPassesTestMixin from django.core.exceptions import PermissionDenied from django.db.models import Q, QuerySet +from django.forms import BaseModelForm from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.urls import reverse @@ -18,6 +19,7 @@ TemplateView, UpdateView, ) +from volpiano_display_utilities.latin_word_syllabification import LatinError from volpiano_display_utilities.cantus_text_syllabification import ( syllabify_text, flatten_syllabified_text, @@ -205,11 +207,14 @@ def get_context_data(self, **kwargs): # syllabification section if chant.volpiano: has_syl_text = bool(chant.manuscript_syllabized_full_text) - text_and_mel, _ = align_text_and_volpiano( - chant.get_best_text_for_syllabizing(), - chant.volpiano, - text_presyllabified=has_syl_text, - ) + try: + text_and_mel, _ = align_text_and_volpiano( + chant.get_best_text_for_syllabizing(), + chant.volpiano, + text_presyllabified=has_syl_text, + ) + except LatinError: + text_and_mel = None context["syllabized_text_with_melody"] = text_and_mel if project := chant.project: @@ -848,16 +853,19 @@ def get_context_data(self, **kwargs: Any) -> dict[Any, Any]: suggested_chants = get_suggested_chants(previous_cantus_id) context["suggested_feasts"] = suggested_feasts context["suggested_chants"] = suggested_chants - return context def form_valid(self, form): - """compute source, incipit; folio/sequence (if left empty) - validate the form: add success/error message + """ + Validates the new chant. + + Custom validation steps are: + - Check if a chant with the same sequence and folio already exists in the source. + - Compute the chant incipit. + - Adds the "created_by" and "updated_by" fields to the chant. """ # compute source - form.instance.source = self.source # same effect as the next line - # form.instance.source = get_object_or_404(Source, pk=self.kwargs['source_pk']) + form.instance.source = self.source # compute incipit, within 30 charactors, keep words complete words = form.instance.manuscript_full_text_std_spelling.split(" ") @@ -893,8 +901,7 @@ def form_valid(self, form): "Chant '" + form.instance.incipit + "' created successfully!", ) return super().form_valid(form) - else: - return super().form_invalid(form) + return super().form_invalid(form) class ChantDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView): @@ -1113,11 +1120,18 @@ def get_context_data(self, **kwargs): has_syl_text = bool(chant.manuscript_syllabized_full_text) # Note: the second value returned is a flag indicating whether the alignment process # encountered errors. In future, this could be used to display a message to the user. - text_and_mel, _ = align_text_and_volpiano( - chant.get_best_text_for_syllabizing(), - chant.volpiano, - text_presyllabified=has_syl_text, - ) + try: + text_and_mel, _ = align_text_and_volpiano( + chant.get_best_text_for_syllabizing(), + chant.volpiano, + text_presyllabified=has_syl_text, + ) + except LatinError as err: + messages.error( + self.request, + "Error in aligning text and melody: " + str(err), + ) + text_and_mel = None context["syllabized_text_with_melody"] = text_and_mel user = self.request.user @@ -1224,12 +1238,20 @@ def get_initial(self): initial = super().get_initial() chant = self.get_object() has_syl_text = bool(chant.manuscript_syllabized_full_text) - syls_text, _ = syllabify_text( - text=chant.get_best_text_for_syllabizing(), - clean_text=True, - text_presyllabified=has_syl_text, - ) - self.flattened_syls_text = flatten_syllabified_text(syls_text) + try: + syls_text, _ = syllabify_text( + text=chant.get_best_text_for_syllabizing(), + clean_text=True, + text_presyllabified=has_syl_text, + ) + self.flattened_syls_text = flatten_syllabified_text(syls_text) + except LatinError as err: + messages.error( + self.request, + "Error in syllabifying text: " + str(err), + ) + syls_text = None + self.flattened_syls_text = "" initial["manuscript_syllabized_full_text"] = self.flattened_syls_text return initial