Merge pull request #2035 from rebeccacremona/alert-frozen-textbooks

Notify admins when a book's export has failed many times

web/config/settings/settings_base.py

@@ -300,7 +300,7 @@ class LoggerConfig(TypedDict, total=False):
     "secret_key": "secretkey",
 }
 
-MAX_EXPORT_ATTEMPTS = 3
+MAX_EXPORT_ATTEMPTS = 5
 MAX_EXPORTS_PER_HOUR = 600
 EXPORT_RATE_FALLOFF = int(MAX_EXPORTS_PER_HOUR / 60)
 

web/main/models.py

@@ -63,6 +63,7 @@
     prefix_ids_hrefs,
     rich_text_export,
     strip_trailing_block_level_whitespace,
+    send_mail,
 )
 
 
@@ -3470,7 +3471,20 @@ def export(
 
     def inc_export_fails(self):
         # This function is used to avoid making a copy of the casebook via CasebookHistory
+        orig = self.export_fails
         Casebook.objects.filter(id=self.id).update(export_fails=F("export_fails") + 1)
+        self.refresh_from_db()
+        if (
+            orig < settings.MAX_EXPORT_ATTEMPTS
+            and self.export_fails >= settings.MAX_EXPORT_ATTEMPTS
+        ):
+            message = f"Export of casebook {self.id} ({self.title}) has failed the maximum allowed times.\n\nPlease investigate, and reset to zero if the failures were spurious."
+            send_mail(
+                f"Export of H2O Casebook {self.id} is frozen.",
+                message,
+                settings.DEFAULT_FROM_EMAIL,
+                settings.ADMINS,
+            )
 
     def reset_export_fails(self):
         # This function is used to avoid making a copy of the casebook via CasebookHistory