Merge branch 'dev' into master

backend/dataset/migrations/0047_speechconversation_final_transcribed_json.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.14 on 2024-05-21 06:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("dataset", "0046_merge_20240416_2233"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="speechconversation",
+            name="final_transcribed_json",
+            field=models.JSONField(
+                blank=True,
+                help_text="Field where data from this standardised_transcription_editing type will be exported.",
+                null=True,
+                verbose_name="final_transcribed_json",
+            ),
+        ),
+    ]

backend/dataset/migrations/0048_ocrdocument_bboxes_relation_prediction_json.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.14 on 2024-06-19 11:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("dataset", "0047_speechconversation_final_transcribed_json"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ocrdocument",
+            name="bboxes_relation_prediction_json",
+            field=models.JSONField(
+                blank=True, null=True, verbose_name="bboxes_relation_prediction_json"
+            ),
+        ),
+    ]

backend/dataset/models.py

@@ -311,6 +311,10 @@ class OCRDocument(DatasetBase):
         verbose_name="bboxes_relation_json", null=True, blank=True
     )
 
+    bboxes_relation_prediction_json = models.JSONField(
+        verbose_name="bboxes_relation_prediction_json", null=True, blank=True
+    )
+
     annotated_document_details_json = models.JSONField(
         verbose_name="annotated_document_details_json", null=True, blank=True
     )
@@ -484,6 +488,14 @@ class SpeechConversation(DatasetBase):
         blank=True,
         help_text=("Prepopulated prediction for the implemented models"),
     )
+    final_transcribed_json = models.JSONField(
+        verbose_name="final_transcribed_json",
+        null=True,
+        blank=True,
+        help_text=(
+            "Field where data from this standardised_transcription_editing type will be exported."
+        ),
+    )
 
     def __str__(self):
         return str(self.id)

backend/dataset/tasks.py

@@ -12,11 +12,9 @@
 #### CELERY SHARED TASKS
 
 
-@shared_task(
-    bind=True,
-)
+@shared_task(queue="default")
 def upload_data_to_data_instance(
-    self, dataset_string, pk, dataset_type, content_type, deduplicate=False
+    dataset_string, pk, dataset_type, content_type, deduplicate=False
 ):
     # sourcery skip: raise-specific-error
     """Celery background task to upload the data to the dataset instance through file upload.
@@ -102,8 +100,8 @@ def upload_data_to_data_instance(
         raise Exception(f"Upload failed for lines: {failed_rows}")
 
 
-@shared_task(bind=True)
-def deduplicate_dataset_instance_items(self, pk, deduplicate_field_list):
+@shared_task(queue="default")
+def deduplicate_dataset_instance_items(pk, deduplicate_field_list):
     if len(deduplicate_field_list) == 0:
         return "Field list cannot be empty"
     try:

backend/functions/tasks.py

@@ -29,7 +29,7 @@
     ANNOTATED,
 )
 from tasks.views import SentenceOperationViewSet
-from users.models import User, LANG_CHOICES
+from users.models import User
 from django.core.mail import EmailMessage
 
 from utils.blob_functions import (
@@ -56,6 +56,7 @@
 import tempfile
 
 from shoonya_backend.locks import Lock
+from utils.constants import LANG_CHOICES
 
 import logging
 

backend/loging/serializers.py

@@ -1,4 +1,5 @@
 from rest_framework import serializers
+import uuid
 
 
 class SelectionSerializer(serializers.Serializer):
@@ -15,3 +16,12 @@ class TransliterationSerializer(serializers.Serializer):
     source = serializers.CharField()
     language = serializers.CharField()
     steps = SelectionSerializer(many=True)
+
+
+class TransliterationLogSerializer(serializers.Serializer):
+    source_english_text = serializers.CharField()
+    indic_translation_text = serializers.CharField()
+    romanised_text = serializers.CharField()
+    edited_romanised_text = serializers.CharField(required=False)
+    language = serializers.CharField()
+    uuid = serializers.UUIDField(default=uuid.uuid4, read_only=True)

backend/loging/tasks.py

@@ -1,8 +1,9 @@
 from celery import shared_task
 from datetime import datetime
 from azure.storage.blob import BlobServiceClient, generate_blob_sas, BlobSasPermissions
-from django.core.mail import EmailMessage
+from django.core.mail import EmailMessage, EmailMultiAlternatives
 from django.conf import settings
+from utils.email_template import send_email_template_with_attachment
 from utils.blob_functions import (
     extract_account_key,
     extract_account_name,
@@ -29,14 +30,32 @@ def get_azure_credentials(connection_string):
 def send_email_with_url(user_email, attachment_url):
     try:
         message = "Here is the link to the generated document:"
-        email = EmailMessage(
+        compiled_msg_code = send_email_template_with_attachment(
             "Transliteration Logs",
+            user_email,
             message,
+        )
+        msg = EmailMultiAlternatives(
+            "Transliteration Logs",
+            compiled_msg_code,
             settings.DEFAULT_FROM_EMAIL,
             [user_email],
         )
-        email.attach("Generated Document", attachment_url, "text/plain")
-        email.send()
+        msg.attach_alternative(compiled_msg_code, "text/html")
+        # also attach the generated document
+        msg.attach("Generated Document", attachment_url, "text/plain")
+        msg.send()
+        # compiled_msg.attach("Generated Document", attachment_url, "text/plain")
+        # compiled_msg.send()
+
+        # email = EmailMessage(
+        #     "Transliteration Logs",
+        #     message,
+        #     settings.DEFAULT_FROM_EMAIL,
+        #     [user_email],
+        # )
+        # email.attach("Generated Document", attachment_url, "text/plain")
+        # email.send()
     except Exception as e:
         print(f"Failed to send email: {str(e)}")
         raise e

backend/loging/urls.py

@@ -1,6 +1,6 @@
 from django.urls import path, include
 from rest_framework import routers
-from loging.views import TransliterationSelectionViewSet
+from loging.views import TransliterationSelectionViewSet, TransliterationLogView
 
 router = routers.DefaultRouter()
 
@@ -11,4 +11,9 @@
         TransliterationSelectionViewSet.as_view(),
         name="transliteration_selection",
     ),
+    path(
+        "transliteration-log/",
+        TransliterationLogView.as_view(),
+        name="transliteration-log",
+    ),
 ]

backend/loging/views.py

@@ -1,12 +1,16 @@
 from rest_framework import status
 from rest_framework.response import Response
 from rest_framework.views import APIView
-from loging.serializers import TransliterationSerializer
-from azure.storage.blob import BlobServiceClient
+from loging.serializers import TransliterationSerializer, TransliterationLogSerializer
+from azure.storage.blob import BlobServiceClient, BlobClient
 from .tasks import retrieve_logs_and_send_through_email
 from users.models import User
 from rest_framework.permissions import IsAuthenticated
 from utils.blob_functions import test_container_connection
+from drf_yasg.utils import swagger_auto_schema
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.decorators import permission_classes
+from rest_framework.decorators import action
 
 import os
 import json
@@ -17,6 +21,7 @@
 
 AZURE_STORAGE_CONNECTION_STRING = os.getenv("AZURE_CONNECTION_STRING")
 TRANSLITERATION_CONTAINER_NAME = os.getenv("TRANSLITERATION_CONTAINER_NAME")
+INDIC_ROMANISED_LOGS = os.getenv("INDIC_ROMANISED_LOGS")
 
 
 class CustomJSONEncoder(json.JSONEncoder):
@@ -140,3 +145,82 @@ def get(self, request):
                 {"message": "Failed to retrieve logs", "error": str(e)},
                 status=status.HTTP_500_INTERNAL_SERVER_ERROR,
             )
+
+
+class TransliterationLogView(APIView):
+    def post(self, request):
+        # Validate that the payload contains exactly three words
+        if len(request.data) != 5:
+            return Response(
+                {"error": "Payload must contain exactly six elements."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        serializer = TransliterationLogSerializer(data=request.data)
+        if serializer.is_valid():
+            # Process the valid data here
+            data = serializer.validated_data
+            self.log_transliteration(data)
+            return Response(
+                {"message": "Data stored in Azure Blob successfully"},
+                status=status.HTTP_201_CREATED,
+            )
+        else:
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+    def log_transliteration(self, data):
+        try:
+            # print('data here',data)
+            current_time = datetime.datetime.now().isoformat()
+            data_with_timestamp = {**data, "timestamp": current_time}
+
+            # Azure Blob Storage setup
+            current_date = datetime.date.today()
+            log_file_name = f"{current_date.isoformat()}.log"
+
+            blob_service_client = BlobServiceClient.from_connection_string(
+                AZURE_STORAGE_CONNECTION_STRING
+            )
+
+            container_client = blob_service_client.get_container_client(
+                INDIC_ROMANISED_LOGS
+            )
+
+            # log_file_name = "romanised_transliteration.log"
+            blob_client = container_client.get_blob_client(log_file_name)
+
+            if not blob_client.exists():
+                # upload some data to the blob
+                blob_client.upload_blob("[]", overwrite=True)
+
+            existing_data = blob_client.download_blob().readall().decode("utf-8")
+            existing_json_data = json.loads(existing_data)
+            existing_json_data.append(data_with_timestamp)
+            updated_content = json.dumps(existing_json_data, cls=CustomJSONEncoder)
+            blob_client.upload_blob(updated_content, overwrite=True)
+
+            create_empty_log_for_next_day(container_client)
+
+            main_transliteration_blob_client = container_client.get_blob_client(
+                "romanised_transliteration.log"
+            )
+
+            if not main_transliteration_blob_client.exists():
+                main_transliteration_blob_client.upload_blob("[]", overwrite=True)
+
+            main_transliteration_data = (
+                main_transliteration_blob_client.download_blob()
+                .readall()
+                .decode("utf-8")
+            )
+            main_transliteration_json_data = json.loads(main_transliteration_data)
+            main_transliteration_json_data.append(data_with_timestamp)
+            main_updated_content = json.dumps(
+                main_transliteration_json_data, cls=CustomJSONEncoder
+            )
+            main_transliteration_blob_client.upload_blob(
+                main_updated_content, overwrite=True
+            )
+            print("Logged transliteration data successfully")
+        except Exception as e:
+            print(f"Failed to log transliteration data: {str(e)}")

backend/notifications/tasks.py

@@ -22,7 +22,7 @@ def delete_excess_Notification(user):
     return 0
 
 
-# @shared_task
+# @shared_task(queue: "default")
 def create_notification_handler(
     title, notification_type, users_ids, project_id=None, task_id=None
 ):