diff --git a/backend/organizations/views.py b/backend/organizations/views.py index 3109cf083..c8ce73c0e 100644 --- a/backend/organizations/views.py +++ b/backend/organizations/views.py @@ -52,6 +52,7 @@ send_project_analytics_mail_org, send_user_analytics_mail_org, ) +from utils.filter_tasks_by_ann_type import filter_tasks_by_ann_type def get_task_count(proj_ids, status, annotator, return_count=True): @@ -2743,24 +2744,41 @@ def cumulative_tasks_count(self, request, pk=None): other_lang = [] for lang in languages: proj_lang_filter = proj_objs.filter(tgt_language=lang) - annotation_tasks_count = 0 - reviewer_task_count = 0 + annotation_tasks = Task.objects.filter( + project_id__in=proj_lang_filter, + task_status__in=[ + "annotated", + "reviewed", + "super_checked", + ], + ) reviewer_tasks = Task.objects.filter( project_id__in=proj_lang_filter, project_id__project_stage__in=[REVIEW_STAGE, SUPERCHECK_STAGE], - task_status__in=["reviewed", "exported", "super_checked"], + task_status__in=["reviewed", "super_checked"], ) - - annotation_tasks = Task.objects.filter( + supercheck_tasks = Task.objects.filter( + project_id__in=proj_lang_filter, + project_id__project_stage__in=[SUPERCHECK_STAGE], + task_status__in=["super_checked"], + ) + annotation_tasks_exported = Task.objects.filter( project_id__in=proj_lang_filter, + project_id__project_stage__in=[ANNOTATION_STAGE], task_status__in=[ - "annotated", - "reviewed", "exported", - "super_checked", ], ) - + reviewer_tasks_exported = Task.objects.filter( + project_id__in=proj_lang_filter, + project_id__project_stage__in=[REVIEW_STAGE], + task_status__in=["exported"], + ) + supercheck_tasks_exported = Task.objects.filter( + project_id__in=proj_lang_filter, + project_id__project_stage__in=[SUPERCHECK_STAGE], + task_status__in=["exported"], + ) if metainfo == True: result = {} @@ -2975,14 +2993,28 @@ def cumulative_tasks_count(self, request, pk=None): } else: - reviewer_task_count = reviewer_tasks.count() + reviewer_task_count = ( + reviewer_tasks.count() + + reviewer_tasks_exported.count() + + supercheck_tasks_exported.count() + ) - annotation_tasks_count = annotation_tasks.count() + annotation_tasks_count = ( + annotation_tasks.count() + + annotation_tasks_exported.count() + + reviewer_tasks_exported.count() + + supercheck_tasks_exported.count() + ) + + supercheck_tasks_count = ( + supercheck_tasks.count() + supercheck_tasks_exported.count() + ) result = { "language": lang, "ann_cumulative_tasks_count": annotation_tasks_count, "rew_cumulative_tasks_count": reviewer_task_count, + "sup_cumulative_tasks_count": supercheck_tasks_count, } if lang == None or lang == "": @@ -2992,6 +3024,7 @@ def cumulative_tasks_count(self, request, pk=None): ann_task_count = 0 rew_task_count = 0 + sup_task_count = 0 ann_word_count = 0 rew_word_count = 0 ann_aud_dur = 0 @@ -3006,6 +3039,7 @@ def cumulative_tasks_count(self, request, pk=None): if metainfo != True: ann_task_count += dat["ann_cumulative_tasks_count"] rew_task_count += dat["rew_cumulative_tasks_count"] + sup_task_count += dat["sup_cumulative_tasks_count"] else: if project_type in get_audio_project_types(): ann_aud_dur += convert_hours_to_seconds( @@ -3048,6 +3082,7 @@ def cumulative_tasks_count(self, request, pk=None): "language": "Others", "ann_cumulative_tasks_count": ann_task_count, "rew_cumulative_tasks_count": rew_task_count, + "sup_cumulative_tasks_count": sup_task_count, } else: if project_type in get_audio_project_types(): diff --git a/backend/shoonya_backend/log_transfer.py b/backend/shoonya_backend/log_transfer.py index c4d348b02..5fd80443e 100644 --- a/backend/shoonya_backend/log_transfer.py +++ b/backend/shoonya_backend/log_transfer.py @@ -19,14 +19,12 @@ log_file_path = log_file_dir + log_file_name -blob_service_client = BlobServiceClient.from_connection_string( - AZURE_STORAGE_CONNECTION_STRING -) - -container_client = blob_service_client.get_container_client(CONTAINER_NAME) - - def get_most_recent_creation_date(): + blob_service_client = BlobServiceClient.from_connection_string( + AZURE_STORAGE_CONNECTION_STRING + ) + + container_client = blob_service_client.get_container_client(CONTAINER_NAME) blobs = list(container_client.list_blobs()) if not blobs: return None @@ -68,6 +66,11 @@ def rotate_logs(): zip_log_file(zip_file_name) zip_file_path_on_disk = os.path.join(log_file_dir, zip_file_name) + blob_service_client = BlobServiceClient.from_connection_string( + AZURE_STORAGE_CONNECTION_STRING + ) + + container_client = blob_service_client.get_container_client(CONTAINER_NAME) blob_client = container_client.get_blob_client(zip_file_name) with open(zip_file_path_on_disk, "rb") as file: diff --git a/backend/utils/filter_tasks_by_ann_type.py b/backend/utils/filter_tasks_by_ann_type.py new file mode 100644 index 000000000..3fb641293 --- /dev/null +++ b/backend/utils/filter_tasks_by_ann_type.py @@ -0,0 +1,47 @@ +from tasks.models import ( + Annotation, + ANNOTATOR_ANNOTATION, + REVIEWER_ANNOTATION, + SUPER_CHECKER_ANNOTATION, + LABELED, + ACCEPTED, + ACCEPTED_WITH_MINOR_CHANGES, + ACCEPTED_WITH_MAJOR_CHANGES, + VALIDATED, + VALIDATED_WITH_CHANGES, +) + + +def filter_tasks_by_ann_type(annotation_tasks, reviewer_tasks, supercheck_tasks): + filtered_annotation_tasks, filtered_reviewer_tasks, filtered_supercheck_tasks = ( + [], + [], + [], + ) + for a in annotation_tasks: + anno = Annotation.objects.filter( + task=a, annotation_type=ANNOTATOR_ANNOTATION, annotation_status=LABELED + )[0] + if anno: + filtered_annotation_tasks.append(a) + for r in reviewer_tasks: + anno = Annotation.objects.filter( + task=r, + annotation_type=REVIEWER_ANNOTATION, + annotation_status__in=[ + ACCEPTED, + ACCEPTED_WITH_MINOR_CHANGES, + ACCEPTED_WITH_MAJOR_CHANGES, + ], + )[0] + if anno: + filtered_reviewer_tasks.append(r) + for s in supercheck_tasks: + anno = Annotation.objects.filter( + task=s, + annotation_type=SUPER_CHECKER_ANNOTATION, + annotation_status__in=[VALIDATED, VALIDATED_WITH_CHANGES], + )[0] + if anno: + filtered_supercheck_tasks.append(s) + return filtered_annotation_tasks, filtered_reviewer_tasks, filtered_supercheck_tasks diff --git a/backend/workspaces/views.py b/backend/workspaces/views.py index e5cd136d9..367a0f93f 100644 --- a/backend/workspaces/views.py +++ b/backend/workspaces/views.py @@ -61,7 +61,7 @@ get_review_reports, get_supercheck_reports, ) - +from utils.filter_tasks_by_ann_type import filter_tasks_by_ann_type # Create your views here. @@ -1404,23 +1404,41 @@ def cumulative_tasks_count_all(self, request, pk=None): other_lang = [] for lang in languages: proj_lang_filter = proj_objs.filter(tgt_language=lang) - annotation_tasks_count = 0 - reviewer_task_count = 0 + annotation_tasks = Task.objects.filter( + project_id__in=proj_lang_filter, + task_status__in=[ + "annotated", + "reviewed", + "super_checked", + ], + ) reviewer_tasks = Task.objects.filter( project_id__in=proj_lang_filter, project_id__project_stage__in=[REVIEW_STAGE, SUPERCHECK_STAGE], - task_status__in=["reviewed", "exported", "super_checked"], + task_status__in=["reviewed", "super_checked"], ) - - annotation_tasks = Task.objects.filter( + supercheck_tasks = Task.objects.filter( + project_id__in=proj_lang_filter, + project_id__project_stage__in=[SUPERCHECK_STAGE], + task_status__in=["super_checked"], + ) + annotation_tasks_exported = Task.objects.filter( project_id__in=proj_lang_filter, + project_id__project_stage__in=[ANNOTATION_STAGE], task_status__in=[ - "annotated", - "reviewed", "exported", - "super_checked", ], ) + reviewer_tasks_exported = Task.objects.filter( + project_id__in=proj_lang_filter, + project_id__project_stage__in=[REVIEW_STAGE], + task_status__in=["exported"], + ) + supercheck_tasks_exported = Task.objects.filter( + project_id__in=proj_lang_filter, + project_id__project_stage__in=[SUPERCHECK_STAGE], + task_status__in=["exported"], + ) if metainfo == True: result = {} @@ -1636,14 +1654,28 @@ def cumulative_tasks_count_all(self, request, pk=None): } else: - reviewer_task_count = reviewer_tasks.count() + reviewer_task_count = ( + reviewer_tasks.count() + + reviewer_tasks_exported.count() + + supercheck_tasks_exported.count() + ) - annotation_tasks_count = annotation_tasks.count() + annotation_tasks_count = ( + annotation_tasks.count() + + annotation_tasks_exported.count() + + reviewer_tasks_exported.count() + + supercheck_tasks_exported.count() + ) + + supercheck_tasks_count = ( + supercheck_tasks.count() + supercheck_tasks_exported.count() + ) result = { "language": lang, "ann_cumulative_tasks_count": annotation_tasks_count, "rew_cumulative_tasks_count": reviewer_task_count, + "sup_cumulative_tasks_count": supercheck_tasks_count, } if lang == None or lang == "": @@ -1653,6 +1685,7 @@ def cumulative_tasks_count_all(self, request, pk=None): ann_task_count = 0 rew_task_count = 0 + sup_task_count = 0 ann_word_count = 0 rew_word_count = 0 ann_aud_dur = 0 @@ -1667,6 +1700,7 @@ def cumulative_tasks_count_all(self, request, pk=None): if metainfo != True: ann_task_count += dat["ann_cumulative_tasks_count"] rew_task_count += dat["rew_cumulative_tasks_count"] + sup_task_count += dat["sup_cumulative_tasks_count"] else: if project_type in get_audio_project_types(): ann_aud_dur += convert_hours_to_seconds( @@ -1710,6 +1744,7 @@ def cumulative_tasks_count_all(self, request, pk=None): "language": "Others", "ann_cumulative_tasks_count": ann_task_count, "rew_cumulative_tasks_count": rew_task_count, + "sup_cumulative_tasks_count": sup_task_count, } else: if project_type in get_audio_project_types():