915 - Add generate computed file command

api/scpca_portal/loader.py

@@ -10,7 +10,14 @@
 
 from scpca_portal import common, metadata_file, s3
 from scpca_portal.config.logging import get_and_configure_logger
-from scpca_portal.models import ComputedFile, Contact, ExternalAccession, Project, Publication
+from scpca_portal.models import (
+    ComputedFile,
+    Contact,
+    ExternalAccession,
+    Project,
+    Publication,
+    Sample,
+)
 
 logger = get_and_configure_logger(__name__)
 
@@ -136,25 +143,61 @@ def create_project(
     return project
 
 
-def _create_computed_file(future, *, update_s3: bool, clean_up_output_data: bool) -> None:
+def _create_computed_file(
+    computed_file: ComputedFile, update_s3: bool, clean_up_output_data: bool
+) -> None:
     """
     Save computed file returned from future to the db.
     Upload file to s3 and clean up output data depending on passed options.
     """
-    if computed_file := future.result():
+    # Only upload and clean up projects and the last sample if multiplexed
+    if computed_file.project or computed_file.sample.is_last_multiplexed_sample:
+        if update_s3:
+            s3.upload_output_file(computed_file.s3_key, computed_file.s3_bucket)
+        if clean_up_output_data:
+            computed_file.clean_up_local_computed_file()
+    computed_file.save()
 
-        # Only upload and clean up projects and the last sample if multiplexed
-        if computed_file.project or computed_file.sample.is_last_multiplexed_sample:
-            if update_s3:
-                s3.upload_output_file(computed_file.s3_key, computed_file.s3_bucket)
-            if clean_up_output_data:
-                computed_file.clean_up_local_computed_file()
-        computed_file.save()
+
+def _create_computed_file_callback(future, *, update_s3: bool, clean_up_output_data: bool) -> None:
+    """
+    Wrap multiprocessing logic by grabbing computed file future and uploading it tohe s3.
+    """
+    if computed_file := future.result():
+        _create_computed_file(computed_file, update_s3, clean_up_output_data)
 
     # Close DB connection for each thread.
     connection.close()
 
 
+def generate_computed_file(
+    *,
+    download_config: Dict,
+    project: Project | None = None,
+    sample: Sample | None = None,
+    update_s3: bool = True,
+) -> None:
+
+    # Purge old computed file
+    if old_computed_file := (project or sample).get_computed_file(download_config):
+        old_computed_file.purge(update_s3)
+
+    if project:
+        computed_file = ComputedFile.get_project_file(
+            project, download_config, project.get_output_file_name(download_config)
+        )
+        _create_computed_file(computed_file, update_s3, clean_up_output_data=False)
+    elif sample:
+        computed_file = ComputedFile.get_sample_file(
+            sample,
+            download_config,
+            sample.get_output_file_name(download_config),
+            Lock(),  # this should be removed when CF::get_sample_file is refactored
+        )
+        _create_computed_file(computed_file, update_s3, clean_up_output_data=False)
+        sample.project.update_downloadable_sample_count()
+
+
 def generate_computed_files(
     project: Project,
     max_workers: int,
@@ -170,7 +213,7 @@ def generate_computed_files(
 
     # Prep callback function
     on_get_file = partial(
-        _create_computed_file,
+        _create_computed_file_callback,
         update_s3=update_s3,
         clean_up_output_data=clean_up_output_data,
     )

api/scpca_portal/management/commands/generate_computed_file.py

@@ -0,0 +1,72 @@
+import logging
+from typing import Dict
+
+from django.core.management.base import BaseCommand
+
+from scpca_portal import common, loader
+from scpca_portal.models import Project, Sample
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+logger.addHandler(logging.StreamHandler())
+
+
+class Command(BaseCommand):
+    help = """
+    This command is meant to be called as an entrypoint to AWS Batch Fargate job instance.
+    Individual files are computed according:
+        - To the project or sample id
+        - An appropriate corresponding download config
+
+    When computation is completed, files are uploaded to S3, and the job is marked as completed.
+
+    At which point the instance which generated this computed file will receive a new job
+    from the job queue and begin computing the next file.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument("--project-id", type=str)
+        parser.add_argument("--sample-id", type=str)
+        parser.add_argument("--download-config", type=dict)
+
+    def handle(self, *args, **kwargs):
+        self.generate_computed_file(**kwargs)
+
+    def generate_computed_file(
+        self,
+        project_id: str,
+        sample_id: str,
+        download_config: Dict,
+        **kwargs,
+    ) -> None:
+        """Generates a project's computed files according predetermined download configurations"""
+        loader.prep_data_dirs()
+
+        if project_id:
+            project = Project.objects.filter(scpca_id=project_id).first()
+            sample = None
+            if not project:
+                logger.error("project doesn't exist")
+                return
+            if download_config not in common.GENERATED_PROJECT_DOWNLOAD_CONFIGS:
+                logger.error("download_config is not valid")
+                return
+        elif sample_id:
+            sample = Sample.objects.filter(scpca_id=sample_id).first()
+            project = None
+            if not sample:
+                logger.error("sample doesn't exist")
+                return
+            if download_config not in common.GENERATED_SAMPLE_DOWNLOAD_CONFIGS:
+                logger.error("download_config is not valid")
+                return
+
+        else:
+            logger.error(
+                "neither project_id nor sample_id were passed. at least one must be passed."
+            )
+            return
+
+        loader.generate_computed_file(
+            download_config=download_config, project=project, sample=sample
+        )

api/scpca_portal/models/computed_file.py

@@ -361,3 +361,9 @@ def zip_file_path(self):
     def clean_up_local_computed_file(self):
         """Delete local computed file."""
         self.zip_file_path.unlink(missing_ok=True)
+
+    def purge(self, delete_from_s3: bool = False) -> None:
+        """Purges a computed file, optionally deleting it from S3."""
+        if delete_from_s3:
+            s3.delete_output_file(self.s3_key, self.s3_bucket)
+        self.delete()

api/scpca_portal/models/project.py

@@ -205,9 +205,9 @@ def get_output_file_name(self, download_config: Dict) -> str:
     def get_computed_file(self, download_config: Dict) -> ComputedFile:
         "Return the project computed file that matches the passed download_config."
         if download_config["metadata_only"]:
-            return self.computed_files.filter(metadata_only=True).first()
+            return self.project_computed_files.filter(metadata_only=True).first()
 
-        return self.computed_files.filter(
+        return self.project_computed_files.filter(
             modality=download_config["modality"],
             format=download_config["format"],
             has_multiplexed_data=(not download_config["excludes_multiplexed"]),
@@ -315,16 +315,11 @@ def purge_computed_files(self, delete_from_s3: bool = False) -> None:
         """Purges all computed files associated with the project instance."""
         # Delete project's sample computed files
         for sample in self.samples.all():
-            for computed_file in sample.computed_files:
-                if delete_from_s3:
-                    s3.delete_output_file(computed_file.s3_key, computed_file.s3_bucket)
-                computed_file.delete()
+            sample.purge_computed_files(delete_from_s3)
 
         # Delete project's project computed files
-        for computed_file in self.computed_files:
-            if delete_from_s3:
-                s3.delete_output_file(computed_file.s3_key, computed_file.s3_bucket)
-            computed_file.delete()
+        for computed_file in self.project_computed_files.all():
+            computed_file.purge(delete_from_s3)
 
     def update_sample_derived_properties(self):
         """

api/scpca_portal/models/sample.py

@@ -123,7 +123,7 @@ def get_metadata(self) -> Dict:
 
     def get_computed_file(self, download_config: Dict) -> ComputedFile:
         "Return the sample computed file that matches the passed download_config."
-        return self.computed_files.filter(
+        return self.sample_computed_files.filter(
             modality=download_config["modality"],
             format=download_config["format"],
         ).first()
@@ -265,3 +265,7 @@ def purge(self) -> None:
             if library.samples.count() == 1:
                 library.delete()
         self.delete()
+
+    def purge_computed_files(self, delete_from_s3: bool = False) -> None:
+        for computed_file in self.sample_computed_files.all():
+            computed_file.purge(delete_from_s3)