-
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #770 from AlexsLemonade/dev
Production Deploy
- Loading branch information
Showing
51 changed files
with
2,231 additions
and
1,158 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| import csv | ||
| import json | ||
| from collections import namedtuple | ||
| from pathlib import Path | ||
| from typing import Dict, List, Tuple | ||
|
|
||
| from scpca_portal import common, utils | ||
|
|
||
| PROJECT_METADATA_KEYS = [ | ||
| # Fields used in Project model object creation | ||
| ("has_bulk", "has_bulk_rna_seq", False), | ||
| ("has_CITE", "has_cite_seq_data", False), | ||
| ("has_multiplex", "has_multiplexed_data", False), | ||
| ("has_spatial", "has_spatial_data", False), | ||
| ("PI", "human_readable_pi_name", None), | ||
| ("submitter", "pi_name", None), | ||
| ("project_title", "title", None), | ||
| # Fields used in Contact model object creation | ||
| ("contact_email", "email", None), | ||
| ("contact_name", "name", None), | ||
| # Fields used in ExternalAccession model object creation | ||
| ("external_accession", "accession", None), | ||
| ("external_accession_raw", "has_raw", False), | ||
| ("external_accession_url", "accession_url", None), | ||
| # Field used in Publication model object creation | ||
| ("citation_doi", "doi", None), | ||
| ] | ||
|
|
||
| SAMPLE_METADATA_KEYS = [ | ||
| ("age", "age_at_diagnosis", None), | ||
| ] | ||
|
|
||
| LIBRARY_METADATA_KEYS = [ | ||
| ("library_id", "scpca_library_id", None), | ||
| ("sample_id", "scpca_sample_id", None), | ||
| # Field only included in Single cell (and Multiplexed) libraries | ||
| ("filtered_cells", "filtered_cell_count", None), | ||
| ] | ||
| KeyTransform = namedtuple("KeyTransform", ["old_key", "new_key", "default_value"]) | ||
|
|
||
|
|
||
| def load_projects_metadata(metadata_file_path: Path): | ||
| """ | ||
| Opens, loads and parses list of project metadata located at inputted metadata_file_path. | ||
| Transforms keys in data dicts to match associated model attributes. | ||
| """ | ||
| with open(metadata_file_path) as raw_file: | ||
| data_dicts = list(csv.DictReader(raw_file)) | ||
|
|
||
| for data_dict in data_dicts: | ||
| transform_keys(data_dict, PROJECT_METADATA_KEYS) | ||
|
|
||
| return data_dicts | ||
|
|
||
|
|
||
| def load_samples_metadata(metadata_file_path: Path): | ||
| """ | ||
| Opens, loads and parses list of sample metadata located at inputted metadata_file_path. | ||
| Transforms keys in data dicts to match associated model attributes. | ||
| """ | ||
| with open(metadata_file_path) as raw_file: | ||
| data_dicts = list(csv.DictReader(raw_file)) | ||
|
|
||
| for data_dict in data_dicts: | ||
| transform_keys(data_dict, SAMPLE_METADATA_KEYS) | ||
|
|
||
| return data_dicts | ||
|
|
||
|
|
||
| def load_library_metadata(metadata_file_path: Path): | ||
| """ | ||
| Opens, loads and parses single library's metadata located at inputted metadata_file_path. | ||
| Transforms keys in data dicts to match associated model attributes. | ||
| """ | ||
| with open(metadata_file_path) as raw_file: | ||
| return transform_keys(json.load(raw_file), LIBRARY_METADATA_KEYS) | ||
|
|
||
|
|
||
| def transform_keys(data_dict: Dict, key_transforms: List[Tuple]): | ||
| """ | ||
| Transforms keys in inputted data dict according to inputted key transforms tuple list. | ||
| """ | ||
| for element in [KeyTransform._make(element) for element in key_transforms]: | ||
| if element.old_key in data_dict: | ||
| data_dict[element.new_key] = data_dict.pop(element.old_key, element.default_value) | ||
|
|
||
| return data_dict | ||
|
|
||
|
|
||
| def write_metadata_dicts(list_of_dicts: List[Dict], output_file_path: str, **kwargs) -> None: | ||
| """ | ||
| Writes a list of dictionaries to a csv-like file. | ||
| Optional modifiers to the csv.DictWriter can be passed to function as kwargs. | ||
| """ | ||
| kwargs["fieldnames"] = kwargs.get( | ||
| "fieldnames", utils.get_sorted_field_names(utils.get_keys_from_dicts(list_of_dicts)) | ||
| ) | ||
| kwargs["delimiter"] = kwargs.get("delimiter", common.TAB) | ||
|
|
||
| sorted_list_of_dicts = sorted( | ||
| list_of_dicts, | ||
| key=lambda k: ( | ||
| k[common.PROJECT_ID_KEY], | ||
| k[common.SAMPLE_ID_KEY], | ||
| k[common.LIBRARY_ID_KEY], | ||
| ), | ||
| ) | ||
|
|
||
| with open(output_file_path, "w", newline="") as raw_file: | ||
| csv_writer = csv.DictWriter(raw_file, **kwargs) | ||
| csv_writer.writeheader() | ||
| csv_writer.writerows(sorted_list_of_dicts) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| # Generated by Django 3.2.25 on 2024-05-24 14:59 | ||
|
|
||
| import django.contrib.postgres.fields | ||
| from django.db import migrations, models | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
|
|
||
| dependencies = [ | ||
| ("scpca_portal", "0042_auto_20240423_2045"), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.CreateModel( | ||
| name="Library", | ||
| fields=[ | ||
| ( | ||
| "id", | ||
| models.AutoField( | ||
| auto_created=True, primary_key=True, serialize=False, verbose_name="ID" | ||
| ), | ||
| ), | ||
| ("created_at", models.DateTimeField(auto_now_add=True)), | ||
| ("updated_at", models.DateTimeField(auto_now=True)), | ||
| ( | ||
| "formats", | ||
| django.contrib.postgres.fields.ArrayField( | ||
| base_field=models.TextField( | ||
| choices=[ | ||
| ("ANN_DATA", "AnnData"), | ||
| ("SINGLE_CELL_EXPERIMENT", "Single cell experiment"), | ||
| ] | ||
| ), | ||
| default=list, | ||
| size=None, | ||
| ), | ||
| ), | ||
| ("is_multiplexed", models.BooleanField(default=False)), | ||
| ("metadata", models.JSONField(default=dict)), | ||
| ( | ||
| "modality", | ||
| models.TextField( | ||
| choices=[("SINGLE_CELL", "Single Cell"), ("SPATIAL", "Spatial")] | ||
| ), | ||
| ), | ||
| ("scpca_id", models.TextField(unique=True)), | ||
| ("workflow_version", models.TextField()), | ||
| ], | ||
| options={ | ||
| "db_table": "libraries", | ||
| "ordering": ["updated_at"], | ||
| "get_latest_by": "updated_at", | ||
| }, | ||
| ), | ||
| migrations.AddField( | ||
| model_name="sample", | ||
| name="libraries", | ||
| field=models.ManyToManyField(to="scpca_portal.Library"), | ||
| ), | ||
| ] |
Oops, something went wrong.