From 8660043591c627d5b24d96ef24183a91fd67a7c4 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 16 Apr 2024 13:34:09 -0600 Subject: [PATCH 01/22] qiita.slurm_resource_allocations (#3391) * Update CHANGELOG.md * add db changes including fix #3368 --- qiita_db/support_files/patches/92.sql | 20 +++ qiita_db/support_files/qiita-db.dbs | 39 ++++- qiita_db/support_files/qiita-db.html | 201 ++++++++++++++++++++++---- 3 files changed, 231 insertions(+), 29 deletions(-) create mode 100644 qiita_db/support_files/patches/92.sql diff --git a/qiita_db/support_files/patches/92.sql b/qiita_db/support_files/patches/92.sql new file mode 100644 index 000000000..a4904adb4 --- /dev/null +++ b/qiita_db/support_files/patches/92.sql @@ -0,0 +1,20 @@ +-- Apr 16, 2024 +-- Adding a new table to contain the basic slurm information to minimize +-- the number of times we need to retrieve this information + +CREATE TABLE qiita.slurm_resource_allocations ( + processing_job_id uuid NOT NULL, + samples integer, + columns integer, + input_size bigint, + extra_info varchar DEFAULT NULL, + memory_used bigint, + walltime_used integer, + CONSTRAINT pk_slurm_resource_allocations_processing_job_id PRIMARY KEY ( + processing_job_id ) + ); + +ALTER TABLE qiita.slurm_resource_allocations + ADD CONSTRAINT fk_slurm_resource_allocations + FOREIGN KEY ( processing_job_id ) + REFERENCES qiita.processing_job ( processing_job_id ); diff --git a/qiita_db/support_files/qiita-db.dbs b/qiita_db/support_files/qiita-db.dbs index 32cdb3f78..c156f4749 100644 --- a/qiita_db/support_files/qiita-db.dbs +++ b/qiita_db/support_files/qiita-db.dbs @@ -1326,6 +1326,22 @@ + + + + + + + + + + + + + + + +
@@ -1338,6 +1354,23 @@
+ + + + + + + + + + + + + + + + +
@@ -2023,14 +2056,14 @@ $function$ - + - + @@ -2091,7 +2124,9 @@ $function$ + + diff --git a/qiita_db/support_files/qiita-db.html b/qiita_db/support_files/qiita-db.html index 177f4b38d..252d8e05c 100644 --- a/qiita_db/support_files/qiita-db.html +++ b/qiita_db/support_files/qiita-db.html @@ -2,7 +2,7 @@ -PostgreSQL +Qiita DB @@ -286,6 +286,11 @@ + + + + + @@ -294,7 +299,7 @@ - PostgreSQL + Qiita DB (c) DbSchema Hover columns to read the comments. @@ -397,21 +402,21 @@ analysis_sample ref artifact ( artifact_id ) artifact_id - + Fk fk_analysis_users_analysis analysis_users ref analysis ( analysis_id ) -Fk fk_analysis_users_analysis +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 480,160L 480,152Q 480,144 488,144L 704,144' ><title>Fk fk_analysis_users_analysis analysis_users ref analysis ( analysis_id ) -analysis_id +analysis_id - + Fk fk_analysis_users_user analysis_users ref qiita_user ( email ) -Fk fk_analysis_users_user +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 368,256L 368,1240Q 368,1248 360,1248L 352,1248' ><title>Fk fk_analysis_users_user analysis_users ref qiita_user ( email ) -email +email Fk fk_archive_feature_value @@ -917,13 +922,13 @@ processing_job ref processing_job_status ( processing_job_status_id ) processing_job_status_id - + Fk fk_processing_job processing_job ref software_command ( command_id ) -Fk fk_processing_job +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 1776,816L 2856,816Q 2864,816 2864,808L 2864,520Q 2864,512 2872,512L 2888,512Q 2896,512 2896,504L 2896,496' ><title>Fk fk_processing_job processing_job ref software_command ( command_id ) -command_id +command_id Fk fk_processing_job_validator_c @@ -997,13 +1002,21 @@ sample_template_filepath ref filepath ( filepath_id ) filepath_id - + Fk fk_study_id sample_template_filepath ref study ( study_id ) -Fk fk_study_id +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2704,1296L 2440,1296Q 2432,1296 2432,1288L 2432,1256Q 2432,1248 2424,1248L 1880,1248Q 1872,1248 1872,1256L 1872,1336Q 1872,1344 1864,1344L 376,1344Q 368,1344 368,1352L 368,1464Q 368,1472 360,1472L 24,1472Q 16,1472 16,1464L 16,872Q 16,864 24,864L 32,864' ><title>Fk fk_study_id sample_template_filepath ref study ( study_id ) study_id + + + Fk fk_slurm_resource_allocations +slurm_resource_allocations ref processing_job ( processing_job_id ) + +Fk fk_slurm_resource_allocations +slurm_resource_allocations ref processing_job ( processing_job_id ) +processing_job_id Fk fk_software_software_type @@ -1085,13 +1098,13 @@ study_artifact ref artifact ( artifact_id ) artifact_id - + Fk fk_study_artifact_study study_artifact ref study ( study_id ) -Fk fk_study_artifact_study +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 320,1136L 328,1136Q 336,1136 336,1128L 336,872Q 336,864 328,864L 320,864' ><title>Fk fk_study_artifact_study study_artifact ref study ( study_id ) -study_id +study_id Fk fk_study_environmental_package_0 @@ -1276,17 +1289,17 @@ <text x='896' y='344' text-anchor='end' class='colType'>bigint</text><a xlink:href='#qiita.analysis_sample.artifact_id'><use id='fk' x='900' y='332' xlink:href='#fk'/><title>References artifact ( artifact_id ) - - - -analysis_usersTable qiita.analysis_users +<rect class='entity' style='stroke:none;' x='352' y='168' width='144' height='80' rx='8' ry='8' /> +<path d='M 352 196 L 352 176 Q 352 168 360 168 L 488 168 Q 496 168 496 176 L 496 196 L352 196 ' style='fill:url(#tbg_D1BEF4);stroke:1;stroke-opacity:0.1;' /> +<rect class='entity' x='352' y='168' width='144' height='80' rx='8' ry='8' style='fill:none;stroke:#28272C'/> +<a xlink:href='#qiita.analysis_users'><text x='376' y='187' >analysis_users</text><title>Table qiita.analysis_users Links analyses to the users they are shared with - Pk idx_analysis_users ( analysis_id, email ) idx_analysis_users_analysis ( analysis_id ) analysis_idanalysis_id + <use id='nn' x='354' y='205' xlink:href='#nn'/><use id='pk' x='354' y='204' xlink:href='#pk'><title>Pk idx_analysis_users ( analysis_id, email ) idx_analysis_users_analysis ( analysis_id ) analysis_idanalysis_id * bigint -bigintReferences analysis ( analysis_id ) - Pk idx_analysis_users ( analysis_id, email ) idx_analysis_users_email ( email ) emailemail +<text x='480' y='216' text-anchor='end' class='colType'>bigint</text><a xlink:href='#qiita.analysis_users.analysis_id'><use id='fk' x='484' y='204' xlink:href='#fk'/><title>References analysis ( analysis_id ) + Pk idx_analysis_users ( analysis_id, email ) idx_analysis_users_email ( email ) emailemail * varchar -varcharReferences qiita_user ( email ) +varcharReferences qiita_user ( email ) @@ -2065,7 +2078,7 @@ processing_jobTable qiita.processing_job - Pk pk_processing_job ( processing_job_id ) processing_job_idprocessing_job_id + <use id='nn' x='1538' y='797' xlink:href='#nn'/><use id='pk' x='1538' y='796' xlink:href='#pk'><title>Pk pk_processing_job ( processing_job_id ) processing_job_idprocessing_job_id * uuid default uuid_generate_v4() uuidReferred by analysis_processing_job ( processing_job_id ) Referred by artifact_output_processing_job ( processing_job_id ) @@ -2075,7 +2088,8 @@ Referred by prep_template_processing_job ( processing_job_id ) Referred by processing_job_validator ( validator_id -> processing_job_id ) Referred by processing_job_validator ( processing_job_id ) -Referred by processing_job_workflow_root ( processing_job_id ) +Referred by processing_job_workflow_root ( processing_job_id ) +Referred by slurm_resource_allocations ( processing_job_id ) idx_processing_job_email ( email ) emailemail * varchar The user that launched the job @@ -2274,6 +2288,24 @@ varchar sample_valuessample_values jsonb jsonb + + + + +settingsTable qiita.settings + testtest +* boolean default true +boolean base_data_dirbase_data_dir +* varchar +varchar base_work_dirbase_work_dir +* varchar +varchar current_patchcurrent_patch +* varchar default 'unpatched'::character varying +varchar max_preparation_samplesmax_preparation_samples +integer default 800 +integer max_artifacts_in_workflowmax_artifacts_in_workflow +integer default 35 +integer @@ -2285,6 +2317,27 @@ Unq idx_severity ( severity ) severityseverity * varchar varchar + + + + +slurm_resource_allocationsTable qiita.slurm_resource_allocations + Pk pk_slurm_resource_allocations_processing_job_id ( processing_job_id ) processing_job_idprocessing_job_id +* uuid +uuidReferences processing_job ( processing_job_id ) + samplessamples +integer +integer columnscolumns +integer +integer input_sizeinput_size +bigint +bigint extra_infoextra_info +varchar default null +varchar memory_usedmemory_used +bigint +bigint walltime_usedwalltime_used +integer +integer @@ -2980,7 +3033,7 @@


-
Table analysis_users
 +
Table analysis_users

Links analyses to the users they are shared with

@@ -5684,6 +5737,46 @@
+

+
Table settings
 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdxField NameData Type
*test boolean DEFAULT true
*base_data_dir varchar
*base_work_dir varchar
*current_patch varchar DEFAULT 'unpatched'::character varying
 max_preparation_samples integer DEFAULT 800
 max_artifacts_in_workflow integer DEFAULT 35
+

Table severity
 @@ -5711,6 +5804,60 @@
+

+
Table slurm_resource_allocations
 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdxField NameData Type
*processing_job_id uuid
 samples integer
 columns integer
 input_size bigint
 extra_info varchar DEFAULT null
 memory_used bigint
 walltime_used integer
Indexes
pk_slurm_resource_allocations_processing_job_id ON processing_job_id
Foreign Keys
fk_slurm_resource_allocations ( processing_job_id ) ref processing_job (processing_job_id)
+

Table software
 From fe506142fef851acd3130efda4590d96d8454c56 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 16 Apr 2024 13:50:11 -0600 Subject: [PATCH 02/22] Pac bio smrt new instruments (#3390) * Update CHANGELOG.md * PacBio_SMRT new instruments --- .../doc/source/checklist-for-ebi-ena-submission.rst | 2 +- qiita_ware/ebi.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst b/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst index 37efbe5b6..1d4191077 100644 --- a/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst +++ b/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst @@ -188,7 +188,7 @@ Remember, metadata is the most important part for an analysis, without it we onl +---------------------+----------------------------------------------------------------------------------------------------------+ | ``Ion_Torrent`` | ``Ion Torrent PGM``, ``Ion Torrent Proton``, ``Ion Torrent S5``, ``Ion Torrent S5 XL`` | +---------------------+----------------------------------------------------------------------------------------------------------+ - | ``PacBio_SMRT`` | ``PacBio RS``, ``PacBio RS II``, ``Sequel``, ``Sequel II`` | + | ``PacBio_SMRT`` | ``PacBio RS``, ``PacBio RS II``, ``Sequel``, ``Sequel II``, ``Sequel IIe``, ``Revio``, ``Onso`` | +---------------------+----------------------------------------------------------------------------------------------------------+ | ``Oxford_Nanopore`` | ``GridION`` | +---------------------+----------------------------------------------------------------------------------------------------------+ diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 35e98dd1f..ec6b084c9 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -117,6 +117,9 @@ class EBISubmission(object): 'PACBIO_SMRT': ['PACBIO RS', 'PACBIO RS II', 'SEQUEL', + 'ONSO', + 'REVIO', + 'SEQUEL IIE', 'SEQUEL II']} xmlns_xsi = "http://www.w3.org/2001/XMLSchema-instance" From 28d719157cfcf68ee8a35596287a332d425ca7dd Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 22 Apr 2024 10:06:53 -0600 Subject: [PATCH 03/22] Adding current human filtering flag to prep_template (#3395) * Update CHANGELOG.md * adding current_human_filtering to PrepTemplate --- qiita_db/artifact.py | 10 +- qiita_db/metadata_template/prep_template.py | 25 ++ .../test/test_prep_template.py | 9 + qiita_db/support_files/patches/92.sql | 4 + qiita_db/support_files/qiita-db.dbs | 7 +- qiita_db/support_files/qiita-db.html | 288 +++++++++--------- qiita_db/test/test_artifact.py | 10 + 7 files changed, 210 insertions(+), 143 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index e4ac92a34..13d72052a 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -1574,7 +1574,15 @@ def being_deleted_by(self): @property def has_human(self): has_human = False - if self.artifact_type == 'per_sample_FASTQ': + # we are going to check the metadata if: + # - the prep data_type is _not_ target gene + # - the prep is not current_human_filtering + # - if the artifact_type is 'per_sample_FASTQ' + pts = self.prep_templates + tgs = qdb.metadata_template.constants.TARGET_GENE_DATA_TYPES + ntg = any([pt.data_type() not in tgs for pt in pts]) + chf = any([not pt.current_human_filtering for pt in pts]) + if ntg and chf and self.artifact_type == 'per_sample_FASTQ': st = self.study.sample_template if 'env_package' in st.categories: sql = f"""SELECT DISTINCT sample_values->>'env_package' diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 4a4cc98fa..77884db17 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -1082,3 +1082,28 @@ def creation_job_id(self, creation_job_id): WHERE prep_template_id = %s""" qdb.sql_connection.TRN.add(sql, [creation_job_id, self.id]) qdb.sql_connection.TRN.execute() + + @property + def current_human_filtering(self): + """If the preparation is current with human filtering + + Returns + ------- + bool + The current_human_filtering of the prep information + """ + with qdb.sql_connection.TRN: + sql = """SELECT current_human_filtering + FROM qiita.prep_template + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, [self.id]) + return qdb.sql_connection.TRN.execute_fetchlast() + + @current_human_filtering.setter + def current_human_filtering(self, current_human_filtering): + with qdb.sql_connection.TRN: + sql = """UPDATE qiita.prep_template + SET current_human_filtering = %s + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, [current_human_filtering, self.id]) + qdb.sql_connection.TRN.execute() diff --git a/qiita_db/metadata_template/test/test_prep_template.py b/qiita_db/metadata_template/test/test_prep_template.py index 81769082d..bfc72ad62 100644 --- a/qiita_db/metadata_template/test/test_prep_template.py +++ b/qiita_db/metadata_template/test/test_prep_template.py @@ -1905,6 +1905,15 @@ def test_name_setter(self): pt.name = 'Prep information 1' self.assertEqual(pt.name, 'Prep information 1') + def test_current_human_filtering(self): + pt = qdb.metadata_template.prep_template.PrepTemplate(1) + # by default it should be false + self.assertFalse(pt.current_human_filtering) + pt.current_human_filtering = True + self.assertTrue(pt.current_human_filtering) + pt.current_human_filtering = False + self.assertFalse(pt.current_human_filtering) + EXP_PREP_TEMPLATE = ( 'sample_name\tbarcode\tcenter_name\tcenter_project_name\t' diff --git a/qiita_db/support_files/patches/92.sql b/qiita_db/support_files/patches/92.sql index a4904adb4..0c2ad5034 100644 --- a/qiita_db/support_files/patches/92.sql +++ b/qiita_db/support_files/patches/92.sql @@ -18,3 +18,7 @@ ALTER TABLE qiita.slurm_resource_allocations ADD CONSTRAINT fk_slurm_resource_allocations FOREIGN KEY ( processing_job_id ) REFERENCES qiita.processing_job ( processing_job_id ); + +-- Apr 21, 2024 +-- Adding a new column: current_human_filtering to qiita.prep_template +ALTER TABLE qiita.prep_template ADD current_human_filtering boolean DEFAULT False; diff --git a/qiita_db/support_files/qiita-db.dbs b/qiita_db/support_files/qiita-db.dbs index c156f4749..7e9a970ca 100644 --- a/qiita_db/support_files/qiita-db.dbs +++ b/qiita_db/support_files/qiita-db.dbs @@ -1001,6 +1001,9 @@ + + + @@ -2148,10 +2151,10 @@ $function$ - \ No newline at end of file + diff --git a/qiita_db/support_files/qiita-db.html b/qiita_db/support_files/qiita-db.html index 252d8e05c..942c14ae8 100644 --- a/qiita_db/support_files/qiita-db.html +++ b/qiita_db/support_files/qiita-db.html @@ -65,9 +65,9 @@ +