From d743a810202a4749a8f38d7c0e9b5451eb71ef46 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 16:31:39 +0100 Subject: [PATCH 01/24] make help email address configurable instead of hard coding (#3384) * Update CHANGELOG.md * make help email address configurable instead of hard coding * codestyle * don't default with email addresses but raise Errors * set mail addresses in example configuration, which is also used for testing * addessing Antonios issues + switch to a non deprecated read fct. * updating dummy help email address in tests --------- Co-authored-by: Antonio Gonzalez --- qiita_core/configuration_manager.py | 30 ++++++++++++++ qiita_core/support_files/config_test.cfg | 6 +++ .../tests/test_configuration_manager.py | 39 ++++++++++++++++++- qiita_db/processing_job.py | 4 +- .../artifact_handlers/base_handlers.py | 2 +- qiita_pet/handlers/auth_handlers.py | 12 +++--- qiita_pet/handlers/download.py | 26 ++++++------- .../study_handlers/sample_template.py | 6 +-- qiita_pet/templates/admin_processing_job.html | 2 +- qiita_pet/templates/error.html | 2 +- qiita_pet/templates/list_analyses.html | 2 +- qiita_pet/templates/redbiom.html | 2 +- qiita_pet/templates/sitebase.html | 2 +- qiita_pet/templates/study_ajax/base_info.html | 2 +- qiita_pet/test/test_download.py | 10 ++--- qiita_ware/test/test_private_plugin.py | 6 +-- 16 files changed, 114 insertions(+), 39 deletions(-) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 4c26583d9..44db36400 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -117,6 +117,10 @@ class ConfigurationManager(object): The script used to start the plugins plugin_dir : str The path to the directory containing the plugin configuration files + help_email : str + The email address a user should write to when asking for help + sysadmin_email : str + The email address, Qiita sends internal notifications to a sys admin Raises ------ @@ -234,6 +238,32 @@ def _get_main(self, config): self.key_file = join(install_dir, 'qiita_core', 'support_files', 'ci_server.key') + self.help_email = config.get('main', 'HELP_EMAIL') + if not self.help_email: + raise ValueError( + "You did not specify the HELP_EMAIL address in the main " + "section of Qiita's config file. This address is essential " + "for users to ask for help as it is displayed at various " + "location throughout Qiita's web pages.") + if (self.help_email == 'foo@bar.com') and \ + (self.test_environment is False): + warnings.warn( + "Using the github fake email for HELP_EMAIL, " + "are you sure this is OK?") + + self.sysadmin_email = config.get('main', 'SYSADMIN_EMAIL') + if not self.sysadmin_email: + raise ValueError( + "You did not specify the SYSADMIN_EMAIL address in the main " + "section of Qiita's config file. Serious issues will " + "automatically be reported to a sys admin, an according " + "address is therefore required!") + if (self.sysadmin_email == 'jeff@bar.com') and \ + (self.test_environment is False): + warnings.warn( + "Using the github fake email for SYSADMIN_EMAIL, " + "are you sure this is OK?") + def _get_job_scheduler(self, config): """Get the configuration of the job_scheduler section""" self.job_scheduler_owner = config.get( diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index e6423f609..8f922e34e 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -68,6 +68,12 @@ COOKIE_SECRET = SECRET # The value used to secure JWTs for delegated permission artifact download. JWT_SECRET = SUPER_SECRET +# Address a user should write to when asking for help +HELP_EMAIL = foo@bar.com + +# The email address, Qiita sends internal notifications to a sys admin +SYSADMIN_EMAIL = jeff@bar.com + # ----------------------------- SMTP settings ----------------------------- [smtp] # The hostname to connect to diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 2a60f353b..ebc7f67f1 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -29,7 +29,7 @@ def setUp(self): self.conf = ConfigParser() with open(self.conf_fp, newline=None) as f: - self.conf.readfp(f) + self.conf.read_file(f) def tearDown(self): if self.old_conf_fp is not None: @@ -132,6 +132,8 @@ def test_get_main(self): # Warning raised if No files will be allowed to be uploaded # Warning raised if no cookie_secret + self.conf.set('main', 'HELP_EMAIL', 'ignore@me') + self.conf.set('main', 'SYSADMIN_EMAIL', 'ignore@me') with warnings.catch_warnings(record=True) as warns: obs._get_main(self.conf) @@ -180,6 +182,35 @@ def test_get_main(self): self.assertEqual(obs.qiita_env, "") + def test_help_email(self): + obs = ConfigurationManager() + + with warnings.catch_warnings(record=True) as warns: + # warning get only issued when in non test environment + self.conf.set('main', 'TEST_ENVIRONMENT', 'FALSE') + + obs._get_main(self.conf) + self.assertEqual(obs.help_email, 'foo@bar.com') + self.assertEqual(obs.sysadmin_email, 'jeff@bar.com') + + obs_warns = [str(w.message) for w in warns] + exp_warns = [ + 'Using the github fake email for HELP_EMAIL, ' + 'are you sure this is OK?', + 'Using the github fake email for SYSADMIN_EMAIL, ' + 'are you sure this is OK?'] + self.assertCountEqual(obs_warns, exp_warns) + + # test if it falls back to qiita.help@gmail.com + self.conf.set('main', 'HELP_EMAIL', '') + with self.assertRaises(ValueError): + obs._get_main(self.conf) + + # test if it falls back to qiita.help@gmail.com + self.conf.set('main', 'SYSADMIN_EMAIL', '') + with self.assertRaises(ValueError): + obs._get_main(self.conf) + def test_get_job_scheduler(self): obs = ConfigurationManager() @@ -274,6 +305,12 @@ def test_get_portal(self): # The value used to secure JWTs for delegated permission artifact download. JWT_SECRET = SUPER_SECRET +# Address a user should write to when asking for help +HELP_EMAIL = foo@bar.com + +# The email address, Qiita sends internal notifications to a sys admin +SYSADMIN_EMAIL = jeff@bar.com + # ----------------------------- SMTP settings ----------------------------- [smtp] # The hostname to connect to diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index a1f7e5baa..11145925b 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -493,7 +493,7 @@ def resource_allocation_info(self): samples, columns, input_size = self.shape parts = [] error_msg = ('Obvious incorrect allocation. Please ' - 'contact qiita.help@gmail.com') + 'contact %s' % qiita_config.help_email) for part in allocation.split('--'): param = '' if part.startswith('time '): @@ -902,7 +902,7 @@ def _set_status(self, value, error_msg=None): if self.user.level in {'admin', 'wet-lab admin'}: if value == 'error': qdb.util.send_email( - 'jdereus@health.ucsd.edu', msg['subject'], + qiita_config.sysadmin_email, msg['subject'], msg['message']) sql = """UPDATE qiita.processing_job diff --git a/qiita_pet/handlers/artifact_handlers/base_handlers.py b/qiita_pet/handlers/artifact_handlers/base_handlers.py index ca60051cb..bdeae6eaf 100644 --- a/qiita_pet/handlers/artifact_handlers/base_handlers.py +++ b/qiita_pet/handlers/artifact_handlers/base_handlers.py @@ -370,7 +370,7 @@ def artifact_patch_request(user, artifact_id, req_op, req_path, req_value=None, sid = artifact.study.id if artifact.visibility == 'awaiting_approval': - email_to = 'qiita.help@gmail.com' + email_to = qiita_config.help_email subject = ('QIITA: Artifact %s awaiting_approval. Study %d, ' 'Prep %d' % (artifact_id, sid, artifact.prep_templates[0].id)) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 38759e77f..4c5e306b1 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -62,8 +62,8 @@ def post(self): url_escape(username), url)) except Exception: msg = ("Unable to send verification email. Please contact the " - "qiita developers at qiita.help@gmail.com") + "qiita developers at %s") % ( + qiita_config.help_email, qiita_config.help_email) self.redirect(u"%s/?level=danger&message=%s" % (qiita_config.portal_dir, url_escape(msg))) return @@ -75,8 +75,9 @@ def post(self): "

If you don't receive your activation email within a " "couple of minutes, check your spam folder. If you still " "don't see it, send us an email at qiita.help@gmail.com" - ".

") + "href=\"mailto:%s\">%s" + ".

") % (qiita_config.help_email, + qiita_config.help_email) self.redirect(u"%s/?level=success&message=%s" % (qiita_config.portal_dir, url_escape(msg))) else: @@ -135,7 +136,8 @@ def post(self): "the verify link. You may need to check your spam " "folder to find the email.
If a verification email" " has not arrived in 15 minutes, please email qiita.help@gmail.com") + "mailto:%s'>%s") % (qiita_config.help_email, + qiita_config.help_email) except QiitaDBUnknownIDError: msg = "Unknown user" except RuntimeError: diff --git a/qiita_pet/handlers/download.py b/qiita_pet/handlers/download.py index 93c81d4c0..bf0242773 100644 --- a/qiita_pet/handlers/download.py +++ b/qiita_pet/handlers/download.py @@ -445,12 +445,12 @@ def get(self): public_raw_download = study.public_raw_download if study.status != 'public': raise HTTPError(404, reason='Study is not public. If this ' - 'is a mistake contact: ' - 'qiita.help@gmail.com') + 'is a mistake contact: %s' % + qiita_config.help_email) elif data == 'raw' and not public_raw_download: raise HTTPError(422, reason='No raw data access. If this ' - 'is a mistake contact: ' - 'qiita.help@gmail.com') + 'is a mistake contact: %s' + % qiita_config.help_email) else: # raw data artifacts = [a for a in study.artifacts(dtype=data_type) @@ -466,8 +466,8 @@ def get(self): if not to_download: raise HTTPError(422, reason='Nothing to download. If ' - 'this is a mistake contact: ' - 'qiita.help@gmail.com') + 'this is a mistake contact: %s' + % qiita_config.help_email) else: self._write_nginx_file_list(to_download) @@ -496,18 +496,18 @@ def get(self): else: if artifact.visibility != 'public': raise HTTPError(404, reason='Artifact is not public. If ' - 'this is a mistake contact: ' - 'qiita.help@gmail.com') + 'this is a mistake contact: %s' + % qiita_config.help_email) elif artifact.has_human: raise HTTPError(404, reason='Artifact has possible human ' 'sequences. If this is a mistake contact: ' - 'qiita.help@gmail.com') + '%s' % qiita_config.help_email) else: to_download = self._list_artifact_files_nginx(artifact) if not to_download: raise HTTPError(422, reason='Nothing to download. If ' - 'this is a mistake contact: ' - 'qiita.help@gmail.com') + 'this is a mistake contact: %s' + % qiita_config.help_email) else: self._write_nginx_file_list(to_download) @@ -600,8 +600,8 @@ def get(self, jti): to_download = self._list_artifact_files_nginx(artifact) if not to_download: raise HTTPError(422, reason='Nothing to download. If ' - 'this is a mistake contact: ' - 'qiita.help@gmail.com') + 'this is a mistake contact: %s' % + qiita_config.help_email) else: self._write_nginx_file_list(to_download) diff --git a/qiita_pet/handlers/study_handlers/sample_template.py b/qiita_pet/handlers/study_handlers/sample_template.py index 0e44ace76..2c3373f87 100644 --- a/qiita_pet/handlers/study_handlers/sample_template.py +++ b/qiita_pet/handlers/study_handlers/sample_template.py @@ -12,7 +12,7 @@ from tornado.web import authenticated, HTTPError -from qiita_core.qiita_settings import r_client +from qiita_core.qiita_settings import r_client, qiita_config from qiita_pet.handlers.util import to_int from qiita_pet.handlers.base_handlers import BaseHandler from qiita_db.util import get_files_from_uploads_folders @@ -214,8 +214,8 @@ def sample_template_handler_patch_request(user, req_op, req_path, # the system filepath = req_value if not exists(filepath): - reason = ('Upload file not found (%s), please report to ' - 'qiita.help@gmail.com' % filepath) + reason = ('Upload file not found (%s), please report to %s' + % (filepath, qiita_config.help_email)) raise HTTPError(404, reason=reason) else: # Check if the file exists diff --git a/qiita_pet/templates/admin_processing_job.html b/qiita_pet/templates/admin_processing_job.html index d341294d4..6f9b81acc 100644 --- a/qiita_pet/templates/admin_processing_job.html +++ b/qiita_pet/templates/admin_processing_job.html @@ -191,7 +191,7 @@ $inp.attr('type', 'file'); } else { - bootstrapAlert("Error: Parameter type (" + p_type + ") not recognized. Please, take a screenshot and contact us", "danger"); + bootstrapAlert("Error: Parameter type (" + p_type + ") not recognized. Please, take a screenshot and contact us", "danger"); } } diff --git a/qiita_pet/templates/error.html b/qiita_pet/templates/error.html index dbd51f70c..b79141735 100644 --- a/qiita_pet/templates/error.html +++ b/qiita_pet/templates/error.html @@ -3,7 +3,7 @@

ERROR: CODE {{status_code}}!

{% raw escape(error) %}

The error has been logged and will be looked at.

-

Need help? Send us an email.

+

Need help? Send us an email.

{% if is_admin %}

Go to the error page to view the logs

{% end %} diff --git a/qiita_pet/templates/list_analyses.html b/qiita_pet/templates/list_analyses.html index 0e890b210..bcfc79207 100644 --- a/qiita_pet/templates/list_analyses.html +++ b/qiita_pet/templates/list_analyses.html @@ -121,7 +121,7 @@ message.append(' '+data[level]); // prepend the "Need help" message if (level == 'warning' || level == 'danger'){ - message.append('

Need help? Send us an email.

'); + message.append('

Need help? Send us an email.

'); } analyses_all_messages.prepend(message); } diff --git a/qiita_pet/templates/redbiom.html b/qiita_pet/templates/redbiom.html index 9c0414568..5f2c31cf8 100644 --- a/qiita_pet/templates/redbiom.html +++ b/qiita_pet/templates/redbiom.html @@ -163,7 +163,7 @@ .fail(function(response, status, error) { var text = 'The query response is larger than is currently allowed, please try another. Track progress on this issue.'; if (response.status != 504) { - text = 'Status code: "' + response.status + '" - ' + error + '.
Please send a screenshot to qiita.help@gmail.com.'; + text = 'Status code: "' + response.status + '" - ' + error + '.
Please send a screenshot to {% raw qiita_config.portal_dir %}.'; } redbiom_info.html( `

-
Table analysis_users
 +
Table analysis_users

Links analyses to the users they are shared with

@@ -5684,6 +5737,46 @@
+

+
Table settings
 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdxField NameData Type
*test boolean DEFAULT true
*base_data_dir varchar
*base_work_dir varchar
*current_patch varchar DEFAULT 'unpatched'::character varying
 max_preparation_samples integer DEFAULT 800
 max_artifacts_in_workflow integer DEFAULT 35
+

Table severity
 @@ -5711,6 +5804,60 @@
+

+
Table slurm_resource_allocations
 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdxField NameData Type
*processing_job_id uuid
 samples integer
 columns integer
 input_size bigint
 extra_info varchar DEFAULT null
 memory_used bigint
 walltime_used integer
Indexes
pk_slurm_resource_allocations_processing_job_id ON processing_job_id
Foreign Keys
fk_slurm_resource_allocations ( processing_job_id ) ref processing_job (processing_job_id)
+

Table software
 From fe506142fef851acd3130efda4590d96d8454c56 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 16 Apr 2024 13:50:11 -0600 Subject: [PATCH 06/24] Pac bio smrt new instruments (#3390) * Update CHANGELOG.md * PacBio_SMRT new instruments --- .../doc/source/checklist-for-ebi-ena-submission.rst | 2 +- qiita_ware/ebi.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst b/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst index 37efbe5b6..1d4191077 100644 --- a/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst +++ b/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst @@ -188,7 +188,7 @@ Remember, metadata is the most important part for an analysis, without it we onl +---------------------+----------------------------------------------------------------------------------------------------------+ | ``Ion_Torrent`` | ``Ion Torrent PGM``, ``Ion Torrent Proton``, ``Ion Torrent S5``, ``Ion Torrent S5 XL`` | +---------------------+----------------------------------------------------------------------------------------------------------+ - | ``PacBio_SMRT`` | ``PacBio RS``, ``PacBio RS II``, ``Sequel``, ``Sequel II`` | + | ``PacBio_SMRT`` | ``PacBio RS``, ``PacBio RS II``, ``Sequel``, ``Sequel II``, ``Sequel IIe``, ``Revio``, ``Onso`` | +---------------------+----------------------------------------------------------------------------------------------------------+ | ``Oxford_Nanopore`` | ``GridION`` | +---------------------+----------------------------------------------------------------------------------------------------------+ diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 35e98dd1f..ec6b084c9 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -117,6 +117,9 @@ class EBISubmission(object): 'PACBIO_SMRT': ['PACBIO RS', 'PACBIO RS II', 'SEQUEL', + 'ONSO', + 'REVIO', + 'SEQUEL IIE', 'SEQUEL II']} xmlns_xsi = "http://www.w3.org/2001/XMLSchema-instance" From 28d719157cfcf68ee8a35596287a332d425ca7dd Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 22 Apr 2024 10:06:53 -0600 Subject: [PATCH 07/24] Adding current human filtering flag to prep_template (#3395) * Update CHANGELOG.md * adding current_human_filtering to PrepTemplate --- qiita_db/artifact.py | 10 +- qiita_db/metadata_template/prep_template.py | 25 ++ .../test/test_prep_template.py | 9 + qiita_db/support_files/patches/92.sql | 4 + qiita_db/support_files/qiita-db.dbs | 7 +- qiita_db/support_files/qiita-db.html | 288 +++++++++--------- qiita_db/test/test_artifact.py | 10 + 7 files changed, 210 insertions(+), 143 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index e4ac92a34..13d72052a 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -1574,7 +1574,15 @@ def being_deleted_by(self): @property def has_human(self): has_human = False - if self.artifact_type == 'per_sample_FASTQ': + # we are going to check the metadata if: + # - the prep data_type is _not_ target gene + # - the prep is not current_human_filtering + # - if the artifact_type is 'per_sample_FASTQ' + pts = self.prep_templates + tgs = qdb.metadata_template.constants.TARGET_GENE_DATA_TYPES + ntg = any([pt.data_type() not in tgs for pt in pts]) + chf = any([not pt.current_human_filtering for pt in pts]) + if ntg and chf and self.artifact_type == 'per_sample_FASTQ': st = self.study.sample_template if 'env_package' in st.categories: sql = f"""SELECT DISTINCT sample_values->>'env_package' diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 4a4cc98fa..77884db17 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -1082,3 +1082,28 @@ def creation_job_id(self, creation_job_id): WHERE prep_template_id = %s""" qdb.sql_connection.TRN.add(sql, [creation_job_id, self.id]) qdb.sql_connection.TRN.execute() + + @property + def current_human_filtering(self): + """If the preparation is current with human filtering + + Returns + ------- + bool + The current_human_filtering of the prep information + """ + with qdb.sql_connection.TRN: + sql = """SELECT current_human_filtering + FROM qiita.prep_template + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, [self.id]) + return qdb.sql_connection.TRN.execute_fetchlast() + + @current_human_filtering.setter + def current_human_filtering(self, current_human_filtering): + with qdb.sql_connection.TRN: + sql = """UPDATE qiita.prep_template + SET current_human_filtering = %s + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, [current_human_filtering, self.id]) + qdb.sql_connection.TRN.execute() diff --git a/qiita_db/metadata_template/test/test_prep_template.py b/qiita_db/metadata_template/test/test_prep_template.py index 81769082d..bfc72ad62 100644 --- a/qiita_db/metadata_template/test/test_prep_template.py +++ b/qiita_db/metadata_template/test/test_prep_template.py @@ -1905,6 +1905,15 @@ def test_name_setter(self): pt.name = 'Prep information 1' self.assertEqual(pt.name, 'Prep information 1') + def test_current_human_filtering(self): + pt = qdb.metadata_template.prep_template.PrepTemplate(1) + # by default it should be false + self.assertFalse(pt.current_human_filtering) + pt.current_human_filtering = True + self.assertTrue(pt.current_human_filtering) + pt.current_human_filtering = False + self.assertFalse(pt.current_human_filtering) + EXP_PREP_TEMPLATE = ( 'sample_name\tbarcode\tcenter_name\tcenter_project_name\t' diff --git a/qiita_db/support_files/patches/92.sql b/qiita_db/support_files/patches/92.sql index a4904adb4..0c2ad5034 100644 --- a/qiita_db/support_files/patches/92.sql +++ b/qiita_db/support_files/patches/92.sql @@ -18,3 +18,7 @@ ALTER TABLE qiita.slurm_resource_allocations ADD CONSTRAINT fk_slurm_resource_allocations FOREIGN KEY ( processing_job_id ) REFERENCES qiita.processing_job ( processing_job_id ); + +-- Apr 21, 2024 +-- Adding a new column: current_human_filtering to qiita.prep_template +ALTER TABLE qiita.prep_template ADD current_human_filtering boolean DEFAULT False; diff --git a/qiita_db/support_files/qiita-db.dbs b/qiita_db/support_files/qiita-db.dbs index c156f4749..7e9a970ca 100644 --- a/qiita_db/support_files/qiita-db.dbs +++ b/qiita_db/support_files/qiita-db.dbs @@ -1001,6 +1001,9 @@ + + + @@ -2148,10 +2151,10 @@ $function$ - \ No newline at end of file + diff --git a/qiita_db/support_files/qiita-db.html b/qiita_db/support_files/qiita-db.html index 252d8e05c..942c14ae8 100644 --- a/qiita_db/support_files/qiita-db.html +++ b/qiita_db/support_files/qiita-db.html @@ -65,9 +65,9 @@ + + +{%end%} +{%block content%} +Listing all users that are not yet validated but registered more than 30 days ago. +
+
+ | |
+
+ + + + {% for head in headers %} + + {% end %} + + + + +
Select{{head}}
+
+
+
+
+
+ + +
+
+
+{% end %}