Merge branch 'dev' of github.com:biocore/qiita into dev

.github/workflows/qiita-ci.yml

@@ -154,8 +154,6 @@ jobs:
 
           echo "5. Setting up qiita"
           conda activate qiita
-          # adapt environment_script for private qiita plugins from travis to github actions.
-          sed 's#export PATH="/home/travis/miniconda3/bin:$PATH"; source #source /home/runner/.profile; conda #' -i qiita_db/support_files/patches/54.sql
           qiita-env make --no-load-ontologies
           qiita-test-install
           qiita plugins update
@@ -203,7 +201,34 @@ jobs:
           QIITA_PID=`cat /tmp/supervisord.pid`
           kill $QIITA_PID
           sleep 10
-          if [[ "$COVER_PACKAGE" != *"qiita_db"* ]]; then test_data_studies/commands.sh; all-qiita-cron-job; fi
+          # due to qiita_db tests being more complex and taking longer than
+          # the other tests we will only add some extra tests to the run that is
+          # not testing qiita_db
+          if [[ "$COVER_PACKAGE" != *"qiita_db"* ]]; then
+              # 1. testing that we can add some "dummy" studies to the db via
+              #    CLI
+              test_data_studies/commands.sh;
+              # 2. making sure that all qiita cron jobs complete as expected
+              all-qiita-cron-job;
+              # 3. making sure than a production system has the expected rows
+              #    in all our tables; steps: a. drop test db, b. change $QIITA_CONFIG_FP
+              #    c. create new production system, c. count rows in the db.
+              qiita-env drop;
+              cp $QIITA_CONFIG_FP ${QIITA_CONFIG_FP}.bk
+              sed 's/TEST_ENVIRONMENT = TRUE/TEST_ENVIRONMENT = FALSE/g' ${QIITA_CONFIG_FP}.bk > $QIITA_CONFIG_FP;
+              qiita-env make --no-load-ontologies;
+
+              export PGPASSWORD=postgres
+              pgport=${{ job.services.postgres.ports[5432] }}
+              row_counts=`psql -h localhost -U postgres -d qiita_test -p $pgport -c "SELECT SUM(c.reltuples) FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE n.nspname = 'qiita' AND c.relkind = 'r' AND n.nspname NOT IN ('information_schema', 'pg_catalog');"`
+              if [[ `echo $row_counts` != *" 0 "* ]]; then
+                 echo "***********";
+                 echo "The number of rows in a production system is not what's expected:";
+                 echo $row_counts;
+                 echo "***********";
+                 exit 1
+              fi
+          fi
 
       - name: Submit coveralls
         uses: AndreMiras/coveralls-python-action@develop

CONTRIBUTING.md

@@ -94,6 +94,17 @@ After the initial production release of Qiita, changes to the database schema wi
 2. We keep fully patched versions of the DBS and HTML files in the repository
 3. We keep a patch file for each patch as required in the `qiita_db/support_files/patches` directory. Note that **the patches will be applied in order based on the natural sort order of their filename** (e.g., `2.sql` will be applied before `10.sql`, and `10.sql` will be applied before `a.sql`)
 
+### Patch 91.sql
+
+In May 2024 we decided to:
+* Merge all patches into the main database schema, this means that there are no patches younger than 92.sql.
+* Added a new folder `patches/test_db_sql/` where we can store sql files that will only be applied for the test environment.
+* Added a test to the GitHub actions to test that the production database has an expected number of rows.
+
+Note that these changes mean:
+1. 92.sql is the current first sql file to patch the database.
+2. If you need to make changes (like INSERTS) _only_ to the tests database you need to add a patch to `patches/test_db_sql/`.
+
 ### Developer Workflow
 
 1. Load the fully patched DBS file (e.g., `qiita-db.dbs`) in [DBSchema](http://www.dbschema.com/)

qiita_db/artifact.py

@@ -359,7 +359,7 @@ def create(cls, filepaths, artifact_type, name=None, prep_template=None,
         # There are three different ways of creating an Artifact, but all of
         # them execute a set of common operations. Declare functions to avoid
         # code duplication. These functions should not be used outside of the
-        # create function, hence declaring them here
+        # CREATE OR REPLACE FUNCTION, hence declaring them here
         def _common_creation_steps(atype, cmd_id, data_type, cmd_parameters):
             gen_timestamp = datetime.now()
             visibility_id = qdb.util.convert_to_id("sandbox", "visibility")

qiita_db/environment_manager.py

@@ -200,7 +200,6 @@ def make_environment(load_ontologies, download_reference, add_demo_user):
             with open(SETTINGS_FP, newline=None) as f:
                 qdb.sql_connection.TRN.add(f.read())
             qdb.sql_connection.TRN.execute()
-
             # Insert the settings values to the database
             sql = """INSERT INTO settings
                      (test, base_data_dir, base_work_dir)
@@ -211,7 +210,6 @@ def make_environment(load_ontologies, download_reference, add_demo_user):
                       qiita_config.working_dir])
             qdb.sql_connection.TRN.execute()
             create_layout(test=test, verbose=verbose)
-
         patch(verbose=verbose, test=test)
 
         if load_ontologies:
@@ -274,7 +272,16 @@ def drop_environment(ask_for_confirmation):
     # Connect to the postgres server
     with qdb.sql_connection.TRN:
         qdb.sql_connection.TRN.add("SELECT test FROM settings")
-        is_test_environment = qdb.sql_connection.TRN.execute_fetchflatten()[0]
+        try:
+            is_test_environment = \
+                qdb.sql_connection.TRN.execute_fetchflatten()[0]
+        except ValueError as e:
+            # if settings doesn't exist then is fine to treat this as a test
+            # environment and clean up
+            if 'UNDEFINED_TABLE. MSG: relation "settings"' in str(e):
+                is_test_environment = True
+            else:
+                raise
     qdb.sql_connection.TRN.close()
 
     if is_test_environment:
@@ -369,15 +376,12 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
     Pulls the current patch from the settings table and applies all subsequent
     patches found in the patches directory.
     """
-    # we are going to open and close 2 main transactions; this is a required
-    # change since patch 68.sql where we transition to jsonb for all info
-    # files. The 2 main transitions are: (1) get the current settings,
-    # (2) each patch in their independent transaction
     with qdb.sql_connection.TRN:
         qdb.sql_connection.TRN.add("SELECT current_patch FROM settings")
         current_patch = qdb.sql_connection.TRN.execute_fetchlast()
         current_sql_patch_fp = join(patches_dir, current_patch)
         corresponding_py_patch = partial(join, patches_dir, 'python_patches')
+        corresponding_test_sql = partial(join, patches_dir, 'test_db_sql')
 
         sql_glob = join(patches_dir, '*.sql')
         sql_patch_files = natsorted(glob(sql_glob))
@@ -389,21 +393,17 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
         else:
             next_patch_index = sql_patch_files.index(current_sql_patch_fp) + 1
 
-    patch_update_sql = "UPDATE settings SET current_patch = %s"
+    if test:
+        with qdb.sql_connection.TRN:
+            _populate_test_db()
 
+    patch_update_sql = "UPDATE settings SET current_patch = %s"
     for sql_patch_fp in sql_patch_files[next_patch_index:]:
         sql_patch_filename = basename(sql_patch_fp)
 
-        py_patch_fp = corresponding_py_patch(
-            splitext(basename(sql_patch_fp))[0] + '.py')
-        py_patch_filename = basename(py_patch_fp)
-
-        # patch 43.sql is when we started testing patches, then in patch
-        # 68.sql is when we transitioned to jsonb for the info files; let's do
-        # this in its own transition
-        if sql_patch_filename == '68.sql' and test:
-            with qdb.sql_connection.TRN:
-                _populate_test_db()
+        patch_prefix = splitext(basename(sql_patch_fp))[0]
+        py_patch_fp = corresponding_py_patch(f'{patch_prefix}.py')
+        test_sql_fp = corresponding_test_sql(f'{patch_prefix}.sql')
 
         with qdb.sql_connection.TRN:
             with open(sql_patch_fp, newline=None) as patch_file:
@@ -413,12 +413,19 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
                 qdb.sql_connection.TRN.add(
                     patch_update_sql, [sql_patch_filename])
 
+            if test and exists(test_sql_fp):
+                if verbose:
+                    print('\t\tApplying test SQL %s...'
+                          % basename(test_sql_fp))
+                with open(test_sql_fp) as test_sql:
+                    qdb.sql_connection.TRN.add(test_sql.read())
+
             qdb.sql_connection.TRN.execute()
 
             if exists(py_patch_fp):
                 if verbose:
                     print('\t\tApplying python patch %s...'
-                          % py_patch_filename)
+                          % basename(py_patch_fp))
                 with open(py_patch_fp) as py_patch:
                     exec(py_patch.read(), globals())
 
@@ -427,7 +434,5 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
         # for the test Study (1) so a lot of the tests actually expect this.
         # Now, trying to regenerate directly in the populate_test_db might
         # require too many dev hours so the easiest is just do it here
-        # UPDATE 01/25/2021: moving to 81.sql as we added timestamps to
-        #                    prep info files
-        if test and sql_patch_filename == '81.sql':
-            qdb.study.Study(1).sample_template.generate_files()
+    if test:
+        qdb.study.Study(1).sample_template.generate_files()