Feature 318 add db for testing (dtcenter#326)

* 318: add mariadb gh actions

* 318: Update tests to use actual db

* Update conftest with fixtures to access and update
test database mv_test

* Add fixure to clean database (drop/recreate)

* Add fixture to access test data in METreformat

* Add example test of met_db_load.py

* 318: remove METdbLoad tests from sonarqube gh workflow.

* 318: fix test, pass tmpdir as list.

.github/workflows/unit_tests.yaml

@@ -27,8 +27,17 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
+
+    services:
+      mariadb:
+        image: mariadb:latest
+        env:
+          MARIADB_ROOT_PASSWORD: root_password
+        ports:
+          - 3306:3306
+        options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
+
     strategy:
       fail-fast: false
       matrix:
@@ -67,6 +76,7 @@ jobs:
 
       - name: Test with pytest
         run: |
+          coverage run --append -m pytest METdbLoad/test
           coverage run --append -m pytest METreformat
           coverage run --append -m pytest METreadnc
           coverage report -m

METdbLoad/conftest.py

@@ -1,12 +1,113 @@
 import pytest
 import sys
 import os
-
+import pymysql
+import logging
 from pathlib import Path
+from unittest.mock import patch
+
+from METdataio.METdbLoad.ush.read_data_files import ReadDataFiles
+from METdataio.METdbLoad.ush.run_sql import RunSql
+
 
 # add METdataio directory to path so packages can be found
-top_dir = str(Path(__file__).parents[1])
-sys.path.insert(0, os.path.abspath(top_dir))
+TOP_DIR = str(Path(__file__).parents[1])
+sys.path.insert(0, os.path.abspath(TOP_DIR))
+
+def parse_sql(filename):
+    """Parse a .sql file and return a list of SQL statements"""
+    data = open(filename, 'r').readlines()
+    stmts = []
+    DELIMITER = ';'
+    stmt = ''
+
+    for line in data:
+        if not line.strip():
+            continue
+
+        if line.startswith('--'):
+            continue
+
+        if (DELIMITER not in line):
+            stmt += line
+            continue
+
+        if stmt:
+            stmt += line
+            stmts.append(stmt.strip())
+            stmt = ''
+        else:
+            stmts.append(line.strip())
+    return stmts
+
+
+def maria_conn():
+    """A databaseless connection to mariaDB server.
+    This will work even if no database has been created.
+    """
+    try:
+        conn = pymysql.connect(
+            host='localhost',
+            port=3306,
+            user='root',
+            password='root_password',
+        )
+
+    except Exception as e:
+        # Test run will fail if db is not found.
+        # TODO: If we want to run tests that don't require a db when db is missing
+        # we could put pytest.skip here instead of raising the exception.
+        raise e
+
+    return conn
+
+
+@pytest.fixture
+def emptyDB():
+    """Drop and recreate the database.
+    Including this fixture in a test will DELETE all data from mv_test.
+    """
+
+    conn = maria_conn()
+    with conn.cursor() as cur:
+        cur.execute("DROP DATABASE IF EXISTS mv_test;")
+        cur.execute("CREATE DATABASE mv_test;")
+    conn.commit()
+    conn.close()
+
+    db_conn = pymysql.connect(
+            host='localhost',
+            port=3306,
+            user='root',
+            password='root_password',
+            database='mv_test',
+            autocommit=True,
+        )
+
+    sql_statements = parse_sql(Path(TOP_DIR) / 'METdbLoad/sql/mv_mysql.sql')
+
+    with db_conn.cursor() as cur:
+        for stm in sql_statements:
+            cur.execute(stm)
+
+    db_conn.close()
+
+
+@pytest.fixture
+def testRunSql():
+    """Return an instance of RunSql with a connection.
+    """
+    connection = {
+            'db_host': 'localhost',
+            'db_port': 3306,
+            'db_user': 'root',
+            'db_password': 'root_password',
+            'db_database': 'mv_test',
+    }
+
+    testRunSql = RunSql()
+    testRunSql.sql_on(connection)
+    return testRunSql
 
 
 # This is a sample of data copied from test file point_stat_DUP_SINGLE_120000L_20120409_120000V.stat
@@ -25,17 +126,17 @@
 
 def _populate_xml_load_spec(met_data_dir,
                             met_tool="point_stat",
-                            host="192.168.0.42"):
+                            host="localhost"):
     """Return the xml load specification with substitute values.
     """
     #TODO: determine if other tags require substitution as well
     return f"""<load_spec>
     <connection>
         <management_system>mysql</management_system>
         <host>{host}:3306</host>
-        <database>mv_load_test</database>
-        <user>user</user>
-        <password>user_pwd</password>
+        <database>mv_test</database>
+        <user>root</user>
+        <password>root_password</password>
     </connection>
 
     <folder_tmpl>{met_data_dir}</folder_tmpl>
@@ -47,10 +148,11 @@ def _populate_xml_load_spec(met_data_dir,
     <drop_indexes>false</drop_indexes>
     <apply_indexes>false</apply_indexes>
     <load_stat>true</load_stat>
-    <load_mode>false</load_mode>
-    <load_mtd>false</load_mtd>
+    <load_mode>true</load_mode>
+    <load_mtd>true</load_mtd>
     <load_mpr>true</load_mpr>
     <load_orank>true</load_orank>
+    <force_dup_file>true</force_dup_file>
     <load_val>
         <field name="met_tool">
         <val>{met_tool}</val>
@@ -61,25 +163,20 @@ def _populate_xml_load_spec(met_data_dir,
     </load_spec>"""
 
 
+# TODO: give access to the other test data
 @pytest.fixture
-def stat_file_dir(tmp_path):
+def point_stat_file_dir(tmp_path):
     """Write test stat file and return parent dir."""
-    stat_files_dir = tmp_path / "stat_files"
-    stat_files_dir.mkdir()
-
-    stat_file = stat_files_dir / "point_stat.stat"
-    with open(stat_file, "w") as text_file:
-        text_file.write(POINT_STAT_DATA)
-    return stat_files_dir
+    return str(Path(TOP_DIR) / 'METreformat/test/data/point_stat' )
 
 
 #TODO: see if we can restrict the scope of this fixture.
 @pytest.fixture
-def get_xml_test_file(tmp_path, stat_file_dir):
+def get_xml_test_file(tmp_path, point_stat_file_dir):
     """Write test_load_specification.xml and return path"""
     xml_path = tmp_path / "test_load_specification.xml"
     with open(xml_path, "w") as text_file:
-        text_file.write(_populate_xml_load_spec(stat_file_dir))
+        text_file.write(_populate_xml_load_spec(point_stat_file_dir))
     return xml_path
 
 

METdbLoad/test/test_load_specification.xml

@@ -1,30 +1,30 @@
 <load_spec>
-  <connection>
-    <management_system>mysql</management_system>
-    <host>localhost:3306</host>
-    <database>mv_load_test</database>
-    <user>user</user>
-    <password>user_pwd</password>
-  </connection>
+    <connection>
+        <management_system>mysql</management_system>
+        <host>localhost:3306</host>
+        <database>mv_test</database>
+        <user>root</user>
+        <password>root_password</password>
+    </connection>
 
-  <folder_tmpl>/path-to/test_data/load_data/load/met_data/point_stat/2011070812/metprd</folder_tmpl>
-  <verbose>true</verbose>
-  <insert_size>1</insert_size>
-  <stat_header_db_check>true</stat_header_db_check>
-  <mode_header_db_check>false</mode_header_db_check>
-  <mtd_header_db_check>false</mtd_header_db_check>
-  <drop_indexes>false</drop_indexes>
-  <apply_indexes>false</apply_indexes>
-  <load_stat>true</load_stat>
-  <load_mode>true</load_mode>
-  <load_mtd>true</load_mtd>
-  <load_mpr>true</load_mpr>
-  <load_orank>true</load_orank>
-  <load_val>
-	<field name="met_tool">
-	  <val>point_stat</val>
-	</field>
-  </load_val>
-  <group>Testing</group>
-  <description>testing DB load</description>
-</load_spec>
+    <folder_tmpl>/METdataio/METreformat/test/data/point_stat</folder_tmpl>
+    <verbose>true</verbose>
+    <insert_size>1</insert_size>
+    <stat_header_db_check>true</stat_header_db_check>
+    <mode_header_db_check>false</mode_header_db_check>
+    <mtd_header_db_check>false</mtd_header_db_check>
+    <drop_indexes>false</drop_indexes>
+    <apply_indexes>false</apply_indexes>
+    <load_stat>true</load_stat>
+    <load_mode>true</load_mode>
+    <load_mtd>true</load_mtd>
+    <load_mpr>true</load_mpr>
+    <load_orank>true</load_orank>
+    <load_val>
+        <field name="met_tool">
+        <val>point_stat</val>
+        </field>
+    </load_val>
+    <group>Testing</group>
+    <description>testing DB load</description>
+    </load_spec>

METdbLoad/test/test_met_db_load.py

@@ -0,0 +1,27 @@
+import argparse
+from METdbLoad.conftest import TOP_DIR
+from METdbLoad.ush.met_db_load import main as load_main
+from METdbLoad.ush.run_sql import RunSql
+
+def test_met_db_load(emptyDB, get_xml_test_file, testRunSql, tmp_path):
+
+    # TODO: parameterize this test data
+    test_data = {
+        "xmlfile": str(get_xml_test_file),
+        "index": True,
+        "tmpdir": [str(tmp_path)],
+    }
+    test_args = argparse.Namespace()
+    for k,v in test_data.items():
+        setattr(test_args, k, v)
+
+    load_main(test_args)
+
+    # Check the correct number of rows written
+    testRunSql.cur.execute("SELECT * FROM line_data_cts")
+    cts_data = testRunSql.cur.fetchall()
+
+    assert len(cts_data) == 24
+
+    #TODO: check all the other metrics and some values.
+

METdbLoad/test/test_read_data_files.py

@@ -19,8 +19,8 @@ def test_counts(get_xml_loadfile):
                         XML_LOADFILE.line_types)
 
     # number of files
-    assert len(XML_LOADFILE.load_files) == 1
+    assert len(XML_LOADFILE.load_files) == 2
     # number of lines of data
-    assert FILE_DATA.stat_data.shape[0] == 6
+    assert FILE_DATA.stat_data.shape[0] == 94
     # number of line types
-    assert FILE_DATA.stat_data.line_type.unique().size == 5
+    assert FILE_DATA.stat_data.line_type.unique().size == 7

METdbLoad/test/test_xml.py

@@ -5,8 +5,8 @@ def test_loadflags(get_xml_loadfile):
     """Read various flags from XML file."""
     XML_LOADFILE = get_xml_loadfile()
     assert XML_LOADFILE.flags['load_stat']
-    assert not XML_LOADFILE.flags['load_mode']
-    assert not XML_LOADFILE.flags['load_mtd']
+    assert XML_LOADFILE.flags['load_mode']
+    assert XML_LOADFILE.flags['load_mtd']
     assert XML_LOADFILE.flags['load_mpr']
     assert XML_LOADFILE.flags['load_orank']
     assert XML_LOADFILE.flags['verbose']
@@ -15,7 +15,7 @@ def test_loadflags(get_xml_loadfile):
     assert XML_LOADFILE.flags['stat_header_db_check']
     assert not XML_LOADFILE.flags['mode_header_db_check']
     assert not XML_LOADFILE.flags['mtd_header_db_check']
-    assert not XML_LOADFILE.flags['force_dup_file']
+    assert XML_LOADFILE.flags['force_dup_file']
     assert XML_LOADFILE.flags['load_xml']
 
 def test_loadgroup(get_xml_loadfile):
@@ -27,10 +27,10 @@ def test_loadgroup(get_xml_loadfile):
 def test_connection(get_xml_loadfile):
     """Read connection tags from XML file."""
     XML_LOADFILE = get_xml_loadfile()
-    assert XML_LOADFILE.connection['db_host'] == "192.168.0.42"
+    assert XML_LOADFILE.connection['db_host'] == "localhost"
     assert XML_LOADFILE.connection['db_port'] == 3306
-    assert XML_LOADFILE.connection['db_database'] == "mv_load_test"
-    assert XML_LOADFILE.connection['db_user'] == "user"
+    assert XML_LOADFILE.connection['db_database'] == "mv_test"
+    assert XML_LOADFILE.connection['db_user'] == "root"
     assert XML_LOADFILE.connection['db_management_system'] == "mysql"
 
 def test_insertsize(get_xml_loadfile):