diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 57180a1..229a8c2 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -9,24 +9,22 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - # Cache docker layers for faster build - - uses: satackey/action-docker-layer-caching@v0.0.8 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true + - uses: actions/setup-python@v5 + name: Setup python + with: + python-version: '3.11' + cache: 'pip' - - name: Build - run: docker build -t harness-testing -f Dockerfile.test . + - name: Install requirements + run: pip install -r requirements.txt + + - run: pip install -r requirements-runners.txt + - run: pip install -r requirements-test.txt - name: Run tests and get output - run: | - echo 'TEST_OUTPUT<> $GITHUB_ENV - echo "$(docker run harness-testing)" >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - - name: Exit if there are any test failures - run: '[[ $TEST_OUTPUT != *FAILED* ]]' + run: pytest check-format: name: Check that code matches Black formatter diff --git a/requirements-test.txt b/requirements-test.txt index 29d5702..b2bba2d 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ pytest==7.4.2 pytest-asyncio==0.23.3 +pytest-httpx==0.30.0 pytest-mock==3.11.1 diff --git a/tests/example_tests.py b/tests/helpers/example_tests.py similarity index 93% rename from tests/example_tests.py rename to tests/helpers/example_tests.py index 16e6578..672fcd1 100644 --- a/tests/example_tests.py +++ b/tests/helpers/example_tests.py @@ -34,7 +34,7 @@ "tags": [], "input_id": "MONDO:0010794", "input_name": "NARP Syndrome", - "input_category": None, + "input_category": "biolink:Disease", "predicate_id": "biolink:treats", "predicate_name": "treats", "output_id": "DRUGBANK:DB00313", @@ -48,7 +48,7 @@ "in_v1": None, "well_known": False, "test_reference": None, - "runner_settings": ["inferred"], + "test_runner_settings": ["inferred"], "test_metadata": { "id": "1", "name": None, @@ -67,7 +67,7 @@ "tags": [], "input_id": "MONDO:0010794", "input_name": "NARP Syndrome", - "input_category": None, + "input_category": "biolink:Disease", "predicate_id": "biolink:treats", "predicate_name": "treats", "output_id": "MESH:D001463", @@ -81,7 +81,7 @@ "in_v1": None, "well_known": False, "test_reference": None, - "runner_settings": ["inferred"], + "test_runner_settings": ["inferred"], "test_metadata": { "id": "1", "name": None, @@ -102,7 +102,7 @@ "test_case_predicate_name": "treats", "test_case_predicate_id": "biolink:treats", "test_case_input_id": "MONDO:0010794", - "test_case_runner_settings": ["inferred"], + "test_runner_settings": ["inferred"], } }, } diff --git a/tests/helpers/logger.py b/tests/helpers/logger.py new file mode 100644 index 0000000..5d9b893 --- /dev/null +++ b/tests/helpers/logger.py @@ -0,0 +1,61 @@ +"""Logging setup.""" + +import logging + + +class ColoredFormatter(logging.Formatter): + """Colored formatter.""" + + prefix = "[%(asctime)s: %(levelname)s/%(name)s]:" + default = f"{prefix} %(message)s" + error_fmt = f"\x1b[31m{prefix}\x1b[0m %(message)s" + warning_fmt = f"\x1b[33m{prefix}\x1b[0m %(message)s" + info_fmt = f"\x1b[32m{prefix}\x1b[0m %(message)s" + debug_fmt = f"\x1b[34m{prefix}\x1b[0m %(message)s" + + def __init__(self, fmt=default): + """Initialize.""" + logging.Formatter.__init__(self, fmt) + + def format(self, record): + """Format record.""" + format_orig = self._style._fmt + if record.levelno == logging.DEBUG: + self._style._fmt = ColoredFormatter.debug_fmt + elif record.levelno == logging.INFO: + self._style._fmt = ColoredFormatter.info_fmt + elif record.levelno == logging.WARNING: + self._style._fmt = ColoredFormatter.warning_fmt + elif record.levelno == logging.ERROR: + self._style._fmt = ColoredFormatter.error_fmt + result = logging.Formatter.format(self, record) + self._style._fmt = format_orig + return result + + +def setup_logger(): + """Set up Test Harness logger.""" + logger = logging.getLogger("harness") + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler() + handler.setLevel(logging.DEBUG) + handler.setFormatter(ColoredFormatter()) + logger.addHandler(handler) + + return logger + + +def assert_no_level(logger, allowed_level, exceptions=0): + """ + Check that the logger has no records greater than + the allowed level. + + Also has a parameter to specify the number of exceptions + to the rule (number of records that will be ignored). + """ + total = 0 + for record in logger.records: + if record.levelno >= allowed_level: + total += 1 + if total > exceptions: + raise Exception(f"Invalid Log Record: {record}") diff --git a/tests/helpers/mock_responses.py b/tests/helpers/mock_responses.py new file mode 100644 index 0000000..24e77a6 --- /dev/null +++ b/tests/helpers/mock_responses.py @@ -0,0 +1,80 @@ +"""Mock Test Responses.""" + +kp_response = { + "message": { + "query_graph": { + "nodes": { + "n0": {"ids": ["MESH:D008687"]}, + "n1": {"categories": ["biolink:Disease"]}, + }, + "edges": { + "n0n1": { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:treats"], + } + }, + }, + "knowledge_graph": { + "nodes": { + "MESH:D008687": { + "categories": ["biolink:SmallMolecule"], + "name": "Metformin", + "attributes": [], + }, + "MONDO:0005148": { + "categories": [ + "biolink:Disease", + ], + "name": "type 2 diabetes mellitus", + "attributes": [], + }, + }, + "edges": { + "n0n1": { + "subject": "MESH:D008687", + "object": "MONDO:0005148", + "predicate": "biolink:treats", + "sources": [ + { + "resource_id": "infores:kp0", + "resource_role": "primary_knowledge_source", + } + ], + "attributes": [], + }, + }, + }, + "results": [ + { + "node_bindings": { + "n0": [ + { + "id": "MESH:D008687", + "attributes": [], + }, + ], + "n1": [ + { + "id": "MONDO:0005148", + "attributes": [], + }, + ], + }, + "analyses": [ + { + "resource_id": "kp0", + "edge_bindings": { + "n0n1": [ + { + "id": "n0n1", + "attributes": [], + }, + ], + }, + } + ], + }, + ], + }, +} \ No newline at end of file diff --git a/tests/helpers/mocks.py b/tests/helpers/mocks.py new file mode 100644 index 0000000..04592da --- /dev/null +++ b/tests/helpers/mocks.py @@ -0,0 +1,68 @@ +from test_harness.reporter import Reporter +from test_harness.slacker import Slacker +from test_harness.runner.query_runner import QueryRunner + +class MockReporter(Reporter): + def __init__(self, base_url=None, refresh_token=None, logger=None): + super().__init__() + self.base_path = base_url + self.test_run_id = 1 + pass + + async def get_auth(self): + pass + + async def create_test_run(self, test_env, suite_name): + return 1 + + async def create_test(self, test, asset): + return 2 + + async def upload_labels(self, test_id, labels): + pass + + async def upload_logs(self, test_id, logs): + pass + + async def upload_artifact_references(self, test_id, artifact_references): + pass + + async def upload_screenshots(self, test_id, screenshot): + pass + + async def upload_log(self, test_id, message): + pass + + async def finish_test(self, test_id, result): + return result + + async def finish_test_run(self): + pass + + +class MockSlacker(Slacker): + def __init__(self): + pass + + async def post_notification(self, messages=[]): + print(f"posting messages: {messages}") + pass + + async def upload_test_results_file(self, filename, extension, results): + pass + + +class MockQueryRunner(QueryRunner): + async def retrieve_registry(self, trapi_version: str): + self.registry = { + "staging": { + "ars": [ + { + "_id": "testing", + "title": "Tester", + "infores": "infores:tester", + "url": "http://tester" + } + ], + }, + } diff --git a/tests/mocker.py b/tests/mocker.py deleted file mode 100644 index 226b1e1..0000000 --- a/tests/mocker.py +++ /dev/null @@ -1,44 +0,0 @@ -class MockReporter: - def __init__(self, base_url=None, refresh_token=None, logger=None): - self.base_path = base_url - self.test_run_id = 1 - pass - - async def get_auth(self): - pass - - async def create_test_run(self, test): - return 1 - - async def create_test(self, test, asset): - return 2 - - async def upload_labels(self, test_id, labels): - pass - - async def upload_logs(self, test_id, logs): - pass - - async def upload_artifact_references(self, test_id, artifact_references): - pass - - async def upload_screenshots(self, test_id, screenshot): - pass - - async def upload_log(self, test_id, message): - pass - - async def finish_test(self, test_id, result): - return result - - async def finish_test_run(self): - pass - - -class MockSlacker: - def __init__(self): - pass - - async def post_notification(self, messages): - print(f"posting messages: {messages}") - pass diff --git a/tests/test_main.py b/tests/test_main.py index d8d0ee0..d3e20b5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,9 @@ import pytest from test_harness.main import main -from .example_tests import example_test_cases -from .mocker import ( +from .helpers.example_tests import example_test_cases +from .helpers.mocks import ( MockReporter, MockSlacker, ) @@ -13,17 +13,16 @@ async def test_main(mocker): """Test the main function.""" # This article is awesome: https://nedbatchelder.com/blog/201908/why_your_mock_doesnt_work.html - run_ars_test = mocker.patch("test_harness.run.run_ars_test", return_value="Fail") run_tests = mocker.patch("test_harness.main.run_tests", return_value={}) - mocker.patch("test_harness.slacker.Slacker", return_value=MockSlacker()) + mocker.patch("test_harness.main.Slacker", return_value=MockSlacker()) mocker.patch("test_harness.main.Reporter", return_value=MockReporter()) await main( { "tests": example_test_cases, + "suite": "testing", "save_to_dashboard": False, "json_output": False, "log_level": "ERROR", } ) - # run_ui_test.assert_called_once() run_tests.assert_called_once() diff --git a/tests/test_run.py b/tests/test_run.py index 2f2751e..f9f1057 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,48 +1,50 @@ +"""Test the Harness run file.""" + import pytest +from pytest_httpx import HTTPXMock from test_harness.run import run_tests -from .example_tests import example_test_cases -from .mocker import ( +from .helpers.example_tests import example_test_cases +from .helpers.mocks import ( MockReporter, MockSlacker, + MockQueryRunner, ) +from .helpers.logger import setup_logger +from .helpers.mock_responses import kp_response + +logger = setup_logger() @pytest.mark.asyncio -async def test_run_tests(mocker): +async def test_run_tests(mocker, httpx_mock: HTTPXMock): """Test the run_tests function.""" # This article is awesome: https://nedbatchelder.com/blog/201908/why_your_mock_doesnt_work.html - run_ars_test = mocker.patch( - "test_harness.run.run_ars_test", - return_value={ - "pks": { - "parent_pk": "123abc", - "merged_pk": "456def", - }, - "results": [ - { - "ars": { - "status": "PASSED", - }, - }, - { - "ars": { - "status": "FAILED", - "message": "", - }, - }, - ], - }, + mocker.patch( + "test_harness.run.QueryRunner", + return_value=MockQueryRunner(logger), + ) + httpx_mock.add_response( + url="http://tester/query", json=kp_response, + ) + httpx_mock.add_response( + url="https://nodenorm.ci.transltr.io/get_normalized_nodes", json={ + "MONDO:0010794": None, + "DRUGBANK:DB00313": None, + "MESH:D001463": None, + } ) - run_benchmarks = mocker.patch("test_harness.run.run_benchmarks", return_value={}) - await run_tests( + full_report = await run_tests( reporter=MockReporter( base_url="http://test", ), slacker=MockSlacker(), tests=example_test_cases, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "1.5.0", + } ) - # run_ui_test.assert_called_once() - run_ars_test.assert_called_once() - run_benchmarks.assert_not_called() + assert full_report["SKIPPED"] == 2