diff --git a/test_harness/download.py b/test_harness/download.py index 6d2e85d..8b79c95 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -13,11 +13,11 @@ def download_tests( - suite: Union[str, List[str]], url: Path, logger: logging.Logger + suite: Union[str, List[str]], url: Path, ) -> List[TestCase]: """Download tests from specified location.""" assert Path(url).suffix == ".zip" - logger.info(f"Downloading tests from {url}...") + print(f"Downloading tests from {url}...") # download file from internet with httpx.Client(follow_redirects=True) as client: tests_zip = client.get(url) @@ -34,7 +34,7 @@ def download_tests( suites = suite if type(suite) == list else [suite] test_case_ids = [] - logger.info(f"Reading in {len(tests_paths)} tests...") + print(f"Reading in {len(tests_paths)} tests...") # do the reading of the tests and make a tests list for test_path in tests_paths: @@ -77,5 +77,5 @@ def download_tests( # test.test_case_type = "acceptance" tests = all_tests # tests = list(filter((lambda x: x for x in all_tests for asset in x.test_assets if asset.output_id), all_tests)) - logger.info(f"Passing along {len(tests)} tests") + print(f"Passing along {len(tests)} tests") return tests diff --git a/test_harness/main.py b/test_harness/main.py index 5d4e2cf..54db3c8 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -7,10 +7,10 @@ from .run import run_tests from .download import download_tests -from .logger import get_logger, setup_logger +# from .logger import get_logger, setup_logger from .reporter import Reporter -setup_logger() +# setup_logger() def url_type(arg): @@ -23,35 +23,35 @@ def url_type(arg): async def main(args): """Main Test Harness entrypoint.""" qid = str(uuid4())[:8] - logger = get_logger(qid, args["log_level"]) + # logger = get_logger(qid, args["log_level"]) tests = [] if "tests_url" in args: - tests = download_tests(args["suite"], args["tests_url"], logger) + tests = download_tests(args["suite"], args["tests_url"]) elif "tests" in args: tests = args["tests"] else: - return logger.error( + return print( "Please run this command with `-h` to see the available options." ) if len(tests) < 1: - return logger.info("No tests to run. Exiting.") + return print("No tests to run. Exiting.") # Create test run in the Information Radiator reporter = Reporter(base_url=args.get("reporter_url"), refresh_token=args.get("reporter_access_token")) await reporter.get_auth() await reporter.create_test_run() - report = await run_tests(reporter, tests, logger) + report = await run_tests(reporter, tests) - logger.info("Saving to Testing Dashboard...") + print("Saving to Testing Dashboard...") await reporter.finish_test_run() if args["json_output"]: - logger.info("Saving report as JSON...") + # logger.info("Saving report as JSON...") with open("test_report.json", "w") as f: json.dump(report, f) - return logger.info("All tests have completed!") + return print("All tests have completed!") def cli(): diff --git a/test_harness/run.py b/test_harness/run.py index 84e74dd..5f8d1a9 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -1,7 +1,7 @@ """Run tests through the Test Runners.""" from collections import defaultdict import json -import logging +# import logging from tqdm import tqdm import traceback from typing import Dict, List @@ -14,17 +14,17 @@ from .reporter import Reporter -async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.Logger) -> Dict: +async def run_tests(reporter: Reporter, tests: List[TestCase]) -> Dict: """Send tests through the Test Runners.""" # tests = [TestCase.parse_obj(test) for test in tests] - logger.info(f"Running {len(tests)} tests...") + print(f"Running {len(tests)} tests...") full_report = {} # loop over all tests for test in tqdm(tests): status = "PASSED" # check if acceptance test if not test.get("test_assets") or not test.get("test_case_objective"): - logger.error(f"Test has missing required fields: {test['id']}") + print(f"Test has missing required fields: {test['id']}") continue if test["test_case_objective"] == "AcceptanceTest": # full_report[test["test_case_input_id"]] = {} @@ -39,7 +39,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L # ) # full_report[test.test_assets[0].input_id]["ui"] = ui_result # except Exception as e: - # logger.error(f"UI test failed with {traceback.format_exc()}") + # print(f"UI test failed with {traceback.format_exc()}") # full_report[test.test_assets[0].input_id]["ui"] = {"error": str(e)} assets = test["test_assets"] test_ids = [] @@ -50,7 +50,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L test_id = await reporter.create_test(test, asset) test_ids.append(test_id) except Exception: - logger.error(f"Failed to create test: {test['id']}") + print(f"Failed to create test: {test['id']}") try: test_input = json.dumps({ # "environment": test["test_env"], @@ -65,8 +65,8 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L test_input=test_input )) except Exception as e: - logger.warning(str(e)) - logger.warning(f"Failed to upload logs to test: {test['id']}, {test_id}") + print(str(e)) + print(f"Failed to upload logs to test: {test['id']}, {test_id}") # group all outputs together to make one Translator query output_ids = [asset["output_id"] for asset in assets] @@ -84,7 +84,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L ars_result = await run_ars_test(*test_inputs) except Exception as e: err_msg = f"ARS Test Runner failed with {traceback.format_exc()}" - logger.error(f"[{test['id']}] {err_msg}") + print(f"[{test['id']}] {err_msg}") ars_result = { "pks": {}, # this will effectively act as a list that we access by index down below @@ -103,7 +103,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L await reporter.upload_log(test_id, json.dumps(test_result, indent=4)) await reporter.finish_test(test_id, status) except Exception as e: - logger.error(f"[{test['id']}] failed to upload logs and finished status.") + print(f"[{test['id']}] failed to upload logs and finished status.") # full_report[test["test_case_input_id"]]["ars"] = ars_result elif test["test_case_objective"] == "QuantitativeTest": continue @@ -111,7 +111,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L try: test_id = await reporter.create_test(test, assets) except Exception: - logger.error(f"Failed to create test: {test['id']}") + print(f"Failed to create test: {test['id']}") try: test_inputs = [ assets["id"], @@ -126,20 +126,20 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L await reporter.upload_screenshot(test_id, screenshot) await reporter.finish_test(test_id, "PASSED") except Exception as e: - logger.error(f"Benchmarks failed with {e}: {traceback.format_exc()}") + print(f"Benchmarks failed with {e}: {traceback.format_exc()}") try: await reporter.upload_log(test_id, traceback.format_exc()) except Exception: - logger.error(f"Failed to upload fail logs for test {test_id}: {traceback.format_exc()}") + print(f"Failed to upload fail logs for test {test_id}: {traceback.format_exc()}") await reporter.finish_test(test_id, "FAILED") else: try: test_id = await reporter.create_test(test, test) - logger.warning(f"Unsupported test type: {test['id']}") + print(f"Unsupported test type: {test['id']}") await reporter.upload_log(test_id, f"Unsupported test type in test: {test['id']}") status = "FAILED" await reporter.finish_test(test_id, status) except Exception: - logger.error(f"Failed to report errors with: {test['id']}") + print(f"Failed to report errors with: {test['id']}") return full_report