Remove logger

test_harness/download.py

@@ -13,11 +13,11 @@
 
 
 def download_tests(
-    suite: Union[str, List[str]], url: Path, logger: logging.Logger
+    suite: Union[str, List[str]], url: Path,
 ) -> List[TestCase]:
     """Download tests from specified location."""
     assert Path(url).suffix == ".zip"
-    logger.info(f"Downloading tests from {url}...")
+    print(f"Downloading tests from {url}...")
     # download file from internet
     with httpx.Client(follow_redirects=True) as client:
         tests_zip = client.get(url)
@@ -34,7 +34,7 @@ def download_tests(
         suites = suite if type(suite) == list else [suite]
         test_case_ids = []
 
-        logger.info(f"Reading in {len(tests_paths)} tests...")
+        print(f"Reading in {len(tests_paths)} tests...")
 
         # do the reading of the tests and make a tests list
         for test_path in tests_paths:
@@ -77,5 +77,5 @@ def download_tests(
     #     test.test_case_type = "acceptance"
     tests = all_tests
     # tests = list(filter((lambda x: x for x in all_tests for asset in x.test_assets if asset.output_id), all_tests))
-    logger.info(f"Passing along {len(tests)} tests")
+    print(f"Passing along {len(tests)} tests")
     return tests

test_harness/main.py

@@ -7,10 +7,10 @@
 
 from .run import run_tests
 from .download import download_tests
-from .logger import get_logger, setup_logger
+# from .logger import get_logger, setup_logger
 from .reporter import Reporter
 
-setup_logger()
+# setup_logger()
 
 
 def url_type(arg):
@@ -23,35 +23,35 @@ def url_type(arg):
 async def main(args):
     """Main Test Harness entrypoint."""
     qid = str(uuid4())[:8]
-    logger = get_logger(qid, args["log_level"])
+    # logger = get_logger(qid, args["log_level"])
     tests = []
     if "tests_url" in args:
-        tests = download_tests(args["suite"], args["tests_url"], logger)
+        tests = download_tests(args["suite"], args["tests_url"])
     elif "tests" in args:
         tests = args["tests"]
     else:
-        return logger.error(
+        return print(
             "Please run this command with `-h` to see the available options."
         )
 
     if len(tests) < 1:
-        return logger.info("No tests to run. Exiting.")
+        return print("No tests to run. Exiting.")
 
     # Create test run in the Information Radiator
     reporter = Reporter(base_url=args.get("reporter_url"), refresh_token=args.get("reporter_access_token"))
     await reporter.get_auth()
     await reporter.create_test_run()
-    report = await run_tests(reporter, tests, logger)
+    report = await run_tests(reporter, tests)
 
-    logger.info("Saving to Testing Dashboard...")
+    print("Saving to Testing Dashboard...")
     await reporter.finish_test_run()
 
     if args["json_output"]:
-        logger.info("Saving report as JSON...")
+        # logger.info("Saving report as JSON...")
         with open("test_report.json", "w") as f:
             json.dump(report, f)
 
-    return logger.info("All tests have completed!")
+    return print("All tests have completed!")
 
 
 def cli():

test_harness/run.py

@@ -1,7 +1,7 @@
 """Run tests through the Test Runners."""
 from collections import defaultdict
 import json
-import logging
+# import logging
 from tqdm import tqdm
 import traceback
 from typing import Dict, List
@@ -14,17 +14,17 @@
 from .reporter import Reporter
 
 
-async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.Logger) -> Dict:
+async def run_tests(reporter: Reporter, tests: List[TestCase]) -> Dict:
     """Send tests through the Test Runners."""
     # tests = [TestCase.parse_obj(test) for test in tests]
-    logger.info(f"Running {len(tests)} tests...")
+    print(f"Running {len(tests)} tests...")
     full_report = {}
     # loop over all tests
     for test in tqdm(tests):
         status = "PASSED"
         # check if acceptance test
         if not test.get("test_assets") or not test.get("test_case_objective"):
-            logger.error(f"Test has missing required fields: {test['id']}")
+            print(f"Test has missing required fields: {test['id']}")
             continue
         if test["test_case_objective"] == "AcceptanceTest":
             # full_report[test["test_case_input_id"]] = {}
@@ -39,7 +39,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L
             #     )
             #     full_report[test.test_assets[0].input_id]["ui"] = ui_result
             # except Exception as e:
-            #     logger.error(f"UI test failed with {traceback.format_exc()}")
+            #     print(f"UI test failed with {traceback.format_exc()}")
             #     full_report[test.test_assets[0].input_id]["ui"] = {"error": str(e)}
             assets = test["test_assets"]
             test_ids = []
@@ -50,7 +50,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L
                     test_id = await reporter.create_test(test, asset)
                     test_ids.append(test_id)
                 except Exception:
-                    logger.error(f"Failed to create test: {test['id']}")
+                    print(f"Failed to create test: {test['id']}")
                 try:
                     test_input = json.dumps({
                         # "environment": test["test_env"],
@@ -65,8 +65,8 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L
                         test_input=test_input
                     ))
                 except Exception as e:
-                    logger.warning(str(e))
-                    logger.warning(f"Failed to upload logs to test: {test['id']}, {test_id}")
+                    print(str(e))
+                    print(f"Failed to upload logs to test: {test['id']}, {test_id}")
 
             # group all outputs together to make one Translator query
             output_ids = [asset["output_id"] for asset in assets]
@@ -84,7 +84,7 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L
                 ars_result = await run_ars_test(*test_inputs)
             except Exception as e:
                 err_msg = f"ARS Test Runner failed with {traceback.format_exc()}"
-                logger.error(f"[{test['id']}] {err_msg}")
+                print(f"[{test['id']}] {err_msg}")
                 ars_result = {
                     "pks": {},
                     # this will effectively act as a list that we access by index down below
@@ -103,15 +103,15 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L
                     await reporter.upload_log(test_id, json.dumps(test_result, indent=4))
                     await reporter.finish_test(test_id, status)
                 except Exception as e:
-                    logger.error(f"[{test['id']}] failed to upload logs and finished status.")
+                    print(f"[{test['id']}] failed to upload logs and finished status.")
             # full_report[test["test_case_input_id"]]["ars"] = ars_result
         elif test["test_case_objective"] == "QuantitativeTest":
             continue
             assets = test["test_assets"][0]
             try:
                 test_id = await reporter.create_test(test, assets)
             except Exception:
-                logger.error(f"Failed to create test: {test['id']}")
+                print(f"Failed to create test: {test['id']}")
             try:
                 test_inputs = [
                     assets["id"],
@@ -126,20 +126,20 @@ async def run_tests(reporter: Reporter, tests: List[TestCase], logger: logging.L
                     await reporter.upload_screenshot(test_id, screenshot)
                 await reporter.finish_test(test_id, "PASSED")
             except Exception as e:
-                logger.error(f"Benchmarks failed with {e}: {traceback.format_exc()}")
+                print(f"Benchmarks failed with {e}: {traceback.format_exc()}")
                 try:
                     await reporter.upload_log(test_id, traceback.format_exc())
                 except Exception:
-                    logger.error(f"Failed to upload fail logs for test {test_id}: {traceback.format_exc()}")
+                    print(f"Failed to upload fail logs for test {test_id}: {traceback.format_exc()}")
                 await reporter.finish_test(test_id, "FAILED")
         else:
             try:
                 test_id = await reporter.create_test(test, test)
-                logger.warning(f"Unsupported test type: {test['id']}")
+                print(f"Unsupported test type: {test['id']}")
                 await reporter.upload_log(test_id, f"Unsupported test type in test: {test['id']}")
                 status = "FAILED"
                 await reporter.finish_test(test_id, status)
             except Exception:
-                logger.error(f"Failed to report errors with: {test['id']}")
+                print(f"Failed to report errors with: {test['id']}")
 
     return full_report