Misc/capture logs when running examples (#1141)

* unmask all signals in App driver's subprocess

* keep log cap only

testplan/testing/py_test.py

@@ -482,6 +482,9 @@ def pytest_runtest_logreport(self, report):
 
             if report.failed:
                 self._current_case_report.status_override = Status.FAILED
+            # XXX: report.skipped set to True when xfail, how to distinguish?
+            # elif report.skipped:
+            #     self._current_case_report.status_override = Status.SKIPPED
             else:
                 self._current_case_report.pass_if_empty()
             self._current_case_report.runtime_status = RuntimeStatus.FINISHED

tests/conftest.py

@@ -4,7 +4,7 @@
 import shutil
 import sys
 import tempfile
-from logging import Logger
+from logging import Logger, INFO
 from logging.handlers import RotatingFileHandler
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "helpers"))
@@ -125,3 +125,13 @@ def rotating_logger(runpath):
     yield logger
 
     logger.handler.close()
+
+
+@pytest.fixture
+def captplog(caplog):
+    from testplan.common.utils.logger import TESTPLAN_LOGGER
+
+    caplog.set_level(INFO)
+    TESTPLAN_LOGGER.addHandler(caplog.handler)
+    yield caplog
+    TESTPLAN_LOGGER.removeHandler(caplog.handler)

tests/functional/examples/test_examples.py

@@ -90,7 +90,7 @@ def _param_formatter(param):
     ],
     ids=_param_formatter,
 )
-def test_example(root, filename, runpath):
+def test_example(root, filename, runpath, captplog):
     file_path = os.path.join(root, filename)
 
     if ON_WINDOWS and any(
@@ -106,5 +106,9 @@ def test_example(root, filename, runpath):
         pytest.skip()
 
     run_example_in_process(
-        filename, root, KNOWN_EXCEPTIONS, ["--runpath", runpath]
+        filename,
+        root,
+        KNOWN_EXCEPTIONS,
+        ["--runpath", runpath],
+        captplog,
     )

tests/helpers/example_runner.py

@@ -18,8 +18,15 @@
 
 
 def run_example_in_process(
-    filename, root, known_exceptions, cmdline_args=None
+    filename,
+    root,
+    known_exceptions,
+    cmdline_args=None,
+    caplog=None,
 ):
+    if caplog is not None:
+        caplog.clear()
+
     sys_path = copy.copy(sys.path)
     sys_argv = copy.copy(sys.argv)
 
@@ -43,9 +50,12 @@ def run_example_in_process(
     except SystemExit as e:
         if e.code not in SUCCES_EXIT_CODES:
             assert (
-                "# This plan contains tests that demonstrate failures as well."
-            ) == second_line, (
-                'Expected "{}" example to pass, it failed'.format(file_path)
+                second_line
+                == "# This plan contains tests that demonstrate failures as well."
+            ), 'Expected "{}" example to pass, it failed'.format(file_path) + (
+                "\nCaptured logs:\n{}".format(caplog.text)
+                if caplog is not None
+                else ""
             )
     except Exception as e:
         for exception in known_exceptions: