diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/Report.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/Report.java index 3e8b610e7..02f099f6d 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/Report.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/Report.java @@ -36,7 +36,6 @@ enum Properties implements IProperties { NAME("name", "Test report"), @Deprecated ACTIVATE_SOURCES("activate.sources", true), - LIST_TESTS("list.tests", false), SOURCE_ROOT("source.root", "src"), SOURCE_LINES_PREFETCH("source.lines.prefetch", 5), SOURCE_EXCLUSION("source.exclusion.regex", ""), diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/ConfigMethodHook.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/ConfigMethodHook.java index fb3d0fe9a..57fc5ae9e 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/ConfigMethodHook.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/ConfigMethodHook.java @@ -19,10 +19,9 @@ * under the License. * */ - package eu.tsystems.mms.tic.testframework.report.hooks; +package eu.tsystems.mms.tic.testframework.report.hooks; import eu.tsystems.mms.tic.testframework.common.Testerra; -import eu.tsystems.mms.tic.testframework.report.Report; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.IConfigureCallBack; @@ -39,15 +38,9 @@ private ConfigMethodHook() { public static void runHook(IConfigureCallBack callBack, ITestResult testResult) { - final ITestNGMethod testNGMethod = testResult.getMethod(); - - if (Report.Properties.LIST_TESTS.asBool()) { - LOGGER.info("Dry run for list tests: " + testNGMethod.getMethodName()); - // no sleep - return; - } if (Testerra.Properties.DRY_RUN.asBool()) { - if (dryRun(testNGMethod)) { + if (dryRun(testResult.getMethod())) { + testResult.setStatus(ITestResult.SUCCESS); return; } } diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/TestMethodHook.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/TestMethodHook.java index 6fecc7d2b..5fc7dd3ad 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/TestMethodHook.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/hooks/TestMethodHook.java @@ -19,10 +19,9 @@ * under the License. * */ - package eu.tsystems.mms.tic.testframework.report.hooks; +package eu.tsystems.mms.tic.testframework.report.hooks; import eu.tsystems.mms.tic.testframework.common.Testerra; -import eu.tsystems.mms.tic.testframework.report.Report; import eu.tsystems.mms.tic.testframework.report.TesterraListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,14 +44,9 @@ public static void runHook(IHookCallBack callBack, ITestResult testResult) { throw new SkipException("Conditional skipping test method (testerra run runHook)"); } - final ITestNGMethod testNGMethod = testResult.getMethod(); - - if (Report.Properties.LIST_TESTS.asBool()) { - LOGGER.info("Dry run for list tests: " + testNGMethod.getMethodName()); - // no sleep - return; - } else if (Testerra.Properties.DRY_RUN.asBool()) { - if (dryRun(testNGMethod)) { + if (Testerra.Properties.DRY_RUN.asBool()) { + if (dryRun(testResult.getMethod())) { + testResult.setStatus(ITestResult.SUCCESS); return; } } diff --git a/docs/src/docs/properties/execution-probs.adoc b/docs/src/docs/properties/execution-probs.adoc index cad3c0b64..8a7ff2a66 100644 --- a/docs/src/docs/properties/execution-probs.adoc +++ b/docs/src/docs/properties/execution-probs.adoc @@ -7,7 +7,6 @@ include::property-attributes.adoc[] | {demomode_timeout} | 2000 | Timeout in ms for visual marks of UiElements. | {dryrun} | false | All testmethods are executed with ignoring all steps. Also all setup methods (before, after) are ignored. + This is useful to check your TestNG suite without executing real testcode. -| {list_tests} | false | Lists all test methods in the current context without executing them. | {on_state_testfailed_skip_shutdown} | false | If true all browser sessions are left open. | {on_state_testfailed_skip_following_tests} | false | If true, all follwoing tests are skipped in case a test failed. | {failed_tests_if_throwable_classes} | na. | Failed tests condition: Throwable Class(~es, devided by ','). diff --git a/docs/src/docs/properties/property-attributes.adoc b/docs/src/docs/properties/property-attributes.adoc index b8557b216..2efc159a9 100644 --- a/docs/src/docs/properties/property-attributes.adoc +++ b/docs/src/docs/properties/property-attributes.adoc @@ -40,7 +40,6 @@ // report :reportdir: tt.report.dir :reportname: tt.report.name -:list_tests: tt.report.list.tests :runcfg: tt.runcfg :screenshotter_active: tt.screenshotter.active :screenshot_on_pageload: tt.screenshot.on.pageload diff --git a/integration-tests/README.md b/integration-tests/README.md index 8a9cbcc3c..c4124dc91 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -2,7 +2,13 @@ ## Run pretest ```shell -gradle test -P pretest=true -D log4j2.configurationFile=src/test/resources/log4j2-pretest.xml +# The pretest to check test status +gradle test -P pretest=true +``` + +```shell +# The pretest to check dry run +gradle test -P pretestDryRun=true ``` ## Start Report prepare tests (deprecated) diff --git a/integration-tests/build.gradle b/integration-tests/build.gradle index 3d005721e..02d0ba48c 100644 --- a/integration-tests/build.gradle +++ b/integration-tests/build.gradle @@ -35,6 +35,16 @@ test { def preTest = findProperty("pretest") if (preTest) { + System.setProperty("logFileName", "logs/pre-test-log4j.log") + System.setProperty("log4j2.configurationFile", "src/test/resources/log4j2-pretest.xml") + s = ['src/test/resources/Pretest_Tests.xml'] + } + + def pretestDryRun = findProperty("pretestDryRun") + if (pretestDryRun) { + System.setProperty("tt.dryrun", "true") + System.setProperty("logFileName", "logs/pre-test-dry-run-log4j.log") + System.setProperty("log4j2.configurationFile", "src/test/resources/log4j2-pretest.xml") s = ['src/test/resources/Pretest_Tests.xml'] } diff --git a/integration-tests/src/test/java/eu/tsystems/mms/tic/testframework/playground/TestNGFeatureTest.java b/integration-tests/src/test/java/eu/tsystems/mms/tic/testframework/playground/TestNGFeatureTest.java new file mode 100644 index 000000000..a434e7bd8 --- /dev/null +++ b/integration-tests/src/test/java/eu/tsystems/mms/tic/testframework/playground/TestNGFeatureTest.java @@ -0,0 +1,46 @@ +package eu.tsystems.mms.tic.testframework.playground; + +import eu.tsystems.mms.tic.testframework.testing.TesterraTest; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Created on 2024-08-30 + * + * @author mgn + */ +public class TestNGFeatureTest extends TesterraTest { + + @BeforeMethod + public void before_method() { + + } + + @BeforeMethod + public void beofre_method_failed() { + Assert.fail(); + } + + @Test + public void T01_testng_passed_test() { + + } + + @Test(dependsOnMethods = "T01_testng_passed_test") + public void T02_testng_depends_method() { + + } + + @Test + public void T03_testng_failed_test() { + Assert.fail(); + } + + @AfterMethod + public void after_method() { + + } + +} diff --git a/integration-tests/src/test/java/io/testerra/test/status/CheckDryRunStatusTest.java b/integration-tests/src/test/java/io/testerra/test/status/CheckDryRunStatusTest.java new file mode 100644 index 000000000..15f9e5ac4 --- /dev/null +++ b/integration-tests/src/test/java/io/testerra/test/status/CheckDryRunStatusTest.java @@ -0,0 +1,125 @@ +/* + * Testerra + * + * (C) 2024, Martin Großmann, Deutsche Telekom MMS GmbH, Deutsche Telekom AG + * + * Deutsche Telekom AG and all other contributors / + * copyright owners license this file to you under the Apache + * License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package io.testerra.test.status; + +import eu.tsystems.mms.tic.testframework.core.utils.Log4jFileReader; +import eu.tsystems.mms.tic.testframework.report.Status; +import eu.tsystems.mms.tic.testframework.testing.TesterraTest; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.List; + +/** + * Created on 2024-08-30 + * + * @author mgn + */ +public class CheckDryRunStatusTest extends TesterraTest { + + // log file reader + private final Log4jFileReader LOG_4_J_FILE_READER = new Log4jFileReader("logs/pre-test-dry-run-log4j.log"); + + // terms for searching log file + private final String METHOD_END_WORKER_SEARCH_TERM = "finish.MethodEndWorker"; + + /** + * Test execution with active tt.dryrun flag are results in passed test methods and past configuration methods + * + * Except: + * For data provider methods there is no TestNG hook. If a data provider method fails, it also fails in a dry run. The corresponding tests are skipped, not passed. + * + */ + + @DataProvider + public static Object[][] provideExpectedStatusPerMethod() { + return new Object[][]{ + {"testT07_NonExistingDataProvider", Status.FAILED}, + {"beforeClassSetup01", Status.PASSED}, + {"beforeClassSetup02", Status.PASSED}, + {"beforeClassSetup03", Status.PASSED}, + {"beforeMethodSetup01", Status.PASSED}, + {"beforeMethodSetup02", Status.PASSED}, + {"afterMethodSetup01", Status.PASSED}, + {"afterMethodSetup02", Status.PASSED}, + {"afterClassSetup01", Status.PASSED}, + {"afterClassSetup02", Status.PASSED}, + {"beforeClassBeforeMethodSetup03", Status.PASSED}, + {"dataProviderThrowingAssertion", Status.FAILED}, + {"dataProviderThrowingException", Status.FAILED}, + {"dataProviderInClassThrowingException", Status.FAILED}, + }; + } + + @DataProvider + public static Object[][] provideTestMethodsSkipped() { + return new Object[][]{ + {"testT01_interceptCrashedDataProvider", Status.SKIPPED, "java.lang.AssertionError"}, + {"testT02_crashedDataProvider", Status.SKIPPED, "java.lang.AssertionError"}, + {"testT03_AssertFailedDataProvider", Status.SKIPPED, "java.lang.AssertionError"}, + }; + } + + @DataProvider + public static Object[][] provideFinalTestResult() { + return new Object[][]{ + {"*** Stats: SuiteContexts: 3", "SuiteContext"}, + {"*** Stats: TestContexts: 3", "TestContext"}, + {"*** Stats: ClassContexts: 16", "ClassContext"}, + {"*** Stats: MethodContexts: 73", "MethodContexts"}, + {"*** Stats: Test Methods Count: 49 (49 relevant)", "Test methods"}, + {"*** Stats: Failed: 1", "Failed tests"}, + {"*** Stats: Skipped: 3", "Skipped tests"}, + {"*** Stats: Passed: 45 ⊃ Repaired: 11", "Passed tests"} + }; + } + + @Test(dataProvider = "provideExpectedStatusPerMethod") + public void testT01_verifySimpleTestStatus(final String methodName, final Status expectedTestStatus) { + LOG_4_J_FILE_READER.assertTestStatusPerMethod(METHOD_END_WORKER_SEARCH_TERM, methodName, expectedTestStatus); + } + + @Test(dataProvider = "provideTestMethodsSkipped") + public void testT02_verifySkippedStatus(final String methodName, final Status expectedInitialStatus, final String skippedInformationInLog) { + + final String expectedInitialStatusTitle = expectedInitialStatus.title; + + // check basic test status in log + final List foundEntries = LOG_4_J_FILE_READER.filterLogForTestMethod(METHOD_END_WORKER_SEARCH_TERM, methodName); + final String logLine = foundEntries.get(0); + + Assert.assertEquals(foundEntries.size(), 1, "correct amount of method end entries found."); + Assert.assertTrue(logLine.contains(expectedInitialStatusTitle), String.format("'%s' has status '%s' in log line '%s'", methodName, expectedInitialStatusTitle, logLine)); + + // check entry with information for skipped + final String skippedMethodEntry = LOG_4_J_FILE_READER.filterLogForFollowingEntry(logLine); + Assert.assertTrue(skippedMethodEntry.contains(skippedInformationInLog), + String.format("status '%s' found in entry '%s'", skippedInformationInLog, skippedMethodEntry)); + } + + @Test(dataProvider = "provideFinalTestResult") + public void testT02_verifyCompleteResult(final String resultString, final String testObject) { + List foundEntries = LOG_4_J_FILE_READER.filterLogForString(resultString); + Assert.assertEquals(foundEntries.size(), 1, String.format("The count of %s should contains in log with the string '%s'.", testObject, resultString)); + } +} diff --git a/integration-tests/src/test/resources/log4j2-pretest.xml b/integration-tests/src/test/resources/log4j2-pretest.xml index d5d0c8d33..2822682ef 100644 --- a/integration-tests/src/test/resources/log4j2-pretest.xml +++ b/integration-tests/src/test/resources/log4j2-pretest.xml @@ -4,7 +4,7 @@ - + %d{dd.MM.yyyy HH:mm:ss.SSS} [%t][%p]%contextIds: %c{2} - %m%n