From b655abea2662b279c4059c2ec6ae08796beb7228 Mon Sep 17 00:00:00 2001 From: Mac Clayton Date: Sun, 5 Sep 2021 14:23:09 -0500 Subject: [PATCH 1/2] Update to Jenkins JUnit 5 xsd --- .../JunitXmlSerializer.cs | 1 - .../JunitXmlValidator.cs | 2 +- test/assets/JUnit.xsd | 212 ------------------ test/assets/jenkins-junit.xsd | 119 ++++++++++ 4 files changed, 120 insertions(+), 214 deletions(-) delete mode 100644 test/assets/JUnit.xsd create mode 100644 test/assets/jenkins-junit.xsd diff --git a/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs b/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs index 425290a..f64d5e3 100644 --- a/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs +++ b/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs @@ -225,7 +225,6 @@ private XElement CreateTestSuiteElement(List results, TestRunCon // single newline. var element = new XElement( "testsuite", - new XElement("properties"), testCaseElements, new XElement("system-out", stdOut.ToString()), new XElement("system-err", stdErr.ToString())); diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs index ea8fc65..e284bae 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs @@ -25,7 +25,7 @@ public bool IsValid(string xml) var xmlReader = new StringReader(xml); var xsdReader = new StringReader( File.ReadAllText( - Path.Combine("..", "..", "..", "..", "assets", "JUnit.xsd"))); + Path.Combine("..", "..", "..", "..", "assets", "jenkins-junit.xsd"))); var schema = XmlSchema.Read( xsdReader, diff --git a/test/assets/JUnit.xsd b/test/assets/JUnit.xsd deleted file mode 100644 index 320779e..0000000 --- a/test/assets/JUnit.xsd +++ /dev/null @@ -1,212 +0,0 @@ - - - - JUnit test result schema for the Apache Ant JUnit and JUnitReport tasks -Copyright © 2011, Windy Road Technology Pty. Limited -The Apache Ant JUnit XML Schema is distributed under the terms of the Apache License Version 2.0 http://www.apache.org/licenses/ -Permission to waive conditions of this license may be requested from Windy Road Support (http://windyroad.org/support). - - - - - - - - - - Contains an aggregation of testsuite results - - - - - - - - - - Derived from testsuite/@name in the non-aggregated documents - - - - - Starts at '0' for the first testsuite and is incremented by 1 for each following testsuite - - - - - - - - - - - - Contains the results of exexuting a testsuite - - - - - Properties (e.g., environment settings) set during test execution - - - - - - - - - - - - - - - - - - - - - - - - - Indicates that the test errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. Contains as a text node relevant data for the error, e.g., a stack trace - - - - - - - The error message. e.g., if a java exception is thrown, the return value of getMessage() - - - - - The type of error that occured. e.g., if a java execption is thrown the full class name of the exception. - - - - - - - - - Indicates that the test failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals. Contains as a text node relevant data for the failure, e.g., a stack trace - - - - - - - The message specified in the assert - - - - - The type of the assert. - - - - - - - - - - Name of the test method - - - - - Full class name for the class the test method is in. - - - - - Time taken (in seconds) to execute the test - - - - - - - Data that was written to standard out while the test was executed - - - - - - - - - - Data that was written to standard error while the test was executed - - - - - - - - - - - Full class name of the test for non-aggregated testsuite documents. Class name without the package for aggregated testsuites documents - - - - - - - - - - when the test was executed. Timezone may not be specified. - - - - - Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined. - - - - - - - - - - The total number of tests in the suite - - - - - The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals - - - - - The total number of tests in the suite that errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. - - - - - The total number of ignored or skipped tests in the suite. - - - - - Time taken (in seconds) to execute the tests in the suite - - - - - - - - - \ No newline at end of file diff --git a/test/assets/jenkins-junit.xsd b/test/assets/jenkins-junit.xsd new file mode 100644 index 0000000..776a58b --- /dev/null +++ b/test/assets/jenkins-junit.xsd @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 507b9cb0f80569e33461be78eafdff87fca1147c Mon Sep 17 00:00:00 2001 From: Mac Clayton Date: Sun, 5 Sep 2021 14:23:42 -0500 Subject: [PATCH 2/2] Add system-out and system-error to each testcase --- README.md | 87 ++++++++++++++++++- .../JunitXmlSerializer.cs | 28 ++++++ .../JUnitTestLoggerAcceptanceTests.cs | 18 +++- 3 files changed, 128 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1d2d05c..45daebb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ JUnit xml report extension for [Visual Studio Test Platform](https://github.com/ | ------ | -------------- | ------------------- | | JUnit | [![NuGet](https://img.shields.io/nuget/v/JUnitXml.TestLogger.svg)](https://www.nuget.org/packages/JUnitXml.TestLogger/) | [![MyGet Pre Release](https://img.shields.io/myget/spekt/vpre/junitxml.testlogger.svg)](https://www.myget.org/feed/spekt/package/nuget/JunitXml.TestLogger) | -If you're looking for `Nunit`, `Xunit` or `appveyor` loggers, visit following repositories: +If you're looking for `NUnit`, `xUnit` or `appveyor` loggers, visit following repositories: - - @@ -20,7 +20,7 @@ If you're looking for `Nunit`, `Xunit` or `appveyor` loggers, visit following re ## Usage -The JUnit Test Logger generates xml reports in the [Ant Junit Format](https://github.com/windyroad/JUnit-Schema), which the JUnit 5 repository refers to as the de-facto standard. While the generated xml complies with that schema, it does not contain values in every case. For example, the logger currently does not log any `properties`. Please [refer to a sample file](docs/assets/TestResults.xml) to see an example. If you find that the format is missing data required by your CI/CD system, please open an issue or PR. +The JUnit Test Logger generates xml reports in the [Jenkins JUnit Format](https://github.com/junit-team/junit5/blob/main/platform-tests/src/test/resources/jenkins-junit.xsd), which the JUnit 5 repository refers to as the de-facto standard. While the generated xml complies with that schema, it does not contain values in every case. For example, the logger currently does not log any `properties`. Please [refer to a sample file](docs/assets/TestResults.xml) to see an example. If you find that the format is missing data required by your CI/CD system, please open an issue or PR. To use the logger, follow these steps: @@ -47,6 +47,89 @@ test logs from over-writing eachother. [config-wiki]: https://github.com/spekt/testlogger/wiki/Logger-Configuration +### Junit Output Notes + +The xml reports will have standard output (stdout) and standard error (stderr) from the test output. Stdout is logged on a per-testcase basis, but stderr + +#### NUnit + +On a testcase failure, NUnit will redirect stderr to stdout for each `TestCase`. Because of this, both stdout and stderr will be captured in the JUnit xml file within the `` tags for each test case. It will also be capture within the `` tags at the `` level. + +##### NUnit Example +```csharp +[TestCase] +public void TestThatFailsWithStdErr() +{ + Console.WriteLine("Output to stderr", Console.Error); + Assert.Fail(); +} +``` +Console Output +``` + Failed TestThatFailsWithStdErr() [71 ms] + Stack Trace: + at NUnit.UnitTests.Tests.TestThatFailsWithStdErr() in /home/runner/work/dotnet_test_output_reporting/dotnet_test_output_reporting/NUnit.UnitTests/UnitTest1.cs:line 42 + + Standard Output Messages: + Output to stderr +``` + +```xml + + at NUnit.UnitTests.Tests.TestThatFailsWithStdOutAndStdErr() in D:\Developer\dotnet_test_output_reporting\NUnit.UnitTests\UnitTest1.cs:line 50 + Output to stdout +Output to stderr + + + +``` + +#### xUnit + +On a testcase failure, by default xUnit will not log any stdout or stderr to the console. Stdout from each `Fact` will be captured in the JUnit xml within the `` tags at the `testsuite` level. + +##### xUnit Example +```csharp +[Fact] +public void TestThatFailsWithStdErr() +{ + Console.WriteLine("Output to stderr", Console.Error); + Assert.True(false, "test failure"); +} +``` +``` +Expected: True +Actual: False + Stack Trace: + at XUnit.UnitTests.UnitTest1.TestThatFailsWithStdErr() in /home/runner/work/dotnet_test_output_reporting/dotnet_test_output_reporting/XUnit.UnitTests/UnitTest1.cs:line 41 + Failed XUnit.UnitTests.UnitTest1.TestThatFailsWithStdOutAndStdErr [< 1 ms] + Error Message: + test failure +``` +```xml + + at XUnit.UnitTests.UnitTest1.TestThatFailsWithStdErr() in D:\Developer\dotnet_test_output_reporting\XUnit.UnitTests\UnitTest1.cs:line 41 + +``` + + +#### MsTest + +#### MsTest Example +```csharp + +``` +``` +Failed TestThatFailsWithStdErr [< 1 ms] +Error Message: + Assert.Fail failed. +Stack Trace: + at MsTest.UnitTests.UnitTest1.TestThatFailsWithStdErr() in /home/runner/work/dotnet_test_output_reporting/dotnet_test_output_reporting/MsTest.UnitTests/UnitTest1.cs:line 42 + +Standard Output Messages: +Output to stderr +``` + ### Customizing Junit XML Contents There are several options to customize how the junit xml is populated. These options exist to diff --git a/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs b/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs index f64d5e3..19c5198 100644 --- a/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs +++ b/src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs @@ -307,6 +307,34 @@ private XElement CreateTestCaseElement(TestResultInfo result) testcaseElement.Add(skippedElement); } + StringBuilder stdErr = new StringBuilder(); + foreach (var m in result.Messages) + { + if (TestResultMessage.StandardErrorCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase)) + { + stdErr.AppendLine(m.Text); + } + } + + if (!string.IsNullOrWhiteSpace(stdErr.ToString())) + { + testcaseElement.Add(new XElement("system-err", stdErr.ToString())); + } + + StringBuilder stdOut = new StringBuilder(); + foreach (var m in result.Messages) + { + if (TestResultMessage.StandardOutCategory.Equals(m.Category, StringComparison.OrdinalIgnoreCase)) + { + stdOut.AppendLine(m.Text); + } + } + + if (!string.IsNullOrWhiteSpace(stdOut.ToString())) + { + testcaseElement.Add(new XElement("system-out", stdOut.ToString())); + } + return testcaseElement; } diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs index eefbb4b..cf808a4 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs @@ -85,8 +85,7 @@ public void TestResultFileShouldContainTestCases() .ToList(); Assert.AreEqual(14, failures.Count()); - Assert.IsTrue(failures.All(x => x.Descendants().Count() == 1)); - Assert.IsTrue(failures.All(x => x.Descendants().First().Attribute("type").Value == "failure")); + Assert.IsTrue(failures.All(x => x.Descendants().Any(element => element.Attribute("type").Value == "failure"))); // Check failures var skips = testcases @@ -96,6 +95,19 @@ public void TestResultFileShouldContainTestCases() Assert.AreEqual(6, skips.Count()); } + [TestMethod] + public void TestCasesShouldContainStandardOut() + { + var node = this.resultsXml.XPathSelectElements("/testsuites/testsuite").Descendants(); + var testcases = node.Where(x => x.Name.LocalName == "testcase").ToList(); + + var testCasesWithSystemOut = testcases + .Where(x => x.Descendants().Any(y => y.Name.LocalName == "system-out")) + .ToList(); + + Assert.AreEqual(2, testCasesWithSystemOut.Count()); + } + [TestMethod] public void TestResultFileShouldContainStandardOut() { @@ -108,7 +120,7 @@ public void TestResultFileShouldContainStandardOut() } [TestMethod] - public void TestResultFileShouldContainErrordOut() + public void TestResultFileShouldContainErrorOut() { var node = this.resultsXml.XPathSelectElement("/testsuites/testsuite/system-err");