From 973712f72f8fae310e516d7af9258110f12766ae Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Sat, 10 Jun 2023 11:59:55 -0500 Subject: [PATCH 1/5] Disable hanging RE test --- .../tests/FunctionalTests/Regex.Match.Tests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs index 6311cdf5f6046..fd8660e961f00 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs @@ -2159,6 +2159,9 @@ public static IEnumerable StressTestDeepNestingOfLoops_TestData() { foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { + if (engine == RegexEngine.NonBacktracking) + yield break; // Hangs, or effectively hangs. https://github.com/dotnet/runtime/issues/84188 + yield return new object[] { engine, "(", "a", ")*", "a", 2000, 1000 }; yield return new object[] { engine, "(", "[aA]", ")+", "aA", 2000, 3000 }; yield return new object[] { engine, "(", "ab", "){0,1}", "ab", 2000, 1000 }; From 2b8eb997aee3c9ebb36fd309a57786861ad56374 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Sun, 11 Jun 2023 10:52:13 -0500 Subject: [PATCH 2/5] feedback --- docs/workflow/testing/libraries/testing.md | 19 +++++++++++++++++-- .../FunctionalTests/Regex.Match.Tests.cs | 9 +++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/workflow/testing/libraries/testing.md b/docs/workflow/testing/libraries/testing.md index 4d90f52af44f2..e7ef4332826a1 100644 --- a/docs/workflow/testing/libraries/testing.md +++ b/docs/workflow/testing/libraries/testing.md @@ -79,13 +79,25 @@ cd src\libraries\System.Collections.Immutable\tests dotnet build /t:Test ``` -It is possible to pass parameters to the underlying xunit runner via the `XUnitOptions` parameter, e.g.: +This one command builds the library under test, then the test library, then runs the tests in the test library. + +### Running only certain tests + +It is possible to pass parameters to the underlying xunit runner via the `XUnitOptions` parameter, e.g., to filter to tests in just one fixture (class): ```cmd dotnet build /t:Test /p:XUnitOptions="-class Test.ClassUnderTests" ``` -Which is very useful when you want to run tests as `x86` on a `x64` machine: +or to just one test method: + +```cmd +dotnet build /t:test /p:outerloop=true /p:xunitoptions="-method System.Text.RegularExpressions.Tests.RegexMatchTests.StressTestDeepNestingOfLoops" +``` + +### Running only specific architectures + +To run tests as `x86` on a `x64` machine: ```cmd dotnet build /t:Test /p:TargetArchitecture=x86 @@ -145,3 +157,6 @@ It is important to highlight that these tests do not use the standard XUnit test - `-nonamespace` - `-parallel` +### Viewing XUnit logs + +It's usually sufficient to see the test failure output in the console. There is a test log file though and you can find it in a location like `...\runtime\artifacts\bin\System.Text.RegularExpressions.Tests\Debug\net8.0\testResults.xml`. It can be helpful for example to grep through a series of failures, or see how long a slow test actually took. \ No newline at end of file diff --git a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs index fd8660e961f00..206e2046806dd 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs @@ -2159,16 +2159,17 @@ public static IEnumerable StressTestDeepNestingOfLoops_TestData() { foreach (RegexEngine engine in RegexHelpers.AvailableEngines) { - if (engine == RegexEngine.NonBacktracking) - yield break; // Hangs, or effectively hangs. https://github.com/dotnet/runtime/issues/84188 + if (engine != RegexEngine.NonBacktracking) // Hangs, or effectively hangs. https://github.com/dotnet/runtime/issues/84188 + { + yield return new object[] { engine, "(", "a", ")*", "a", 2000, 1 }; + } - yield return new object[] { engine, "(", "a", ")*", "a", 2000, 1000 }; yield return new object[] { engine, "(", "[aA]", ")+", "aA", 2000, 3000 }; yield return new object[] { engine, "(", "ab", "){0,1}", "ab", 2000, 1000 }; } } - [OuterLoop("Can take over 10 seconds")] + [OuterLoop("Can take a few seconds")] [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess))] // consumes a lot of memory [MemberData(nameof(StressTestDeepNestingOfLoops_TestData))] public async Task StressTestDeepNestingOfLoops(RegexEngine engine, string begin, string inner, string end, string input, int pattern_repetition, int input_repetition) From 31783b996f0a662e9e884974f701aca00497ec9b Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Sun, 11 Jun 2023 10:55:03 -0500 Subject: [PATCH 3/5] Oops --- .../tests/FunctionalTests/Regex.Match.Tests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs index 206e2046806dd..f8919a8b8d806 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs @@ -2161,7 +2161,7 @@ public static IEnumerable StressTestDeepNestingOfLoops_TestData() { if (engine != RegexEngine.NonBacktracking) // Hangs, or effectively hangs. https://github.com/dotnet/runtime/issues/84188 { - yield return new object[] { engine, "(", "a", ")*", "a", 2000, 1 }; + yield return new object[] { engine, "(", "a", ")*", "a", 2000, 1000 }; } yield return new object[] { engine, "(", "[aA]", ")+", "aA", 2000, 3000 }; From 90d65c02c9297e9b62b071f97b7aa6b4a3bfa14b Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Mon, 12 Jun 2023 15:53:17 -0500 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Stephen Toub --- docs/workflow/testing/libraries/testing.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/workflow/testing/libraries/testing.md b/docs/workflow/testing/libraries/testing.md index e7ef4332826a1..6ef725027db73 100644 --- a/docs/workflow/testing/libraries/testing.md +++ b/docs/workflow/testing/libraries/testing.md @@ -79,7 +79,6 @@ cd src\libraries\System.Collections.Immutable\tests dotnet build /t:Test ``` -This one command builds the library under test, then the test library, then runs the tests in the test library. ### Running only certain tests @@ -159,4 +158,4 @@ It is important to highlight that these tests do not use the standard XUnit test ### Viewing XUnit logs -It's usually sufficient to see the test failure output in the console. There is a test log file though and you can find it in a location like `...\runtime\artifacts\bin\System.Text.RegularExpressions.Tests\Debug\net8.0\testResults.xml`. It can be helpful for example to grep through a series of failures, or see how long a slow test actually took. \ No newline at end of file +It's usually sufficient to see the test failure output in the console. There is also a test log file, which you can find in a location like `...\runtime\artifacts\bin\System.Text.RegularExpressions.Tests\Debug\net8.0\testResults.xml`. It can be helpful, for example, to grep through a series of failures, or to see how long a slow test actually took. \ No newline at end of file From 35a208a490ff4707c4f7d137cb34e0ab2a44294a Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Mon, 12 Jun 2023 15:53:36 -0500 Subject: [PATCH 5/5] Update testing.md --- docs/workflow/testing/libraries/testing.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/workflow/testing/libraries/testing.md b/docs/workflow/testing/libraries/testing.md index 6ef725027db73..1ebd87ac40715 100644 --- a/docs/workflow/testing/libraries/testing.md +++ b/docs/workflow/testing/libraries/testing.md @@ -79,7 +79,6 @@ cd src\libraries\System.Collections.Immutable\tests dotnet build /t:Test ``` - ### Running only certain tests It is possible to pass parameters to the underlying xunit runner via the `XUnitOptions` parameter, e.g., to filter to tests in just one fixture (class): @@ -158,4 +157,4 @@ It is important to highlight that these tests do not use the standard XUnit test ### Viewing XUnit logs -It's usually sufficient to see the test failure output in the console. There is also a test log file, which you can find in a location like `...\runtime\artifacts\bin\System.Text.RegularExpressions.Tests\Debug\net8.0\testResults.xml`. It can be helpful, for example, to grep through a series of failures, or to see how long a slow test actually took. \ No newline at end of file +It's usually sufficient to see the test failure output in the console. There is also a test log file, which you can find in a location like `...\runtime\artifacts\bin\System.Text.RegularExpressions.Tests\Debug\net8.0\testResults.xml`. It can be helpful, for example, to grep through a series of failures, or to see how long a slow test actually took.