diff --git a/src/NLU.DevOps.CommandLine/Compare/CompareCommand.cs b/src/NLU.DevOps.CommandLine/Compare/CompareCommand.cs index 160bb67..c2a3a2e 100644 --- a/src/NLU.DevOps.CommandLine/Compare/CompareCommand.cs +++ b/src/NLU.DevOps.CommandLine/Compare/CompareCommand.cs @@ -7,8 +7,8 @@ namespace NLU.DevOps.CommandLine.Compare using System.Globalization; using System.IO; using System.Linq; + using Core; using ModelPerformance; - using Models; using NUnitLite; using static Serializer; @@ -23,6 +23,7 @@ public static int Run(CompareOptions options) (ConfigurationConstants.ExpectedUtterancesPathKey, options.ExpectedUtterancesPath), (ConfigurationConstants.ActualUtterancesPathKey, options.ActualUtterancesPath), (ConfigurationConstants.CompareTextKey, options.CompareText.ToString(CultureInfo.InvariantCulture)), + (ConfigurationConstants.EvaluateKey, options.Evaluate.ToString(CultureInfo.InvariantCulture)), (ConfigurationConstants.TestLabelKey, options.TestLabel)); var arguments = new List { $"-p:{parameters}" }; @@ -34,8 +35,11 @@ public static int Run(CompareOptions options) if (options.Metadata) { var expectedUtterances = Read>(options.ExpectedUtterancesPath); - var actualUtterances = Read>(options.ActualUtterancesPath); - var compareResults = TestCaseSource.GetNLUCompareResults(expectedUtterances, actualUtterances, options.CompareText); + var actualUtterances = Read>(options.ActualUtterancesPath); + TestCaseSource.ShouldCompareText = options.CompareText; + TestCaseSource.ShouldEvaluate = options.Evaluate; + TestCaseSource.TestLabel = options.TestLabel; + var compareResults = TestCaseSource.GetNLUCompareResults(expectedUtterances, actualUtterances); var metadataPath = options.OutputFolder != null ? Path.Combine(options.OutputFolder, TestMetadataFileName) : TestMetadataFileName; var statisticsPath = options.OutputFolder != null ? Path.Combine(options.OutputFolder, TestStatisticsFileName) : TestStatisticsFileName; Write(metadataPath, compareResults.TestCases); diff --git a/src/NLU.DevOps.CommandLine/Compare/CompareOptions.cs b/src/NLU.DevOps.CommandLine/Compare/CompareOptions.cs index 645ba82..6928385 100644 --- a/src/NLU.DevOps.CommandLine/Compare/CompareOptions.cs +++ b/src/NLU.DevOps.CommandLine/Compare/CompareOptions.cs @@ -20,6 +20,9 @@ internal class CompareOptions [Option('m', "metadata", HelpText = "Return test case metadata in addition to NUnit test results.", Required = false)] public bool Metadata { get; set; } + [Option('x', "evaluate", HelpText = "Evaluate inline scripts.", Required = false)] + public bool Evaluate { get; set; } + [Option('t', "text", HelpText = "Run text comparison test cases.", Required = false)] public bool CompareText { get; set; } diff --git a/src/NLU.DevOps.CommandLine/Serializer.cs b/src/NLU.DevOps.CommandLine/Serializer.cs index ea12e61..17c65a1 100644 --- a/src/NLU.DevOps.CommandLine/Serializer.cs +++ b/src/NLU.DevOps.CommandLine/Serializer.cs @@ -17,6 +17,7 @@ public static T Read(string path) { var serializer = JsonSerializer.CreateDefault(); serializer.Converters.Add(new LabeledUtteranceConverter()); + serializer.DateParseHandling = DateParseHandling.None; using (var jsonReader = new JsonTextReader(File.OpenText(path))) { return serializer.Deserialize(jsonReader); diff --git a/src/NLU.DevOps.Core/LabeledUtteranceContext.cs b/src/NLU.DevOps.Core/LabeledUtteranceContext.cs new file mode 100644 index 0000000..0105d0a --- /dev/null +++ b/src/NLU.DevOps.Core/LabeledUtteranceContext.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NLU.DevOps.Core +{ + using System; + using System.Globalization; + + /// + /// Labeled utterance context. + /// + public class LabeledUtteranceContext + { + /// + /// Initializes a new instance of the class. + /// + /// Timestamp. + public LabeledUtteranceContext(string timestamp) + { + this.Timestamp = timestamp; + } + + private LabeledUtteranceContext() + : this(DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)) + { + } + + /// + /// Gets the timestamp for the labeled utterance. + /// + public string Timestamp { get; } + + /// + /// Creates default instance of . + /// + /// The default instance. + public static LabeledUtteranceContext CreateDefault() => new LabeledUtteranceContext(); + } +} diff --git a/src/NLU.DevOps.Luis.Shared/ScoredEntity.cs b/src/NLU.DevOps.Core/PredictedEntity.cs similarity index 78% rename from src/NLU.DevOps.Luis.Shared/ScoredEntity.cs rename to src/NLU.DevOps.Core/PredictedEntity.cs index 1bd54ea..b02a58a 100644 --- a/src/NLU.DevOps.Luis.Shared/ScoredEntity.cs +++ b/src/NLU.DevOps.Core/PredictedEntity.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -namespace NLU.DevOps.Luis +namespace NLU.DevOps.Core { using Models; using Newtonsoft.Json.Linq; @@ -9,17 +9,17 @@ namespace NLU.DevOps.Luis /// /// Entity appearing in utterance with confidence score. /// - public class ScoredEntity : Entity + public class PredictedEntity : Entity { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// Entity type name. /// Entity value, generally a canonical form of the entity. /// Matching text in the utterance. /// Occurrence index of matching token in the utterance. /// Confidence score for the entity. - public ScoredEntity(string entityType, JToken entityValue, string matchText, int matchIndex, double score) + public PredictedEntity(string entityType, JToken entityValue, string matchText, int matchIndex, double score) : base(entityType, entityValue, matchText, matchIndex) { this.Score = score; diff --git a/src/NLU.DevOps.Core/PredictedLabeledUtterance.cs b/src/NLU.DevOps.Core/PredictedLabeledUtterance.cs new file mode 100644 index 0000000..67e99ab --- /dev/null +++ b/src/NLU.DevOps.Core/PredictedLabeledUtterance.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NLU.DevOps.Core +{ + using System.Collections.Generic; + using Models; + using Newtonsoft.Json; + + /// + /// Labeled utterance with confidence score. + /// + public class PredictedLabeledUtterance : LabeledUtterance + { + /// + /// Initializes a new instance of the class. + /// + /// Text of the utterance. + /// Intent of the utterance. + /// Confidence score for the intent label. + /// Confidence score for speech-to-text. + /// Entities referenced in the utterance. + /// Labeled utterance context. + [JsonConstructor] + public PredictedLabeledUtterance(string text, string intent, double score, double textScore, IReadOnlyList entities, LabeledUtteranceContext context) + : this(text, intent, score, textScore, (IReadOnlyList)entities, context) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Text of the utterance. + /// Intent of the utterance. + /// Confidence score for the intent label. + /// Confidence score for speech-to-text. + /// Entities referenced in the utterance. + /// Labeled utterance context. + public PredictedLabeledUtterance(string text, string intent, double score, double textScore, IReadOnlyList entities, LabeledUtteranceContext context) + : base(text, intent, entities) + { + this.Context = context; + this.Score = score; + this.TextScore = textScore; + } + + /// + /// Gets the context of the labeled utterance. + /// + public LabeledUtteranceContext Context { get; } + + /// + /// Gets the confidence score for the intent label. + /// + public double Score { get; } + + /// + /// Gets the confidence score for speech-to-text. + /// + public double TextScore { get; } + } +} diff --git a/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs b/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs index c0cef9b..3113813 100644 --- a/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs +++ b/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs @@ -73,10 +73,14 @@ protected override async Task TestAsync(string utterance, Canc { var client = await this.GetSessionClientAsync(cancellationToken).ConfigureAwait(false); var result = await client.DetectIntentAsync(sessionName, queryInput, cancellationToken).ConfigureAwait(false); - return new LabeledUtterance( + var context = LabeledUtteranceContext.CreateDefault(); + return new PredictedLabeledUtterance( result.QueryResult.QueryText, result.QueryResult.Intent.DisplayName, - result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList()); + result.QueryResult.IntentDetectionConfidence, + result.QueryResult.SpeechRecognitionConfidence, + result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList(), + context); }, cancellationToken) .ConfigureAwait(false); @@ -107,10 +111,14 @@ protected override async Task TestSpeechAsync(string speechFil { var client = await this.GetSessionClientAsync(cancellationToken).ConfigureAwait(false); var result = await client.DetectIntentAsync(request, cancellationToken).ConfigureAwait(false); - return new LabeledUtterance( + var context = LabeledUtteranceContext.CreateDefault(); + return new PredictedLabeledUtterance( result.QueryResult.QueryText, result.QueryResult.Intent.DisplayName, - result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList()); + result.QueryResult.IntentDetectionConfidence, + result.QueryResult.SpeechRecognitionConfidence, + result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList(), + context); }, cancellationToken) .ConfigureAwait(false); diff --git a/src/NLU.DevOps.Lex/LexNLUTestClient.cs b/src/NLU.DevOps.Lex/LexNLUTestClient.cs index 1689492..2a0eee0 100644 --- a/src/NLU.DevOps.Lex/LexNLUTestClient.cs +++ b/src/NLU.DevOps.Lex/LexNLUTestClient.cs @@ -94,10 +94,14 @@ protected override async Task TestAsync(string utterance, Canc .Select(slot => new Entity(slot.Key, slot.Value, null, 0)) .ToArray(); - return new LabeledUtterance( + var context = LabeledUtteranceContext.CreateDefault(); + return new PredictedLabeledUtterance( utterance, postTextResponse.IntentName, - entities); + 0, + 0, + entities, + context); } /// @@ -127,10 +131,14 @@ protected override async Task TestSpeechAsync(string speechFil .ToArray() : null; - return new LabeledUtterance( + var context = LabeledUtteranceContext.CreateDefault(); + return new PredictedLabeledUtterance( postContentResponse.InputTranscript, postContentResponse.IntentName, - slots); + 0, + 0, + slots, + context); } } diff --git a/src/NLU.DevOps.Luis.Shared/NLU.DevOps.Luis.Shared.projitems b/src/NLU.DevOps.Luis.Shared/NLU.DevOps.Luis.Shared.projitems index 2b72566..9b65a03 100644 --- a/src/NLU.DevOps.Luis.Shared/NLU.DevOps.Luis.Shared.projitems +++ b/src/NLU.DevOps.Luis.Shared/NLU.DevOps.Luis.Shared.projitems @@ -15,8 +15,6 @@ - - diff --git a/src/NLU.DevOps.Luis.Shared/ScoredLabeledUtterance.cs b/src/NLU.DevOps.Luis.Shared/ScoredLabeledUtterance.cs deleted file mode 100644 index c705689..0000000 --- a/src/NLU.DevOps.Luis.Shared/ScoredLabeledUtterance.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace NLU.DevOps.Luis -{ - using System.Collections.Generic; - using Models; - - /// - /// Labeled utterance with confidence score. - /// - public class ScoredLabeledUtterance : LabeledUtterance - { - /// - /// Initializes a new instance of the class. - /// - /// Text of the utterance. - /// Intent of the utterance. - /// Confidence score for the intent label. - /// Confidence score for speech-to-text. - /// Entities referenced in the utterance. - public ScoredLabeledUtterance(string text, string intent, double score, double textScore, IReadOnlyList entities) - : base(text, intent, entities) - { - this.Score = score; - this.TextScore = textScore; - } - - /// - /// Gets the confidence score for the intent label. - /// - public double Score { get; } - - /// - /// Gets the confidence score for speech-to-text. - /// - public double TextScore { get; } - } -} diff --git a/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs b/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs index 92d51ea..a1a1d75 100644 --- a/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs +++ b/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs @@ -7,6 +7,7 @@ namespace NLU.DevOps.Luis.Tests using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using Core; using FluentAssertions; using FluentAssertions.Json; using Microsoft.Azure.CognitiveServices.Language.LUIS.Runtime.Models; @@ -204,8 +205,8 @@ public static async Task TestSpeechWithTextScore() var result = await luis.TestSpeechAsync(testFile).ConfigureAwait(false); result.Text.Should().Be(test); result.Intent.Should().Be("intent"); - result.As().TextScore.Should().Be(0.5); - result.As().Score.Should().Be(0); + result.As().TextScore.Should().Be(0.5); + result.As().Score.Should().Be(0); } } @@ -312,8 +313,8 @@ public static async Task WithLabeledIntentScore() using (var luis = builder.Build()) { var result = await luis.TestAsync(test).ConfigureAwait(false); - result.Should().BeOfType(typeof(ScoredLabeledUtterance)); - result.As().Score.Should().Be(0.42); + result.Should().BeOfType(typeof(PredictedLabeledUtterance)); + result.As().Score.Should().Be(0.42); } } @@ -385,8 +386,8 @@ public static async Task WithEntityScore() { var result = await luis.TestAsync(test).ConfigureAwait(false); result.Entities.Count.Should().Be(1); - result.Entities[0].Should().BeOfType(typeof(ScoredEntity)); - result.Entities[0].As().Score.Should().Be(0.42); + result.Entities[0].Should().BeOfType(typeof(PredictedEntity)); + result.Entities[0].As().Score.Should().Be(0.42); } } diff --git a/src/NLU.DevOps.Luis/LuisNLUTestClient.cs b/src/NLU.DevOps.Luis/LuisNLUTestClient.cs index 5014f0d..0b04e5c 100644 --- a/src/NLU.DevOps.Luis/LuisNLUTestClient.cs +++ b/src/NLU.DevOps.Luis/LuisNLUTestClient.cs @@ -141,16 +141,18 @@ Entity getEntity(EntityModel entity) } return entityScore.HasValue - ? new ScoredEntity(entityType, entityValue, matchText, matchIndex, entityScore.Value) + ? new PredictedEntity(entityType, entityValue, matchText, matchIndex, entityScore.Value) : new Entity(entityType, entityValue, matchText, matchIndex); } + var query = speechLuisResult.LuisResult.Query; var intent = speechLuisResult.LuisResult.TopScoringIntent?.Intent; var score = speechLuisResult.LuisResult.TopScoringIntent?.Score; var entities = speechLuisResult.LuisResult.Entities?.Select(getEntity).ToList(); + var context = LabeledUtteranceContext.CreateDefault(); return !score.HasValue && Math.Abs(speechLuisResult.TextScore) < Epsilon - ? new LabeledUtterance(speechLuisResult.LuisResult.Query, intent, entities) - : new ScoredLabeledUtterance(speechLuisResult.LuisResult.Query, intent, score ?? 0, speechLuisResult.TextScore, entities); + ? new LabeledUtterance(query, intent, entities) + : new PredictedLabeledUtterance(query, intent, score ?? 0, speechLuisResult.TextScore, entities, context); } } } diff --git a/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs b/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs index f57f2e3..d580742 100644 --- a/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs +++ b/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs @@ -7,6 +7,7 @@ namespace NLU.DevOps.Luis.Tests using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using Core; using FluentAssertions; using FluentAssertions.Json; using Microsoft.Azure.CognitiveServices.Language.LUIS.Runtime.Models; @@ -152,8 +153,8 @@ public static async Task TestSpeechWithTextScore() var result = await luis.TestSpeechAsync(testFile).ConfigureAwait(false); result.Text.Should().Be(test); result.Intent.Should().Be("intent"); - result.As().TextScore.Should().Be(0.5); - result.As().Score.Should().Be(0); + result.As().TextScore.Should().Be(0.5); + result.As().Score.Should().Be(0); } } @@ -271,8 +272,8 @@ public static async Task WithLabeledIntentScore() using (var luis = builder.Build()) { var result = await luis.TestAsync(test).ConfigureAwait(false); - result.Should().BeOfType(typeof(ScoredLabeledUtterance)); - result.As().Score.Should().Be(0.42); + result.Should().BeOfType(typeof(PredictedLabeledUtterance)); + result.As().Score.Should().Be(0.42); } } @@ -350,8 +351,8 @@ public static async Task WithEntityScore() { var result = await luis.TestAsync(test).ConfigureAwait(false); result.Entities.Count.Should().Be(1); - result.Entities[0].Should().BeOfType(typeof(ScoredEntity)); - result.Entities[0].As().Score.Should().Be(0.42); + result.Entities[0].Should().BeOfType(typeof(PredictedEntity)); + result.Entities[0].As().Score.Should().Be(0.42); } } diff --git a/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs b/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs index 27f28dc..ed44077 100644 --- a/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs +++ b/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs @@ -8,6 +8,7 @@ namespace NLU.DevOps.Luis using System.Linq; using System.Threading; using System.Threading.Tasks; + using Core; using Microsoft.Azure.CognitiveServices.Language.LUIS.Runtime.Models; using Models; using Newtonsoft.Json.Linq; @@ -109,7 +110,7 @@ Entity getEntity(string entityType, JToken entityJson, JToken entityMetadata) } return score.HasValue - ? new ScoredEntity(modifiedEntityType, entityValue, matchText, matchIndex, score.Value) + ? new PredictedEntity(modifiedEntityType, entityValue, matchText, matchIndex, score.Value) : new Entity(modifiedEntityType, entityValue, matchText, matchIndex); } @@ -175,11 +176,13 @@ private LabeledUtterance LuisResultToLabeledUtterance(SpeechPredictionResponse s mappedTypes)? .ToList(); + var query = speechPredictionResponse.PredictionResponse.Query; var intentData = default(Intent); speechPredictionResponse.PredictionResponse.Prediction.Intents?.TryGetValue(intent, out intentData); + var context = LabeledUtteranceContext.CreateDefault(); return (intentData != null && intentData.Score.HasValue) || Math.Abs(speechPredictionResponse.TextScore) > Epsilon - ? new ScoredLabeledUtterance(speechPredictionResponse.PredictionResponse.Query, intent, intentData?.Score ?? 0, speechPredictionResponse.TextScore, entities) - : new LabeledUtterance(speechPredictionResponse.PredictionResponse.Query, intent, entities); + ? new PredictedLabeledUtterance(query, intent, intentData?.Score ?? 0, speechPredictionResponse.TextScore, entities, context) + : new LabeledUtterance(query, intent, entities); } } } diff --git a/src/NLU.DevOps.ModelPerformance.Tests/JTokenExtensionsTests.cs b/src/NLU.DevOps.ModelPerformance.Tests/JTokenExtensionsTests.cs new file mode 100644 index 0000000..7f5830f --- /dev/null +++ b/src/NLU.DevOps.ModelPerformance.Tests/JTokenExtensionsTests.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NLU.DevOps.ModelPerformance.Tests +{ + using FluentAssertions; + using Newtonsoft.Json.Linq; + using NUnit.Framework; + + [TestFixture] + internal static class JTokenExtensionsTests + { + [Test] + public static void ContainsSubtreeNull() + { + JTokenExtensions.ContainsSubtree(null, JValue.CreateNull()).Should().BeTrue(); + JTokenExtensions.ContainsSubtree(JValue.CreateNull(), null).Should().BeFalse(); + } + + [Test] + [TestCase("null", "null")] + [TestCase("42", "42")] + [TestCase("\"foo\"", "\"foo\"")] + [TestCase("[1]", "[1]")] + [TestCase("[1,2]", "[1,2]")] + [TestCase("[1,2]", "[2,1]")] + [TestCase("[1]", "[1,2]")] + [TestCase("[1,[2]]", "[1,[2]]")] + [TestCase("[1,[2]]", "[1,[2,3]]")] + [TestCase("{\"foo\": 42}", "{\"foo\": 42}")] + [TestCase("{\"foo\": 42}", "{\"foo\": 42, \"bar\": 7}")] + public static void ContainsSubtreeTrue(string expectedJson, string actualJson) + { + var expected = JToken.Parse(expectedJson); + var actual = JToken.Parse(actualJson); + expected.ContainsSubtree(actual).Should().BeTrue(); + } + + [Test] + [TestCase("null", "42")] + [TestCase("42", "\"foo\"")] + [TestCase("[1]", "[null]")] + [TestCase("[1,2]", "[1]")] + [TestCase("[1,[2,3]]", "[1,[2]]")] + [TestCase("{\"foo\": 42}", "{\"foo\": null}")] + [TestCase("{\"foo\": 42, \"bar\": 7}", "{\"foo\": 42}")] + public static void ContainsSubtreeFalse(string expectedJson, string actualJson) + { + var expected = JToken.Parse(expectedJson); + var actual = JToken.Parse(actualJson); + expected.ContainsSubtree(actual).Should().BeFalse(); + } + + [Test] + public static void EvaluateNull() + { + JTokenExtensions.Evaluate(null, null).Should().BeNull(); + } + + [Test] + [TestCase("csharp(1+1)", "2")] + [TestCase("[csharp(1+1)]", "[2]")] + [TestCase("{\"foo\":csharp(1+1)}", "{\"foo\":2}")] + [TestCase("{\"foo\":[csharp(1+1)]}", "{\"foo\":[2]}")] + public static void Evaluate(string actualJson, string expectedJson) + { + var expected = JToken.Parse(expectedJson); + var actual = JToken.Parse(actualJson); + JToken.DeepEquals(expected.Evaluate(null), actual).Should().BeTrue(); + } + + [Test] + [TestCase("null")] + [TestCase("42")] + [TestCase("\"foo\"")] + [TestCase("[]")] + [TestCase("[42]")] + [TestCase("{}")] + [TestCase("{\"foo\":42}")] + [TestCase("{\"foo\":[42, \"foo\"]}")] + public static void EvaluateNoChange(string jsonString) + { + var json = JToken.Parse(jsonString); + json.Evaluate(null).Should().BeSameAs(json); + } + + [TestCase("csharp(x+1)", "3")] + [TestCase("[csharp(x+1)]", "[3]")] + [TestCase("{\"foo\":csharp(x+1)}", "{\"foo\":3}")] + [TestCase("{\"foo\":[csharp(x+1)]}", "{\"foo\":[3]}")] + public static void EvaluateWithGlobals(string actualJson, string expectedJson) + { + var globals = new { x = 2 }; + var expected = JToken.Parse(expectedJson); + var actual = JToken.Parse(actualJson); + JToken.DeepEquals(expected.Evaluate(globals), actual).Should().BeTrue(); + } + } +} diff --git a/src/NLU.DevOps.ModelPerformance.Tests/TestCaseSourceTests.cs b/src/NLU.DevOps.ModelPerformance.Tests/TestCaseSourceTests.cs index a51f44f..bf82b2e 100644 --- a/src/NLU.DevOps.ModelPerformance.Tests/TestCaseSourceTests.cs +++ b/src/NLU.DevOps.ModelPerformance.Tests/TestCaseSourceTests.cs @@ -7,6 +7,7 @@ namespace NLU.DevOps.ModelPerformance.Tests using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; + using Core; using FluentAssertions; using Models; using Newtonsoft.Json.Linq; @@ -395,10 +396,11 @@ public static void GetNLUCompareResultsTextStatistics( { var expectedUtterance = new LabeledUtterance(expected, null, null); var actualUtterance = new LabeledUtterance(actual, null, null); + TestCaseSource.ShouldCompareText = true; var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - true); + new[] { actualUtterance }); + TestCaseSource.ShouldCompareText = false; compareResults.Statistics.Text.TruePositive.Should().Be(truePositive); compareResults.Statistics.Text.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Text.FalsePositive.Should().Be(falsePositive); @@ -427,8 +429,7 @@ public static void GetNLUCompareResultsIntentStatistics( var actualUtterance = new LabeledUtterance(null, actual, null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); compareResults.Statistics.Intent.TruePositive.Should().Be(truePositive); compareResults.Statistics.Intent.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Intent.FalsePositive.Should().Be(falsePositive); @@ -465,8 +466,7 @@ public static void GetNLUCompareResultsEntityStatistics( var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); compareResults.Statistics.Entity.TruePositive.Should().Be(truePositive); compareResults.Statistics.Entity.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Entity.FalsePositive.Should().Be(falsePositive); @@ -504,8 +504,7 @@ public static void GetNLUCompareResultsEntityValueStatistics( var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); compareResults.Statistics.EntityValue.TruePositive.Should().Be(truePositive); compareResults.Statistics.EntityValue.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.EntityValue.FalsePositive.Should().Be(falsePositive); @@ -531,8 +530,7 @@ public static void GetNLUCompareResultsFalsePositiveEntityDifferentType() var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); compareResults.Statistics.Entity.TruePositive.Should().Be(0); compareResults.Statistics.Entity.TrueNegative.Should().Be(0); compareResults.Statistics.Entity.FalsePositive.Should().Be(1); @@ -558,8 +556,7 @@ public static void GetNLUCompareResultsDefaultScores() var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); compareResults.TestCases.Select(t => t.Score).Should().AllBeEquivalentTo(0); } @@ -567,11 +564,12 @@ public static void GetNLUCompareResultsDefaultScores() public static void GetNLUCompareResultsExtractsIntentAndTextScore() { var expectedUtterance = new LabeledUtterance(null, null, null); - var actualUtterance = new ScoredLabeledUtterance(null, null, 0.5, 0.1, null); + var actualUtterance = new PredictedLabeledUtterance(null, null, 0.5, 0.1, null, null); + TestCaseSource.ShouldCompareText = true; var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - true); + new[] { actualUtterance }); + TestCaseSource.ShouldCompareText = false; var intentTestCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Intent); intentTestCase.Should().NotBeNull(); intentTestCase.Score.Should().Be(0.5); @@ -586,13 +584,12 @@ public static void GetNLUCompareResultsExtractsEntityScore() var entityType = Guid.NewGuid().ToString(); var matchText = Guid.NewGuid().ToString(); var expectedEntity = new[] { new Entity(entityType, null, matchText, 0) }; - var actualEntity = new[] { new ScoredEntity(entityType, null, matchText, 0, 0.5) }; + var actualEntity = new[] { new PredictedEntity(entityType, null, matchText, 0, 0.5) }; var expectedUtterance = new LabeledUtterance(null, null, expectedEntity); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); var testCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Entity); testCase.Should().NotBeNull(); testCase.Score.Should().Be(0.5); @@ -603,13 +600,12 @@ public static void GetNLUCompareResultsExtractsFalsePositiveEntityScore() { var entityType = Guid.NewGuid().ToString(); var matchText = Guid.NewGuid().ToString(); - var actualEntity = new[] { new ScoredEntity(entityType, null, matchText, 0, 0.5) }; + var actualEntity = new[] { new PredictedEntity(entityType, null, matchText, 0, 0.5) }; var expectedUtterance = new LabeledUtterance(null, null, null); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); var testCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Entity); testCase.Should().NotBeNull(); testCase.Score.Should().Be(0.5); @@ -622,8 +618,7 @@ public static void UsesIndexAsUtteranceId() var actualUtterance = new LabeledUtterance(null, "Greeting", null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance, expectedUtterance }, - new[] { actualUtterance, actualUtterance }, - false); + new[] { actualUtterance, actualUtterance }); compareResults.TestCases.Count.Should().Be(4); compareResults.TestCases.Where(t => t.UtteranceId == "0").Count().Should().Be(2); compareResults.TestCases.Where(t => t.UtteranceId == "1").Count().Should().Be(2); @@ -637,8 +632,7 @@ public static void UsesInputUtteranceId() var actualUtterance = new LabeledUtterance(null, "Greeting", null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, - new[] { actualUtterance }, - false); + new[] { actualUtterance }); compareResults.TestCases.Count.Should().Be(2); compareResults.TestCases.Where(t => t.UtteranceId == utteranceId).Count().Should().Be(2); } diff --git a/src/NLU.DevOps.ModelPerformance.Tests/appsettings.json b/src/NLU.DevOps.ModelPerformance.Tests/appsettings.json index b0e9884..235a350 100644 --- a/src/NLU.DevOps.ModelPerformance.Tests/appsettings.json +++ b/src/NLU.DevOps.ModelPerformance.Tests/appsettings.json @@ -1,4 +1,5 @@ { "expected": "models/expectedUtterances.json", - "actual": "models/actualUtterances.json" + "actual": "models/actualUtterances.json", + "evaluate": true } diff --git a/src/NLU.DevOps.ModelPerformance.Tests/models/actualUtterances.json b/src/NLU.DevOps.ModelPerformance.Tests/models/actualUtterances.json index 05b3d81..956c559 100644 --- a/src/NLU.DevOps.ModelPerformance.Tests/models/actualUtterances.json +++ b/src/NLU.DevOps.ModelPerformance.Tests/models/actualUtterances.json @@ -73,5 +73,28 @@ ], "intent": "AddAccount", "text": "create a new account for Bob with email bob@example.com" + }, + { + "entities": [ + { + "matchText": "two", + "entityValue": 2 + } + ], + "intent": "None", + "text": "add two salads to my order" + }, + { + "entities": [ + { + "matchText": "tomorrow", + "entityValue": "1970-01-02T00:00:00Z" + } + ], + "intent": "None", + "text": "what's on my calendar tomorrow", + "context": { + "timestamp": "1970-01-01T00:00:00Z" + } } ] diff --git a/src/NLU.DevOps.ModelPerformance.Tests/models/expectedUtterances.json b/src/NLU.DevOps.ModelPerformance.Tests/models/expectedUtterances.json index 0d7051e..84720c5 100644 --- a/src/NLU.DevOps.ModelPerformance.Tests/models/expectedUtterances.json +++ b/src/NLU.DevOps.ModelPerformance.Tests/models/expectedUtterances.json @@ -71,5 +71,25 @@ ], "intent": "AddAccount", "text": "create a new account for Bob with email bob@example.com" + }, + { + "entities": [ + { + "matchText": "two", + "entityValue": "csharp(1+1)" + } + ], + "intent": "None", + "text": "add two salads to my order", + }, + { + "entities": [ + { + "matchText": "tomorrow", + "entityValue": "csharp(DateTimeOffset.Parse(Timestamp).AddDays(1).Date.ToString(\"yyyy-MM-ddTHH:mm:ssZ\"))" + } + ], + "intent": "None", + "text": "what's on my calendar tomorrow" } ] diff --git a/src/NLU.DevOps.ModelPerformance/ConfigurationConstants.cs b/src/NLU.DevOps.ModelPerformance/ConfigurationConstants.cs index 9acf465..fc1eef1 100644 --- a/src/NLU.DevOps.ModelPerformance/ConfigurationConstants.cs +++ b/src/NLU.DevOps.ModelPerformance/ConfigurationConstants.cs @@ -27,5 +27,10 @@ public static class ConfigurationConstants /// The test label key. /// public const string TestLabelKey = "testLabel"; + + /// + /// A Boolean value that signals whether or not inline evaluation of scripts should be performed. + /// + public const string EvaluateKey = "evaluate"; } } diff --git a/src/NLU.DevOps.ModelPerformance/JTokenExtensions.cs b/src/NLU.DevOps.ModelPerformance/JTokenExtensions.cs new file mode 100644 index 0000000..e7f6acc --- /dev/null +++ b/src/NLU.DevOps.ModelPerformance/JTokenExtensions.cs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NLU.DevOps.ModelPerformance +{ + using System; + using System.Linq; + using System.Text.RegularExpressions; + using System.Threading.Tasks; + using Logging; + using Microsoft.CodeAnalysis.CSharp.Scripting; + using Microsoft.CodeAnalysis.Scripting; + using Microsoft.Extensions.Logging; + using Newtonsoft.Json.Linq; + + internal static class JTokenExtensions + { + private static readonly Regex CSharpScriptRegex = new Regex(@"csharp\((.*)\)"); + + private static ILogger Logger => LazyLogger.Value; + + private static Lazy LazyLogger { get; } = new Lazy(() => ApplicationLogger.LoggerFactory.CreateLogger(typeof(JTokenExtensions))); + + public static JToken Evaluate(this JToken json, object globals) + { + try + { + return json.EvaluateAsync(globals).Result; + } + catch (Exception ex) + { + Logger.LogWarning(ex, ex.Message); + return json; + } + } + + public static bool ContainsSubtree(this JToken expected, JToken actual) + { + if (expected == null) + { + return true; + } + + if (actual == null) + { + return false; + } + + switch (expected) + { + case JObject expectedObject: + var actualObject = actual as JObject; + if (actualObject == null) + { + return false; + } + + foreach (var expectedProperty in expectedObject.Properties()) + { + var actualProperty = actualObject.Property(expectedProperty.Name, StringComparison.Ordinal); + if (!ContainsSubtree(expectedProperty.Value, actualProperty?.Value)) + { + return false; + } + } + + return true; + case JArray expectedArray: + var actualArray = actual as JArray; + if (actualArray == null) + { + return false; + } + + foreach (var expectedItem in expectedArray) + { + // Order is not asserted + if (!actualArray.Any(actualItem => ContainsSubtree(expectedItem, actualItem))) + { + return false; + } + } + + return true; + default: + return JToken.DeepEquals(expected, actual); + } + } + + private static async Task EvaluateAsync(this JToken json, object globals) + { + if (json == null) + { + return null; + } + + switch (json) + { + case JObject jsonObject: + var updatedObject = default(JObject); + foreach (var jsonProperty in jsonObject.Properties()) + { + var propertyValue = jsonObject.Property(jsonProperty.Name, StringComparison.Ordinal); + var evaluatedValue = await EvaluateAsync(propertyValue.Value, globals).ConfigureAwait(false); + if (evaluatedValue != propertyValue.Value) + { + updatedObject = updatedObject ?? (JObject)jsonObject.DeepClone(); + updatedObject[jsonProperty.Name] = evaluatedValue; + } + } + + return updatedObject ?? jsonObject; + case JArray jsonArray: + var updatedArray = default(JArray); + for (var i = 0; i < jsonArray.Count; ++i) + { + var evaluatedValue = await EvaluateAsync(jsonArray[i], globals).ConfigureAwait(false); + if (evaluatedValue != jsonArray[i]) + { + updatedArray = updatedArray ?? (JArray)jsonArray.DeepClone(); + updatedArray[i] = evaluatedValue; + } + } + + return updatedArray ?? jsonArray; + case JValue jsonValue: + if (jsonValue.Type == JTokenType.String) + { + var scriptMatch = CSharpScriptRegex.Match(jsonValue.Value()); + if (scriptMatch.Success) + { + var script = scriptMatch.Groups[1].Value; + var result = await EvaluateScriptAsync(script, globals).ConfigureAwait(false); + return JToken.FromObject(result); + } + } + + return json; + default: + return json; + } + } + + private static async Task EvaluateScriptAsync(string script, object globals) + { + var options = ScriptOptions.Default.WithImports("System"); + try + { + return await CSharpScript.EvaluateAsync(script, options, globals).ConfigureAwait(false); + } + catch (Exception ex) + { + throw new Exception($"Failed to evaluate script '{script}'.", ex); + } + } + } +} diff --git a/src/NLU.DevOps.ModelPerformance/NLU.DevOps.ModelPerformance.csproj b/src/NLU.DevOps.ModelPerformance/NLU.DevOps.ModelPerformance.csproj index 747204d..26333fd 100644 --- a/src/NLU.DevOps.ModelPerformance/NLU.DevOps.ModelPerformance.csproj +++ b/src/NLU.DevOps.ModelPerformance/NLU.DevOps.ModelPerformance.csproj @@ -16,6 +16,7 @@ + diff --git a/src/NLU.DevOps.ModelPerformance/ScoredEntity.cs b/src/NLU.DevOps.ModelPerformance/ScoredEntity.cs deleted file mode 100644 index 122eda5..0000000 --- a/src/NLU.DevOps.ModelPerformance/ScoredEntity.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace NLU.DevOps.ModelPerformance -{ - using Models; - using Newtonsoft.Json.Linq; - - /// - /// Entity appearing in utterance with confidence score. - /// - public class ScoredEntity : Entity - { - /// - /// Initializes a new instance of the class. - /// - /// Entity type name. - /// Entity value, generally a canonical form of the entity. - /// Matching text in the utterance. - /// Occurrence index of matching token in the utterance. - /// Confidence score for the entity. - public ScoredEntity(string entityType, JToken entityValue, string matchText, int matchIndex, double score) - : base(entityType, entityValue, matchText, matchIndex) - { - this.Score = score; - } - - /// - /// Gets the confidence score for the entity. - /// - public double Score { get; } - } -} diff --git a/src/NLU.DevOps.ModelPerformance/ScoredLabeledUtterance.cs b/src/NLU.DevOps.ModelPerformance/ScoredLabeledUtterance.cs deleted file mode 100644 index 5332cbd..0000000 --- a/src/NLU.DevOps.ModelPerformance/ScoredLabeledUtterance.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace NLU.DevOps.ModelPerformance -{ - using System.Collections.Generic; - using System.Linq; - using Models; - - /// - /// Labeled utterance with confidence score. - /// - public class ScoredLabeledUtterance : LabeledUtterance - { - /// - /// Initializes a new instance of the class. - /// - /// Text of the utterance. - /// Intent of the utterance. - /// Confidence score for the intent label. - /// Confidence score for speech-to-text. - /// Entities referenced in the utterance. - public ScoredLabeledUtterance(string text, string intent, double score, double textScore, IReadOnlyList entities) - : base(text, intent, entities) - { - this.Score = score; - this.TextScore = textScore; - } - - /// - /// Gets the confidence score for the intent label. - /// - public double Score { get; } - - /// - /// Gets the confidence score for speech-to-text. - /// - public double TextScore { get; } - - /// - /// Gets the entities referenced in the utterance. - /// - public new IReadOnlyList Entities => base.Entities?.OfType().ToList(); - } -} diff --git a/src/NLU.DevOps.ModelPerformance/TestCaseSource.cs b/src/NLU.DevOps.ModelPerformance/TestCaseSource.cs index 85500ee..6b67e10 100644 --- a/src/NLU.DevOps.ModelPerformance/TestCaseSource.cs +++ b/src/NLU.DevOps.ModelPerformance/TestCaseSource.cs @@ -24,6 +24,21 @@ public static class TestCaseSource private const string AppSettingsPath = "appsettings.json"; private const string AppSettingsLocalPath = "appsettings.local.json"; + /// + /// Gets or sets a flag signaling whether text comparison tests should be run. + /// + public static bool? ShouldCompareText { get; set; } + + /// + /// Gets or sets a flag signaling whether inline scripts should be evaluated. + /// + public static bool? ShouldEvaluate { get; set; } + + /// + /// Gets or sets the test label value. + /// + public static string TestLabel { get; set; } + /// /// Gets the passing tests. /// @@ -40,7 +55,14 @@ public static class TestCaseSource .Where(IsFalse) .Select(ToTestCaseData); - private static string TestLabel => TestContext.Parameters.Get(ConfigurationConstants.TestLabelKey) ?? Configuration[ConfigurationConstants.TestLabelKey]; + private static bool ShouldCompareTextSetting => + GetConfigurationBoolean(ConfigurationConstants.CompareTextKey, ShouldCompareText); + + private static bool ShouldEvaluateSetting => + GetConfigurationBoolean(ConfigurationConstants.EvaluateKey, ShouldEvaluate); + + private static string TestLabelSetting => + GetConfiguration(ConfigurationConstants.TestLabelKey, TestLabel); private static IConfiguration Configuration { get; } = new ConfigurationBuilder() .AddJsonFile(AppSettingsPath, true) @@ -57,11 +79,9 @@ public static class TestCaseSource /// The test cases. /// Expected utterances. /// Actual utterances. - /// Signals whether to generate text comparison test cases. public static NLUCompareResults GetNLUCompareResults( IReadOnlyList expectedUtterances, - IReadOnlyList actualUtterances, - bool compareText) + IReadOnlyList actualUtterances) { if (expectedUtterances.Count != actualUtterances.Count) { @@ -82,7 +102,7 @@ string getUtteranceId(LabeledUtterance utterance, int index) var testCases = zippedUtterances.Select(ToIntentTestCase) .Concat(zippedUtterances.SelectMany(ToEntityTestCases)); - if (compareText) + if (ShouldCompareTextSetting) { testCases = testCases.Concat(zippedUtterances.Select(ToTextTestCase)); } @@ -96,7 +116,7 @@ internal static TestCase ToTextTestCase(LabeledUtterancePair pair) var actualUtterance = pair.Actual; var expected = expectedUtterance.Text; var actual = actualUtterance.Text; - var score = actualUtterance is ScoredLabeledUtterance scoredUtterance + var score = actualUtterance is PredictedLabeledUtterance scoredUtterance ? scoredUtterance.TextScore : 0; @@ -158,7 +178,7 @@ internal static TestCase ToIntentTestCase(LabeledUtterancePair pair) { var expectedUtterance = pair.Expected; var actualUtterance = pair.Actual; - var score = actualUtterance is ScoredLabeledUtterance scoredUtterance + var score = actualUtterance is PredictedLabeledUtterance scoredUtterance ? scoredUtterance.Score : 0; @@ -229,6 +249,9 @@ internal static IEnumerable ToEntityTestCases(LabeledUtterancePair pai var text = expectedUtterance.Text; var expected = expectedUtterance.Entities; var actual = actualUtterance.Entities; + var globals = actualUtterance is PredictedLabeledUtterance predictedUtterance + ? predictedUtterance.Context + : null; if ((expected == null || expected.Count == 0) && (actual == null || actual.Count == 0)) { @@ -264,7 +287,7 @@ bool isEntityValueMatch(Entity expectedEntity, Entity actualEntity) { /* Required case to support NLU providers that do not specify matched text */ return actualEntity.MatchText == null - && (EqualsNormalizedJson(expectedEntity.EntityValue, actualEntity.EntityValue) + && (Evaluate(expectedEntity.EntityValue, globals).ContainsSubtree(actualEntity.EntityValue) || EqualsNormalizedJson(expectedEntity.MatchText, actualEntity.EntityValue)); } @@ -279,7 +302,7 @@ bool isEntityValueMatch(Entity expectedEntity, Entity actualEntity) ? actual.FirstOrDefault(actualEntity => isEntityMatch(entity, actualEntity)) : null; - var score = matchedEntity is ScoredEntity scoredEntity + var score = matchedEntity is PredictedEntity scoredEntity ? scoredEntity.Score : 0; @@ -315,7 +338,7 @@ bool isEntityValueMatch(Entity expectedEntity, Entity actualEntity) if (entity.EntityValue != null && entity.EntityValue.Type != JTokenType.Null) { var formattedEntityValue = entity.EntityValue.ToString(Formatting.None); - if (!ContainsSubtree(entity.EntityValue, matchedEntity.EntityValue)) + if (!Evaluate(entity.EntityValue, globals).ContainsSubtree(matchedEntity.EntityValue)) { yield return FalseNegative( pair.UtteranceId, @@ -350,7 +373,7 @@ bool isEntityValueMatch(Entity expectedEntity, Entity actualEntity) { foreach (var entity in actual) { - var score = entity is ScoredEntity scoredEntity ? scoredEntity.Score : 0; + var score = entity is PredictedEntity scoredEntity ? scoredEntity.Score : 0; var entityValue = entity.MatchText ?? entity.EntityValue; if (expected == null || !expected.Any(expectedEntity => isEntityMatch(expectedEntity, entity))) { @@ -402,41 +425,22 @@ private static IReadOnlyList LoadTestCases() throw new InvalidOperationException("Could not find configuration for expected or actual utterances."); } - var compareTextString = TestContext.Parameters.Get(ConfigurationConstants.CompareTextKey); - var compareText = false; - if (compareTextString != null && bool.TryParse(compareTextString, out var parsedValue)) - { - compareText = parsedValue; - } - - var expected = Read(expectedPath); - var actual = Read(actualPath); - return GetNLUCompareResults(expected, actual, compareText).TestCases; + var expected = Read(expectedPath); + var actual = Read(actualPath); + return GetNLUCompareResults(expected, actual).TestCases; } - private static List Read(string path) + private static List Read(string path) { var serializer = JsonSerializer.CreateDefault(); serializer.Converters.Add(new LabeledUtteranceConverter()); + serializer.DateParseHandling = DateParseHandling.None; using (var jsonReader = new JsonTextReader(File.OpenText(path))) { - return serializer.Deserialize>(jsonReader); + return serializer.Deserialize>(jsonReader); } } - private static bool EqualsNormalizedJson(JToken x, JToken y) - { - // Entity is not a match if both values are null - if (x == null && y == null) - { - return false; - } - - return x?.Type == JTokenType.String - ? EqualsNormalizedJson(x.Value(), y) - : false; - } - private static bool EqualsNormalizedJson(string x, JToken y) { // Entity is not a match if both values are null @@ -467,57 +471,19 @@ string normalize(string s) return string.Equals(normalize(x), normalize(y), StringComparison.OrdinalIgnoreCase); } - private static bool ContainsSubtree(JToken expected, JToken actual) + private static JToken Evaluate(JToken token, object globals) { - if (expected == null) - { - return true; - } - - if (actual == null) - { - return false; - } - - switch (expected) - { - case JObject expectedObject: - var actualObject = actual as JObject; - if (actualObject == null) - { - return false; - } - - foreach (var expectedProperty in expectedObject.Properties()) - { - var actualProperty = actualObject.Property(expectedProperty.Name, StringComparison.Ordinal); - if (!ContainsSubtree(expectedProperty.Value, actualProperty?.Value)) - { - return false; - } - } - - return true; - case JArray expectedArray: - var actualArray = actual as JArray; - if (actualArray == null) - { - return false; - } + return ShouldEvaluateSetting ? token.Evaluate(globals) : token; + } - foreach (var expectedItem in expectedArray) - { - // Order is not asserted - if (!actualArray.Any(actualItem => ContainsSubtree(expectedItem, actualItem))) - { - return false; - } - } + private static bool GetConfigurationBoolean(string key, bool? overrideValue) + { + return overrideValue ?? (bool.TryParse(GetConfiguration(key, null), out var flag) ? flag : false); + } - return true; - default: - return JToken.DeepEquals(expected, actual); - } + private static string GetConfiguration(string key, string overrideValue) + { + return overrideValue ?? TestContext.Parameters.Get(key) ?? Configuration[key]; } private static TestCase TruePositive( @@ -628,7 +594,7 @@ private static TestCase CreateTestCase( string because, IEnumerable categories) { - var testLabel = TestLabel != null ? $"[{TestLabel}] " : string.Empty; + var testLabel = TestLabelSetting != null ? $"[{TestLabelSetting}] " : string.Empty; var categoriesWithGroup = categories; if (group != null) {