Adds support for dynamic entity value matching

src/NLU.DevOps.CommandLine/Compare/CompareCommand.cs

@@ -7,8 +7,8 @@ namespace NLU.DevOps.CommandLine.Compare
     using System.Globalization;
     using System.IO;
     using System.Linq;
+    using Core;
     using ModelPerformance;
-    using Models;
     using NUnitLite;
     using static Serializer;
 
@@ -23,6 +23,7 @@ public static int Run(CompareOptions options)
                 (ConfigurationConstants.ExpectedUtterancesPathKey, options.ExpectedUtterancesPath),
                 (ConfigurationConstants.ActualUtterancesPathKey, options.ActualUtterancesPath),
                 (ConfigurationConstants.CompareTextKey, options.CompareText.ToString(CultureInfo.InvariantCulture)),
+                (ConfigurationConstants.EvaluateKey, options.Evaluate.ToString(CultureInfo.InvariantCulture)),
                 (ConfigurationConstants.TestLabelKey, options.TestLabel));
 
             var arguments = new List<string> { $"-p:{parameters}" };
@@ -34,8 +35,11 @@ public static int Run(CompareOptions options)
             if (options.Metadata)
             {
                 var expectedUtterances = Read<List<CompareLabeledUtterance>>(options.ExpectedUtterancesPath);
-                var actualUtterances = Read<List<ScoredLabeledUtterance>>(options.ActualUtterancesPath);
-                var compareResults = TestCaseSource.GetNLUCompareResults(expectedUtterances, actualUtterances, options.CompareText);
+                var actualUtterances = Read<List<PredictedLabeledUtterance>>(options.ActualUtterancesPath);
+                TestCaseSource.ShouldCompareText = options.CompareText;
+                TestCaseSource.ShouldEvaluate = options.Evaluate;
+                TestCaseSource.TestLabel = options.TestLabel;
+                var compareResults = TestCaseSource.GetNLUCompareResults(expectedUtterances, actualUtterances);
                 var metadataPath = options.OutputFolder != null ? Path.Combine(options.OutputFolder, TestMetadataFileName) : TestMetadataFileName;
                 var statisticsPath = options.OutputFolder != null ? Path.Combine(options.OutputFolder, TestStatisticsFileName) : TestStatisticsFileName;
                 Write(metadataPath, compareResults.TestCases);

src/NLU.DevOps.CommandLine/Compare/CompareOptions.cs

@@ -20,6 +20,9 @@ internal class CompareOptions
         [Option('m', "metadata", HelpText = "Return test case metadata in addition to NUnit test results.", Required = false)]
         public bool Metadata { get; set; }
 
+        [Option('x', "evaluate", HelpText = "Evaluate inline scripts.", Required = false)]
+        public bool Evaluate { get; set; }
+
         [Option('t', "text", HelpText = "Run text comparison test cases.", Required = false)]
         public bool CompareText { get; set; }
 

src/NLU.DevOps.CommandLine/Serializer.cs

@@ -17,6 +17,7 @@ public static T Read<T>(string path)
         {
             var serializer = JsonSerializer.CreateDefault();
             serializer.Converters.Add(new LabeledUtteranceConverter());
+            serializer.DateParseHandling = DateParseHandling.None;
             using (var jsonReader = new JsonTextReader(File.OpenText(path)))
             {
                 return serializer.Deserialize<T>(jsonReader);

src/NLU.DevOps.Core/LabeledUtteranceContext.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace NLU.DevOps.Core
+{
+    using System;
+    using System.Globalization;
+
+    /// <summary>
+    /// Labeled utterance context.
+    /// </summary>
+    public class LabeledUtteranceContext
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LabeledUtteranceContext"/> class.
+        /// </summary>
+        /// <param name="timestamp">Timestamp.</param>
+        public LabeledUtteranceContext(string timestamp)
+        {
+            this.Timestamp = timestamp;
+        }
+
+        private LabeledUtteranceContext()
+            : this(DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture))
+        {
+        }
+
+        /// <summary>
+        /// Gets the timestamp for the labeled utterance.
+        /// </summary>
+        public string Timestamp { get; }
+
+        /// <summary>
+        /// Creates default instance of <see cref="LabeledUtteranceContext"/>.
+        /// </summary>
+        /// <returns>The default instance.</returns>
+        public static LabeledUtteranceContext CreateDefault() => new LabeledUtteranceContext();
+    }
+}

src/NLU.DevOps.Luis.Shared/ScoredEntity.cs → src/NLU.DevOps.Core/PredictedEntity.cs

@@ -1,25 +1,25 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-namespace NLU.DevOps.Luis
+namespace NLU.DevOps.Core
 {
     using Models;
     using Newtonsoft.Json.Linq;
 
     /// <summary>
     /// Entity appearing in utterance with confidence score.
     /// </summary>
-    public class ScoredEntity : Entity
+    public class PredictedEntity : Entity
     {
         /// <summary>
-        /// Initializes a new instance of the <see cref="ScoredEntity"/> class.
+        /// Initializes a new instance of the <see cref="PredictedEntity"/> class.
         /// </summary>
         /// <param name="entityType">Entity type name.</param>
         /// <param name="entityValue">Entity value, generally a canonical form of the entity.</param>
         /// <param name="matchText">Matching text in the utterance.</param>
         /// <param name="matchIndex">Occurrence index of matching token in the utterance.</param>
         /// <param name="score">Confidence score for the entity.</param>
-        public ScoredEntity(string entityType, JToken entityValue, string matchText, int matchIndex, double score)
+        public PredictedEntity(string entityType, JToken entityValue, string matchText, int matchIndex, double score)
             : base(entityType, entityValue, matchText, matchIndex)
         {
             this.Score = score;

src/NLU.DevOps.Core/PredictedLabeledUtterance.cs

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace NLU.DevOps.Core
+{
+    using System.Collections.Generic;
+    using Models;
+    using Newtonsoft.Json;
+
+    /// <summary>
+    /// Labeled utterance with confidence score.
+    /// </summary>
+    public class PredictedLabeledUtterance : LabeledUtterance
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="PredictedLabeledUtterance"/> class.
+        /// </summary>
+        /// <param name="text">Text of the utterance.</param>
+        /// <param name="intent">Intent of the utterance.</param>
+        /// <param name="score">Confidence score for the intent label.</param>
+        /// <param name="textScore">Confidence score for speech-to-text.</param>
+        /// <param name="entities">Entities referenced in the utterance.</param>
+        /// <param name="context">Labeled utterance context.</param>
+        [JsonConstructor]
+        public PredictedLabeledUtterance(string text, string intent, double score, double textScore, IReadOnlyList<PredictedEntity> entities, LabeledUtteranceContext context)
+            : this(text, intent, score, textScore, (IReadOnlyList<Entity>)entities, context)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="PredictedLabeledUtterance"/> class.
+        /// </summary>
+        /// <param name="text">Text of the utterance.</param>
+        /// <param name="intent">Intent of the utterance.</param>
+        /// <param name="score">Confidence score for the intent label.</param>
+        /// <param name="textScore">Confidence score for speech-to-text.</param>
+        /// <param name="entities">Entities referenced in the utterance.</param>
+        /// <param name="context">Labeled utterance context.</param>
+        public PredictedLabeledUtterance(string text, string intent, double score, double textScore, IReadOnlyList<Entity> entities, LabeledUtteranceContext context)
+            : base(text, intent, entities)
+        {
+            this.Context = context;
+            this.Score = score;
+            this.TextScore = textScore;
+        }
+
+        /// <summary>
+        /// Gets the context of the labeled utterance.
+        /// </summary>
+        public LabeledUtteranceContext Context { get; }
+
+        /// <summary>
+        /// Gets the confidence score for the intent label.
+        /// </summary>
+        public double Score { get; }
+
+        /// <summary>
+        /// Gets the confidence score for speech-to-text.
+        /// </summary>
+        public double TextScore { get; }
+    }
+}

src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs

@@ -73,10 +73,14 @@ protected override async Task<LabeledUtterance> TestAsync(string utterance, Canc
                     {
                         var client = await this.GetSessionClientAsync(cancellationToken).ConfigureAwait(false);
                         var result = await client.DetectIntentAsync(sessionName, queryInput, cancellationToken).ConfigureAwait(false);
-                        return new LabeledUtterance(
+                        var context = LabeledUtteranceContext.CreateDefault();
+                        return new PredictedLabeledUtterance(
                             result.QueryResult.QueryText,
                             result.QueryResult.Intent.DisplayName,
-                            result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList());
+                            result.QueryResult.IntentDetectionConfidence,
+                            result.QueryResult.SpeechRecognitionConfidence,
+                            result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList(),
+                            context);
                     },
                     cancellationToken)
                 .ConfigureAwait(false);
@@ -107,10 +111,14 @@ protected override async Task<LabeledUtterance> TestSpeechAsync(string speechFil
                     {
                         var client = await this.GetSessionClientAsync(cancellationToken).ConfigureAwait(false);
                         var result = await client.DetectIntentAsync(request, cancellationToken).ConfigureAwait(false);
-                        return new LabeledUtterance(
+                        var context = LabeledUtteranceContext.CreateDefault();
+                        return new PredictedLabeledUtterance(
                             result.QueryResult.QueryText,
                             result.QueryResult.Intent.DisplayName,
-                            result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList());
+                            result.QueryResult.IntentDetectionConfidence,
+                            result.QueryResult.SpeechRecognitionConfidence,
+                            result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList(),
+                            context);
                     },
                     cancellationToken)
                 .ConfigureAwait(false);

src/NLU.DevOps.Lex/LexNLUTestClient.cs

@@ -94,10 +94,14 @@ protected override async Task<LabeledUtterance> TestAsync(string utterance, Canc
                 .Select(slot => new Entity(slot.Key, slot.Value, null, 0))
                 .ToArray();
 
-            return new LabeledUtterance(
+            var context = LabeledUtteranceContext.CreateDefault();
+            return new PredictedLabeledUtterance(
                 utterance,
                 postTextResponse.IntentName,
-                entities);
+                0,
+                0,
+                entities,
+                context);
         }
 
         /// <inheritdoc />
@@ -127,10 +131,14 @@ protected override async Task<LabeledUtterance> TestSpeechAsync(string speechFil
                         .ToArray()
                     : null;
 
-                return new LabeledUtterance(
+                var context = LabeledUtteranceContext.CreateDefault();
+                return new PredictedLabeledUtterance(
                     postContentResponse.InputTranscript,
                     postContentResponse.IntentName,
-                    slots);
+                    0,
+                    0,
+                    slots,
+                    context);
             }
         }
 

src/NLU.DevOps.Luis.Shared/NLU.DevOps.Luis.Shared.projitems

@@ -15,8 +15,6 @@
     <Compile Include="$(MSBuildThisFileDirectory)LuisTrainClient.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)LuisSettings.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)LuisNLUTrainClient.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)ScoredEntity.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)ScoredLabeledUtterance.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)LuisConfiguration.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ILuisConfiguration.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)TestLuisConfiguration.cs" />

src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs

@@ -7,6 +7,7 @@ namespace NLU.DevOps.Luis.Tests
     using System.Collections.Generic;
     using System.Threading;
     using System.Threading.Tasks;
+    using Core;
     using FluentAssertions;
     using FluentAssertions.Json;
     using Microsoft.Azure.CognitiveServices.Language.LUIS.Runtime.Models;
@@ -204,8 +205,8 @@ public static async Task TestSpeechWithTextScore()
                 var result = await luis.TestSpeechAsync(testFile).ConfigureAwait(false);
                 result.Text.Should().Be(test);
                 result.Intent.Should().Be("intent");
-                result.As<ScoredLabeledUtterance>().TextScore.Should().Be(0.5);
-                result.As<ScoredLabeledUtterance>().Score.Should().Be(0);
+                result.As<PredictedLabeledUtterance>().TextScore.Should().Be(0.5);
+                result.As<PredictedLabeledUtterance>().Score.Should().Be(0);
             }
         }
 
@@ -312,8 +313,8 @@ public static async Task WithLabeledIntentScore()
             using (var luis = builder.Build())
             {
                 var result = await luis.TestAsync(test).ConfigureAwait(false);
-                result.Should().BeOfType(typeof(ScoredLabeledUtterance));
-                result.As<ScoredLabeledUtterance>().Score.Should().Be(0.42);
+                result.Should().BeOfType(typeof(PredictedLabeledUtterance));
+                result.As<PredictedLabeledUtterance>().Score.Should().Be(0.42);
             }
         }
 
@@ -385,8 +386,8 @@ public static async Task WithEntityScore()
             {
                 var result = await luis.TestAsync(test).ConfigureAwait(false);
                 result.Entities.Count.Should().Be(1);
-                result.Entities[0].Should().BeOfType(typeof(ScoredEntity));
-                result.Entities[0].As<ScoredEntity>().Score.Should().Be(0.42);
+                result.Entities[0].Should().BeOfType(typeof(PredictedEntity));
+                result.Entities[0].As<PredictedEntity>().Score.Should().Be(0.42);
             }
         }
 

src/NLU.DevOps.Luis/LuisNLUTestClient.cs

@@ -141,16 +141,18 @@ Entity getEntity(EntityModel entity)
                 }
 
                 return entityScore.HasValue
-                    ? new ScoredEntity(entityType, entityValue, matchText, matchIndex, entityScore.Value)
+                    ? new PredictedEntity(entityType, entityValue, matchText, matchIndex, entityScore.Value)
                     : new Entity(entityType, entityValue, matchText, matchIndex);
             }
 
+            var query = speechLuisResult.LuisResult.Query;
             var intent = speechLuisResult.LuisResult.TopScoringIntent?.Intent;
             var score = speechLuisResult.LuisResult.TopScoringIntent?.Score;
             var entities = speechLuisResult.LuisResult.Entities?.Select(getEntity).ToList();
+            var context = LabeledUtteranceContext.CreateDefault();
             return !score.HasValue && Math.Abs(speechLuisResult.TextScore) < Epsilon
-                ? new LabeledUtterance(speechLuisResult.LuisResult.Query, intent, entities)
-                : new ScoredLabeledUtterance(speechLuisResult.LuisResult.Query, intent, score ?? 0, speechLuisResult.TextScore, entities);
+                ? new LabeledUtterance(query, intent, entities)
+                : new PredictedLabeledUtterance(query, intent, score ?? 0, speechLuisResult.TextScore, entities, context);
         }
     }
 }