From 5a3e5cfbd3ae5ff6566a7f9a6c7e94217468f72e Mon Sep 17 00:00:00 2001 From: Eric Rozell Date: Thu, 10 Oct 2019 22:42:52 -0400 Subject: [PATCH] Convert timestamp to opt-in feature Not convinced that we need to do this, but we can spare a few bytes in the NLU results files by only including the timestamp when requested via CLI argument. --- docs/Test.md | 5 +++ docs/UtteranceExtensions.md | 4 +-- .../Test/TestCommand.cs | 34 ++++++++++++++++++- .../Test/TestOptions.cs | 3 ++ .../DialogflowNLUTestClient.cs | 6 ++-- .../LexNLUTestClientTests.cs | 4 --- src/NLU.DevOps.Lex/LexNLUTestClient.cs | 6 ++-- .../LuisNLUTestClientTests.cs | 1 - src/NLU.DevOps.Luis/LuisNLUTestClient.cs | 3 +- .../LuisNLUTestClientTests.cs | 1 - src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs | 3 +- 11 files changed, 49 insertions(+), 21 deletions(-) diff --git a/docs/Test.md b/docs/Test.md index be82c64..57c993e 100644 --- a/docs/Test.md +++ b/docs/Test.md @@ -140,6 +140,11 @@ See [Caching speech-to-text transcriptions](#caching-speech-to-text-transcriptio This is currently only used for LUIS, see the section on LUIS prebuilt entities in [Configuring prebuilt entities](LuisModelConfiguration.md#configuring-prebuilt-entities). +### `--timestamp` +(Optional) Signals whether to add a timestamp to each NLU test result. + +See the documentation on the [`timestamp` property](UtteranceExtensions.md#returning-timestamps-for-each-query) for more details. + ### `-i, --include` (Optional) Path to custom NLU provider DLL. See documentation about [Specifying the include path](https://github.com/microsoft/NLU.DevOps/blob/master/docs/CliExtensions.md#specifying-the-include-path) for more details. diff --git a/docs/UtteranceExtensions.md b/docs/UtteranceExtensions.md index c976b8e..781629d 100644 --- a/docs/UtteranceExtensions.md +++ b/docs/UtteranceExtensions.md @@ -22,7 +22,7 @@ When an NLU provider in NLU.DevOps returns a prediction result, the value will b ``` In this case, the intent confidence score was `0.99` and the text transcription confidence score was `0.95`. This is useful context when debugging false predictions, as a low confidence score may indicate that the model could be improved with more training examples. The recognized `genre` entity also includes a confidence score of `0.80`, although it should be noted that only the LUIS provider currently returns confidence score for entity types trained from examples. -## Labeled utterance timestamps +## Returning timestamps for each query When analyzing results for a set of NLU predictions, it is often important context to understand when the test was run. For example, for Dialogflow `date` and `time` entities, the service only returns a date time string, and no indication of what token(s) triggered that entity to be recognized. For example, the result from a query like `"Call a taxi in 15 minutes"` may look like the following: ```json @@ -38,7 +38,7 @@ When analyzing results for a set of NLU predictions, it is often important conte "timestamp": "2020-01-01T00:00:00-04:00" } ``` -Without the context provided by the `timestamp` property, we wouldn't be able to make any assertion about the correctness of the `entityValue` property for time. Currently, LUIS, Lex, and Dialogflow return a timestamp for each prediction result. +Without the context provided by the `timestamp` property, we wouldn't be able to make any assertion about the correctness of the `entityValue` property for time. Currently, you must specify the [`--timestamp`](Test.md#--timestamp) option to ensure a timestamp is assigned to each NLU prediction result. ## Adjusting entity compare results diff --git a/src/NLU.DevOps.CommandLine/Test/TestCommand.cs b/src/NLU.DevOps.CommandLine/Test/TestCommand.cs index 666c8ac..4a65be7 100644 --- a/src/NLU.DevOps.CommandLine/Test/TestCommand.cs +++ b/src/NLU.DevOps.CommandLine/Test/TestCommand.cs @@ -6,7 +6,9 @@ namespace NLU.DevOps.CommandLine.Test using System; using System.Collections.Generic; using System.IO; + using System.Threading; using System.Threading.Tasks; + using Core; using Models; using Newtonsoft.Json.Linq; using static Serializer; @@ -31,7 +33,8 @@ public override int Main() protected override INLUTestClient CreateNLUTestClient() { - return NLUClientFactory.CreateTestInstance(this.Options, this.Configuration, this.Options.SettingsPath); + var client = NLUClientFactory.CreateTestInstance(this.Options, this.Configuration, this.Options.SettingsPath); + return this.Options.Timestamp ? new TimestampNLUTestClient(client) : client; } private static void EnsureDirectory(string filePath) @@ -129,5 +132,34 @@ private void SaveTranscriptions() yield return (query, speechFile); } } + + private class TimestampNLUTestClient : INLUTestClient + { + public TimestampNLUTestClient(INLUTestClient client) + { + this.Client = client; + } + + private INLUTestClient Client { get; } + + public async Task TestAsync(JToken query, CancellationToken cancellationToken) + { + var timestamp = DateTimeOffset.Now; + var result = await this.Client.TestAsync(query, cancellationToken).ConfigureAwait(false); + return result.WithTimestamp(timestamp); + } + + public async Task TestSpeechAsync(string speechFile, JToken query, CancellationToken cancellationToken) + { + var timestamp = DateTimeOffset.Now; + var result = await this.Client.TestSpeechAsync(speechFile, query, cancellationToken).ConfigureAwait(false); + return result.WithTimestamp(timestamp); + } + + public void Dispose() + { + this.Client.Dispose(); + } + } } } diff --git a/src/NLU.DevOps.CommandLine/Test/TestOptions.cs b/src/NLU.DevOps.CommandLine/Test/TestOptions.cs index 18d0791..b5eaac2 100644 --- a/src/NLU.DevOps.CommandLine/Test/TestOptions.cs +++ b/src/NLU.DevOps.CommandLine/Test/TestOptions.cs @@ -25,5 +25,8 @@ internal class TestOptions : BaseOptions [Option('p', "parallelism", HelpText = "Numeric value to determine the numer of parallel tests. Default value is 3.", Required = false)] public int Parallelism { get; set; } = 3; + + [Option("timestamp", HelpText = "Assign a timestamp to each utterance result.", Required = false)] + public bool Timestamp { get; set; } } } diff --git a/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs b/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs index a9e0a4a..919d5d3 100644 --- a/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs +++ b/src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs @@ -78,8 +78,7 @@ protected override async Task TestAsync(string utterance, Canc result.QueryResult.Intent.DisplayName, result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList()) .WithScore(result.QueryResult.IntentDetectionConfidence) - .WithTextScore(result.QueryResult.SpeechRecognitionConfidence) - .WithTimestamp(DateTimeOffset.Now); + .WithTextScore(result.QueryResult.SpeechRecognitionConfidence); }, cancellationToken) .ConfigureAwait(false); @@ -115,8 +114,7 @@ protected override async Task TestSpeechAsync(string speechFil result.QueryResult.Intent.DisplayName, result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList()) .WithScore(result.QueryResult.IntentDetectionConfidence) - .WithTextScore(result.QueryResult.SpeechRecognitionConfidence) - .WithTimestamp(DateTimeOffset.Now); + .WithTextScore(result.QueryResult.SpeechRecognitionConfidence); }, cancellationToken) .ConfigureAwait(false); diff --git a/src/NLU.DevOps.Lex.Tests/LexNLUTestClientTests.cs b/src/NLU.DevOps.Lex.Tests/LexNLUTestClientTests.cs index de32368..9839986 100644 --- a/src/NLU.DevOps.Lex.Tests/LexNLUTestClientTests.cs +++ b/src/NLU.DevOps.Lex.Tests/LexNLUTestClientTests.cs @@ -86,9 +86,6 @@ public static async Task TestsWithSpeech(string slots, string entityType, string // assert reads content from file (file contents are "hello world") content.Should().Be("hello world"); - // assert result type - result.Should().BeOfType(); - // assert intent and text result.Intent.Should().Be(intent); result.Text.Should().Be(transcript); @@ -121,7 +118,6 @@ public static async Task CreatesLabeledUtterances() using (var lex = new LexNLUTestClient(string.Empty, string.Empty, mockClient.Object)) { var response = await lex.TestAsync(text).ConfigureAwait(false); - response.Should().BeOfType(); response.Text.Should().Be(text); response.Intent.Should().Be(intent); response.Entities.Should().BeEmpty(); diff --git a/src/NLU.DevOps.Lex/LexNLUTestClient.cs b/src/NLU.DevOps.Lex/LexNLUTestClient.cs index f980d88..0407917 100644 --- a/src/NLU.DevOps.Lex/LexNLUTestClient.cs +++ b/src/NLU.DevOps.Lex/LexNLUTestClient.cs @@ -87,8 +87,7 @@ protected override async Task TestAsync(string utterance, Canc .Select(slot => new Entity(slot.Key, slot.Value, null, 0)) .ToArray(); - return new LabeledUtterance(utterance, postTextResponse.IntentName, entities) - .WithTimestamp(DateTimeOffset.Now); + return new LabeledUtterance(utterance, postTextResponse.IntentName, entities); } /// @@ -118,8 +117,7 @@ protected override async Task TestSpeechAsync(string speechFil .ToArray() : null; - return new JsonLabeledUtterance(postContentResponse.InputTranscript, postContentResponse.IntentName, slots) - .WithTimestamp(DateTimeOffset.Now); + return new LabeledUtterance(postContentResponse.InputTranscript, postContentResponse.IntentName, slots); } } diff --git a/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs b/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs index d392b1e..37cfa6a 100644 --- a/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs +++ b/src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs @@ -66,7 +66,6 @@ public static async Task TestModel() using (var luis = builder.Build()) { var result = await luis.TestAsync(test).ConfigureAwait(false); - result.Should().BeOfType(); result.Text.Should().Be(test); result.Intent.Should().Be("intent"); result.Entities.Count.Should().Be(1); diff --git a/src/NLU.DevOps.Luis/LuisNLUTestClient.cs b/src/NLU.DevOps.Luis/LuisNLUTestClient.cs index 0a4bb94..d9da717 100644 --- a/src/NLU.DevOps.Luis/LuisNLUTestClient.cs +++ b/src/NLU.DevOps.Luis/LuisNLUTestClient.cs @@ -134,8 +134,7 @@ roleValue is string role && speechLuisResult.LuisResult.Entities?.Select(getEntity).ToList()) .WithProperty("intents", speechLuisResult.LuisResult.Intents) .WithScore(speechLuisResult.LuisResult.TopScoringIntent?.Score) - .WithTextScore(speechLuisResult.TextScore) - .WithTimestamp(DateTimeOffset.Now); + .WithTextScore(speechLuisResult.TextScore); } } } diff --git a/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs b/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs index 4fd9d01..9f9e565 100644 --- a/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs +++ b/src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs @@ -70,7 +70,6 @@ public static async Task TestModel() using (var luis = builder.Build()) { var result = await luis.TestAsync(test).ConfigureAwait(false); - result.Should().BeOfType(); result.Text.Should().Be(test); result.Intent.Should().Be("intent"); result.Entities.Count.Should().Be(1); diff --git a/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs b/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs index dd10f35..0cb780f 100644 --- a/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs +++ b/src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs @@ -208,8 +208,7 @@ private LabeledUtterance LuisResultToLabeledUtterance(SpeechPredictionResponse s return new LabeledUtterance(query, intent, entities) .WithProperty("intents", intents) .WithScore(intentData?.Score) - .WithTextScore(speechPredictionResponse.TextScore) - .WithTimestamp(DateTimeOffset.Now); + .WithTextScore(speechPredictionResponse.TextScore); } } }