Feature/cdms-223 Decision Analytics & additional tests

Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs

@@ -3,8 +3,8 @@
 
 namespace Btms.Analytics.Tests.Helpers;
 
-public class MultiSeriesDatasetAssertions(List<Series>? test)
-    : GenericCollectionAssertions<Series>(test)
+public class MultiSeriesDatasetAssertions(List<Series<ByNumericDimensionResult>>? test)
+    : GenericCollectionAssertions<Series<ByNumericDimensionResult>>(test)
 {
     [CustomAssertion]
     public void BeSameLength(string because = "", params object[] becauseArgs)

Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs

@@ -2,7 +2,7 @@ namespace Btms.Analytics.Tests.Helpers;
 
 public static class TestAssertionExtensions
 {
-    public static MultiSeriesDatasetAssertions Should(this List<Series>? instance)
+    public static MultiSeriesDatasetAssertions Should(this List<Series<ByNumericDimensionResult>>? instance)
     {
         return new MultiSeriesDatasetAssertions(instance);
     }

Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs

@@ -12,7 +12,7 @@
 namespace Btms.Analytics.Tests;
 
 public class ImportNotificationsByCommoditiesTests(ITestOutputHelper output)
-    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDatasetName, reloadData:false)
+    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDatasetName)
 {
 
     [Fact]
@@ -37,7 +37,6 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
         });
 
         result.Should().HaveResults();
-
         result.Should().BeSameLength();
     }
 }

Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs

@@ -61,6 +61,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
             r.Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1);
         });
 
+        // result.shoul
         result.Sum(r => r.Periods.Sum(p => p.Value))
             .Should().BeGreaterThan(0);
     }

Btms.Analytics.Tests/MovementsByDecisionsTests.cs

@@ -11,21 +11,18 @@
 namespace Btms.Analytics.Tests;
 
 public class MovementsByDecisionsTests(ITestOutputHelper output)
-    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDatasetName)
+    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDecisionsDatasetName)
 {
-
-    [Fact(Skip = "Needs revisiting - needs more assertions, perhaps switch to individual scenario test")]
+    [Fact]
+    // [Fact(Skip = "Needs revisiting - needs more assertions, perhaps switch to individual scenario test")]
     public async Task WhenCalled_ReturnExpectedAggregation()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
-        var result = (await GetMovementsAggregationService()
-            .ByDecision(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
-            .Result;
+        var result = await MovementsAggregationService
+            .ByDecision(DateTime.MinValue, DateTime.MaxValue)!;
 
-        TestOutputHelper.WriteLine("{0} aggregated items found", result.Count);
+        TestOutputHelper.WriteLine("{0} aggregated items found", result!.Result.Count());
 
-        result.Count.Should().BeGreaterThan(1);
-        // result.Select(r => r.Key).Order().Should()
-        //     .Equal("ALVS Linked : H01", "BTMS Linked : C03", "BTMS Linked : X00", "BTMS Not Linked : X00");
+        result.Result.Count.Should().BeGreaterThan(1);
     }
 }

Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs

@@ -1,3 +1,4 @@
+using System.Text.Json.Nodes;
 using Btms.Common.Extensions;
 using FluentAssertions;
 using Xunit;
@@ -6,37 +7,39 @@
 using Btms.Analytics.Tests.Fixtures;
 using Btms.Analytics.Tests.Helpers;
 using TestDataGenerator.Config;
+using TestDataGenerator.Scenarios.SpecificFiles;
 using TestGenerator.IntegrationTesting.Backend;
+using TestGenerator.IntegrationTesting.Backend.Extensions;
 
 namespace Btms.Analytics.Tests;
 
 public class MovementsByUniqueDocumentReferenceTests(ITestOutputHelper output)
-    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDatasetName)
+    : ScenarioGeneratorBaseTest<Mrn24GBDDJER3ZFRMZAR9ScenarioGenerator>(output)
 {
-
-    [Fact(Skip = "Needs revisiting - needs more assertions, perhaps switch to individual scenario test")]
-    public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
+    [Fact]
+    public async Task ShouldReturnExpectedAggregation()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
-        var result = (await GetMovementsAggregationService()
-            .ByUniqueDocumentReferenceCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
-            .Series
-            .ToList();
-
-        TestOutputHelper.WriteLine("{0} aggregated items found", result.Count);
+        var result = await Client
+            .GetAnalyticsDashboard(["movementsByUniqueDocumentReferenceCount"],
+                dateFrom: DateTime.MinValue, dateTo: DateTime.MaxValue);
+
+        TestOutputHelper.WriteLine($"{result.StatusCode} status");
+        result.IsSuccessStatusCode.Should().BeTrue(result.StatusCode.ToString());
 
-        result.Count().Should().Be(3);
-        result.Select(r => r.Name).Order().Should().Equal("Investigate", "Linked", "Not Linked");
+        var chart = await result
+            .AnalyticsMultiSeriesChartAs<MultiSeriesDataset<ByNumericDimensionResult>,ByNumericDimensionResult>("movementsByUniqueDocumentReferenceCount")!;
+
+        TestOutputHelper.WriteLine("{0} aggregated items found", chart.Series!.Count);
 
-        result.Should().AllSatisfy(d =>
+        chart.Series!.Count.Should().Be(3);
+        chart.Series.Should().HaveResults();
+        chart.Series.Should().BeSameLength();
+
+        chart.Series!.Should().AllSatisfy(d =>
         {
             d.Dimension.Should().Be("Document Reference Count");
-            d.Results.Count().Should().NotBe(0);
-            d.Results.Sum(r => ((ByNumericDimensionResult)r).Value).Should().BeGreaterThan(0);
         });
 
-        result.Should().HaveResults();
-
-        result.Should().BeSameLength();
     }
 }

Btms.Analytics.Tests/MovementsExceptionsTests.cs

@@ -6,24 +6,35 @@
 using Btms.Analytics.Tests.Fixtures;
 using Btms.Analytics.Tests.Helpers;
 using TestDataGenerator.Config;
+using TestDataGenerator.Scenarios;
 using TestGenerator.IntegrationTesting.Backend;
+using TestGenerator.IntegrationTesting.Backend.Extensions;
 
 namespace Btms.Analytics.Tests;
 
 public class MovementsExceptionsTests(ITestOutputHelper output)
-    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDatasetName)
+    : ScenarioGeneratorBaseTest<CrNoMatchSingleItemWithDecisionScenarioGenerator>(output)
 {
 
-    [Fact(Skip="Change to an individiual scenario test that generates an exception. Possibly a seperate test for each exception type")]
+    [Fact] //(Skip="Change to an individiual scenario test that generates an exception. Possibly a seperate test for each exception type")]
     public async Task WhenCalled_ReturnExpectedAggregation()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
-        var result = (await GetMovementsAggregationService()
-            .ByDecision(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
-            .Result;
-
-        TestOutputHelper.WriteLine("{0} aggregated items found", result.Count);
 
-        result.Count.Should().BeGreaterThan(1);
+        var result = await Client
+            .GetAnalyticsDashboard(["movementsExceptions"],
+                dateFrom: DateTime.MinValue, dateTo: DateTime.MaxValue);
+
+        var chart = await result
+            .AnalyticsChartAs<SingleSeriesDataset>("movementsExceptions")!;
+
+        chart.Values.Count
+            .Should().BeGreaterThan(1);
+
+        chart.Values
+            .Single(v => v.Key == "Alvs has match decisions but no Btms links")
+            .Value
+            .Should().Be(1);
+
     }
 }

Btms.Analytics/Dataset.cs

@@ -16,9 +16,10 @@ public class SummarisedDataset<TSummary, TResult> : IDataset
 
 }
 
-public class MultiSeriesDataset : IDataset
+public class MultiSeriesDataset<TDimensionResult> : IDataset
+    where TDimensionResult : IDimensionResult
 {
-    public List<Series> Series { get; set; } = [];
+    public List<Series<TDimensionResult>> Series { get; set; } = [];
 }
 
 public class SingleSeriesDataset : IDataset, IDimensionResult
@@ -32,6 +33,7 @@ public class TabularDataset<TColumn> : IDataset where TColumn : IDimensionResult
 }
 
 public class EntityDataset<T>(IEnumerable<T> items) : IDataset
+    where T : IDimensionResult
 {
     public IEnumerable<T> Items { get; set; } = items;
 }
@@ -41,6 +43,15 @@ public class MultiSeriesDatetimeDataset : IDataset
     public List<DatetimeSeries> Series { get; set; } = [];
 }
 
+public class ScenarioItem : IDimensionResult
+{
+    [JsonInclude]
+    public required string Scenario;
+
+    [JsonInclude]
+    public required string[] Keys;
+}
+
 /// <summary>
 /// Serialise the derived types of IDataset
 /// </summary>
@@ -49,18 +60,36 @@ public class DatasetResultTypeMappingConverter<TType> : JsonConverter<TType> whe
 {
     [return: MaybeNull]
     public override TType Read(
-        ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) =>
-        throw new NotImplementedException();
+        ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        var converters = options
+            .Converters
+            .Where(c => c is not DatasetResultTypeMappingConverter<TType>);
+
+        var newOptions = new JsonSerializerOptions
+        {
+            PropertyNamingPolicy = options.PropertyNamingPolicy
+        };
+
+        foreach (var jsonConverter in converters)
+        {
+            newOptions.Converters.Add(jsonConverter);
+        }
+
+        TType result = JsonSerializer.Deserialize<TType>(ref reader, newOptions)!;
+
+        return result;   
+    }
 
     public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOptions options)
     {
         if (value is MultiSeriesDatetimeDataset)
         {
             JsonSerializer.Serialize(writer, value as MultiSeriesDatetimeDataset, options);
         }
-        else if (value is MultiSeriesDataset)
+        else if (value is MultiSeriesDataset<ByNumericDimensionResult>)
         {
-            JsonSerializer.Serialize(writer, value as MultiSeriesDataset, options);
+            JsonSerializer.Serialize(writer, value as MultiSeriesDataset<ByNumericDimensionResult>, options);
         }
         else if (value is SingleSeriesDataset)
         {

Btms.Analytics/DatasetDimensions.cs

@@ -54,6 +54,7 @@ public class StringBucketDimensionResult : IDimensionResult
 }
 
 public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceApiPrefix, string resourceId)
+    : IDimensionResult 
 {
     public AuditEntry AuditEntry { get; set; } = auditEntry;
     public string ResourceType { get; set; } = resourceType;
@@ -67,11 +68,11 @@ public class DatetimeSeries(string name)
     public List<ByDateTimeResult> Periods { get; set; } = [];
 }
 
-public class Series
+public class Series<TDimensionResult>
 {
     public required string Name { get; set; }
     public required string Dimension { get; set; }
-    public required List<IDimensionResult> Results { get; set; }
+    public required List<TDimensionResult> Results { get; set; }
 }
 
 /// <summary>
@@ -82,8 +83,17 @@ public class DimensionResultTypeMappingConverter<TType> : JsonConverter<TType> w
 {
     [return: MaybeNull]
     public override TType Read(
-        ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) =>
-        throw new NotImplementedException();
+        ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        var newOptions = new JsonSerializerOptions
+        {
+            PropertyNamingPolicy = options.PropertyNamingPolicy
+        };
+
+        TType result = JsonSerializer.Deserialize<TType>(ref reader, newOptions)!;
+
+        return result;
+    }
 
     public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOptions options)
     {

Btms.Analytics/Extensions/AnalyticsExtensions.cs

@@ -256,7 +256,7 @@ public static async Task<IDataset> AsIDataset(this Task<MultiSeriesDatetimeDatas
         return (IDataset)ms.Result;
     }
 
-    public static async Task<IDataset> AsIDataset(this Task<MultiSeriesDataset> ms)
+    public static async Task<IDataset> AsIDataset(this Task<MultiSeriesDataset<ByNumericDimensionResult>> ms)
     {
         await ms;
         return (IDataset)ms.Result;

Btms.Analytics/Extensions/AnalyticsHelpers.cs

@@ -1,4 +1,5 @@
 using Btms.Model;
+using Btms.Model.Cds;
 using MongoDB.Bson;
 namespace Btms.Analytics.Extensions;
 
@@ -51,8 +52,8 @@ public static string[] GetImportNotificationSegments()
             .ToArray();
     }
 
-    public static string[] GetMovementStatusSegments()
+    public static LinkStatusEnum[] GetMovementStatusSegments()
     {
-        return ["Linked", "Not Linked", "Investigate"];
+        return Enum.GetValues<LinkStatusEnum>();
     }
 }

Btms.Analytics/Extensions/MovementExtensions.cs

@@ -1,4 +1,5 @@
 using Btms.Model;
+using Btms.Model.Cds;
 using Btms.Model.Ipaffs;
 
 namespace Btms.Analytics.Extensions;
@@ -21,7 +22,7 @@ public class MovementWithLinkStatus
     {
         public required Movement Movement;
         public required DateTime CreatedSource;
-        public required string Description;
+        public required LinkStatusEnum Status;
     }
 
     public static IQueryable<MovementWithLinkStatus> SelectLinkStatus(this IQueryable<Movement> source)
@@ -30,11 +31,7 @@ public static IQueryable<MovementWithLinkStatus> SelectLinkStatus(this IQueryabl
             .Select(m => new MovementWithLinkStatus() {
                 Movement = m,
                 CreatedSource = m.CreatedSource!.Value,
-                Description =
-                    m.BtmsStatus.LinkStatus
-                    // m.Relationships.Notifications.Data.Count > 0 ? "Linked" :
-                    // m.AlvsDecisionStatus!.Context!.AlvsCheckStatus!.AnyMatch ? "Investigate" :
-                    // "Not Linked"
+                Status = m.BtmsStatus.LinkStatus
             });
 
         return m;

Btms.Analytics/IImportNotificationsAggregationService.cs

@@ -3,10 +3,11 @@
 namespace Btms.Analytics;
 
 public interface IImportNotificationsAggregationService
-{
+{   
     public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime? from = null, DateTime? to = null);
-    public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByCommodityCount(DateTime from, DateTime to);
     public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public EntityDataset<ScenarioItem> Scenarios(DateTime? from, DateTime? to);
 }

Btms.Analytics/IMovementsAggregationService.cs

@@ -7,10 +7,10 @@ public interface IMovementsAggregationService
 {
     public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime? from = null, DateTime? to = null);
-    public Task<MultiSeriesDataset> ByItemCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByItemCount(DateTime from, DateTime to);
     public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
     // public Task<TabularDataset<ByNameDimensionResult>> ByDecisionAndLinkStatus(DateTime from, DateTime to);
-    public Task<MultiSeriesDataset> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
     public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
     // public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
     public Task<EntityDataset<AuditHistory>?> GetHistory(string movementId);