CDMS-200 captures ched types of movements & adds to analytics. One of…

Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs

@@ -39,6 +39,9 @@ public MultiItemDataTestFixture(IMessageSink messageSink)
             app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig<CrNoMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
                 .GetAwaiter().GetResult();
 
+            app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig<CrNoMatchNoDecisionScenarioGenerator>(10, 3, arrivalDateRange: 0))
+                .GetAwaiter().GetResult();
+
             app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig<ChedASimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
                 .GetAwaiter().GetResult();
 

Btms.Analytics.Tests/ImportNotificationsByVersionTests.cs → Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs

@@ -4,11 +4,12 @@
 using Xunit.Abstractions;
 
 using Btms.Analytics.Tests.Fixtures;
+using Btms.Model.Ipaffs;
 
 namespace Btms.Analytics.Tests;
 
 [Collection(nameof(BasicSampleDataTestCollection))]
-public class ImportNotificationsByVersionTests(
+public class ImportNotificationsByMaxVersionTests(
     BasicSampleDataTestFixture basicSampleDataTestFixture,
     ITestOutputHelper testOutputHelper)
 {
@@ -30,13 +31,11 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
     {
         testOutputHelper.WriteLine("Querying for aggregated data");
         var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
-            .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
 
         testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 
-        result.Values.Count.Should().Be(8);
-        result.Values.Keys.Order().Should().Equal(
-            "CHEDA Linked", "CHEDA Not Linked", "CHEDD Linked", "CHEDD Not Linked", "CHEDP Linked", "CHEDP Not Linked", "CHEDPP Linked", "CHEDPP Not Linked");
+        result.Values.Count.Should().Be(1);
 
     }
 
@@ -45,10 +44,34 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
     {
         testOutputHelper.WriteLine("Querying for aggregated data");
         var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
-            .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
+            .ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
 
         testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 
-        result.Values.Count.Should().Be(8);
+        result.Values.Count.Should().Be(0);
+    }
+
+    [Fact]
+    public async Task WhenCalledWithChedType_ReturnsResults()
+    {
+        testOutputHelper.WriteLine("Querying for aggregated data");
+        var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]));
+
+        testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
+
+        result.Values.Count.Should().Be(1);
+    }
+
+    [Fact]
+    public async Task WhenCalledWithCountry_ReturnsResults()
+    {
+        testOutputHelper.WriteLine("Querying for aggregated data");
+        var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL"));
+
+        testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
+
+        result.Values.Count.Should().Be(1);
     }
 }

Btms.Analytics.Tests/MovementsByCreatedDateTests.cs

@@ -22,13 +22,13 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
 
         testOutputHelper.WriteLine(result.ToJsonString());
 
-        result.Count.Should().Be(2);
+        result.Count.Should().Be(3);
 
-        result[0].Name.Should().Be("Linked");
-        result[0].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
-        result[0].Periods.Count.Should().Be(48);
+        result[1].Name.Should().Be("Linked");
+        result[1].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
+        result[1].Periods.Count.Should().Be(48);
 
-        result[1].Name.Should().Be("Not Linked");
+        result[2].Name.Should().Be("Not Linked");
     }
 
     [Fact]
@@ -44,9 +44,9 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
 
         testOutputHelper.WriteLine(result.ToJsonString());
 
-        result.Count.Should().Be(2);
+        result.Count.Should().Be(3);
 
-        result.Select(r => r.Name).Should().Equal("Linked", "Not Linked");
+        result.Select(r => r.Name).Should().Equal("Investigate", "Linked", "Not Linked");
 
         result.Should().AllSatisfy(r =>
         {
@@ -69,12 +69,12 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
 
         testOutputHelper.WriteLine(result.ToJsonString());
 
-        result.Count.Should().Be(2);
+        result.Count.Should().Be(3);
 
-        result[0].Name.Should().Be("Linked");
-        result[0].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
-        result[0].Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1);
+        result[1].Name.Should().Be("Linked");
+        result[1].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
+        result[1].Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1);
 
-        result[1].Name.Should().Be("Not Linked");
+        result[2].Name.Should().Be("Not Linked");
     }
 }

Btms.Analytics.Tests/MovementsByItemsTests.cs

@@ -25,8 +25,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
 
         testOutputHelper.WriteLine("{0} aggregated items found", result.Count);
 
-        result.Count.Should().Be(2);
-        result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked");
+        result.Count.Should().Be(3);
+        result.Select(r => r.Name).Order().Should().Equal("Investigate", "Linked", "Not Linked");
 
         result.Should().AllSatisfy(r =>
         {

Btms.Analytics.Tests/MovementsByMaxVersionTests.cs

@@ -0,0 +1,78 @@
+using Btms.Common.Extensions;
+using FluentAssertions;
+using Xunit;
+using Xunit.Abstractions;
+
+using Btms.Analytics.Tests.Fixtures;
+using Btms.Model.Ipaffs;
+
+namespace Btms.Analytics.Tests;
+
+[Collection(nameof(BasicSampleDataTestCollection))]
+public class MovementsByMaxVersionTests(
+    BasicSampleDataTestFixture basicSampleDataTestFixture,
+    ITestOutputHelper testOutputHelper)
+{
+    // [Fact]
+    // public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
+    // {
+    //     testOutputHelper.WriteLine("Querying for aggregated data");
+    //     var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
+    //         .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()));
+    //
+    //     testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
+    //     
+    //     result.Values.Count.Should().Be(2);
+    //     result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
+    // }
+    //
+    // [Fact]
+    // public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
+    // {
+    //     testOutputHelper.WriteLine("Querying for aggregated data");
+    //     var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
+    //         .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
+    //
+    //     testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
+    //     
+    //     result.Values.Count.Should().Be(2);
+    //     result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
+    // }
+    //
+    // [Fact]
+    // public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation()
+    // {
+    //     testOutputHelper.WriteLine("Querying for aggregated data");
+    //     var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
+    //         .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
+    //
+    //     testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
+    //     
+    //     result.Values.Count.Should().Be(2);
+    //     result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
+    // }
+
+    [Fact]
+    public async Task WhenCalledWithChedType_ReturnsResults()
+    {
+        testOutputHelper.WriteLine("Querying for aggregated data");
+        var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]));
+
+        testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
+
+        result.Values.Count.Should().Be(2);
+    }
+
+    [Fact]
+    public async Task WhenCalledWithCountry_ReturnsResults()
+    {
+        testOutputHelper.WriteLine("Querying for aggregated data");
+        var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL"));
+
+        testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
+
+        result.Values.Count.Should().Be(1);
+    }
+}

Btms.Analytics.Tests/MovementsByStatusTests.cs

@@ -21,8 +21,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
 
         testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
 
-        result.Values.Count.Should().Be(2);
-        result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
+        result.Values.Count.Should().Be(3);
+        result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked");
     }
 
     [Fact]
@@ -34,8 +34,8 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
 
         testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 
-        result.Values.Count.Should().Be(2);
-        result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
+        result.Values.Count.Should().Be(3);
+        result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked");
     }
 
     [Fact]
@@ -47,7 +47,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
 
         testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 
-        result.Values.Count.Should().Be(2);
-        result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
+        result.Values.Count.Should().Be(3);
+        result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked");
     }
 }

Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs

@@ -25,8 +25,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
 
         testOutputHelper.WriteLine("{0} aggregated items found", result.Count);
 
-        result.Count().Should().Be(2);
-        result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked");
+        result.Count().Should().Be(3);
+        result.Select(r => r.Name).Order().Should().Equal("Investigate", "Linked", "Not Linked");
 
         result.Should().AllSatisfy(d =>
         {

Btms.Analytics/DatasetDimensions.cs

@@ -2,6 +2,7 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using Btms.Model.Auditing;
+using Btms.Model.Ipaffs;
 
 namespace Btms.Analytics;
 
@@ -19,6 +20,7 @@ public class ExceptionResult : IDimensionResult
     public required DateTime Updated { get; set; }
 
     public required int ItemCount { get; set; }
+    public required ImportNotificationTypeEnum[] ChedTypes { get; set; }
     public required int MaxEntryVersion { get; set; }
     public required int MaxDecisionNumber { get; set; }
     public required int LinkedCheds { get; set; }

Btms.Analytics/Extensions/AnalyticsHelpers.cs

@@ -21,6 +21,11 @@ public static class AnalyticsHelpers
 {
     internal static DateTime AggregateDateCreator(this BsonValue b) => b["dateToUse"].BsonType != BsonType.Null ? b["dateToUse"].ToUniversalTime() : DateTime.MinValue;
 
+    internal static string GetLinkedName(string linked, string type)
+    {
+        return $"{type} {linked}";
+    }
+
     internal static string GetLinkedName(bool linked, string type)
     {
         return $"{type} {GetLinkedName(linked)}";
@@ -46,8 +51,8 @@ public static string[] GetImportNotificationSegments()
             .ToArray();
     }
 
-    public static string[] GetMovementSegments()
+    public static string[] GetMovementStatusSegments()
     {
-        return ["Linked", "Not Linked"];
+        return ["Linked", "Not Linked", "Investigate"];
     }
 }

Btms.Analytics/Extensions/MovementExtensions.cs

@@ -0,0 +1,42 @@
+using Btms.Model;
+using Btms.Model.Ipaffs;
+
+namespace Btms.Analytics.Extensions;
+
+public static class MovementExtensions
+{   
+    public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
+        ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    {
+        return source
+            .Where(m => (m.CreatedSource >= from && m.CreatedSource < to)
+                        && (country == null || m.DispatchCountryCode == country)
+                        && (chedTypes == null || !chedTypes!.Any() ||
+                            !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() ||
+                            m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c))));
+
+    }
+
+    public class MovementWithLinkStatus
+    {
+        public required Movement Movement;
+        public required DateTime CreatedSource;
+        public required string Description;
+    }
+
+    public static IQueryable<MovementWithLinkStatus> SelectLinkStatus(this IQueryable<Movement> source)
+    {
+        var m = source
+            .Select(m => new MovementWithLinkStatus() {
+                Movement = m,
+                CreatedSource = m.CreatedSource!.Value,
+                Description =
+                    m.Status.LinkStatus
+                    // m.Relationships.Notifications.Data.Count > 0 ? "Linked" :
+                    // m.AlvsDecisionStatus!.Context!.AlvsCheckStatus!.AnyMatch ? "Investigate" :
+                    // "Not Linked"
+            });
+
+        return m;
+    } 
+}

Btms.Analytics/IImportNotificationsAggregationService.cs

@@ -1,3 +1,5 @@
+using Btms.Model.Ipaffs;
+
 namespace Btms.Analytics;
 
 public interface IImportNotificationsAggregationService
@@ -6,5 +8,5 @@ public interface IImportNotificationsAggregationService
     public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to);
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
 }

Btms.Analytics/IMovementsAggregationService.cs

@@ -1,4 +1,5 @@
 using Btms.Model.Auditing;
+using Btms.Model.Ipaffs;
 
 namespace Btms.Analytics;
 
@@ -7,15 +8,15 @@ public interface IMovementsAggregationService
     public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset> ByItemCount(DateTime from, DateTime to);
-    public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
+    public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
     // public Task<TabularDataset<ByNameDimensionResult>> ByDecisionAndLinkStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
     public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
     // public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
     public Task<EntityDataset<AuditHistory>?> GetHistory(string movementId);
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
-    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
-    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
-    public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
 
 }