From c803f377a600cff99a566ed8eb608a05b5cefc4b Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 31 Dec 2024 13:49:16 +0000 Subject: [PATCH 1/9] CDMS-200 captures ched types of movements & adds to analytics. One of the tests is failing so needs investigation --- Btms.Analytics/DatasetDimensions.cs | 2 + .../IImportNotificationsAggregationService.cs | 4 +- .../IMovementsAggregationService.cs | 11 +- .../ImportNotificationsAggregationService.cs | 10 +- Btms.Analytics/MovementExceptions.cs | 16 ++- Btms.Analytics/MovementsAggregationService.cs | 33 ++--- .../DecisionTests/ChedPSimpleTests.cs | 124 ++++++++++++++---- Btms.Backend/Config/AnalyticsDashboards.cs | 3 +- Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 7 +- .../MovementPreProcessingTests.cs | 1 + .../Decisions/NoMatchDecisionsTest.cs | 1 + .../Services/Matching/MatchingServiceTests.cs | 1 + .../Builders}/MovementBuilder.cs | 23 +++- .../ImportNotificationTypeEnumExtensions.cs | 20 +++ .../Extensions/ServiceCollectionExtensions.cs | 1 + .../PreProcessing/MovementPreProcessor.cs | 1 + .../Decisions/DecisionMessageBuilder.cs | 16 +-- .../ClearanceRequestConsumerTests.cs | 1 + Btms.Consumers/DecisionsConsumer.cs | 1 + Btms.Model/Cds/AlvsDecision.cs | 5 +- .../Ipaffs/ImportNotificationTypeEnum.g.cs | 1 + 21 files changed, 210 insertions(+), 72 deletions(-) rename {Btms.Model => Btms.Business/Builders}/MovementBuilder.cs (96%) create mode 100644 Btms.Business/Extensions/ImportNotificationTypeEnumExtensions.cs diff --git a/Btms.Analytics/DatasetDimensions.cs b/Btms.Analytics/DatasetDimensions.cs index 1bf9d458..8520561a 100644 --- a/Btms.Analytics/DatasetDimensions.cs +++ b/Btms.Analytics/DatasetDimensions.cs @@ -2,6 +2,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Btms.Model.Auditing; +using Btms.Model.Ipaffs; namespace Btms.Analytics; @@ -19,6 +20,7 @@ public class ExceptionResult : IDimensionResult public required DateTime Updated { get; set; } public required int ItemCount { get; set; } + public required ImportNotificationTypeEnum[] ChedTypes { get; set; } public required int MaxEntryVersion { get; set; } public required int MaxDecisionNumber { get; set; } public required int LinkedCheds { get; set; } diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index 49ea7787..427db297 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -1,3 +1,5 @@ +using Btms.Model.Ipaffs; + namespace Btms.Analytics; public interface IImportNotificationsAggregationService @@ -6,5 +8,5 @@ public interface IImportNotificationsAggregationService public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByCommodityCount(DateTime from, DateTime to); - public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); + public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index 968a08f6..7291f4bc 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -1,4 +1,5 @@ using Btms.Model.Auditing; +using Btms.Model.Ipaffs; namespace Btms.Analytics; @@ -7,15 +8,15 @@ public interface IMovementsAggregationService public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByItemCount(DateTime from, DateTime to); - public Task> ByDecision(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); + public Task> ByDecision(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); // public Task> ByDecisionAndLinkStatus(DateTime from, DateTime to); public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); // public Task ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); public Task?> GetHistory(string movementId); - public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); - public Task ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); - public Task> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); - public Task ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); + public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 8443d374..b4a182b8 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -115,12 +115,16 @@ public Task ByCommodityCount(DateTime from, DateTime to) }); } - public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) + public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var data = context .Notifications - .Where(n => n.CreatedSource >= from && n.CreatedSource < to) - .Where(m => country == null || m.PartOne!.Route!.TransitingStates!.Contains(country)) + // .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .Where(n => (n.CreatedSource >= from && n.CreatedSource < to) + && (country == null || n.PartOne!.Route!.TransitingStates!.Contains(country)) + && (chedTypes == null || n.ImportNotificationType == null || Array.IndexOf(chedTypes!, n.ImportNotificationType) > -1) + ) + // .Where(m => country == null || m.PartOne!.Route!.TransitingStates!.Contains(country)) .GroupBy(n => new { MaxVersion = n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version ) }) diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs index baaaf956..688b5b5d 100644 --- a/Btms.Analytics/MovementExceptions.cs +++ b/Btms.Analytics/MovementExceptions.cs @@ -1,5 +1,6 @@ using Btms.Analytics.Extensions; using Btms.Backend.Data; +using Btms.Model.Ipaffs; using Microsoft.Extensions.Logging; namespace Btms.Analytics; @@ -9,15 +10,17 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) //Returns a summary of the exceptions or a list // Means we can share the same anonymous / query code without needing to create loads // of classes - public (SingleSeriesDataset summary, List) GetAllExceptions(DateTime from, DateTime to, bool summary, string[]? chedTypes = null, string? country = null) + public (SingleSeriesDataset summary, List) GetAllExceptions(DateTime from, DateTime to, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var exceptionsSummary = new SingleSeriesDataset(); var exceptionsResult = new List(); var simplifiedMovementView = context .Movements - .Where(n => n.CreatedSource >= from && n.CreatedSource < to) - .Where(m => country == null || m.DispatchCountryCode == country ) + .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) + && (country == null || m.DispatchCountryCode == country) + && (chedTypes == null || m.AlvsDecisionStatus.Context.ChedTypes!.Intersect(chedTypes).Count() != 0) + ) .Select(m => new { // TODO - we should think about pre-calculating this stuff and storing it on the movement... @@ -29,7 +32,8 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) MaxEntryVersion = m.ClearanceRequests.Max(c => c.Header!.EntryVersionNumber) ?? 0, LinkedCheds = m.Relationships.Notifications.Data.Count, ItemCount = m.Items.Count, - HasMatchDecisions = m.AlvsDecisionStatus.Context != null && m.AlvsDecisionStatus.Context.AlvsCheckStatus != null && m.AlvsDecisionStatus.Context.AlvsCheckStatus.AnyMatch, + ChedTypes = m.AlvsDecisionStatus.Context.ChedTypes, + HasMatchDecisions = m.AlvsDecisionStatus.Context.AlvsCheckStatus != null && m.AlvsDecisionStatus.Context.AlvsCheckStatus.AnyMatch, DecisionMatched = !m.AlvsDecisionStatus.Decisions .OrderBy(d => d.Context.AlvsDecisionNumber) .Reverse() @@ -48,6 +52,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) MaxEntryVersion = m.MaxEntryVersion, LinkedCheds = m.LinkedCheds, ItemCount = m.ItemCount, + ChedTypes = m.ChedTypes, HasMatchDecisions = m.HasMatchDecisions, HasNotificationRelationships = m.HasNotificationRelationships, Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount, @@ -78,6 +83,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) UpdatedSource = r.UpdatedSource!.Value, Updated = r.Updated, ItemCount = r.ItemCount, + ChedTypes = r.ChedTypes!, MaxEntryVersion = r.MaxEntryVersion, MaxDecisionNumber = r.MaxDecisionNumber, LinkedCheds = r.LinkedCheds, @@ -107,6 +113,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) UpdatedSource = r.UpdatedSource!.Value, Updated = r.Updated, ItemCount = r.ItemCount, + ChedTypes = r.ChedTypes!, MaxEntryVersion = r.MaxEntryVersion, MaxDecisionNumber = r.MaxDecisionNumber, LinkedCheds = r.LinkedCheds, @@ -138,6 +145,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) UpdatedSource = r.UpdatedSource!.Value, Updated = r.Updated, ItemCount = r.ItemCount, + ChedTypes = r.ChedTypes!, MaxEntryVersion = r.MaxEntryVersion, MaxDecisionNumber = r.MaxDecisionNumber, LinkedCheds = r.LinkedCheds, diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 35c68c1b..31711f0c 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -187,7 +187,7 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime return new EntityDataset(entries); } - public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) + public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var data = context .Movements @@ -205,12 +205,14 @@ public Task ByMaxVersion(DateTime from, DateTime to, string }); } - public Task ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) + public Task ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var data = context .Movements - .Where(n => n.CreatedSource >= from && n.CreatedSource < to) - .Where(m => country == null || m.DispatchCountryCode == country ) + .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) + && (country == null || m.DispatchCountryCode == country) + && (chedTypes == null || m.AlvsDecisionStatus.Context.ChedTypes!.Intersect(chedTypes).Count() != 0) + ) .GroupBy(n => new { MaxVersion = n.Decisions.Max(a => a.Header!.DecisionNumber ) }) @@ -223,28 +225,24 @@ public Task ByMaxDecisionNumber(DateTime from, DateTime to, }); } - public Task> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) + public Task> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var movementExceptions = new MovementExceptions(context, logger); - var (_, result) = movementExceptions.GetAllExceptions(from, to, false, chedTypes, country); + var (_, result) = movementExceptions + .GetAllExceptions(from, to, false, chedTypes, country); return Task.FromResult(result); } - public Task ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) + public Task ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var movementExceptions = new MovementExceptions(context, logger); - var (summary, _) = movementExceptions.GetAllExceptions(from, to, true, chedTypes, country); + var (summary, _) = movementExceptions + .GetAllExceptions(from, to, true, chedTypes, country); return Task.FromResult(summary); } - // TODO : remove - // public Task ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) - // { - // return Task.FromResult(new MultiSeriesDataset() ); - // } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -277,11 +275,14 @@ private Task Aggregate(DateTime[] dateRange, Func /// public Task> ByDecision(DateTime from, - DateTime to, string[]? chedTypes = null, string? country = null) + DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var mongoQuery = context .Movements - .Where(m => m.CreatedSource >= from && m.CreatedSource < to) + .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) + && (country == null || m.DispatchCountryCode == country) + && (chedTypes == null || m.AlvsDecisionStatus.Context.ChedTypes!.Intersect(chedTypes).Count() != 0) + ) .SelectMany(m => m.AlvsDecisionStatus.Decisions.Select( d => new {Decision = d, Movement = m } )) .SelectMany(d => d.Decision.Checks.Select(c => new { d.Decision, d.Movement, Check = c})) diff --git a/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs b/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs index 375ece84..d267ee1d 100644 --- a/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs +++ b/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs @@ -22,6 +22,7 @@ using TestDataGenerator.Scenarios.ChedP; using Xunit; using Xunit.Abstractions; +using ImportNotification = Btms.Types.Ipaffs.ImportNotification; namespace Btms.Backend.IntegrationTests.DecisionTests; @@ -31,27 +32,32 @@ public class ChedPSimpleTests(InMemoryScenarioApplicationFactory - d is { message: AlvsClearanceRequest }) - .message; - - var chedPNotification = (Types.Ipaffs.ImportNotification)loadedData.Single(d => - d is { message: Types.Ipaffs.ImportNotification }) - .message; - // Act - var jsonClientResponse = Client.AsJsonApiClient().GetById(chedPMovement!.Header!.EntryReference!, "api/movements"); + var movementResource = Client.AsJsonApiClient() + .Get("api/movements") + .Data + .Single() + .Relationships!.Count.Should().Be(1); + } + + [Fact] + public void ShouldHaveCorrectDecisions() + { + var chedPNotification = (Types.Ipaffs.ImportNotification)factory + .LoadedData + .Single(d => + d is { message: Types.Ipaffs.ImportNotification } + ) + .message; // Assert - jsonClientResponse.Should().NotBeNull(); - jsonClientResponse.Data.Relationships!.Count.Should().Be(1); + var movement = Client.AsJsonApiClient() + .Get("api/movements") + .GetResourceObjects() + .Single(); - var movement = jsonClientResponse.GetResourceObject(); movement.Decisions.Count.Should().Be(2); movement.AlvsDecisionStatus.Decisions.Count.Should().Be(1); @@ -59,7 +65,32 @@ public void SimpleChedPScenario_ShouldBeLinkedAndMatchDecision() .First() .Context.DecisionMatched .Should().BeTrue(); + + var decisionWithLinkAndContext = movement.AuditEntries + .Where(a => a is { CreatedBy: "Btms", Status: "Decision" }) + .MaxBy(a => a.Version)!; + + decisionWithLinkAndContext.Context!.ImportNotifications + .Should().NotBeNull(); + + decisionWithLinkAndContext.Context!.ImportNotifications! + .Select(n => (n.Id, n.Version)) + .Should() + .Equal([ + ( chedPNotification.ReferenceNumber!, 1 ) + ]); + } + [Fact] + public void ShouldHaveCorrectAuditTrail() + { + + // Assert + var movement = Client.AsJsonApiClient() + .Get("api/movements") + .GetResourceObjects() + .Single(); + movement.AuditEntries .Select(a => (a.CreatedBy, a.Status, a.Version)) .Should() @@ -70,21 +101,58 @@ public void SimpleChedPScenario_ShouldBeLinkedAndMatchDecision() ("Btms", "Decision", 2), ("Alvs", "Decision", 1) ]); + } - var decisionWithLinkAndContext = movement.AuditEntries - .Where(a => a.CreatedBy == "Btms" && a.Status == "Decision") - .MaxBy(a => a.Version)!; + [Fact] + public void ShouldHaveDecisionMatched() + { + + // Assert + var movement = Client.AsJsonApiClient() + .Get("api/movements") + .GetResourceObjects() + .Single(); - decisionWithLinkAndContext.Context!.ImportNotifications - .Should().NotBeNull(); + movement + .AlvsDecisionStatus + .Context! + .DecisionMatched + .Should() + .BeTrue(); + } + + [Fact] + public void ShouldHaveChedType() + { - decisionWithLinkAndContext.Context!.ImportNotifications! - .Select(n => (n.Id, n.Version)) - .Should().Equal([ - ( chedPNotification.ReferenceNumber!, 1 ) - ]); + // Assert + var movement = Client.AsJsonApiClient() + .Get("api/movements") + .GetResourceObjects() + .Single(); + + movement + .AlvsDecisionStatus + .Context! + .ChedTypes + .Should() + .Equal(ImportNotificationTypeEnum.Cvedp); + } + + [Fact(Skip = "Relationships aren't being deserialised correctly")] + // TODO : for some reason whilst jsonClientResponse contains the notification relationship, + // but movement from .GetResourceObject(s)(); doesn't! + public void ShouldHaveNotificationRelationships() + { + + // Assert + var movement = Client.AsJsonApiClient() + .Get("api/movements") + .GetResourceObjects() + .Single(); - // TODO : for some reason whilst jsonClientResponse contains the notification relationship, movement doesn't! - // movement.Relationships.Notifications.Data.Count.Should().Be(1); + movement + .Relationships.Notifications.Data + .Should().NotBeEmpty(); } } \ No newline at end of file diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index 69cd7cf1..39c59848 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -1,6 +1,7 @@ using Btms.Analytics; using Btms.Analytics.Extensions; using Btms.Common.Extensions; +using Btms.Model.Ipaffs; using FluentAssertions; using MongoDB.Driver.Linq; @@ -13,7 +14,7 @@ public static async Task> GetCharts( IImportNotificationsAggregationService importService, IMovementsAggregationService movementsService, string[] chartsToRender, - string[] chedTypes, + ImportNotificationTypeEnum[] chedTypes, string? country, DateTime? dateFrom, DateTime? dateTo diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index 402709a9..d213019b 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -4,6 +4,7 @@ using Btms.Common; using Btms.Common.Extensions; using Btms.Model.Extensions; +using Btms.Model.Ipaffs; using Microsoft.AspNetCore.Mvc; namespace Btms.Backend.Endpoints; @@ -35,13 +36,13 @@ private static async Task Timeline( } private static async Task Exceptions( [FromServices] IMovementsAggregationService movementsService, - [FromQuery(Name = "chedType")] string[] chedTypes, + [FromQuery(Name = "chedType")] ImportNotificationTypeEnum[] chedTypes, [FromQuery(Name = "country")] string? country, [FromQuery(Name = "dateFrom")] DateTime? dateFrom, [FromQuery(Name = "dateTo")] DateTime? dateTo) { var result - = await movementsService.GetExceptions(dateFrom ?? DateTime.MinValue, dateTo ?? DateTime.Today); + = await movementsService.GetExceptions(dateFrom ?? DateTime.MinValue, dateTo ?? DateTime.Today, chedTypes, country); if (result.HasValue()) { @@ -62,7 +63,7 @@ private static async Task GetDashboard( [FromServices] IImportNotificationsAggregationService importService, [FromServices] IMovementsAggregationService movementsService, [FromQuery] string[] chartsToRender, - [FromQuery(Name = "chedType")] string[] chedTypes, + [FromQuery(Name = "chedType")] ImportNotificationTypeEnum[] chedTypes, [FromQuery(Name = "coo")] string? countryOfOrigin, [FromQuery(Name = "dateFrom")] DateTime? dateFrom, [FromQuery(Name = "dateTo")] DateTime? dateTo) diff --git a/Btms.Business.Tests/PreProcessing/MovementPreProcessingTests.cs b/Btms.Business.Tests/PreProcessing/MovementPreProcessingTests.cs index 0c33c354..a4140d79 100644 --- a/Btms.Business.Tests/PreProcessing/MovementPreProcessingTests.cs +++ b/Btms.Business.Tests/PreProcessing/MovementPreProcessingTests.cs @@ -1,4 +1,5 @@ using Btms.Backend.Data.InMemory; +using Btms.Business.Builders; using Btms.Business.Pipelines.PreProcessing; using Btms.Model; using Btms.Types.Alvs; diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index 7977cef9..492e6fe2 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -1,3 +1,4 @@ +using Btms.Business.Builders; using Btms.Business.Pipelines.PreProcessing; using Btms.Business.Services.Decisions; using Btms.Business.Services.Matching; diff --git a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs index 7d705378..8d948a18 100644 --- a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs +++ b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs @@ -1,3 +1,4 @@ +using Btms.Business.Builders; using Btms.Business.Pipelines.PreProcessing; using Btms.Business.Services.Matching; using Btms.Model; diff --git a/Btms.Model/MovementBuilder.cs b/Btms.Business/Builders/MovementBuilder.cs similarity index 96% rename from Btms.Model/MovementBuilder.cs rename to Btms.Business/Builders/MovementBuilder.cs index 63fc96c0..78200adc 100644 --- a/Btms.Model/MovementBuilder.cs +++ b/Btms.Business/Builders/MovementBuilder.cs @@ -4,8 +4,11 @@ using Btms.Model.Cds; using Btms.Model.ChangeLog; using Microsoft.Extensions.Logging; +using Btms.Business.Extensions; +using Btms.Model; +using Btms.Model.Ipaffs; -namespace Btms.Model; +namespace Btms.Business.Builders; public class MovementBuilder(ILogger logger) { @@ -35,6 +38,10 @@ public MovementBuilder From(Model.Cds.CdsClearanceRequest request) Items = request.Items?.Select(x => x).ToList()!, }; + _movement + .AlvsDecisionStatus.Context + .ChedTypes = GetChedTypes(); + return this; } @@ -45,6 +52,18 @@ public MovementBuilder From(Movement movement) return this; } + private ImportNotificationTypeEnum[] GetChedTypes() + { + return _movement!.Items? + .SelectMany(i => i.Documents!) + .Select(d => + d.DocumentCode!.GetChedType() + ) + .Where(ct => ct.HasValue()) + .Select(ct => ct!.Value) + .ToArray()!; + } + public string Id { get @@ -151,7 +170,7 @@ public MovementBuilder MergeDecision(string path, Model.Cds.CdsClearanceRequest } context.ImportNotifications = notificationContext; - // context.ChedTypes = + context.ChedTypes = GetChedTypes(); var auditEntry = AuditEntry.CreateDecision( BuildNormalizedDecisionPath(path), diff --git a/Btms.Business/Extensions/ImportNotificationTypeEnumExtensions.cs b/Btms.Business/Extensions/ImportNotificationTypeEnumExtensions.cs new file mode 100644 index 00000000..f7bac76f --- /dev/null +++ b/Btms.Business/Extensions/ImportNotificationTypeEnumExtensions.cs @@ -0,0 +1,20 @@ +using Btms.Model.Ipaffs; + +namespace Btms.Business.Extensions; + +public static class ImportNotificationTypeEnumExtensions +{ + public static ImportNotificationTypeEnum? GetChedType(this string documentCode) + { + //This is the mapping from https://eaflood.atlassian.net/wiki/spaces/ALVS/pages/5177016349/DocumentCode+Field + return documentCode switch + { + "9115"or "C633" or "N002" or "N851" => ImportNotificationTypeEnum.Chedpp, + "N852" or "C678" => ImportNotificationTypeEnum.Ced, + "C640" => ImportNotificationTypeEnum.Cveda, + "C641" or "C673" or "N853" => ImportNotificationTypeEnum.Cvedp, + "9HCG" => null, //TODO : should this be mapped to a ched type? + _ => null + }; + } +} \ No newline at end of file diff --git a/Btms.Business/Extensions/ServiceCollectionExtensions.cs b/Btms.Business/Extensions/ServiceCollectionExtensions.cs index a9c0eb3f..c11c5119 100644 --- a/Btms.Business/Extensions/ServiceCollectionExtensions.cs +++ b/Btms.Business/Extensions/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using Btms.Backend.Data.Extensions; using Btms.BlobService; using Btms.BlobService.Extensions; +using Btms.Business.Builders; using Btms.Common.Extensions; using Btms.Metrics.Extensions; using Btms.SensitiveData; diff --git a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs index 4d19606d..891e7ddb 100644 --- a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs +++ b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs @@ -1,4 +1,5 @@ using Btms.Backend.Data; +using Btms.Business.Builders; using Btms.Model; using Btms.Model.Auditing; using Btms.Model.ChangeLog; diff --git a/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs b/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs index 1fcd7714..4e1be389 100644 --- a/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs +++ b/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs @@ -1,3 +1,5 @@ +using Btms.Business.Extensions; +using Btms.Common.Extensions; using Btms.Model; using Btms.Model.Ipaffs; using Btms.Types.Alvs; @@ -99,15 +101,13 @@ public static string[] BuildDecisionReasons(Model.Cds.Items item, DecisionCode d private static string MapToChedType(string documentCode) { - return documentCode switch - { - "N002" or "N851" or "9115" => ImportNotificationTypeEnum.Chedpp.ToString(), - "N852" or "C678" => ImportNotificationTypeEnum.Ced.ToString(), - "C640" => ImportNotificationTypeEnum.Cveda.ToString(), - "C641" or "C673" or "N853" => ImportNotificationTypeEnum.Cvedp.ToString(), - _ => throw new ArgumentOutOfRangeException(nameof(documentCode), documentCode, null) - }; + var ct = documentCode.GetChedType(); + if (!ct.HasValue()) + { + throw new ArgumentOutOfRangeException(nameof(documentCode), documentCode, null); + } + return ct.ToString()!; } } \ No newline at end of file diff --git a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs index 14292283..94ee1872 100644 --- a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs +++ b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs @@ -1,3 +1,4 @@ +using Btms.Business.Builders; using Btms.Business.Pipelines.PreProcessing; using Btms.Business.Services.Decisions; using Btms.Business.Services.Linking; diff --git a/Btms.Consumers/DecisionsConsumer.cs b/Btms.Consumers/DecisionsConsumer.cs index 0250c1c8..4ea3e9b6 100644 --- a/Btms.Consumers/DecisionsConsumer.cs +++ b/Btms.Consumers/DecisionsConsumer.cs @@ -1,4 +1,5 @@ using Btms.Backend.Data; +using Btms.Business.Builders; using Btms.Model; using Btms.Model.Cds; using Btms.Types.Alvs; diff --git a/Btms.Model/Cds/AlvsDecision.cs b/Btms.Model/Cds/AlvsDecision.cs index 71e6384d..d6a2d08d 100644 --- a/Btms.Model/Cds/AlvsDecision.cs +++ b/Btms.Model/Cds/AlvsDecision.cs @@ -96,6 +96,9 @@ public partial class DecisionContext : IAuditContext // [System.ComponentModel.Description("")] public List? ImportNotifications { get; set; } + [Attr] + [System.ComponentModel.Description("")] + [MongoDB.Bson.Serialization.Attributes.BsonRepresentation(MongoDB.Bson.BsonType.String)] public ImportNotificationTypeEnum[]? ChedTypes { get; set; } [Attr] @@ -143,7 +146,7 @@ public partial class AlvsDecisionStatus // [Attr] [System.ComponentModel.Description("")] - public DecisionContext? Context { get; set; } = default; + public DecisionContext Context { get; set; } = new DecisionContext(); // TODO - should we put the checks into context, and so into audit log? [Attr] diff --git a/Btms.Model/Ipaffs/ImportNotificationTypeEnum.g.cs b/Btms.Model/Ipaffs/ImportNotificationTypeEnum.g.cs index 9d474fb0..83583d72 100644 --- a/Btms.Model/Ipaffs/ImportNotificationTypeEnum.g.cs +++ b/Btms.Model/Ipaffs/ImportNotificationTypeEnum.g.cs @@ -6,6 +6,7 @@ namespace Btms.Model.Ipaffs; +[JsonConverter(typeof(JsonStringEnumConverter))] public enum ImportNotificationTypeEnum { From c962eec693c124f9cbb28b28fe9c17534459fb25 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 12:10:49 +0000 Subject: [PATCH 2/9] CDMS-200 finalising country filtering --- ...> ImportNotificationsByMaxVersionTests.cs} | 37 +++++++-- .../MovementsByMaxVersionTests.cs | 78 +++++++++++++++++++ .../ImportNotificationsAggregationService.cs | 13 +++- Btms.Analytics/MovementsAggregationService.cs | 6 +- TestDataGenerator.Tests/ScenarioTests.cs | 29 +++++++ TestDataGenerator/ClearanceRequestBuilder.cs | 5 ++ .../Extensions/BuilderExtensions.cs | 29 ++++--- .../ChedASimpleMatchScenarioGenerator.cs | 1 + 8 files changed, 175 insertions(+), 23 deletions(-) rename Btms.Analytics.Tests/{ImportNotificationsByVersionTests.cs => ImportNotificationsByMaxVersionTests.cs} (53%) create mode 100644 Btms.Analytics.Tests/MovementsByMaxVersionTests.cs create mode 100644 TestDataGenerator.Tests/ScenarioTests.cs diff --git a/Btms.Analytics.Tests/ImportNotificationsByVersionTests.cs b/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs similarity index 53% rename from Btms.Analytics.Tests/ImportNotificationsByVersionTests.cs rename to Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs index 96c70d0b..d7ff84da 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByVersionTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs @@ -4,11 +4,12 @@ using Xunit.Abstractions; using Btms.Analytics.Tests.Fixtures; +using Btms.Model.Ipaffs; namespace Btms.Analytics.Tests; [Collection(nameof(BasicSampleDataTestCollection))] -public class ImportNotificationsByVersionTests( +public class ImportNotificationsByMaxVersionTests( BasicSampleDataTestFixture basicSampleDataTestFixture, ITestOutputHelper testOutputHelper) { @@ -30,13 +31,11 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) - .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); - result.Values.Count.Should().Be(8); - result.Values.Keys.Order().Should().Equal( - "CHEDA Linked", "CHEDA Not Linked", "CHEDD Linked", "CHEDD Not Linked", "CHEDP Linked", "CHEDP Not Linked", "CHEDPP Linked", "CHEDPP Not Linked"); + result.Values.Count.Should().Be(1); } @@ -45,10 +44,34 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati { testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) - .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); + .ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); - result.Values.Count.Should().Be(8); + result.Values.Count.Should().Be(0); + } + + [Fact] + public async Task WhenCalledWithChedType_ReturnsResults() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda])); + + testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + + result.Values.Count.Should().Be(1); + } + + [Fact] + public async Task WhenCalledWithCountry_ReturnsResults() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL")); + + testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + + result.Values.Count.Should().Be(1); } } \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByMaxVersionTests.cs b/Btms.Analytics.Tests/MovementsByMaxVersionTests.cs new file mode 100644 index 00000000..780c18f3 --- /dev/null +++ b/Btms.Analytics.Tests/MovementsByMaxVersionTests.cs @@ -0,0 +1,78 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Model.Ipaffs; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(BasicSampleDataTestCollection))] +public class MovementsByMaxVersionTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + // [Fact] + // public async Task WhenCalledLastWeek_ReturnExpectedAggregation() + // { + // testOutputHelper.WriteLine("Querying for aggregated data"); + // var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) + // .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())); + // + // testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); + // + // result.Values.Count.Should().Be(2); + // result.Values.Keys.Order().Should().Equal("Linked", "Not Linked"); + // } + // + // [Fact] + // public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() + // { + // testOutputHelper.WriteLine("Querying for aggregated data"); + // var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) + // .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); + // + // testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + // + // result.Values.Count.Should().Be(2); + // result.Values.Keys.Order().Should().Equal("Linked", "Not Linked"); + // } + // + // [Fact] + // public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation() + // { + // testOutputHelper.WriteLine("Querying for aggregated data"); + // var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) + // .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); + // + // testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + // + // result.Values.Count.Should().Be(2); + // result.Values.Keys.Order().Should().Equal("Linked", "Not Linked"); + // } + + [Fact] + public async Task WhenCalledWithChedType_ReturnsResults() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda])); + + testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + + result.Values.Count.Should().Be(1); + } + + [Fact] + public async Task WhenCalledWithCountry_ReturnsResults() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL")); + + testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + + result.Values.Count.Should().Be(1); + } +} \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index b4a182b8..3110afc1 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -119,12 +119,19 @@ public Task ByMaxVersion(DateTime from, DateTime to, Import { var data = context .Notifications - // .Where(n => n.CreatedSource >= from && n.CreatedSource < to) .Where(n => (n.CreatedSource >= from && n.CreatedSource < to) && (country == null || n.PartOne!.Route!.TransitingStates!.Contains(country)) - && (chedTypes == null || n.ImportNotificationType == null || Array.IndexOf(chedTypes!, n.ImportNotificationType) > -1) + + && (chedTypes == null || chedTypes!.Length == 0 || n.ImportNotificationType == null || chedTypes!.Contains(n.ImportNotificationType!.Value)) ) - // .Where(m => country == null || m.PartOne!.Route!.TransitingStates!.Contains(country)) + + // .Select(n => new { Notification = n, IsChedTypeMatch = chedTypes == null || n.ImportNotificationType == null || Array.IndexOf(chedTypes!, n.ImportNotificationType) > -1 }) + // .Where(n => (n.Notification.CreatedSource >= from && n.Notification.CreatedSource < to) + // && (country == null || n.Notification.PartOne!.Route!.TransitingStates!.Contains(country)) + // && n.IsChedTypeMatch + // ) + // .Select(n => n.Notification) + .GroupBy(n => new { MaxVersion = n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version ) }) diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 31711f0c..f58e798b 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -191,8 +191,10 @@ public Task ByMaxVersion(DateTime from, DateTime to, Import { var data = context .Movements - .Where(n => n.CreatedSource >= from && n.CreatedSource < to) - .Where(m => country == null || m.DispatchCountryCode == country ) + .Where(m => m.CreatedSource >= from && m.CreatedSource < to + && (country == null || m.DispatchCountryCode == country) + && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || + m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c)))) .GroupBy(n => new { MaxVersion = n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber ) }) diff --git a/TestDataGenerator.Tests/ScenarioTests.cs b/TestDataGenerator.Tests/ScenarioTests.cs new file mode 100644 index 00000000..f27d8405 --- /dev/null +++ b/TestDataGenerator.Tests/ScenarioTests.cs @@ -0,0 +1,29 @@ +using Btms.BlobService; +using Btms.BlobService.Extensions; +using Microsoft.Extensions.DependencyInjection; +using TestDataGenerator.Extensions; +using Xunit; + +namespace TestDataGenerator.Tests; + +public class ScenarioTests +{ + [Fact] + public void EnsureAllScenarioDefaultsAreValid() + { + // var (configuration, _) = BuilderExtensions.GetConfig("Scenarios/Samples"); + // + // var sp = new ServiceCollection() + // .AddBlobStorage(configuration) + // .AddSingleton() + // .ConfigureTestGenerationServices() + // .BuildServiceProvider(); + + var scenarioTypes = BuilderExtensions.GetAllScenarios(); + + foreach (var scenarioType in scenarioTypes) + { + + } + } +} \ No newline at end of file diff --git a/TestDataGenerator/ClearanceRequestBuilder.cs b/TestDataGenerator/ClearanceRequestBuilder.cs index 92bc4c34..fc6e4954 100644 --- a/TestDataGenerator/ClearanceRequestBuilder.cs +++ b/TestDataGenerator/ClearanceRequestBuilder.cs @@ -64,6 +64,11 @@ public ClearanceRequestBuilder IncrementCreationDate(TimeSpan t) { return Do(x => x.ServiceHeader!.ServiceCallTimestamp = x.ServiceHeader!.ServiceCallTimestamp!.Value.Add(t)); } + + public ClearanceRequestBuilder WithDispatchCountryCode(string? countryCode) + { + return Do(x => x.Header!.DispatchCountryCode = countryCode); + } public ClearanceRequestBuilder WithCreationDate(DateTime entryDate, bool randomTime = true) { diff --git a/TestDataGenerator/Extensions/BuilderExtensions.cs b/TestDataGenerator/Extensions/BuilderExtensions.cs index d2f22d9f..c883c67d 100644 --- a/TestDataGenerator/Extensions/BuilderExtensions.cs +++ b/TestDataGenerator/Extensions/BuilderExtensions.cs @@ -38,22 +38,29 @@ public static object[] BuildAll(this IBaseBuilder[] builders) .OrderBy(m => m.CreatedDate()) .ToArray(); } + + /// + /// // Find all scenario generators + /// + /// + public static IEnumerable GetAllScenarios() + { + Type scenarioService = typeof(ScenarioGenerator); + + return AppDomain + .CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where( + p => scenarioService.IsAssignableFrom(p) + && !p.IsAbstract + && p != scenarioService); + } public static IServiceCollection ConfigureTestGenerationServices(this IServiceCollection services) { services.AddHttpClient(); - Type scenarionService = typeof(ScenarioGenerator); - - // Find scenario generators and add them to the DI container - - foreach (var type in AppDomain - .CurrentDomain.GetAssemblies() - .SelectMany(s => s.GetTypes()) - .Where( - p => scenarionService.IsAssignableFrom(p) - && !p.IsAbstract - && p != scenarionService)) + foreach (var type in GetAllScenarios()) { services.AddSingleton(type); } diff --git a/TestDataGenerator/Scenarios/ChedASimpleMatchScenarioGenerator.cs b/TestDataGenerator/Scenarios/ChedASimpleMatchScenarioGenerator.cs index cc5925fd..dafb0bcf 100644 --- a/TestDataGenerator/Scenarios/ChedASimpleMatchScenarioGenerator.cs +++ b/TestDataGenerator/Scenarios/ChedASimpleMatchScenarioGenerator.cs @@ -21,6 +21,7 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD .WithCreationDate(entryDate) .WithArrivalDateTimeOffset(notification.PartOne!.ArrivalDate, notification.PartOne!.ArrivalTime) .WithReferenceNumberOneToOne(notification.ReferenceNumber!) + .WithDispatchCountryCode(notification.PartOne!.Route!.TransitingStates!.FirstOrDefault()) .ValidateAndBuild(); logger.LogInformation("Created {EntryReference}", clearanceRequest.Header!.EntryReference); From cbe21c498a8287e8bbf47b0c096c4b2e53ca2b23 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 12:46:05 +0000 Subject: [PATCH 3/9] Adds test generation unit test that instantiates and validates all scenario generators --- TestDataGenerator.Tests/ScenarioTests.cs | 36 ++++++++++++------- .../Extensions/BuilderExtensions.cs | 11 ++++++ .../ImportNotificationBuilder.cs | 8 ++--- .../AllChedsNoMatchScenarioGenerator.cs | 4 +++ .../Samples/cheda-one-commodity.json | 2 +- .../Samples/chedd-one-commodity.json | 2 +- .../Samples/chedpp-multiple-commodity.json | 2 +- .../Scenarios/Samples/cr-one-item.json | 4 +-- .../Scenarios/Samples/decision-one-item.json | 2 +- 9 files changed, 48 insertions(+), 23 deletions(-) diff --git a/TestDataGenerator.Tests/ScenarioTests.cs b/TestDataGenerator.Tests/ScenarioTests.cs index f27d8405..384052ea 100644 --- a/TestDataGenerator.Tests/ScenarioTests.cs +++ b/TestDataGenerator.Tests/ScenarioTests.cs @@ -1,29 +1,39 @@ using Btms.BlobService; using Btms.BlobService.Extensions; +using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using TestDataGenerator.Extensions; +using TestDataGenerator.Scenarios; using Xunit; namespace TestDataGenerator.Tests; public class ScenarioTests { - [Fact] - public void EnsureAllScenarioDefaultsAreValid() + ServiceProvider _serviceProvider = BuilderExtensions.GetDefaultServiceProvider(); + + public static IEnumerable GetAllScenarios() + { + return BuilderExtensions + .GetAllScenarios() + .Select(s => new object[] { s }); + } + + [Theory] + [MemberData(nameof(GetAllScenarios))] + public void EnsureAllScenarioDefaultsAreValid(Type scenarioType) { - // var (configuration, _) = BuilderExtensions.GetConfig("Scenarios/Samples"); - // - // var sp = new ServiceCollection() - // .AddBlobStorage(configuration) - // .AddSingleton() - // .ConfigureTestGenerationServices() - // .BuildServiceProvider(); - var scenarioTypes = BuilderExtensions.GetAllScenarios(); - foreach (var scenarioType in scenarioTypes) + var scenario = (ScenarioGenerator)_serviceProvider.GetRequiredService(scenarioType); + var config = new ScenarioConfig() { - - } + Name = "Test", ArrivalDateRange = 1, Count = 1, Generator = scenario, CreationDateRange = 1 + }; + + var messages = scenario.Generate(1, 1, DateTime.Today, config); + + messages.Should().NotBeNull(); + messages.Should().NotBeEmpty(); } } \ No newline at end of file diff --git a/TestDataGenerator/Extensions/BuilderExtensions.cs b/TestDataGenerator/Extensions/BuilderExtensions.cs index c883c67d..9bc2bb73 100644 --- a/TestDataGenerator/Extensions/BuilderExtensions.cs +++ b/TestDataGenerator/Extensions/BuilderExtensions.cs @@ -55,6 +55,17 @@ public static IEnumerable GetAllScenarios() && !p.IsAbstract && p != scenarioService); } + + public static ServiceProvider GetDefaultServiceProvider() + { + var (configuration, _) = BuilderExtensions.GetConfig("Scenarios/Samples"); + + return new ServiceCollection() + .AddBlobStorage(configuration) + .AddSingleton() + .ConfigureTestGenerationServices() + .BuildServiceProvider(); + } public static IServiceCollection ConfigureTestGenerationServices(this IServiceCollection services) { diff --git a/TestDataGenerator/ImportNotificationBuilder.cs b/TestDataGenerator/ImportNotificationBuilder.cs index 5c6d8ce7..81b13f35 100644 --- a/TestDataGenerator/ImportNotificationBuilder.cs +++ b/TestDataGenerator/ImportNotificationBuilder.cs @@ -199,10 +199,10 @@ protected override ImportNotificationBuilder Validate() n.Version.AssertHasValue("Import Notification Version missing"); // NB - this may not be correct... - if (n.ImportNotificationType != ImportNotificationTypeEnum.Cveda) - { - n.PartTwo!.InspectionRequired.AssertHasValue("Import Notification PartTwo InspectionRequired missing"); - } + // if (n.ImportNotificationType != ImportNotificationTypeEnum.Cveda && n.PartTwo.HasValue()) + // { + // n.PartTwo!.InspectionRequired.AssertHasValue("Import Notification PartTwo InspectionRequired missing"); + // } }); } diff --git a/TestDataGenerator/Scenarios/AllChedsNoMatchScenarioGenerator.cs b/TestDataGenerator/Scenarios/AllChedsNoMatchScenarioGenerator.cs index 9c006886..3103a10a 100644 --- a/TestDataGenerator/Scenarios/AllChedsNoMatchScenarioGenerator.cs +++ b/TestDataGenerator/Scenarios/AllChedsNoMatchScenarioGenerator.cs @@ -11,6 +11,7 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD .WithCreationDate(entryDate) .WithRandomArrivalDateTime(config.ArrivalDateRange) .WithReferenceNumber(ImportNotificationTypeEnum.Cveda, scenario, entryDate, item) + .WithVersionNumber() .ValidateAndBuild(); logger.LogInformation("Created {NotificationReferenceNumber}", @@ -20,6 +21,7 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD .WithCreationDate(entryDate) .WithRandomArrivalDateTime(config.ArrivalDateRange) .WithReferenceNumber(ImportNotificationTypeEnum.Cvedp, scenario, entryDate, item) + .WithVersionNumber() .ValidateAndBuild(); logger.LogInformation("Created {NotificationReferenceNumber}", @@ -29,6 +31,7 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD .WithCreationDate(entryDate) .WithRandomArrivalDateTime(config.ArrivalDateRange) .WithReferenceNumber(ImportNotificationTypeEnum.Ced, scenario, entryDate, item) + .WithVersionNumber() .ValidateAndBuild(); logger.LogInformation("Created {NotificationReferenceNumber}", @@ -38,6 +41,7 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD .WithCreationDate(entryDate) .WithRandomArrivalDateTime(config.ArrivalDateRange) .WithReferenceNumber(ImportNotificationTypeEnum.Chedpp, scenario, entryDate, item) + .WithVersionNumber() .ValidateAndBuild(); logger.LogInformation("Created {NotificationReferenceNumber}", diff --git a/TestDataGenerator/Scenarios/Samples/cheda-one-commodity.json b/TestDataGenerator/Scenarios/Samples/cheda-one-commodity.json index 97e9a9da..0572afa4 100644 --- a/TestDataGenerator/Scenarios/Samples/cheda-one-commodity.json +++ b/TestDataGenerator/Scenarios/Samples/cheda-one-commodity.json @@ -1,6 +1,6 @@ { "referenceNumber": null, - "version": 1, + "version": null, "lastUpdated": null, "lastUpdatedBy": { "displayName": "Percy Inspector-Tester", diff --git a/TestDataGenerator/Scenarios/Samples/chedd-one-commodity.json b/TestDataGenerator/Scenarios/Samples/chedd-one-commodity.json index 6bc64054..091b5ca8 100644 --- a/TestDataGenerator/Scenarios/Samples/chedd-one-commodity.json +++ b/TestDataGenerator/Scenarios/Samples/chedd-one-commodity.json @@ -1,7 +1,7 @@ { "id": 3989, "referenceNumber": "CHEDD.GB.2024.1003989", - "version": 1, + "version": null, "lastUpdated": "2024-08-19T08:45:36.18Z", "lastUpdatedBy": { "displayName": "Percy Inspector-Tester", diff --git a/TestDataGenerator/Scenarios/Samples/chedpp-multiple-commodity.json b/TestDataGenerator/Scenarios/Samples/chedpp-multiple-commodity.json index 7b57b706..73524467 100644 --- a/TestDataGenerator/Scenarios/Samples/chedpp-multiple-commodity.json +++ b/TestDataGenerator/Scenarios/Samples/chedpp-multiple-commodity.json @@ -1,7 +1,7 @@ { "id": 3991, "referenceNumber": "CHEDPP.GB.2024.1003991", - "version": 1, + "version": null, "lastUpdated": "2024-08-19T08:50:00.193Z", "lastUpdatedBy": { "displayName": "Percy Inspector-Tester", diff --git a/TestDataGenerator/Scenarios/Samples/cr-one-item.json b/TestDataGenerator/Scenarios/Samples/cr-one-item.json index e4eeabf4..063127a0 100644 --- a/TestDataGenerator/Scenarios/Samples/cr-one-item.json +++ b/TestDataGenerator/Scenarios/Samples/cr-one-item.json @@ -7,8 +7,8 @@ }, "header": { "entryReference": null, - "entryVersionNumber": 2, - "previousVersionNumber": 1, + "entryVersionNumber": null, + "previousVersionNumber": null, "declarationUCR": null, "declarationPartNumber": null, "declarationType": null, diff --git a/TestDataGenerator/Scenarios/Samples/decision-one-item.json b/TestDataGenerator/Scenarios/Samples/decision-one-item.json index 7b2bb3d9..c2dca9a0 100644 --- a/TestDataGenerator/Scenarios/Samples/decision-one-item.json +++ b/TestDataGenerator/Scenarios/Samples/decision-one-item.json @@ -7,7 +7,7 @@ }, "header": { "entryReference": null, - "entryVersionNumber": 1, + "entryVersionNumber": null, "decisionNumber": null }, "items": [ From 54eaca1441bb93e53faf09a8fd7b411cd78c3ba2 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 13:40:14 +0000 Subject: [PATCH 4/9] Refactors to reuse chedtype & country filtering --- .../Extensions/MovementExtensions.cs | 19 +++++++++++ Btms.Analytics/MovementExceptions.cs | 11 ++++--- Btms.Analytics/MovementsAggregationService.cs | 32 ++++++++++++------- TestDataGenerator.Tests/ScenarioTests.cs | 2 +- 4 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 Btms.Analytics/Extensions/MovementExtensions.cs diff --git a/Btms.Analytics/Extensions/MovementExtensions.cs b/Btms.Analytics/Extensions/MovementExtensions.cs new file mode 100644 index 00000000..669912ae --- /dev/null +++ b/Btms.Analytics/Extensions/MovementExtensions.cs @@ -0,0 +1,19 @@ +using Btms.Model; +using Btms.Model.Ipaffs; + +namespace Btms.Analytics.Extensions; + +public static class MovementExtensions +{ + public static IQueryable WhereFilteredByCreatedDateAndParams(this IQueryable source, DateTime from, DateTime to, + ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + { + return source + .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) + && (country == null || m.DispatchCountryCode == country) + && (chedTypes == null || !chedTypes!.Any() || + !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || + m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c)))); + + } +} \ No newline at end of file diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs index 688b5b5d..3ece1291 100644 --- a/Btms.Analytics/MovementExceptions.cs +++ b/Btms.Analytics/MovementExceptions.cs @@ -17,10 +17,13 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) var simplifiedMovementView = context .Movements - .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) - && (country == null || m.DispatchCountryCode == country) - && (chedTypes == null || m.AlvsDecisionStatus.Context.ChedTypes!.Intersect(chedTypes).Count() != 0) - ) + .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + + // .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) + // && (country == null || m.DispatchCountryCode == country) + // && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || + // m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c))) + // ) .Select(m => new { // TODO - we should think about pre-calculating this stuff and storing it on the movement... diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index f58e798b..21fb9b7c 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.Logging; using System.Linq.Expressions; using Btms.Analytics.Extensions; @@ -16,6 +17,7 @@ namespace Btms.Analytics; public class MovementsAggregationService(IMongoDbContext context, ILogger logger) : IMovementsAggregationService { + /// /// Aggregates movements by createdSource and returns counts by date period. Could be refactored to use a generic/interface in time /// @@ -191,10 +193,12 @@ public Task ByMaxVersion(DateTime from, DateTime to, Import { var data = context .Movements - .Where(m => m.CreatedSource >= from && m.CreatedSource < to - && (country == null || m.DispatchCountryCode == country) - && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || - m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c)))) + // .Where(FilterByDateAndParams) + .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + // .Where(m => m.CreatedSource >= from && m.CreatedSource < to + // && (country == null || m.DispatchCountryCode == country) + // && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || + // m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c)))) .GroupBy(n => new { MaxVersion = n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber ) }) @@ -211,10 +215,11 @@ public Task ByMaxDecisionNumber(DateTime from, DateTime to, { var data = context .Movements - .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) - && (country == null || m.DispatchCountryCode == country) - && (chedTypes == null || m.AlvsDecisionStatus.Context.ChedTypes!.Intersect(chedTypes).Count() != 0) - ) + .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + // .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) + // && (country == null || m.DispatchCountryCode == country) + // && (chedTypes == null || m.AlvsDecisionStatus.Context.ChedTypes!.Intersect(chedTypes).Count() != 0) + // ) .GroupBy(n => new { MaxVersion = n.Decisions.Max(a => a.Header!.DecisionNumber ) }) @@ -281,10 +286,13 @@ public Task> { var mongoQuery = context .Movements - .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) - && (country == null || m.DispatchCountryCode == country) - && (chedTypes == null || m.AlvsDecisionStatus.Context.ChedTypes!.Intersect(chedTypes).Count() != 0) - ) + .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + + // .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) + // && (country == null || m.DispatchCountryCode == country) + // && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || + // m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c))) + // ) .SelectMany(m => m.AlvsDecisionStatus.Decisions.Select( d => new {Decision = d, Movement = m } )) .SelectMany(d => d.Decision.Checks.Select(c => new { d.Decision, d.Movement, Check = c})) diff --git a/TestDataGenerator.Tests/ScenarioTests.cs b/TestDataGenerator.Tests/ScenarioTests.cs index 384052ea..9e3c0835 100644 --- a/TestDataGenerator.Tests/ScenarioTests.cs +++ b/TestDataGenerator.Tests/ScenarioTests.cs @@ -10,7 +10,7 @@ namespace TestDataGenerator.Tests; public class ScenarioTests { - ServiceProvider _serviceProvider = BuilderExtensions.GetDefaultServiceProvider(); + readonly ServiceProvider _serviceProvider = BuilderExtensions.GetDefaultServiceProvider(); public static IEnumerable GetAllScenarios() { From 369d35d75ef713c35aaba6c5005cb3201af6c638 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 13:44:37 +0000 Subject: [PATCH 5/9] Removing comments --- Btms.Analytics/MovementExceptions.cs | 6 ------ Btms.Analytics/MovementsAggregationService.cs | 11 ----------- 2 files changed, 17 deletions(-) diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs index 3ece1291..a62aca94 100644 --- a/Btms.Analytics/MovementExceptions.cs +++ b/Btms.Analytics/MovementExceptions.cs @@ -18,12 +18,6 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) var simplifiedMovementView = context .Movements .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) - - // .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) - // && (country == null || m.DispatchCountryCode == country) - // && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || - // m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c))) - // ) .Select(m => new { // TODO - we should think about pre-calculating this stuff and storing it on the movement... diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 21fb9b7c..256ab31b 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -193,12 +193,7 @@ public Task ByMaxVersion(DateTime from, DateTime to, Import { var data = context .Movements - // .Where(FilterByDateAndParams) .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) - // .Where(m => m.CreatedSource >= from && m.CreatedSource < to - // && (country == null || m.DispatchCountryCode == country) - // && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || - // m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c)))) .GroupBy(n => new { MaxVersion = n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber ) }) @@ -287,12 +282,6 @@ public Task> var mongoQuery = context .Movements .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) - - // .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) - // && (country == null || m.DispatchCountryCode == country) - // && (chedTypes == null || !chedTypes!.Any() || !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || - // m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c))) - // ) .SelectMany(m => m.AlvsDecisionStatus.Decisions.Select( d => new {Decision = d, Movement = m } )) .SelectMany(d => d.Decision.Checks.Select(c => new { d.Decision, d.Movement, Check = c})) From 89b8ff7dcce4794c2f76213caa053ead5c2398f0 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 14:13:44 +0000 Subject: [PATCH 6/9] Refactoring analytics to re-use link status analysis --- Btms.Analytics/Extensions/AnalyticsHelpers.cs | 5 +++++ .../Extensions/MovementExtensions.cs | 19 +++++++++++++++++++ Btms.Analytics/MovementsAggregationService.cs | 16 +++++++++------- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Btms.Analytics/Extensions/AnalyticsHelpers.cs b/Btms.Analytics/Extensions/AnalyticsHelpers.cs index 3b830f76..722fff78 100644 --- a/Btms.Analytics/Extensions/AnalyticsHelpers.cs +++ b/Btms.Analytics/Extensions/AnalyticsHelpers.cs @@ -21,6 +21,11 @@ public static class AnalyticsHelpers { internal static DateTime AggregateDateCreator(this BsonValue b) => b["dateToUse"].BsonType != BsonType.Null ? b["dateToUse"].ToUniversalTime() : DateTime.MinValue; + internal static string GetLinkedName(string linked, string type) + { + return $"{type} {linked}"; + } + internal static string GetLinkedName(bool linked, string type) { return $"{type} {GetLinkedName(linked)}"; diff --git a/Btms.Analytics/Extensions/MovementExtensions.cs b/Btms.Analytics/Extensions/MovementExtensions.cs index 669912ae..d33493e2 100644 --- a/Btms.Analytics/Extensions/MovementExtensions.cs +++ b/Btms.Analytics/Extensions/MovementExtensions.cs @@ -15,5 +15,24 @@ public static IQueryable WhereFilteredByCreatedDateAndParams(this IQue !m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() || m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c)))); + } + + public class MovementWithLinkStatus + { + public required Movement Movement; + public required DateTime CreatedSource; + public required string Description; + } + + public static IQueryable SelectLinkStatus(this IQueryable source) + { + var m = source + .Select(m => new MovementWithLinkStatus() { + Movement = m, + CreatedSource = m.CreatedSource!.Value, + Description = m.Relationships.Notifications.Data.Count > 0 ? "Linked" : "Not Linked" + }); + + return m; } } \ No newline at end of file diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 256ab31b..18905735 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -41,9 +41,9 @@ public Task ByStatus(DateTime from, DateTime to) { var data = context .Movements - .Select(m => new { m.CreatedSource, IsLinked = m.Relationships.Notifications.Data.Count > 0 ? "Linked" : "Not Linked" }) + .SelectLinkStatus() .Where(n => n.CreatedSource >= from && n.CreatedSource < to) - .GroupBy(m => m.IsLinked) + .GroupBy(m => m.Description) .Select(g => new { g.Key, Count = g.Count() }) .ToDictionary(g => g.Key, g => g.Count); @@ -58,7 +58,8 @@ public Task ByItemCount(DateTime from, DateTime to) var mongoQuery = context .Movements .Where(n => n.CreatedSource >= from && n.CreatedSource < to) - .GroupBy(m => new { Linked = m.Relationships.Notifications.Data.Count > 0, ItemCount = m.Items.Count }) + .SelectLinkStatus() + .GroupBy(m => new { Linked = m.Description, ItemCount = m.Movement.Items.Count }) .Select(g => new { g.Key.Linked, g.Key.ItemCount, Count = g.Count() }); var mongoResult = mongoQuery @@ -66,7 +67,7 @@ public Task ByItemCount(DateTime from, DateTime to) .ToList(); var dictionary = mongoResult - .ToDictionary(g => new { Title = AnalyticsHelpers.GetLinkedName(g.Linked), g.ItemCount }, g => g.Count); + .ToDictionary(g => new { Title = g.Linked, g.ItemCount }, g => g.Count); var maxCount = mongoResult.Count > 0 ? mongoResult.Max(r => r.Count) : 0; @@ -95,10 +96,11 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, Da var mongoQuery = context .Movements .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .SelectLinkStatus() .GroupBy(m => new { - Linked = m.Relationships.Notifications.Data.Count > 0, - DocumentReferenceCount = m.Items + Linked = m.Description, + DocumentReferenceCount = m.Movement.Items .SelectMany(i => i.Documents == null ? new string[] {} : i.Documents.Select(d => d.DocumentReference)) .Distinct() .Count() @@ -109,7 +111,7 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, Da var dictionary = mongoResult .ToDictionary( - g => new { Title = AnalyticsHelpers.GetLinkedName(g.Linked), g.DocumentReferenceCount }, + g => new { Title = g.Linked, g.DocumentReferenceCount }, g => g.MovementCount); var maxReferences = mongoResult.Count > 0 ? From 1014f0a22857bf190b1643902985d8940eee315f Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 15:22:35 +0000 Subject: [PATCH 7/9] Refactored Link status out of analytics to be calculated at write time and stored on movement --- .../MovementsByStatusTests.cs | 2 +- .../Unit/MovementLinkStatus.cs | 86 +++++++++++++++++++ Btms.Analytics/Extensions/AnalyticsHelpers.cs | 4 +- .../Extensions/MovementExtensions.cs | 6 +- Btms.Analytics/MovementExceptions.cs | 8 +- Btms.Analytics/MovementsAggregationService.cs | 8 +- .../Services/Linking/LinkingServiceTests.cs | 9 +- Btms.Business/Builders/MovementBuilder.cs | 13 ++- .../Extensions/MovementExtensions.cs | 23 +++++ .../Services/Linking/LinkingService.cs | 2 + Btms.Model/Cds/AlvsDecision.cs | 49 +++++++---- Btms.Model/Movement.cs | 8 ++ .../Scenarios/CRNoMatchScenarioGenerator.cs | 21 ++++- 13 files changed, 204 insertions(+), 35 deletions(-) create mode 100644 Btms.Analytics.Tests/Unit/MovementLinkStatus.cs create mode 100644 Btms.Business/Extensions/MovementExtensions.cs diff --git a/Btms.Analytics.Tests/MovementsByStatusTests.cs b/Btms.Analytics.Tests/MovementsByStatusTests.cs index a66f0a42..aa14dc31 100644 --- a/Btms.Analytics.Tests/MovementsByStatusTests.cs +++ b/Btms.Analytics.Tests/MovementsByStatusTests.cs @@ -22,7 +22,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); result.Values.Count.Should().Be(2); - result.Values.Keys.Order().Should().Equal("Linked", "Not Linked"); + result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked"); } [Fact] diff --git a/Btms.Analytics.Tests/Unit/MovementLinkStatus.cs b/Btms.Analytics.Tests/Unit/MovementLinkStatus.cs new file mode 100644 index 00000000..e3422056 --- /dev/null +++ b/Btms.Analytics.Tests/Unit/MovementLinkStatus.cs @@ -0,0 +1,86 @@ +using Btms.Analytics.Extensions; +using Btms.Common.Extensions; +using Btms.Model; +using Btms.Model.Cds; +using Btms.Model.Relationships; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace Btms.Analytics.Tests.Unit; + +public static class MovementExtensions +{ + public static string GetDescription(this Movement m) + { + return new List() { m } + .AsQueryable() + .SelectLinkStatus() + .First() + .Description; + } + +} +public class MovementLinkStatus +{ + [Fact] + public void WhenRelationshipPresent_ShouldBeLinked() + { + new Movement() + { + CreatedSource = DateTime.Now, + Relationships = new MovementTdmRelationships() + { + Notifications = new TdmRelationshipObject() + { + Data = [new RelationshipDataItem()] + } + }, + Status = MovementStatus.Default() + } + .GetDescription() + .Should() + .Be("Linked"); + } + + [Fact] + public void ShouldBeNotLinked() + { + new Movement() + { + CreatedSource = DateTime.Now, + Status = new MovementStatus() + { + ChedTypes = [] + } + } + .GetDescription() + .Should() + .Be("Not Linked"); + } + + [Fact] + public void WhenNoRelationshipPresentAnd_ShouldBeLinked() + { + var m = new Movement() + { + CreatedSource = DateTime.Now, + AlvsDecisionStatus = new AlvsDecisionStatus() + { + Context = new DecisionContext() + { + AlvsCheckStatus = new StatusChecker() + { + AnyMatch = true + } + } + }, + Status = MovementStatus.Default() + }; + + m + .GetDescription() + .Should() + .Be("Investigate"); + } +} \ No newline at end of file diff --git a/Btms.Analytics/Extensions/AnalyticsHelpers.cs b/Btms.Analytics/Extensions/AnalyticsHelpers.cs index 722fff78..0b5811ad 100644 --- a/Btms.Analytics/Extensions/AnalyticsHelpers.cs +++ b/Btms.Analytics/Extensions/AnalyticsHelpers.cs @@ -51,8 +51,8 @@ public static string[] GetImportNotificationSegments() .ToArray(); } - public static string[] GetMovementSegments() + public static string[] GetMovementStatusSegments() { - return ["Linked", "Not Linked"]; + return ["Linked", "Not Linked", "Investigate"]; } } \ No newline at end of file diff --git a/Btms.Analytics/Extensions/MovementExtensions.cs b/Btms.Analytics/Extensions/MovementExtensions.cs index d33493e2..4da3dc1b 100644 --- a/Btms.Analytics/Extensions/MovementExtensions.cs +++ b/Btms.Analytics/Extensions/MovementExtensions.cs @@ -30,7 +30,11 @@ public static IQueryable SelectLinkStatus(this IQueryabl .Select(m => new MovementWithLinkStatus() { Movement = m, CreatedSource = m.CreatedSource!.Value, - Description = m.Relationships.Notifications.Data.Count > 0 ? "Linked" : "Not Linked" + Description = + m.Status.LinkStatus + // m.Relationships.Notifications.Data.Count > 0 ? "Linked" : + // m.AlvsDecisionStatus!.Context!.AlvsCheckStatus!.AnyMatch ? "Investigate" : + // "Not Linked" }); return m; diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs index a62aca94..bca93c76 100644 --- a/Btms.Analytics/MovementExceptions.cs +++ b/Btms.Analytics/MovementExceptions.cs @@ -30,7 +30,8 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) LinkedCheds = m.Relationships.Notifications.Data.Count, ItemCount = m.Items.Count, ChedTypes = m.AlvsDecisionStatus.Context.ChedTypes, - HasMatchDecisions = m.AlvsDecisionStatus.Context.AlvsCheckStatus != null && m.AlvsDecisionStatus.Context.AlvsCheckStatus.AnyMatch, + // HasMatchDecisions = m.AlvsDecisionStatus.Context.AlvsCheckStatus != null && m.AlvsDecisionStatus.Context.AlvsCheckStatus.AnyMatch, + Status = m.Status, DecisionMatched = !m.AlvsDecisionStatus.Decisions .OrderBy(d => d.Context.AlvsDecisionNumber) .Reverse() @@ -50,7 +51,8 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) LinkedCheds = m.LinkedCheds, ItemCount = m.ItemCount, ChedTypes = m.ChedTypes, - HasMatchDecisions = m.HasMatchDecisions, + // HasMatchDecisions = m.HasMatchDecisions, + Status = m.Status, HasNotificationRelationships = m.HasNotificationRelationships, Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount, // TODO - can we include CHED versions here too? @@ -90,7 +92,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) } var movementsWhereAlvsLinksButNotBtmsQuery = simplifiedMovementView - .Where(r => r.HasMatchDecisions && !r.HasNotificationRelationships); + .Where(r => r.Status.LinkStatus == "Investigate"); if (summary) { diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 18905735..a7555b78 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -49,7 +49,7 @@ public Task ByStatus(DateTime from, DateTime to) return Task.FromResult(new SingleSeriesDataset { - Values = AnalyticsHelpers.GetMovementSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) + Values = AnalyticsHelpers.GetMovementStatusSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) }); } @@ -74,7 +74,7 @@ public Task ByItemCount(DateTime from, DateTime to) return Task.FromResult(new MultiSeriesDataset() { - Series = AnalyticsHelpers.GetMovementSegments() + Series = AnalyticsHelpers.GetMovementStatusSegments() .Select(title => new Series() { Name = title, @@ -119,7 +119,7 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, Da return Task.FromResult(new MultiSeriesDataset() { - Series = AnalyticsHelpers.GetMovementSegments() + Series = AnalyticsHelpers.GetMovementStatusSegments() .Select(title => new Series() { Name = title, @@ -263,7 +263,7 @@ private Task Aggregate(DateTime[] dateRange, Func mongoResult.AsDataset(dateRange, title)) .AsOrderedArray(m => m.Name); diff --git a/Btms.Business.Tests/Services/Linking/LinkingServiceTests.cs b/Btms.Business.Tests/Services/Linking/LinkingServiceTests.cs index c787ab87..043553a0 100644 --- a/Btms.Business.Tests/Services/Linking/LinkingServiceTests.cs +++ b/Btms.Business.Tests/Services/Linking/LinkingServiceTests.cs @@ -296,7 +296,8 @@ private MovementLinkContext CreateMovementContext(Movement? movement, List x).ToList()!, + Items = request.Items?.ToList()!, + Status = new MovementStatus() + { + ChedTypes = GetChedTypes(request.Items!.ToList()) + } }; _movement @@ -52,9 +56,9 @@ public MovementBuilder From(Movement movement) return this; } - private ImportNotificationTypeEnum[] GetChedTypes() + private ImportNotificationTypeEnum[] GetChedTypes(List? items = null) { - return _movement!.Items? + return items? .SelectMany(i => i.Documents!) .Select(d => d.DocumentCode!.GetChedType() @@ -161,7 +165,8 @@ public MovementBuilder MergeDecision(string path, Model.Cds.CdsClearanceRequest } _movement.AlvsDecisionStatus.Decisions.Add(alvsDecision); - + + _movement.AddLinkStatus(); } else { diff --git a/Btms.Business/Extensions/MovementExtensions.cs b/Btms.Business/Extensions/MovementExtensions.cs new file mode 100644 index 00000000..c1e2e9e1 --- /dev/null +++ b/Btms.Business/Extensions/MovementExtensions.cs @@ -0,0 +1,23 @@ +using Btms.Model; + +namespace Btms.Business.Extensions; + +public static class MovementExtensions +{ + public static void AddLinkStatus(this Movement movement) + { + var linkStatus = "Not Linked"; + + if (movement.Relationships.Notifications.Data.Count > 0) + { + linkStatus = "Linked"; + } + else if (movement.AlvsDecisionStatus?.Context?.AlvsCheckStatus?.AnyMatch ?? false) + { + linkStatus = "Investigate"; + } + + movement.Status.LinkStatus = linkStatus; + } + +} \ No newline at end of file diff --git a/Btms.Business/Services/Linking/LinkingService.cs b/Btms.Business/Services/Linking/LinkingService.cs index b8fcc46b..a467f07a 100644 --- a/Btms.Business/Services/Linking/LinkingService.cs +++ b/Btms.Business/Services/Linking/LinkingService.cs @@ -1,6 +1,7 @@ using System.Text.RegularExpressions; using Btms.Backend.Data; using Btms.Backend.Data.Extensions; +using Btms.Business.Extensions; using Btms.Metrics; using Btms.Model; using Btms.Model.ChangeLog; @@ -106,6 +107,7 @@ public async Task Link(LinkContext linkContext, CancellationToken ca notification._MatchReference) ] }); + movement.AddLinkStatus(); await dbContext.Movements.Update(movement, movement._Etag, transaction, cancellationToken); await dbContext.Notifications.Update(notification, notification._Etag, transaction, diff --git a/Btms.Model/Cds/AlvsDecision.cs b/Btms.Model/Cds/AlvsDecision.cs index d6a2d08d..2c9ba1d5 100644 --- a/Btms.Model/Cds/AlvsDecision.cs +++ b/Btms.Model/Cds/AlvsDecision.cs @@ -47,60 +47,79 @@ public class DecisionImportNotifications public required DateTime UpdatedSource { get; set; } } -public class StatusChecker { - +public class StatusChecker +{ + [Attr] [System.ComponentModel.Description("")] - public required bool AllMatch { get; set; } + public bool AllMatch { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AnyMatch { get; set; } + public bool AnyMatch { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AllNoMatch { get; set; } + public bool AllNoMatch { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AnyNoMatch { get; set; } + public bool AnyNoMatch { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AllHold { get; set; } + public bool AllHold { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AnyHold { get; set; } + public bool AnyHold { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AllRefuse { get; set; } + public bool AllRefuse { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AnyRefuse { get; set; } + public bool AnyRefuse { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AllRelease { get; set; } + public bool AllRelease { get; set; } = default; [Attr] [System.ComponentModel.Description("")] - public required bool AnyRelease { get; set; } + public bool AnyRelease { get; set; } = default; } -public partial class DecisionContext : IAuditContext // +public class MovementStatus { + public static MovementStatus Default() + { + return new MovementStatus() { ChedTypes = [] }; + } + [Attr] [System.ComponentModel.Description("")] - public List? ImportNotifications { get; set; } - + [MongoDB.Bson.Serialization.Attributes.BsonRepresentation(MongoDB.Bson.BsonType.String)] + public required ImportNotificationTypeEnum[]? ChedTypes { get; set; } + + [Attr] + [System.ComponentModel.Description("")] + public string LinkStatus { get; set; } = "Not Linked"; + +} +public partial class DecisionContext : IAuditContext // +{ + //TODO : Remove in favour of MovementStatus [Attr] [System.ComponentModel.Description("")] [MongoDB.Bson.Serialization.Attributes.BsonRepresentation(MongoDB.Bson.BsonType.String)] public ImportNotificationTypeEnum[]? ChedTypes { get; set; } + [Attr] + [System.ComponentModel.Description("")] + public List? ImportNotifications { get; set; } + [Attr] [System.ComponentModel.Description("")] public int AlvsDecisionNumber { get; set; } = default; diff --git a/Btms.Model/Movement.cs b/Btms.Model/Movement.cs index 67f29b4a..f2896b54 100644 --- a/Btms.Model/Movement.cs +++ b/Btms.Model/Movement.cs @@ -22,6 +22,10 @@ public class Movement : IMongoIdentifiable, IDataEntity, IAuditable { private List matchReferences = []; + [ChangeSetIgnore] //TODO : should we ignore this or not? + [Attr] + public required MovementStatus Status { get; set; } + // This field is used by the jsonapi-consumer to control the correct casing in the type field [ChangeSetIgnore] public string Type { get; set; } = "movements"; @@ -113,6 +117,10 @@ public void AddRelationship(TdmRelationshipObject relationship) .All(itemNumber => Relationships.Notifications.Data.Exists(x => x.Matched.GetValueOrDefault() && x.SourceItem == itemNumber)); + //TODO : This would be the right time to call AddLinkStatus I think + // but relies on linking being moved into Business + // this.AddLinkStatus(); + if (linked) { AuditEntries.Add(AuditEntry.CreateLinked(String.Empty, this.AuditEntries.FirstOrDefault()?.Version ?? 1)); diff --git a/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs b/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs index d7b8ce28..b0f113e7 100644 --- a/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs +++ b/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs @@ -9,15 +9,32 @@ public class CrNoMatchScenarioGenerator(ILogger logg { public override GeneratorResult Generate(int scenario, int item, DateTime entryDate, ScenarioConfig config) { + var reference = DataHelpers.GenerateReferenceNumber(ImportNotificationTypeEnum.Cveda, scenario, entryDate, item); + var clearanceRequest = GetClearanceRequestBuilder("cr-one-item") .WithCreationDate(entryDate) .WithArrivalDateTimeOffset(DateTime.Today.ToDate(), DateTime.Now.ToTime()) - .WithReferenceNumberOneToOne(DataHelpers.GenerateReferenceNumber(ImportNotificationTypeEnum.Cveda, scenario, entryDate, item)) + .WithReferenceNumberOneToOne(reference) .WithRandomItems(10, 100) .ValidateAndBuild(); logger.LogInformation("Created {EntryReference}", clearanceRequest.Header!.EntryReference); - return new GeneratorResult([clearanceRequest]); + // TODO - check with Matt what a sensible checkCode, decision & other fields we need to + // implement a 'real world' test here + var checkCode = "H2019"; + var decisionCode = "H01"; + + var alvsDecision = GetDecisionBuilder("decision-one-item") + .WithCreationDate(clearanceRequest.ServiceHeader!.ServiceCallTimestamp!.Value.AddHours(1), false) + .WithReferenceNumber(reference) + .WithEntryVersionNumber(1) + .WithDecisionVersionNumber() + .WithItemAndCheck(1, checkCode, decisionCode) + .ValidateAndBuild(); + + logger.LogInformation("Created {EntryReference}", alvsDecision.Header!.EntryReference); + + return new GeneratorResult([clearanceRequest, alvsDecision]); } } \ No newline at end of file From 50cc40c662f87d1745c1a17db6e7554f46a97650 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 15:43:19 +0000 Subject: [PATCH 8/9] Fixed tests --- .../Fixtures/BasicSampleDataTestFixture.cs | 2 +- .../Fixtures/MultiItemDataTestFixture.cs | 3 + .../MovementsByCreatedDateTests.cs | 24 +++--- Btms.Analytics.Tests/MovementsByItemsTests.cs | 4 +- .../MovementsByMaxVersionTests.cs | 2 +- .../MovementsByStatusTests.cs | 10 +-- ...MovementsByUniqueDocumentReferenceTests.cs | 4 +- .../Unit/MovementLinkStatus.cs | 86 ------------------- .../CRNoMatchNoDecisionScenarioGenerator.cs | 25 ++++++ 9 files changed, 51 insertions(+), 109 deletions(-) delete mode 100644 Btms.Analytics.Tests/Unit/MovementLinkStatus.cs create mode 100644 TestDataGenerator/Scenarios/CRNoMatchNoDecisionScenarioGenerator.cs diff --git a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs index 15760199..0a7a3253 100644 --- a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs +++ b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs @@ -28,7 +28,7 @@ public BasicSampleDataTestFixture(IMessageSink messageSink) _mongoDbContext = rootScope.ServiceProvider.GetRequiredService(); // Would like to pick this up from env/config/DB state - var insertToMongo = true; + var insertToMongo = false; if (insertToMongo) { diff --git a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs index c0324ba1..87ce88a2 100644 --- a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs +++ b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs @@ -39,6 +39,9 @@ public MultiItemDataTestFixture(IMessageSink messageSink) app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) .GetAwaiter().GetResult(); + app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + .GetAwaiter().GetResult(); + app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) .GetAwaiter().GetResult(); diff --git a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs index c90975f0..f983cab0 100644 --- a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs @@ -22,13 +22,13 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() testOutputHelper.WriteLine(result.ToJsonString()); - result.Count.Should().Be(2); + result.Count.Should().Be(3); - result[0].Name.Should().Be("Linked"); - result[0].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today); - result[0].Periods.Count.Should().Be(48); + result[1].Name.Should().Be("Linked"); + result[1].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today); + result[1].Periods.Count.Should().Be(48); - result[1].Name.Should().Be("Not Linked"); + result[2].Name.Should().Be("Not Linked"); } [Fact] @@ -44,9 +44,9 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati testOutputHelper.WriteLine(result.ToJsonString()); - result.Count.Should().Be(2); + result.Count.Should().Be(3); - result.Select(r => r.Name).Should().Equal("Linked", "Not Linked"); + result.Select(r => r.Name).Should().Equal("Investigate", "Linked", "Not Linked"); result.Should().AllSatisfy(r => { @@ -69,12 +69,12 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation() testOutputHelper.WriteLine(result.ToJsonString()); - result.Count.Should().Be(2); + result.Count.Should().Be(3); - result[0].Name.Should().Be("Linked"); - result[0].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today); - result[0].Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1); + result[1].Name.Should().Be("Linked"); + result[1].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today); + result[1].Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1); - result[1].Name.Should().Be("Not Linked"); + result[2].Name.Should().Be("Not Linked"); } } \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByItemsTests.cs b/Btms.Analytics.Tests/MovementsByItemsTests.cs index 0d82f73b..081beda8 100644 --- a/Btms.Analytics.Tests/MovementsByItemsTests.cs +++ b/Btms.Analytics.Tests/MovementsByItemsTests.cs @@ -25,8 +25,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("{0} aggregated items found", result.Count); - result.Count.Should().Be(2); - result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked"); + result.Count.Should().Be(3); + result.Select(r => r.Name).Order().Should().Equal("Investigate", "Linked", "Not Linked"); result.Should().AllSatisfy(r => { diff --git a/Btms.Analytics.Tests/MovementsByMaxVersionTests.cs b/Btms.Analytics.Tests/MovementsByMaxVersionTests.cs index 780c18f3..9f117a60 100644 --- a/Btms.Analytics.Tests/MovementsByMaxVersionTests.cs +++ b/Btms.Analytics.Tests/MovementsByMaxVersionTests.cs @@ -61,7 +61,7 @@ public async Task WhenCalledWithChedType_ReturnsResults() testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); - result.Values.Count.Should().Be(1); + result.Values.Count.Should().Be(2); } [Fact] diff --git a/Btms.Analytics.Tests/MovementsByStatusTests.cs b/Btms.Analytics.Tests/MovementsByStatusTests.cs index aa14dc31..d4b973f2 100644 --- a/Btms.Analytics.Tests/MovementsByStatusTests.cs +++ b/Btms.Analytics.Tests/MovementsByStatusTests.cs @@ -21,7 +21,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); - result.Values.Count.Should().Be(2); + result.Values.Count.Should().Be(3); result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked"); } @@ -34,8 +34,8 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); - result.Values.Count.Should().Be(2); - result.Values.Keys.Order().Should().Equal("Linked", "Not Linked"); + result.Values.Count.Should().Be(3); + result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked"); } [Fact] @@ -47,7 +47,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); - result.Values.Count.Should().Be(2); - result.Values.Keys.Order().Should().Equal("Linked", "Not Linked"); + result.Values.Count.Should().Be(3); + result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked"); } } \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs index 507aabdf..2c8238c0 100644 --- a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs +++ b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs @@ -25,8 +25,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("{0} aggregated items found", result.Count); - result.Count().Should().Be(2); - result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked"); + result.Count().Should().Be(3); + result.Select(r => r.Name).Order().Should().Equal("Investigate", "Linked", "Not Linked"); result.Should().AllSatisfy(d => { diff --git a/Btms.Analytics.Tests/Unit/MovementLinkStatus.cs b/Btms.Analytics.Tests/Unit/MovementLinkStatus.cs deleted file mode 100644 index e3422056..00000000 --- a/Btms.Analytics.Tests/Unit/MovementLinkStatus.cs +++ /dev/null @@ -1,86 +0,0 @@ -using Btms.Analytics.Extensions; -using Btms.Common.Extensions; -using Btms.Model; -using Btms.Model.Cds; -using Btms.Model.Relationships; -using FluentAssertions; -using Xunit; -using Xunit.Abstractions; - -namespace Btms.Analytics.Tests.Unit; - -public static class MovementExtensions -{ - public static string GetDescription(this Movement m) - { - return new List() { m } - .AsQueryable() - .SelectLinkStatus() - .First() - .Description; - } - -} -public class MovementLinkStatus -{ - [Fact] - public void WhenRelationshipPresent_ShouldBeLinked() - { - new Movement() - { - CreatedSource = DateTime.Now, - Relationships = new MovementTdmRelationships() - { - Notifications = new TdmRelationshipObject() - { - Data = [new RelationshipDataItem()] - } - }, - Status = MovementStatus.Default() - } - .GetDescription() - .Should() - .Be("Linked"); - } - - [Fact] - public void ShouldBeNotLinked() - { - new Movement() - { - CreatedSource = DateTime.Now, - Status = new MovementStatus() - { - ChedTypes = [] - } - } - .GetDescription() - .Should() - .Be("Not Linked"); - } - - [Fact] - public void WhenNoRelationshipPresentAnd_ShouldBeLinked() - { - var m = new Movement() - { - CreatedSource = DateTime.Now, - AlvsDecisionStatus = new AlvsDecisionStatus() - { - Context = new DecisionContext() - { - AlvsCheckStatus = new StatusChecker() - { - AnyMatch = true - } - } - }, - Status = MovementStatus.Default() - }; - - m - .GetDescription() - .Should() - .Be("Investigate"); - } -} \ No newline at end of file diff --git a/TestDataGenerator/Scenarios/CRNoMatchNoDecisionScenarioGenerator.cs b/TestDataGenerator/Scenarios/CRNoMatchNoDecisionScenarioGenerator.cs new file mode 100644 index 00000000..df459305 --- /dev/null +++ b/TestDataGenerator/Scenarios/CRNoMatchNoDecisionScenarioGenerator.cs @@ -0,0 +1,25 @@ +using Btms.Common.Extensions; +using Btms.Types.Ipaffs; +using Microsoft.Extensions.Logging; +using TestDataGenerator.Helpers; + +namespace TestDataGenerator.Scenarios; + +public class CrNoMatchNoDecisionScenarioGenerator(ILogger logger) : ScenarioGenerator +{ + public override GeneratorResult Generate(int scenario, int item, DateTime entryDate, ScenarioConfig config) + { + var reference = DataHelpers.GenerateReferenceNumber(ImportNotificationTypeEnum.Cveda, scenario, entryDate, item); + + var clearanceRequest = GetClearanceRequestBuilder("cr-one-item") + .WithCreationDate(entryDate) + .WithArrivalDateTimeOffset(DateTime.Today.ToDate(), DateTime.Now.ToTime()) + .WithReferenceNumberOneToOne(reference) + .WithRandomItems(10, 100) + .ValidateAndBuild(); + + logger.LogInformation("Created {EntryReference}", clearanceRequest.Header!.EntryReference); + + return new GeneratorResult([clearanceRequest]); + } +} \ No newline at end of file From 26c10f4c1af3bd12a12a58b463d8d90bea87caff Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 2 Jan 2025 16:27:40 +0000 Subject: [PATCH 9/9] Fixes tests --- .../Fixtures/BasicSampleDataTestFixture.cs | 2 +- .../DecisionTests/ChedPSimpleTests.cs | 13 +++++- .../DecisionTests/ManyItemTests.cs | 46 +++++++++++++++++++ .../Decisions/NoMatchDecisionsTest.cs | 18 +++++--- .../Services/Matching/MatchingServiceTests.cs | 17 +++---- Btms.Business/Builders/MovementBuilder.cs | 8 ++-- 6 files changed, 83 insertions(+), 21 deletions(-) create mode 100644 Btms.Backend.IntegrationTests/DecisionTests/ManyItemTests.cs diff --git a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs index 0a7a3253..15760199 100644 --- a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs +++ b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs @@ -28,7 +28,7 @@ public BasicSampleDataTestFixture(IMessageSink messageSink) _mongoDbContext = rootScope.ServiceProvider.GetRequiredService(); // Would like to pick this up from env/config/DB state - var insertToMongo = false; + var insertToMongo = true; if (insertToMongo) { diff --git a/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs b/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs index d267ee1d..2aeaf7b4 100644 --- a/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs +++ b/Btms.Backend.IntegrationTests/DecisionTests/ChedPSimpleTests.cs @@ -131,12 +131,21 @@ public void ShouldHaveChedType() .GetResourceObjects() .Single(); + movement - .AlvsDecisionStatus - .Context! + .Status .ChedTypes .Should() .Equal(ImportNotificationTypeEnum.Cvedp); + + // + // movement + // .AlvsDecisionStatus + // .Context! + // .ChedTypes + // .Should() + // .Equal(ImportNotificationTypeEnum.Cvedp); + } [Fact(Skip = "Relationships aren't being deserialised correctly")] diff --git a/Btms.Backend.IntegrationTests/DecisionTests/ManyItemTests.cs b/Btms.Backend.IntegrationTests/DecisionTests/ManyItemTests.cs new file mode 100644 index 00000000..187e02f2 --- /dev/null +++ b/Btms.Backend.IntegrationTests/DecisionTests/ManyItemTests.cs @@ -0,0 +1,46 @@ +using System.Diagnostics.CodeAnalysis; +using Btms.Common.Extensions; +using Btms.Model; +using Btms.SyncJob; +using Btms.Backend.IntegrationTests.JsonApiClient; +using FluentAssertions; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Btms.Backend.IntegrationTests.Extensions; +using Btms.Backend.IntegrationTests.Helpers; +using Btms.Business.Commands; +using Btms.Model.Cds; +using Btms.Model.Ipaffs; +using Btms.Types.Alvs; +using TestDataGenerator.Scenarios; +using Json.More; +using Microsoft.Extensions.Logging; +using TestDataGenerator.Scenarios.ChedP; +using Xunit; +using Xunit.Abstractions; +using ImportNotification = Btms.Types.Ipaffs.ImportNotification; + +namespace Btms.Backend.IntegrationTests.DecisionTests; + +[Trait("Category", "Integration")] +public class ManyItemTests(InMemoryScenarioApplicationFactory factory, ITestOutputHelper testOutputHelper) + : BaseApiTests(factory, testOutputHelper, "DecisionTests"), IClassFixture> +{ + + [Fact] + public void ShouldHaveOneChedType() + { + // Act + var movementResource = Client.AsJsonApiClient() + .Get("api/movements") + // .Data + .GetResourceObjects() + .Single() + .Status.ChedTypes!.Count().Should().Be(1); + } + +} \ No newline at end of file diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index 492e6fe2..2ef160c1 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -78,13 +78,17 @@ private static List GenerateMovements(bool hasChecks) : new CrNoMatchNoChecksScenarioGenerator(NullLogger.Instance); var config = ScenarioFactory.CreateScenarioConfig(generator, 1, 1); - - var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config); + var movementBuilder = new MovementBuilder(NullLogger.Instance); - return generatorResult.Select(x => - { - var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)x); - return movementBuilder.From(internalClearanceRequest).Build(); - }).ToList(); + var generatorResult = generator + .Generate(1, 1, DateTime.UtcNow, config) + .First(x => x is AlvsClearanceRequest); + + var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)generatorResult); + var movement = movementBuilder + .From(internalClearanceRequest) + .Build(); + + return [movement]; } } \ No newline at end of file diff --git a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs index 8d948a18..0654b078 100644 --- a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs +++ b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs @@ -53,15 +53,16 @@ private static List GenerateMovements() var movementBuilder = new MovementBuilder(NullLogger.Instance); var config = ScenarioFactory.CreateScenarioConfig(generator, 1, 1); - var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config); + var generatorResult = generator + .Generate(1, 1, DateTime.UtcNow, config) + .First(x => x is AlvsClearanceRequest); - return generatorResult - .Select(x => - { - var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)x); - return movementBuilder.From(internalClearanceRequest).Build(); - }) - .ToList(); + var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)generatorResult); + var movement = movementBuilder + .From(internalClearanceRequest) + .Build(); + + return [movement]; } private static (List Notifications, List Movements) GenerateSimpleMatch() diff --git a/Btms.Business/Builders/MovementBuilder.cs b/Btms.Business/Builders/MovementBuilder.cs index 3db8df15..b692a0e4 100644 --- a/Btms.Business/Builders/MovementBuilder.cs +++ b/Btms.Business/Builders/MovementBuilder.cs @@ -42,9 +42,10 @@ public MovementBuilder From(Model.Cds.CdsClearanceRequest request) } }; - _movement - .AlvsDecisionStatus.Context - .ChedTypes = GetChedTypes(); + //TODO : Remove + // _movement + // .AlvsDecisionStatus.Context + // .ChedTypes = GetChedTypes(); return this; } @@ -63,6 +64,7 @@ private ImportNotificationTypeEnum[] GetChedTypes(List? items = null) .Select(d => d.DocumentCode!.GetChedType() ) + .Distinct() .Where(ct => ct.HasValue()) .Select(ct => ct!.Value) .ToArray()!;