Skip to content

Commit

Permalink
CDMS-200 switches country ched searching to use country of origin & r…
Browse files Browse the repository at this point in the history
…efactoring of test fixtures (#40)

* CDMS-200 switches country ched searching to use country of origin

CDMS-200 Refactored no match tests to re-use code

Refactoring Test Fixtures

Switched Analytics MaxVersion tests to use new fixtures

Some small refactoring to remove some generics

CDMS-200 adds additional decision tests, moves alvs decision check analytics into context so it appears in audit entry

Refactoring fixtures

Refactoring fixtures

* Finished refactoring of BaseTest class :fingers-crossed
  • Loading branch information
craigedmunds authored Jan 3, 2025
1 parent 4f680ac commit f50eea6
Show file tree
Hide file tree
Showing 62 changed files with 1,114 additions and 413 deletions.
1 change: 1 addition & 0 deletions Btms.Analytics.Tests/Btms.Analytics.Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
<ProjectReference Include="..\Btms.Analytics\Btms.Analytics.csproj" />
<ProjectReference Include="..\Btms.Consumers\Btms.Consumers.csproj" />
<ProjectReference Include="..\TestDataGenerator\TestDataGenerator.csproj" />
<ProjectReference Include="..\TestGenerator.IntegrationTesting.Backend\TestGenerator.IntegrationTesting.Backend.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using TestDataGenerator.Scenarios;
using TestGenerator.IntegrationTesting.Backend.Extensions;
using Xunit.Abstractions;

namespace Btms.Analytics.Tests.Fixtures;
Expand Down
1 change: 1 addition & 0 deletions Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using TestDataGenerator.Scenarios;
using TestGenerator.IntegrationTesting.Backend.Extensions;
using Xunit.Abstractions;

namespace Btms.Analytics.Tests.Fixtures;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public async Task WhenCalledWithCountry_ReturnsResults()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL"));
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "ES"));

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

Expand Down
193 changes: 137 additions & 56 deletions Btms.Analytics.Tests/MovementsByMaxVersionTests.cs
Original file line number Diff line number Diff line change
@@ -1,78 +1,159 @@
using Btms.Common.Extensions;
using Btms.Model.Ipaffs;
using FluentAssertions;
using TestDataGenerator.Scenarios;
using TestGenerator.IntegrationTesting.Backend;
using TestGenerator.IntegrationTesting.Backend.Extensions;
using TestGenerator.IntegrationTesting.Backend.Fixtures;
using Xunit;
using Xunit.Abstractions;

using Btms.Analytics.Tests.Fixtures;
using Btms.Model.Ipaffs;

namespace Btms.Analytics.Tests;

[Collection(nameof(BasicSampleDataTestCollection))]
public class MovementsByMaxVersionTests(
BasicSampleDataTestFixture basicSampleDataTestFixture,
ITestOutputHelper testOutputHelper)
// [Collection(nameof(BasicSampleDataTestCollection))]
// public class MovementsByMaxVersionTests(
// BasicSampleDataTestFixture basicSampleDataTestFixture,
// ITestOutputHelper testOutputHelper)
public class MovementsByMaxVersionTests(ITestOutputHelper output)
: BaseTest<CrNoMatchSingleItemWithDecisionScenarioGenerator>(output)
{
// [Fact]
// public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
// {
// testOutputHelper.WriteLine("Querying for aggregated data");
// var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
// .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()));
//
// testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
//
// result.Values.Count.Should().Be(2);
// result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
// }
//
// [Fact]
// public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
// {
// testOutputHelper.WriteLine("Querying for aggregated data");
// var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
// .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
//
// testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
//
// result.Values.Count.Should().Be(2);
// result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
// }
//
// [Fact]
// public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation()
// {
// testOutputHelper.WriteLine("Querying for aggregated data");
// var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
// .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
//
// testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
//
// result.Values.Count.Should().Be(2);
// result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
// }

[Fact]
public async Task WhenCalled_ReturnsResults()
{
TestOutputHelper.WriteLine("Querying for aggregated data");

var result = await BackendFixture
.BtmsClient
.GetAnalyticsDashboard(["movementsByMaxEntryVersion"],
dateFrom:DateTime.Today.AddDays(-1),
dateTo:DateTime.Today.AddDays(1));

TestOutputHelper.WriteLine($"{result.StatusCode} status");
result.IsSuccessStatusCode.Should().BeTrue(result.StatusCode.ToString());

var charts = await result.ToJsonDictionary();

TestOutputHelper.WriteLine($"movementsByMaxEntryVersion keys : {charts["movementsByMaxEntryVersion"].GetKeys()}");
TestOutputHelper.WriteLine($"result keys : {charts["movementsByMaxEntryVersion"]["values"]!.GetKeys()}");

charts["movementsByMaxEntryVersion"]["values"]!
.GetKeys()
.Length.Should().Be(1);

var val = charts["movementsByMaxEntryVersion"]["values"]!["1"]!
.GetValue<int>()
.Should()
.Be(1);
}

[Fact]
public async Task WhenCalledWithChedType_ReturnsResults()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]));
TestOutputHelper.WriteLine("Querying for aggregated data");

var result = await BackendFixture
.BtmsClient
.GetAnalyticsDashboard(["movementsByMaxEntryVersion"],
dateFrom:DateTime.Today.AddDays(-1),
dateTo:DateTime.Today.AddDays(1),
chedTypes: [ImportNotificationTypeEnum.Cvedp]);

TestOutputHelper.WriteLine($"{result.StatusCode} status");
result.IsSuccessStatusCode.Should().BeTrue(result.StatusCode.ToString());

var charts = await result.ToJsonDictionary();

TestOutputHelper.WriteLine($"movementsByMaxEntryVersion keys : {charts["movementsByMaxEntryVersion"].GetKeys()}");
TestOutputHelper.WriteLine($"result keys : {charts["movementsByMaxEntryVersion"]["values"]!.GetKeys()}");

charts["movementsByMaxEntryVersion"]["values"]!
.GetKeys()
.Length.Should().Be(1);

var val = charts["movementsByMaxEntryVersion"]["values"]!["1"]!
.GetValue<int>()
.Should()
.Be(1);
}

[Fact]
public async Task WhenCalledWithWrongChedType_ReturnsResults()
{
TestOutputHelper.WriteLine("Querying for aggregated data");

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
var result = await BackendFixture
.BtmsClient
.GetAnalyticsDashboard(["movementsByMaxEntryVersion"],
dateFrom:DateTime.Today.AddDays(-1),
dateTo:DateTime.Today.AddDays(1),
chedTypes: [ImportNotificationTypeEnum.Cveda]);

TestOutputHelper.WriteLine($"{result.StatusCode} status");
result.IsSuccessStatusCode.Should().BeTrue(result.StatusCode.ToString());

var charts = await result.ToJsonDictionary();

result.Values.Count.Should().Be(2);
TestOutputHelper.WriteLine($"movementsByMaxEntryVersion keys : {charts["movementsByMaxEntryVersion"].GetKeys()}");
TestOutputHelper.WriteLine($"result keys : {charts["movementsByMaxEntryVersion"]["values"]!.GetKeys()}");

charts["movementsByMaxEntryVersion"]["values"]!
.GetKeys()
.Length.Should().Be(0);
}

[Fact]
public async Task WhenCalledWithCountry_ReturnsResults()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL"));
TestOutputHelper.WriteLine("Querying for aggregated data");

var result = await BackendFixture
.BtmsClient
.GetAnalyticsDashboard(["movementsByMaxEntryVersion"],
dateFrom:DateTime.Today.AddDays(-1),
dateTo:DateTime.Today.AddDays(1),
country:"FR");

TestOutputHelper.WriteLine($"{result.StatusCode} status");
result.IsSuccessStatusCode.Should().BeTrue(result.StatusCode.ToString());

var charts = await result.ToJsonDictionary();

TestOutputHelper.WriteLine($"movementsByMaxEntryVersion keys : {charts["movementsByMaxEntryVersion"].GetKeys()}");
TestOutputHelper.WriteLine($"result keys : {charts["movementsByMaxEntryVersion"]["values"]!.GetKeys()}");

charts["movementsByMaxEntryVersion"]["values"]!
.GetKeys()
.Length.Should().Be(1);

var val = charts["movementsByMaxEntryVersion"]["values"]!["1"]!
.GetValue<int>()
.Should()
.Be(1);
}


[Fact]
public async Task WhenCalledWithWrongCountry_ReturnsResults()
{
TestOutputHelper.WriteLine("Querying for aggregated data");

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
var result = await BackendFixture
.BtmsClient
.GetAnalyticsDashboard(["movementsByMaxEntryVersion"],
dateFrom:DateTime.Today.AddDays(-1),
dateTo:DateTime.Today.AddDays(1),
country:"ES");

TestOutputHelper.WriteLine($"{result.StatusCode} status");
result.IsSuccessStatusCode.Should().BeTrue(result.StatusCode.ToString());

result.Values.Count.Should().Be(1);
var charts = await result.ToJsonDictionary();

TestOutputHelper.WriteLine($"movementsByMaxEntryVersion keys : {charts["movementsByMaxEntryVersion"].GetKeys()}");
TestOutputHelper.WriteLine($"result keys : {charts["movementsByMaxEntryVersion"]["values"]!.GetKeys()}");

charts["movementsByMaxEntryVersion"]["values"]!
.GetKeys()
.Length.Should().Be(0);
}
}
6 changes: 3 additions & 3 deletions Btms.Analytics/Extensions/MovementExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQue
.Where(m => (m.CreatedSource >= from && m.CreatedSource < to)
&& (country == null || m.DispatchCountryCode == country)
&& (chedTypes == null || !chedTypes!.Any() ||
!m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() ||
m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c))));
!m.BtmsStatus.ChedTypes!.Any() ||
m.BtmsStatus.ChedTypes!.Any(c => chedTypes!.Contains(c))));

}

Expand All @@ -31,7 +31,7 @@ public static IQueryable<MovementWithLinkStatus> SelectLinkStatus(this IQueryabl
Movement = m,
CreatedSource = m.CreatedSource!.Value,
Description =
m.Status.LinkStatus
m.BtmsStatus.LinkStatus
// m.Relationships.Notifications.Data.Count > 0 ? "Linked" :
// m.AlvsDecisionStatus!.Context!.AlvsCheckStatus!.AnyMatch ? "Investigate" :
// "Not Linked"
Expand Down
11 changes: 1 addition & 10 deletions Btms.Analytics/ImportNotificationsAggregationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -120,18 +120,9 @@ public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, Import
var data = context
.Notifications
.Where(n => (n.CreatedSource >= from && n.CreatedSource < to)
&& (country == null || n.PartOne!.Route!.TransitingStates!.Contains(country))

&& (country == null || n.CommoditiesSummary!.CountryOfOrigin! == country)
&& (chedTypes == null || chedTypes!.Length == 0 || n.ImportNotificationType == null || chedTypes!.Contains(n.ImportNotificationType!.Value))
)

// .Select(n => new { Notification = n, IsChedTypeMatch = chedTypes == null || n.ImportNotificationType == null || Array.IndexOf(chedTypes!, n.ImportNotificationType) > -1 })
// .Where(n => (n.Notification.CreatedSource >= from && n.Notification.CreatedSource < to)
// && (country == null || n.Notification.PartOne!.Route!.TransitingStates!.Contains(country))
// && n.IsChedTypeMatch
// )
// .Select(n => n.Notification)

.GroupBy(n => new { MaxVersion =
n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version )
})
Expand Down
7 changes: 3 additions & 4 deletions Btms.Analytics/MovementExceptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
var simplifiedMovementView = context
.Movements
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.Where(m => !m.BtmsStatus.Linked)
.Select(m => new
{
// TODO - we should think about pre-calculating this stuff and storing it on the movement...
Expand All @@ -29,9 +30,8 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
MaxEntryVersion = m.ClearanceRequests.Max(c => c.Header!.EntryVersionNumber) ?? 0,
LinkedCheds = m.Relationships.Notifications.Data.Count,
ItemCount = m.Items.Count,
ChedTypes = m.AlvsDecisionStatus.Context.ChedTypes,
// HasMatchDecisions = m.AlvsDecisionStatus.Context.AlvsCheckStatus != null && m.AlvsDecisionStatus.Context.AlvsCheckStatus.AnyMatch,
Status = m.Status,
ChedTypes = m.BtmsStatus.ChedTypes,
Status = m.BtmsStatus,
DecisionMatched = !m.AlvsDecisionStatus.Decisions
.OrderBy(d => d.Context.AlvsDecisionNumber)
.Reverse()
Expand All @@ -51,7 +51,6 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
LinkedCheds = m.LinkedCheds,
ItemCount = m.ItemCount,
ChedTypes = m.ChedTypes,
// HasMatchDecisions = m.HasMatchDecisions,
Status = m.Status,
HasNotificationRelationships = m.HasNotificationRelationships,
Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount,
Expand Down
6 changes: 3 additions & 3 deletions Btms.Analytics/MovementsAggregationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.SelectMany(m => m.AlvsDecisionStatus.Decisions.Select(
d => new {Decision = d, Movement = m } ))
.SelectMany(d => d.Decision.Checks.Select(c => new { d.Decision, d.Movement, Check = c}))
.SelectMany(d => d.Decision.Context.Checks.Select(c => new { d.Decision, d.Movement, Check = c}))
.GroupBy(d => new
{
d.Decision.Context.DecisionStatus,
Expand All @@ -306,7 +306,7 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
// Works
var summary = new SingleSeriesDataset() {
Values = mongoQuery
.GroupBy(q => q.Key.DecisionStatus ?? "TBC")
.GroupBy(q => q.Key.DecisionStatus ?? "Investigation Needed")
.ToDictionary(
g => g.Key,
g => g.Sum(k => k.Count)
Expand All @@ -320,7 +320,7 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
{
Fields = new Dictionary<string, string>()
{
{ "Classification", a.Key.DecisionStatus ?? "TBC" },
{ "Classification", a.Key.DecisionStatus ?? "Investigation Needed" },
{ "CheckCode", a.Key.CheckCode! },
{ "AlvsDecisionCode", a.Key.AlvsDecisionCode! },
{ "BtmsDecisionCode", a.Key.BtmsDecisionCode! }
Expand Down
21 changes: 5 additions & 16 deletions Btms.Backend.IntegrationTests/AnalyticsTests.cs
Original file line number Diff line number Diff line change
@@ -1,18 +1,7 @@
using System.Diagnostics.CodeAnalysis;
using Btms.Common.Extensions;
using Btms.Model;
using Btms.SyncJob;
using Btms.Backend.IntegrationTests.JsonApiClient;
using FluentAssertions;
using System.Net;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using Btms.Backend.IntegrationTests.Extensions;
using Btms.Backend.IntegrationTests.Helpers;
using Json.More;
using FluentAssertions;
using TestGenerator.IntegrationTesting.Backend.Extensions;
using Xunit;
using Xunit.Abstractions;

Expand All @@ -27,7 +16,7 @@ public class AnalyticsTests(ApplicationFactory factory, ITestOutputHelper testOu
public async Task GetIndividualMultiSeriesDatetimeChart()
{
//Act
var response = await Client.GetAnalyticsDashboard("importNotificationLinkingByCreated");
var response = await Client.GetAnalyticsDashboard(["importNotificationLinkingByCreated"]);

// Assert
response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString());
Expand All @@ -44,7 +33,7 @@ public async Task GetIndividualMultiSeriesDatetimeChart()
public async Task GetIndividualMultiSeriesChart()
{
//Act
var response = await Client.GetAnalyticsDashboard("lastMonthMovementsByItemCount");
var response = await Client.GetAnalyticsDashboard(["lastMonthMovementsByItemCount"]);

// Assert
response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString());
Expand All @@ -61,7 +50,7 @@ public async Task GetIndividualMultiSeriesChart()
public async Task GetIndividualSingleSeriesChart()
{
//Act
var response = await Client.GetAnalyticsDashboard("last7DaysImportNotificationsLinkingStatus");
var response = await Client.GetAnalyticsDashboard(["last7DaysImportNotificationsLinkingStatus"]);

// Assert
response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString());
Expand Down
Loading

0 comments on commit f50eea6

Please sign in to comment.