1. Updated KeyDataPairsToDictionaryStringObjectJsonConverter to suppo…

…rt when data is null, and added some tests

2. Updated audit log to use a double when ValueKind is a number to support real numbers, and added some tests
3. Updated document to documents
4. Fixed an issue with concurrency within Linking due to the etag being empty

Btms.Backend.Data/InMemory/MemoryCollectionSet.cs

@@ -48,7 +48,7 @@ public Task Update(T item, string etag, IMongoDbTransaction transaction = defaul
 
         if ((existingItem._Etag ?? "") != etag)
         {
-            throw new ConcurrencyException("Concurrency Error, change this to a Concurrency exception");
+            throw new ConcurrencyException(item.Id!, etag);
         }
 
         item._Etag = BsonObjectIdGenerator.Instance.GenerateId(null, null).ToString()!;

Btms.Backend.Data/MigrationException.cs

@@ -1,3 +1,8 @@
 namespace Btms.Backend.Data;
 
-public class ConcurrencyException(string message) : Exception(message);
+public class ConcurrencyException(string entityId, string entityEtag) : Exception($"Failed up update {entityId} with etag {entityEtag}")
+{
+    public string EntityId { get; } = entityId;
+
+    public string EntityEtag { get; } = entityEtag;
+}

Btms.Backend.Data/Mongo/MongoCollectionSet.cs

@@ -50,6 +50,7 @@ public Task Insert(T item, IMongoDbTransaction transaction = null!, Cancellation
         public async Task Update(T item, string etag, IMongoDbTransaction transaction = null!,
             CancellationToken cancellationToken = default)
         {
+            ArgumentNullException.ThrowIfNull(etag);
             var builder = Builders<T>.Filter;
 
             var filter = builder.Eq(x => x.Id, item.Id) & builder.Eq(x => x._Etag, etag);
@@ -66,7 +67,7 @@ public async Task Update(T item, string etag, IMongoDbTransaction transaction =
 
             if (updateResult.ModifiedCount == 0)
             {
-                throw new ConcurrencyException("Concurrency Error, change this to a Concurrency exception");
+                throw new ConcurrencyException(item.Id!, etag);
             }
         }
 

Btms.Backend.Test/Auditing/AuditingTests.cs

@@ -0,0 +1,43 @@
+using Btms.Model.Auditing;
+using FluentAssertions;
+
+namespace Btms.Backend.Test.Auditing;
+
+public class AuditingTests
+{
+    public class TestClassOne
+    {
+        public double NumberValue { get; set; }
+    }
+
+    [Fact]
+    public void CreateAuditWhenDifferentIsDouble()
+    {
+        var previous = new TestClassOne() { NumberValue = 1.2 };
+        var current = new TestClassOne() { NumberValue = 2.2 };
+        var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow);
+
+        auditEntry.Should().NotBeNull();
+        auditEntry.Diff.Count.Should().Be(1);
+        auditEntry.Diff[0].Value.Should().Be(2.2);
+        auditEntry.Diff[0].Op.Should().Be("Replace");
+        auditEntry.Diff[0].Path.Should().Be("/NumberValue");
+
+    }
+
+
+    [Fact]
+    public void CreateAuditWhenDifferentIsInt()
+    {
+        var previous = new TestClassOne() { NumberValue = 1 };
+        var current = new TestClassOne() { NumberValue = 2 };
+        var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow);
+
+        auditEntry.Should().NotBeNull();
+        auditEntry.Diff.Count.Should().Be(1);
+        auditEntry.Diff[0].Value.Should().Be(2);
+        auditEntry.Diff[0].Op.Should().Be("Replace");
+        auditEntry.Diff[0].Path.Should().Be("/NumberValue");
+
+    }
+}

Btms.Backend.Test/JsonCoverters/KeyDataPairsToDictionaryStringObjectJsonConverterTests.cs

@@ -0,0 +1,47 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Btms.Types.Ipaffs;
+using FluentAssertions;
+
+namespace Btms.Backend.Test.JsonCoverters;
+
+public class KeyDataPairsToDictionaryStringObjectJsonConverterTests
+{
+    public class TestClass
+    {
+        [JsonPropertyName("keyDataPair")]
+        [JsonConverter(typeof(KeyDataPairsToDictionaryStringObjectJsonConverter))]
+        public IDictionary<string, object>? KeyDataPairs { get; set; }
+    }
+
+    [Fact]
+    public void GivenDataIsNotPresentInJson_ThenItShouldBeDeserializedAsNull()
+    {
+        var json =
+            "{\"keyDataPair\": [\r\n                        {\r\n                            \"key\": \"netweight\"\r\n                        },\r\n                        {\r\n                            \"key\": \"number_package\",\r\n                            \"data\": \"0\"\r\n                        },\r\n                        {\r\n                            \"key\": \"type_package\",\r\n                            \"data\": \"Case\"\r\n                        }\r\n                    ]}";
+
+        var result = JsonSerializer.Deserialize<TestClass>(json);
+
+        result.Should().NotBeNull();
+        result?.KeyDataPairs?.Count.Should().Be(3);
+        result?.KeyDataPairs?["netweight"].Should().BeNull();
+        result?.KeyDataPairs?["number_package"].Should().Be(0);
+        result?.KeyDataPairs?["type_package"].Should().Be("Case");
+
+    }
+
+    [Fact]
+    public void GivenValidJson_ThenEverythingShouldBeDeserialized()
+    {
+        var json =
+            "{\r\n\"keyDataPair\": [\r\n                        {\r\n                            \"key\": \"number_package\",\r\n                            \"data\": \"0\"\r\n                        },\r\n                        {\r\n                            \"key\": \"type_package\",\r\n                            \"data\": \"Case\"\r\n                        }\r\n                    ]\r\n}";
+
+        var result = JsonSerializer.Deserialize<TestClass>(json);
+
+        result.Should().NotBeNull();
+        result?.KeyDataPairs?.Count.Should().Be(2);
+        result?.KeyDataPairs?["number_package"].Should().Be(0);
+        result?.KeyDataPairs?["type_package"].Should().Be("Case");
+
+    }
+}

Btms.Business/Services/ImportNotificationLinkContext.cs

@@ -2,10 +2,10 @@
 
 namespace Btms.Business.Services;
 
-public record ImportNotificationLinkContext(ImportNotification ReceivedImportNotification, ImportNotification? ExistingImportNotification) : LinkContext
+public record ImportNotificationLinkContext(ImportNotification PersistedImportNotification, ImportNotification? ExistingImportNotification) : LinkContext
 {
     public override string GetIdentifiers()
     {
-        return ReceivedImportNotification._MatchReference;
+        return PersistedImportNotification._MatchReference;
     }
 }

Btms.Business/Services/LinkingService.cs

@@ -53,7 +53,7 @@ public async Task<LinkResult> Link(LinkContext linkContext, CancellationToken ca
                             return new LinkResult(LinkState.NotLinked);
                         }
 
-                        result = await FindMovementLinks(movementLinkContext.ReceivedMovement, cancellationToken);
+                        result = await FindMovementLinks(movementLinkContext.PersistedMovement, cancellationToken);
                         break;
                     case ImportNotificationLinkContext notificationLinkContext:
                         if (!ShouldLink(notificationLinkContext))
@@ -62,7 +62,7 @@ public async Task<LinkResult> Link(LinkContext linkContext, CancellationToken ca
                             return new LinkResult(LinkState.NotLinked);
                         }
 
-                        result = await FindImportNotificationLinks(notificationLinkContext.ReceivedImportNotification,
+                        result = await FindImportNotificationLinks(notificationLinkContext.PersistedImportNotification,
                             cancellationToken);
                         break;
                     default: throw new ArgumentException("context type not supported");
@@ -115,7 +115,6 @@ await dbContext.Notifications.Update(notification, notification._Etag, transacti
             }
             catch (Exception e)
             {
-                logger.LinkingFailed(e, linkContext.GetType().Name, linkContext.GetIdentifiers());
                 metrics.Faulted(e);
                 throw new LinkException(e);
             }
@@ -137,7 +136,7 @@ private static bool ShouldLink(MovementLinkContext movContext)
         if (movContext.ExistingMovement is null) return true;
 
         var existingItems = movContext.ExistingMovement.Items is null ? [] : movContext.ExistingMovement.Items;
-        var receivedItems = movContext.ReceivedMovement.Items is null ? [] : movContext.ReceivedMovement.Items;
+        var receivedItems = movContext.PersistedMovement.Items is null ? [] : movContext.PersistedMovement.Items;
 
         // Diff movements for fields of interest
         var existingDocs = existingItems
@@ -169,7 +168,7 @@ private static bool ShouldLink(ImportNotificationLinkContext notifContext)
                 c.CommodityId,
                 c.CommodityDescription
             }).ToList();
-        var receivedCommodities = notifContext.ReceivedImportNotification.Commodities?
+        var receivedCommodities = notifContext.PersistedImportNotification.Commodities?
             .Select(c => new
             {
                 c.CommodityId,

Btms.Business/Services/MovementLinkContext.cs

@@ -2,10 +2,10 @@
 
 namespace Btms.Business.Services;
 
-public record MovementLinkContext(Movement ReceivedMovement, Movement? ExistingMovement) : LinkContext
+public record MovementLinkContext(Movement PersistedMovement, Movement? ExistingMovement) : LinkContext
 {
     public override string GetIdentifiers()
     {
-        return string.Join(',', ReceivedMovement._MatchReferences);
+        return string.Join(',', PersistedMovement._MatchReferences);
     }
 }

Btms.Consumers/AlvsClearanceRequestConsumer.cs

@@ -8,6 +8,7 @@
 using SlimMessageBus;
 using System.Diagnostics.CodeAnalysis;
 using Btms.Consumers.Extensions;
+using Force.DeepCloner;
 using Items = Btms.Model.Alvs.Items;
 
 namespace Btms.Consumers
@@ -35,13 +36,15 @@ public async Task OnHandle(AlvsClearanceRequest message)
                 var internalClearanceRequest = AlvsClearanceRequestMapper.Map(message);
                 var movement = BuildMovement(internalClearanceRequest);
                 var existingMovement = await dbContext.Movements.Find(movement.Id!);
+                Movement persistedMovement = null!;
 
                 if (existingMovement is not null)
                 {
+                    persistedMovement = existingMovement.DeepClone();
                     if (movement.ClearanceRequests[0].Header?.EntryVersionNumber >
                         existingMovement.ClearanceRequests[0].Header?.EntryVersionNumber)
                     {
-                        movement.AuditEntries = existingMovement.AuditEntries;
+                        ////movement.AuditEntries = existingMovement.AuditEntries;
                         var auditEntry = AuditEntry.CreateUpdated(existingMovement.ClearanceRequests[0],
                             movement.ClearanceRequests[0],
                             BuildNormalizedAlvsPath(auditId!),
@@ -69,6 +72,7 @@ public async Task OnHandle(AlvsClearanceRequest message)
                 }
                 else
                 {
+                    persistedMovement = movement!;
                     var auditEntry = AuditEntry.CreateCreatedEntry(
                         movement.ClearanceRequests[0],
                         BuildNormalizedAlvsPath(auditId!),
@@ -78,7 +82,8 @@ public async Task OnHandle(AlvsClearanceRequest message)
                     await dbContext.Movements.Insert(movement);
                 }
 
-                var linkContext = new MovementLinkContext(movement, existingMovement);
+                //This should be existing, pre update (may need to clone)
+                var linkContext = new MovementLinkContext(persistedMovement, existingMovement);
                 var linkResult = await linkingService.Link(linkContext, Context.CancellationToken);
             }
         }

Btms.Consumers/Btms.Consumers.csproj

@@ -17,6 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="DeepCloner" Version="0.10.4" />
     <PackageReference Include="SlimMessageBus" Version="2.0.4" />
     <PackageReference Include="SlimMessageBus.Host.Memory" Version="2.5.3" />
   </ItemGroup>

Btms.Consumers/Extensions/ConsumerContextExtensions.cs

@@ -38,7 +38,7 @@ public static ActivityContext GetActivityContext(this IConsumerContext consumerC
 
     public static void Skipped(this IConsumerContext consumerContext)
     {
-        consumerContext.Properties.Add(MessageBusHeaders.Skipped, true);
+        consumerContext.Properties.TryAdd(MessageBusHeaders.Skipped, true);
     }
 
     public static bool WasSkipped(this IConsumerContext consumerContext)

Btms.Consumers/Interceptors/InMemoryConsumerErrorHandler.cs

@@ -18,7 +18,7 @@ private async Task<ConsumerErrorHandlerResult> AttemptRetry(T message, IConsumer
 
         if (retryCount > 5)
         {
-            logger.LogError(exception, "Error Consuming Message Retry count {RetryCount} - {Record}", retryCount, message?.ToJson());
+            logger.LogError(exception, "Error Consuming Message Retry count {RetryCount}", retryCount);
             return ConsumerErrorHandlerResult.Failure;
         }
 

Btms.Consumers/NotificationConsumer.cs

@@ -6,6 +6,7 @@
 using SlimMessageBus;
 using System.Diagnostics.CodeAnalysis;
 using Btms.Consumers.Extensions;
+using Force.DeepCloner;
 using Microsoft.Extensions.Logging;
 
 namespace Btms.Consumers
@@ -34,8 +35,10 @@ public async Task OnHandle(ImportNotification message)
 
 
                 var existingNotification = await dbContext.Notifications.Find(message.ReferenceNumber!);
+                Model.Ipaffs.ImportNotification persistedNotification = null!;
                 if (existingNotification is not null)
                 {
+                    persistedNotification = existingNotification.DeepClone();
                     if (internalNotification.UpdatedSource.TrimMicroseconds() >
                         existingNotification.UpdatedSource.TrimMicroseconds())
                     {
@@ -54,9 +57,10 @@ public async Task OnHandle(ImportNotification message)
                 {
                     internalNotification.Create(BuildNormalizedIpaffsPath(auditId!));
                     await dbContext.Notifications.Insert(internalNotification);
+                    persistedNotification = internalNotification!;
                 }
 
-                var linkContext = new ImportNotificationLinkContext(internalNotification, existingNotification);
+                var linkContext = new ImportNotificationLinkContext(persistedNotification, existingNotification);
                 var linkResult = await linkingService.Link(linkContext, Context.CancellationToken);
             }
         }

Btms.Model/Auditing/AuditEntry.cs

@@ -149,7 +149,7 @@ internal static AuditDiffEntry Internal(PatchOperation operation)
                         value = operation.Value.GetValue<string>();
                         break;
                     case JsonValueKind.Number:
-                        value = operation.Value.GetValue<long>();
+                        value = operation.Value.GetValue<double>();
                         break;
                     case JsonValueKind.True:
                     case JsonValueKind.False:

Btms.Model/MatchIdentifier.cs

@@ -50,4 +50,18 @@ public static MatchIdentifier FromCds(string reference)
 
         return new MatchIdentifier(identifier);
     }
+
+    public static bool TryFromCds(string reference, out MatchIdentifier matchIdentifier)
+    {
+        try
+        {
+            matchIdentifier = MatchIdentifier.FromCds(reference);
+            return true;
+        }
+        catch (Exception)
+        {
+            matchIdentifier = default;
+            return false;
+        }
+    }
 }

Btms.Model/Movement.cs

@@ -71,14 +71,18 @@ public List<string> _MatchReferences
                 {
                     foreach (var itemDocument in item.Documents!)
                     {
-                        list.Add(MatchIdentifier.FromCds(itemDocument.DocumentReference!).Identifier);
+                        if (MatchIdentifier.TryFromCds(itemDocument.DocumentReference!, out var identifier))
+                        {
+                            list.Add(identifier.Identifier);
+                        }
                     }
                 }
 
                 matchReferences = list.ToList();
             }
 
             return matchReferences;
+
         }
         set => matchReferences = value;
     }

Btms.Types.Alvs.V1/Items.g.cs

@@ -94,7 +94,7 @@ public partial class Items  //
     /// <summary>
     /// 
     /// </summary
-    [JsonPropertyName("document")]
+    [JsonPropertyName("documents")]
     public Document[]? Documents { get; set; }
 
 

Btms.Types.Ipaffs.V1/KeyDataPairsToDictionaryStringObjectJsonConverter.cs

@@ -58,11 +58,15 @@ public override bool CanConvert(Type typeToConvert)
                 var key = ExtractValue(ref reader, options);
 
                 reader.Read();
-                reader.Read();
-                var v = ExtractValue(ref reader, options);
+                object? value = null;
+                if (reader.TokenType != JsonTokenType.EndObject)
+                {
+                    reader.Read();
+                    value = ExtractValue(ref reader, options);
+                }
 
 
-                dictionary.Add(key?.ToString()!, v);
+                dictionary.Add(key?.ToString()!, value);
             }
         }