Fix "UTF-8" invalid encoding error

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## Changelog
 
+### 0.9.89 - 09/10/2024
+* Fixed a bug with case-insensitive string comparisons in the database. This would cause an "Invalid UTF-8" exception to be thrown
+
 ### 0.9.88 - 09/10/2024
 * Added support for historical databases. These are instances of `IDb` that contain all datoms, inserted, retracted, and historical.
 Can be useful for analytics or viewing the changes of an entity over time

src/NexusMods.MnemonicDB.Abstractions/ElementComparers/ValueComparer.cs

@@ -102,7 +102,7 @@ public static unsafe int CompareValues(ValueTags typeA, byte* aVal, int aLen, Va
             ValueTags.Float64 => CompareInternal<double>(aVal, bVal),
             ValueTags.Ascii => CompareAscii(aVal, aLen, bVal, bLen),
             ValueTags.Utf8 => CompareUtf8(aVal, aLen, bVal, bLen),
-            ValueTags.Utf8Insensitive => Utf8Comparer.Utf8CaseInsensitiveCompare(aVal, aLen, bVal, bLen),
+            ValueTags.Utf8Insensitive => CompareUtf8Insensitive(aVal, aLen, bVal, bLen),
             ValueTags.Blob => CompareBlobInternal(aVal, aLen, bVal, bLen),
             // HashedBlob is a special case, we compare the hashes not the blobs
             ValueTags.HashedBlob => CompareInternal<ulong>(aVal, bVal),
@@ -113,6 +113,16 @@ public static unsafe int CompareValues(ValueTags typeA, byte* aVal, int aLen, Va
         };
     }
 
+    private static unsafe int CompareUtf8Insensitive(byte* aVal, int aLen, byte* bVal, int bLen)
+    {
+        var aValOffset = aVal + sizeof(uint);
+        var bValOffset = bVal + sizeof(uint);
+
+        var aLenOffset = aLen - sizeof(uint);
+        var bLenOffset = bLen - sizeof(uint);
+        return Utf8Comparer.Utf8CaseInsensitiveCompare(aValOffset, aLenOffset, bValOffset, bLenOffset);
+    }
+
     private static unsafe int CompareTuples2(byte* aVal, int aLen, byte* bVal, int bLen)
     {
         var typeA1 = (ValueTags)aVal[0];

tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs

@@ -132,6 +132,20 @@ await Verify(merged.ToTable(AttributeCache))
             .UseParameters(type);
     }
 
+    [Fact]
+    public async Task CaseInsenstiveUTF8DoesntCrashTheComparator()
+    {
+        using var segment = new IndexSegmentBuilder(AttributeCache);
+        var id1 = NextTempId();
+        var id2 = NextTempId();
+        var id3 = NextTempId();
+        segment.Add(id1, File.Path, "/foo/bar");
+        segment.Add(id2, File.Path, "/foo/bar");
+        segment.Add(id3, File.Path, "/foo/bar");
+
+        var (tx, _) = await DatomStore.TransactAsync(segment.Build());
+    }
+
     private static Func<Datom, Datom, int> CompareDatoms(IDatomComparator comparer)
     {
         return (a, b) => comparer.CompareInstance(a, b);

tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs

@@ -18,10 +18,19 @@ public IDb AsOf(TxId txId)
         throw new NotSupportedException();
     }
 
+    public IDb History()
+    {
+        throw new NotImplementedException();
+    }
+
     public ITransaction BeginTransaction()
     {
         throw new NotSupportedException();
     }
 
     public IAnalyzer[] Analyzers => throw new NotSupportedException();
+    public Task<ulong> Excise(EntityId[] entityIds)
+    {
+        throw new NotImplementedException();
+    }
 }