Reduce Serializer and Index Complexity

benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs

@@ -4,7 +4,7 @@
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
 using NexusMods.MnemonicDB.Abstractions;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.Paths;
 

benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadThenWriteBenchmarks.cs

@@ -1,7 +1,7 @@
 using System.Linq;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.Paths;

benchmarks/OneBillionDatomsTest/Program.cs

@@ -1,14 +1,12 @@
 using System;
 using System.Diagnostics;
-using System.Threading;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using NexusMods.MnemonicDB;
 using NexusMods.MnemonicDB.Abstractions;
-using NexusMods.MnemonicDB.Storage;
 using NexusMods.MnemonicDB.TestModel;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.Paths;
 
 var host = Host.CreateDefaultBuilder()

src/NexusMods.MnemonicDB.Abstractions/Attribute.cs

@@ -1,34 +1,30 @@
 using System;
 using System.Buffers;
-using System.Collections.Generic;
-using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-using System.Text;
 using NexusMods.MnemonicDB.Abstractions.DatomIterators;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
-using NexusMods.MnemonicDB.Abstractions.Exceptions;
 using NexusMods.MnemonicDB.Abstractions.Internals;
 using NexusMods.MnemonicDB.Abstractions.Models;
-using Reloaded.Memory.Extensions;
 
 namespace NexusMods.MnemonicDB.Abstractions;
 
 /// <summary>
 ///     Interface for a specific attribute
 /// </summary>
-/// <typeparam name="TValueType"></typeparam>
-public abstract class Attribute<TValueType, TLowLevelType> : IAttribute<TValueType>
+public abstract class Attribute<TValueType, TLowLevelType, TSerializer> : 
+      IWritableAttribute<TValueType>,
+      IReadableAttribute<TValueType>
+      where TSerializer : IValueSerializer<TLowLevelType>
 {
     protected Attribute(
-        ValueTag lowLevelType,
         string ns,
         string name,
         bool isIndexed = false,
         bool noHistory = false,
         Cardinality cardinality = Cardinality.One)
     {
-        LowLevelType = lowLevelType;
+
         Id = Symbol.Intern(ns, name);
         Cardinalty = cardinality;
         IsIndexed = isIndexed;
@@ -44,9 +40,9 @@ protected Attribute(
     /// Converts a high-level value to a low-level value
     /// </summary>
     protected abstract TValueType FromLowLevel(TLowLevelType value, AttributeResolver resolver);
-    
+
     /// <inheritdoc />
-    public ValueTag LowLevelType { get; }
+    public ValueTag LowLevelType => TSerializer.ValueTag;
 
     /// <inheritdoc />
     public Symbol Id { get; }
@@ -92,12 +88,20 @@ public ReadDatom Resolve(in Datom datom, AttributeResolver resolver)
         return new ReadDatom(in prefix, ReadValue(datom.ValueSpan, datom.Prefix.ValueTag, resolver), this);
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
+    private void AssertTag(ValueTag tag)
+    {
+        if (tag != LowLevelType)
+            throw new InvalidCastException($"Invalid value tag for attribute {Id.Name}");
+    }
+
     /// <summary>
     /// Reads the high level value from the given span
     /// </summary>
     public TValueType ReadValue(ReadOnlySpan<byte> span, ValueTag tag, AttributeResolver resolver)
     {
-        return FromLowLevel(tag.Read<TLowLevelType>(span), resolver);
+        AssertTag(tag);
+        return FromLowLevel(TSerializer.Read(span), resolver);
     }
 
     /// <summary>
@@ -180,7 +184,7 @@ public override string ToString()
         /// <summary>
         ///     Typed datom for this attribute
         /// </summary>
-        public ReadDatom(in KeyPrefix prefix, TValueType v, Attribute<TValueType, TLowLevelType> a)
+        public ReadDatom(in KeyPrefix prefix, TValueType v, Attribute<TValueType, TLowLevelType, TSerializer> a)
         {
             Prefix = prefix;
             TypedAttribute = a;
@@ -190,7 +194,7 @@ public ReadDatom(in KeyPrefix prefix, TValueType v, Attribute<TValueType, TLowLe
         /// <summary>
         /// The typed attribute for this datom
         /// </summary>
-        public readonly Attribute<TValueType, TLowLevelType> TypedAttribute;
+        public readonly Attribute<TValueType, TLowLevelType, TSerializer> TypedAttribute;
 
         /// <summary>
         /// The abstract attribute for this datom
@@ -218,7 +222,7 @@ public ReadDatom(in KeyPrefix prefix, TValueType v, Attribute<TValueType, TLowLe
         /// <inheritdoc />
         public void Retract(ITransaction tx)
         {
-            tx.Add(E, (Attribute<TValueType, TLowLevelType>)A, V, true);
+            tx.Add(E, (Attribute<TValueType, TLowLevelType, TSerializer>)A, V, true);
         }
 
         /// <inheritdoc />

src/NexusMods.MnemonicDB.Abstractions/AttributeCache.cs

@@ -50,7 +50,6 @@ public AttributeCache()
     /// <summary>
     /// Resets the cache, causing it to re-query the database for the latest definitions.
     /// </summary>
-    /// <param name="idb"></param>
     public void Reset(IDb db)
     {
         var symbols = db.Datoms(AttributeDefinition.UniqueId);

src/NexusMods.MnemonicDB.Abstractions/Attributes/CardinalityAttribute.cs

@@ -1,11 +1,12 @@
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// Used to mark the cardinality of an attribute in the database
 /// </summary>
-public sealed class CardinalityAttribute(string ns, string name) : ScalarAttribute<Cardinality, byte>(ValueTag.UInt8, ns, name)
+public sealed class CardinalityAttribute(string ns, string name) : ScalarAttribute<Cardinality, byte, UInt8Serializer>(ns, name)
 {
     /// <inheritdoc />
     protected override byte ToLowLevel(Cardinality value) => (byte)value;

src/NexusMods.MnemonicDB.Abstractions/Attributes/CollectionAttribute.cs

@@ -1,22 +1,20 @@
-using System.Collections.Generic;
-using NexusMods.MnemonicDB.Abstractions.ElementComparers;
-using NexusMods.MnemonicDB.Abstractions.IndexSegments;
+using NexusMods.MnemonicDB.Abstractions.IndexSegments;
 using NexusMods.MnemonicDB.Abstractions.Models;
-using NexusMods.MnemonicDB.Abstractions.Query;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// An attribute that represents a collection of values
 /// </summary>
-public abstract class CollectionAttribute<TValue, TLowLevel>(ValueTag tag, string ns, string name)
-    : Attribute<TValue, TLowLevel>(tag, ns, name, cardinality: Cardinality.Many)
+public abstract class CollectionAttribute<TValue, TLowLevel, TSerializer>(string ns, string name)
+    : Attribute<TValue, TLowLevel, TSerializer>(ns, name, cardinality: Cardinality.Many) 
+    where TSerializer : IValueSerializer<TLowLevel>
 {
 
     /// <summary>
     /// Gets all values for this attribute on the given entity
     /// </summary>
-    public Values<TValue, TLowLevel> Get(IHasIdAndIndexSegment ent)
+    public Values<TValue> Get(IHasIdAndIndexSegment ent)
     {
         var segment = ent.IndexSegment;
         var dbId = ent.Db.AttributeCache.GetAttributeId(Id);
@@ -30,16 +28,16 @@ public Values<TValue, TLowLevel> Get(IHasIdAndIndexSegment ent)
             {
                 i++;
             }
-            return new Values<TValue, TLowLevel>(segment, start, i, this, ent.Db.Connection.AttributeResolver);
+            return new Values<TValue>(segment, start, i, this, ent.Db.Connection.AttributeResolver);
         }
-        return new Values<TValue, TLowLevel>(segment, 0, 0, this, ent.Db.Connection.AttributeResolver);
+        return new Values<TValue>(segment, 0, 0, this, ent.Db.Connection.AttributeResolver);
     }
 
     /// <summary>
     /// Gets all values for this attribute on the given entity, this performs a lookup in the database
     /// so prefer using the overload with IHasIdAndIndexSegment if you already have the segment.
     /// </summary>
-    protected Values<TValue, TLowLevel> Get(IHasEntityIdAndDb ent)
+    protected Values<TValue> Get(IHasEntityIdAndDb ent)
     {
         var segment = ent.Db.Get(ent.Id);
         var dbId = ent.Db.AttributeCache.GetAttributeId(Id);
@@ -53,9 +51,9 @@ protected Values<TValue, TLowLevel> Get(IHasEntityIdAndDb ent)
             {
                 i++;
             }
-            return new Values<TValue, TLowLevel>(segment, start, i, this, ent.Db.Connection.AttributeResolver);
+            return new Values<TValue>(segment, start, i, this, ent.Db.Connection.AttributeResolver);
         }
-        return new Values<TValue, TLowLevel>(segment, 0, 0, this, ent.Db.Connection.AttributeResolver);
+        return new Values<TValue>(segment, 0, 0, this, ent.Db.Connection.AttributeResolver);
     }
 
     /// <summary>

src/NexusMods.MnemonicDB.Abstractions/Attributes/MarkerAttribute.cs

@@ -1,5 +1,6 @@
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.Models;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
@@ -9,7 +10,7 @@ namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 /// </summary>
 /// <param name="ns"></param>
 /// <param name="name"></param>
-public class MarkerAttribute(string ns, string name) : Attribute<Null, Null>(ValueTag.Null, ns, name)
+public class MarkerAttribute(string ns, string name) : Attribute<Null, Null, NullSerializer>(ns, name)
 {
     /// <inheritdoc />
     protected override Null ToLowLevel(Null value) => value;

src/NexusMods.MnemonicDB.Abstractions/Attributes/ReferenceAttribute.cs

@@ -1,12 +1,12 @@
-using NexusMods.MnemonicDB.Abstractions.ElementComparers;
-using NexusMods.MnemonicDB.Abstractions.Models;
+using NexusMods.MnemonicDB.Abstractions.Models;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// An attribute that references another entity.
 /// </summary>
-public class ReferenceAttribute(string ns, string name) : ScalarAttribute<EntityId, EntityId>(ValueTag.Reference, ns, name)
+public class ReferenceAttribute(string ns, string name) : ScalarAttribute<EntityId, EntityId, EntityIdSerializer>(ns, name)
 {
     /// <inheritdoc />
     protected override EntityId ToLowLevel(EntityId value) => value;

src/NexusMods.MnemonicDB.Abstractions/Attributes/ReferencesAttribute.cs

@@ -1,12 +1,13 @@
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.Models;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// Represents a collection of references to other entities.
 /// </summary>
-public class ReferencesAttribute(string ns, string name) : CollectionAttribute<EntityId, EntityId>(ValueTag.Reference, ns, name)
+public class ReferencesAttribute(string ns, string name) : CollectionAttribute<EntityId, EntityId, EntityIdSerializer>(ns, name)
 {
     /// <inheritdoc />
     protected override EntityId ToLowLevel(EntityId value) => value;

src/NexusMods.MnemonicDB.Abstractions/Attributes/ScalarAttribute.cs

@@ -8,8 +8,10 @@ namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 /// <summary>
 /// An attribute that represents a scalar value, where there is a 1:1 ratio between the attribute and the value.
 /// </summary>
-public abstract class ScalarAttribute<TValue, TLowLevel>(ValueTag tag, string ns, string name) :
-    Attribute<TValue, TLowLevel>(tag, ns, name) where TValue : notnull
+public abstract class ScalarAttribute<TValue, TLowLevel, TSerializer>(string ns, string name) :
+    Attribute<TValue, TLowLevel, TSerializer>(ns, name)
+    where TSerializer : IValueSerializer<TLowLevel>
+    where TValue : notnull
 {
     /// <summary>
     /// True if the attribute is optional, and not required by models

src/NexusMods.MnemonicDB.Abstractions/Attributes/StringAttribute.cs

@@ -1,11 +1,11 @@
-using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// An attribute that represents a string.
 /// </summary>
-public sealed class StringAttribute(string ns, string name) : ScalarAttribute<string, string>(ValueTag.Utf8, ns, name)
+public sealed class StringAttribute(string ns, string name) : ScalarAttribute<string, string, Utf8Serializer>(ns, name)
 {
     /// <inheritdoc />
     protected override string ToLowLevel(string value) => value;

src/NexusMods.MnemonicDB.Abstractions/Attributes/SymbolAttribute.cs

@@ -1,11 +1,11 @@
-using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// An attribute that encodes a symbol.
 /// </summary>
-public class SymbolAttribute(string ns, string name) : ScalarAttribute<Symbol, string>(ValueTag.Ascii, ns, name)
+public class SymbolAttribute(string ns, string name) : ScalarAttribute<Symbol, string, AsciiSerializer>(ns, name)
 {
     /// <inheritdoc />
     protected override string ToLowLevel(Symbol value) => value.Id;

src/NexusMods.MnemonicDB.Abstractions/Attributes/TimestampAttribute.cs

@@ -1,13 +1,14 @@
 using System;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.Models;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// An attribute that holds a timestamp
 /// </summary>
-public class TimestampAttribute(string ns, string name) : ScalarAttribute<DateTimeOffset, long>(ValueTag.Int64, ns, name)
+public class TimestampAttribute(string ns, string name) : ScalarAttribute<DateTimeOffset, long, Int64Serializer>(ns, name)
 {
     /// <inheritdoc />
     protected override long ToLowLevel(DateTimeOffset value) => value.ToUnixTimeMilliseconds();

src/NexusMods.MnemonicDB.Abstractions/Attributes/ULongAttribute.cs

@@ -1,11 +1,11 @@
-using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// UInt64 attribute (ulong)
 /// </summary>
-public sealed class ULongAttribute(string ns, string name) : ScalarAttribute<ulong, ulong>(ValueTag.UInt64, ns, name)
+public sealed class ULongAttribute(string ns, string name) : ScalarAttribute<ulong, ulong, UInt64Serializer>(ns, name)
 {
     /// <inheritdoc />
     protected override ulong ToLowLevel(ulong value) => value;

src/NexusMods.MnemonicDB.Abstractions/Attributes/ValuesTagAttribute.cs

@@ -1,11 +1,12 @@
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.Abstractions.Attributes;
 
 /// <summary>
 /// An attribute that represents a value tag value
 /// </summary>
-public sealed class ValuesTagAttribute(string ns, string name) : ScalarAttribute<ValueTag, byte>(ValueTag.UInt8, ns, name)
+public sealed class ValuesTagAttribute(string ns, string name) : ScalarAttribute<ValueTag, byte, UInt8Serializer>(ns, name)
 {
     /// <inheritdoc />
     protected override byte ToLowLevel(ValueTag value) => (byte)value;