Switch to TX functions for all transactions

CHANGELOG.md

@@ -1,5 +1,13 @@
 ## Changelog
 
+### 0.9.90 - 15/10/2024
+* Reworked serialization of Tuple attributes. The result more efficient (in space and time) but limited in the number and types of elements that can be stored in a Tuple.
+* Reworked the internals of the DatomStore to support more types of specialized transactions such as Excision
+* Implemented a basic form of schema migration.
+ * Indexes can be added and removed
+ * Attributes can be added and removed (removed attributes are ignored)
+ * Attribute types can be changed (in a very limited fashion, to be expanded in the future)
+
 ### 0.9.89 - 09/10/2024
 * Fixed a bug with case-insensitive string comparisons in the database. This would cause an "Invalid UTF-8" exception to be thrown
 

src/NexusMods.MnemonicDB.Abstractions/AttributeCache.cs

@@ -19,6 +19,7 @@ public sealed class AttributeCache
     private BitArray _isReference;
     private BitArray _isIndexed;
     private Symbol[] _symbols;
+    private ValueTag[] _valueTags;
     private BitArray _isNoHistory;
 
     public AttributeCache()
@@ -29,12 +30,14 @@ public AttributeCache()
         _isIndexed = new BitArray(maxId);
         _isNoHistory = new BitArray(maxId);
         _symbols = new Symbol[maxId];
+        _valueTags = new ValueTag[maxId];
 
         foreach (var kv in AttributeDefinition.HardcodedIds)
         {
             _attributeIdsBySymbol[kv.Key.Id] = AttributeId.From(kv.Value);
             _isIndexed[kv.Value] = kv.Key.IsIndexed;
             _symbols[kv.Value] = kv.Key.Id;
+            _valueTags[kv.Value] = kv.Key.LowLevelType;
         }
 
     }
@@ -104,7 +107,16 @@ public void Reset(IDb db)
             newIsCardinalityMany[(int)id] = AttributeDefinition.Cardinality.ReadValue(datom.ValueSpan, datom.Prefix.ValueTag, null!) == Cardinality.Many;
         }
         _isCardinalityMany = newIsCardinalityMany;
-
+
+        var valueTags = db.Datoms(AttributeDefinition.ValueType);
+        var newValueTags = new ValueTag[maxIndex];
+        foreach (var datom in valueTags)
+        {
+            var id = datom.E.Value;
+            var type = AttributeDefinition.ValueType.ReadValue(datom.ValueSpan, datom.Prefix.ValueTag, null!);
+            newValueTags[id] = type;
+        }
+        _valueTags = newValueTags;
     }
 
     /// <summary>
@@ -155,4 +167,28 @@ public Symbol GetSymbol(AttributeId id)
     {
         return _symbols[id.Value];
     }
+
+    /// <summary>
+    /// Returns true if the attribute is defined in the database.
+    /// </summary>
+    public bool Has(Symbol attribute)
+    {
+        return _attributeIdsBySymbol.ContainsKey(attribute);
+    }
+
+    /// <summary>
+    /// Try to get the AttributeId for the given attribute name
+    /// </summary>
+    public bool TryGetAttributeId(Symbol attribute, out AttributeId id)
+    {
+        return _attributeIdsBySymbol.TryGetValue(attribute, out id);
+    }
+
+    /// <summary>
+    /// Return the value tag type for the given attribute id
+    /// </summary>
+    public ValueTag GetValueTag(AttributeId aid)
+    {
+        return _valueTags[aid.Value];
+    }
 }

src/NexusMods.MnemonicDB.Abstractions/BuiltInEntities/AttributeDefinition.cs

@@ -84,13 +84,7 @@ public static void Insert(ITransaction tx, IAttribute attribute, ushort id = 0)
         { Documentation, 7 },
         { Transaction.Timestamp, 8}
     };
-
-    /// <summary>
-    /// All hardcoded attributes as a DbAttribute enumerable
-    /// </summary>
-    public static IEnumerable<DbAttribute> HardcodedDbAttributes => HardcodedIds.Select(a =>
-        new DbAttribute(a.Key.Id, AttributeId.From(a.Value), a.Key.LowLevelType, a.Key));
-
+
     /// <summary>
     /// Adds the initial set of attributes to the transaction, these will be created when the transaction is committed.
     /// </summary>

src/NexusMods.MnemonicDB.Abstractions/Datom.cs

@@ -95,7 +95,7 @@ public Datom Clone()
     /// <inheritdoc />
     public override string ToString()
     {
-        return $"[E: {E}, A: {A}, T: {T}, IsRetract: {IsRetract}, Value: {ValueSpan.Length}]";
+        return $"[E: {E}, A: {A}, T: {T}, IsRetract: {IsRetract}, Value: {Prefix.ValueTag.Read<object>(ValueSpan)}]";
     }
 
     /// <summary>

src/NexusMods.MnemonicDB.Abstractions/IConnection.cs

@@ -6,22 +6,6 @@
 
 namespace NexusMods.MnemonicDB.Abstractions;
 
-/// <summary>
-/// A database revision, which includes a datom and the datoms added to it.
-/// </summary>
-public struct Revision
-{
-    /// <summary>
-    /// The database for the most recent transaction
-    /// </summary>
-    public IDb Database;
-
-    /// <summary>
-    /// The datoms that were added in the most recent transaction
-    /// </summary>
-    public IndexSegment AddedDatoms;
-}
-
 /// <summary>
 ///     Represents a connection to a database.
 /// </summary>
@@ -87,5 +71,10 @@ public interface IConnection
     /// Deletes the entities with the given ids, also deleting them from any historical indexes. Returns the total number
     /// of datoms that were excised.
     /// </summary>
-    public Task<ulong> Excise(EntityId[] entityIds);
+    public Task<ICommitResult> Excise(EntityId[] entityIds);
+
+    /// <summary>
+    /// Update the database's schema with the given attributes.
+    /// </summary>
+    public Task UpdateSchema(params IAttribute[] attribute);
 }

src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs

@@ -1,10 +1,7 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
-using NexusMods.MnemonicDB.Abstractions.DatomIterators;
 using NexusMods.MnemonicDB.Abstractions.IndexSegments;
-using NexusMods.MnemonicDB.Abstractions.TxFunctions;
 
 namespace NexusMods.MnemonicDB.Abstractions;
 
@@ -39,31 +36,20 @@ public interface IDatomStore : IDisposable
     /// Any existing data will be deleted before importing.
     /// </summary>
     public Task ImportAsync(Stream stream);
-
-    /// <summary>
-    ///     Transacts (adds) the given datoms into the store.
-    /// </summary>
-    public Task<(StoreResult, IDb)> TransactAsync(IndexSegment datoms, HashSet<ITxFunction>? txFunctions = null);
-
-
+
     /// <summary>
-    ///     Transacts (adds) the given datoms into the store.
+    ///    Transacts (adds) the given datoms into the store.
     /// </summary>
-    public (StoreResult, IDb) Transact(IndexSegment datoms, HashSet<ITxFunction>? txFunctions = null);
-
+    public (StoreResult, IDb) Transact(IndexSegment segment);
+
+
     /// <summary>
-    ///     Registers new attributes with the store.
+    ///   Transacts (adds) the given datoms into the store.
     /// </summary>
-    /// <param name="newAttrs"></param>
-    void RegisterAttributes(IEnumerable<DbAttribute> newAttrs);
-
+    public Task<(StoreResult, IDb)> TransactAsync(IndexSegment segment);
+
     /// <summary>
     ///     Create a snapshot of the current state of the store.
     /// </summary>
     ISnapshot GetSnapshot();
-
-    /// <summary>
-    /// Deletes these datoms from any of the indexes they are in.
-    /// </summary>
-    ValueTask Excise(List<Datom> datomsToRemove);
 }

src/NexusMods.MnemonicDB.Abstractions/IInternalTxFunction.cs

@@ -0,0 +1,7 @@
+namespace NexusMods.MnemonicDB.Abstractions;
+
+/// <summary>
+/// A marker attribute for functions that are internal to the MnemonicDB, used mostly for schema updates
+/// and other bulk operations that are started via the TX Queue but are exposed to users through non transaction APIs
+/// </summary>
+public interface IInternalTxFunction;

src/NexusMods.MnemonicDB.Abstractions/ITransaction.cs

@@ -37,8 +37,7 @@ public interface ITransaction : IDisposable
     ///     Adds a new datom to the transaction
     /// </summary>
     void Add<TVal, TLowLevel>(EntityId entityId, Attribute<TVal, TLowLevel> attribute, TVal val, bool isRetract = false);
-
-
+
     /// <summary>
     ///     Adds datoms for adding the given ids to the transaction under the given attribute
     /// </summary>

src/NexusMods.MnemonicDB.Abstractions/Internals/KeyPrefix.cs

@@ -95,7 +95,7 @@ public bool IsRetract
     public ValueTag ValueTag
     {
         get => (ValueTag)((_lower >> 1) & 0x7F);
-        init => _lower = (_lower & 0xFF00000000000001) | ((ulong)value << 1);
+        init => _lower = (_lower & 0xFFFFFFFFFFFFFF01) | ((ulong)value << 1);
     }
 
     /// <summary>

src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs

@@ -110,6 +110,10 @@ public static SliceDescriptor Create(TxId tx)
     /// </summary>
     public static SliceDescriptor Create<THighLevel, TLowLevel>(Attribute<THighLevel, TLowLevel> attr, THighLevel value, AttributeCache attributeCache)
     {
+        var id = attributeCache.GetAttributeId(attr.Id);
+        if (attributeCache.GetValueTag(id) != ValueTag.Reference && !attributeCache.IsIndexed(id))
+            throw new InvalidOperationException($"Attribute {attr.Id} must be indexed or a reference");
+
         return new SliceDescriptor
         {
             Index = attr.IsReference ? IndexType.VAETCurrent : IndexType.AVETCurrent,
@@ -162,11 +166,11 @@ public static SliceDescriptor Create(AttributeId referenceAttribute, EntityId po
     /// Creates a slice descriptor for the given attribute from the current AEVT index
     /// reverse lookup.
     /// </summary>
-    public static SliceDescriptor Create(AttributeId referenceAttribute)
+    public static SliceDescriptor Create(AttributeId referenceAttribute, IndexType indexType = IndexType.AEVTCurrent)
     {
         return new SliceDescriptor
         {
-            Index = IndexType.AEVTCurrent,
+            Index = indexType,
             From = Datom(EntityId.MinValueNoPartition, referenceAttribute, TxId.MinValue, false),
             To = Datom(EntityId.MaxValueNoPartition, referenceAttribute, TxId.MaxValue, false)
         };

src/NexusMods.MnemonicDB.Abstractions/Serializer.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Buffers;
+using System.Collections.Generic;
 using System.IO.Hashing;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -189,8 +190,8 @@ private static void WriteAscii<TWriter>(string value, TWriter writer)
     {
         var length = (uint)value.Length;
         var span = writer.GetSpan(sizeof(uint) + value.Length);
-        MemoryMarshal.Write(span, ref length);
-        ASCII.GetBytes(value, span.Slice(sizeof(uint)));
+        MemoryMarshal.Write(span, length);
+        ASCII.GetBytes(value, span.SliceFast(sizeof(uint)));
         writer.Advance(sizeof(uint) + value.Length);
     }
 
@@ -199,8 +200,8 @@ private static void WriteUtf8<TWriter>(string value, TWriter writer)
     {
         var length = (uint)value.Length;
         var span = writer.GetSpan(sizeof(uint) + value.Length);
-        MemoryMarshal.Write(span, ref length);
-        UTF8.GetBytes(value, span.Slice(sizeof(uint)));
+        MemoryMarshal.Write(span, length);
+        UTF8.GetBytes(value, span.SliceFast(sizeof(uint)));
         writer.Advance(sizeof(uint) + value.Length);
     }
 
@@ -209,8 +210,8 @@ private static void WriteBlob<TWriter>(Memory<byte> value, TWriter writer)
     {
         var length = (uint)value.Length;
         var span = writer.GetSpan(sizeof(uint) + value.Length);
-        MemoryMarshal.Write(span, ref length);
-        value.Span.CopyTo(span.Slice(sizeof(uint)));
+        MemoryMarshal.Write(span, length);
+        value.Span.CopyTo(span.SliceFast(sizeof(uint)));
         writer.Advance(sizeof(uint) + value.Length);
     }
 
@@ -396,5 +397,37 @@ public static void Remap(this ValueTag tag, Span<byte> span, Func<EntityId, Enti
     }
     #endregion
 
+    #region ValueConversion
+
+    public static void ConvertValue<TWriter>(this ValueTag srcTag, ReadOnlySpan<byte> srcSpan, ValueTag destTag, TWriter destWriter)
+        where TWriter : IBufferWriter<byte>
+    {
+
+        try
+        {
+            switch (srcTag, destTag)
+            {
+                case (ValueTag.UInt8, ValueTag.UInt16):
+                    WriteUnmanaged((ushort)MemoryMarshal.Read<byte>(srcSpan), destWriter);
+                    break;
+                case (ValueTag.Utf8, ValueTag.UInt64):
+                {
+                    var val = srcTag.Read<string>(srcSpan);
+                    WriteUnmanaged(Convert.ToUInt64(val), destWriter);
+                    break;
+                }
+
+                default:
+                    throw new NotSupportedException("Conversion not supported from " + srcTag + " to " + destTag);
+            }
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException($"Failed to convert ({srcTag.Read<object>(srcSpan)}) value from " + srcTag + " to " + destTag, e);
+        }
+    }
+
+    #endregion
+
 }