Skip to content

Commit

Permalink
Merge pull request #27 from Nexus-Mods/support-multi-value-attributes
Browse files Browse the repository at this point in the history 
Added support for multi-value attributes
  • Loading branch information
@halgari
halgari authored Apr 1, 2024
2 parents 1b54eb4 + 377f651 commit b883f9d
Show file tree
Hide file tree
Showing 33 changed files with 284 additions and 30 deletions.
7 changes: 7 additions & 0 deletions src/NexusMods.MneumonicDB.Abstractions/IDb.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,11 @@ public IEnumerable<IReadDatom> Datoms<TAttribute>()
/// Create a new iterator for the given index type.
/// </summary>
IDatomSource Iterate(IndexType index);

/// <summary>
/// Gets all values for the given attribute on the given entity. There's no reason to use this
/// on attributes that are not multi-valued.
/// </summary>
IEnumerable<TValueType> GetAll<TAttribute, TValueType>(ref ModelHeader model, EntityId modelId)
where TAttribute : IAttribute<TValueType>;
}
12 changes: 9 additions & 3 deletions src/NexusMods.MneumonicDB.Abstractions/ScalarAttribute.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using NexusMods.MneumonicDB.Abstractions.Internals;
using NexusMods.MneumonicDB.Abstractions.Models;

Expand All @@ -21,11 +22,11 @@ public class ScalarAttribute<TAttribute, TValueType> : IAttribute<TValueType>
protected ScalarAttribute(string uniqueName = "",
bool isIndexed = false,
bool keepHistory = true,
bool multiArity = false)
bool multiValued = false)
{
IsIndexed = isIndexed;
KeepHistory = keepHistory;
MultiArity = multiArity;
Multivalued = multiValued;
Id = uniqueName == "" ? Symbol.Intern(typeof(TAttribute).FullName!) : Symbol.InternPreSanitized(uniqueName);
}

Expand All @@ -37,7 +38,7 @@ protected ScalarAttribute(Symbol symbol)
Id = symbol;
}

public bool MultiArity { get; }
public bool Multivalued { get; }

public bool KeepHistory { get; }

Expand Down Expand Up @@ -128,6 +129,11 @@ public static TValueType Get(ref ModelHeader model)
return model.Db.Get<TAttribute, TValueType>(ref model, model.Id);
}

public static IEnumerable<TValueType> GetAll(ref ModelHeader model)
{
return model.Db.GetAll<TAttribute, TValueType>(ref model, model.Id);
}

/// <inheritdoc />
public static void Add(ref ModelHeader model, TValueType value)
{
Expand Down
5 changes: 1 addition & 4 deletions src/NexusMods.MneumonicDB.Storage/DatomStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,10 +330,7 @@ private unsafe void Log(PendingTransaction pendingTransaction, out StoreResult r
foreach (var datom in pendingTransaction.Data)
{
_writer.Reset();
unsafe
{
_writer.Advance(sizeof(KeyPrefix));
}
_writer.Advance(sizeof(KeyPrefix));

var isRemapped = Ids.IsPartition(datom.E.Value, Ids.Partition.Tmp);
datom.Explode(_registry, remapFn, out var e, out var a, _writer, out var isRetract);
Expand Down
13 changes: 9 additions & 4 deletions src/NexusMods.MneumonicDB/Db.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ internal class Db : IDb
private readonly Connection _connection;
private readonly AttributeRegistry _registry;
private readonly EntityCache _cache;
private readonly ConcurrentDictionary<EntityId, EntityId[]> _reverseCache = new();
private readonly ConcurrentDictionary<(EntityId, Type), EntityId[]> _reverseCache = new();
internal readonly ISnapshot Snapshot;

public Db(ISnapshot snapshot, Connection connection, TxId txId, AttributeRegistry registry)
Expand All @@ -39,7 +39,6 @@ public IEnumerable<TModel> Get<TModel>(IEnumerable<EntityId> ids)
{
yield return Get<TModel>(id);
}

}

public TValue Get<TAttribute, TValue>(ref ModelHeader header, EntityId id)
Expand All @@ -65,7 +64,7 @@ public TModel[] GetReverse<TAttribute, TModel>(EntityId id)
where TAttribute : IAttribute<EntityId>
where TModel : struct, IEntity
{
if (!_reverseCache.TryGetValue(id, out var eIds))
if (!_reverseCache.TryGetValue((id, typeof(TAttribute)), out var eIds))
{
using var attrSource = Snapshot.GetIterator(IndexType.VAETCurrent);
var attrId = _registry.GetAttributeId<TAttribute>();
Expand All @@ -76,7 +75,7 @@ public TModel[] GetReverse<TAttribute, TModel>(EntityId id)
.Select(c => c.CurrentKeyPrefix().E)
.ToArray();

_reverseCache[id] = eIds;
_reverseCache[(id, typeof(TAttribute))] = eIds;
}

var results = GC.AllocateUninitializedArray<TModel>(eIds.Length);
Expand Down Expand Up @@ -121,5 +120,11 @@ public IDatomSource Iterate(IndexType index)
return Snapshot.GetIterator(index);
}

public IEnumerable<TValueType> GetAll<TAttribute, TValueType>(ref ModelHeader model, EntityId modelId)
where TAttribute : IAttribute<TValueType>
{
return _cache.GetAll<TAttribute, TValueType>(ref model);
}

public void Dispose() { }
}
40 changes: 40 additions & 0 deletions src/NexusMods.MneumonicDB/EntityCache.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,4 +114,44 @@ private class Entry
public ushort[] Offsets = null!;
public byte[] Data = null!;
}

public IEnumerable<TValue> GetAll<TAttribute, TValue>(ref ModelHeader header)
where TAttribute : IAttribute<TValue>
{
Entry entry;
if (header.InlineCache is Entry inline)
entry = inline;
else if (!cache.TryGetValue(header.Id, out entry!))
entry = Cache(header.Id);

header.InlineCache = entry;

return GetAllInner<TAttribute, TValue>(entry);
}

private IEnumerable<TValue> GetAllInner<TAttribute, TValue>(Entry entry) where TAttribute : IAttribute<TValue>
{
var localCache = InlineCache<TAttribute, TValue>.Instance;
if (localCache.Registry != registry)
{
localCache = Recache<TAttribute, TValue>();
}

var attributeIndex = Array.IndexOf(entry.Attributes, localCache.Id);
if (attributeIndex == -1)
yield break;

while (attributeIndex < entry.Attributes.Length)
{
if (entry.Attributes[attributeIndex] != localCache.Id)
break;

var offset = entry.Offsets[attributeIndex];
var length = entry.Offsets[attributeIndex + 1] - offset;

localCache.Serializer.Read(entry.Data.AsSpan().SliceFast(offset, length), out var value);
yield return value;
attributeIndex++;
}
}
}
16 changes: 14 additions & 2 deletions tests/NexusMods.MneumonicDB.Storage.Tests/ABackendTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ public async Task InsertedDatomsShowUpInTheIndex(IndexType type)

var modId1 = NextTempId();
var modId2 = NextTempId();
var loadoutId = NextTempId();
var collectionId = NextTempId();

var tx = await DatomStore.Transact([
FileAttributes.Path.Assert(id1, "/foo/bar"),
Expand All @@ -44,16 +46,26 @@ public async Task InsertedDatomsShowUpInTheIndex(IndexType type)
FileAttributes.ModId.Assert(id1, modId1),
FileAttributes.ModId.Assert(id2, modId1),
ModAttributes.Name.Assert(modId1, "Test Mod 1"),
ModAttributes.Name.Assert(modId2, "Test Mod 2")
ModAttributes.LoadoutId.Assert(modId1, loadoutId),
ModAttributes.Name.Assert(modId2, "Test Mod 2"),
ModAttributes.LoadoutId.Assert(modId2, loadoutId),
LoadoutAttributes.Name.Assert(loadoutId, "Test Loadout 1"),
CollectionAttributes.Name.Assert(collectionId, "Test Collection 1"),
CollectionAttributes.LoadoutId.Assert(collectionId, loadoutId),
CollectionAttributes.Mods.Assert(collectionId, modId1),
CollectionAttributes.Mods.Assert(collectionId, modId2)
]);

id1 = tx.Remaps[id1];
id2 = tx.Remaps[id2];
collectionId = tx.Remaps[collectionId];

tx = await DatomStore.Transact([
// Rename file 1 and move file 1 to mod 2
FileAttributes.Path.Assert(id2, "/foo/qux"),
FileAttributes.ModId.Assert(id1, modId2)
FileAttributes.ModId.Assert(id1, modId2),
// Remove mod2 from collection
CollectionAttributes.Mods.Retract(collectionId, modId2),
]);

using var iterator = tx.Snapshot.GetIterator(type);
Expand Down
12 changes: 11 additions & 1 deletion tests/NexusMods.MneumonicDB.Storage.Tests/AStorageTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,17 @@ protected AStorageTest(IServiceProvider provider, Func<AttributeRegistry, IStore
new DbAttribute(Symbol.Intern<FileAttributes.ModId>(), AttributeId.From(23),
Symbol.Intern<EntityIdSerializer>()),
new DbAttribute(Symbol.Intern<ModAttributes.Name>(), AttributeId.From(24),
Symbol.Intern<StringSerializer>())
Symbol.Intern<StringSerializer>()),
new DbAttribute(Symbol.Intern<ModAttributes.LoadoutId>(), AttributeId.From(25),
Symbol.Intern<EntityIdSerializer>()),
new DbAttribute(Symbol.Intern<LoadoutAttributes.Name>(), AttributeId.From(26),
Symbol.Intern<StringSerializer>()),
new DbAttribute(Symbol.Intern<CollectionAttributes.Name>(), AttributeId.From(27),
Symbol.Intern<StringSerializer>()),
new DbAttribute(Symbol.Intern<CollectionAttributes.LoadoutId>(), AttributeId.From(28),
Symbol.Intern<EntityIdSerializer>()),
new DbAttribute(Symbol.Intern<CollectionAttributes.Mods>(), AttributeId.From(29),
Symbol.Intern<EntityIdSerializer>())
]);
_path = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory).Combine("tests_datomstore" + Guid.NewGuid());

Expand Down
10 changes: 8 additions & 2 deletions ...TestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,13 @@
+ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002
2 changes: 2 additions & 0 deletions ...TestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
+ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003
10 changes: 8 additions & 2 deletions ...TestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,16 @@
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
+ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003
+ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002
2 changes: 2 additions & 0 deletions ...TestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003
12 changes: 10 additions & 2 deletions ...ackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,14 @@
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
+ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003
6 changes: 5 additions & 1 deletion ...TestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,6 @@
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
2 changes: 2 additions & 0 deletions ...TestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003
10 changes: 8 additions & 2 deletions ...ackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,13 @@
+ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
+ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003
+ | 0200000000000001 | (0017) ModId | 0200000000000007 | 0100000000000003
+ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002
+ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000005 | (0018) Name | Test Mod 2 | 0100000000000002
+ | 0200000000000003 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000005 | (0019) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000004 | (001A) Name | Test Loadout 1 | 0100000000000002
+ | 0200000000000006 | (001B) Name | Test Collection 1 | 0100000000000002
+ | 0200000000000006 | (001C) LoadoutId | 0200000000000004 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000005 | 0100000000000002
2 changes: 2 additions & 0 deletions ...ackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
+ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002
+ | 0200000000000006 | (001D) Mods | 0200000000000003 | 0100000000000002
- | 0200000000000006 | (001D) Mods | 0200000000000007 | 0100000000000003
Loading

0 comments on commit b883f9d

Please sign in to comment.