diff --git a/NexusMods.MnemonicDB.sln b/NexusMods.MnemonicDB.sln index 50a1bb69..959fba1d 100644 --- a/NexusMods.MnemonicDB.sln +++ b/NexusMods.MnemonicDB.sln @@ -41,6 +41,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Storag EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneBillionDatomsTest", "benchmarks\OneBillionDatomsTest\OneBillionDatomsTest.csproj", "{EA397BAE-9726-486F-BC9B-87BD86DF157F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.SourceGenerator", "src\NexusMods.MnemonicDB.SourceGenerator\NexusMods.MnemonicDB.SourceGenerator.csproj", "{0EE4BFCE-9E72-4BCC-B179-416E16136A1E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -55,6 +57,7 @@ Global {73E074F9-250F-4D8A-8038-5B12DB761E98} = {0377EBE6-F147-4233-86AD-32C821B9567E} {33A3DA79-D3FD-46DC-8D14-82E23D5B608D} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {EA397BAE-9726-486F-BC9B-87BD86DF157F} = {72AFE85F-8C12-436A-894E-638ED2C92A76} + {0EE4BFCE-9E72-4BCC-B179-416E16136A1E} = {0377EBE6-F147-4233-86AD-32C821B9567E} EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {A92DED3D-BC67-4E04-9A06-9A1B302B3070}.Debug|Any CPU.ActiveCfg = Debug|Any CPU @@ -89,5 +92,9 @@ Global {EA397BAE-9726-486F-BC9B-87BD86DF157F}.Debug|Any CPU.Build.0 = Debug|Any CPU {EA397BAE-9726-486F-BC9B-87BD86DF157F}.Release|Any CPU.ActiveCfg = Release|Any CPU {EA397BAE-9726-486F-BC9B-87BD86DF157F}.Release|Any CPU.Build.0 = Release|Any CPU + {0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs index 568d0ede..1966e9b7 100644 --- a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs @@ -60,14 +60,14 @@ public async Task Setup() _db = Connection.Db; - _preLoaded = _entityIds.Select(id => File.Get(_db, id)).ToArray(); + _preLoaded = _entityIds.Select(id => File.As(_db, id)).ToArray(); } [Benchmark] public ulong ReadFiles() { ulong sum = 0; - sum += File.Get(_db, _readId).Size.Value; + sum += File.As(_db, _readId).Size.Value; return sum; } @@ -80,7 +80,7 @@ public ulong ReadProperty() [Benchmark] public long ReadAll() { - return _entityIds.Select(id => File.Get(_db, id)) + return _entityIds.Select(id => File.As(_db, id)) .Sum(e => (long)e.Size.Value); } @@ -94,7 +94,7 @@ public long ReadAllPreloaded() [Benchmark] public ulong ReadAllFromMod() { - var mod = Mod.Get(_db, _modId); + var mod = Mod.As(_db, _modId); ulong sum = 0; foreach (var file in mod.Files) { diff --git a/docs/index.md b/docs/index.md index 32be0531..18183700 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,125 +41,113 @@ RocksDB. However, the storage layer is abstracted, and any system that supports and backward), and atomic updates across multiple keys can be used. Currently, the only other storage backend is a in-memory backed based on Microsoft's `System.Collections.Immutable` library which contains the `ImmutableSortedSet` class. -## Datamodel -At the core of the application is a set of attribute definitions. These are defined in C# as implementations of the `IAttribute` -class. Most often these are defined as classes inside a containing class to better group them by name. Here is an example -of a simple set of attributes: +## Conceptual Overview +MnemonicDB stores data (as mentioned) in tuples of `[Entity, Attribute, Value, Transaction, Assert/Retract]`. *Everything* +in the database is stored on-disk in this format. This includes the schema, the indexes, and the data itself. Thus defining +a datamodel in this database starts by defining attributes that will be collected into "models". + +## Defining Attributes +Attributes are simply implementations of the `IAttribute` interface, and must inherit from the `Attribute` abstract +class. There is a lot of logic in these classes however, so it is recommended to use one of the provided helper classes such +as `ScalarAttribute` or `ReferenceAttribute`. In the definition of `Attribute`, `THighLevel` refers +to the C# type that the attribute will contain values of, and `TLowLevel` refers to the type that the database will use to +store the values. The attribute itself contains conversions to and from these types. ```csharp -public class FileAttributes -{ - /// - /// The path of the file - /// - public class Path() : Attribute(isIndexed: true); - - /// - /// The size of the file - /// - public class Size : Attribute; - - /// - /// The hashcode of the file - /// - public class Hash() : Attribute(isIndexed: true); - - /// - /// The mod this file belongs to - /// - public class ModId : Attribute; -} +var attr = new BooleanAttribute("Test.Namespace", "IsTest") { IsIndexed = true }; ``` -A few interesting features of the above code: - -* Each attribute is defined as a class, this is so the class can later be used in C# attributes such as `[From]` -for defining read models (more on this below). -* Attributes can be indexed, this is a hint to the database that this attribute will be queried on often, and it should -be included in the secondary or reverse index. In the above example all the entities where `Hash` is `X` can be found -very quickly, via a single lookup and scan of an index. -* Attributes define a value type. Also in the app, should be a `IValueSerializer` for each of these value types. But -this system is open-ended. The serializer is used to convert the value to and from a byte array for storage in the database, -and to perform comparisons on the values, so longs need not be stored in BigEndian order as they are not compared as byte -arrays, but as longs. In addition, arbitrary types can be stored in the database, so long as they can be compared in their -C# form. +In this example a new attribute is defined with the name `Test.Namespace/IsTest`. Names in MneumonicDB are in the format of +`namespace/name` and are used to uniquely identify the attribute. The `IsIndexed` property is a hint to the database that +this attribute will be queried on often, and it should be included in the secondary or reverse index. +However, now what the attribute is created, how is it used? It can't yet, instead it must be registered with the DI container. +When the database starts it will query all the instances of `IAttribute` in the container and register them with the database. -Once the attributes are defined, they must be registered with the DI container. +A shorthand for this is to make the attributes static members of a class, and then register them all at once with the `AddAttributeCollection` +extension method: ```csharp -services.AddAttributeCollection(); -``` +public class Person +{ + public const string Namespace = "Test.Model.Person"; -While attributes can be registered individually, it is recommended to group them into classes as shown above, and register -them at once via the `AddAttributeCollection` method. + public static readonly StringAttribute Name = new(Namespace, nameof(Name)) { IsIndexed = true }; + public static readonly UInt32Attribute Age = new(Namespace, nameof(Age)); -## Read Models +} + +services.AddAttributeCollection(typeof(Person)); +``` -For ease of use, MnemonicDB supports the concept of read models. These are classes that define a set of attributes that -are commonly queried together. There is nothing that requires these attributes to always be grouped in the same way, and -users are encouraged to define read models that make sense for their application. +By convention, the namespace is stored in a constant string, and the attributes are stored as static readonly members of the +class, using the `nameof` operator to get the name of the attribute. + +## Using Source Generators +Since a lot of code is required to easily define attributes, a source generator is provided to generate this code for you. +To define such a model, simply subclass your model from `IModelDefinition`. Be sure to include the `partial` keyword on +the class definition, as the source generator will generate the other half of the class. ```csharp -public class File(ITransaction? tx) : AReadModel(tx) +public partial class Person : IModelDefinition { - [From] - public required RelativePath Name { get; set; } - - [From] - public required Size Size { get; set; } - - [From] - public required Hash Hash { get; set; } - - [From] - public required EntityId ModId { get; init; } - - public static File Create(ITransaction tx, string name, Mod mod, Size size, Hash hash) - { - return new File(tx) - { - Name = name, - Size = size, - Hash = hash, - ModId = mod.Id - }; - } + private const string Namespace = "Test.Person"; + public static readonly StringAttribute Name = new(Namespace, nameof(Path)) {IsIndexed = true}; + public static readonly UInt32Attribute Age = new(Namespace, nameof(Age)); } ``` -The above class defines a read model for a file. It is a simple class with properties that are decorated with the `[From]` -attribute. This attribute is used to tell the database that when this read model is queried, it should include the values -for the given DB attributes. The `ITransaction` parameter is used during writes to the database, and can be ignored for now. +Once the source generator runs, you will find that a lot of helper methods and code has been added to the class, including: + +### Lookup Methods +The methods `.All()`, and `.Get()` are added to the class. These methods can be used to look up all models of this type, or +just a specific model by its entity ID. -Now that the read model is defined, it can be used to query the database. +### Write Model +A `Person.New` class is created so that new instances of this model can be created easily: ```csharp - var file = db.Get(eId); - Console.WriteLine($"File: {file.Name} Size: {file.Size} Hash: {file.Hash}"); +var txn = db.NewTransaction(); +var person = Person.New(txn) +{ + Name = "Test", + Age = 32 +}; + +await txn.Commit(); + ``` +Every member of the model is required (unless the data is optional), and the `New` method will return a new instance of the +model that has been attached to the transaction. Committing the transaction will write the data to the database. -## Writing data -Datoms (tuples of data) are written to the database via transactions. Since MnemonicDB is a single-writer database, transactions -are not applied directly and do not lock the database. Instead, transactions are created, and then shipped off to the -writer task which serializes writes to the backing store. +### ReadOnly Model +A `Person.ReadOnly` class is created so that instances of this model can be read from the database, these are read-only methods +and are returned by any methods that query the database. The `.Remap` method on the `.New` class will return a `ReadOnly` instance +given a transaction result. -Read models can be used to create new entities in the database. Here is an example of creating a new file entity using -the above constructor method. +!!!info + During creation of a new entity, the entity ID is not known until the transaction is committed. Thus the `.New` class +will often have a `.Id` that has a `Partition` type of `Tmp`, meaning it's a temporary Id and never exists in the database +as that specific id. During the commit, the logging methods in the database will assign a new entity ID for each used Temporary +id, and those will be returned in the transaction result. The `.Remap` method will then replace the temporary id with the +newly assigned id, and return a `ReadOnly` instance of the model. ```csharp - var tx = db.BeginTransaction(); - var file = File.Create(tx, "file.txt", mod, 1024, hash); - var txResult = await tx.Commit(); -``` -A key thing to note in this code is that each entity when it is created fresh, is given a temporary ID. These ids are -not unique, but are unique to the given transaction. Once `txResult` is returned, the tempId assigned to the entity -can be resolved to a real ID: +var txn = db.NewTransaction(); +var personNew = Person.New(txn) +{ + Name = "Test", + Age = 32 +}; -```csharp - file = db.Get(txResult[file.Id]); -``` +var result = await txn.Commit(); +var person = personNew.Remap(result); +// person.Name == "Test" +// person.Age == 32 +// person.Id != personNew.Id + +``` diff --git a/src/NexusMods.MnemonicDB.SourceGenerator/Analyzer.cs b/src/NexusMods.MnemonicDB.SourceGenerator/Analyzer.cs index cc4e529e..7884fdf8 100644 --- a/src/NexusMods.MnemonicDB.SourceGenerator/Analyzer.cs +++ b/src/NexusMods.MnemonicDB.SourceGenerator/Analyzer.cs @@ -1,5 +1,7 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; +using System.Security.Cryptography; using System.Text; using Microsoft.CodeAnalysis; @@ -29,7 +31,9 @@ public void Execute(GeneratorExecutionContext context) { var writer = new StringWriter(); Templates.RenderModel(modelAnalyzer, writer); - context.AddSource($"{modelAnalyzer.Name}.Generated.cs", writer.ToString()); + + var full = modelAnalyzer.Namespace.ToDisplayString().Replace(".", "_") + "_" + modelAnalyzer.Name; + context.AddSource($"{full}.Generated.cs", writer.ToString()); } return; diff --git a/src/NexusMods.MnemonicDB.SourceGenerator/NexusMods.MnemonicDB.SourceGenerator.csproj b/src/NexusMods.MnemonicDB.SourceGenerator/NexusMods.MnemonicDB.SourceGenerator.csproj index be07835d..062324f4 100644 --- a/src/NexusMods.MnemonicDB.SourceGenerator/NexusMods.MnemonicDB.SourceGenerator.csproj +++ b/src/NexusMods.MnemonicDB.SourceGenerator/NexusMods.MnemonicDB.SourceGenerator.csproj @@ -17,4 +17,5 @@ + diff --git a/src/NexusMods.MnemonicDB.SourceGenerator/Properties/launchSettings.json b/src/NexusMods.MnemonicDB.SourceGenerator/Properties/launchSettings.json new file mode 100644 index 00000000..3e2b1708 --- /dev/null +++ b/src/NexusMods.MnemonicDB.SourceGenerator/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "Generators": { + "commandName": "DebugRoslynComponent", + "targetProject": "../../tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj" + } + } +} diff --git a/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave b/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave index 60a55c9d..af421e65 100644 --- a/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave +++ b/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave @@ -2,6 +2,8 @@ @methodname RenderModel @model ModelAnalyzer +#nullable enable + using System; namespace {{= model.Namespace}}; @@ -12,6 +14,7 @@ using __DI__ = Microsoft.Extensions.DependencyInjection; using __COMPARERS__ = NexusMods.MnemonicDB.Abstractions.ElementComparers; using Microsoft.Extensions.DependencyInjection; using NexusMods.MnemonicDB.Abstractions; +using System.Diagnostics.CodeAnalysis; {{= model.Comments}} public partial class {{= model.Name}} { @@ -42,7 +45,7 @@ public partial class {{= model.Name}} { /// Assumes that the id given points to a {{= model.Name}} entity, and /// returns a ReadOnly model of the entity. /// - public static {{= model.Name}}.ReadOnly Get(__ABSTRACTIONS__.IDb db, __ABSTRACTIONS__.EntityId id) { + public static {{= model.Name}}.ReadOnly As(__ABSTRACTIONS__.IDb db, __ABSTRACTIONS__.EntityId id) { return new {{= model.Name}}.ReadOnly(db, id); } @@ -61,7 +64,24 @@ public partial class {{= model.Name}} { /// Returns all {{= model.Name}} entities in the database. /// public static IEnumerable<{{= model.Name}}.ReadOnly> All(__ABSTRACTIONS__.IDb db) { - return Get(db, __ABSTRACTIONS__.IDbExtensions.Intersection(db, RequiredAttributes)); + foreach (var id in __ABSTRACTIONS__.IDbExtensions.Intersection(db, RequiredAttributes)) + { + yield return new {{= model.Name}}.ReadOnly(db, id); + } + } + + /// + /// Tries to get the entity with the given id from the database, if the model is invalid, it returns false. + /// + public static bool TryGet(__ABSTRACTIONS__.IDb db, __ABSTRACTIONS__.EntityId id, [NotNullWhen(true)] out {{= model.Name}}.ReadOnly? result) + { + var model = As(db, id); + if (model.IsValid()) { + result = model; + return true; + } + result = default; + return false; } /// @@ -158,7 +178,7 @@ public partial class {{= model.Name}} { protected override string ModelName => "{{= model.Name}}"; - internal bool IsValid() + public virtual bool IsValid() { {{each attr in model.Attributes}} {{if !attr.IsMarker}} @@ -222,7 +242,7 @@ public static class {{= model.Name}}Extensions { } {{each include in model.Includes}} - public static bool TryGetAs{{= model.Name}}(this {{= include.ToDisplayString()}}.ReadOnly model, out {{= include.ToDisplayString()}}.ReadOnly result) { + public static bool TryGetAs{{= model.Name}}(this {{= include.ToDisplayString()}}.ReadOnly model, out {{= include.ToDisplayString()}}.ReadOnly? result) { var casted = new {{= include.ToDisplayString()}}.ReadOnly(model.Db, model.Id); if (casted.IsValid()) { result = casted; diff --git a/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs b/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs index d1700e95..35919cc8 100644 --- a/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs +++ b/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs @@ -108,7 +108,7 @@ protected SettingsTask VerifyTable(IEnumerable datoms) } await tx2.Commit(); - return Loadout.Get(Connection.Db, loadoutWritten.Id); + return Loadout.As(Connection.Db, loadoutWritten.Id); } public void Dispose() diff --git a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs index bdcc99ce..e9d9ad97 100644 --- a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs +++ b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs @@ -104,7 +104,7 @@ public async Task DbIsImmutable() var originalDb = Connection.Db; // Validate the data - var found = File.Get(originalDb, realId); + var found = File.As(originalDb, realId); await VerifyModel(found).UseTextForParameters("original data"); @@ -121,11 +121,11 @@ public async Task DbIsImmutable() newDb.BasisTxId.Value.Should().Be(originalDb.BasisTxId.Value + 1UL + (ulong)i, "transaction id should be incremented by 1 for each mutation at iteration " + i); - var newFound = File.Get(newDb, realId); + var newFound = File.As(newDb, realId); await VerifyModel(newFound).UseTextForParameters("mutated data " + i); // Validate the original data - var orignalFound = File.Get(originalDb, realId); + var orignalFound = File.As(originalDb, realId); await VerifyModel(orignalFound).UseTextForParameters("original data" + i); } } @@ -157,12 +157,12 @@ public async Task ReadModelsCanHaveExtraAttributes() var db = Connection.Db; // Original data exists - var readModel = File.Get(db, realId); + var readModel = File.As(db, realId); await VerifyModel(readModel).UseTextForParameters("file data"); // Extra data exists and can be read with a different read model - var archiveReadModel = ArchiveFile.Get(db, realId); + var archiveReadModel = ArchiveFile.As(db, realId); await VerifyModel(archiveReadModel).UseTextForParameters("archive file data"); readModel.Id.Should().Be(archiveReadModel.Id, "both models are the same entity"); @@ -360,7 +360,7 @@ public async Task CanCreateTempEntities() loadout.AddTo(tx); var result = await tx.Commit(); - var loaded = Loadout.Get(result.Db, result[loadout.Id!.Value]); + var loaded = Loadout.As(result.Db, result[loadout.Id!.Value]); loaded.Name.Should().Be("Test Loadout"); loadout.GetFirst(Loadout.Name).Should().Be("Test Loadout"); @@ -381,7 +381,7 @@ public async Task CanWorkWithMarkerAttributes() mod.AddTo(tx); var result = await tx.Commit(); - var reloaded = Mod.Get(result.Db, result[mod.Id!.Value]); + var reloaded = Mod.As(result.Db, result[mod.Id!.Value]); reloaded.IsMarked.Should().BeTrue(); } @@ -418,7 +418,7 @@ public async Task CanExecuteTxFunctions() await Task.WhenAll(tasks); var db = Connection.Db; - var loadoutRO = Loadout.Get(db, id); + var loadoutRO = Loadout.As(db, id); loadoutRO.Name.Should().Be("Test Loadout: 1001"); return; @@ -427,7 +427,7 @@ public async Task CanExecuteTxFunctions() // by the transaction executor void AddToName(ITransaction tx, IDb db, EntityId eid, int amount) { - var loadout = Loadout.Get(db, eid); + var loadout = Loadout.As(db, eid); var oldAmount = int.Parse(loadout.Name.Split(":")[1].Trim()); tx.Add(loadout.Id, Loadout.Name, $"Test Loadout: {(oldAmount + amount)}"); }