More template/codegen fixes

NexusMods.MnemonicDB.sln

@@ -41,6 +41,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Storag
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneBillionDatomsTest", "benchmarks\OneBillionDatomsTest\OneBillionDatomsTest.csproj", "{EA397BAE-9726-486F-BC9B-87BD86DF157F}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.SourceGenerator", "src\NexusMods.MnemonicDB.SourceGenerator\NexusMods.MnemonicDB.SourceGenerator.csproj", "{0EE4BFCE-9E72-4BCC-B179-416E16136A1E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -55,6 +57,7 @@ Global
 		{73E074F9-250F-4D8A-8038-5B12DB761E98} = {0377EBE6-F147-4233-86AD-32C821B9567E}
 		{33A3DA79-D3FD-46DC-8D14-82E23D5B608D} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE}
 		{EA397BAE-9726-486F-BC9B-87BD86DF157F} = {72AFE85F-8C12-436A-894E-638ED2C92A76}
+		{0EE4BFCE-9E72-4BCC-B179-416E16136A1E} = {0377EBE6-F147-4233-86AD-32C821B9567E}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{A92DED3D-BC67-4E04-9A06-9A1B302B3070}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -89,5 +92,9 @@ Global
 		{EA397BAE-9726-486F-BC9B-87BD86DF157F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EA397BAE-9726-486F-BC9B-87BD86DF157F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EA397BAE-9726-486F-BC9B-87BD86DF157F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0EE4BFCE-9E72-4BCC-B179-416E16136A1E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs

@@ -60,14 +60,14 @@ public async Task Setup()
 
         _db = Connection.Db;
 
-        _preLoaded = _entityIds.Select(id => File.Get(_db, id)).ToArray();
+        _preLoaded = _entityIds.Select(id => File.As(_db, id)).ToArray();
     }
 
     [Benchmark]
     public ulong ReadFiles()
     {
         ulong sum = 0;
-        sum += File.Get(_db, _readId).Size.Value;
+        sum += File.As(_db, _readId).Size.Value;
         return sum;
     }
 
@@ -80,7 +80,7 @@ public ulong ReadProperty()
     [Benchmark]
     public long ReadAll()
     {
-        return _entityIds.Select(id => File.Get(_db, id))
+        return _entityIds.Select(id => File.As(_db, id))
             .Sum(e => (long)e.Size.Value);
     }
 
@@ -94,7 +94,7 @@ public long ReadAllPreloaded()
     [Benchmark]
     public ulong ReadAllFromMod()
     {
-        var mod = Mod.Get(_db, _modId);
+        var mod = Mod.As(_db, _modId);
         ulong sum = 0;
         foreach (var file in mod.Files)
         {

docs/index.md

@@ -41,125 +41,113 @@ RocksDB. However, the storage layer is abstracted, and any system that supports
 and backward), and atomic updates across multiple keys can be used. Currently, the only other storage backend is a in-memory
 backed based on Microsoft's `System.Collections.Immutable` library which contains the `ImmutableSortedSet` class.
 
-## Datamodel
-At the core of the application is a set of attribute definitions. These are defined in C# as implementations of the `IAttribute`
-class. Most often these are defined as classes inside a containing class to better group them by name. Here is an example
-of a simple set of attributes:
+## Conceptual Overview
+MnemonicDB stores data (as mentioned) in tuples of `[Entity, Attribute, Value, Transaction, Assert/Retract]`. *Everything*
+in the database is stored on-disk in this format. This includes the schema, the indexes, and the data itself. Thus defining
+a datamodel in this database starts by defining attributes that will be collected into "models".
+
+## Defining Attributes
+Attributes are simply implementations of the `IAttribute` interface, and must inherit from the `Attribute<THighLevel, TLowLevel>` abstract
+class. There is a lot of logic in these classes however, so it is recommended to use one of the provided helper classes such
+as `ScalarAttribute<T>` or `ReferenceAttribute<T>`. In the definition of `Attribute<THighLevel, TLowLevel>`, `THighLevel` refers
+to the C# type that the attribute will contain values of, and `TLowLevel` refers to the type that the database will use to
+store the values. The attribute itself contains conversions to and from these types.
 
 ```csharp
-public class FileAttributes
-{
-    /// <summary>
-    ///     The path of the file
-    /// </summary>
-    public class Path() : Attribute<Path, RelativePath>(isIndexed: true);
-
-    /// <summary>
-    ///     The size of the file
-    /// </summary>
-    public class Size : Attribute<Size, Paths.Size>;
-
-    /// <summary>
-    ///     The hashcode of the file
-    /// </summary>
-    public class Hash() : Attribute<Hash, Hashing.xxHash64.Hash>(isIndexed: true);
-
-    /// <summary>
-    ///     The mod this file belongs to
-    /// </summary>
-    public class ModId : Attribute<ModId, EntityId>;
-}
+var attr = new BooleanAttribute("Test.Namespace", "IsTest") { IsIndexed = true };
 ```
 
-A few interesting features of the above code:
-
-* Each attribute is defined as a class, this is so the class can later be used in C# attributes such as `[From<Path>]`
-for defining read models (more on this below).
-* Attributes can be indexed, this is a hint to the database that this attribute will be queried on often, and it should
-be included in the secondary or reverse index. In the above example all the entities where `Hash` is `X` can be found
-very quickly, via a single lookup and scan of an index.
-* Attributes define a value type. Also in the app, should be a `IValueSerializer<T>` for each of these value types. But
-this system is open-ended. The serializer is used to convert the value to and from a byte array for storage in the database,
-and to perform comparisons on the values, so longs need not be stored in BigEndian order as they are not compared as byte
-arrays, but as longs. In addition, arbitrary types can be stored in the database, so long as they can be compared in their
-C# form.
+In this example a new attribute is defined with the name `Test.Namespace/IsTest`. Names in MneumonicDB are in the format of
+`namespace/name` and are used to uniquely identify the attribute. The `IsIndexed` property is a hint to the database that
+this attribute will be queried on often, and it should be included in the secondary or reverse index.
 
+However, now what the attribute is created, how is it used? It can't yet, instead it must be registered with the DI container.
+When the database starts it will query all the instances of `IAttribute` in the container and register them with the database.
 
-Once the attributes are defined, they must be registered with the DI container.
+A shorthand for this is to make the attributes static members of a class, and then register them all at once with the `AddAttributeCollection`
+extension method:
 
 ```csharp
-services.AddAttributeCollection<FileAttributes>();
-```
+public class Person
+{
+    public const string Namespace = "Test.Model.Person";
 
-While attributes can be registered individually, it is recommended to group them into classes as shown above, and register
-them at once via the `AddAttributeCollection` method.
+    public static readonly StringAttribute Name = new(Namespace, nameof(Name)) { IsIndexed = true };
+    public static readonly UInt32Attribute Age = new(Namespace, nameof(Age));
 
-## Read Models
+}
+
+services.AddAttributeCollection(typeof(Person));
+```
 
-For ease of use, MnemonicDB supports the concept of read models. These are classes that define a set of attributes that
-are commonly queried together. There is nothing that requires these attributes to always be grouped in the same way, and
-users are encouraged to define read models that make sense for their application.
+By convention, the namespace is stored in a constant string, and the attributes are stored as static readonly members of the
+class, using the `nameof` operator to get the name of the attribute.
+
+## Using Source Generators
+Since a lot of code is required to easily define attributes, a source generator is provided to generate this code for you.
+To define such a model, simply subclass your model from `IModelDefinition`. Be sure to include the `partial` keyword on
+the class definition, as the source generator will generate the other half of the class.
 
 ```csharp
-public class File(ITransaction? tx) : AReadModel<File>(tx)
+public partial class Person : IModelDefinition
 {
-    [From<FileAttributes.Path>]
-    public required RelativePath Name { get; set; }
-
-    [From<FileAttributes.Size>]
-    public required Size Size { get; set; }
-
-    [From<FileAttributes.Hash>]
-    public required Hash Hash { get; set; }
-
-    [From<FileAttributes.ModId>]
-    public required EntityId ModId { get; init; }
-
-    public static File Create(ITransaction tx, string name, Mod mod, Size size, Hash hash)
-    {
-        return new File(tx)
-        {
-            Name = name,
-            Size = size,
-            Hash = hash,
-            ModId = mod.Id
-        };
-    }
+    private const string Namespace = "Test.Person";
+    public static readonly StringAttribute Name = new(Namespace, nameof(Path)) {IsIndexed = true};
+    public static readonly UInt32Attribute Age = new(Namespace, nameof(Age));
 }
 ```
 
-The above class defines a read model for a file. It is a simple class with properties that are decorated with the `[From<Attribute>]`
-attribute. This attribute is used to tell the database that when this read model is queried, it should include the values
-for the given DB attributes. The `ITransaction` parameter is used during writes to the database, and can be ignored for now.
+Once the source generator runs, you will find that a lot of helper methods and code has been added to the class, including:
+
+### Lookup Methods
+The methods `.All()`, and `.Get()` are added to the class. These methods can be used to look up all models of this type, or
+just a specific model by its entity ID.
 
-Now that the read model is defined, it can be used to query the database.
+### Write Model
+A `Person.New` class is created so that new instances of this model can be created easily:
 
 ```csharp
-    var file = db.Get<File>(eId);
 
-    Console.WriteLine($"File: {file.Name} Size: {file.Size} Hash: {file.Hash}");
+var txn = db.NewTransaction();
+var person = Person.New(txn)
+{
+    Name = "Test",
+    Age = 32
+};
+
+await txn.Commit();
+
 ```
+Every member of the model is required (unless the data is optional), and the `New` method will return a new instance of the
+model that has been attached to the transaction. Committing the transaction will write the data to the database.
 
-## Writing data
-Datoms (tuples of data) are written to the database via transactions. Since MnemonicDB is a single-writer database, transactions
-are not applied directly and do not lock the database. Instead, transactions are created, and then shipped off to the
-writer task which serializes writes to the backing store.
+### ReadOnly Model
+A `Person.ReadOnly` class is created so that instances of this model can be read from the database, these are read-only methods
+and are returned by any methods that query the database. The `.Remap` method on the `.New` class will return a `ReadOnly` instance
+given a transaction result.
 
-Read models can be used to create new entities in the database. Here is an example of creating a new file entity using
-the above constructor method.
+!!!info
+    During creation of a new entity, the entity ID is not known until the transaction is committed. Thus the `.New` class
+will often have a `.Id` that has a `Partition` type of `Tmp`, meaning it's a temporary Id and never exists in the database
+as that specific id. During the commit, the logging methods in the database will assign a new entity ID for each used Temporary
+id, and those will be returned in the transaction result. The `.Remap` method will then replace the temporary id with the
+newly assigned id, and return a `ReadOnly` instance of the model.
 
 ```csharp
-    var tx = db.BeginTransaction();
-    var file = File.Create(tx, "file.txt", mod, 1024, hash);
-    var txResult = await tx.Commit();
-```
 
-A key thing to note in this code is that each entity when it is created fresh, is given a temporary ID. These ids are
-not unique, but are unique to the given transaction. Once `txResult` is returned, the tempId assigned to the entity
-can be resolved to a real ID:
+var txn = db.NewTransaction();
+var personNew = Person.New(txn)
+{
+    Name = "Test",
+    Age = 32
+};
 
-```csharp
-    file = db.Get<File>(txResult[file.Id]);
-```
+var result = await txn.Commit();
 
+var person = personNew.Remap(result);
 
+// person.Name == "Test"
+// person.Age == 32
+// person.Id != personNew.Id
+
+```

src/NexusMods.MnemonicDB.SourceGenerator/Analyzer.cs

@@ -1,5 +1,7 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Security.Cryptography;
 using System.Text;
 using Microsoft.CodeAnalysis;
 
@@ -29,7 +31,9 @@ public void Execute(GeneratorExecutionContext context)
         {
             var writer = new StringWriter();
             Templates.RenderModel(modelAnalyzer, writer);
-            context.AddSource($"{modelAnalyzer.Name}.Generated.cs", writer.ToString());
+
+            var full = modelAnalyzer.Namespace.ToDisplayString().Replace(".", "_") + "_" + modelAnalyzer.Name;
+            context.AddSource($"{full}.Generated.cs", writer.ToString());
         }
         return;
 

src/NexusMods.MnemonicDB.SourceGenerator/NexusMods.MnemonicDB.SourceGenerator.csproj

@@ -17,4 +17,5 @@
       <WeaveTemplate Include="Template.weave" />
     </ItemGroup>
 
+    <Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))"/>
 </Project>

src/NexusMods.MnemonicDB.SourceGenerator/Properties/launchSettings.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "Generators": {
+      "commandName": "DebugRoslynComponent",
+      "targetProject": "../../tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj"
+    }
+  }
+}

src/NexusMods.MnemonicDB.SourceGenerator/Template.weave

@@ -2,6 +2,8 @@
 @methodname RenderModel
 @model ModelAnalyzer
 
+#nullable enable
+
 using System;
 
 namespace {{= model.Namespace}};
@@ -12,6 +14,7 @@ using __DI__ = Microsoft.Extensions.DependencyInjection;
 using __COMPARERS__ = NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using Microsoft.Extensions.DependencyInjection;
 using NexusMods.MnemonicDB.Abstractions;
+using System.Diagnostics.CodeAnalysis;
 
 {{= model.Comments}}
 public partial class {{= model.Name}} {
@@ -42,7 +45,7 @@ public partial class {{= model.Name}} {
     /// Assumes that the id given points to a {{= model.Name}} entity, and
     /// returns a ReadOnly model of the entity.
     /// </summary>
-    public static {{= model.Name}}.ReadOnly Get(__ABSTRACTIONS__.IDb db, __ABSTRACTIONS__.EntityId id) {
+    public static {{= model.Name}}.ReadOnly As(__ABSTRACTIONS__.IDb db, __ABSTRACTIONS__.EntityId id) {
         return new {{= model.Name}}.ReadOnly(db, id);
     }
 
@@ -61,7 +64,24 @@ public partial class {{= model.Name}} {
     /// Returns all {{= model.Name}} entities in the database.
     /// </summary>
     public static IEnumerable<{{= model.Name}}.ReadOnly> All(__ABSTRACTIONS__.IDb db) {
-        return Get(db, __ABSTRACTIONS__.IDbExtensions.Intersection(db, RequiredAttributes));
+        foreach (var id in __ABSTRACTIONS__.IDbExtensions.Intersection(db, RequiredAttributes))
+        {
+            yield return new {{= model.Name}}.ReadOnly(db, id);
+        }
+    }
+
+    /// <summary>
+    /// Tries to get the entity with the given id from the database, if the model is invalid, it returns false.
+    /// </summary>
+    public static bool TryGet(__ABSTRACTIONS__.IDb db, __ABSTRACTIONS__.EntityId id, [NotNullWhen(true)] out {{= model.Name}}.ReadOnly? result)
+    {
+        var model = As(db, id);
+        if (model.IsValid()) {
+            result = model;
+            return true;
+        }
+        result = default;
+        return false;
     }
 
     /// <summary>
@@ -158,7 +178,7 @@ public partial class {{= model.Name}} {
            protected override string ModelName => "{{= model.Name}}";
 
 
-           internal bool IsValid()
+           public virtual bool IsValid()
            {
                {{each attr in model.Attributes}}
                {{if !attr.IsMarker}}
@@ -222,7 +242,7 @@ public static class {{= model.Name}}Extensions {
     }
 
     {{each include in model.Includes}}
-    public static bool TryGetAs{{= model.Name}}(this {{= include.ToDisplayString()}}.ReadOnly model, out {{= include.ToDisplayString()}}.ReadOnly result) {
+    public static bool TryGetAs{{= model.Name}}(this {{= include.ToDisplayString()}}.ReadOnly model, out {{= include.ToDisplayString()}}.ReadOnly? result) {
         var casted = new {{= include.ToDisplayString()}}.ReadOnly(model.Db, model.Id);
         if (casted.IsValid()) {
             result = casted;

tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs

@@ -108,7 +108,7 @@ protected SettingsTask VerifyTable(IEnumerable<IReadDatom> datoms)
         }
         await tx2.Commit();
 
-        return Loadout.Get(Connection.Db, loadoutWritten.Id);
+        return Loadout.As(Connection.Db, loadoutWritten.Id);
     }
 
     public void Dispose()