Merge pull request #22 from Nexus-Mods/rename-project

Rename project to MneumonicDB
Nexus-Mods · Mar 28, 2024 · 0bef8b1 · 0bef8b1
2 parents 66639ac + effa6a4
commit 0bef8b1
Show file tree

Hide file tree

Showing 193 changed files with 1,647 additions and 1,115 deletions.
diff --git a/NexusMods.EventSourcing.sln → NexusMods.MneumonicDB.sln b/NexusMods.EventSourcing.sln → NexusMods.MneumonicDB.sln
@@ -23,21 +23,21 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{6ED01F9D
 		tests\SharedUsings.cs = tests\SharedUsings.cs
 	EndProjectSection
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing", "src\NexusMods.EventSourcing\NexusMods.EventSourcing.csproj", "{A92DED3D-BC67-4E04-9A06-9A1B302B3070}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MneumonicDB", "src\NexusMods.MneumonicDB\NexusMods.MneumonicDB.csproj", "{A92DED3D-BC67-4E04-9A06-9A1B302B3070}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Abstractions", "src\NexusMods.EventSourcing.Abstractions\NexusMods.EventSourcing.Abstractions.csproj", "{6737673E-5898-42EC-B0B2-60DE2CFFF0AF}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MneumonicDB.Abstractions", "src\NexusMods.MneumonicDB.Abstractions\NexusMods.MneumonicDB.Abstractions.csproj", "{6737673E-5898-42EC-B0B2-60DE2CFFF0AF}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmarks", "benchmarks", "{72AFE85F-8C12-436A-894E-638ED2C92A76}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.TestModel", "tests\NexusMods.EventSourcing.TestModel\NexusMods.EventSourcing.TestModel.csproj", "{EC1570A4-18B9-4A76-84FF-275BAA76A357}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MneumonicDB.TestModel", "tests\NexusMods.MneumonicDB.TestModel\NexusMods.MneumonicDB.TestModel.csproj", "{EC1570A4-18B9-4A76-84FF-275BAA76A357}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Tests", "tests\NexusMods.EventSourcing.Tests\NexusMods.EventSourcing.Tests.csproj", "{07E2C578-8644-474D-8F07-B25CFEB28408}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MneumonicDB.Tests", "tests\NexusMods.MneumonicDB.Tests\NexusMods.MneumonicDB.Tests.csproj", "{07E2C578-8644-474D-8F07-B25CFEB28408}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Benchmarks", "benchmarks\NexusMods.EventSourcing.Benchmarks\NexusMods.EventSourcing.Benchmarks.csproj", "{930B3AB7-56EA-48D6-B603-24D79C7DD00A}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MneumonicDB.Benchmarks", "benchmarks\NexusMods.MneumonicDB.Benchmarks\NexusMods.MneumonicDB.Benchmarks.csproj", "{930B3AB7-56EA-48D6-B603-24D79C7DD00A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Storage", "src\NexusMods.EventSourcing.Storage\NexusMods.EventSourcing.Storage.csproj", "{73E074F9-250F-4D8A-8038-5B12DB761E98}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MneumonicDB.Storage", "src\NexusMods.MneumonicDB.Storage\NexusMods.MneumonicDB.Storage.csproj", "{73E074F9-250F-4D8A-8038-5B12DB761E98}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Storage.Tests", "tests\NexusMods.EventSourcing.Storage.Tests\NexusMods.EventSourcing.Storage.Tests.csproj", "{33A3DA79-D3FD-46DC-8D14-82E23D5B608D}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MneumonicDB.Storage.Tests", "tests\NexusMods.MneumonicDB.Storage.Tests\NexusMods.MneumonicDB.Storage.Tests.csproj", "{33A3DA79-D3FD-46DC-8D14-82E23D5B608D}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneBillionDatomsTest", "benchmarks\OneBillionDatomsTest\OneBillionDatomsTest.csproj", "{EA397BAE-9726-486F-BC9B-87BD86DF157F}"
 EndProject

diff --git a/NexusMods.EventSourcing.sln.DotSettings → NexusMods.MneumonicDB.sln.DotSettings b/NexusMods.EventSourcing.sln.DotSettings → NexusMods.MneumonicDB.sln.DotSettings
diff --git a/README.md b/README.md
@@ -1,14 +1,14 @@
-# NexusMods.EventSourcing
+# NexusMods.MnemonicDB
 
-EventSourcing and CQRS library for .NET. This library is designed for single process multi threaded applications, and
-provides
-extremely high performance event sourcing and CQRS capabilities. The current datastore is based on RocksDB, but the
-interface is
-designed to be easily swapped out for other storage systems.
+This is a simple, fast, and type-safe in-process temporal database for .NET applications. It borrows heavily from
+[Datomic](https://www.datomic.com/), [Datahike](https://github.com/replikativ/datahike) and heavily leverages
+[RocksDB](https://rocksdb.org/) for storage. Performance wise, on modern desktop hardware it can write roughly
+400K tuples per second sustained, and read roughly 1M tuples per second sustained, this includes the overhead
+of maintaining the indexes.
 
 ## Documentation
 
-The full docs can be found [here](https://nexus-mods.github.io/NexusMods.EventSourcing/)
+The full docs can be found [here](https://nexus-mods.github.io/NexusMods.MnemonicDB/)
 
 ## License
 

diff --git a/...rcing.Benchmarks/Benchmarks/ABenchmark.cs → ...nicDB.Benchmarks/Benchmarks/ABenchmark.cs b/...rcing.Benchmarks/Benchmarks/ABenchmark.cs → ...nicDB.Benchmarks/Benchmarks/ABenchmark.cs
@@ -2,13 +2,13 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
-using NexusMods.EventSourcing.Abstractions;
-using NexusMods.EventSourcing.Storage;
-using NexusMods.EventSourcing.TestModel;
+using NexusMods.MneumonicDB.Abstractions;
+using NexusMods.MneumonicDB.Storage;
+using NexusMods.MneumonicDB.TestModel;
 using NexusMods.Paths;
 using Xunit;
 
-namespace NexusMods.EventSourcing.Benchmarks.Benchmarks;
+namespace NexusMods.MneumonicDB.Benchmarks.Benchmarks;
 
 public class ABenchmark : IAsyncLifetime
 {
@@ -22,9 +22,9 @@ public async Task InitializeAsync()
         var builder = Host.CreateDefaultBuilder()
             .ConfigureServices(services =>
             {
-                services.AddEventSourcingStorage()
+                services.AddMneumonicDBStorage()
                     .AddRocksDbBackend()
-                    .AddEventSourcing()
+                    .AddMneumonicDB()
                     .AddTestModel()
                     .AddDatomStoreSettings(new DatomStoreSettings
                     {
@@ -33,7 +33,7 @@ public async Task InitializeAsync()
             });
 
         _host = builder.Build();
-        Connection = await EventSourcing.Connection.Start(Services);
+        Connection = await MneumonicDB.Connection.Start(Services);
     }
 
     public async Task DisposeAsync()

diff --git a/...urcing.Benchmarks/Benchmarks/ReadTests.cs → ...onicDB.Benchmarks/Benchmarks/ReadTests.cs b/...urcing.Benchmarks/Benchmarks/ReadTests.cs → ...onicDB.Benchmarks/Benchmarks/ReadTests.cs
@@ -1,13 +1,16 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
-using NexusMods.EventSourcing.Abstractions;
-using NexusMods.EventSourcing.TestModel.Model;
+using NexusMods.MneumonicDB.Abstractions;
+using NexusMods.MneumonicDB.TestModel.ComplexModel.ReadModels;
+using NexusMods.Hashing.xxHash64;
+using NexusMods.Paths;
 
 // ReSharper disable MemberCanBePrivate.Global
 
-namespace NexusMods.EventSourcing.Benchmarks.Benchmarks;
+namespace NexusMods.MneumonicDB.Benchmarks.Benchmarks;
 
 [MemoryDiagnoser]
 public class ReadTests : ABenchmark
@@ -25,13 +28,22 @@ public async Task Setup()
         await InitializeAsync();
         var tx = Connection.BeginTransaction();
         var entityIds = new List<EntityId>();
+
+        var modId = new Mod(tx)
+        {
+            Name = "TestMod",
+            Source = new Uri("https://www.nexusmods.com"),
+            LoadoutId = EntityId.From(1)
+        };
+
         for (var i = 0; i < Count; i++)
         {
             var file = new File(tx)
             {
-                Hash = (ulong)i,
+                Hash = Hash.From((ulong)i),
                 Path = $"C:\\test_{i}.txt",
-                Index = (ulong)i
+                Size = Size.From((ulong)i),
+                ModId = modId.Id,
             };
             entityIds.Add(file.Id);
         }
@@ -49,14 +61,14 @@ public async Task Setup()
     public ulong ReadFiles()
     {
         ulong sum = 0;
-        sum += _db.Get<File>(_readId).Index;
+        sum += _db.Get<File>(_readId).Size.Value;
         return sum;
     }
 
     [Benchmark]
     public long ReadAll()
     {
         return _db.Get<File>(_entityIds)
-            .Sum(e => (long)e.Index);
+            .Sum(e => (long)e.Size.Value);
     }
 }
diff --git a/...rcing.Benchmarks/Benchmarks/WriteTests.cs → ...nicDB.Benchmarks/Benchmarks/WriteTests.cs b/...rcing.Benchmarks/Benchmarks/WriteTests.cs → ...nicDB.Benchmarks/Benchmarks/WriteTests.cs
@@ -1,4 +1,4 @@
-namespace NexusMods.EventSourcing.Benchmarks.Benchmarks;
+namespace NexusMods.MneumonicDB.Benchmarks.Benchmarks;
 
 public class WriteTests
 {

diff --git a/...NexusMods.EventSourcing.Benchmarks.csproj → ...s/NexusMods.MneumonicDB.Benchmarks.csproj b/...NexusMods.EventSourcing.Benchmarks.csproj → ...s/NexusMods.MneumonicDB.Benchmarks.csproj
@@ -14,9 +14,9 @@
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\..\src\NexusMods.EventSourcing.Abstractions\NexusMods.EventSourcing.Abstractions.csproj"/>
-        <ProjectReference Include="..\..\src\NexusMods.EventSourcing\NexusMods.EventSourcing.csproj"/>
-        <ProjectReference Include="..\..\tests\NexusMods.EventSourcing.TestModel\NexusMods.EventSourcing.TestModel.csproj"/>
+        <ProjectReference Include="..\..\src\NexusMods.MneumonicDB.Abstractions\NexusMods.MneumonicDB.Abstractions.csproj"/>
+        <ProjectReference Include="..\..\src\NexusMods.MneumonicDB\NexusMods.MneumonicDB.csproj"/>
+        <ProjectReference Include="..\..\tests\NexusMods.MneumonicDB.TestModel\NexusMods.MneumonicDB.TestModel.csproj"/>
     </ItemGroup>
 
 </Project>
diff --git a/...sMods.EventSourcing.Benchmarks/Program.cs → ...xusMods.MneumonicDB.Benchmarks/Program.cs b/...sMods.EventSourcing.Benchmarks/Program.cs → ...xusMods.MneumonicDB.Benchmarks/Program.cs
@@ -2,7 +2,7 @@
 
 using System;
 using System.Diagnostics;
-using NexusMods.EventSourcing.Benchmarks.Benchmarks;
+using NexusMods.MneumonicDB.Benchmarks.Benchmarks;
 
 
 //#if DEBUG

diff --git a/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj b/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj
@@ -13,10 +13,10 @@
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\..\src\NexusMods.EventSourcing.Storage\NexusMods.EventSourcing.Storage.csproj"/>
-        <ProjectReference Include="..\..\src\NexusMods.EventSourcing\NexusMods.EventSourcing.csproj"/>
-        <ProjectReference Include="..\..\tests\NexusMods.EventSourcing.Storage.Tests\NexusMods.EventSourcing.Storage.Tests.csproj"/>
-        <ProjectReference Include="..\..\tests\NexusMods.EventSourcing.TestModel\NexusMods.EventSourcing.TestModel.csproj"/>
+        <ProjectReference Include="..\..\src\NexusMods.MneumonicDB.Storage\NexusMods.MneumonicDB.Storage.csproj"/>
+        <ProjectReference Include="..\..\src\NexusMods.MneumonicDB\NexusMods.MneumonicDB.csproj"/>
+        <ProjectReference Include="..\..\tests\NexusMods.MneumonicDB.Storage.Tests\NexusMods.MneumonicDB.Storage.Tests.csproj"/>
+        <ProjectReference Include="..\..\tests\NexusMods.MneumonicDB.TestModel\NexusMods.MneumonicDB.TestModel.csproj"/>
     </ItemGroup>
 
 </Project>
diff --git a/benchmarks/OneBillionDatomsTest/Program.cs b/benchmarks/OneBillionDatomsTest/Program.cs
@@ -1,20 +1,26 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
-using NexusMods.EventSourcing;
-using NexusMods.EventSourcing.Abstractions;
-using NexusMods.EventSourcing.Storage;
-using NexusMods.EventSourcing.TestModel;
-using NexusMods.EventSourcing.TestModel.Model;
+using NexusMods.MneumonicDB;
+using NexusMods.MneumonicDB.Abstractions;
+using NexusMods.MneumonicDB.Abstractions.DatomIterators;
+using NexusMods.MneumonicDB.Storage;
+using NexusMods.MneumonicDB.TestModel;
+using NexusMods.MneumonicDB.TestModel.ComplexModel.Attributes;
+using NexusMods.MneumonicDB.TestModel.ComplexModel.ReadModels;
+using NexusMods.Hashing.xxHash64;
 using NexusMods.Paths;
 
 var host = Host.CreateDefaultBuilder()
     .ConfigureServices(s =>
     {
-        s.AddEventSourcingStorage()
-            .AddEventSourcing()
+        s.AddMneumonicDBStorage()
+            .AddMneumonicDB()
             .AddRocksDbBackend()
             .AddTestModel()
             .AddSingleton<DatomStoreSettings>(_ => new DatomStoreSettings
@@ -48,18 +54,58 @@
 var globalSw = Stopwatch.StartNew();
 ulong fileNumber = 0;
 var lastPrint = DateTime.UtcNow;
+
+async Task<(TimeSpan Time, int loaded)> ReadCheckpoint(IDb db)
+{
+    var tasks = new List<Task<(TimeSpan Time, int loaded)>>();
+
+
+    for (int i = 0; i < 4; i++)
+    {
+        var task = Task.Run(() =>
+        {
+            var sw = Stopwatch.StartNew();
+            using var iterator = db.Iterate(IndexType.TxLog);
+
+            var count = 0;
+            var scanned = 0;
+            foreach (var itm in iterator.SeekLast().Reverse().Resolve())
+            {
+                if (count == 1024 * 128) break;
+                if (itm is FileAttributes.Hash.ReadDatom hashDatom)
+                {
+                    var file = db.Get<File>(hashDatom.E);
+                    count++;
+                }
+
+                scanned++;
+            }
+
+            return (sw.Elapsed, (count * 4) + scanned);
+        });
+        tasks.Add(task);
+    }
+
+    var results = await Task.WhenAll(tasks);
+
+    return (results.Max(r => r.Time), results.Sum(r => r.loaded));
+
+}
+
+var checkPointTask = ReadCheckpoint(connection.Db);
 for (ulong i = 0; i < batches; i++)
 {
     var tx = connection.BeginTransaction();
 
     for (var j = 0; j < (int)batchSize; j++)
     {
         fileNumber += 1;
-        var file = new File(tx)
+        var _ = new File(tx)
         {
             Path = $"c:\\test_{i}_{j}.txt",
-            Hash = fileNumber,
-            Index = entityCount - fileNumber
+            Hash = Hash.From(fileNumber % 0xFFFF),
+            Size = Size.From(fileNumber),
+            ModId = EntityId.From(1)
         };
     }
 
@@ -69,10 +115,14 @@
 
     if (DateTime.UtcNow - lastPrint > TimeSpan.FromSeconds(1))
     {
+        var checkpointStatus = await checkPointTask;
         var estimatedRemaining = (batches - i) * (globalSw.Elapsed.TotalSeconds / i);
         Console.WriteLine(
             $"({i}/{batches}) Elapsed: {globalSw.Elapsed} - Datoms per second: {perSecond} - ETA: {TimeSpan.FromSeconds(estimatedRemaining)}");
+        Console.WriteLine(" - Read Checkpoint: " + checkpointStatus.loaded + " datoms in " + checkpointStatus.Time.Milliseconds + "ms");
         lastPrint = DateTime.UtcNow;
+
+        checkPointTask = ReadCheckpoint(connection.Db);
     }
 }