Merge pull request #54 from Nexus-Mods/transaction-fixes

Transactor failsafes
Nexus-Mods · May 6, 2024 · 943d5e1 · 943d5e1
2 parents 6c80a23 + 4d02eaa
commit 943d5e1
Show file tree

Hide file tree

Showing 11 changed files with 40 additions and 140 deletions.
diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/NexusMods.MnemonicDB.Benchmarks.csproj b/benchmarks/NexusMods.MnemonicDB.Benchmarks/NexusMods.MnemonicDB.Benchmarks.csproj
@@ -9,7 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="BenchmarkDotNet" Version="0.13.12"/>
-        <PackageReference Include="JetBrains.Profiler.Api" Version="1.4.2" />
+        <PackageReference Include="JetBrains.Profiler.Api" Version="1.4.3" />
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0"/>
         <PackageReference Update="JetBrains.Annotations" Version="2023.3.0"/>
     </ItemGroup>

diff --git a/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj b/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj
@@ -6,9 +6,8 @@
 
     <ItemGroup>
         <PackageReference Include="DynamicData" Version="8.4.1"/>
-        <PackageReference Include="MemoryPack.Core" Version="1.20.5"/>
         <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1"/>
-        <PackageReference Include="NexusMods.Paths" Version="0.9.0"/>
+        <PackageReference Include="NexusMods.Paths" Version="0.9.4" />
         <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
         <PackageReference Include="TransparentValueObjects" Version="1.0.1" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false"/>
         <PackageReference Update="JetBrains.Annotations" Version="2023.3.0"/>

diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs
@@ -41,6 +41,8 @@ public class DatomStore : IDatomStore
     private readonly Task _startupTask;
     private TxId _asOfTx = TxId.MinValue;
 
+    private static readonly TimeSpan TransactionTimeout = TimeSpan.FromMinutes(120);
+
     /// <summary>
     /// Cached version of the registry ID to avoid the overhead of looking it up every time
     /// </summary>
@@ -111,6 +113,7 @@ public DatomStore(ILogger<DatomStore> logger, AttributeRegistry registry, DatomS
     public async Task<StoreResult> Transact(IndexSegment datoms, HashSet<ITxFunction>? txFunctions = null,
         Func<ISnapshot, IDb>? factoryFn = null)
     {
+
         var pending = new PendingTransaction
         {
             Data = datoms,
@@ -120,22 +123,20 @@ public async Task<StoreResult> Transact(IndexSegment datoms, HashSet<ITxFunction
         if (!_txChannel.Writer.TryWrite(pending))
             throw new InvalidOperationException("Failed to write to the transaction channel");
 
-        return await pending.CompletionSource.Task;
+        var task = pending.CompletionSource.Task;
+        if (await Task.WhenAny(task, Task.Delay(TransactionTimeout)) == task)
+        {
+            return await task;
+        }
+        _logger.LogError("Transaction didn't complete after {Timeout}", TransactionTimeout);
+        throw new TimeoutException($"Transaction didn't complete after {TransactionTimeout}");
+
     }
 
     /// <inheritdoc />
     public async Task<StoreResult> Sync()
     {
-        var pending = new PendingTransaction
-        {
-            Data = new IndexSegment(),
-            TxFunctions = null,
-            DatabaseFactory = null
-        };
-        if (!_txChannel.Writer.TryWrite(pending))
-            throw new InvalidOperationException("Failed to write to the transaction channel");
-
-        return await pending.CompletionSource.Task;
+        return await Transact(new IndexSegment());
     }
 
     public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog => _updatesSubject;
@@ -202,6 +203,7 @@ private async Task ConsumeTransactions()
                 }
                 catch (Exception ex)
                 {
+                    _logger.LogError(ex, "While commiting transaction");
                     pendingTransaction.CompletionSource.TrySetException(ex);
                 }
             }

diff --git a/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj b/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj
@@ -8,8 +8,6 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="LinqGen" Version="0.3.1"/>
-        <PackageReference Include="LinqGen.Generator" Version="0.3.1"/>
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1"/>
         <PackageReference Include="Reloaded.Memory" Version="9.4.1"/>
         <PackageReference Include="RocksDB" Version="8.11.3.46984"/>

diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs b/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs
@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Generic;
 using System.Runtime.InteropServices;
+using DynamicData;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Abstractions.DatomComparators;
 using NexusMods.MnemonicDB.Abstractions.DatomIterators;
@@ -21,6 +23,15 @@ public class IndexStore<TComparator> : IRocksDBIndexStore
     private IntPtr _namePtr;
     private ColumnFamilyOptions _options = null!;
 
+    /// <summary>
+    /// This is a bit of a hack, but we throw all our interop delegates in here, and then they
+    /// live for the entire life of the application. It seems that RocksDB will occasionally call
+    /// delegates after we think we've disposed of the handles. It really doesn't matter as these
+    /// things will never amount to more than a few dozen objects.
+    /// </summary>
+    /// <returns></returns>
+    private static List<object> _roots = new ();
+
     public IndexStore(string handleName, IndexType type, AttributeRegistry registry)
     {
         Type = type;
@@ -47,6 +58,9 @@ public void SetupColumnFamily(IIndex index, ColumnFamilies columnFamilies)
             }
         };
 
+        // Save these as roots so they never get GC'd
+        _roots.Add((_nameDelegate, _destructorDelegate, _comparatorDelegate));
+
         _comparator =
             Native.Instance.rocksdb_comparator_create(IntPtr.Zero, _destructorDelegate, _comparatorDelegate,
                 _nameDelegate);

diff --git a/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj b/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj
@@ -5,9 +5,8 @@
     </PropertyGroup>
     <ItemGroup>
         <PackageReference Include="DynamicData" Version="8.4.1"/>
-        <PackageReference Include="MemoryPack" Version="1.20.5"/>
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1"/>
-        <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="8.0.3"/>
+        <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="8.0.4" />
         <PackageReference Include="Reloaded.Memory" Version="9.4.1"/>
         <PackageReference Include="System.Reactive" Version="6.0.0"/>
         <PackageReference Include="TransparentValueObjects" Version="1.0.1" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false"/>

diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props
@@ -6,16 +6,6 @@
         <IsTestProject>true</IsTestProject>
     </PropertyGroup>
 
-    <ItemGroup>
-        <PackageReference Include="Moq" Version="4.*"/>
-    </ItemGroup>
-
-    <ItemGroup>
-        <PackageReference Include="AutoFixture" Version="4.*"/>
-        <PackageReference Include="AutoFixture.AutoMoq" Version="4.*"/>
-        <PackageReference Include="AutoFixture.Xunit2" Version="4.*"/>
-    </ItemGroup>
-
     <ItemGroup>
         <PackageReference Include="FluentAssertions" Version="6.*"/>
         <PackageReference Include="FluentAssertions.OneOf" Version="0.0.5"/>
@@ -26,15 +16,19 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="xunit" Version="2.4.*"/>
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.*">
+        <PackageReference Include="xunit" Version="2.8.0"/>
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.0">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
         <PackageReference Include="Xunit.SkippableFact" Version="1.4.*"/>
         <PackageReference Include="Xunit.DependencyInjection" Version="8.*"/>
         <PackageReference Include="Xunit.DependencyInjection.Logging" Version="8.*"/>
         <PackageReference Include="Xunit.DependencyInjection.SkippableFact" Version="8.*"/>
+        <PackageReference Include="NexusMods.Hashing.xxHash64"/>
+        <PackageReference Include="Argon" Version="0.17.0"/>
+        <PackageReference Include="Verify" Version="24.1.0"/>
+        <PackageReference Include="Verify.Xunit" Version="24.1.0"/>
     </ItemGroup>
 
     <ItemGroup>

diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj b/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj
@@ -17,47 +17,6 @@
         <IsTestProject>true</IsTestProject>
     </PropertyGroup>
 
-    <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/>
-        <PackageReference Include="xunit" Version="2.4.2"/>
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="6.0.0">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-        <PackageReference Update="JetBrains.Annotations" Version="2023.3.0"/>
-        <PackageReference Update="Moq" Version="4.20.70"/>
-        <PackageReference Update="AutoFixture" Version="4.18.1"/>
-        <PackageReference Update="AutoFixture.AutoMoq" Version="4.18.1"/>
-        <PackageReference Update="AutoFixture.Xunit2" Version="4.18.1"/>
-        <PackageReference Update="FluentAssertions" Version="6.12.0"/>
-        <PackageReference Update="FluentAssertions.Analyzers" Version="0.31.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="xunit" Version="2.7.0"/>
-        <PackageReference Update="xunit.runner.visualstudio" Version="2.5.7">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="Xunit.SkippableFact" Version="1.4.13"/>
-        <PackageReference Update="Xunit.DependencyInjection" Version="9.0.1"/>
-        <PackageReference Update="Xunit.DependencyInjection.Logging" Version="9.0.0"/>
-        <PackageReference Update="Xunit.DependencyInjection.SkippableFact" Version="9.0.0"/>
-        <PackageReference Update="Microsoft.NET.Test.Sdk" Version="17.9.0"/>
-        <PackageReference Update="coverlet.collector" Version="6.0.2">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="GitHubActionsTestLogger" Version="2.3.3">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-    </ItemGroup>
-
     <ItemGroup>
         <ProjectReference Include="..\..\src\NexusMods.MnemonicDB.Storage\NexusMods.MnemonicDB.Storage.csproj"/>
         <ProjectReference Include="..\NexusMods.MnemonicDB.TestModel\NexusMods.MnemonicDB.TestModel.csproj"/>
@@ -67,4 +26,8 @@
       <None Remove="InMemoryTests.CanStoreDataInBlobs.verified.txt" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Update="Verify" Version="24.1.0" />
+    </ItemGroup>
+
 </Project>
diff --git a/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj b/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj
@@ -10,38 +10,4 @@
         <ProjectReference Include="..\..\src\NexusMods.MnemonicDB.Abstractions\NexusMods.MnemonicDB.Abstractions.csproj"/>
     </ItemGroup>
 
-    <ItemGroup>
-        <PackageReference Include="NexusMods.Hashing.xxHash64" Version="2.0.0"/>
-        <PackageReference Include="NexusMods.Paths" Version="0.9.0"/>
-        <PackageReference Update="JetBrains.Annotations" Version="2023.3.0"/>
-        <PackageReference Update="Moq" Version="4.20.70"/>
-        <PackageReference Update="AutoFixture" Version="4.18.1"/>
-        <PackageReference Update="AutoFixture.AutoMoq" Version="4.18.1"/>
-        <PackageReference Update="AutoFixture.Xunit2" Version="4.18.1"/>
-        <PackageReference Update="FluentAssertions" Version="6.12.0"/>
-        <PackageReference Update="FluentAssertions.Analyzers" Version="0.31.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="xunit" Version="2.7.0"/>
-        <PackageReference Update="xunit.runner.visualstudio" Version="2.5.7">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="Xunit.SkippableFact" Version="1.4.13"/>
-        <PackageReference Update="Xunit.DependencyInjection" Version="9.0.1"/>
-        <PackageReference Update="Xunit.DependencyInjection.Logging" Version="9.0.0"/>
-        <PackageReference Update="Xunit.DependencyInjection.SkippableFact" Version="9.0.0"/>
-        <PackageReference Update="Microsoft.NET.Test.Sdk" Version="17.9.0"/>
-        <PackageReference Update="coverlet.collector" Version="6.0.2">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="GitHubActionsTestLogger" Version="2.3.3">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Include="Verify.Xunit" Version="23.5.2"/>
-    </ItemGroup>
-
 </Project>
diff --git a/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj b/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj
@@ -5,39 +5,6 @@
         <RootNamespace>NexusMods.MnemonicDB.Tests</RootNamespace>
     </PropertyGroup>
 
-
-    <ItemGroup>
-        <PackageReference Update="FluentAssertions" Version="6.12.0"/>
-        <PackageReference Update="FluentAssertions.Analyzers" Version="0.31.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="xunit" Version="2.7.0"/>
-        <PackageReference Update="xunit.runner.visualstudio" Version="2.5.7">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="JetBrains.Annotations" Version="2023.3.0"/>
-        <PackageReference Update="Moq" Version="4.20.70"/>
-        <PackageReference Update="AutoFixture" Version="4.18.1"/>
-        <PackageReference Update="AutoFixture.AutoMoq" Version="4.18.1"/>
-        <PackageReference Update="AutoFixture.Xunit2" Version="4.18.1"/>
-        <PackageReference Update="Xunit.SkippableFact" Version="1.4.13"/>
-        <PackageReference Update="Xunit.DependencyInjection" Version="9.0.1"/>
-        <PackageReference Update="Xunit.DependencyInjection.Logging" Version="9.0.0"/>
-        <PackageReference Update="Xunit.DependencyInjection.SkippableFact" Version="9.0.0"/>
-        <PackageReference Update="Microsoft.NET.Test.Sdk" Version="17.9.0"/>
-        <PackageReference Update="coverlet.collector" Version="6.0.2">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Update="GitHubActionsTestLogger" Version="2.3.3">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Include="Verify.Xunit" Version="23.5.2"/>
-    </ItemGroup>
-
     <ItemGroup>
         <ProjectReference Include="..\..\src\NexusMods.MnemonicDB\NexusMods.MnemonicDB.csproj"/>
         <ProjectReference Include="..\NexusMods.MnemonicDB.TestModel\NexusMods.MnemonicDB.TestModel.csproj"/>

diff --git a/tests/SharedUsings.cs b/tests/SharedUsings.cs
@@ -2,8 +2,6 @@
 
 global using Xunit;
 global using FluentAssertions;
-global using AutoFixture;
-global using AutoFixture.Xunit2;
 using System.Runtime.CompilerServices;
 using NexusMods.MnemonicDB.TestModel.Helpers;