From b8e84ce16a76d32fd791552a6c9b57f208f9c7d6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 6 May 2024 15:24:00 -0600 Subject: [PATCH 1/5] bit of cleanup --- src/NexusMods.MnemonicDB.Storage/DatomStore.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs index f3601746..cbc9ef82 100644 --- a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs @@ -41,6 +41,8 @@ public class DatomStore : IDatomStore private readonly Task _startupTask; private TxId _asOfTx = TxId.MinValue; + private static readonly TimeSpan TransactionTimeout = TimeSpan.FromSeconds(10); + /// /// Cached version of the registry ID to avoid the overhead of looking it up every time /// @@ -111,6 +113,7 @@ public DatomStore(ILogger logger, AttributeRegistry registry, DatomS public async Task Transact(IndexSegment datoms, HashSet? txFunctions = null, Func? factoryFn = null) { + var pending = new PendingTransaction { Data = datoms, @@ -120,7 +123,15 @@ public async Task Transact(IndexSegment datoms, HashSet @@ -202,6 +213,7 @@ private async Task ConsumeTransactions() } catch (Exception ex) { + _logger.LogError(ex, "While commiting transaction"); pendingTransaction.CompletionSource.TrySetException(ex); } } From ae199822b5a6579fa55969b334273548d66bed1d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 6 May 2024 15:32:53 -0600 Subject: [PATCH 2/5] Update deps --- .../NexusMods.MnemonicDB.Benchmarks.csproj | 2 +- .../NexusMods.MnemonicDB.Abstractions.csproj | 3 +-- src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj | 3 +-- .../NexusMods.MnemonicDB.Storage.Tests.csproj | 6 +++--- .../NexusMods.MnemonicDB.TestModel.csproj | 12 ++++++------ .../NexusMods.MnemonicDB.Tests.csproj | 10 +++++----- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/NexusMods.MnemonicDB.Benchmarks.csproj b/benchmarks/NexusMods.MnemonicDB.Benchmarks/NexusMods.MnemonicDB.Benchmarks.csproj index e4fdabcf..21988099 100644 --- a/benchmarks/NexusMods.MnemonicDB.Benchmarks/NexusMods.MnemonicDB.Benchmarks.csproj +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/NexusMods.MnemonicDB.Benchmarks.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj b/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj index 61aece7c..981ca21f 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj +++ b/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj @@ -6,9 +6,8 @@ - - + diff --git a/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj b/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj index 097d79d0..f97e6aa9 100644 --- a/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj +++ b/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj @@ -5,9 +5,8 @@ - - + diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj b/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj index ab63f92f..a73bb6e1 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj @@ -38,13 +38,13 @@ all runtime; build; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj b/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj index 0b868bd3..020d8cf4 100644 --- a/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj +++ b/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj @@ -11,8 +11,8 @@ - - + + @@ -23,13 +23,13 @@ all runtime; build; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + @@ -41,7 +41,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj b/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj index 67dd722d..a745a16c 100644 --- a/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj +++ b/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj @@ -12,8 +12,8 @@ all runtime; build; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -23,10 +23,9 @@ - + - all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -35,7 +34,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + + From d48ae09056c619346a2db94a831d24114cad2361 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 6 May 2024 15:57:50 -0600 Subject: [PATCH 3/5] Fixup the broken tests and the deps mess --- .../DatomStore.cs | 13 +++--- .../NexusMods.MnemonicDB.Storage.csproj | 2 - tests/Directory.Build.props | 18 +++----- .../NexusMods.MnemonicDB.Storage.Tests.csproj | 45 ++----------------- .../NexusMods.MnemonicDB.TestModel.csproj | 34 -------------- .../NexusMods.MnemonicDB.Tests.csproj | 33 -------------- tests/SharedUsings.cs | 2 - 7 files changed, 16 insertions(+), 131 deletions(-) diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs index cbc9ef82..af4a424c 100644 --- a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs @@ -123,15 +123,14 @@ public async Task Transact(IndexSegment datoms, HashSet diff --git a/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj b/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj index 8baa98b2..c03952a1 100644 --- a/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj +++ b/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj @@ -8,8 +8,6 @@ - - diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 136c3ab4..38a84242 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -6,16 +6,6 @@ true - - - - - - - - - - @@ -26,8 +16,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -35,6 +25,10 @@ + + + + diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj b/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj index a73bb6e1..38e3d886 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj @@ -17,47 +17,6 @@ true - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - - all - runtime; build; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - @@ -67,4 +26,8 @@ + + + + diff --git a/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj b/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj index 020d8cf4..c84e3f6b 100644 --- a/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj +++ b/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj @@ -10,38 +10,4 @@ - - - - - - - - - - - all - runtime; build; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - diff --git a/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj b/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj index a745a16c..40a59bc7 100644 --- a/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj +++ b/tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj @@ -5,39 +5,6 @@ NexusMods.MnemonicDB.Tests - - - - - all - runtime; build; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - diff --git a/tests/SharedUsings.cs b/tests/SharedUsings.cs index 67636b23..d99c0f88 100644 --- a/tests/SharedUsings.cs +++ b/tests/SharedUsings.cs @@ -2,8 +2,6 @@ global using Xunit; global using FluentAssertions; -global using AutoFixture; -global using AutoFixture.Xunit2; using System.Runtime.CompilerServices; using NexusMods.MnemonicDB.TestModel.Helpers; From a125cded727f2cf253fc3b366b5df457614ed648 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 6 May 2024 15:59:49 -0600 Subject: [PATCH 4/5] Set the Tx timeout to 2 minutes, because GH actions are so stupidly slow --- src/NexusMods.MnemonicDB.Storage/DatomStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs index af4a424c..fb8cdfd5 100644 --- a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs @@ -41,7 +41,7 @@ public class DatomStore : IDatomStore private readonly Task _startupTask; private TxId _asOfTx = TxId.MinValue; - private static readonly TimeSpan TransactionTimeout = TimeSpan.FromSeconds(10); + private static readonly TimeSpan TransactionTimeout = TimeSpan.FromMinutes(120); /// /// Cached version of the registry ID to avoid the overhead of looking it up every time From 4d02eaabdca94835153736a69d67864559adf3ba Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 6 May 2024 16:14:25 -0600 Subject: [PATCH 5/5] Fix a strange race condition with interop delegates --- src/NexusMods.MnemonicDB.Storage/DatomStore.cs | 11 +---------- .../RocksDbBackend/IndexStore.cs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs index fb8cdfd5..1b8ae4ce 100644 --- a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs @@ -136,16 +136,7 @@ public async Task Transact(IndexSegment datoms, HashSet public async Task Sync() { - var pending = new PendingTransaction - { - Data = new IndexSegment(), - TxFunctions = null, - DatabaseFactory = null - }; - if (!_txChannel.Writer.TryWrite(pending)) - throw new InvalidOperationException("Failed to write to the transaction channel"); - - return await pending.CompletionSource.Task; + return await Transact(new IndexSegment()); } public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog => _updatesSubject; diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs b/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs index 00799ea1..004502b4 100644 --- a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs +++ b/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.Runtime.InteropServices; +using DynamicData; using NexusMods.MnemonicDB.Abstractions; using NexusMods.MnemonicDB.Abstractions.DatomComparators; using NexusMods.MnemonicDB.Abstractions.DatomIterators; @@ -21,6 +23,15 @@ public class IndexStore : IRocksDBIndexStore private IntPtr _namePtr; private ColumnFamilyOptions _options = null!; + /// + /// This is a bit of a hack, but we throw all our interop delegates in here, and then they + /// live for the entire life of the application. It seems that RocksDB will occasionally call + /// delegates after we think we've disposed of the handles. It really doesn't matter as these + /// things will never amount to more than a few dozen objects. + /// + /// + private static List _roots = new (); + public IndexStore(string handleName, IndexType type, AttributeRegistry registry) { Type = type; @@ -47,6 +58,9 @@ public void SetupColumnFamily(IIndex index, ColumnFamilies columnFamilies) } }; + // Save these as roots so they never get GC'd + _roots.Add((_nameDelegate, _destructorDelegate, _comparatorDelegate)); + _comparator = Native.Instance.rocksdb_comparator_create(IntPtr.Zero, _destructorDelegate, _comparatorDelegate, _nameDelegate);