Merge pull request #94 from koculu/93-enhancement-provide-operation-i…

…ndex-to-support-asynchronous-replication-and-audits Added operation index to support asynchronous replication and audits
koculu · Sep 7, 2024 · bbf27b3 · bbf27b3
2 parents c4746e2 + e060896
commit bbf27b3
Show file tree

Hide file tree

Showing 17 changed files with 212 additions and 80 deletions.
diff --git a/README.md b/README.md
@@ -175,7 +175,7 @@ zoneTree.TryAtomicAddOrUpdate(39, "a", (ref string x) =>
 {
     x += "b";
     return true;
-});
+}, out var opIndex);
 ```
 
 ---

diff --git a/src/Playground/Test1.cs b/src/Playground/Test1.cs
@@ -108,7 +108,7 @@ public static void TestReverseIterator(
                     {
                         x += " ooops!";
                         return true;
-                    });
+                    }, out _);
             }
             maintainer.WaitForBackgroundThreads();
         }

diff --git a/src/ZoneTree.UnitTests/AtomicUpdateTests.cs b/src/ZoneTree.UnitTests/AtomicUpdateTests.cs
@@ -48,7 +48,8 @@ public void IntIntAtomicIncrement(WriteAheadLogMode walMode)
                         {
                             ++y;
                             return true;
-                        }
+                        },
+                        out _
                     );
                     Interlocked.Increment(ref off);
                 }
@@ -116,7 +117,7 @@ public void IntIntAtomicIncrementForBTree(WriteAheadLogMode walMode)
                         {
                             ++y;
                             return true;
-                        });
+                        }, out _);
                     Interlocked.Increment(ref off);
                 }
 
@@ -182,7 +183,7 @@ public void IntIntMutableSegmentOnlyAtomicIncrement(WriteAheadLogMode walMode)
                         {
                             ++y;
                             return true;
-                        });
+                        }, out _);
                     Interlocked.Increment(ref off);
                 }
 

diff --git a/src/ZoneTree.UnitTests/FixedSizeKeyAndValueTests.cs b/src/ZoneTree.UnitTests/FixedSizeKeyAndValueTests.cs
@@ -75,10 +75,10 @@ public void IntStringDeleteTest()
         using var data = new ZoneTreeFactory<int, string>()
             .SetDataDirectory(dataPath)
             .OpenOrCreate();
-        data.TryAtomicAdd(1, "1");
-        data.TryAtomicAdd(2, "2");
-        data.TryAtomicAdd(3, "3");
-        data.TryDelete(2);
+        data.TryAtomicAdd(1, "1", out _);
+        data.TryAtomicAdd(2, "2", out _);
+        data.TryAtomicAdd(3, "3", out _);
+        data.TryDelete(2, out _);
         Assert.That(data.ContainsKey(1), Is.True);
         Assert.That(data.ContainsKey(2), Is.False);
         Assert.That(data.ContainsKey(3), Is.True);
@@ -95,10 +95,10 @@ public void IntNullableIntDeleteTest()
             .SetDataDirectory(dataPath)
             .SetValueSerializer(new NullableInt32Serializer())
             .OpenOrCreate();
-        data.TryAtomicAdd(1, 1);
-        data.TryAtomicAdd(2, 2);
-        data.TryAtomicAdd(3, 3);
-        data.TryDelete(2);
+        data.TryAtomicAdd(1, 1, out _);
+        data.TryAtomicAdd(2, 2, out _);
+        data.TryAtomicAdd(3, 3, out _);
+        data.TryDelete(2, out _);
         Assert.That(data.ContainsKey(1), Is.True);
         Assert.That(data.ContainsKey(2), Is.False);
         Assert.That(data.ContainsKey(3), Is.True);
@@ -116,13 +116,13 @@ public void IntStringGarbageCollectionTest()
             using var data = new ZoneTreeFactory<int, string>()
                 .SetDataDirectory(dataPath)
                 .OpenOrCreate();
-            data.TryAtomicAdd(1, "1");
-            data.TryAtomicAdd(2, "2");
-            data.TryAtomicAdd(3, "3");
-            data.TryDelete(2);
-            data.TryAtomicAdd(4, "4");
-            data.TryAtomicUpdate(3, "33");
-            data.TryDelete(2);
+            data.TryAtomicAdd(1, "1", out _);
+            data.TryAtomicAdd(2, "2", out _);
+            data.TryAtomicAdd(3, "3", out _);
+            data.TryDelete(2, out _);
+            data.TryAtomicAdd(4, "4", out _);
+            data.TryAtomicUpdate(3, "33", out _);
+            data.TryDelete(2, out _);
             Assert.That(data.ContainsKey(1), Is.True);
             Assert.That(data.ContainsKey(2), Is.False);
             Assert.That(data.ContainsKey(3), Is.True);
@@ -161,13 +161,13 @@ public void IntStringReadOnlySegmentLoadingTest()
             using var data = new ZoneTreeFactory<int, string>()
                 .SetDataDirectory(dataPath)
                 .OpenOrCreate();
-            data.TryAtomicAdd(1, "1");
-            data.TryAtomicAdd(2, "2");
-            data.TryAtomicAdd(3, "3");
-            data.TryDelete(2);
-            data.TryAtomicAdd(4, "4");
-            data.TryAtomicUpdate(3, "33");
-            data.TryDelete(2);
+            data.TryAtomicAdd(1, "1", out _);
+            data.TryAtomicAdd(2, "2", out _);
+            data.TryAtomicAdd(3, "3", out _);
+            data.TryDelete(2, out _);
+            data.TryAtomicAdd(4, "4", out _);
+            data.TryAtomicUpdate(3, "33", out _);
+            data.TryDelete(2, out _);
             Assert.That(data.ContainsKey(1), Is.True);
             Assert.That(data.ContainsKey(2), Is.False);
             Assert.That(data.ContainsKey(3), Is.True);
@@ -207,10 +207,10 @@ public void IntStringDiskSegmentLoadingTest()
             using var data = new ZoneTreeFactory<int, string>()
                 .SetDataDirectory(dataPath)
                 .OpenOrCreate();
-            data.TryAtomicAdd(1, "1");
-            data.TryAtomicAdd(2, "2");
-            data.TryAtomicAdd(3, "3");
-            data.TryDelete(2);
+            data.TryAtomicAdd(1, "1", out _);
+            data.TryAtomicAdd(2, "2", out _);
+            data.TryAtomicAdd(3, "3", out _);
+            data.TryDelete(2, out _);
             Assert.That(data.ContainsKey(1), Is.True);
             Assert.That(data.ContainsKey(2), Is.False);
             Assert.That(data.ContainsKey(3), Is.True);

diff --git a/src/ZoneTree.UnitTests/OpIndexTests.cs b/src/ZoneTree.UnitTests/OpIndexTests.cs
@@ -0,0 +1,51 @@
+using System.Collections.Concurrent;
+using Tenray.ZoneTree.PresetTypes;
+using Tenray.ZoneTree.Serializers;
+
+namespace Tenray.ZoneTree.UnitTests;
+
+public sealed class OpIndexTests
+{
+    [Test]
+    public void TestOpIndex()
+    {
+        var dataPath = "data/TestOpIndex";
+        if (Directory.Exists(dataPath))
+            Directory.Delete(dataPath, true);
+        var recordCount = 10_000;
+        var opIndexes = new ConcurrentBag<long>();
+        void CreateData()
+        {
+            using var zoneTree = new ZoneTreeFactory<int, int>()
+                .SetDataDirectory(dataPath)
+                .SetMutableSegmentMaxItemCount(100)
+                .OpenOrCreate();
+
+            using var maintainer = zoneTree.CreateMaintainer();
+            Parallel.For(0, recordCount, (i) =>
+            {
+                var opIndex = zoneTree.Upsert(i, i);
+                opIndexes.Add(opIndex);
+            });
+            maintainer.EvictToDisk();
+            maintainer.WaitForBackgroundThreads();
+        }
+
+        void ReloadData()
+        {
+            using var zoneTree = new ZoneTreeFactory<int, int>()
+                .SetDataDirectory(dataPath)
+                .SetMutableSegmentMaxItemCount(100)
+                .Open();
+
+            var opIndex = zoneTree.Upsert(recordCount + 1, recordCount + 1);
+            Assert.That(opIndex, Is.EqualTo(recordCount + 1));
+            zoneTree.Maintenance.Drop();
+        }
+        CreateData();
+        ReloadData();
+        Assert.IsTrue(
+            opIndexes.Order().ToArray()
+            .SequenceEqual(Enumerable.Range(1, recordCount).Select(x => (long)x)));
+    }
+}
diff --git a/src/ZoneTree.UnitTests/StringTreeTests.cs b/src/ZoneTree.UnitTests/StringTreeTests.cs
@@ -114,7 +114,7 @@ public void HelloWorldTest2()
             {
                 x += "b";
                 return true;
-            });
+            }, out _);
         zoneTree.TryGet(39, out value);
         Assert.That(value, Is.EqualTo("Hello Zone Tree!b"));
     }

diff --git a/src/ZoneTree.UnitTests/TTLTests.cs b/src/ZoneTree.UnitTests/TTLTests.cs
@@ -35,13 +35,15 @@ public void TestTTL()
             5,
             out v1,
             bool (ref TTLValue<int> v) =>
-                v.SlideExpiration(TimeSpan.FromMilliseconds(300)));
+                v.SlideExpiration(TimeSpan.FromMilliseconds(300)),
+            out _);
         Thread.Sleep(450); // initial expiration (300) + slided expiration (300) - Thread.Sleep(150)
         f2 = zoneTree.TryGetAndUpdate(
             5,
             out v2,
             bool (ref TTLValue<int> v) =>
-                v.SlideExpiration(TimeSpan.FromMilliseconds(300)));
+                v.SlideExpiration(TimeSpan.FromMilliseconds(300)),
+            out _);
 
         Assert.That(f1, Is.True);
         Assert.That(f2, Is.False);

diff --git a/src/ZoneTree/Core/ZoneTree.Merge.cs b/src/ZoneTree/Core/ZoneTree.Merge.cs
@@ -36,6 +36,7 @@ void MoveMutableSegmentForward(IMutableSegment<TKey, TValue> mutableSegment)
 
             mutableSegment.Freeze();
             ReadOnlySegmentQueue.Enqueue(mutableSegment);
+            MetaWal.EnqueueMaximumOpIndex(mutableSegment.MaximumOpIndex);
             MetaWal.EnqueueReadOnlySegment(mutableSegment.SegmentId);
 
             MutableSegment = new MutableSegment<TKey, TValue>(
-Original file line number
+Diff line change
@@ Expand Up / @@ -175,7 +175,7 @@ zoneTree.TryAtomicAddOrUpdate(39, "a", (ref string x) => @@
     {
         x += "b";
         return true;
-    });
+    }, out var opIndex);
     ```
     ---
@@ Expand Down @@