Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MoveMutableSegmentForward should wait until all writes to the previous mutable segment finalized. #100

Merged
merged 4 commits into from
Oct 16, 2024
Merged

MoveMutableSegmentForward should wait until all writes to the previous mutable segment finalized. #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
309836e
MoveMutableSegmentForward should wait until all writes to the previou…
koculu Oct 15, 2024
a4b7c1e
Add xml docs.
koculu Oct 16, 2024
61c82cc
Use thread-safe random number generator.
koculu Oct 16, 2024
99199b1
Assign opIndex only if needed.
koculu Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Playground/Benchmark/OldTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public static void ShowBottomSegments(WriteAheadLogMode mode, int count)
var iterate = false;
if (iterate)
{
var random = new Random();
var random = Random.Shared;
Parallel.For(0, 750000, (i) =>
{
var key = random.Next(0, 999_999_999);
Expand Down
2 changes: 1 addition & 1 deletion src/Playground/DeadlockFinder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ void TreeLoop(int taskNo)
const int upsertLogFrequency = 1_000_000;
const int iterationLogFrequency = 10_000_000;
const int iterationYieldFrequency = 1000;
var rand = new Random();
var rand = Random.Shared;

void resetTree()
{
Expand Down
8 changes: 4 additions & 4 deletions src/ZoneTree.UnitTests/AtomicUpdateTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public void IntIntAtomicIncrement(WriteAheadLogMode walMode)
}
data.Maintenance.MoveMutableSegmentForward();
data.Maintenance.StartMergeOperation().Join();
var random = new Random();
var random = Random.Shared;
var off = -1;
Parallel.For(0, 1001, (x) =>
{
Expand Down Expand Up @@ -100,7 +100,7 @@ public void IntIntAtomicIncrementForBTree(WriteAheadLogMode walMode)
{
data.Upsert(i, i + i);
}
var random = new Random();
var random = Random.Shared;
var off = -1;
Parallel.For(0, 1001, (x) =>
{
Expand Down Expand Up @@ -166,7 +166,7 @@ public void IntIntMutableSegmentOnlyAtomicIncrement(WriteAheadLogMode walMode)
{
data.Upsert(i, i + i);
}
var random = new Random();
var random = Random.Shared;
var off = -1;
Parallel.For(0, 1001, (x) =>
{
Expand Down Expand Up @@ -228,7 +228,7 @@ public void IntIntMutableSegmentSeveralUpserts(WriteAheadLogMode walMode)
.ConfigureWriteAheadLogOptions(x => x.WriteAheadLogMode = walMode)
.OpenOrCreate();
var n = 1000;
var random = new Random();
var random = Random.Shared;
Parallel.For(0, 1000, (x) =>
{
try
Expand Down
2 changes: 1 addition & 1 deletion src/ZoneTree.UnitTests/ExceptionlessTransactionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public async Task TransactionWithFluentAPI(int compactionThreshold)

zoneTree.Maintenance.TransactionLog.CompactionThreshold = compactionThreshold;

var random = new Random();
var random = Random.Shared;
await Parallel.ForEachAsync(Enumerable.Range(0, 1000), async (x, cancel) =>
{
using var transaction =
Expand Down
10 changes: 5 additions & 5 deletions src/ZoneTree.UnitTests/IteratorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ public void IntIntIteratorParallelInserts()
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);

var random = new Random();
var random = Random.Shared;
var insertCount = 100000;
var iteratorCount = 1000;

Expand Down Expand Up @@ -289,7 +289,7 @@ public void IntIntReverseIteratorParallelInserts(bool reverse)
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);

var random = new Random();
var random = Random.Shared;
var insertCount = 1000000;
var iteratorCount = 1000;

Expand Down Expand Up @@ -352,7 +352,7 @@ public void IntIntSnapshotIteratorParallelInserts()
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);

var random = new Random();
var random = Random.Shared;
var insertCount = 100000;
var iteratorCount = 1000;

Expand Down Expand Up @@ -512,7 +512,7 @@ public void SeekIteratorsAfterMerge(
if (merge)
zoneTree.Maintenance.StartMergeOperation()?.Join();

var random = new Random();
var random = Random.Shared;
DoPrefixSearch(zoneTree, list, random);
zoneTree.Maintenance.DiskSegment.InitSparseArray(100);
DoPrefixSearch(zoneTree, list, random);
Expand Down Expand Up @@ -604,7 +604,7 @@ public void SeekIteratorsAfterMergeReload(
x.MaximumRecordCount = maximumRecordCount;
})
.Open();
var random = new Random();
var random = Random.Shared;
DoPrefixSearch(zoneTree, list, random);
zoneTree.Maintenance.DiskSegment.InitSparseArray(100);
DoPrefixSearch(zoneTree, list, random);
Expand Down
93 changes: 83 additions & 10 deletions src/ZoneTree.UnitTests/ReplicatorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,6 @@ namespace Tenray.ZoneTree.UnitTests;

public sealed class ReplicatorTests
{
[Test]
public void TestReplicator2()
{
for (int i = 0; i < 100; i++)
{
TestReplicator();
}
}

[Test]
public void TestReplicator()
{
Expand All @@ -21,14 +12,17 @@ public void TestReplicator()
Directory.Delete(dataPath, true);
var recordCount = 50_000;
var keyCount = 15_000;
var maxMemory = 2_000;
void CreateData()
{
using var zoneTree = new ZoneTreeFactory<int, int>()
.SetDataDirectory(dataPath + "/source")
.SetMutableSegmentMaxItemCount(maxMemory)
.OpenOrCreate();

using var replica = new ZoneTreeFactory<int, int>()
.SetDataDirectory(dataPath + "/replica")
.SetMutableSegmentMaxItemCount(maxMemory)
.OpenOrCreate();

using var replicator = new Replicator<int, int>(replica, dataPath + "/replica-op-index");
Expand Down Expand Up @@ -84,4 +78,83 @@ void TestEqual()
CreateData();
TestEqual();
}
}

[Test]
public void TestReplicator2()
{
for (int i = 0; i < 5; i++)
{
var dataPath = "data/TestReplicator";
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);

var recordCount = 50_000;
var keyCount = 15_000;
var maxMemory = 2_000;

void CreateData()
{
using var zoneTree = new ZoneTreeFactory<int, int>()
.SetDataDirectory(dataPath + "/source")
.SetMutableSegmentMaxItemCount(maxMemory)
.OpenOrCreate();

using var replica = new ZoneTreeFactory<int, int>()
.SetDataDirectory(dataPath + "/replica")
.SetMutableSegmentMaxItemCount(maxMemory)
.OpenOrCreate();

using var replicator = new Replicator<int, int>(replica, dataPath + "/replica-op-index");
using var maintainer1 = zoneTree.CreateMaintainer();
using var maintainer2 = replica.CreateMaintainer();
int replicated = 0;
var k = 0;
Parallel.For(0, recordCount, (i) =>
{
var key = i % keyCount;
var value = Interlocked.Increment(ref k);
var opIndex = zoneTree.Upsert(key, value);
Task.Run(() =>
{
replicator.OnUpsert(key, value, opIndex);
Interlocked.Increment(ref replicated);
});
});
while (replicated < recordCount) Task.Delay(500).Wait();
maintainer1.EvictToDisk();
maintainer2.EvictToDisk();
maintainer1.WaitForBackgroundThreads();
maintainer2.WaitForBackgroundThreads();
}

void TestEqual()
{
using var zoneTree = new ZoneTreeFactory<int, int>()
.SetDataDirectory(dataPath + "/source")
.Open();

using var replica = new ZoneTreeFactory<int, int>()
.SetDataDirectory(dataPath + "/replica")
.Open();

using var iterator1 = zoneTree.CreateIterator();
using var iterator2 = replica.CreateIterator();
var i = 0;
while (true)
{
var n1 = iterator1.Next();
var n2 = iterator2.Next();
Assert.That(n2, Is.EqualTo(n1));
if (!n1) break;
Assert.That(iterator2.Current, Is.EqualTo(iterator1.Current));
++i;
}
Assert.That(i, Is.EqualTo(keyCount));
zoneTree.Maintenance.Drop();
replica.Maintenance.Drop();
}
CreateData();
TestEqual();
}
}
}
16 changes: 8 additions & 8 deletions src/ZoneTree.UnitTests/SafeBplusTreeTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ public void BTreeIteration(BTreeLockMode lockMode)
tree.TryInsert(i, i + i, out _);

var iterator = new BTreeSeekableIterator<int, int>(tree);
var j = 0;
var j = 0;
while (iterator.Next())
{
Assert.That(iterator.CurrentKey, Is.EqualTo(j));
Assert.That(iterator.CurrentValue, Is.EqualTo(j + j));
++j;
}
}

iterator.SeekEnd();
j = n - 1;
Expand Down Expand Up @@ -92,7 +92,7 @@ public void BTreeLowerOrEqualBound(BTreeLockMode lockMode)
Assert.That(GetLastNodeSmallerOrEqual(iterator, 4), Is.EqualTo(3));
Assert.That(GetLastNodeSmallerOrEqual(iterator, 3), Is.EqualTo(3));
Assert.Throws<IndexOutOfRangeException>(
() => GetLastNodeSmallerOrEqual(iterator ,- 1));
() => GetLastNodeSmallerOrEqual(iterator, -1));
Assert.That(GetLastNodeSmallerOrEqual(iterator, 10), Is.EqualTo(9));
Assert.That(GetLastNodeSmallerOrEqual(iterator, 9), Is.EqualTo(9));
Assert.That(GetLastNodeSmallerOrEqual(iterator, 1), Is.EqualTo(1));
Expand Down Expand Up @@ -135,7 +135,7 @@ int GetFirstNodeGreaterOrEqual(BTreeSeekableIterator<int, int> iterator, int key
[TestCase(BTreeLockMode.NodeLevelMonitor)]
public void BTreeIteratorParallelInserts(BTreeLockMode lockMode)
{
var random = new Random();
var random = Random.Shared;
var insertCount = 100000;
var iteratorCount = 1000;

Expand Down Expand Up @@ -190,7 +190,7 @@ public void BTreeIteratorParallelInserts(BTreeLockMode lockMode)
[TestCase(BTreeLockMode.NodeLevelMonitor)]
public void BTreeReverseIteratorParallelInserts(BTreeLockMode lockMode)
{
var random = new Random();
var random = Random.Shared;
var insertCount = 100000;
var iteratorCount = 1550;

Expand All @@ -203,7 +203,7 @@ public void BTreeReverseIteratorParallelInserts(BTreeLockMode lockMode)
{
var key = random.Next();
tree.AddOrUpdate(key,
AddOrUpdateResult (ref int x) =>
AddOrUpdateResult (ref int x) =>
{
x = key + key;
return AddOrUpdateResult.ADDED;
Expand Down Expand Up @@ -253,7 +253,7 @@ public void BTreeReverseIteratorParallelInserts(BTreeLockMode lockMode)
[TestCase(BTreeLockMode.NodeLevelMonitor)]
public void IntIntDuplicateRecords(BTreeLockMode lockMode)
{
var random = new Random();
var random = Random.Shared;
var insertCount = 1000000;
var iteratorCount = 1000;

Expand Down Expand Up @@ -297,7 +297,7 @@ public void IntIntDuplicateRecords(BTreeLockMode lockMode)
[TestCase(BTreeLockMode.NodeLevelMonitor)]
public void IntIntDuplicateReverseRecords(BTreeLockMode lockMode)
{
var random = new Random();
var random = Random.Shared;
var insertCount = 1000000;
var iteratorCount = 1000;

Expand Down
Loading
Loading