Merge pull request #20 from Nexus-Mods/perf-experiments

Merging Some Perf Related Experiments from Last Week

NexusMods.Archives.Nx.Cli/Program.cs

@@ -39,8 +39,8 @@
     new Option<string>("--source", "[Required] Source folder to pack files from.") { IsRequired = true },
     new Option<string>("--target", "[Required] Target location to place packed archive to.") { IsRequired = true },
     new Option<int?>("--blocksize", () => defaultPackerSettings.BlockSize,
-        "Size of SOLID blocks. Range is 32767 to 67108863 (64 MiB). This is a power of 2 (minus one) and must be smaller than chunk size."),
-    new Option<int?>("--chunksize", () => defaultPackerSettings.ChunkSize, "Size of large file chunks. Range is 4194304 (4 MiB) to 536870912 (512 MiB)."),
+        "Size of SOLID blocks. Range is 4096 to 67108863 (64 MiB). This is a power of 2 (minus one) and must be smaller than chunk size."),
+    new Option<int?>("--chunksize", () => defaultPackerSettings.ChunkSize, "Size of large file chunks. Range is 32768 (32K) to 1073741824 (1GiB). Must be power of 2."),
     new Option<int?>("--solidlevel", () => defaultPackerSettings.SolidCompressionLevel, "Compression level to use for SOLID data. ZStandard has Range -5 - 22. LZ4 has Range: 1 - 12."),
     new Option<int?>("--chunkedlevel", () => defaultPackerSettings.ChunkedCompressionLevel, "Compression level to use for chunks of large data. ZStandard has Range -5 - 22. LZ4 has Range: 1 - 12."),
     new Option<CompressionPreference?>("--solid-algorithm", () => defaultPackerSettings.SolidBlockAlgorithm, "Compression algorithm used for compressing SOLID blocks."),
@@ -58,7 +58,7 @@
     benchmarkCommand
 };
 
-// Parse the incoming args and invoke the handler 
+// Parse the incoming args and invoke the handler
 rootCommand.Invoke(args);
 
 void Extract(string source, string target, int? threads)
@@ -72,58 +72,58 @@ void Extract(string source, string target, int? threads)
 
     if (threads.HasValue)
         builder.WithMaxNumThreads(threads.Value);
-    
+
     Console.WriteLine("Initialized in {0}ms", initializeTimeTaken.ElapsedMilliseconds);
 
     // Progress Reporting.
     var unpackingTimeTaken = Stopwatch.StartNew();
     AnsiConsole.Progress()
-        .Start(ctx => 
+        .Start(ctx =>
         {
             // Define tasks
             var packTask = ctx.AddTask("[green]Unpacking Files[/]");
             var progress = new Progress<double>(d => packTask.Value = d * 100);
             builder.WithProgress(progress);
             builder.Extract();
         });
-    
+
     Console.WriteLine("Unpacked in {0}ms", unpackingTimeTaken.ElapsedMilliseconds);
 }
 
 void Pack(string source, string target, int? blocksize, int? chunksize, int? solidLevel, int? chunkedLevel, CompressionPreference? solidAlgorithm, CompressionPreference? chunkedAlgorithm, int? threads)
 {
     Console.WriteLine($"Packing {source} to {target} with {threads} threads, blocksize [{blocksize}], chunksize [{chunksize}], solidLevel [{solidLevel}], chunkedLevel [{chunkedLevel}], solidAlgorithm [{solidAlgorithm}], chunkedAlgorithm [{chunkedAlgorithm}].");
-    
+
     var builder = new NxPackerBuilder();
     builder.AddFolder(source);
     builder.WithOutput(new FileStream(target, FileMode.Create, FileAccess.ReadWrite));
-    
+
     if (blocksize.HasValue)
         builder.WithBlockSize(blocksize.Value);
-    
+
     if (chunksize.HasValue)
         builder.WithChunkSize(chunksize.Value);
 
     if (solidLevel.HasValue)
         builder.WithSolidCompressionLevel(solidLevel.Value);
-    
+
     if (chunkedLevel.HasValue)
         builder.WithChunkedLevel(chunkedLevel.Value);
-    
+
     if (solidAlgorithm.HasValue)
         builder.WithSolidBlockAlgorithm(solidAlgorithm.Value);
-    
+
     if (chunkedAlgorithm.HasValue)
         builder.WithChunkedFileAlgorithm(chunkedAlgorithm.Value);
-    
+
     if (threads.HasValue)
         builder.WithMaxNumThreads(threads.Value);
 
     var packingTimeTaken = Stopwatch.StartNew();
 
     // Progress Reporting.
     AnsiConsole.Progress()
-        .Start(ctx => 
+        .Start(ctx =>
         {
             // Define tasks
             var packTask = ctx.AddTask("[green]Packing Files[/]");
@@ -148,7 +148,7 @@ void Benchmark(string source, int? threads, int? attempts)
         builder.WithMaxNumThreads(threads.Value);
 
     long totalTimeTaken = 0;
-    
+
     // Warmup, get that JIT to promote all the way to max tier.
     // With .NET 8, and R2R, this might take 2 (* 40) executions.
     for (var x = 0; x < 80; x++)
@@ -157,7 +157,7 @@ void Benchmark(string source, int? threads, int? attempts)
         builder.Extract();
         Console.WriteLine("[Warmup] Unpacked in {0}ms", unpackingTimeTaken.ElapsedMilliseconds);
     }
-    
+
     attempts = attempts.GetValueOrDefault(25);
     for (var x = 0; x < attempts; x++)
     {
@@ -170,4 +170,4 @@ void Benchmark(string source, int? threads, int? attempts)
     var averageMs = (totalTimeTaken / (float)attempts);
     Console.WriteLine("Average {0:###.00}ms", averageMs);
     Console.WriteLine("Throughput {0:###.00}GiB/s", outputs.Sum(x => (long)x.Data.Length) / averageMs / 1048576F);
-}
+}

NexusMods.Archives.Nx.Tests/Tests/Headers/NativeHeaderPackingTests.cs

@@ -24,7 +24,7 @@ public class NativeHeaderPackingTests
     [AutoNativeHeaders]
     public void VersionAndBlockSizeCanBePacked(NativeFileHeader header)
     {
-        foreach (var currentVersion in Permutations.GetBitPackingOverlapTestValues(4))
+        foreach (var currentVersion in Permutations.GetBitPackingOverlapTestValues(3))
             foreach (var currentBlockSize in Permutations.GetBitPackingOverlapTestValues(4))
                 PackingTestHelpers.TestPackedProperties(
                     ref header,
@@ -39,11 +39,11 @@ public void VersionAndBlockSizeCanBePacked(NativeFileHeader header)
 
     [Theory]
     [AutoNativeHeaders]
-    public void VersionShouldBe4Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits(
+    public void VersionShouldBe3Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits(
         ref header,
         (ref NativeFileHeader instance, long value) => instance.Version = (byte)value,
         (ref NativeFileHeader instance) => instance.Version,
-        4);
+        3);
 
     [Theory]
     [AutoNativeHeaders]
@@ -76,11 +76,11 @@ public void ChunkSizeAndPageCountCanBePacked(NativeFileHeader header)
 
     [Theory]
     [AutoNativeHeaders]
-    public void ChunkSizeShouldBe3Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits(
+    public void ChunkSizeShouldBe4Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits(
         ref header,
         (ref NativeFileHeader instance, long value) => instance.ChunkSize = (byte)value,
         (ref NativeFileHeader instance) => instance.ChunkSize,
-        3);
+        4);
 
     [Theory]
     [AutoNativeHeaders]
@@ -99,25 +99,26 @@ public void ReverseEndian_ReversesExpectedValues(NativeFileHeader header)
     {
         // Setting values that are guaranteed not to mirror in hex
         header.Magic = 1234;
-        header._largeChunkSizeAndPageCount = 5678;
+        header._headerData = 5678;
 
         // Copy and assert they are reversed.
         var header2 = header;
         header2.ReverseEndian();
 
         header2.Magic.Should().NotBe(header.Magic);
-        header2._largeChunkSizeAndPageCount.Should().NotBe(header._largeChunkSizeAndPageCount);
+        header2._headerData.Should().NotBe(header._headerData);
 
         // Now reverse again and doubly make sure
         header2.ReverseEndian();
         header2.Magic.Should().Be(header.Magic);
-        header2._largeChunkSizeAndPageCount.Should().Be(header._largeChunkSizeAndPageCount);
+        header2._headerData.Should().Be(header._headerData);
     }
 
     [Theory]
-    [InlineData(0, 32767)]
-    [InlineData(1, 65535)]
-    [InlineData(11, 67108863)]
+    [InlineData(0, 4095)]
+    [InlineData(1, 8191)]
+    [InlineData(14, 67108863)]
+    [InlineData(15, 134217727)]
     public void BlockSizeBytes_IsCorrectlyConverted(int rawValue, int numBytes)
     {
         var header = new NativeFileHeader();
@@ -132,9 +133,10 @@ public void BlockSizeBytes_IsCorrectlyConverted(int rawValue, int numBytes)
     }
 
     [Theory]
-    [InlineData(0, 4194304)]
-    [InlineData(1, 8388608)]
-    [InlineData(7, 536870912)]
+    [InlineData(0, 32768)]
+    [InlineData(1, 65536)]
+    [InlineData(14, 536870912)]
+    [InlineData(15, 1073741824)]
     public void ChunkSizeBytes_IsCorrectlyConverted(int rawValue, int numBytes)
     {
         var header = new NativeFileHeader();

NexusMods.Archives.Nx.Tests/Tests/Packing/PackerSettingsTests.cs

@@ -7,10 +7,10 @@ namespace NexusMods.Archives.Nx.Tests.Tests.Packing;
 public class PackerSettingsTests
 {
     [Theory]
-    [InlineData(536870913, 536870912)]
-    [InlineData(int.MaxValue, 536870912)]
-    [InlineData(4194303, 4194304)]
-    [InlineData(int.MinValue, 4194304)]
+    [InlineData(1073741825, 1073741824)]
+    [InlineData(int.MaxValue, 1073741824)]
+    [InlineData(1048575, 1048576)]
+    [InlineData(int.MinValue, 1048576)]
     public void ChunkSize_IsClamped(int chunkSize, int expected)
     {
         var settings = new PackerSettings { Output = Stream.Null };
@@ -22,8 +22,8 @@ public void ChunkSize_IsClamped(int chunkSize, int expected)
     [Theory]
     [InlineData(67108864, 67108863)]
     [InlineData(int.MaxValue, 67108863)]
-    [InlineData(32766, 32767)]
-    [InlineData(int.MinValue, 32767)]
+    [InlineData(4094, 4095)]
+    [InlineData(int.MinValue, 4095)]
     public void BlockSize_IsClamped(int value, int expected)
     {
         var settings = new PackerSettings { Output = Stream.Null };

NexusMods.Archives.Nx/FileProviders/FileData/MemoryMappedFileData.cs

@@ -1,6 +1,10 @@
 using System.IO.MemoryMappedFiles;
 using JetBrains.Annotations;
 using NexusMods.Archives.Nx.Interfaces;
+#if NET5_0_OR_GREATER
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+#endif
 
 // ReSharper disable IntroduceOptionalParameters.Global
 
@@ -77,6 +81,30 @@ private unsafe void InitFromMmf(long start, uint length, bool isReadOnly = false
         _mappedFileView = _mappedFile!.CreateViewAccessor(start, length, access);
         Data = (byte*)_mappedFileView.SafeMemoryMappedViewHandle.DangerousGetHandle();
         DataLength = length;
+
+        // Provide some OS specific hints
+        // POSIX compliant
+#if NET5_0_OR_GREATER
+        if (OperatingSystem.IsLinux())
+        {
+            // Also tried MADV_SEQUENTIAL, but didn't yield a benefit (on Linux) strangely.
+            madvise(Data, length, 3); // MADV_WILLNEED
+        }
+        else if (OperatingSystem.IsAndroid())
+        {
+            madvise_android(Data, length, 3); // MADV_WILLNEED
+        }
+        else if (OperatingSystem.IsMacOS() || OperatingSystem.IsIOS())
+        {
+            madvise_libSystem(Data, length, 3); // MADV_WILLNEED
+        }
+        else if (OperatingSystem.IsWindows())
+        {
+            var entries = stackalloc MemoryRangeEntry[1];
+            entries[0] = new MemoryRangeEntry { VirtualAddress = (nint)Data, NumberOfBytes = DataLength };
+            PrefetchVirtualMemory(Process.GetCurrentProcess().Handle, (nuint)1, entries, 0);
+        }
+#endif
     }
 
     private unsafe void InitEmpty()
@@ -89,4 +117,37 @@ private unsafe void InitEmpty()
 
     /// <inheritdoc />
     ~MemoryMappedFileData() => Dispose();
+
+    #region Memory Access Hints for OSes
+
+#if NET5_0_OR_GREATER
+    // POSIX Compatible
+    [DllImport("libc.so.6", EntryPoint = "madvise")]
+    private static extern unsafe int madvise(byte* addr, nuint length, int advice);
+
+    // POSIX Compatible
+    [DllImport("libc.so", EntryPoint = "madvise")]
+    private static extern unsafe int madvise_android(byte* addr, nuint length, int advice);
+
+    // OSX
+    [DllImport("libSystem", EntryPoint = "madvise")]
+    private static extern unsafe int madvise_libSystem(byte* addr, nuint length, int advice);
+
+    // Windows-Like
+    [StructLayout(LayoutKind.Sequential)]
+    private struct MemoryRangeEntry
+    {
+        public nint VirtualAddress;
+        public nuint NumberOfBytes;
+    }
+
+    [DllImport("kernel32.dll", SetLastError = true)]
+    private static extern unsafe bool PrefetchVirtualMemory(
+        IntPtr hProcess,
+        UIntPtr numberOfEntries,
+        MemoryRangeEntry* memoryRanges,
+        uint flags);
+#endif
+
+    #endregion
 }

NexusMods.Archives.Nx/FileProviders/OutputFileProvider.cs

@@ -21,8 +21,7 @@ public sealed class OutputFileProvider : IOutputDataProvider
     /// </summary>
     public string FullPath { get; init; }
 
-    private readonly MemoryMappedFile? _mappedFile;
-    private readonly FileStream _fileStream;
+    private readonly MemoryMappedFile _mappedFile;
     private bool _isDisposed;
 
     /// <summary>
@@ -43,31 +42,14 @@ public OutputFileProvider(string outputFolder, string relativePath, FileEntry en
     trycreate:
         try
         {
-#if NET7_0_OR_GREATER
-            _fileStream = new FileStream(FullPath, new FileStreamOptions
-            {
-                PreallocationSize = (long)entry.DecompressedSize,
-                Access = FileAccess.ReadWrite,
-                Mode = FileMode.Create,
-                Share = FileShare.ReadWrite,
-                BufferSize = 0,
-                Options = FileOptions.SequentialScan
-            });
-#else
-            _fileStream = new FileStream(FullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
-            _fileStream.SetLength((long)entry.DecompressedSize);
-#endif
+            _mappedFile = MemoryMappedFile.CreateFromFile(FullPath, FileMode.Create, null, (long)entry.DecompressedSize, MemoryMappedFileAccess.ReadWrite);
         }
         catch (DirectoryNotFoundException)
         {
             // This is written this way because explicit check is slow.
             Directory.CreateDirectory(Path.GetDirectoryName(FullPath)!);
             goto trycreate;
         }
-
-        if (entry.DecompressedSize > 0)
-            _mappedFile = MemoryMappedFile.CreateFromFile(_fileStream, null, (long)entry.DecompressedSize, MemoryMappedFileAccess.ReadWrite,
-                HandleInheritability.None, true);
     }
 
     /// <inheritdoc />
@@ -83,8 +65,7 @@ public void Dispose()
             return;
 
         _isDisposed = true;
-        _mappedFile?.Dispose();
-        _fileStream.Dispose();
+        _mappedFile.Dispose();
         GC.SuppressFinalize(this);
     }
 }