diff --git a/NexusMods.Archives.Nx.Cli/Program.cs b/NexusMods.Archives.Nx.Cli/Program.cs index f14d2cf..522b62d 100644 --- a/NexusMods.Archives.Nx.Cli/Program.cs +++ b/NexusMods.Archives.Nx.Cli/Program.cs @@ -39,8 +39,8 @@ new Option("--source", "[Required] Source folder to pack files from.") { IsRequired = true }, new Option("--target", "[Required] Target location to place packed archive to.") { IsRequired = true }, new Option("--blocksize", () => defaultPackerSettings.BlockSize, - "Size of SOLID blocks. Range is 32767 to 67108863 (64 MiB). This is a power of 2 (minus one) and must be smaller than chunk size."), - new Option("--chunksize", () => defaultPackerSettings.ChunkSize, "Size of large file chunks. Range is 4194304 (4 MiB) to 536870912 (512 MiB)."), + "Size of SOLID blocks. Range is 4096 to 67108863 (64 MiB). This is a power of 2 (minus one) and must be smaller than chunk size."), + new Option("--chunksize", () => defaultPackerSettings.ChunkSize, "Size of large file chunks. Range is 32768 (32K) to 1073741824 (1GiB). Must be power of 2."), new Option("--solidlevel", () => defaultPackerSettings.SolidCompressionLevel, "Compression level to use for SOLID data. ZStandard has Range -5 - 22. LZ4 has Range: 1 - 12."), new Option("--chunkedlevel", () => defaultPackerSettings.ChunkedCompressionLevel, "Compression level to use for chunks of large data. ZStandard has Range -5 - 22. LZ4 has Range: 1 - 12."), new Option("--solid-algorithm", () => defaultPackerSettings.SolidBlockAlgorithm, "Compression algorithm used for compressing SOLID blocks."), @@ -58,7 +58,7 @@ benchmarkCommand }; -// Parse the incoming args and invoke the handler +// Parse the incoming args and invoke the handler rootCommand.Invoke(args); void Extract(string source, string target, int? threads) @@ -72,13 +72,13 @@ void Extract(string source, string target, int? threads) if (threads.HasValue) builder.WithMaxNumThreads(threads.Value); - + Console.WriteLine("Initialized in {0}ms", initializeTimeTaken.ElapsedMilliseconds); // Progress Reporting. var unpackingTimeTaken = Stopwatch.StartNew(); AnsiConsole.Progress() - .Start(ctx => + .Start(ctx => { // Define tasks var packTask = ctx.AddTask("[green]Unpacking Files[/]"); @@ -86,36 +86,36 @@ void Extract(string source, string target, int? threads) builder.WithProgress(progress); builder.Extract(); }); - + Console.WriteLine("Unpacked in {0}ms", unpackingTimeTaken.ElapsedMilliseconds); } void Pack(string source, string target, int? blocksize, int? chunksize, int? solidLevel, int? chunkedLevel, CompressionPreference? solidAlgorithm, CompressionPreference? chunkedAlgorithm, int? threads) { Console.WriteLine($"Packing {source} to {target} with {threads} threads, blocksize [{blocksize}], chunksize [{chunksize}], solidLevel [{solidLevel}], chunkedLevel [{chunkedLevel}], solidAlgorithm [{solidAlgorithm}], chunkedAlgorithm [{chunkedAlgorithm}]."); - + var builder = new NxPackerBuilder(); builder.AddFolder(source); builder.WithOutput(new FileStream(target, FileMode.Create, FileAccess.ReadWrite)); - + if (blocksize.HasValue) builder.WithBlockSize(blocksize.Value); - + if (chunksize.HasValue) builder.WithChunkSize(chunksize.Value); if (solidLevel.HasValue) builder.WithSolidCompressionLevel(solidLevel.Value); - + if (chunkedLevel.HasValue) builder.WithChunkedLevel(chunkedLevel.Value); - + if (solidAlgorithm.HasValue) builder.WithSolidBlockAlgorithm(solidAlgorithm.Value); - + if (chunkedAlgorithm.HasValue) builder.WithChunkedFileAlgorithm(chunkedAlgorithm.Value); - + if (threads.HasValue) builder.WithMaxNumThreads(threads.Value); @@ -123,7 +123,7 @@ void Pack(string source, string target, int? blocksize, int? chunksize, int? sol // Progress Reporting. AnsiConsole.Progress() - .Start(ctx => + .Start(ctx => { // Define tasks var packTask = ctx.AddTask("[green]Packing Files[/]"); @@ -148,7 +148,7 @@ void Benchmark(string source, int? threads, int? attempts) builder.WithMaxNumThreads(threads.Value); long totalTimeTaken = 0; - + // Warmup, get that JIT to promote all the way to max tier. // With .NET 8, and R2R, this might take 2 (* 40) executions. for (var x = 0; x < 80; x++) @@ -157,7 +157,7 @@ void Benchmark(string source, int? threads, int? attempts) builder.Extract(); Console.WriteLine("[Warmup] Unpacked in {0}ms", unpackingTimeTaken.ElapsedMilliseconds); } - + attempts = attempts.GetValueOrDefault(25); for (var x = 0; x < attempts; x++) { @@ -170,4 +170,4 @@ void Benchmark(string source, int? threads, int? attempts) var averageMs = (totalTimeTaken / (float)attempts); Console.WriteLine("Average {0:###.00}ms", averageMs); Console.WriteLine("Throughput {0:###.00}GiB/s", outputs.Sum(x => (long)x.Data.Length) / averageMs / 1048576F); -} \ No newline at end of file +} diff --git a/NexusMods.Archives.Nx.Tests/Tests/Headers/NativeHeaderPackingTests.cs b/NexusMods.Archives.Nx.Tests/Tests/Headers/NativeHeaderPackingTests.cs index 2352031..24fa1f0 100644 --- a/NexusMods.Archives.Nx.Tests/Tests/Headers/NativeHeaderPackingTests.cs +++ b/NexusMods.Archives.Nx.Tests/Tests/Headers/NativeHeaderPackingTests.cs @@ -24,7 +24,7 @@ public class NativeHeaderPackingTests [AutoNativeHeaders] public void VersionAndBlockSizeCanBePacked(NativeFileHeader header) { - foreach (var currentVersion in Permutations.GetBitPackingOverlapTestValues(4)) + foreach (var currentVersion in Permutations.GetBitPackingOverlapTestValues(3)) foreach (var currentBlockSize in Permutations.GetBitPackingOverlapTestValues(4)) PackingTestHelpers.TestPackedProperties( ref header, @@ -39,11 +39,11 @@ public void VersionAndBlockSizeCanBePacked(NativeFileHeader header) [Theory] [AutoNativeHeaders] - public void VersionShouldBe4Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits( + public void VersionShouldBe3Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits( ref header, (ref NativeFileHeader instance, long value) => instance.Version = (byte)value, (ref NativeFileHeader instance) => instance.Version, - 4); + 3); [Theory] [AutoNativeHeaders] @@ -76,11 +76,11 @@ public void ChunkSizeAndPageCountCanBePacked(NativeFileHeader header) [Theory] [AutoNativeHeaders] - public void ChunkSizeShouldBe3Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits( + public void ChunkSizeShouldBe4Bits(NativeFileHeader header) => PackingTestHelpers.AssertSizeBits( ref header, (ref NativeFileHeader instance, long value) => instance.ChunkSize = (byte)value, (ref NativeFileHeader instance) => instance.ChunkSize, - 3); + 4); [Theory] [AutoNativeHeaders] @@ -99,25 +99,26 @@ public void ReverseEndian_ReversesExpectedValues(NativeFileHeader header) { // Setting values that are guaranteed not to mirror in hex header.Magic = 1234; - header._largeChunkSizeAndPageCount = 5678; + header._headerData = 5678; // Copy and assert they are reversed. var header2 = header; header2.ReverseEndian(); header2.Magic.Should().NotBe(header.Magic); - header2._largeChunkSizeAndPageCount.Should().NotBe(header._largeChunkSizeAndPageCount); + header2._headerData.Should().NotBe(header._headerData); // Now reverse again and doubly make sure header2.ReverseEndian(); header2.Magic.Should().Be(header.Magic); - header2._largeChunkSizeAndPageCount.Should().Be(header._largeChunkSizeAndPageCount); + header2._headerData.Should().Be(header._headerData); } [Theory] - [InlineData(0, 32767)] - [InlineData(1, 65535)] - [InlineData(11, 67108863)] + [InlineData(0, 4095)] + [InlineData(1, 8191)] + [InlineData(14, 67108863)] + [InlineData(15, 134217727)] public void BlockSizeBytes_IsCorrectlyConverted(int rawValue, int numBytes) { var header = new NativeFileHeader(); @@ -132,9 +133,10 @@ public void BlockSizeBytes_IsCorrectlyConverted(int rawValue, int numBytes) } [Theory] - [InlineData(0, 4194304)] - [InlineData(1, 8388608)] - [InlineData(7, 536870912)] + [InlineData(0, 32768)] + [InlineData(1, 65536)] + [InlineData(14, 536870912)] + [InlineData(15, 1073741824)] public void ChunkSizeBytes_IsCorrectlyConverted(int rawValue, int numBytes) { var header = new NativeFileHeader(); diff --git a/NexusMods.Archives.Nx.Tests/Tests/Packing/PackerSettingsTests.cs b/NexusMods.Archives.Nx.Tests/Tests/Packing/PackerSettingsTests.cs index 50145eb..b16915b 100644 --- a/NexusMods.Archives.Nx.Tests/Tests/Packing/PackerSettingsTests.cs +++ b/NexusMods.Archives.Nx.Tests/Tests/Packing/PackerSettingsTests.cs @@ -7,10 +7,10 @@ namespace NexusMods.Archives.Nx.Tests.Tests.Packing; public class PackerSettingsTests { [Theory] - [InlineData(536870913, 536870912)] - [InlineData(int.MaxValue, 536870912)] - [InlineData(4194303, 4194304)] - [InlineData(int.MinValue, 4194304)] + [InlineData(1073741825, 1073741824)] + [InlineData(int.MaxValue, 1073741824)] + [InlineData(1048575, 1048576)] + [InlineData(int.MinValue, 1048576)] public void ChunkSize_IsClamped(int chunkSize, int expected) { var settings = new PackerSettings { Output = Stream.Null }; @@ -22,8 +22,8 @@ public void ChunkSize_IsClamped(int chunkSize, int expected) [Theory] [InlineData(67108864, 67108863)] [InlineData(int.MaxValue, 67108863)] - [InlineData(32766, 32767)] - [InlineData(int.MinValue, 32767)] + [InlineData(4094, 4095)] + [InlineData(int.MinValue, 4095)] public void BlockSize_IsClamped(int value, int expected) { var settings = new PackerSettings { Output = Stream.Null }; diff --git a/NexusMods.Archives.Nx/FileProviders/FileData/MemoryMappedFileData.cs b/NexusMods.Archives.Nx/FileProviders/FileData/MemoryMappedFileData.cs index d5a22c7..15363e1 100644 --- a/NexusMods.Archives.Nx/FileProviders/FileData/MemoryMappedFileData.cs +++ b/NexusMods.Archives.Nx/FileProviders/FileData/MemoryMappedFileData.cs @@ -1,6 +1,10 @@ using System.IO.MemoryMappedFiles; using JetBrains.Annotations; using NexusMods.Archives.Nx.Interfaces; +#if NET5_0_OR_GREATER +using System.Diagnostics; +using System.Runtime.InteropServices; +#endif // ReSharper disable IntroduceOptionalParameters.Global @@ -77,6 +81,30 @@ private unsafe void InitFromMmf(long start, uint length, bool isReadOnly = false _mappedFileView = _mappedFile!.CreateViewAccessor(start, length, access); Data = (byte*)_mappedFileView.SafeMemoryMappedViewHandle.DangerousGetHandle(); DataLength = length; + + // Provide some OS specific hints + // POSIX compliant +#if NET5_0_OR_GREATER + if (OperatingSystem.IsLinux()) + { + // Also tried MADV_SEQUENTIAL, but didn't yield a benefit (on Linux) strangely. + madvise(Data, length, 3); // MADV_WILLNEED + } + else if (OperatingSystem.IsAndroid()) + { + madvise_android(Data, length, 3); // MADV_WILLNEED + } + else if (OperatingSystem.IsMacOS() || OperatingSystem.IsIOS()) + { + madvise_libSystem(Data, length, 3); // MADV_WILLNEED + } + else if (OperatingSystem.IsWindows()) + { + var entries = stackalloc MemoryRangeEntry[1]; + entries[0] = new MemoryRangeEntry { VirtualAddress = (nint)Data, NumberOfBytes = DataLength }; + PrefetchVirtualMemory(Process.GetCurrentProcess().Handle, (nuint)1, entries, 0); + } +#endif } private unsafe void InitEmpty() @@ -89,4 +117,37 @@ private unsafe void InitEmpty() /// ~MemoryMappedFileData() => Dispose(); + + #region Memory Access Hints for OSes + +#if NET5_0_OR_GREATER + // POSIX Compatible + [DllImport("libc.so.6", EntryPoint = "madvise")] + private static extern unsafe int madvise(byte* addr, nuint length, int advice); + + // POSIX Compatible + [DllImport("libc.so", EntryPoint = "madvise")] + private static extern unsafe int madvise_android(byte* addr, nuint length, int advice); + + // OSX + [DllImport("libSystem", EntryPoint = "madvise")] + private static extern unsafe int madvise_libSystem(byte* addr, nuint length, int advice); + + // Windows-Like + [StructLayout(LayoutKind.Sequential)] + private struct MemoryRangeEntry + { + public nint VirtualAddress; + public nuint NumberOfBytes; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern unsafe bool PrefetchVirtualMemory( + IntPtr hProcess, + UIntPtr numberOfEntries, + MemoryRangeEntry* memoryRanges, + uint flags); +#endif + + #endregion } diff --git a/NexusMods.Archives.Nx/FileProviders/OutputFileProvider.cs b/NexusMods.Archives.Nx/FileProviders/OutputFileProvider.cs index a1676e8..83b9f4f 100644 --- a/NexusMods.Archives.Nx/FileProviders/OutputFileProvider.cs +++ b/NexusMods.Archives.Nx/FileProviders/OutputFileProvider.cs @@ -21,8 +21,7 @@ public sealed class OutputFileProvider : IOutputDataProvider /// public string FullPath { get; init; } - private readonly MemoryMappedFile? _mappedFile; - private readonly FileStream _fileStream; + private readonly MemoryMappedFile _mappedFile; private bool _isDisposed; /// @@ -43,20 +42,7 @@ public OutputFileProvider(string outputFolder, string relativePath, FileEntry en trycreate: try { -#if NET7_0_OR_GREATER - _fileStream = new FileStream(FullPath, new FileStreamOptions - { - PreallocationSize = (long)entry.DecompressedSize, - Access = FileAccess.ReadWrite, - Mode = FileMode.Create, - Share = FileShare.ReadWrite, - BufferSize = 0, - Options = FileOptions.SequentialScan - }); -#else - _fileStream = new FileStream(FullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite); - _fileStream.SetLength((long)entry.DecompressedSize); -#endif + _mappedFile = MemoryMappedFile.CreateFromFile(FullPath, FileMode.Create, null, (long)entry.DecompressedSize, MemoryMappedFileAccess.ReadWrite); } catch (DirectoryNotFoundException) { @@ -64,10 +50,6 @@ public OutputFileProvider(string outputFolder, string relativePath, FileEntry en Directory.CreateDirectory(Path.GetDirectoryName(FullPath)!); goto trycreate; } - - if (entry.DecompressedSize > 0) - _mappedFile = MemoryMappedFile.CreateFromFile(_fileStream, null, (long)entry.DecompressedSize, MemoryMappedFileAccess.ReadWrite, - HandleInheritability.None, true); } /// @@ -83,8 +65,7 @@ public void Dispose() return; _isDisposed = true; - _mappedFile?.Dispose(); - _fileStream.Dispose(); + _mappedFile.Dispose(); GC.SuppressFinalize(this); } } diff --git a/NexusMods.Archives.Nx/Headers/Native/NativeFileHeader.cs b/NexusMods.Archives.Nx/Headers/Native/NativeFileHeader.cs index dd067bc..2954aab 100644 --- a/NexusMods.Archives.Nx/Headers/Native/NativeFileHeader.cs +++ b/NexusMods.Archives.Nx/Headers/Native/NativeFileHeader.cs @@ -29,13 +29,7 @@ public struct NativeFileHeader : ICanConvertToLittleEndian public uint Magic; // Packed Values - private byte _versionAndBlockSize; - internal ushort _largeChunkSizeAndPageCount; - - /// - /// [u8] Reserved. - /// - public byte FeatureFlags; + internal uint _headerData; // Get/Setters @@ -50,42 +44,54 @@ public struct NativeFileHeader : ICanConvertToLittleEndian public void SetMagic() => Magic = ExpectedMagic.AsLittleEndian(); /// - /// [u4] Gets or sets the archive version. + /// [u3] Gets or sets the archive version. /// public byte Version { - get => (byte)((_versionAndBlockSize >> 4) & 0xF); // Extract the first 4 bits (upper bits) - set => _versionAndBlockSize = (byte)((_versionAndBlockSize & 0x0F) | (value << 4)); + get => (byte)((_headerData >> 29) & 0b111); + set => _headerData = (_headerData & 0b00011111_11111111_11111111_11111111) | ((uint)value << 29); } /// - /// [u4] Gets or sets the block size in its encoded raw value.
+ /// [u4] Gets or sets the block size in its encoded raw value.
/// (Blocks are encoded as (32768 << blockSize) - 1) /// public byte BlockSize { - get => (byte)(_versionAndBlockSize & 0xF); // Extract the next 4 bits (lower bits) - set => _versionAndBlockSize = (byte)((_versionAndBlockSize & 0xF0) | value); + get => (byte)((_headerData >> 25) & 0b1111); + set => _headerData = (_headerData & 0b11100001_11111111_11111111_11111111) | ((uint)value << 25); } /// - /// [u3] Gets or sets the large chunk size in its encoded raw value (0-7).
- /// (Blocks are encoded as (32768 << blockSize) - 1) + /// [u4] Gets or sets the large chunk size in its encoded raw value (0-15).
+ /// (Chunks are encoded as (16384 << chunkSize)) /// public byte ChunkSize { - get => (byte)((_largeChunkSizeAndPageCount >> 13) & 0b111); // Extract the first 3 bits (upper bits) - set => _largeChunkSizeAndPageCount = (ushort)((_largeChunkSizeAndPageCount & 0x1FFF) | (value << 13)); + get => (byte)((_headerData >> 21) & 0b1111); + set => _headerData = (_headerData & 0b11111110_00011111_11111111_11111111) | ((uint)value << 21); } /// - /// [u13] Gets or sets the number of compressed pages to store the entire ToC (incl. compressed stringpool).
- /// (Blocks are encoded as (32768 << blockSize) - 1) + /// [u13] Gets or sets the number of compressed pages used to store the entire ToC (incl. compressed stringpool). /// public ushort HeaderPageCount { - get => (ushort)(_largeChunkSizeAndPageCount & 0x1FFF); // Extract the next 13 bits (lower bits) - set => _largeChunkSizeAndPageCount = (ushort)((_largeChunkSizeAndPageCount & 0xE000) | value); + get => (ushort)((_headerData >> 8) & 0b11111_11111111); + set => _headerData = (_headerData & 0b11111110_11100000_00000000_11111111) | ((uint)value << 8); + } + + /// + /// [u8] Gets/sets the 'feature flags' for this structure. + /// A feature flag represents an extension to the format, such as storing time/date.
+ /// + /// + /// This is internal until any feature flags are actually implemented. + /// + internal ushort FeatureFlags + { + get => (ushort)(_headerData & 0b11111111); + set => _headerData = (_headerData & 0b11111110_11111111_11111111_00000000) | value; } // Note: Not adding a constructor since it could technically be skipped, if not explicitly init'ed by `new`. @@ -104,8 +110,8 @@ public int HeaderPageBytes /// public int BlockSizeBytes { - get => (32768 << BlockSize) - 1; - set => BlockSize = (byte)Math.Log((value + 1) >> 15, 2); + get => (4096 << BlockSize) - 1; + set => BlockSize = (byte)Math.Log((value + 1) >> 12, 2); } /// @@ -113,8 +119,8 @@ public int BlockSizeBytes /// public int ChunkSizeBytes { - get => 4194304 << ChunkSize; - set => ChunkSize = (byte)Math.Log(value >> 22, 2); + get => 32768 << ChunkSize; + set => ChunkSize = (byte)Math.Log(value >> 15, 2); } /// @@ -131,11 +137,12 @@ public int ChunkSizeBytes public static unsafe void Init(NativeFileHeader* header, ArchiveVersion version, int blockSizeBytes, int chunkSizeBytes, int headerPageCountBytes) { header->Magic = ExpectedMagic; + header->_headerData = 0; // Zero out before assigning bits via packing. + header->Version = (byte)version; header->BlockSizeBytes = blockSizeBytes; header->ChunkSizeBytes = chunkSizeBytes; header->HeaderPageBytes = headerPageCountBytes.RoundUp4096(); - header->FeatureFlags = 0; header->ReverseEndianIfNeeded(); } @@ -157,6 +164,6 @@ public void ReverseEndianIfNeeded() internal void ReverseEndian() { Magic = BinaryPrimitives.ReverseEndianness(Magic); - _largeChunkSizeAndPageCount = BinaryPrimitives.ReverseEndianness(_largeChunkSizeAndPageCount); + _headerData = BinaryPrimitives.ReverseEndianness(_headerData); } } diff --git a/NexusMods.Archives.Nx/Structs/PackerSettings.cs b/NexusMods.Archives.Nx/Structs/PackerSettings.cs index e562c81..8eda5cf 100644 --- a/NexusMods.Archives.Nx/Structs/PackerSettings.cs +++ b/NexusMods.Archives.Nx/Structs/PackerSettings.cs @@ -28,20 +28,21 @@ public class PackerSettings /// /// Maximum number of threads allowed. /// - public int MaxNumThreads { get; set; } = Environment.ProcessorCount; + public int MaxNumThreads { get; set; } = NxEnvironment.PhysicalCoreCount; + // HT/SMT hurts here, due to sub-par concurrency handling and fact core cache is split. /// /// Size of SOLID blocks. - /// Range is 32767 to 67108863 (64 MiB). + /// Range is 4095 to 67108863 (64 MiB). /// Must be smaller than . /// public int BlockSize { get; set; } = 1048575; /// /// Size of large file chunks. - /// Range is 4194304 (4 MiB) to 536870912 (512 MiB). + /// Range is 32768 (32K) to 1073741824 (1 GiB). /// - public int ChunkSize { get; set; } = 16777216; + public int ChunkSize { get; set; } = 1048576; /// /// Compression level to use for SOLID data. @@ -85,8 +86,8 @@ public void Sanitize() BlockSize = Polyfills.RoundUpToPowerOf2NoOverflow(BlockSize) - 1; ChunkSize = Polyfills.RoundUpToPowerOf2NoOverflow(ChunkSize); - BlockSize = Polyfills.Clamp(BlockSize, 32767, 67108863); - ChunkSize = Polyfills.Clamp(ChunkSize, 4194304, 536870912); + BlockSize = Polyfills.Clamp(BlockSize, 4095, 67108863); + ChunkSize = Polyfills.Clamp(ChunkSize, 32768, 1073741824); // 1GiB because we can't pool more with standard ArrayPool. if (ChunkSize <= BlockSize) ChunkSize = BlockSize + 1; diff --git a/NexusMods.Archives.Nx/Utilities/NxEnvironment.cs b/NexusMods.Archives.Nx/Utilities/NxEnvironment.cs new file mode 100644 index 0000000..25c12aa --- /dev/null +++ b/NexusMods.Archives.Nx/Utilities/NxEnvironment.cs @@ -0,0 +1,90 @@ +using System.Runtime.InteropServices; + +namespace NexusMods.Archives.Nx.Utilities; + +/// +/// Platform specific utilities. +/// +internal class NxEnvironment +{ + public static int PhysicalCoreCount { get; } = GetPhysicalCoreCount(); + + /// + /// Returns the physical core count on this machine (excluding hyperthreads). + /// + private static unsafe int GetPhysicalCoreCount() + { + #if NET5_0_OR_GREATER + var isLinux = OperatingSystem.IsLinux(); // intrinsic/constant + var isWindows = OperatingSystem.IsWindows(); // intrinsic/constant + #else + var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + #endif + + if (isLinux) + { + using var file = new StreamReader("/proc/cpuinfo"); + while (file.ReadLine() is { } line) + { + if (!line.Contains("cpu cores")) + continue; + + return int.Parse(line.Split(':')[1].Trim()); + } + + return Environment.ProcessorCount; + } + + if (isWindows) + { + uint returnLength = 0; + GetLogicalProcessorInformation(IntPtr.Zero, ref returnLength); + if (returnLength == 0) + return Environment.ProcessorCount; + + var info = stackalloc SYSTEM_LOGICAL_PROCESSOR_INFORMATION[(int)returnLength / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)]; + if (!GetLogicalProcessorInformation((IntPtr)info, ref returnLength)) + return Environment.ProcessorCount; + + var count = (int)returnLength / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + var physicalCoreCount = 0; + for (var x = 0; x < count; x++) + { + if (info[x].Relationship == LOGICAL_PROCESSOR_RELATIONSHIP.RelationProcessorCore) + physicalCoreCount++; + } + + return physicalCoreCount; + } + + return Environment.ProcessorCount; + } + + #region Windows Specific + [StructLayout(LayoutKind.Sequential)] + // ReSharper disable once InconsistentNaming + private struct SYSTEM_LOGICAL_PROCESSOR_INFORMATION + { + public IntPtr ProcessorMask; + public LOGICAL_PROCESSOR_RELATIONSHIP Relationship; + public ProcessorInformation ProcessorInformation; + } + + // ReSharper disable once InconsistentNaming + internal enum LOGICAL_PROCESSOR_RELATIONSHIP : int + { + RelationProcessorCore, + } + + [StructLayout(LayoutKind.Explicit, Size = 16)] + private struct ProcessorInformation + { + [FieldOffset(0)] + public byte Flags; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetLogicalProcessorInformation(nint buffer, ref uint returnLength); + #endregion +} diff --git a/NexusMods.Archives.Nx/Utilities/Polyfills.cs b/NexusMods.Archives.Nx/Utilities/Polyfills.cs index 4e992d1..ce4c253 100644 --- a/NexusMods.Archives.Nx/Utilities/Polyfills.cs +++ b/NexusMods.Archives.Nx/Utilities/Polyfills.cs @@ -48,6 +48,9 @@ public static int RoundUpToPowerOf2NoOverflow(int value) if (value == int.MaxValue) return value; + if (value > int.MaxValue >> 1) + return int.MaxValue; + #if NET7_0_OR_GREATER return (int)BitOperations.RoundUpToPowerOf2((uint)value); #else diff --git a/docs/Benchmarks.md b/docs/Benchmarks.md index 0dd0963..8feef4c 100644 --- a/docs/Benchmarks.md +++ b/docs/Benchmarks.md @@ -1,7 +1,5 @@ # Benchmarks -!!! tip "Coming Soon (TM)" - !!! info "Spoiler: This bottlenecks any NVMe 😀" !!! info @@ -15,6 +13,10 @@ - `OS`: Windows 11 22H2 (Build 22621) - `Storage`: Samsung 980 Pro 1TB (NVMe) [PCI-E 3.0 x4] +!!! note "Some of the compression benchmarks are very dated, as ZStd 1.5.5 has has huge improvements in handling incompressible data since." + + Nonetheless, hopefully they are useful for reference. + ## Common Data Sets !!! tip "These are the data sets which are used in multiple benchmarks below." @@ -30,8 +32,6 @@ Therefore, having a good compression ratio on this data set is important. ### Log Files -!!! tip "[Available Here](./R2Logs.nx)" - !!! info "This dataset consists of 189 [Reloaded-II Logs](https://reloaded-project.github.io/Reloaded-II/) from end users, across various games, with a total size of 12.4MiB" ### Lightly Compressed Files diff --git a/docs/R2Logs.nx b/docs/R2Logs.nx deleted file mode 100644 index 9ba6ccf..0000000 Binary files a/docs/R2Logs.nx and /dev/null differ diff --git a/docs/Specification/010Template.bt b/docs/Specification/010Template.bt index 5c797de..23ed83e 100644 --- a/docs/Specification/010Template.bt +++ b/docs/Specification/010Template.bt @@ -24,12 +24,12 @@ string HeaderSize(short r) string ChunkSize(short r) { - return "Chunk Size: " + Str("%d", 4194304 << r); + return "Chunk Size: " + Str("%d", 32768 << r); } string BlockSize(short r) { - return "Block Size: " + Str("%d", (32768 << r) - 1); + return "Block Size: " + Str("%d", (4096 << r) - 1); } string Version(short r) @@ -54,11 +54,11 @@ string Compression(short r) struct Header { byte Magic[4]; - byte Version : 4 ; - byte BlockSize : 4 ; - uint16 ChunkSize : 3 ; - uint16 PageCount : 13 ; - byte FeatureFlags; + uint32 Version : 3 ; + uint32 BlockSize : 4 ; + uint32 ChunkSize : 4 ; + uint32 PageCount : 13 ; + uint32 FeatureFlags : 8; }; struct TableOfContentsHeader diff --git a/docs/Specification/File-Header.md b/docs/Specification/File-Header.md index e20acfa..b8208f2 100644 --- a/docs/Specification/File-Header.md +++ b/docs/Specification/File-Header.md @@ -3,15 +3,15 @@ 8 bytes: - `u8[4]` Magic (`"NXUS"`) -- `u4` [Version/Variant](#versionvariant) +- `u3` [Version/Variant](#versionvariant) - `u4` [BlockSize](#block-size) -- `u3` [Large File Chunk Size](#large-file-chunk-size) -- `u13` [HeaderPageCount](#headerpagecount) +- `u4` [Large File Chunk Size](#large-file-chunk-size) +- `u13` [Header Page Count](#header-page-count) - `u8` [Feature Flags](#feature-flags) ## Version/Variant -Size: `4 bits` (0-15) +Size: `3 bits` (0-7) - `0`: - Most common variant covering 99.99% of cases. @@ -30,12 +30,15 @@ Limitation of 1 million files is inferred from [FileEntry -> FilePathIndex](./Ta Stored so the decompressor knows how big each block is. -Size: `4 bits`, but restricted (0-11) due to [Table of Contents File Entries](./Table-Of-Contents.md). -Parsed as `(32768 << blockSize) - 1`. +Size: `4 bits`. +Parsed as `(4096 << blockSize) - 1`. -i.e. BlockSize = 11 is `67108863` (i.e. `64MiB - 1` or `2^26 - 1`). +Limited to BlockSize = 14, `67108863` (i.e. `64MiB - 1` or `2^26 - 1`). +Due to [Table of Contents File Entries](./Table-Of-Contents.md). -We remove -1 from the value to avoid collisions with [Chunk Size](#large-file-chunk-size). +!!! note "A future version/flag may allow 128MiB SOLID blocks, however for now, we haven't found a need for it." + +!!! note "We remove -1 from the value to avoid collisions with [Chunk Size](#large-file-chunk-size)" ## Large File Chunk Size @@ -46,10 +49,10 @@ We remove -1 from the value to avoid collisions with [Chunk Size](#large-file-ch Stored so the decompressor knows how many chunks a file is split into; and how much memory to allocate. Also limits memory use on 4+GB archives. -Size: `3 bits`, (0-7). -Parsed as `4194304 << chunkSize`. +Size: `4 bits`, (0-15). +Parsed as `32768 << chunkSize`. -i.e. ChunkSize = 7 is `536870912` (512MiB, i.e. 2^29). +i.e. ChunkSize = 15 is `1073741824` (1GiB, i.e. 2^31). !!! note diff --git a/mkdocs.yml b/mkdocs.yml index 8390681..5a9e65f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,8 +36,8 @@ markdown_extensions: - pymdownx.tabbed: alternate_style: true - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg theme: name: material