Merge pull request #24 from Nexus-Mods/use-xxhash3

xxHash3 updates and various code cleanups, switch to .Net 8, etc.

.github/workflows/BuildAndTest.yml

@@ -16,24 +16,13 @@ jobs:
           - windows-latest
           - ubuntu-latest
           - macos-13 # macos-latest (macos-12) currently has issues with failing to run some AVX2 instructions.
+          - macos-latest
         targetFramework:
-          - net7.0
-          - net5.0
-          - netcoreapp3.1
+          - net8.0
         platform:
           - x64
           # I'd like to add arm64 here, but GitHub runners don't support it. The native port of Zstandard does support it though.
           # Modern Linux and OSX environments also don't support x86 backcompat; and we don't have runners for them either.
-        include:
-          - os: windows-latest
-            targetFramework: net7.0
-            platform: x86
-          - os: windows-latest
-            targetFramework: net5.0
-            platform: x86
-          - os: windows-latest
-            targetFramework: netcoreapp3.1
-            platform: x86
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -47,11 +36,6 @@ jobs:
         run: dotnet build -c Release -f ${{ matrix.targetFramework }} ./NexusMods.Archives.Nx.Tests/NexusMods.Archives.Nx.Tests.csproj
       - name: Run Tests
         run: dotnet test -c Release -f ${{ matrix.targetFramework }} ./NexusMods.Archives.Nx.Tests/NexusMods.Archives.Nx.Tests.csproj --collect:"XPlat Code Coverage;Format=cobertura;" --results-directory "Coverage" --no-build 
-      - name: "Upload Coverage"
-        uses: actions/upload-artifact@v3
-        with:
-          name: coverage-${{ matrix.os }}-${{ matrix.targetFramework }}
-          path: Coverage/*/coverage.cobertura.xml
   upload:
     needs: build
     runs-on: ubuntu-latest

NexusMods.Archives.Nx.Benchmarks/NexusMods.Archives.Nx.Benchmarks.csproj

@@ -2,7 +2,7 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>

NexusMods.Archives.Nx.Cli/NexusMods.Archives.Nx.Cli.csproj

@@ -2,7 +2,6 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFrameworks>net7.0;net8.0;net9.0</TargetFrameworks>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
 
@@ -24,6 +23,7 @@
         <HttpActivityPropagationSupport>false</HttpActivityPropagationSupport>
         <InvariantGlobalization>true</InvariantGlobalization>
         <MetadataUpdaterSupport>false</MetadataUpdaterSupport>
+        <TargetFramework>net8.0</TargetFramework>
     </PropertyGroup>
 
     <ItemGroup>

NexusMods.Archives.Nx.Tests/NexusMods.Archives.Nx.Tests.csproj

@@ -2,14 +2,14 @@
 
     <PropertyGroup>
         <!-- net462 disabled due to ZStd dependency not supporting it right now -->
-        <TargetFrameworks>net7.0;netcoreapp3.1;net5.0</TargetFrameworks>
         <Nullable>enable</Nullable>
 
         <IsPackable>false</IsPackable>
         <IsTestProject>true</IsTestProject>
         <LangVersion>preview</LangVersion>
         <XunitStartupAssembly>NexusMods.Archives.Nx.Tests</XunitStartupAssembly>
         <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
+        <TargetFramework>net8.0</TargetFramework>
     </PropertyGroup>
 
     <ItemGroup>

NexusMods.Archives.Nx.Tests/Tests/Packing/ChunkedFileDeduplicationTests.cs

@@ -1,3 +1,4 @@
+using System.IO.Hashing;
 using FluentAssertions;
 using NexusMods.Archives.Nx.FileProviders;
 using NexusMods.Archives.Nx.Packing;
@@ -11,6 +12,8 @@ public void DeduplicateChunkedBlocks_IdenticalFiles_HaveSameStartingBlockIndex()
     {
         // Arrange
         var fileContent = PackingTests.MakeDummyFile(2 * 1024 * 1024); // 2 MB file
+        var fileHash = XxHash3.HashToUInt64(fileContent);
+
 
         var packerBuilder = new NxPackerBuilder();
         packerBuilder.WithChunkSize(1024 * 1024); // 1 MB chunks
@@ -34,6 +37,7 @@ public void DeduplicateChunkedBlocks_IdenticalFiles_HaveSameStartingBlockIndex()
         fileEntries[1].Entry.FirstBlockIndex.Should().Be(fileEntries[0].Entry.FirstBlockIndex);
         fileEntries[0].Entry.DecompressedSize.Should().Be(2 * 1024 * 1024);
         fileEntries[1].Entry.DecompressedSize.Should().Be(2 * 1024 * 1024);
+        fileEntries[0].Entry.Hash.Should().Be(fileHash);
 
         // Extract and verify content
         unpackerBuilder.AddFilesWithArrayOutput(fileEntries, out var extractedFiles);

NexusMods.Archives.Nx.Tests/Tests/Packing/PackingTests.cs

@@ -1,3 +1,4 @@
+using System.IO.Hashing;
 using AutoFixture;
 using AutoFixture.Xunit2;
 using FluentAssertions;
@@ -254,7 +255,7 @@ internal static PackerFile[] GetRandomDummyFiles(IFixture fixture, int numFiles,
 
     internal static byte[] MakeDummyFile(int length)
     {
-        var result = Polyfills.AllocateUninitializedArray<byte>(length);
+        var result = GC.AllocateUninitializedArray<byte>(length, false);
         for (var x = 0; x < length; x++)
             result[x] = (byte)(x % 255);
 
@@ -300,5 +301,5 @@ public static class SpanExtensions
     /// </summary>
     /// <param name="data">The data to hash.</param>
     /// <returns>Hash for the given data.</returns>
-    public static ulong XxHash64(this ReadOnlySpan<byte> data) => XxHash64Algorithm.HashBytes(data);
+    public static ulong XxHash64(this ReadOnlySpan<byte> data) => XxHash3.HashToUInt64(data);
 }

NexusMods.Archives.Nx.Tests/Tests/Pooling/StringPoolTests.cs

@@ -22,7 +22,7 @@ public void CreateAndVerifyPool_WithMultiCharOnly()
         };
 
         using var createPool = StringPool.Pack(items.AsSpan());
-        var strings = Polyfills.ToHashSet(StringPool.Unpack(createPool.Span));
+        var strings = StringPool.Unpack(createPool.Span).ToHashSet();
 
         foreach (var item in items)
             strings.Should().Contain(item.RelativePath);
@@ -40,7 +40,7 @@ public void CreateAndVerifyPool()
         };
 
         using var createPool = StringPool.Pack(items.AsSpan());
-        var strings = Polyfills.ToHashSet(StringPool.Unpack(createPool.Span));
+        var strings = StringPool.Unpack(createPool.Span).ToHashSet();
 
         foreach (var item in items)
             strings.Should().Contain(item.RelativePath);

NexusMods.Archives.Nx.Tests/Tests/Utilities/FindOffsetsTests.cs

@@ -1,3 +1,5 @@
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
 using FluentAssertions;
 using NexusMods.Archives.Nx.Utilities;
 
@@ -19,7 +21,6 @@ public enum FindOffsetMethod
     ///     Determines if offsets for StringPool can be correctly found using various optimized approaches.
     /// </summary>
     [Theory]
-#if NETCOREAPP3_1_OR_GREATER
     [InlineData(65, new[] { 1, 4, 8 }, FindOffsetMethod.Avx2)] // above register
     [InlineData(64, new[] { 1, 4, 8 }, FindOffsetMethod.Avx2)] // on register
     [InlineData(64, new[] { 63 }, FindOffsetMethod.Avx2)] // last element
@@ -30,12 +31,18 @@ public enum FindOffsetMethod
     [InlineData(32, new[] { 31 }, FindOffsetMethod.Sse2)] // last element
     [InlineData(32, new[] { 0, 31 }, FindOffsetMethod.Sse2)] // first element
     [InlineData(31, new[] { 1, 4, 8 }, FindOffsetMethod.Sse2)] // below register
-#endif
     [InlineData(8, new[] { 1, 4, 7 }, FindOffsetMethod.Fallback)] // above register
     [InlineData(8, new[] { 7 }, FindOffsetMethod.Fallback)] // last element
     [InlineData(8, new[] { 0, 7 }, FindOffsetMethod.Fallback)] // first element
     public void FindOffset(int numBytes, int[] expectedOffsets, FindOffsetMethod findOffsetMethod)
     {
+        switch (findOffsetMethod)
+        {
+            case FindOffsetMethod.Sse2 when !Sse2.IsSupported:
+            case FindOffsetMethod.Avx2 when !Avx2.IsSupported:
+                return;
+        }
+
         var bytes = GenerateRandomBytes(numBytes, expectedOffsets);
         var offsets = FindOffsetWithMethod(bytes, findOffsetMethod);
 
@@ -51,14 +58,12 @@ private unsafe List<int> FindOffsetWithMethod(byte[] data, FindOffsetMethod meth
             var results = new List<int>();
             switch (method)
             {
-#if NETCOREAPP3_1_OR_GREATER
                 case FindOffsetMethod.Avx2:
                     SpanExtensions.FindAllOffsetsOfByteAvx2(bytePtr, data.Length, 0, results);
                     break;
                 case FindOffsetMethod.Sse2:
                     SpanExtensions.FindAllOffsetsOfByteSse2(bytePtr, data.Length, 0, results);
                     break;
-#endif
                 case FindOffsetMethod.Fallback:
                     SpanExtensions.FindAllOffsetsOfByteFallback(bytePtr, data.Length, 0, 0, results);
                     break;

NexusMods.Archives.Nx/FileProviders/FileData/MemoryMappedFileData.cs

@@ -1,10 +1,8 @@
 using System.IO.MemoryMappedFiles;
 using JetBrains.Annotations;
 using NexusMods.Archives.Nx.Interfaces;
-#if NET5_0_OR_GREATER
 using System.Diagnostics;
 using System.Runtime.InteropServices;
-#endif
 
 // ReSharper disable IntroduceOptionalParameters.Global
 
@@ -85,7 +83,6 @@ private unsafe void InitFromMmf(ulong start, ulong length, bool isReadOnly = fal
 
         // Provide some OS specific hints
         // POSIX compliant
-#if NET5_0_OR_GREATER
         if (OperatingSystem.IsLinux())
         {
             // Also tried MADV_SEQUENTIAL, but didn't yield a benefit (on Linux) strangely.
@@ -107,7 +104,6 @@ private unsafe void InitFromMmf(ulong start, ulong length, bool isReadOnly = fal
             // ReSharper disable once RedundantCast
             PrefetchVirtualMemory(Process.GetCurrentProcess().Handle, (nuint)1, entries, 0);
         }
-#endif
     }
 
     private unsafe void InitEmpty()
@@ -123,7 +119,6 @@ private unsafe void InitEmpty()
 
     #region Memory Access Hints for OSes
 
-#if NET5_0_OR_GREATER
     // POSIX Compatible
     [DllImport("libc.so.6", EntryPoint = "madvise")]
     private static extern unsafe int madvise(byte* addr, nuint length, int advice);
@@ -150,7 +145,5 @@ private static extern unsafe bool PrefetchVirtualMemory(
         UIntPtr numberOfEntries,
         MemoryRangeEntry* memoryRanges,
         uint flags);
-#endif
-
     #endregion
 }

NexusMods.Archives.Nx/FileProviders/FileData/MemoryMappedOutputFileData.cs

@@ -84,7 +84,7 @@ public void Dispose()
         // On Windows, flushing the view leads to somewhat asynchronous write in any case.
         // But .NET Runtime does it synchronously on Linux.
         // This actually brings our platforms closer to parity.
-        if (Polyfills.IsWindows())
+        if (OperatingSystem.IsWindows())
         {
             // On Windows flushing acts as a hint of 'start writing asynchronously now'.
             // so it's desirable to keep the full flush.

NexusMods.Archives.Nx/FileProviders/FromExistingNxBlock.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics.CodeAnalysis;
+using System.Runtime.InteropServices;
 using NexusMods.Archives.Nx.Enums;
 using NexusMods.Archives.Nx.FileProviders.FileData;
 using NexusMods.Archives.Nx.Headers.Managed;
@@ -209,7 +210,7 @@ public void ConsiderFile(FileEntry entry)
             if (_data != null)
                 return _data;
 
-            var data = AllocNativeMemory((nuint)_numBytesToDecompress);
+            var data = NativeMemory.Alloc((nuint)_numBytesToDecompress);
             using var rawBlockData = _sourceNxDataProvider.GetFileData(_blockOffset, _compressedBlockLength);
             Compression.Decompress(_compression, rawBlockData.Data, (int)rawBlockData.DataLength, (byte*)data, (int)_numBytesToDecompress);
             _data = data;
@@ -237,7 +238,7 @@ private void ReleaseUnmanagedResources()
             throw new InvalidOperationException("Memory is being released while there are still references to it.");
 #endif
 
-        FreeNativeMemory(_data);
+        NativeMemory.Free(_data);
         _data = null;
     }
 }

NexusMods.Archives.Nx/FileProviders/FromStreamProvider.cs

@@ -45,7 +45,8 @@ public IFileData GetFileData(ulong start, ulong length)
             var pooledData = new ArrayRental((int)length);
 
             // In case of old framework, or Stream which doesn't implement span overload, don't use span here.
-            var numRead = Polyfills.ReadAtLeast(Stream, pooledData.Array, (int)length);
+            int minimumBytes = (int)length;
+            var numRead = Stream.ReadAtLeast(pooledData.Array.AsSpan(0, minimumBytes), minimumBytes, true);
             return new RentedArrayFileData(new ArrayRentalSlice(pooledData, numRead));
         }
     }

NexusMods.Archives.Nx/FileProviders/OutputArrayProvider.cs

@@ -33,7 +33,7 @@ public OutputArrayProvider(string relativePath, FileEntry entry)
 
         RelativePath = relativePath;
         Entry = entry;
-        Data = Polyfills.AllocateUninitializedArray<byte>((int)Entry.DecompressedSize);
+        Data = GC.AllocateUninitializedArray<byte>((int)Entry.DecompressedSize, false);
     }
 
     /// <inheritdoc />

NexusMods.Archives.Nx/Headers/Managed/ParsedHeader.cs

@@ -26,7 +26,7 @@ public void Init()
     {
         var currentOffset = (ulong)Header.HeaderPageBytes;
         var numBlocks = Blocks.Length;
-        BlockOffsets = Polyfills.AllocateUninitializedArray<ulong>(numBlocks);
+        BlockOffsets = GC.AllocateUninitializedArray<ulong>(numBlocks, false);
         if (numBlocks <= 0)
             return;
 

NexusMods.Archives.Nx/Headers/Managed/TableOfContents.cs

@@ -62,9 +62,9 @@ public class TableOfContents : IEquatable<TableOfContents>
         var reader = new LittleEndianReader(dataPtr);
         var tocHeader = NativeTocHeader.FromRaw(reader.ReadULong());
 
-        var entries = Polyfills.AllocateUninitializedArray<FileEntry>(tocHeader.FileCount);
-        var blocks = Polyfills.AllocateUninitializedArray<BlockSize>(tocHeader.BlockCount);
-        var blockCompressions = Polyfills.AllocateUninitializedArray<CompressionPreference>(tocHeader.BlockCount);
+        var entries = GC.AllocateUninitializedArray<FileEntry>(tocHeader.FileCount, false);
+        var blocks = GC.AllocateUninitializedArray<BlockSize>(tocHeader.BlockCount, false);
+        var blockCompressions = GC.AllocateUninitializedArray<CompressionPreference>(tocHeader.BlockCount, false);
 
         // Unavoidable bounds check in DangerousGetReferenceAt on older frameworks, when 0 blocks.
         // So despite the code handling 0 blocks properly, code still throws there; so we have to add this

NexusMods.Archives.Nx/Headers/StringPool.cs

@@ -117,7 +117,7 @@ public static unsafe string[] Unpack(byte* poolPtr, int compressedDataSize, int
         using var decompressed = Compression.DecompressZStd(poolPtr, compressedDataSize);
         var decompressedSpan = decompressed.Span;
         var offsets = decompressedSpan.Length > 0 ? decompressedSpan.FindAllOffsetsOfByte(0, fileCountHint) : new List<int>();
-        var items = Polyfills.AllocateUninitializedArray<string>(offsets.Count);
+        var items = GC.AllocateUninitializedArray<string>(offsets.Count, false);
 
         var currentOffset = 0;
         fixed (byte* spanPtr = decompressedSpan)
@@ -168,13 +168,6 @@ internal static class StringPoolExtensions
 {
     public static void SortLexicographically<T>(this Span<T> items) where T : IHasRelativePath
     {
-#if NET5_0_OR_GREATER
         items.Sort((a, b) => string.Compare(a.RelativePath, b.RelativePath, StringComparison.Ordinal));
-#else
-        // No way to sort a span on older frameworks; this is going to suck, but I guess we have to.
-        var copy = items.ToArray();
-        Array.Sort(copy, (a, b) => string.Compare(a.RelativePath, b.RelativePath, StringComparison.Ordinal));
-        copy.CopyTo(items);
-#endif
     }
 }

NexusMods.Archives.Nx/Headers/TableOfContentsBuilder.cs

@@ -103,7 +103,7 @@ private void Init<TWithRelativePath>(List<IBlock<T>> blocks, Span<TWithRelativeP
             Version = TableOfContentsVersion.V1;
 
         // Populate file name dictionary and names.
-        var poolPaths = Polyfills.AllocateUninitializedArray<string>(relativeFilePaths.Length);
+        var poolPaths = GC.AllocateUninitializedArray<string>(relativeFilePaths.Length, false);
         FileNameToIndexDictionary = new Dictionary<string, int>(relativeFilePaths.Length);
         for (var x = 0; x < relativeFilePaths.Length; x++)
         {
@@ -117,9 +117,9 @@ private void Init<TWithRelativePath>(List<IBlock<T>> blocks, Span<TWithRelativeP
         Toc = new TableOfContents
         {
             PoolSize = poolSize,
-            BlockCompressions = Polyfills.AllocateUninitializedArray<CompressionPreference>(blocks.Count),
-            Blocks = Polyfills.AllocateUninitializedArray<BlockSize>(blocks.Count),
-            Entries = Polyfills.AllocateUninitializedArray<FileEntry>(relativeFilePaths.Length),
+            BlockCompressions = GC.AllocateUninitializedArray<CompressionPreference>(blocks.Count, false),
+            Blocks = GC.AllocateUninitializedArray<BlockSize>(blocks.Count, false),
+            Entries = GC.AllocateUninitializedArray<FileEntry>(relativeFilePaths.Length, false),
             Pool = poolPaths
         };
     }