Adding CI, fixing large arrays (#33)

Improvements: - Increased CHUNK size as per miniz instructions - Added tests for very large arrays in NPZ files - Added some CI tests to catch issues across platforms - Removed the internal IO streams in favor of just using stringstream - NPZs can now be read from and written to memory Bugfixes: - Fixed an issue where very large arrays in NPZ files would throw an error - Fixed a bug with mac builds due to deprecated APIs Signed-off-by: Matthew Johnson <[email protected]>
matajoh · Nov 2, 2024 · 57fcaea · 57fcaea
1 parent 9329797
commit 57fcaea
Show file tree

Hide file tree

Showing 36 changed files with 2,448 additions and 2,813 deletions.
diff --git a/.clang_format b/.clang_format
@@ -0,0 +1,108 @@
+---
+Language: Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: DontAlign
+AlignOperands: false
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterClass: true
+  AfterControlStatement: true
+  AfterEnum: true
+  AfterFunction: true
+  AfterNamespace: true
+  AfterObjCDeclaration: true
+  AfterStruct: true
+  AfterUnion: true
+  AfterExternBlock: true
+  BeforeCatch: true
+  BeforeElse: true
+  IndentBraces: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Custom
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: false
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: AfterColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit: 80
+CommentPragmas: "^ IWYU pragma:"
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 2
+ContinuationIndentWidth: 2
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat: false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: false
+ForEachMacros:
+  - FOREACH
+IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex: '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority: 2
+  - Regex: '^(<|"(gtest|gmock|isl|json)/)'
+    Priority: 3
+  - Regex: ".*"
+    Priority: 1
+IncludeIsMainRegex: "(Test)?$"
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth: 2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ""
+MacroBlockEnd: ""
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: All
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 600
+PointerAlignment: Left
+ReflowComments: true
+SortIncludes: true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard: Cpp11
+TabWidth: 2
+UseTab: Never
+---
diff --git a/.github/workflows/prgate.yaml b/.github/workflows/prgate.yaml
@@ -0,0 +1,156 @@
+name: PR Gate
+
+on:
+  pull_request:
+    branches: ["main"]
+  workflow_dispatch:
+
+jobs:
+  cpp-format:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        path: 
+          - check: src
+            exclude: (/miniz/)
+          - check: test
+            exclude: ''
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run clang-format style check
+        uses: jidicula/[email protected]
+        with:
+          clang-format-version: '18'
+          check-path: ${{matrix.path['check']}}
+          exclude-regex: ${{matrix.path['exclude']}}
+
+  linux:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get dependencies
+        run: |
+          sudo apt-get install ninja-build
+
+      - name: CMake config
+        run: cmake -B ${{github.workspace}}/build --preset release-clang
+
+      - name: CMake build
+        working-directory: ${{github.workspace}}/build
+        run: ninja
+
+      - name: Use Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Generate large NPZ files
+        working-directory: ${{github.workspace}}/
+        run: |
+          pip install numpy
+          python ${{github.workspace}}/test/generate_large_test.py
+
+      - name: CMake test
+        working-directory: ${{github.workspace}}/build
+        run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test
+
+  linux-asan:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get dependencies
+        run: |
+          sudo apt-get install ninja-build
+
+      - name: CMake config
+        run: cmake -B ${{github.workspace}}/build --preset release-clang -DLIBNPY_SANITIZE=address
+
+      - name: CMake build
+        working-directory: ${{github.workspace}}/build
+        run: ninja
+
+      - name: Use Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Generate large NPZ files
+        working-directory: ${{github.workspace}}/
+        run: |
+          pip install numpy
+          python ${{github.workspace}}/test/generate_large_test.py
+
+      - name: CMake test
+        working-directory: ${{github.workspace}}/build
+        run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test
+
+  windows:
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: CMake config
+        run: |
+          cmake -B ${{github.workspace}}/build --preset release
+      
+      - name: CMake build
+        working-directory: ${{github.workspace}}/build
+        run: cmake --build . --config Release
+
+      - name: Use Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Generate large NPZ files
+        working-directory: ${{github.workspace}}/
+        run: |
+          pip install numpy
+          python ${{github.workspace}}/test/generate_large_test.py
+
+      - name: CMake test
+        working-directory: ${{github.workspace}}/build
+        run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test
+
+  macos:
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get dependencies
+        run: |
+          brew update && brew install ninja
+
+      - name: CMake config
+        run: cmake -B ${{github.workspace}}/build --preset release-clang
+
+      - name: CMake build
+        working-directory: ${{github.workspace}}/build
+        run: ninja
+
+      - name: Use Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Generate large NPZ files
+        working-directory: ${{github.workspace}}/
+        run: |
+          pip install numpy
+          python ${{github.workspace}}/test/generate_large_test.py
+
+      - name: CMake test
+        working-directory: ${{github.workspace}}/build
+        run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test
diff --git a/.gitignore b/.gitignore
@@ -33,4 +33,7 @@
 
 # Misc
 .vscode
-build*
+build*
+assets/test/*_large*.npz
+.cache
+.env
diff --git a/.travis.yml b/.travis.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # Changelog
 
+## [2024-11-01 - Version 1.5.3](https://github.com/matajoh/libnpy/releases/tag/v1.5.3)
+
+Improvements:
+- Increased CHUNK size as per miniz instructions
+- Added tests for very large arrays in NPZ files
+- Added some CI tests to catch issues across platforms
+- Removed the internal IO streams in favor of just using stringstream
+- NPZs can now be read from and written to memory
+
+Bugfixes:
+- Fixed an issue where very large arrays in NPZ files would throw an error
+- Fixed a bug with mac builds due to deprecated APIs
+
 ## [2021-10-05 - Version 1.5.2](https://github.com/matajoh/libnpy/releases/tag/v1.5.2)
 
 Removing `using namespace std` to simplify library use