From 57fcaeaf957af01396b0d468e480b2eb79c8a3f0 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Sat, 2 Nov 2024 02:34:30 +0000 Subject: [PATCH] Adding CI, fixing large arrays (#33) Improvements: - Increased CHUNK size as per miniz instructions - Added tests for very large arrays in NPZ files - Added some CI tests to catch issues across platforms - Removed the internal IO streams in favor of just using stringstream - NPZs can now be read from and written to memory Bugfixes: - Fixed an issue where very large arrays in NPZ files would throw an error - Fixed a bug with mac builds due to deprecated APIs Signed-off-by: Matthew Johnson --- .clang_format | 108 +++++ .github/workflows/prgate.yaml | 156 +++++++ .gitignore | 5 +- .travis.yml | 22 - CHANGELOG.md | 13 + CMakeLists.txt | 80 ++-- CMakePresets.json | 49 ++ CSharpWrapper/NumpyIONative.i | 4 +- README.md | 104 +---- RELEASE_NOTES | 9 +- include/npy/core.h | 128 +----- include/npy/npy.h | 485 +++++++++----------- include/npy/npz.h | 350 +++++++------- include/npy/tensor.h | 661 ++++++++++++-------------- src/CMakeLists.txt | 8 +- src/dtype.cpp | 68 +-- src/memstream.cpp | 238 ---------- src/npy.cpp | 226 ++++----- src/npz.cpp | 841 ++++++++++++++++------------------ src/tensor.cpp | 69 +-- src/zip.cpp | 289 ++++++------ src/zip.h | 12 +- test/crc32.cpp | 31 +- test/exceptions.cpp | 167 ++++--- test/generate_large_test.py | 17 + test/libnpy_tests.cpp | 142 +++--- test/libnpy_tests.h | 366 +++++++-------- test/memstream.cpp | 70 ++- test/npy_peek.cpp | 48 +- test/npy_read.cpp | 40 +- test/npy_read.h | 42 +- test/npy_write.cpp | 104 ++--- test/npz_peek.cpp | 66 +-- test/npz_read.cpp | 89 +++- test/npz_write.cpp | 86 ++-- test/tensor.cpp | 68 ++- 36 files changed, 2448 insertions(+), 2813 deletions(-) create mode 100644 .clang_format create mode 100644 .github/workflows/prgate.yaml delete mode 100644 .travis.yml create mode 100644 CMakePresets.json delete mode 100644 src/memstream.cpp create mode 100644 test/generate_large_test.py diff --git a/.clang_format b/.clang_format new file mode 100644 index 0000000..ed1ba13 --- /dev/null +++ b/.clang_format @@ -0,0 +1,108 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: -2 +AlignAfterOpenBracket: AlwaysBreak +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: DontAlign +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: true + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: AfterColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: "^ IWYU pragma:" +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 2 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: false +ForEachMacros: + - FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: ".*" + Priority: 1 +IncludeIsMainRegex: "(Test)?$" +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: "" +MacroBlockEnd: "" +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: All +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 600 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 2 +UseTab: Never +--- \ No newline at end of file diff --git a/.github/workflows/prgate.yaml b/.github/workflows/prgate.yaml new file mode 100644 index 0000000..9b24ede --- /dev/null +++ b/.github/workflows/prgate.yaml @@ -0,0 +1,156 @@ +name: PR Gate + +on: + pull_request: + branches: ["main"] + workflow_dispatch: + +jobs: + cpp-format: + runs-on: ubuntu-latest + strategy: + matrix: + path: + - check: src + exclude: (/miniz/) + - check: test + exclude: '' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run clang-format style check + uses: jidicula/clang-format-action@v4.8.0 + with: + clang-format-version: '18' + check-path: ${{matrix.path['check']}} + exclude-regex: ${{matrix.path['exclude']}} + + linux: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get dependencies + run: | + sudo apt-get install ninja-build + + - name: CMake config + run: cmake -B ${{github.workspace}}/build --preset release-clang + + - name: CMake build + working-directory: ${{github.workspace}}/build + run: ninja + + - name: Use Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Generate large NPZ files + working-directory: ${{github.workspace}}/ + run: | + pip install numpy + python ${{github.workspace}}/test/generate_large_test.py + + - name: CMake test + working-directory: ${{github.workspace}}/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + linux-asan: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get dependencies + run: | + sudo apt-get install ninja-build + + - name: CMake config + run: cmake -B ${{github.workspace}}/build --preset release-clang -DLIBNPY_SANITIZE=address + + - name: CMake build + working-directory: ${{github.workspace}}/build + run: ninja + + - name: Use Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Generate large NPZ files + working-directory: ${{github.workspace}}/ + run: | + pip install numpy + python ${{github.workspace}}/test/generate_large_test.py + + - name: CMake test + working-directory: ${{github.workspace}}/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + windows: + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CMake config + run: | + cmake -B ${{github.workspace}}/build --preset release + + - name: CMake build + working-directory: ${{github.workspace}}/build + run: cmake --build . --config Release + + - name: Use Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Generate large NPZ files + working-directory: ${{github.workspace}}/ + run: | + pip install numpy + python ${{github.workspace}}/test/generate_large_test.py + + - name: CMake test + working-directory: ${{github.workspace}}/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + macos: + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get dependencies + run: | + brew update && brew install ninja + + - name: CMake config + run: cmake -B ${{github.workspace}}/build --preset release-clang + + - name: CMake build + working-directory: ${{github.workspace}}/build + run: ninja + + - name: Use Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Generate large NPZ files + working-directory: ${{github.workspace}}/ + run: | + pip install numpy + python ${{github.workspace}}/test/generate_large_test.py + + - name: CMake test + working-directory: ${{github.workspace}}/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test diff --git a/.gitignore b/.gitignore index 6170147..24176df 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,7 @@ # Misc .vscode -build* \ No newline at end of file +build* +assets/test/*_large*.npz +.cache +.env \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index df8210b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -dist: bionic -language: cpp - -addons: - apt: - update: true - -before_install: - - sudo apt-get remove cmake - - sudo apt-get update - - sudo apt-get install apt-transport-https ca-certificates gnupg software-properties-common wget - - wget -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add - - - sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' - - sudo apt-get update - - sudo apt-get install -y cmake swig doxygen - -script: - - mkdir build - - cd build - - /usr/bin/cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTS=1 .. - - /usr/bin/cmake --build . - - /usr/bin/ctest -C RelWithDebInfo diff --git a/CHANGELOG.md b/CHANGELOG.md index 10647b9..91b6bac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [2024-11-01 - Version 1.5.3](https://github.com/matajoh/libnpy/releases/tag/v1.5.3) + +Improvements: +- Increased CHUNK size as per miniz instructions +- Added tests for very large arrays in NPZ files +- Added some CI tests to catch issues across platforms +- Removed the internal IO streams in favor of just using stringstream +- NPZs can now be read from and written to memory + +Bugfixes: +- Fixed an issue where very large arrays in NPZ files would throw an error +- Fixed a bug with mac builds due to deprecated APIs + ## [2021-10-05 - Version 1.5.2](https://github.com/matajoh/libnpy/releases/tag/v1.5.2) Removing `using namespace std` to simplify library use diff --git a/CMakeLists.txt b/CMakeLists.txt index e010a08..41ce88d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,50 +20,14 @@ project( libnpy VERSION ${LIBNPY_VERSION} LANGUAGES CXX) # -------------------- Options -------------------------------- -option( BUILD_TESTS "Specifies whether to build the tests" OFF ) -option( BUILD_SAMPLES "Specifies whether to build the samples" OFF ) -option( BUILD_DOCUMENTATION "Specifies whether to build the documentation for the API and XML" OFF ) -option( INCLUDE_CSHARP "Specifies whether to build libnpy with C# bindings" OFF ) - -# ------------------- Detect the system ----------------------- - -if( APPLE ) - set( SYSTEM_NAME "Mac" ) - set( LIBNPY_NUGET_LIB "libnpy.a") - set( SYSTEM_TOOLKIT "${CMAKE_CXX_COMPILER_VERSION}") -elseif( UNIX ) - set( SYSTEM_NAME "Linux" ) - set( LIBNPY_NUGET_LIB "libnpy.a") - execute_process(COMMAND lsb_release -is OUTPUT_VARIABLE LSB_ID) - execute_process(COMMAND lsb_release -rs OUTPUT_VARIABLE LSB_RELEASE) - if( LSB_ID AND LSB_RELEASE ) - string(STRIP "${LSB_ID}" LSB_ID) - string(STRIP "${LSB_RELEASE}" LSB_RELEASE) - set( SYSTEM_NAME "${SYSTEM_NAME}-${LSB_ID}-${LSB_RELEASE}") - elseif( EXISTS "/etc/debian_version") - file( READ /etc/debian_version DEBIAN_VERSION ) - set( SYSTEM_NAME "${SYSTEM_NAME}-${DEBIAN_VERSION}") - elseif( EXISTS "/etc/os-release") - execute_process(COMMAND "sed" "-ne" "s/^ID=\"\\?\\([a-z]\\+\\)\"\\?$/\\1/p" "/etc/os-release" OUTPUT_VARIABLE OS_RELEASE_ID OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND "sed" "-ne" "s/^VERSION_ID=\"\\?\\([0-9\\.]\\+\\)\"\\?$/\\1/p" "/etc/os-release" OUTPUT_VARIABLE OS_RELEASE_VERSION_ID OUTPUT_STRIP_TRAILING_WHITESPACE) - set( SYSTEM_NAME "${SYSTEM_NAME}-${OS_RELEASE_ID}-${OS_RELEASE_VERSION_ID}") - elseif( EXISTS "/etc/redhat-release") - set( SYSTEM_NAME "${SYSTEM_NAME}-Redhat") - endif() - set( SYSTEM_TOOLKIT "gcc${CMAKE_CXX_COMPILER_VERSION}") -elseif( WIN32 ) - set( LIBNPY_NUGET_LIB "npy.lib") - set( SYSTEM_NAME "Windows" ) - set( SYSTEM_TOOLKIT "v${MSVC_TOOLSET_VERSION}") -endif() +option( LIBNPY_BUILD_TESTS "Specifies whether to build the tests" OFF ) +option( LIBNPY_BUILD_SAMPLES "Specifies whether to build the samples" OFF ) +option( LIBNPY_BUILD_DOCUMENTATION "Specifies whether to build the documentation for the API and XML" OFF ) +option( LIBNPY_INCLUDE_CSHARP "Specifies whether to build libnpy with C# bindings" OFF ) +set( LIBNPY_SANITIZE "" CACHE STRING "Argument to pass to sanitize (disabled by default)") -if( CMAKE_SIZEOF_VOID_P EQUAL 8 ) - set( SYSTEM_NAME "${SYSTEM_NAME}-64bit" ) - set( SYSTEM_BITS "x64" ) -else() - set( SYSTEM_NAME "${SYSTEM_NAME}-32bit" ) - set( SYSTEM_BITS "x86" ) -endif() +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # -------------------- Find packages -------------------------- @@ -91,19 +55,41 @@ if( BUILD_DOCUMENTATION ) find_package( Doxygen REQUIRED ) endif() +find_program(CLANG_FORMAT NAMES clang-format-10 clang-format-14 clang-format-18 ) + +string(COMPARE EQUAL ${CLANG_FORMAT} "CLANG_FORMAT-NOTFOUND" CLANG_FORMAT_NOT_FOUND) +if(CLANG_FORMAT_NOT_FOUND) + message("libnpy_format target not defined: no clang-format tool found") +else() + file(GLOB ALL_SOURCE_FILES CONFIGURE_DEPENDS + src/*.cpp + src/*.h + include/libnpy/*.h + test/*.cpp + test/*.h + examples/*.cpp + ) + + add_custom_target(libnpy_format + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${CLANG_FORMAT} + -i + ${ALL_SOURCE_FILES}) +endif() + # -------------------- Walk the subdirectories -------------------- add_subdirectory( src ) -if( BUILD_SAMPLES ) +if( LIBNPY_BUILD_SAMPLES ) add_subdirectory( samples ) endif() -if( BUILD_DOCUMENTATION ) +if( LIBNPY_BUILD_DOCUMENTATION ) add_subdirectory( doc ) endif() -if( INCLUDE_CSHARP ) +if( LIBNPY_INCLUDE_CSHARP ) add_subdirectory( CSharpWrapper ) endif() @@ -117,7 +103,7 @@ target_include_directories(npy # -------------------- Testing ------------------------------------ -if( BUILD_TESTS ) +if( LIBNPY_BUILD_TESTS ) if( MSVC ) set( LIBNPY_CSHARP_DIR ${CMAKE_BINARY_DIR}/CSharpWrapper/$ ) endif() diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..31efae6 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,49 @@ +{ + "version": 3, + "configurePresets": [ + { + "name": "debug-clang", + "displayName": "Debug Build using clang", + "description": "Sets up a debug build that uses Clang++", + "generator": "Ninja", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", + "CMAKE_CXX_COMPILER": "clang++", + "LIBNPY_BUILD_TESTS": "ON" + } + }, + { + "name": "debug", + "displayName": "Debug Build", + "description": "Sets up a debug build that uses the default compiler and generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", + "LIBNPY_BUILD_TESTS": "ON" + } + }, + { + "name": "release-clang", + "displayName": "Release Build using clang", + "description": "Sets up a release build that uses Clang++", + "generator": "Ninja", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", + "CMAKE_CXX_COMPILER": "clang++", + "LIBNPY_BUILD_TESTS": "ON" + } + }, + { + "name": "release", + "displayName": "Release Build", + "description": "Sets up a release build that uses the default compiler and generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", + "LIBNPY_BUILD_TESTS": "ON" + } + } + ] + } \ No newline at end of file diff --git a/CSharpWrapper/NumpyIONative.i b/CSharpWrapper/NumpyIONative.i index 12a4a85..e6a2fa3 100644 --- a/CSharpWrapper/NumpyIONative.i +++ b/CSharpWrapper/NumpyIONative.i @@ -16,7 +16,7 @@ %rename(DataType) data_type_t; %typemap(csbase) data_type_t "byte"; -enum class data_type_t : std::uint8_t { +enum class data_type_t : char { INT8, UINT8, INT16, @@ -32,7 +32,7 @@ enum class data_type_t : std::uint8_t { %rename(Endian) endian_t; %typemap(csbase) endian_t "byte"; -enum class endian_t : std::uint8_t { +enum class endian_t : char { NATIVE, BIG, LITTLE diff --git a/README.md b/README.md index 31312c4..06f8b7a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # libnpy -[![Build Status](https://travis-ci.com/matajoh/libnpy.svg?token=mQKh8ae3m6BDSeGHqxyY&branch=main)](https://travis-ci.com/matajoh/libnpy) - `libnpy` is a multi-platform C++ library for reading and writing NPY and NPZ files, with an additional .NET interface. It was built with the intention of making it easier for multi-language projects to use NPZ and @@ -18,98 +16,34 @@ format documents: ## Getting Started -There are two main ways to use the library: as a statically linked C++ -library, and as a .NET DLL (on Windows using Visual Studio). In this -guide we will walk through how to compile the library and run the tests -on our currently supported platforms. These directions will likely -work for other platforms as well (the codebase is written to be clean, -portable C++ 11). If you have problems on your platform, please raise -it as an [issue](https://github.com/matajoh/libnpy/issues). - - -### Ubuntu 18.04 [gcc 7.3.0], Ubuntu 16.04 [gcc 5.4.0] - -First, install all of the necessary dependencies: - - sudo apt-get install git cmake build-essential - -If you want to build the documentation, you will also need: - - sudo apt-get install doxygen - -You may also find that `cmake` is easier to use via the curses GUI: - - sudo apt-get install cmake-curses-gui - -Once everything is in place, you can clone the repository and generate the -makefiles: - - git clone https://github.com/matajoh/libnpy.git - mkdir libnpy/build - cd libnpy/build - cmake -DCMAKE_BUILD_TYPE=Debug .. - -Your other build options are `Release` and `RelWithDebInfo`. - -### Windows 10 - -On Windows, you can download and install the dependencies from the following -locations: - -#### Install CMake -Download and run e.g. `v3.19/cmake-3.19.0-win64-x64.msi` from -https://cmake.org/files/. - -#### Install git and Visual Studio. -Get the latest Windows git from https://git-scm.com/downloads. Download a -version of Visual Studio from https://visualstudio.microsoft.com/vs/. You -will need the C++ compiler (and C# compiler if needed). - -#### Install SWIG (optional, only for C#) -Browse to http://swig.org/download.html and download the latest version of -`swigwin`. Unzip the directory and copy it to your `C:\` drive. Add (e.g.) -`C:\swigwin-4.0.2` to your PATH. CMake should then find swig automatically. - -#### Download and install Doxygen (optional) -If you want to build the documentation, you should also download -[Doxygen](http://www.doxygen.nl/). - -#### Generate MSBuild -Now that everything is ready, cmake can generate the MSBuild files necessary -for the project. Run the following commands in a command prompt once you have -navigated to your desired source code folder: +Start by installing [CMake](https://cmake.org/) in the way appropriate for your +environment. - git clone https://github.com/matajoh/libnpy.git - mkdir libnpy\build - cd libnpy\build - cmake .. +### Linux -If building the C# library, you will also need to do the following: +Create a build directory and initialize the cmake project: - cmake --build . --target NumpyIONative - cmake .. + mkdir build + cd build + cmake .. --preset release -The reason for the above is that SWIG autogenerates the C# files for the -interface in the first pass, after which CMake needs to scan the generated -directory to build the wrapper library. +You can then build and run the tests using: -### Build and Test -You are now able to build the test the library. Doing so is the same -regardless of your platform. First, navigate to the `build` folder you -created above. Then run the following commands: + make + ctest - cmake --build . --config +### Windows -Where `` is one of `Release|Debug|RelWithDebInfo`. This will build -the project, including the tests and (if selected) the documentation. You -can then do the following: +Create a build directory and initialize the cmake project: - ctest -C + mkdir build + cd build + cmake .. --preset release -Where again you replace `` as above will run all of the tests. -If you want to install the library, run: +You can then build and run the tests using: - cmake --build . --config --target INSTALL + cmake --build . --config Release + ctest -C Release ## Sample code @@ -198,4 +132,4 @@ The generated documentation contains more details on all of the functionality. We hope you find that the library fulfills your needs and is easy to use, but if you have any difficulties please create [issues](https://github.com/matajoh/libnpy/issues) so the maintainers can make -the library even better. Thanks! \ No newline at end of file +the library even better. Thanks! diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 9afd24f..1c8b71e 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,5 +1,10 @@ Improvements: -- CMake build now uses the highest compiler warning/error setting +- Increased CHUNK size as per miniz instructions +- Added tests for very large arrays in NPZ files +- Added some CI tests to catch issues across platforms +- Removed the internal IO streams in favor of just using stringstream +- NPZs can now be read from and written to memory Bugfixes: -- Fixed some bugs exposed by heightened compiler warnings \ No newline at end of file +- Fixed an issue where very large arrays in NPZ files would throw an error +- Fixed a bug with mac builds due to deprecated APIs diff --git a/include/npy/core.h b/include/npy/core.h index 81238ec..ede7a6e 100644 --- a/include/npy/core.h +++ b/include/npy/core.h @@ -12,18 +12,15 @@ #define _CORE_H_ #include -#include #include -#include +#include -namespace npy -{ +namespace npy { /** Enumeration which represents a type of endianness */ -enum class endian_t : std::uint8_t -{ - /** Indicates that the native endianness should be used. Native in this case means - * that of the hardware the program is currently running on. - */ +enum class endian_t : char { + /** Indicates that the native endianness should be used. Native in this case + * means that of the hardware the program is currently running on. + */ NATIVE, /** Indicates the use of big-endian encoding */ BIG, @@ -32,8 +29,7 @@ enum class endian_t : std::uint8_t }; /** This function will return the endianness of the current hardware. */ -inline endian_t native_endian() -{ +inline endian_t native_endian() { union { std::uint32_t i; char c[4]; @@ -42,9 +38,9 @@ inline endian_t native_endian() return endian_test.c[0] == 1 ? endian_t::BIG : endian_t::LITTLE; }; -/** This enum represents the different types of tensor data that can be stored. */ -enum class data_type_t : std::uint8_t -{ +/** This enum represents the different types of tensor data that can be stored. + */ +enum class data_type_t : char { /** 8 bit signed integer */ INT8, /** 8 bit unsigned integer */ @@ -71,10 +67,11 @@ enum class data_type_t : std::uint8_t /** Convert a data type and endianness to a NPY dtype string. * \param dtype the data type - * \param endian the endianness. Defaults to the current endianness of the caller. - * \return the NPY dtype string + * \param endian the endianness. Defaults to the current endianness of the + * caller. \return the NPY dtype string */ -const std::string &to_dtype(data_type_t dtype, endian_t endian = endian_t::NATIVE); +const std::string &to_dtype(data_type_t dtype, + endian_t endian = endian_t::NATIVE); /** Converts from an NPY dtype string to a data type and endianness. * \param dtype the NPY dtype string @@ -82,101 +79,8 @@ const std::string &to_dtype(data_type_t dtype, endian_t endian = endian_t::NATIV */ const std::pair &from_dtype(const std::string &dtype); -/** Implementation of an in-memory stream buffer that can be moved */ -class membuf : public std::basic_streambuf -{ -public: - /** Default Constructor. */ - membuf(); - - /** Constructor. - * \param n the default starting capacity for the buffer - */ - membuf(size_t n); - - /** Copy constructor. - * \param buffer the values copied and used for the buffer - */ - membuf(const std::vector &buffer); - - /** Move constructor. - * \param buffer the values moved and used for the buffer - */ - membuf(std::vector &&buffer); - - /** Returns a reference to the internal buffer object. */ - std::vector &buf(); - - /** Returns a const reference to the internal buffer object. */ - const std::vector &buf() const; - -protected: - membuf *setbuf(std::uint8_t *s, std::streamsize n) override; - pos_type seekoff(off_type off, std::ios_base::seekdir way, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out) override; - pos_type seekpos(pos_type pos, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out) override; - std::streamsize showmanyc() override; - std::streamsize xsgetn(std::uint8_t *s, std::streamsize n) override; - int_type underflow() override; - int_type pbackfail(int_type c = traits_type::eof()) override; - std::streamsize xsputn(const std::uint8_t *s, std::streamsize n) override; - int_type overflow(int_type c = traits_type::eof()) override; - -private: - std::vector m_buffer; - std::vector::iterator m_posg; - std::vector::iterator m_posp; -}; - -/** An input stream which uses a moveable, in-memory buffer */ -class imemstream : public std::basic_istream -{ -public: - /** Copy constructor. - * \param buffer the buffer copied and used for the stream - */ - imemstream(const std::vector &buffer); - - /** Move constructor. - * \param buffer the buffer moved and used for the stream - */ - imemstream(std::vector &&buffer); - - /** Returns a reference to the underlying byte vector */ - std::vector &buf(); - - /** Returns a const reference to the underlying byte vector */ - const std::vector &buf() const; - -private: - membuf m_buffer; -}; - -/** An output stream which uses a moveable, in-memory byte vector */ -class omemstream : public std::basic_ostream -{ -public: - /** Default constructor */ - omemstream(); - - /** Move constructor. - * \param buffer the buffer to use for writing. Values in the buffer will be overwritten - */ - omemstream(std::vector &&buffer); - - /** Constructor. - * \param capacity the initial capacity for the output buffer - */ - omemstream(std::streamsize capacity); - - /** Returns a reference to the underlying byte vector */ - std::vector &buf(); - - /** Returns a const reference to the underlying byte vector */ - const std::vector &buf() const; - -private: - membuf m_buffer; -}; +typedef std::basic_istringstream imemstream; +typedef std::basic_ostringstream omemstream; std::ostream &operator<<(std::ostream &os, const endian_t &obj); std::ostream &operator<<(std::ostream &os, const data_type_t &obj); diff --git a/include/npy/npy.h b/include/npy/npy.h index 0420148..cec5f4e 100644 --- a/include/npy/npy.h +++ b/include/npy/npy.h @@ -14,120 +14,113 @@ #define _NPY_H_ #include -#include -#include -#include +#include +#include #include -#include +#include #include -#include -#include +#include +#include #include "core.h" const int STATIC_HEADER_LENGTH = 10; -namespace npy -{ +namespace npy { /** Class representing the header info for an NPY file */ -struct header_info -{ - /** Constructor. - * \param dictionary a Python-encoded dictionary containing the header information - */ - explicit header_info(const std::string &dictionary); - - /** Constructor */ - header_info(data_type_t dtype, - npy::endian_t endianness, - bool fortran_order, - const std::vector &shape); - - /** The data type of the NPY file */ - data_type_t dtype; - - /** The endianness of the data in the NPY file */ - npy::endian_t endianness; - - /** Whether the values in the tensor are stored in FORTRAN, or column major, order */ - bool fortran_order; - - /** A vector of values indicating the shape of each dimension of the tensor. */ - std::vector shape; - - /** Value used to indicate the maximum length of an element (used by Unicode strings) */ - std::size_t max_element_length; +struct header_info { + /** Constructor. + * \param dictionary a Python-encoded dictionary containing the header + * information + */ + explicit header_info(const std::string &dictionary); + + /** Constructor */ + header_info(data_type_t dtype, npy::endian_t endianness, bool fortran_order, + const std::vector &shape); + + /** The data type of the NPY file */ + data_type_t dtype; + + /** The endianness of the data in the NPY file */ + npy::endian_t endianness; + + /** Whether the values in the tensor are stored in FORTRAN, or column major, + * order */ + bool fortran_order; + + /** A vector of values indicating the shape of each dimension of the tensor. + */ + std::vector shape; + + /** Value used to indicate the maximum length of an element (used by Unicode + * strings) */ + std::size_t max_element_length; }; /** Writes an NPY header to the provided stream. * \param output the output stream - * \param dtype the NPY-encoded dtype string (includes data type and endianness) - * \param fortran_order whether the data is encoded in FORTRAN (i.e. column major) order - * \param shape a sequence of values indicating the shape of each dimension of the tensor - * \sa npy::to_dtype + * \param dtype the NPY-encoded dtype string (includes data type and + * endianness) \param fortran_order whether the data is encoded in FORTRAN (i.e. + * column major) order \param shape a sequence of values indicating the shape of + * each dimension of the tensor \sa npy::to_dtype */ template void write_npy_header(std::basic_ostream &output, - const std::string &dtype, - bool fortran_order, - const std::vector &shape) -{ - std::ostringstream buff; - buff << "{'descr': '" << dtype; - buff << "', 'fortran_order': " << (fortran_order ? "True" : "False"); - buff << ", 'shape': ("; - for (auto dim = shape.begin(); dim < shape.end(); ++dim) - { - buff << *dim; - if (dim < shape.end() - 1) - { - buff << ", "; - } - } - - if(shape.size() == 1) - { - buff << ","; - } - - buff << "), }"; - std::string dictionary = buff.str(); - auto dict_length = dictionary.size() + 1; - std::string end = "\n"; - auto header_length = dict_length + STATIC_HEADER_LENGTH; - if (header_length % 64 != 0) - { - header_length = ((header_length / 64) + 1) * 64; - dict_length = header_length - STATIC_HEADER_LENGTH; - end = std::string(dict_length - dictionary.length(), ' '); - end.back() = '\n'; + const std::string &dtype, bool fortran_order, + const std::vector &shape) { + std::ostringstream buff; + buff << "{'descr': '" << dtype; + buff << "', 'fortran_order': " << (fortran_order ? "True" : "False"); + buff << ", 'shape': ("; + for (auto dim = shape.begin(); dim < shape.end(); ++dim) { + buff << *dim; + if (dim < shape.end() - 1) { + buff << ", "; } - - std::uint8_t header[STATIC_HEADER_LENGTH] = {0x93, 'N', 'U', 'M', 'P', 'Y', 0x01, 0x00, - static_cast(dict_length), - 0x00}; - output.write(reinterpret_cast(header), STATIC_HEADER_LENGTH); - output.write(reinterpret_cast(dictionary.data()), dictionary.length()); - output.write(reinterpret_cast(end.data()), end.length()); + } + + if (shape.size() == 1) { + buff << ","; + } + + buff << "), }"; + std::string dictionary = buff.str(); + auto dict_length = dictionary.size() + 1; + std::string end = "\n"; + auto header_length = dict_length + STATIC_HEADER_LENGTH; + if (header_length % 64 != 0) { + header_length = ((header_length / 64) + 1) * 64; + dict_length = header_length - STATIC_HEADER_LENGTH; + end = std::string(dict_length - dictionary.length(), ' '); + end.back() = '\n'; + } + + const char header[STATIC_HEADER_LENGTH] = { + static_cast(0x93), 'N', 'U', + 'M', 'P', 'Y', + 0x01, 0x00, static_cast(dict_length), + 0x00}; + output.write(header, STATIC_HEADER_LENGTH); + output.write(reinterpret_cast(dictionary.data()), + dictionary.length()); + output.write(reinterpret_cast(end.data()), end.length()); } -template -void copy_to(const T* data_ptr, std::size_t num_elements, std::basic_ostream& output, npy::endian_t endianness) -{ - if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) - { - output.write(reinterpret_cast(data_ptr), num_elements * sizeof(T)); +template +void copy_to(const T *data_ptr, std::size_t num_elements, + std::basic_ostream &output, npy::endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), + num_elements * sizeof(T)); + } else { + CHAR buffer[sizeof(T)]; + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const CHAR *start = reinterpret_cast(curr); + std::reverse_copy(start, start + sizeof(T), buffer); + output.write(buffer, sizeof(T)); } - else - { - CHAR buffer[sizeof(T)]; - for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) - { - const CHAR *start = reinterpret_cast(curr); - std::reverse_copy(start, start + sizeof(T), buffer); - output.write(buffer, sizeof(T)); - } - } + } } /** Saves a tensor to the provided stream. @@ -138,17 +131,13 @@ void copy_to(const T* data_ptr, std::size_t num_elements, std::basic_ostream class TENSOR, - typename CHAR, +template class TENSOR, typename CHAR, std::enable_if_t::value, int> = 42> -void save(std::basic_ostream &output, - const TENSOR &tensor, - endian_t endianness = npy::endian_t::NATIVE) -{ - auto dtype = to_dtype(tensor.dtype(), endianness); - write_npy_header(output, dtype, tensor.fortran_order(), tensor.shape()); - copy_to(tensor.data(), tensor.size(), output, endianness); +void save(std::basic_ostream &output, const TENSOR &tensor, + endian_t endianness = npy::endian_t::NATIVE) { + auto dtype = to_dtype(tensor.dtype(), endianness); + write_npy_header(output, dtype, tensor.fortran_order(), tensor.shape()); + copy_to(tensor.data(), tensor.size(), output, endianness); }; /** Saves a unicode string tensor to the provided stream. @@ -158,54 +147,43 @@ void save(std::basic_ostream &output, * \param endianness the endianness to use in saving the tensor * \sa npy::tensor */ -template class TENSOR, - typename CHAR, +template class TENSOR, typename CHAR, std::enable_if_t::value, int> = 42> -void save(std::basic_ostream &output, - const TENSOR &tensor, - endian_t endianness = npy::endian_t::NATIVE) -{ - std::size_t max_length = 0; - for(const auto& element : tensor) - { - if(element.size() > max_length) - { - max_length = element.size(); - } +void save(std::basic_ostream &output, const TENSOR &tensor, + endian_t endianness = npy::endian_t::NATIVE) { + std::size_t max_length = 0; + for (const auto &element : tensor) { + if (element.size() > max_length) { + max_length = element.size(); } - - if(endianness == npy::endian_t::NATIVE) - { - endianness = native_endian(); - } - - std::string dtype = ">U" + std::to_string(max_length); - if(endianness == npy::endian_t::LITTLE) - { - dtype = " unicode(tensor.size() * max_length, 0); + auto word_start = unicode.begin(); + for (const auto &element : tensor) { + auto char_it = word_start; + for (const auto &wchar : element) { + *char_it = static_cast(wchar); + char_it += 1; } - write_npy_header(output, dtype, tensor.fortran_order(), tensor.shape()); + word_start += max_length; + } - std::vector unicode(tensor.size() * max_length, 0); - auto word_start = unicode.begin(); - for(const auto& element : tensor) - { - auto char_it = word_start; - for(const auto& wchar : element) - { - *char_it = static_cast(wchar); - char_it += 1; - } - - word_start += max_length; - } - - copy_to(unicode.data(), unicode.size(), output, endianness); + copy_to(unicode.data(), unicode.size(), output, endianness); }; - /** Saves a tensor to the provided location on disk. * \tparam T the data type * \tparam TENSOR the tensor type. @@ -214,19 +192,15 @@ void save(std::basic_ostream &output, * \param endianness the endianness to use in saving the tensor * \sa npy::tensor */ -template class TENSOR> -void save(const std::string &path, - const TENSOR &tensor, - endian_t endianness = npy::endian_t::NATIVE) -{ - std::ofstream output(path, std::ios::out | std::ios::binary); - if (!output.is_open()) - { - throw std::invalid_argument("path"); - } - - save(output, tensor, endianness); +template class TENSOR> +void save(const std::string &path, const TENSOR &tensor, + endian_t endianness = npy::endian_t::NATIVE) { + std::ofstream output(path, std::ios::out | std::ios::binary); + if (!output.is_open()) { + throw std::invalid_argument("path"); + } + + save(output, tensor, endianness); }; /** Read an NPY header from the provided stream. @@ -234,148 +208,121 @@ void save(const std::string &path, * \return the header information */ template -header_info read_npy_header(std::basic_istream &input) -{ - std::uint8_t header[STATIC_HEADER_LENGTH]; - input.read(reinterpret_cast(header), STATIC_HEADER_LENGTH); - assert(header[0] == 0x93); - assert(header[1] == 'N'); - assert(header[2] == 'U'); - assert(header[3] == 'M'); - assert(header[4] == 'P'); - assert(header[5] == 'Y'); - size_t dict_length = 0; - if (header[6] == 0x01 && header[7] == 0x00) - { - dict_length = header[8] | (header[9] << 8); - } - else if (header[6] == 0x02 && header[7] == 0x00) - { - std::uint8_t extra[2]; - input.read(reinterpret_cast(extra), 2); - dict_length = header[8] | (header[9] << 8) | (extra[0] << 16) | (extra[1] << 24); - } - - std::vector buffer(dict_length); - input.read(buffer.data(), dict_length); - std::string dictionary(buffer.begin(), buffer.end()); - return header_info(dictionary); +header_info read_npy_header(std::basic_istream &input) { + std::uint8_t header[STATIC_HEADER_LENGTH]; + input.read(reinterpret_cast(header), STATIC_HEADER_LENGTH); + assert(header[0] == 0x93); + assert(header[1] == 'N'); + assert(header[2] == 'U'); + assert(header[3] == 'M'); + assert(header[4] == 'P'); + assert(header[5] == 'Y'); + size_t dict_length = 0; + if (header[6] == 0x01 && header[7] == 0x00) { + dict_length = header[8] | (header[9] << 8); + } else if (header[6] == 0x02 && header[7] == 0x00) { + std::uint8_t extra[2]; + input.read(reinterpret_cast(extra), 2); + dict_length = + header[8] | (header[9] << 8) | (extra[0] << 16) | (extra[1] << 24); + } + + std::vector buffer(dict_length); + input.read(buffer.data(), dict_length); + std::string dictionary(buffer.begin(), buffer.end()); + return header_info(dictionary); } template -void copy_to(std::basic_istream &input, T* data_ptr, std::size_t num_elements, npy::endian_t endianness) -{ - if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) - { - CHAR *start = reinterpret_cast(data_ptr); - input.read(start, num_elements * sizeof(T)); +void copy_to(std::basic_istream &input, T *data_ptr, + std::size_t num_elements, npy::endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + CHAR *start = reinterpret_cast(data_ptr); + input.read(start, num_elements * sizeof(T)); + } else { + CHAR buffer[sizeof(T)]; + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + input.read(buffer, sizeof(T)); + CHAR *start = reinterpret_cast(curr); + std::reverse_copy(buffer, buffer + sizeof(T), start); } - else - { - CHAR buffer[sizeof(T)]; - for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) - { - input.read(buffer, sizeof(T)); - CHAR *start = reinterpret_cast(curr); - std::reverse_copy(buffer, buffer + sizeof(T), start); - } - } + } } -/** Loads a tensor in NPY format from the provided stream. The type of the tensor - * must match the data to be read. - * \tparam T the data type - * \tparam TENSOR the tensor type - * \param input the input stream - * \return an object of type TENSOR read from the stream - * \sa npy::tensor +/** Loads a tensor in NPY format from the provided stream. The type of the + * tensor must match the data to be read. \tparam T the data type \tparam TENSOR + * the tensor type \param input the input stream \return an object of type + * TENSOR read from the stream \sa npy::tensor */ -template class TENSOR, - typename CHAR, +template class TENSOR, typename CHAR, std::enable_if_t::value, int> = 42> -TENSOR load(std::basic_istream &input) -{ - header_info info = read_npy_header(input); - TENSOR tensor(info.shape, info.fortran_order); - if (info.dtype != tensor.dtype()) - { - throw std::logic_error("requested dtype does not match stream's dtype"); - } - - copy_to(input, tensor.data(), tensor.size(), info.endianness); - return tensor; +TENSOR load(std::basic_istream &input) { + header_info info = read_npy_header(input); + TENSOR tensor(info.shape, info.fortran_order); + if (info.dtype != tensor.dtype()) { + throw std::logic_error("requested dtype does not match stream's dtype"); + } + + copy_to(input, tensor.data(), tensor.size(), info.endianness); + return tensor; } - -/** Loads a unicode string tensor in NPY format from the provided stream. The type of the tensor - * must match the data to be read. - * \tparam T the data type +/** Loads a unicode string tensor in NPY format from the provided stream. The + * type of the tensor must match the data to be read. \tparam T the data type * \tparam TENSOR the tensor type * \param input the input stream * \return an object of type TENSOR read from the stream * \sa npy::tensor */ -template class TENSOR, - typename CHAR, +template class TENSOR, typename CHAR, std::enable_if_t::value, int> = 42> -TENSOR load(std::basic_istream &input) -{ - header_info info = read_npy_header(input); - TENSOR tensor(info.shape, info.fortran_order); - if (info.dtype != tensor.dtype()) - { - throw std::logic_error("requested dtype does not match stream's dtype"); +TENSOR load(std::basic_istream &input) { + header_info info = read_npy_header(input); + TENSOR tensor(info.shape, info.fortran_order); + if (info.dtype != tensor.dtype()) { + throw std::logic_error("requested dtype does not match stream's dtype"); + } + + std::vector unicode(tensor.size() * info.max_element_length, 0); + copy_to(input, unicode.data(), unicode.size(), info.endianness); + + auto word_start = unicode.begin(); + for (auto &element : tensor) { + auto char_it = word_start; + for (std::size_t i = 0; i < info.max_element_length && *char_it > 0; + ++i, ++char_it) { + element.push_back(static_cast(*char_it)); } - std::vector unicode(tensor.size() * info.max_element_length, 0); - copy_to(input, unicode.data(), unicode.size(), info.endianness); - - auto word_start = unicode.begin(); - for(auto& element : tensor) - { - auto char_it = word_start; - for(std::size_t i=0; i 0; ++i, ++char_it) - { - element.push_back(static_cast(*char_it)); - } + word_start += info.max_element_length; + } - word_start += info.max_element_length; - } - - return tensor; + return tensor; } -/** Loads a tensor in NPY format from the specified location on the disk. The type of the tensor - * must match the data to be read. - * \tparam T the data type +/** Loads a tensor in NPY format from the specified location on the disk. The + * type of the tensor must match the data to be read. \tparam T the data type * \tparam TENSOR the tensor type * \param path a valid location on the disk * \return an object of type TENSOR read from the stream * \sa npy::tensor */ -template class TENSOR> -TENSOR load(const std::string &path) -{ - std::ifstream input(path, std::ios::in | std::ios::binary); - if (!input.is_open()) - { - throw std::invalid_argument("path"); - } - - return load(input); +template class TENSOR> +TENSOR load(const std::string &path) { + std::ifstream input(path, std::ios::in | std::ios::binary); + if (!input.is_open()) { + throw std::invalid_argument("path"); + } + + return load(input); } /** Return the header information for an NPY file. * \param input the input stream containing the NPY-encoded bytes * \return the NPY header information */ -template -header_info peek(std::basic_istream &input) -{ - return read_npy_header(input); +template header_info peek(std::basic_istream &input) { + return read_npy_header(input); } /** Return the header information for an NPY file. diff --git a/include/npy/npz.h b/include/npy/npz.h index ed87b9a..bedd9eb 100644 --- a/include/npy/npz.h +++ b/include/npy/npz.h @@ -13,204 +13,200 @@ #ifndef _NPZ_H_ #define _NPZ_H_ -#include -#include -#include +#include #include +#include +#include +#include #include -#include #include -#include #include "core.h" #include "npy.h" #include "tensor.h" -namespace npy -{ -/** Enumeration indicating the compression method to use for data in the NPZ archive. */ -enum class compression_method_t : std::uint16_t -{ - /** Store the data with no compression */ - STORED = 0, - /** Use the DEFLATE algorithm to compress the data */ - DEFLATED = 8 +namespace npy { +/** Enumeration indicating the compression method to use for data in the NPZ + * archive. */ +enum class compression_method_t : std::uint16_t { + /** Store the data with no compression */ + STORED = 0, + /** Use the DEFLATE algorithm to compress the data */ + DEFLATED = 8 }; /** Struct representing a file in the NPZ archive. */ -struct file_entry -{ - /** The name of the file */ - std::string filename; - /** The CRC32 checksum of the uncompressed data */ - std::uint32_t crc32; - /** The size of the compressed data */ - std::uint64_t compressed_size; - /** The size of the uncompressed data */ - std::uint64_t uncompressed_size; - /** The method used to compress the data */ - std::uint16_t compression_method; - /** The offset of the file in the archive */ - std::uint64_t offset; - - /** Check if this entry matches another entry - * \param other the other entry - * \return if these entries match - */ - bool check(const file_entry &other) const; +struct file_entry { + /** The name of the file */ + std::string filename; + /** The CRC32 checksum of the uncompressed data */ + std::uint32_t crc32; + /** The size of the compressed data */ + std::uint64_t compressed_size; + /** The size of the uncompressed data */ + std::uint64_t uncompressed_size; + /** The method used to compress the data */ + std::uint16_t compression_method; + /** The offset of the file in the archive */ + std::uint64_t offset; + + /** Check if this entry matches another entry + * \param other the other entry + * \return if these entries match + */ + bool check(const file_entry &other) const; }; /** Class representing an output stream for an NPZ archive file */ -class onpzstream -{ - public: - /** Constructor. - * \param path the path to the file on disk - * \param compression how the entries should be compressed - * \param endianness the endianness to use in writing the entries - */ - onpzstream(const std::string &path, - compression_method_t compression = compression_method_t::STORED, - endian_t endianness = npy::endian_t::NATIVE); - - /** Whether the underlying stream has successfully been opened. */ - bool is_open() const; - - /** Closes this stream. This will write the directory and close - * the underlying stream as well. */ - void close(); - - /** Write a tensor to the NPZ archive. - * \tparam T the data type - * \tparam TENSOR the tensor type - * \param filename the name of the file in the archive - * \param tensor the tensor to write - */ - template class TENSOR> - void write(const std::string &filename, const TENSOR &tensor) - { - if (m_closed) - { - throw std::logic_error("Stream is closed"); - } - - omemstream output; - save(output, tensor, m_endianness); - - std::string suffix = ".npy"; - std::string name = filename; - if(name.size() < 4 || !std::equal(suffix.rbegin(), suffix.rend(), name.rbegin())) - { - name += ".npy"; - } - - write_file(name, std::move(output.buf())); +class onpzstream { +public: + /** Constructor + * \param output the output stream to write to + * \param compression how the entries should be compressed + * \param endianness the endianness to use in writing the entries + */ + onpzstream(const std::shared_ptr &output, + compression_method_t compression = compression_method_t::STORED, + endian_t endianness = npy::endian_t::NATIVE); + + /** Constructor. + * \param path the path to the file on disk + * \param compression how the entries should be compressed + * \param endianness the endianness to use in writing the entries + */ + onpzstream(const std::string &path, + compression_method_t compression = compression_method_t::STORED, + endian_t endianness = npy::endian_t::NATIVE); + + /** Whether the underlying stream has successfully been opened. */ + bool is_open() const; + + /** Closes this stream. This will write the directory and close + * the underlying stream as well. */ + void close(); + + /** Write a tensor to the NPZ archive. + * \tparam T the data type + * \tparam TENSOR the tensor type + * \param filename the name of the file in the archive + * \param tensor the tensor to write + */ + template class TENSOR> + void write(const std::string &filename, const TENSOR &tensor) { + if (m_closed) { + throw std::logic_error("Stream is closed"); } - /** Write a tensor to the NPZ archive. - * \tparam T the data type - * \param filename the name of the file in the archive - * \param tensor the tensor to write - */ - template - void write(const std::string &filename, const tensor &tensor) - { - write(filename, tensor); + omemstream output; + save(output, tensor, m_endianness); + + std::string suffix = ".npy"; + std::string name = filename; + if (name.size() < 4 || + !std::equal(suffix.rbegin(), suffix.rend(), name.rbegin())) { + name += ".npy"; } - /** Destructor. This will call - * \link npy::onpzstream::close \endlink, if it has not been called already. - */ - ~onpzstream(); - - private: - /** Write a file to the stream. - * \param filename the name of the file - * \param bytes the file data - */ - void write_file(const std::string &filename, - std::vector &&bytes); - - bool m_closed; - std::ofstream m_output; - compression_method_t m_compression_method; - endian_t m_endianness; - std::vector m_entries; + write_file(name, output.str()); + } + + /** Write a tensor to the NPZ archive. + * \tparam T the data type + * \param filename the name of the file in the archive + * \param tensor the tensor to write + */ + template + void write(const std::string &filename, const tensor &tensor) { + write(filename, tensor); + } + + /** Destructor. This will call + * \link npy::onpzstream::close \endlink, if it has not been called already. + */ + ~onpzstream(); + +private: + /** Write a file to the stream. + * \param filename the name of the file + * \param bytes the file data + */ + void write_file(const std::string &filename, std::string &&bytes); + + bool m_closed; + std::shared_ptr m_output; + compression_method_t m_compression_method; + endian_t m_endianness; + std::vector m_entries; }; /** Class representing an input stream from an NPZ archive file */ -class inpzstream -{ - public: - /** Constructor. - * \param path the path to the NPZ file on the disk - */ - inpzstream(const std::string &path); - - /** Whether the underlying stream has successfully been opened. */ - bool is_open() const; - - /** Closes the underlying stream. */ - void close(); - - /** The keys of the tensors in the NPZ */ - const std::vector& keys() const; - - /** Returns whether this NPZ contains the specified tensor - * \param filename the name of the tensor in the archive - * \return whether the tensor is in the archive - */ - bool contains(const std::string &filename); - - /** Returns the header for a specified tensor. - * \param filename the name of the tensor in the archive - * \return the header for the tensor - */ - header_info peek(const std::string &filename); - - /** Read a tensor from the archive. This method will thrown an exception if - * the tensor does not exist, or if the data type of the tensor does not match - * the template type. - * \tparam T the data type - * \tparam TENSOR the tensor type - * \param filename the name of the tensor in the archive. - * \return a instance of TENSOR read from the archive. - * \sa npy::tensor - */ - template class TENSOR> - TENSOR read(const std::string &filename) - { - imemstream stream(read_file(filename)); - return load(stream); - } - - /** Read a tensor from the archive. This method will thrown an exception if - * the tensor does not exist, or if the data type of the tensor does not match. - * \tparam T the data type - * \param filename the name of the tensor in the archive. - * \return a instance of tensor read from the archive. - */ - template - tensor read(const std::string &filename) - { - return read(filename); - } - - private: - /** Reads the bytes for a file from the archive. - * \param filename the name of the file - * \return the raw file bytes - */ - std::vector read_file(const std::string &filename); - - /** Read all entries from the directory. */ - void read_entries(); - - std::ifstream m_input; - std::map m_entries; - std::vector m_keys; +class inpzstream { +public: + /** Constructor. + * \param stream the input stream to read from + */ + inpzstream(const std::shared_ptr &stream); + + /** Constructor. + * \param path the path to the NPZ file on the disk + */ + inpzstream(const std::string &path); + + /** Whether the underlying stream has successfully been opened. */ + bool is_open() const; + + /** Closes the underlying stream. */ + void close(); + + /** The keys of the tensors in the NPZ */ + const std::vector &keys() const; + + /** Returns whether this NPZ contains the specified tensor + * \param filename the name of the tensor in the archive + * \return whether the tensor is in the archive + */ + bool contains(const std::string &filename); + + /** Returns the header for a specified tensor. + * \param filename the name of the tensor in the archive + * \return the header for the tensor + */ + header_info peek(const std::string &filename); + + /** Read a tensor from the archive. This method will thrown an exception if + * the tensor does not exist, or if the data type of the tensor does not + * match the template type. \tparam T the data type \tparam TENSOR the tensor + * type \param filename the name of the tensor in the archive. \return a + * instance of TENSOR read from the archive. \sa npy::tensor + */ + template class TENSOR> + TENSOR read(const std::string &filename) { + imemstream stream(read_file(filename)); + return load(stream); + } + + /** Read a tensor from the archive. This method will thrown an exception if + * the tensor does not exist, or if the data type of the tensor does not + * match. \tparam T the data type \param filename the name of the tensor in + * the archive. \return a instance of tensor read from the archive. + */ + template tensor read(const std::string &filename) { + return read(filename); + } + +private: + /** Reads the bytes for a file from the archive. + * \param filename the name of the file + * \return the raw file bytes + */ + std::string read_file(const std::string &filename); + + /** Read all entries from the directory. */ + void read_entries(); + + std::shared_ptr m_input; + std::map m_entries; + std::vector m_keys; }; } // namespace npy diff --git a/include/npy/tensor.h b/include/npy/tensor.h index 0a4256c..d2763fe 100644 --- a/include/npy/tensor.h +++ b/include/npy/tensor.h @@ -12,17 +12,16 @@ #define _TENSOR_H_ #include -#include #include #include #include #include +#include #include "core.h" #include "npy.h" -namespace npy -{ +namespace npy { /** The default tensor class. This class can be used as a data exchange format * for the library, but the methods and classes will also work with your own * tensor implementation. The library methods require the following methods to @@ -32,373 +31,303 @@ namespace npy * - \link size \endlink * - \link dtype \endlink * - \link fortran_order \endlink - * + * * As long as these are present and have the same semantics, the library should * handle them in the same was as this implementation. Only certain type of * tensor objects are natively supported (see \link npy::data_type_t \endlink). */ -template -class tensor -{ - public: - /** Constructor. - * \param path the path to an NPY file on the disk - */ - explicit tensor(const std::string &path) : tensor(npy::load(path)) {} - - /** Constructor. This will allocate a data buffer of the appropriate size in row-major order. - * \param shape the shape of the tensor - */ - tensor(const std::vector &shape) : tensor(shape, false) - { - } - - /** Constructor. This will allocate a data buffer of the appropriate size. - * \param shape the shape of the tensor - * \param fortran_order whether the data is stored in FORTRAN, or column major, order - */ - tensor(const std::vector &shape, - bool fortran_order) : m_shape(shape), - m_ravel_strides(tensor::get_ravel_strides(shape, fortran_order)), - m_fortran_order(fortran_order), - m_dtype(tensor::get_dtype()), - m_values(tensor::get_size(shape)) - { - } - - /** Copy constructor. */ - tensor(const tensor &other) : m_shape(other.m_shape), - m_ravel_strides(other.m_ravel_strides), - m_fortran_order(other.m_fortran_order), - m_dtype(other.m_dtype), - m_values(other.m_values) - { - } - - /** Move constructor. */ - tensor(tensor &&other) : m_shape(std::move(other.m_shape)), - m_ravel_strides(std::move(other.m_ravel_strides)), - m_fortran_order(other.m_fortran_order), - m_dtype(other.m_dtype), - m_values(std::move(other.m_values)) - { - } - - /** Variable parameter index function. - * \param index an index into the tensor. Can be negative (in which case it will work as in numpy) - * \return the value at the provided index - */ - template - const T &operator()(Indices... index) const - { - return m_values[ravel(std::vector({index...}))]; - } - - /** Index function. - * \param multi_index the index into the tensor. - * \return the value at the provided index - */ - const T &operator()(const std::vector &multi_index) const - { - return m_values[ravel(multi_index)]; - } - - /** Variable parameter index function. - * \param index an index into the tensor. Can be negative (in which case it will work as in numpy) - * \return the value at the provided index - */ - template - T &operator()(Indices... index) - { - return m_values[ravel(std::vector({index...}))]; - } - - /** Index function. - * \param multi_index the index into the tensor. - * \return the value at the provided index - */ - T &operator()(const std::vector &multi_index) - { - return m_values[ravel(multi_index)]; - } - - /** Iterator pointing at the beginning of the tensor in memory. */ - typename std::vector::iterator begin() - { - return m_values.begin(); - } - - /** Iterator pointing at the beginning of the tensor in memory. */ - typename std::vector::const_iterator begin() const - { - return m_values.begin(); - } - - /** Iterator pointing at the end of the tensor in memory. */ - typename std::vector::iterator end() - { - return m_values.end(); - } - - /** Iterator pointing at the end of the tensor in memory. */ - typename std::vector::const_iterator end() const - { - return m_values.end(); - } - - /** Sets the value at the provided index. - * \param multi_index an index into the tensor - * \param value the value to set - */ - void set(const std::vector &multi_index, const T &value) - { - m_values[ravel(multi_index)] = value; - } - - /** Gets the value at the provided index. - * \param multi_index the index into the tensor - * \return the value at the provided index - */ - const T &get(const std::vector &multi_index) const - { - return m_values[ravel(multi_index)]; - } - - /** The data type of the tensor. */ - const data_type_t dtype() const - { - return m_dtype; - } - - /** The underlying values buffer. */ - const std::vector &values() const - { - return m_values; - } - - /** Copy values from the source to this tensor. - * \param source pointer to the start of the source buffer - * \param nitems the number of items to copy. Should be equal to \link size \endlink. - */ - void copy_from(const T *source, size_t nitems) - { - if (nitems != size()) - { - throw std::invalid_argument("nitems"); - } - - std::copy(source, source + nitems, m_values.begin()); - } - - /** Copy values from the provided vector. - * \param source the source vector. Should have the same size as \link values \endlink. - */ - void copy_from(const std::vector &source) - { - if (source.size() != size()) - { - throw std::invalid_argument("source.size"); - } - - std::copy(source.begin(), source.end(), m_values.begin()); - } - - /** Move values from the provided vector. - * \param source the source vector. Should have the same size as \link values \endlink. - */ - void move_from(std::vector &&source) - { - if (source.size() != size()) - { - throw std::invalid_argument("source.size"); - } - - m_values = std::move(source); - } - - /** A pointer to the start of the underlying values buffer. */ - T *data() - { - return m_values.data(); - } - - /** A pointer to the start of the underlying values buffer. */ - const T *data() const - { - return m_values.data(); - } - - /** The number of elements in the tensor. */ - size_t size() const - { - return m_values.size(); - } - - /** The shape of the vector. Each element is the size of the - * corresponding dimension. */ - const std::vector &shape() const - { - return m_shape; - } - - /** Returns the dimensionality of the tensor at the specified index. - * \param index index into the shape - * \return the dimensionality at the index - */ - const size_t shape(int index) const - { - return m_shape[index]; - } - - /** Whether the tensor data is stored in FORTRAN, or column-major, order. */ - bool fortran_order() const - { - return m_fortran_order; - } - - /** Copy assignment operator. */ - tensor &operator=(const tensor &other) - { - m_shape = other.m_shape; - m_ravel_strides = other.m_ravel_strides; - m_fortran_order = other.m_fortran_order; - m_dtype = other.m_dtype; - m_values = other.m_values; - return *this; - } - - /** Move assignment operator. */ - tensor &operator=(tensor &&other) - { - m_shape = std::move(other.m_shape); - m_ravel_strides = std::move(other.m_ravel_strides); - m_fortran_order = other.m_fortran_order; - m_dtype = other.m_dtype; - m_values = std::move(other.m_values); - return *this; - } - - /** Save this tensor to the provided location on disk. - * \param path a valid location on disk - * \param endianness the endianness to use in writing the tensor - */ - void save(const std::string &path, endian_t endianness = npy::endian_t::NATIVE) - { - npy::save(path, *this, endianness); - } - - /** Ravels a multi-index into a single value indexing the buffer. - * \tparam INDEX_IT the index iterator class - * \tparam SHAPE_IT the shape iterator class - * \param index the multi-index iterator - * \param shape the shape iterator - * \return the single value in the buffer corresponding to the multi-index - */ - template - size_t ravel(INDEX_IT index, SHAPE_IT shape) const - { - std::size_t ravel = 0; - for (auto stride = m_ravel_strides.begin(); - stride < m_ravel_strides.end(); - ++index, ++shape, ++stride) - { - if (*index >= *shape) - { - throw std::out_of_range("multi_index"); - } - - ravel += *index * *stride; - } - - return ravel; - } - - /** Ravels a multi-index into a single value indexing the buffer. - * \param multi_index the multi-index value - * \return the single value in the buffer corresponding to the multi-index - */ - size_t ravel(const std::vector &multi_index) const - { - if (multi_index.size() != m_shape.size()) - { - throw std::invalid_argument("multi_index"); - } - - std::vector abs_multi_index(multi_index.size()); - std::transform(multi_index.begin(), multi_index.end(), m_shape.begin(), abs_multi_index.begin(), - [](std::int32_t index, std::size_t shape) -> std::size_t{ - if(index < 0) - { - return static_cast(shape + index); - } - - return static_cast(index); - }); - - return ravel(abs_multi_index); - } - - /** Ravels a multi-index into a single value indexing the buffer. - * \param abs_multi_index the multi-index value - * \return the single value in the buffer corresponding to the multi-index - */ - size_t ravel(const std::vector &abs_multi_index) const - { - if (m_fortran_order) - { - return ravel(abs_multi_index.rbegin(), m_shape.rbegin()); - } - - return ravel(abs_multi_index.begin(), m_shape.begin()); - } - - private: - std::vector m_shape; - std::vector m_ravel_strides; - bool m_fortran_order; - data_type_t m_dtype; - std::vector m_values; - - /** Returns the data type for this tensor. */ - static data_type_t get_dtype(); - - /** Gets the size of a tensor given its shape */ - static size_t get_size(const std::vector &shape) - { - size_t size = 1; - for (auto &dim : shape) - { - size *= dim; - } - - return size; - } - - /** Gets the strides for ravelling */ - static std::vector get_ravel_strides(const std::vector &shape, bool fortran_order) - { - std::vector ravel_strides(shape.size()); - size_t stride = 1; - auto ravel = ravel_strides.rbegin(); - if (fortran_order) - { - for (auto max_index = shape.begin(); max_index < shape.end(); ++max_index, ++ravel) - { - *ravel = stride; - stride *= *max_index; - } - } - else - { - for (auto max_index = shape.rbegin(); max_index < shape.rend(); ++max_index, ++ravel) - { - *ravel = stride; - stride *= *max_index; - } - } - - return ravel_strides; - } +template class tensor { +public: + /** Constructor. + * \param path the path to an NPY file on the disk + */ + explicit tensor(const std::string &path) + : tensor(npy::load(path)) {} + + /** Constructor. This will allocate a data buffer of the appropriate size in + * row-major order. \param shape the shape of the tensor + */ + tensor(const std::vector &shape) : tensor(shape, false) {} + + /** Constructor. This will allocate a data buffer of the appropriate size. + * \param shape the shape of the tensor + * \param fortran_order whether the data is stored in FORTRAN, or column + * major, order + */ + tensor(const std::vector &shape, bool fortran_order) + : m_shape(shape), + m_ravel_strides(tensor::get_ravel_strides(shape, fortran_order)), + m_fortran_order(fortran_order), m_dtype(tensor::get_dtype()), + m_values(tensor::get_size(shape)) {} + + /** Copy constructor. */ + tensor(const tensor &other) + : m_shape(other.m_shape), m_ravel_strides(other.m_ravel_strides), + m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), + m_values(other.m_values) {} + + /** Move constructor. */ + tensor(tensor &&other) + : m_shape(std::move(other.m_shape)), + m_ravel_strides(std::move(other.m_ravel_strides)), + m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), + m_values(std::move(other.m_values)) {} + + /** Variable parameter index function. + * \param index an index into the tensor. Can be negative (in which case it + * will work as in numpy) \return the value at the provided index + */ + template const T &operator()(Indices... index) const { + return m_values[ravel(std::vector({index...}))]; + } + + /** Index function. + * \param multi_index the index into the tensor. + * \return the value at the provided index + */ + const T &operator()(const std::vector &multi_index) const { + return m_values[ravel(multi_index)]; + } + + /** Variable parameter index function. + * \param index an index into the tensor. Can be negative (in which case it + * will work as in numpy) \return the value at the provided index + */ + template T &operator()(Indices... index) { + return m_values[ravel(std::vector({index...}))]; + } + + /** Index function. + * \param multi_index the index into the tensor. + * \return the value at the provided index + */ + T &operator()(const std::vector &multi_index) { + return m_values[ravel(multi_index)]; + } + + /** Iterator pointing at the beginning of the tensor in memory. */ + typename std::vector::iterator begin() { return m_values.begin(); } + + /** Iterator pointing at the beginning of the tensor in memory. */ + typename std::vector::const_iterator begin() const { + return m_values.begin(); + } + + /** Iterator pointing at the end of the tensor in memory. */ + typename std::vector::iterator end() { return m_values.end(); } + + /** Iterator pointing at the end of the tensor in memory. */ + typename std::vector::const_iterator end() const { return m_values.end(); } + + /** Sets the value at the provided index. + * \param multi_index an index into the tensor + * \param value the value to set + */ + void set(const std::vector &multi_index, const T &value) { + m_values[ravel(multi_index)] = value; + } + + /** Gets the value at the provided index. + * \param multi_index the index into the tensor + * \return the value at the provided index + */ + const T &get(const std::vector &multi_index) const { + return m_values[ravel(multi_index)]; + } + + /** The data type of the tensor. */ + const data_type_t dtype() const { return m_dtype; } + + /** The underlying values buffer. */ + const std::vector &values() const { return m_values; } + + /** Copy values from the source to this tensor. + * \param source pointer to the start of the source buffer + * \param nitems the number of items to copy. Should be equal to \link size + * \endlink. + */ + void copy_from(const T *source, size_t nitems) { + if (nitems != size()) { + throw std::invalid_argument("nitems"); + } + + std::copy(source, source + nitems, m_values.begin()); + } + + /** Copy values from the provided vector. + * \param source the source vector. Should have the same size as \link values + * \endlink. + */ + void copy_from(const std::vector &source) { + if (source.size() != size()) { + throw std::invalid_argument("source.size"); + } + + std::copy(source.begin(), source.end(), m_values.begin()); + } + + /** Move values from the provided vector. + * \param source the source vector. Should have the same size as \link values + * \endlink. + */ + void move_from(std::vector &&source) { + if (source.size() != size()) { + throw std::invalid_argument("source.size"); + } + + m_values = std::move(source); + } + + /** A pointer to the start of the underlying values buffer. */ + T *data() { return m_values.data(); } + + /** A pointer to the start of the underlying values buffer. */ + const T *data() const { return m_values.data(); } + + /** The number of elements in the tensor. */ + size_t size() const { return m_values.size(); } + + /** The shape of the vector. Each element is the size of the + * corresponding dimension. */ + const std::vector &shape() const { return m_shape; } + + /** Returns the dimensionality of the tensor at the specified index. + * \param index index into the shape + * \return the dimensionality at the index + */ + const size_t shape(int index) const { return m_shape[index]; } + + /** Whether the tensor data is stored in FORTRAN, or column-major, order. */ + bool fortran_order() const { return m_fortran_order; } + + /** Copy assignment operator. */ + tensor &operator=(const tensor &other) { + m_shape = other.m_shape; + m_ravel_strides = other.m_ravel_strides; + m_fortran_order = other.m_fortran_order; + m_dtype = other.m_dtype; + m_values = other.m_values; + return *this; + } + + /** Move assignment operator. */ + tensor &operator=(tensor &&other) { + m_shape = std::move(other.m_shape); + m_ravel_strides = std::move(other.m_ravel_strides); + m_fortran_order = other.m_fortran_order; + m_dtype = other.m_dtype; + m_values = std::move(other.m_values); + return *this; + } + + /** Save this tensor to the provided location on disk. + * \param path a valid location on disk + * \param endianness the endianness to use in writing the tensor + */ + void save(const std::string &path, + endian_t endianness = npy::endian_t::NATIVE) { + npy::save(path, *this, endianness); + } + + /** Ravels a multi-index into a single value indexing the buffer. + * \tparam INDEX_IT the index iterator class + * \tparam SHAPE_IT the shape iterator class + * \param index the multi-index iterator + * \param shape the shape iterator + * \return the single value in the buffer corresponding to the multi-index + */ + template + size_t ravel(INDEX_IT index, SHAPE_IT shape) const { + std::size_t ravel = 0; + for (auto stride = m_ravel_strides.begin(); stride < m_ravel_strides.end(); + ++index, ++shape, ++stride) { + if (*index >= *shape) { + throw std::out_of_range("multi_index"); + } + + ravel += *index * *stride; + } + + return ravel; + } + + /** Ravels a multi-index into a single value indexing the buffer. + * \param multi_index the multi-index value + * \return the single value in the buffer corresponding to the multi-index + */ + size_t ravel(const std::vector &multi_index) const { + if (multi_index.size() != m_shape.size()) { + throw std::invalid_argument("multi_index"); + } + + std::vector abs_multi_index(multi_index.size()); + std::transform(multi_index.begin(), multi_index.end(), m_shape.begin(), + abs_multi_index.begin(), + [](std::int32_t index, std::size_t shape) -> std::size_t { + if (index < 0) { + return static_cast(shape + index); + } + + return static_cast(index); + }); + + return ravel(abs_multi_index); + } + + /** Ravels a multi-index into a single value indexing the buffer. + * \param abs_multi_index the multi-index value + * \return the single value in the buffer corresponding to the multi-index + */ + size_t ravel(const std::vector &abs_multi_index) const { + if (m_fortran_order) { + return ravel(abs_multi_index.rbegin(), m_shape.rbegin()); + } + + return ravel(abs_multi_index.begin(), m_shape.begin()); + } + +private: + std::vector m_shape; + std::vector m_ravel_strides; + bool m_fortran_order; + data_type_t m_dtype; + std::vector m_values; + + /** Returns the data type for this tensor. */ + static data_type_t get_dtype(); + + /** Gets the size of a tensor given its shape */ + static size_t get_size(const std::vector &shape) { + size_t size = 1; + for (auto &dim : shape) { + size *= dim; + } + + return size; + } + + /** Gets the strides for ravelling */ + static std::vector get_ravel_strides(const std::vector &shape, + bool fortran_order) { + std::vector ravel_strides(shape.size()); + size_t stride = 1; + auto ravel = ravel_strides.rbegin(); + if (fortran_order) { + for (auto max_index = shape.begin(); max_index < shape.end(); + ++max_index, ++ravel) { + *ravel = stride; + stride *= *max_index; + } + } else { + for (auto max_index = shape.rbegin(); max_index < shape.rend(); + ++max_index, ++ravel) { + *ravel = stride; + stride *= *max_index; + } + } + + return ravel_strides; + } }; } // namespace npy diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 75bd080..66f8af4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,5 @@ set( SOURCES dtype.cpp - memstream.cpp npy.cpp npz.cpp tensor.cpp @@ -13,8 +12,13 @@ add_definitions( -DLIBNPY_VERSION=${LIBNPY_VERSION} ) add_library( npy STATIC ${SOURCES} ) add_library( npy::npy ALIAS npy ) +if (LIBNPY_SANITIZE) + target_compile_options(npy PUBLIC -g -fsanitize=${REGOCPP_SANITIZE} -fno-omit-frame-pointer) + target_link_libraries(npy PUBLIC -fsanitize=${REGOCPP_SANITIZE}) +endif() + if ( CMAKE_COMPILER_IS_GNUCC ) - target_compile_options(npy PRIVATE "-Wall -Wextra") + target_compile_options(npy PRIVATE "-Wall" "-Wextra") elseif( MSVC ) target_compile_options(npy PRIVATE "/W4") endif() diff --git a/src/dtype.cpp b/src/dtype.cpp index c8550f4..3968cb6 100644 --- a/src/dtype.cpp +++ b/src/dtype.cpp @@ -1,34 +1,15 @@ -#include #include +#include #include #include "npy/core.h" -namespace -{ +namespace { std::array BIG_ENDIAN_DTYPES = { - "|i1", - "|u1", - ">i2", - ">u2", - ">i4", - ">u4", - ">i8", - ">u8", - ">f4", - ">f8"}; + "|i1", "|u1", ">i2", ">u2", ">i4", ">u4", ">i8", ">u8", ">f4", ">f8"}; std::array LITTLE_ENDIAN_DTYPES = { - "|i1", - "|u1", - "> DTYPE_MAP = { {"|u1", {npy::data_type_t::UINT8, npy::endian_t::NATIVE}}, @@ -52,38 +33,31 @@ std::map> DTYPE_MAP = { }; } // namespace -namespace npy -{ -const std::string &to_dtype(data_type_t dtype, endian_t endianness) -{ - if (endianness == npy::endian_t::NATIVE) - { - endianness = native_endian(); - } +namespace npy { +const std::string &to_dtype(data_type_t dtype, endian_t endianness) { + if (endianness == npy::endian_t::NATIVE) { + endianness = native_endian(); + } - if (endianness == npy::endian_t::BIG) - { - return BIG_ENDIAN_DTYPES[static_cast(dtype)]; - } + if (endianness == npy::endian_t::BIG) { + return BIG_ENDIAN_DTYPES[static_cast(dtype)]; + } - return LITTLE_ENDIAN_DTYPES[static_cast(dtype)]; + return LITTLE_ENDIAN_DTYPES[static_cast(dtype)]; } -const std::pair &from_dtype(const std::string &dtype) -{ - return DTYPE_MAP[dtype]; +const std::pair &from_dtype(const std::string &dtype) { + return DTYPE_MAP[dtype]; } -std::ostream &operator<<(std::ostream &os, const data_type_t &value) -{ - os << static_cast(value); - return os; +std::ostream &operator<<(std::ostream &os, const data_type_t &value) { + os << static_cast(value); + return os; } -std::ostream &operator<<(std::ostream &os, const endian_t &value) -{ - os << static_cast(value); - return os; +std::ostream &operator<<(std::ostream &os, const endian_t &value) { + os << static_cast(value); + return os; } } // namespace npy \ No newline at end of file diff --git a/src/memstream.cpp b/src/memstream.cpp deleted file mode 100644 index 77ab0f3..0000000 --- a/src/memstream.cpp +++ /dev/null @@ -1,238 +0,0 @@ -#include "npy/core.h" - -#include - -namespace { - const int BUFFER_SIZE = 64 * 1024; -} - -namespace npy -{ -membuf::membuf() : membuf(BUFFER_SIZE) -{ - seekpos(0); -} - -membuf::membuf(size_t n) -{ - m_buffer.reserve(n); - seekpos(0); -} - -membuf::membuf(const std::vector &buffer) : m_buffer(buffer) -{ - seekpos(0); -} - -membuf::membuf(std::vector &&buffer) : m_buffer(std::move(buffer)) -{ - seekpos(0); -} - -membuf *membuf::setbuf(std::uint8_t *s, std::streamsize n) -{ - m_buffer = std::vector(s, s + n); - m_posg = m_buffer.begin(); - m_posp = m_buffer.begin(); - return this; -} - -membuf::pos_type membuf::seekoff(membuf::off_type off, std::ios_base::seekdir way, std::ios_base::openmode which) -{ - membuf::pos_type result(membuf::off_type(-1)); - if (which & std::ios_base::in) - { - switch (way) - { - case std::ios_base::beg: - m_posg = m_buffer.begin() + off; - break; - - case std::ios_base::end: - m_posg = m_buffer.end() + off; - break; - - case std::ios_base::cur: - m_posg += off; - break; - - default: - std::cerr << "Unsupported seek direction." << std::endl; - break; - } - - result = static_cast(m_posg - m_buffer.begin()); - } - - if (which & std::ios::out) - { - switch (way) - { - case std::ios_base::beg: - m_posp = m_buffer.begin() + off; - break; - - case std::ios_base::end: - m_posp = m_buffer.end() + off; - break; - - case std::ios_base::cur: - m_posp += off; - break; - - default: - std::cerr << "Unsupported seek direction." << std::endl; - break; - } - - result = static_cast(m_posp - m_buffer.begin()); - } - - return result; -} - -membuf::pos_type membuf::seekpos(membuf::pos_type pos, std::ios_base::openmode which) -{ - membuf::pos_type result(membuf::off_type(-1)); - if (which & std::ios_base::in) - { - m_posg = m_buffer.begin() + pos; - result = static_cast(m_posg - m_buffer.begin()); - } - - if (which & std::ios::out) - { - m_posp = m_buffer.begin() + pos; - result = static_cast(m_posp - m_buffer.begin()); - } - - return result; -} - -std::streamsize membuf::showmanyc() -{ - return m_buffer.end() - m_posg; -} - -std::streamsize membuf::xsgetn(std::uint8_t *s, std::streamsize n) -{ - std::streamsize bytes_read = showmanyc(); - bytes_read = n < bytes_read ? n : bytes_read; - auto end = m_posg + bytes_read; - std::copy(m_posg, end, s); - m_posg = end; - return bytes_read; -} - -membuf::int_type membuf::underflow() -{ - int_type result = membuf::traits_type::eof(); - if (m_posg < m_buffer.end()) - { - result = membuf::traits_type::to_int_type(*m_posg); - ++m_posg; - } - - return result; -} - -membuf::int_type membuf::pbackfail(membuf::int_type c) -{ - if (c != membuf::traits_type::eof()) - { - *m_posg = membuf::traits_type::to_char_type(c); - } - else - { - c = membuf::traits_type::to_int_type(*m_posg); - } - - return c; -} - -std::streamsize membuf::xsputn(const std::uint8_t *s, std::streamsize n) -{ - std::streamsize num_copy = m_buffer.end() - m_posp; - num_copy = n < num_copy ? n : num_copy; - std::streamsize num_insert = n - num_copy; - - std::copy(s, s + num_copy, m_posp); - if (num_insert > 0) - { - auto diffg = m_posg - m_buffer.begin(); - m_buffer.insert(m_buffer.end(), s + num_copy, s + n); - m_posp = m_buffer.end(); - m_posg = m_buffer.begin() + diffg; - - }else{ - m_posp += num_copy; - } - - return n; -} - -membuf::int_type membuf::overflow(membuf::int_type c) -{ - if (c != membuf::traits_type::eof()) - { - m_buffer.push_back(membuf::traits_type::to_char_type(c)); - } - - return c; -} - -std::vector &membuf::buf() -{ - return m_buffer; -} - -const std::vector &membuf::buf() const -{ - return m_buffer; -} - -imemstream::imemstream(const std::vector &buffer) : std::basic_istream(&m_buffer), - m_buffer(buffer) -{ -} - -imemstream::imemstream(std::vector &&buffer) : std::basic_istream(&m_buffer), - m_buffer(std::move(buffer)) -{ -} - -std::vector &imemstream::buf() -{ - return m_buffer.buf(); -} - -const std::vector &imemstream::buf() const -{ - return m_buffer.buf(); -} - -omemstream::omemstream() : std::basic_ostream(&m_buffer) -{ -} - -omemstream::omemstream(std::vector &&buffer) : std::basic_ostream(&m_buffer), - m_buffer(std::move(buffer)) -{ -} - -omemstream::omemstream(std::streamsize capacity) : std::basic_ostream(&m_buffer), - m_buffer(capacity) -{ -} - -std::vector &omemstream::buf() -{ - return m_buffer.buf(); -} - -const std::vector &omemstream::buf() const -{ - return m_buffer.buf(); -} - -} // namespace npy \ No newline at end of file diff --git a/src/npy.cpp b/src/npy.cpp index e1d049c..8966983 100644 --- a/src/npy.cpp +++ b/src/npy.cpp @@ -1,167 +1,137 @@ +#include #include -#include -#include #include -#include +#include #include #include "npy/npy.h" -namespace -{ +namespace { const int BUFFER_SIZE = 64 * 1024; char BUFFER[BUFFER_SIZE]; -void read(std::istream &input, char expected) -{ - char actual; - input.get(actual); - assert(actual == expected); +void read(std::istream &input, char expected) { + char actual; + input.get(actual); + assert(actual == expected); } -void read(std::istream &input, const std::string &expected) -{ - input.read(BUFFER, expected.length()); - std::string actual(BUFFER, BUFFER + expected.length()); - assert(actual == expected); +void read(std::istream &input, const std::string &expected) { + input.read(BUFFER, expected.length()); + std::string actual(BUFFER, BUFFER + expected.length()); + assert(actual == expected); } -void skip_whitespace(std::istream &input) -{ - char skip; - while (std::isspace(input.peek())) - { - input.get(skip); - } +void skip_whitespace(std::istream &input) { + char skip; + while (std::isspace(input.peek())) { + input.get(skip); + } } -std::string read_to(std::istream &input, char delim) -{ - if(input.peek() == delim) - { - return ""; - } +std::string read_to(std::istream &input, char delim) { + if (input.peek() == delim) { + return ""; + } - input.get(BUFFER, BUFFER_SIZE, delim); - auto length = input.gcount(); - assert(length < BUFFER_SIZE); + input.get(BUFFER, BUFFER_SIZE, delim); + auto length = input.gcount(); + assert(length < BUFFER_SIZE); - return std::string(BUFFER, BUFFER + length); + return std::string(BUFFER, BUFFER + length); } -std::string read_string(std::istream &input) -{ - read(input, '\''); - std::string token = read_to(input, '\''); - read(input, '\''); - return token; +std::string read_string(std::istream &input) { + read(input, '\''); + std::string token = read_to(input, '\''); + read(input, '\''); + return token; } -bool read_bool(std::istream &input) -{ - if (input.peek() == 'T') - { - read(input, "True"); - return true; - } - else if (input.peek() == 'F') - { - read(input, "False"); - return false; - } +bool read_bool(std::istream &input) { + if (input.peek() == 'T') { + read(input, "True"); + return true; + } else if (input.peek() == 'F') { + read(input, "False"); + return false; + } - throw std::logic_error("Dictionary value is not a boolean"); + throw std::logic_error("Dictionary value is not a boolean"); } -std::vector read_shape(std::istream &input) -{ - read(input, '('); - std::stringstream tuple(read_to(input, ')')); - read(input, ')'); - - std::vector shape; - size_t size; - - while (tuple >> size) - { - shape.push_back(size); - if (tuple.peek() == ',') - { - read(tuple, ','); - skip_whitespace(tuple); - } +std::vector read_shape(std::istream &input) { + read(input, '('); + std::stringstream tuple(read_to(input, ')')); + read(input, ')'); + + std::vector shape; + size_t size; + + while (tuple >> size) { + shape.push_back(size); + if (tuple.peek() == ',') { + read(tuple, ','); + skip_whitespace(tuple); } + } - return shape; + return shape; } } // namespace -namespace npy -{ -header_info::header_info(const std::string &dictionary) -{ - std::istringstream input(dictionary); - read(input, '{'); - while (input.peek() != '}') - { - skip_whitespace(input); - std::string key = read_string(input); - skip_whitespace(input); - read(input, ':'); - skip_whitespace(input); - if (key == "descr") - { - std::string dtype_code = read_string(input); - if(dtype_code[1] == 'U') - { - this->dtype = npy::data_type_t::UNICODE_STRING; - endianness = dtype_code[0] == '>' ? npy::endian_t::BIG : npy::endian_t::LITTLE; - max_element_length = std::stoi(dtype_code.substr(2)); - } - else - { - std::tie(this->dtype, endianness) = from_dtype(dtype_code); - max_element_length = 0; - } - - } - else if (key == "fortran_order") - { - fortran_order = read_bool(input); - } - else if (key == "shape") - { - shape = read_shape(input); - } - else - { - throw std::logic_error("Unsupported key: " + key); - } - - read(input, ','); - skip_whitespace(input); +namespace npy { +header_info::header_info(const std::string &dictionary) { + std::istringstream input(dictionary); + read(input, '{'); + while (input.peek() != '}') { + skip_whitespace(input); + std::string key = read_string(input); + skip_whitespace(input); + read(input, ':'); + skip_whitespace(input); + if (key == "descr") { + std::string dtype_code = read_string(input); + if (dtype_code[1] == 'U') { + this->dtype = npy::data_type_t::UNICODE_STRING; + endianness = + dtype_code[0] == '>' ? npy::endian_t::BIG : npy::endian_t::LITTLE; + max_element_length = std::stoi(dtype_code.substr(2)); + } else { + std::tie(this->dtype, endianness) = from_dtype(dtype_code); + max_element_length = 0; + } + + } else if (key == "fortran_order") { + fortran_order = read_bool(input); + } else if (key == "shape") { + shape = read_shape(input); + } else { + throw std::logic_error("Unsupported key: " + key); } - read(input, '}'); + read(input, ','); + skip_whitespace(input); + } + + read(input, '}'); } -header_info::header_info(data_type_t dtype, npy::endian_t endianness, bool fortran_order, const std::vector &shape) -{ - this->dtype = dtype; - this->endianness = endianness; - this->fortran_order = fortran_order; - this->shape = shape; +header_info::header_info(data_type_t dtype, npy::endian_t endianness, + bool fortran_order, const std::vector &shape) { + this->dtype = dtype; + this->endianness = endianness; + this->fortran_order = fortran_order; + this->shape = shape; } -header_info peek(const std::string &path) -{ - std::ifstream input(path, std::ios::in | std::ios::binary); - if (!input.is_open()) - { - throw std::invalid_argument("path"); - } +header_info peek(const std::string &path) { + std::ifstream input(path, std::ios::in | std::ios::binary); + if (!input.is_open()) { + throw std::invalid_argument("path"); + } - return peek(input); + return peek(input); } } // namespace npy \ No newline at end of file diff --git a/src/npz.cpp b/src/npz.cpp index 1fb9c65..a192d14 100644 --- a/src/npz.cpp +++ b/src/npz.cpp @@ -1,17 +1,14 @@ -#include -#include -#include #include -#include -#include #include +#include +#include #include +#include #include "npy/npz.h" #include "zip.h" -namespace -{ +namespace { const std::array LOCAL_HEADER_SIG = {0x50, 0x4B, 0x03, 0x04}; const std::array CD_HEADER_SIG = {0x50, 0x4B, 0x01, 0x02}; const std::array CD_END_SIG = {0x50, 0x4B, 0x05, 0x06}; @@ -19,509 +16,471 @@ const std::array CD_END_SIG = {0x50, 0x4B, 0x05, 0x06}; const std::array EXTERNAL_ATTR = {0x00, 0x00, 0x80, 0x01}; const std::array TIME = {0x00, 0x00, 0x21, 0x00}; const int CD_END_SIZE = 22; -const std::uint16_t STANDARD_VERSION = 20; // 2.0 File is encrypted using traditional PKWARE encryption -const std::uint16_t ZIP64_VERSION = 45; // 4.5 File uses ZIP64 format extensions +const std::uint16_t STANDARD_VERSION = + 20; // 2.0 File is encrypted using traditional PKWARE encryption +const std::uint16_t ZIP64_VERSION = 45; // 4.5 File uses ZIP64 format extensions const std::uint16_t ZIP64_TAG = 1; const std::uint64_t ZIP64_LIMIT = 0x8FFFFFFF; const std::uint32_t ZIP64_PLACEHOLDER = 0xFFFFFFFF; -void write(std::ostream &stream, std::uint16_t value) -{ - stream.put(value & 0x00FF); - stream.put(value >> 8); +void write(std::ostream &stream, std::uint16_t value) { + stream.put(value & 0x00FF); + stream.put(value >> 8); } -void write(std::ostream &stream, std::uint32_t value) -{ - for (int i = 0; i < 4; ++i) - { - stream.put(value & 0x000000FF); - value >>= 8; - } +void write(std::ostream &stream, std::uint32_t value) { + for (int i = 0; i < 4; ++i) { + stream.put(value & 0x000000FF); + value >>= 8; + } } -void write32(std::ostream & stream, std::uint64_t value) -{ - if(value > ZIP64_LIMIT) - { - write(stream, ZIP64_PLACEHOLDER); - } - else - { - write(stream, static_cast(value)); - } +void write32(std::ostream &stream, std::uint64_t value) { + if (value > ZIP64_LIMIT) { + write(stream, ZIP64_PLACEHOLDER); + } else { + write(stream, static_cast(value)); + } } -void write(std::ostream &stream, std::uint64_t value) -{ - for(int i=0; i<8; ++i) - { - stream.put(value & 0x00000000000000FF); - value >>= 8; - } +void write(std::ostream &stream, std::uint64_t value) { + for (int i = 0; i < 8; ++i) { + stream.put(value & 0x00000000000000FF); + value >>= 8; + } } -std::uint16_t read16(std::istream &stream) -{ - std::uint16_t low = static_cast(stream.get()); - std::uint16_t high = static_cast(stream.get()); - return low | (high << 8); +std::uint16_t read16(std::istream &stream) { + std::uint16_t low = static_cast(stream.get()); + std::uint16_t high = static_cast(stream.get()); + return low | (high << 8); } -std::uint32_t read32(std::istream &stream) -{ - std::uint32_t result = 0; - int shift = 0; - for (int i = 0; i < 4; ++i, shift += 8) - { - std::uint32_t part = stream.get(); - result |= part << shift; - } +std::uint32_t read32(std::istream &stream) { + std::uint32_t result = 0; + int shift = 0; + for (int i = 0; i < 4; ++i, shift += 8) { + std::uint32_t part = stream.get(); + result |= part << shift; + } - return result; + return result; } -std::uint64_t read64(std::istream &stream) -{ - std::uint64_t result = 0; - int shift = 0; - for(int i=0; i<8; ++i, shift += 8) - { - std::uint64_t part = stream.get(); - result |= part << shift; - } - - return result; -} +std::uint64_t read64(std::istream &stream) { + std::uint64_t result = 0; + int shift = 0; + for (int i = 0; i < 8; ++i, shift += 8) { + std::uint64_t part = stream.get(); + result |= part << shift; + } -void assert_sig(std::istream &stream, const std::array &expected) -{ - std::array actual; - stream.read(reinterpret_cast(actual.data()), actual.size()); - if (actual != expected) - { - throw std::logic_error("Invalid signature (Not a valid NPZ file)"); - } + return result; } -std::uint16_t determine_extra_length(const npy::file_entry &header, bool include_offset) -{ - std::uint16_t length = 0; - if(header.compressed_size > ZIP64_LIMIT) - { - length += 8; - } - - if(header.uncompressed_size > ZIP64_LIMIT) - { - length += 8; - } - - if(include_offset && header.offset > ZIP64_LIMIT) - { - length += 8; - } - - return length; +void assert_sig(std::istream &stream, + const std::array &expected) { + std::array actual; + stream.read(reinterpret_cast(actual.data()), actual.size()); + if (actual != expected) { + throw std::logic_error("Invalid signature (Not a valid NPZ file)"); + } } -void write_zip64_extra(std::ostream &stream, const npy::file_entry &header, bool include_offset) -{ - std::vector extra; - if(header.uncompressed_size > ZIP64_LIMIT) - { - extra.push_back(header.uncompressed_size); - } - - if(header.compressed_size > ZIP64_LIMIT) - { - extra.push_back(header.compressed_size); - } +std::uint16_t determine_extra_length(const npy::file_entry &header, + bool include_offset) { + std::uint16_t length = 0; + if (header.compressed_size > ZIP64_LIMIT) { + length += 8; + } + + if (header.uncompressed_size > ZIP64_LIMIT) { + length += 8; + } + + if (include_offset && header.offset > ZIP64_LIMIT) { + length += 8; + } + + return length; +} + +void write_zip64_extra(std::ostream &stream, const npy::file_entry &header, + bool include_offset) { + std::vector extra; + if (header.uncompressed_size > ZIP64_LIMIT) { + extra.push_back(header.uncompressed_size); + } + + if (header.compressed_size > ZIP64_LIMIT) { + extra.push_back(header.compressed_size); + } + + if (include_offset && header.offset > ZIP64_LIMIT) { + extra.push_back(header.offset); + } - if(include_offset && header.offset > ZIP64_LIMIT) - { - extra.push_back(header.offset); - } + write(stream, ZIP64_TAG); + write(stream, static_cast(extra.size() * 8)); + for (auto val : extra) { + write(stream, val); + } +} + +void read_zip64_extra(std::istream &stream, npy::file_entry &header, + bool include_offset) { + std::uint16_t tag = read16(stream); + if (tag != ZIP64_TAG) { + throw std::logic_error("Invalid tag (expected ZIP64)"); + } + + std::uint16_t actual_size = read16(stream); + std::uint16_t expected_size = 0; + + if (header.uncompressed_size == ZIP64_PLACEHOLDER) { + header.uncompressed_size = read64(stream); + expected_size += 8; + } + + if (header.compressed_size == ZIP64_PLACEHOLDER) { + header.compressed_size = read64(stream); + expected_size += 8; + } + + if (include_offset && header.offset == ZIP64_PLACEHOLDER) { + header.offset = read64(stream); + expected_size += 8; + } + + if (actual_size < expected_size) { + throw std::logic_error("ZIP64 extra info missing"); + } + + if (actual_size > expected_size) { + // this can be the result of force_zip64 being set in Python's zipfile + stream.seekg(actual_size - expected_size, std::ios::cur); + } +} + +void write_shared_header(std::ostream &stream, const npy::file_entry &header) { + std::uint16_t general_purpose_big_flag = 0; + write(stream, general_purpose_big_flag); + write(stream, header.compression_method); + stream.write(reinterpret_cast(TIME.data()), TIME.size()); + write(stream, header.crc32); + write32(stream, header.compressed_size); + write32(stream, header.uncompressed_size); + write(stream, static_cast(header.filename.length())); +} + +std::uint16_t read_shared_header(std::istream &stream, + npy::file_entry &header) { + read16(stream); // general purpose bit flag + header.compression_method = read16(stream); + read32(stream); // time + header.crc32 = read32(stream); + header.compressed_size = read32(stream); + header.uncompressed_size = read32(stream); + return read16(stream); +} + +void write_local_header(std::ostream &stream, const npy::file_entry &header, + bool zip64) { + stream.write(reinterpret_cast(LOCAL_HEADER_SIG.data()), + LOCAL_HEADER_SIG.size()); + write(stream, zip64 ? ZIP64_VERSION : STANDARD_VERSION); + write_shared_header(stream, header); + std::uint16_t extra_field_length = determine_extra_length(header, false); + write(stream, extra_field_length); + stream.write(header.filename.data(), header.filename.length()); + if (extra_field_length > 0) { + write_zip64_extra(stream, header, false); + } +} + +npy::file_entry read_local_header(std::istream &stream) { + assert_sig(stream, LOCAL_HEADER_SIG); + std::uint16_t version = read16(stream); + if (version > ZIP64_VERSION) { + throw std::logic_error("Unsupported NPZ version"); + } + + npy::file_entry entry; + + std::uint16_t filename_length = read_shared_header(stream, entry); + std::uint16_t extra_field_length = read16(stream); + std::vector buffer(filename_length); + stream.read(buffer.data(), filename_length); + entry.filename = std::string(buffer.begin(), buffer.end()); + + if (extra_field_length > 0) { + read_zip64_extra(stream, entry, false); + } + + return entry; +} + +void write_central_directory_header(std::ostream &stream, + const npy::file_entry &header) { + std::uint16_t extra_field_length = determine_extra_length(header, true); + stream.write(reinterpret_cast(CD_HEADER_SIG.data()), + CD_HEADER_SIG.size()); + write(stream, STANDARD_VERSION); + write(stream, extra_field_length > 0 ? ZIP64_VERSION : STANDARD_VERSION); + write_shared_header(stream, header); + write(stream, extra_field_length); + std::uint16_t file_comment_length = 0; + write(stream, file_comment_length); + std::uint16_t disk_number_start = 0; + write(stream, disk_number_start); + std::uint16_t internal_file_attributes = 0; + write(stream, internal_file_attributes); + stream.write(reinterpret_cast(EXTERNAL_ATTR.data()), + EXTERNAL_ATTR.size()); + write32(stream, header.offset); + stream.write(header.filename.data(), header.filename.length()); + if (extra_field_length > 0) { + write_zip64_extra(stream, header, true); + } +} + +npy::file_entry read_central_directory_header(std::istream &stream) { + assert_sig(stream, CD_HEADER_SIG); + read16(stream); // version made by + std::uint16_t version = read16(stream); + if (version > ZIP64_VERSION) { + throw std::logic_error("Unsupported NPZ version"); + } + + npy::file_entry entry; + std::uint16_t filename_length = read_shared_header(stream, entry); + std::uint16_t extra_field_length = read16(stream); + read16(stream); // file comment length + read16(stream); // disk number start + read16(stream); // internal file attributes + read32(stream); // external file attributes + entry.offset = read32(stream); + + std::vector buffer(filename_length); + stream.read(buffer.data(), filename_length); + entry.filename = std::string(buffer.begin(), buffer.end()); + + if (extra_field_length > 0) { + read_zip64_extra(stream, entry, true); + } + + return entry; +} + +struct CentralDirectory { + std::uint16_t num_entries; + std::uint32_t size; + std::uint32_t offset; +}; - write(stream, ZIP64_TAG); - write(stream, static_cast(extra.size() * 8)); - for(auto val : extra) - { - write(stream, val); - } +void write_end_of_central_directory(std::ostream &stream, + const CentralDirectory &dir) { + stream.write(reinterpret_cast(CD_END_SIG.data()), + CD_END_SIG.size()); + uint16_t disk_number = 0; + write(stream, disk_number); + write(stream, disk_number); + write(stream, dir.num_entries); + write(stream, dir.num_entries); + write(stream, dir.size); + write(stream, dir.offset); + std::uint16_t file_comment_length = 0; + write(stream, file_comment_length); +} + +CentralDirectory read_end_of_central_directory(std::istream &stream) { + assert_sig(stream, CD_END_SIG); + + CentralDirectory result; + read16(stream); // number of this disk + read16(stream); // number of the disk with the start of the central directory + result.num_entries = read16(stream); + read16(stream); // num_entries_on_disk + result.size = read32(stream); + result.offset = read32(stream); + return result; } -void read_zip64_extra(std::istream &stream, npy::file_entry &header, bool include_offset) -{ - std::uint16_t tag = read16(stream); - if(tag != ZIP64_TAG) - { - throw std::logic_error("Invalid tag (expected ZIP64)"); - } - - std::uint16_t actual_size = read16(stream); - std::uint16_t expected_size = 0; - - if(header.uncompressed_size == ZIP64_PLACEHOLDER) - { - header.uncompressed_size = read64(stream); - expected_size += 8; - } - - if(header.compressed_size == ZIP64_PLACEHOLDER) - { - header.compressed_size = read64(stream); - expected_size += 8; - } - - if(include_offset && header.offset == ZIP64_PLACEHOLDER) - { - header.offset = read64(stream); - expected_size += 8; - } - - if(actual_size < expected_size) - { - throw std::logic_error("ZIP64 extra info missing"); - } - - if(actual_size > expected_size) - { - // this can be the result of force_zip64 being set in Python's zipfile - stream.seekg(actual_size - expected_size, std::ios::cur); - } -} +} // namespace -void write_shared_header(std::ostream &stream, const npy::file_entry &header) -{ - std::uint16_t general_purpose_big_flag = 0; - write(stream, general_purpose_big_flag); - write(stream, header.compression_method); - stream.write(reinterpret_cast(TIME.data()), TIME.size()); - write(stream, header.crc32); - write32(stream, header.compressed_size); - write32(stream, header.uncompressed_size); - write(stream, static_cast(header.filename.length())); -} - -std::uint16_t read_shared_header(std::istream &stream, npy::file_entry &header) -{ - read16(stream); // general purpose bit flag - header.compression_method = read16(stream); - read32(stream); // time - header.crc32 = read32(stream); - header.compressed_size = read32(stream); - header.uncompressed_size = read32(stream); - return read16(stream); -} - -void write_local_header(std::ostream &stream, const npy::file_entry &header, bool zip64) -{ - stream.write(reinterpret_cast(LOCAL_HEADER_SIG.data()), LOCAL_HEADER_SIG.size()); - write(stream, zip64 ? ZIP64_VERSION : STANDARD_VERSION); - write_shared_header(stream, header); - std::uint16_t extra_field_length = determine_extra_length(header, false); - write(stream, extra_field_length); - stream.write(header.filename.data(), header.filename.length()); - if(extra_field_length > 0) - { - write_zip64_extra(stream, header, false); - } +namespace npy { +bool file_entry::check(const file_entry &other) const { + return !(other.filename != this->filename || other.crc32 != this->crc32 || + other.compression_method != this->compression_method || + other.compressed_size != this->compressed_size || + other.uncompressed_size != this->uncompressed_size); } -npy::file_entry read_local_header(std::istream &stream) -{ - assert_sig(stream, LOCAL_HEADER_SIG); - std::uint16_t version = read16(stream); - if (version > ZIP64_VERSION) - { - throw std::logic_error("Unsupported NPZ version"); - } - - npy::file_entry entry; +onpzstream::onpzstream(const std::shared_ptr &output, + compression_method_t compression, endian_t endianness) + : m_closed(false), m_output(output), m_compression_method(compression), + m_endianness(endianness) {} - std::uint16_t filename_length = read_shared_header(stream, entry); - std::uint16_t extra_field_length = read16(stream); - std::vector buffer(filename_length); - stream.read(buffer.data(), filename_length); - entry.filename = std::string(buffer.begin(), buffer.end()); +onpzstream::onpzstream(const std::string &path, compression_method_t method, + endian_t endianness) + : m_closed(false), m_output(std::make_shared( + path, std::ios::out | std::ios::binary)), + m_compression_method(method), m_endianness(endianness) {} - if(extra_field_length > 0) - { - read_zip64_extra(stream, entry, false); - } - - return entry; -} - -void write_central_directory_header(std::ostream &stream, const npy::file_entry &header) -{ - std::uint16_t extra_field_length = determine_extra_length(header, true); - stream.write(reinterpret_cast(CD_HEADER_SIG.data()), CD_HEADER_SIG.size()); - write(stream, STANDARD_VERSION); - write(stream, extra_field_length > 0 ? ZIP64_VERSION : STANDARD_VERSION); - write_shared_header(stream, header); - write(stream, extra_field_length); - std::uint16_t file_comment_length = 0; - write(stream, file_comment_length); - std::uint16_t disk_number_start = 0; - write(stream, disk_number_start); - std::uint16_t internal_file_attributes = 0; - write(stream, internal_file_attributes); - stream.write(reinterpret_cast(EXTERNAL_ATTR.data()), EXTERNAL_ATTR.size()); - write32(stream, header.offset); - stream.write(header.filename.data(), header.filename.length()); - if(extra_field_length > 0) - { - write_zip64_extra(stream, header, true); - } +onpzstream::~onpzstream() { + if (!m_closed) { + close(); + } } -npy::file_entry read_central_directory_header(std::istream &stream) -{ - assert_sig(stream, CD_HEADER_SIG); - read16(stream); // version made by - std::uint16_t version = read16(stream); - if (version > ZIP64_VERSION) - { - throw std::logic_error("Unsupported NPZ version"); - } +void onpzstream::write_file(const std::string &filename, std::string &&bytes) { + std::uint32_t uncompressed_size = static_cast(bytes.size()); + std::uint32_t compressed_size = 0; + std::string compressed_bytes; + std::uint32_t checksum = npy_crc32(bytes); + if (m_compression_method == compression_method_t::STORED) { + compressed_bytes = bytes; + compressed_size = uncompressed_size; + } else if (m_compression_method == compression_method_t::DEFLATED) { + compressed_bytes = npy_deflate(std::move(bytes)); + compressed_size = static_cast(compressed_bytes.size()); + } else { + throw std::invalid_argument("m_compression_method"); + } - npy::file_entry entry; - std::uint16_t filename_length = read_shared_header(stream, entry); - std::uint16_t extra_field_length = read16(stream); - read16(stream); // file comment length - read16(stream); // disk number start - read16(stream); // internal file attributes - read32(stream); // external file attributes - entry.offset = read32(stream); - - std::vector buffer(filename_length); - stream.read(buffer.data(), filename_length); - entry.filename = std::string(buffer.begin(), buffer.end()); - - if(extra_field_length > 0) - { - read_zip64_extra(stream, entry, true); - } + file_entry entry = {filename, + checksum, + compressed_size, + uncompressed_size, + static_cast(m_compression_method), + static_cast(m_output->tellp())}; - return entry; + bool zip64 = uncompressed_size > ZIP64_LIMIT || compressed_size > ZIP64_LIMIT; + write_local_header(*m_output, entry, zip64); + m_output->write(compressed_bytes.data(), compressed_size); + m_entries.push_back(std::move(entry)); } -struct CentralDirectory -{ - std::uint16_t num_entries; - std::uint32_t size; - std::uint32_t offset; -}; +bool onpzstream::is_open() const { + std::shared_ptr output = + std::dynamic_pointer_cast(m_output); + if (output) { + return output->is_open(); + } -void write_end_of_central_directory(std::ostream &stream, const CentralDirectory &dir) -{ - stream.write(reinterpret_cast(CD_END_SIG.data()), CD_END_SIG.size()); - uint16_t disk_number = 0; - write(stream, disk_number); - write(stream, disk_number); - write(stream, dir.num_entries); - write(stream, dir.num_entries); - write(stream, dir.size); - write(stream, dir.offset); - std::uint16_t file_comment_length = 0; - write(stream, file_comment_length); -} - -CentralDirectory read_end_of_central_directory(std::istream &stream) -{ - assert_sig(stream, CD_END_SIG); - - CentralDirectory result; - read16(stream); // number of this disk - read16(stream); // number of the disk with the start of the central directory - result.num_entries = read16(stream); - read16(stream); // num_entries_on_disk - result.size = read32(stream); - result.offset = read32(stream); - return result; + return true; } -} // namespace - -namespace npy -{ -bool file_entry::check(const file_entry &other) const -{ - return !(other.filename != this->filename || - other.crc32 != this->crc32 || - other.compression_method != this->compression_method || - other.compressed_size != this->compressed_size || - other.uncompressed_size != this->uncompressed_size); -} - -onpzstream::onpzstream(const std::string &path, - compression_method_t method, - endian_t endianness) : m_closed(false), - m_output(path, std::ios::out | std::ios::binary), - m_compression_method(method), - m_endianness(endianness) -{ -} - -onpzstream::~onpzstream() -{ - if (!m_closed) - { - close(); +void onpzstream::close() { + if (!m_closed) { + CentralDirectory dir; + dir.offset = static_cast(m_output->tellp()); + for (auto &header : m_entries) { + write_central_directory_header(*m_output, header); } -} -void onpzstream::write_file(const std::string &filename, - std::vector &&bytes) -{ - std::uint32_t uncompressed_size = static_cast(bytes.size()); - std::uint32_t compressed_size = 0; - std::vector compressed_bytes; - std::uint32_t checksum = npy_crc32(bytes); - if (m_compression_method == compression_method_t::STORED) - { - compressed_bytes = bytes; - compressed_size = uncompressed_size; - } - else if (m_compression_method == compression_method_t::DEFLATED) - { - compressed_bytes = npy_deflate(std::move(bytes)); - compressed_size = static_cast(compressed_bytes.size()); - } - else - { - throw std::invalid_argument("m_compression_method"); - } + dir.size = static_cast(m_output->tellp()) - dir.offset; + dir.num_entries = static_cast(m_entries.size()); + write_end_of_central_directory(*m_output, dir); - file_entry entry = { - filename, - checksum, - compressed_size, - uncompressed_size, - static_cast(m_compression_method), - static_cast(m_output.tellp())}; - - bool zip64 = uncompressed_size > ZIP64_LIMIT || compressed_size > ZIP64_LIMIT; - write_local_header(m_output, entry, zip64); - m_output.write(reinterpret_cast(compressed_bytes.data()), compressed_size); - m_entries.push_back(std::move(entry)); -} - -bool onpzstream::is_open() const -{ - return m_output.is_open(); -} - -void onpzstream::close() -{ - if (!m_closed) - { - CentralDirectory dir; - dir.offset = static_cast(m_output.tellp()); - for (auto &header : m_entries) - { - write_central_directory_header(m_output, header); - } - - dir.size = static_cast(m_output.tellp()) - dir.offset; - dir.num_entries = static_cast(m_entries.size()); - write_end_of_central_directory(m_output, dir); - m_output.close(); - m_closed = true; + std::shared_ptr output = + std::dynamic_pointer_cast(m_output); + if (output) { + output->close(); } + + m_closed = true; + } } -inpzstream::inpzstream(const std::string &path) : m_input(path, std::ios::out | std::ios::binary) -{ - read_entries(); +inpzstream::inpzstream(const std::shared_ptr &stream) + : m_input(stream) { + read_entries(); } -void inpzstream::read_entries() -{ - if (!m_input.is_open()) - { - throw std::invalid_argument("path"); - } +inpzstream::inpzstream(const std::string &path) { + m_input = + std::make_shared(path, std::ios::in | std::ios::binary); + read_entries(); +} - m_input.seekg(-CD_END_SIZE, std::ios::end); - CentralDirectory dir = read_end_of_central_directory(m_input); +void inpzstream::read_entries() { + m_input->seekg(-CD_END_SIZE, std::ios::end); + CentralDirectory dir = read_end_of_central_directory(*m_input); - m_input.seekg(dir.offset, std::ios::beg); + m_input->seekg(dir.offset, std::ios::beg); - for (size_t i = 0; i < dir.num_entries; ++i) - { - file_entry entry = read_central_directory_header(m_input); - m_entries[entry.filename] = entry; - m_keys.push_back(entry.filename); - } + for (size_t i = 0; i < dir.num_entries; ++i) { + file_entry entry = read_central_directory_header(*m_input); + m_entries[entry.filename] = entry; + m_keys.push_back(entry.filename); + } - std::sort(m_keys.begin(), m_keys.end()); + std::sort(m_keys.begin(), m_keys.end()); } -const std::vector& inpzstream::keys() const -{ - return m_keys; -} +const std::vector &inpzstream::keys() const { return m_keys; } -std::vector inpzstream::read_file(const std::string &temp_filename) -{ - std::string filename = temp_filename; - if (m_entries.count(filename) == 0) - { - filename += ".npy"; - if(m_entries.count(filename) == 0) - { - throw std::invalid_argument("filename"); - } +std::string inpzstream::read_file(const std::string &temp_filename) { + std::string filename = temp_filename; + if (m_entries.count(filename) == 0) { + filename += ".npy"; + if (m_entries.count(filename) == 0) { + throw std::invalid_argument("filename"); } + } - const file_entry &entry = m_entries[filename]; - m_input.seekg(entry.offset, std::ios::beg); + const file_entry &entry = m_entries[filename]; + m_input->seekg(entry.offset, std::ios::beg); - file_entry local = read_local_header(m_input); - if (!entry.check(local)) - { - throw std::logic_error("Central directory and local headers disagree"); - } + file_entry local = read_local_header(*m_input); + if (!entry.check(local)) { + throw std::logic_error("Central directory and local headers disagree"); + } - std::vector uncompressed_bytes(entry.compressed_size); - m_input.read(reinterpret_cast(uncompressed_bytes.data()), uncompressed_bytes.size()); - compression_method_t cmethod = static_cast(entry.compression_method); - if (cmethod == compression_method_t::DEFLATED) - { - uncompressed_bytes = npy_inflate(std::move(uncompressed_bytes)); - } + std::string uncompressed_bytes; + uncompressed_bytes.resize(entry.compressed_size); + m_input->read(reinterpret_cast(uncompressed_bytes.data()), + uncompressed_bytes.size()); + compression_method_t cmethod = + static_cast(entry.compression_method); + if (cmethod == compression_method_t::DEFLATED) { + uncompressed_bytes = npy_inflate(std::move(uncompressed_bytes)); + } - std::uint32_t actual_crc32 = npy_crc32(uncompressed_bytes); - if (actual_crc32 != entry.crc32) - { - throw std::logic_error("CRC mismatch"); - } + std::uint32_t actual_crc32 = npy_crc32(uncompressed_bytes); + if (actual_crc32 != entry.crc32) { + throw std::logic_error("CRC mismatch"); + } - return uncompressed_bytes; + return uncompressed_bytes; } -bool inpzstream::is_open() const -{ - return m_input.is_open(); +bool inpzstream::is_open() const { + std::shared_ptr input = + std::dynamic_pointer_cast(m_input); + if (input) { + return input->is_open(); + } + + return true; } -void inpzstream::close() -{ - m_input.close(); +void inpzstream::close() { + std::shared_ptr input = + std::dynamic_pointer_cast(m_input); + if (input) { + input->close(); + } } -bool inpzstream::contains(const std::string &filename) -{ - return m_entries.count(filename); +bool inpzstream::contains(const std::string &filename) { + return m_entries.count(filename); } -header_info inpzstream::peek(const std::string &filename) -{ - imemstream stream(read_file(filename)); - return npy::peek(stream); +header_info inpzstream::peek(const std::string &filename) { + imemstream stream(read_file(filename)); + return npy::peek(stream); } } // namespace npy \ No newline at end of file diff --git a/src/tensor.cpp b/src/tensor.cpp index 0fc904b..04497f2 100644 --- a/src/tensor.cpp +++ b/src/tensor.cpp @@ -1,71 +1,48 @@ #include "npy/tensor.h" -namespace npy -{ -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::INT8; +namespace npy { +template <> data_type_t tensor::get_dtype() { + return data_type_t::INT8; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::UINT8; +template <> data_type_t tensor::get_dtype() { + return data_type_t::UINT8; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::INT16; +template <> data_type_t tensor::get_dtype() { + return data_type_t::INT16; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::UINT16; +template <> data_type_t tensor::get_dtype() { + return data_type_t::UINT16; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::INT32; +template <> data_type_t tensor::get_dtype() { + return data_type_t::INT32; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::UINT32; +template <> data_type_t tensor::get_dtype() { + return data_type_t::UINT32; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::INT64; +template <> data_type_t tensor::get_dtype() { + return data_type_t::INT64; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::UINT64; +template <> data_type_t tensor::get_dtype() { + return data_type_t::UINT64; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::FLOAT32; +template <> data_type_t tensor::get_dtype() { + return data_type_t::FLOAT32; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::FLOAT64; +template <> data_type_t tensor::get_dtype() { + return data_type_t::FLOAT64; }; -template <> -data_type_t tensor::get_dtype() -{ - return data_type_t::UNICODE_STRING; +template <> data_type_t tensor::get_dtype() { + return data_type_t::UNICODE_STRING; } } // namespace npy \ No newline at end of file diff --git a/src/zip.cpp b/src/zip.cpp index acc98fb..1269704 100644 --- a/src/zip.cpp +++ b/src/zip.cpp @@ -1,177 +1,158 @@ +#include "miniz/miniz.h" #include #include -#include "miniz/miniz.h" #include "npy/core.h" #include "zip.h" -namespace -{ -const size_t CHUNK = 32 * 1024; +namespace { +const size_t CHUNK = 1024 * 1024; const int WINDOW_BITS = -15; const int MEM_LEVEL = 8; } // namespace -namespace npy -{ +namespace npy { -std::uint32_t npy_crc32(const std::vector &bytes) -{ - uLong crc = ::crc32(0L, Z_NULL, 0); - const Bytef *buf = reinterpret_cast(bytes.data()); - uInt len = static_cast(bytes.size()); - return ::crc32(crc, buf, len); +std::uint32_t npy_crc32(const std::string &bytes) { + uLong crc = ::crc32(0L, Z_NULL, 0); + const Bytef *buf = reinterpret_cast(bytes.data()); + uInt len = static_cast(bytes.size()); + return ::crc32(crc, buf, len); } -std::vector npy_deflate(std::vector &&bytes) -{ - int ret, flush; - unsigned have; - z_stream strm; - unsigned char in[CHUNK]; - unsigned char out[CHUNK]; - - /* allocate deflate state */ - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, - WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) - { - throw std::logic_error("Unable to initialize deflate algorithm"); +std::string npy_deflate(std::string &&bytes) { + int ret, flush; + unsigned have; + z_stream strm; + std::vector in(CHUNK); + std::vector out(CHUNK); + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, WINDOW_BITS, + MEM_LEVEL, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + throw std::logic_error("Unable to initialize deflate algorithm"); + } + + std::string str(bytes.begin(), bytes.end()); + imemstream input(std::move(str)); + omemstream output; + + /* compress until end of file */ + do { + input.read(reinterpret_cast(in.data()), CHUNK); + strm.avail_in = static_cast(input.gcount()); + if (input.eof()) { + flush = Z_FINISH; + } else { + if (input.fail() || input.bad()) { + (void)deflateEnd(&strm); + throw std::logic_error("Error reading from input stream"); + } + + flush = Z_NO_FLUSH; } - imemstream input(std::move(bytes)); - omemstream output; - - /* compress until end of file */ - do - { - input.read(in, CHUNK); - strm.avail_in = static_cast(input.gcount()); - if (input.eof()) - { - flush = Z_FINISH; - } - else - { - if (input.fail() || input.bad()) - { - (void)deflateEnd(&strm); - throw std::logic_error("Error reading from input stream"); - } - - flush = Z_NO_FLUSH; - } - - strm.next_in = in; - - /* run deflate() on input until output buffer not full, finish - compression if all of source has been read in */ - do - { - strm.avail_out = CHUNK; - strm.next_out = out; - ret = deflate(&strm, flush); - assert(ret != Z_STREAM_ERROR); /* state not clobbered */ - have = CHUNK - strm.avail_out; - output.write(out, have); - if (output.fail() || output.bad()) - { - (void)deflateEnd(&strm); - throw std::logic_error("Error writing to output stream"); - } - } while (strm.avail_out == 0); - assert(strm.avail_in == 0); /* all input will be used */ - /* done when last data in file processed */ - } while (flush != Z_FINISH); - assert(ret == Z_STREAM_END); /* stream will be complete */ - - /* clean up and return */ - (void)deflateEnd(&strm); - - return std::move(output.buf()); + strm.next_in = in.data(); + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out.data(); + ret = deflate(&strm, flush); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + output.write(reinterpret_cast(out.data()), have); + if (output.fail() || output.bad()) { + (void)deflateEnd(&strm); + throw std::logic_error("Error writing to output stream"); + } + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + + return output.str(); } -std::vector npy_inflate(std::vector &&bytes) -{ - int ret; - unsigned have; - z_stream strm; - unsigned char in[CHUNK]; - unsigned char out[CHUNK]; - /* allocate inflate state */ - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = 0; - strm.next_in = Z_NULL; - ret = inflateInit2(&strm, WINDOW_BITS); - if (ret != Z_OK) - { - throw std::logic_error("Unable to initialize inflate algorithm"); +std::string npy_inflate(std::string &&bytes) { + int ret; + unsigned have; + z_stream strm; + std::vector in(CHUNK); + std::vector out(CHUNK); + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, WINDOW_BITS); + if (ret != Z_OK) { + throw std::logic_error("Unable to initialize inflate algorithm"); + } + + std::string str(bytes.begin(), bytes.end()); + imemstream input(std::move(str)); + omemstream output; + + /* decompress until deflate stream ends or end of file */ + do { + input.read(reinterpret_cast(in.data()), CHUNK); + strm.avail_in = static_cast(input.gcount()); + if ((input.fail() && !input.eof()) || input.bad()) { + (void)inflateEnd(&strm); + throw std::logic_error("Error reading from input stream"); } - imemstream input(std::move(bytes)); - omemstream output; - - /* decompress until deflate stream ends or end of file */ - do - { - input.read(in, CHUNK); - strm.avail_in = static_cast(input.gcount()); - if ((input.fail() && !input.eof()) || input.bad()) - { - (void)inflateEnd(&strm); - throw std::logic_error("Error reading from input stream"); - } - - if (strm.avail_in == 0) - { - break; - } - - strm.next_in = in; - - /* run inflate() on input until output buffer not full */ - do - { - strm.avail_out = CHUNK; - strm.next_out = out; - ret = inflate(&strm, Z_NO_FLUSH); - assert(ret != Z_STREAM_ERROR); /* state not clobbered */ - switch (ret) - { - case Z_NEED_DICT: - ret = Z_DATA_ERROR; /* and fall through */ - case Z_DATA_ERROR: - case Z_MEM_ERROR: - (void)inflateEnd(&strm); - throw std::logic_error("Error inflating stream"); - } - - have = CHUNK - strm.avail_out; - output.write(out, have); - if (output.fail() || output.bad()) - { - (void)inflateEnd(&strm); - throw std::logic_error("Error writing to output stream"); - } - } while (strm.avail_out == 0); - - /* done when inflate() says it's done */ - } while (ret != Z_STREAM_END); - - /* clean up and return */ - (void)inflateEnd(&strm); - - if (ret == Z_STREAM_END) - { - return std::move(output.buf()); + if (strm.avail_in == 0) { + break; } - throw std::logic_error("Error inflating stream"); + strm.next_in = in.data(); + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out.data(); + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + throw std::logic_error("Error inflating stream"); + } + + have = CHUNK - strm.avail_out; + output.write(reinterpret_cast(out.data()), have); + if (output.fail() || output.bad()) { + (void)inflateEnd(&strm); + throw std::logic_error("Error writing to output stream"); + } + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + + if (ret == Z_STREAM_END) { + return output.str(); + } + + throw std::logic_error("Error inflating stream"); } -} // namespace zip \ No newline at end of file +} // namespace npy \ No newline at end of file diff --git a/src/zip.h b/src/zip.h index 40a91e7..1810410 100644 --- a/src/zip.h +++ b/src/zip.h @@ -11,29 +11,27 @@ #ifndef _ZIP_H_ #define _ZIP_H_ -#include -#include #include +#include -namespace npy -{ +namespace npy { /** Deflate the bytes and return the compressed result. * \param bytes the raw bytes * \return the compressed bytes */ -std::vector npy_deflate(std::vector &&bytes); +std::string npy_deflate(std::string &&bytes); /** Inflate the bytes and return the decompressed result. * \param bytes the compressed bytes * \return the raw bytes */ -std::vector npy_inflate(std::vector &&bytes); +std::string npy_inflate(std::string &&bytes); /** Perform a fast CRC32 checksum of a set of bytes. * \param bytes the bytes to check * \return the CRC32 checksum */ -std::uint32_t npy_crc32(const std::vector &bytes); +std::uint32_t npy_crc32(const std::string &bytes); } // namespace npy #endif \ No newline at end of file diff --git a/test/crc32.cpp b/test/crc32.cpp index 87913e9..a52c5a1 100644 --- a/test/crc32.cpp +++ b/test/crc32.cpp @@ -1,26 +1,23 @@ #include -#include -#include #include "libnpy_tests.h" #include "zip.h" const static int BUF_SIZE = 4096; -int test_crc32() -{ - int result = EXIT_SUCCESS; - std::ifstream stream(test::asset_path("float32.npy"), std::ios_base::in | std::ios_base::binary); - char buffer[BUF_SIZE]; - std::vector bytes; - while(stream.good()) - { - stream.read(buffer, BUF_SIZE); - std::copy(buffer, buffer + stream.gcount(), std::back_inserter(bytes)); - } +int test_crc32() { + int result = EXIT_SUCCESS; + std::ifstream stream(test::asset_path("float32.npy"), + std::ios_base::in | std::ios_base::binary); + char buffer[BUF_SIZE]; + std::string bytes; + while (stream.good()) { + stream.read(buffer, BUF_SIZE); + std::copy(buffer, buffer + stream.gcount(), std::back_inserter(bytes)); + } - int actual = npy::npy_crc32(bytes); - int expected = 928602993; - test::assert_equal(expected, actual, result, "crc32"); - return result; + int actual = npy::npy_crc32(bytes); + int expected = 928602993; + test::assert_equal(expected, actual, result, "crc32"); + return result; } \ No newline at end of file diff --git a/test/exceptions.cpp b/test/exceptions.cpp index 378214c..b6006db 100644 --- a/test/exceptions.cpp +++ b/test/exceptions.cpp @@ -3,119 +3,118 @@ #include "npy/npz.h" #include "npy/tensor.h" -namespace -{ -npy::tensor TENSOR(std::vector({ 5, 2, 5 })); +namespace { +npy::tensor TENSOR(std::vector({5, 2, 5})); -void save_invalid_path() -{ - npy::save(test::path_join({"does_not_exist", "bad.npy"}), TENSOR); +void save_invalid_path() { + npy::save(test::path_join({"does_not_exist", "bad.npy"}), TENSOR); } -void load_invalid_path() -{ - npy::load(test::path_join({"does_not_exist", "bad.npy"})); +void load_invalid_path() { + npy::load( + test::path_join({"does_not_exist", "bad.npy"})); } -void peek_invalid_path() -{ - npy::peek(test::path_join({"does_not_exist", "bad.npy"})); +void peek_invalid_path() { + npy::peek(test::path_join({"does_not_exist", "bad.npy"})); } -void inpzstream_invalid_path() -{ - npy::inpzstream(test::path_join({"does_not_exist", "bad.npz"})); +void inpzstream_invalid_path() { + npy::inpzstream(test::path_join({"does_not_exist", "bad.npz"})); } -void inpzstream_read_invalid_filename() -{ - npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); - npy::tensor tensor = stream.read("not_there.npy"); +void inpzstream_read_invalid_filename() { + npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); + npy::tensor tensor = stream.read("not_there.npy"); } -void inpzstream_peek_invalid_filename() -{ - npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); - npy::header_info header = stream.peek("not_there.npy"); +void inpzstream_peek_invalid_filename() { + npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); + npy::header_info header = stream.peek("not_there.npy"); } -void onpzstream_compression() -{ - npy::compression_method_t compression_method = static_cast(99); - npy::onpzstream stream("test.npz", compression_method); - stream.write("test.npy", TENSOR); +void onpzstream_compression() { + npy::compression_method_t compression_method = + static_cast(99); + npy::onpzstream stream("test.npz", compression_method); + stream.write("test.npy", TENSOR); } -void tensor_copy_from_0() -{ - std::vector buffer; - TENSOR.copy_from(buffer.data(), buffer.size()); +void tensor_copy_from_0() { + std::vector buffer; + TENSOR.copy_from(buffer.data(), buffer.size()); } -void tensor_copy_from_1() -{ - std::vector buffer; - TENSOR.copy_from(buffer); +void tensor_copy_from_1() { + std::vector buffer; + TENSOR.copy_from(buffer); } -void tensor_move_from() -{ - std::vector buffer; - TENSOR.copy_from(std::move(buffer)); +void tensor_move_from() { + std::vector buffer; + TENSOR.copy_from(std::move(buffer)); } -void tensor_index_size() -{ - std::uint8_t value = TENSOR(0, 0); -} +void tensor_index_size() { std::uint8_t value = TENSOR(0, 0); } -void tensor_index_range() -{ - std::uint8_t value = TENSOR(2, 3, 3); -} +void tensor_index_range() { std::uint8_t value = TENSOR(2, 3, 3); } -void load_wrong_dtype() -{ - npy::tensor tensor = npy::load(test::path_join({"assets", "test", "uint8.npy"})); +void load_wrong_dtype() { + npy::tensor tensor = npy::load( + test::path_join({"assets", "test", "uint8.npy"})); } -void onpzstream_closed() -{ - npy::onpzstream stream("test.npz"); - stream.close(); - stream.write("error.npy", TENSOR); +void onpzstream_closed() { + npy::onpzstream stream("test.npz"); + stream.close(); + stream.write("error.npy", TENSOR); } -void inpzstream_invalid_file() -{ - npy::inpzstream stream(test::path_join({"assets", "test", "uint8.npy"})); +void inpzstream_invalid_file() { + npy::inpzstream stream(test::path_join({"assets", "test", "uint8.npy"})); } } // namespace -int test_exceptions() -{ - int result = EXIT_SUCCESS; - - test::assert_throws(peek_invalid_path, result, "peek_invalid_path"); - test::assert_throws(save_invalid_path, result, "save_invalid_path"); - test::assert_throws(load_invalid_path, result, "load_invalid_path"); - test::assert_throws(inpzstream_invalid_path, result, "inpzstream_invalid_path"); - test::assert_throws(inpzstream_read_invalid_filename, result, "inpzstream_read_invalid_filename"); - test::assert_throws(inpzstream_peek_invalid_filename, result, "inpzstream_peek_invalid_filename"); - test::assert_throws(onpzstream_compression, result, "onpzstream_compression"); - test::assert_throws(tensor_copy_from_0, result, "tensor_copy_from_0"); - test::assert_throws(tensor_copy_from_1, result, "tensor_copy_from_1"); - test::assert_throws(tensor_move_from, result, "tensor_move_from"); - test::assert_throws(tensor_index_size, result, "tensor_index"); - - test::assert_throws(tensor_index_range, result, "tensor_index_range"); - - test::assert_throws(load_wrong_dtype, result, "load_wrong_dtype"); - test::assert_throws(onpzstream_closed, result, "onpzstream_closed"); - test::assert_throws(inpzstream_invalid_file, result, "inpzstream_invalid_file"); - - std::remove("test.npz"); - - return result; +int test_exceptions() { + int result = EXIT_SUCCESS; + + test::assert_throws(peek_invalid_path, result, + "peek_invalid_path"); + test::assert_throws(save_invalid_path, result, + "save_invalid_path"); + test::assert_throws(load_invalid_path, result, + "load_invalid_path"); + test::assert_throws(inpzstream_invalid_path, result, + "inpzstream_invalid_path"); + test::assert_throws( + inpzstream_read_invalid_filename, result, + "inpzstream_read_invalid_filename"); + test::assert_throws( + inpzstream_peek_invalid_filename, result, + "inpzstream_peek_invalid_filename"); + test::assert_throws(onpzstream_compression, result, + "onpzstream_compression"); + test::assert_throws(tensor_copy_from_0, result, + "tensor_copy_from_0"); + test::assert_throws(tensor_copy_from_1, result, + "tensor_copy_from_1"); + test::assert_throws(tensor_move_from, result, + "tensor_move_from"); + test::assert_throws(tensor_index_size, result, + "tensor_index"); + + test::assert_throws(tensor_index_range, result, + "tensor_index_range"); + + test::assert_throws(load_wrong_dtype, result, + "load_wrong_dtype"); + test::assert_throws(onpzstream_closed, result, + "onpzstream_closed"); + test::assert_throws(inpzstream_invalid_file, result, + "inpzstream_invalid_file"); + + std::remove("test.npz"); + + return result; } \ No newline at end of file diff --git a/test/generate_large_test.py b/test/generate_large_test.py new file mode 100644 index 0000000..65bb4d9 --- /dev/null +++ b/test/generate_large_test.py @@ -0,0 +1,17 @@ +import os + +import numpy as np + +if __name__ == "__main__": + test_int = np.arange(1000000).astype(np.int32) + test_int = test_int.reshape(200, 5, 1000) + test_float = np.arange(1000000).astype(np.float32) + test_float = test_float.reshape(1000, 5, 20, 10) + + path = os.path.join(os.path.dirname(__file__), "..", + "assets", "test", "test_large.npz") + np.savez(path, test_int=test_int, test_float=test_float) + + path = os.path.join(os.path.dirname(__file__), "..", + "assets", "test", "test_large_compressed.npz") + np.savez_compressed(path, test_int=test_int, test_float=test_float) diff --git a/test/libnpy_tests.cpp b/test/libnpy_tests.cpp index ee6f141..8f70fa3 100644 --- a/test/libnpy_tests.cpp +++ b/test/libnpy_tests.cpp @@ -1,113 +1,93 @@ -#include -#include -#include #include #include +#include +#include +#include #include #include "libnpy_tests.h" -namespace -{ -inline char sep() -{ +namespace { +inline char sep() { #if defined(_WIN32) || defined(WIN32) - return '\\'; + return '\\'; #else - return '/'; + return '/'; #endif } } // namespace -namespace test -{ +namespace test { -std::string path_join(const std::vector &parts) -{ - std::stringstream result; - result << parts.front(); +std::string path_join(const std::vector &parts) { + std::stringstream result; + result << parts.front(); - for (auto it = parts.begin() + 1; it < parts.end(); ++it) - { - result << sep() << *it; - } + for (auto it = parts.begin() + 1; it < parts.end(); ++it) { + result << sep() << *it; + } - return result.str(); + return result.str(); } -std::string read_file(const std::string &path) -{ - std::ifstream file(path, std::ios::in | std::ios::binary); - if (!file.is_open()) - { - throw std::invalid_argument("path"); - } +std::string read_file(const std::string &path) { + std::ifstream file(path, std::ios::in | std::ios::binary); + if (!file.is_open()) { + throw std::invalid_argument("path"); + } - std::ostringstream stream; - stream << file.rdbuf(); - return stream.str(); + std::ostringstream stream; + stream << file.rdbuf(); + return stream.str(); } -std::string asset_path(const std::string &filename) -{ - return path_join({"assets", "test", filename}); +std::string asset_path(const std::string &filename) { + return path_join({"assets", "test", filename}); } -std::string read_asset(const std::string &filename) -{ - return read_file(asset_path(filename)); +std::string read_asset(const std::string &filename) { + return read_file(asset_path(filename)); } } // namespace test typedef std::function TestFunction; -int main(int argc, char **argv) -{ - std::map tests; - - tests["crc32"] = test_crc32; - tests["exceptions"] = test_exceptions; - tests["memstream"] = test_memstream; - tests["npy_peek"] = test_npy_peek; - tests["npy_read"] = test_npy_read; - tests["npy_write"] = test_npy_write; - tests["npz_peek"] = test_npz_peek; - tests["npz_read"] = test_npz_read; - tests["npz_write"] = test_npz_write; - tests["tensor"] = test_tensor; - - if (argc == 2) - { - std::string test(argv[1]); - if (tests.count(test)) - { - return tests[test](); - } - else - { - std::cout << "Invalid test: " << test << std::endl; - return EXIT_FAILURE; - } +int main(int argc, char **argv) { + std::map tests; + + tests["crc32"] = test_crc32; + tests["exceptions"] = test_exceptions; + tests["memstream"] = test_memstream; + tests["npy_peek"] = test_npy_peek; + tests["npy_read"] = test_npy_read; + tests["npy_write"] = test_npy_write; + tests["npz_peek"] = test_npz_peek; + tests["npz_read"] = test_npz_read; + tests["npz_write"] = test_npz_write; + tests["tensor"] = test_tensor; + + if (argc == 2) { + std::string test(argv[1]); + if (tests.count(test)) { + return tests[test](); + } else { + std::cout << "Invalid test: " << test << std::endl; + return EXIT_FAILURE; } - else - { - int result = EXIT_SUCCESS; - for (auto &test : tests) - { - std::cout << "Running " << test.first << "..." << std::endl; - if (test.second()) - { - result = EXIT_FAILURE; - std::cout << test.first << " failed." << std::endl; - } - else - { - std::cout << test.first << " succeeded." << std::endl; - } - } - - return result; + } else { + int result = EXIT_SUCCESS; + for (auto &test : tests) { + std::cout << "Running " << test.first << "..." << std::endl; + if (test.second()) { + result = EXIT_FAILURE; + std::cout << test.first << " failed." << std::endl; + } else { + std::cout << test.first << " succeeded." << std::endl; + } } + + return result; + } } \ No newline at end of file diff --git a/test/libnpy_tests.h b/test/libnpy_tests.h index 5fff5d0..0c96471 100644 --- a/test/libnpy_tests.h +++ b/test/libnpy_tests.h @@ -2,13 +2,13 @@ #define _LIBNPY_TESTS_H_ #include +#include #include #include -#include #include "npy/core.h" -#include "npy/tensor.h" #include "npy/npy.h" +#include "npy/tensor.h" int test_crc32(); int test_exceptions(); @@ -21,255 +21,219 @@ int test_npz_read(); int test_npz_write(); int test_tensor(); -namespace test -{ +namespace test { template -void assert_equal(const T &expected, - const T &actual, - int &result, - const std::string &tag) -{ - if (expected != actual) - { - result = EXIT_FAILURE; - std::cout << tag << " is incorrect: " << actual << " != " << expected << std::endl; - } +void assert_equal(const T &expected, const T &actual, int &result, + const std::string &tag) { + if (expected != actual) { + result = EXIT_FAILURE; + std::cout << tag << " is incorrect: " << actual << " != " << expected + << std::endl; + } } template -void assert_equal(const std::vector &expected, - const std::vector &actual, - int &result, - const std::string &tag) -{ - assert_equal(expected.size(), actual.size(), result, tag + " size"); - if (result == EXIT_SUCCESS) - { - for (std::size_t i = 0; i < expected.size(); ++i) - { - assert_equal(expected[i], actual[i], result, tag + "[" + std::to_string(i) + "]"); - if (result == EXIT_FAILURE) - { - break; - } - } +void assert_equal(const std::vector &expected, const std::vector &actual, + int &result, const std::string &tag) { + assert_equal(expected.size(), actual.size(), result, tag + " size"); + if (result == EXIT_SUCCESS) { + for (std::size_t i = 0; i < expected.size(); ++i) { + assert_equal(expected[i], actual[i], result, + tag + "[" + std::to_string(i) + "]"); + if (result == EXIT_FAILURE) { + break; + } } + } } template -void assert_equal(const npy::tensor &expected, - const npy::tensor &actual, - int &result, - const std::string &tag) -{ - assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, tag + " dtype"); - assert_equal(expected.fortran_order(), actual.fortran_order(), result, tag + " fortran_order"); - assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); - assert_equal(expected.values(), actual.values(), result, tag); +void assert_equal(const npy::tensor &expected, const npy::tensor &actual, + int &result, const std::string &tag) { + assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, + tag + " dtype"); + assert_equal(expected.fortran_order(), actual.fortran_order(), result, + tag + " fortran_order"); + assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); + assert_equal(expected.values(), actual.values(), result, tag); } template <> inline void assert_equal(const std::string &expected, - const std::string &actual, - int &result, - const std::string &tag) -{ - assert_equal(expected.length(), actual.length(), result, tag + " length"); - if (result == EXIT_SUCCESS) - { - for (std::size_t i = 0; i < expected.size(); ++i) - { - int expected_val = static_cast(expected[i]); - int actual_val = static_cast(actual[i]); - assert_equal(expected_val, actual_val, result, tag + "[" + std::to_string(i) + "]"); - if (result == EXIT_FAILURE) - { - break; - } - } + const std::string &actual, int &result, + const std::string &tag) { + assert_equal(expected.length(), actual.length(), result, tag + " length"); + if (result == EXIT_SUCCESS) { + for (std::size_t i = 0; i < expected.size(); ++i) { + int expected_val = static_cast(expected[i]); + int actual_val = static_cast(actual[i]); + assert_equal(expected_val, actual_val, result, + tag + "[" + std::to_string(i) + "]"); + if (result == EXIT_FAILURE) { + break; + } } + } } -template<> +template <> inline void assert_equal(const std::wstring &expected, - const std::wstring &actual, - int &result, - const std::string &tag) -{ - assert_equal(expected.length(), actual.length(), result, tag + " length"); - if (result == EXIT_SUCCESS) - { - for (std::size_t i = 0; i < expected.size(); ++i) - { - int expected_val = static_cast(expected[i]); - int actual_val = static_cast(actual[i]); - assert_equal(expected_val, actual_val, result, tag + "[" + std::to_string(i) + "]"); - if (result == EXIT_FAILURE) - { - break; - } - } - } + const std::wstring &actual, int &result, + const std::string &tag) { + assert_equal(expected.length(), actual.length(), result, tag + " length"); + if (result == EXIT_SUCCESS) { + for (std::size_t i = 0; i < expected.size(); ++i) { + int expected_val = static_cast(expected[i]); + int actual_val = static_cast(actual[i]); + assert_equal(expected_val, actual_val, result, + tag + "[" + std::to_string(i) + "]"); + if (result == EXIT_FAILURE) { + break; + } + } + } } template <> inline void assert_equal(const npy::header_info &expected, const npy::header_info &actual, int &result, - const std::string &tag) -{ - assert_equal(expected.dtype, actual.dtype, result, tag + " dtype"); - assert_equal(expected.endianness, actual.endianness, result, tag + " endianness"); - assert_equal(expected.fortran_order, actual.fortran_order, result, tag + " fortran_order"); - assert_equal(expected.shape, actual.shape, result, tag + " shape"); + const std::string &tag) { + assert_equal(expected.dtype, actual.dtype, result, tag + " dtype"); + assert_equal(expected.endianness, actual.endianness, result, + tag + " endianness"); + assert_equal(expected.fortran_order, actual.fortran_order, result, + tag + " fortran_order"); + assert_equal(expected.shape, actual.shape, result, tag + " shape"); } -template<> -inline void assert_equal>(const npy::tensor &expected, - const npy::tensor &actual, - int &result, - const std::string &tag) -{ - assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, tag + " dtype"); - assert_equal(expected.fortran_order(), actual.fortran_order(), result, tag + " fortran_order"); - assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); - - auto expected_it = expected.begin(); - auto actual_it = actual.begin(); - for (std::size_t i = 0; i < expected.size(); ++i, ++expected_it, ++actual_it) - { - if (*expected_it != *actual_it) - { - result = EXIT_FAILURE; - std::wcout << std::wstring(tag.begin(), tag.end()) << " is incorrect: " << *actual_it << " != " << *expected_it << std::endl; - break; - } +template <> +inline void assert_equal>( + const npy::tensor &expected, + const npy::tensor &actual, int &result, + const std::string &tag) { + assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, + tag + " dtype"); + assert_equal(expected.fortran_order(), actual.fortran_order(), result, + tag + " fortran_order"); + assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); + + auto expected_it = expected.begin(); + auto actual_it = actual.begin(); + for (std::size_t i = 0; i < expected.size(); + ++i, ++expected_it, ++actual_it) { + if (*expected_it != *actual_it) { + result = EXIT_FAILURE; + std::wcout << std::wstring(tag.begin(), tag.end()) + << " is incorrect: " << *actual_it << " != " << *expected_it + << std::endl; + break; } + } } template -void assert_throws(void (*function)(), int &result, const std::string &tag) -{ - try - { - function(); - result = EXIT_FAILURE; - std::cout << tag << " did not throw an exception" << std::endl; - } - catch (EXCEPTION &) - { - } - catch (std::exception &e) - { - result = EXIT_FAILURE; - std::cout << tag << " threw unexpected exception: " << e.what() << std::endl; - } +void assert_throws(void (*function)(), int &result, const std::string &tag) { + try { + function(); + result = EXIT_FAILURE; + std::cout << tag << " did not throw an exception" << std::endl; + } catch (EXCEPTION &) { + } catch (std::exception &e) { + result = EXIT_FAILURE; + std::cout << tag << " threw unexpected exception: " << e.what() + << std::endl; + } } template -npy::tensor test_tensor(const std::vector &shape) -{ - npy::tensor tensor(shape); - std::vector values(tensor.size()); - auto curr = values.begin(); - for (int i = 0; curr < values.end(); ++i, ++curr) - { - *curr = static_cast(i); - } - tensor.copy_from(values); - - return tensor; +npy::tensor test_tensor(const std::vector &shape) { + npy::tensor tensor(shape); + std::vector values(tensor.size()); + auto curr = values.begin(); + for (int i = 0; curr < values.end(); ++i, ++curr) { + *curr = static_cast(i); + } + tensor.copy_from(values); + + return tensor; }; template <> -inline npy::tensor test_tensor(const std::vector &shape) -{ - npy::tensor tensor(shape); - int i=0; - for(auto& word : tensor) - { - word = std::to_wstring(i); - i += 1; - } - - return tensor; +inline npy::tensor test_tensor(const std::vector &shape) { + npy::tensor tensor(shape); + int i = 0; + for (auto &word : tensor) { + word = std::to_wstring(i); + i += 1; + } + + return tensor; } -template -npy::tensor test_fortran_tensor() -{ - std::vector values = { - 0, 10, 20, 30, 40, 5, 15, 25, 35, 45, - 1, 11, 21, 31, 41, 6, 16, 26, 36, 46, - 2, 12, 22, 32, 42, 7, 17, 27, 37, 47, - 3, 13, 23, 33, 43, 8, 18, 28, 38, 48, - 4, 14, 24, 34, 44, 9, 19, 29, 39, 49}; - npy::tensor tensor({5, 2, 5}, true); - auto dst = tensor.data(); - auto src = values.begin(); - for (; dst < tensor.data() + tensor.size(); ++src, ++dst) - { - *dst = static_cast(*src); - } - - return tensor; +template npy::tensor test_fortran_tensor() { + std::vector values = {0, 10, 20, 30, 40, 5, 15, 25, 35, 45, 1, 11, 21, + 31, 41, 6, 16, 26, 36, 46, 2, 12, 22, 32, 42, 7, + 17, 27, 37, 47, 3, 13, 23, 33, 43, 8, 18, 28, 38, + 48, 4, 14, 24, 34, 44, 9, 19, 29, 39, 49}; + npy::tensor tensor({5, 2, 5}, true); + auto dst = tensor.data(); + auto src = values.begin(); + for (; dst < tensor.data() + tensor.size(); ++src, ++dst) { + *dst = static_cast(*src); + } + + return tensor; } -template <> -inline npy::tensor test_fortran_tensor() -{ - std::vector values = { - 0, 10, 20, 30, 40, 5, 15, 25, 35, 45, - 1, 11, 21, 31, 41, 6, 16, 26, 36, 46, - 2, 12, 22, 32, 42, 7, 17, 27, 37, 47, - 3, 13, 23, 33, 43, 8, 18, 28, 38, 48, - 4, 14, 24, 34, 44, 9, 19, 29, 39, 49}; - npy::tensor tensor({5, 2, 5}, true); - auto dst = tensor.data(); - auto src = values.begin(); - for (; dst < tensor.data() + tensor.size(); ++src, ++dst) - { - *dst = std::to_wstring(*src); - } - - return tensor; +template <> inline npy::tensor test_fortran_tensor() { + std::vector values = {0, 10, 20, 30, 40, 5, 15, 25, 35, 45, 1, 11, 21, + 31, 41, 6, 16, 26, 36, 46, 2, 12, 22, 32, 42, 7, + 17, 27, 37, 47, 3, 13, 23, 33, 43, 8, 18, 28, 38, + 48, 4, 14, 24, 34, 44, 9, 19, 29, 39, 49}; + npy::tensor tensor({5, 2, 5}, true); + auto dst = tensor.data(); + auto src = values.begin(); + for (; dst < tensor.data() + tensor.size(); ++src, ++dst) { + *dst = std::to_wstring(*src); + } + + return tensor; } template -std::string npy_stream(npy::endian_t endianness = npy::endian_t::NATIVE) -{ - std::ostringstream actual_stream; - npy::tensor tensor = test_tensor({5, 2, 5}); - npy::save(actual_stream, tensor, endianness); - return actual_stream.str(); +std::string npy_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { + std::ostringstream actual_stream; + npy::tensor tensor = test_tensor({5, 2, 5}); + npy::save(actual_stream, tensor, endianness); + return actual_stream.str(); } template -std::string npy_scalar_stream(npy::endian_t endianness = npy::endian_t::NATIVE) -{ - std::ostringstream actual_stream; - npy::tensor tensor = test_tensor({}); - *tensor.data() = static_cast(42); - npy::save(actual_stream, tensor, endianness); - return actual_stream.str(); +std::string +npy_scalar_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { + std::ostringstream actual_stream; + npy::tensor tensor = test_tensor({}); + *tensor.data() = static_cast(42); + npy::save(actual_stream, tensor, endianness); + return actual_stream.str(); } template -std::string npy_array_stream(npy::endian_t endianness = npy::endian_t::NATIVE) -{ - std::ostringstream actual_stream; - npy::tensor tensor = test_tensor({25}); - npy::save(actual_stream, tensor, endianness); - return actual_stream.str(); +std::string npy_array_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { + std::ostringstream actual_stream; + npy::tensor tensor = test_tensor({25}); + npy::save(actual_stream, tensor, endianness); + return actual_stream.str(); } template -std::string npy_fortran_stream(npy::endian_t endianness = npy::endian_t::NATIVE) -{ - std::ostringstream actual_stream; - npy::tensor tensor = test_fortran_tensor(); - npy::save(actual_stream, tensor, endianness); - return actual_stream.str(); +std::string +npy_fortran_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { + std::ostringstream actual_stream; + npy::tensor tensor = test_fortran_tensor(); + npy::save(actual_stream, tensor, endianness); + return actual_stream.str(); } std::string read_file(const std::string &path); diff --git a/test/memstream.cpp b/test/memstream.cpp index 69a5c27..e042ec5 100644 --- a/test/memstream.cpp +++ b/test/memstream.cpp @@ -1,56 +1,54 @@ #include #include "libnpy_tests.h" -#include "zip.h" namespace { - const size_t SIZE = 50; +const size_t SIZE = 50; - void test_read(int& result) - { - std::vector expected(SIZE); - std::iota(expected.begin(), expected.end(), 0); - - npy::imemstream stream(expected); - std::vector actual(SIZE); - stream.read(actual.data(), SIZE); +void test_read(int &result) { + std::vector values(SIZE); + std::iota(values.begin(), values.end(), 0); + std::string expected(values.begin(), values.end()); - test::assert_equal(expected, actual, result, "memstream_test_copy_read"); + npy::imemstream stream(expected); + stream.read(values.data(), SIZE); + std::string actual(values.begin(), values.end()); - stream = npy::imemstream(std::move(expected)); - std::fill(actual.begin(), actual.end(), 0); - stream.read(actual.data(), SIZE); + test::assert_equal(expected, actual, result, "memstream_test_copy_read"); - expected = std::move(stream.buf()); + stream = npy::imemstream(std::move(expected)); + std::fill(actual.begin(), actual.end(), 0); + stream.read(actual.data(), SIZE); - test::assert_equal(expected, actual, result, "memstream_test_move_read"); - } + expected = std::move(stream.str()); - void test_write(int& result) - { - std::vector expected(SIZE); - std::iota(expected.begin(), expected.end(), 0); + test::assert_equal(expected, actual, result, "memstream_test_move_read"); +} + +void test_write(int &result) { + std::vector values(SIZE); + std::iota(values.begin(), values.end(), 0); + std::string expected(values.begin(), values.end()); - npy::omemstream stream; - stream.write(expected.data(), SIZE); + npy::omemstream stream; + stream.write(expected.data(), SIZE); - std::vector actual = stream.buf(); - test::assert_equal(expected, actual, result, "memstream_test_copy_write"); + std::string actual = stream.str(); + test::assert_equal(expected, actual, result, "memstream_test_copy_write"); - std::fill(actual.begin(), actual.end(), 0); - stream = npy::omemstream(std::move(actual)); - stream.write(expected.data(), SIZE); - actual = std::move(stream.buf()); + std::fill(actual.begin(), actual.end(), 0); + stream = npy::omemstream(std::move(actual)); + stream.write(expected.data(), SIZE); + actual = std::move(stream.str()); - test::assert_equal(expected, actual, result, "memstream_test_move_write"); - } + test::assert_equal(expected, actual, result, "memstream_test_move_write"); } +} // namespace -int test_memstream() -{ - int result = EXIT_SUCCESS; +int test_memstream() { + int result = EXIT_SUCCESS; - test_read(result); + test_read(result); - return result; + return result; } \ No newline at end of file diff --git a/test/npy_peek.cpp b/test/npy_peek.cpp index 0255f9f..f1c5672 100644 --- a/test/npy_peek.cpp +++ b/test/npy_peek.cpp @@ -1,36 +1,32 @@ #include "libnpy_tests.h" #include "npy/npy.h" -namespace -{ -void test_peek(int &result, - const std::string &tag, - npy::data_type_t data_type, +namespace { +void test_peek(int &result, const std::string &tag, npy::data_type_t data_type, npy::endian_t endianness = npy::endian_t::LITTLE, - bool fortran_order = false) -{ - npy::header_info expected = {data_type, endianness, fortran_order, {5, 2, 5}}; - npy::header_info actual = npy::peek(test::asset_path(tag + ".npy")); - test::assert_equal(expected, actual, result, tag); + bool fortran_order = false) { + npy::header_info expected = {data_type, endianness, fortran_order, {5, 2, 5}}; + npy::header_info actual = npy::peek(test::asset_path(tag + ".npy")); + test::assert_equal(expected, actual, result, tag); } } // namespace -int test_npy_peek() -{ - int result = EXIT_SUCCESS; +int test_npy_peek() { + int result = EXIT_SUCCESS; - test_peek(result, "uint8", npy::data_type_t::UINT8, npy::endian_t::NATIVE); - test_peek(result, "uint8_fortran", npy::data_type_t::UINT8, npy::endian_t::NATIVE, true); - test_peek(result, "int8", npy::data_type_t::INT8, npy::endian_t::NATIVE); - test_peek(result, "uint16", npy::data_type_t::UINT16); - test_peek(result, "int16", npy::data_type_t::INT16); - test_peek(result, "uint32", npy::data_type_t::UINT32); - test_peek(result, "int32", npy::data_type_t::INT32); - test_peek(result, "int32_big", npy::data_type_t::INT32, npy::endian_t::BIG); - test_peek(result, "uint64", npy::data_type_t::UINT64); - test_peek(result, "int64", npy::data_type_t::INT64); - test_peek(result, "float32", npy::data_type_t::FLOAT32); - test_peek(result, "float64", npy::data_type_t::FLOAT64); + test_peek(result, "uint8", npy::data_type_t::UINT8, npy::endian_t::NATIVE); + test_peek(result, "uint8_fortran", npy::data_type_t::UINT8, + npy::endian_t::NATIVE, true); + test_peek(result, "int8", npy::data_type_t::INT8, npy::endian_t::NATIVE); + test_peek(result, "uint16", npy::data_type_t::UINT16); + test_peek(result, "int16", npy::data_type_t::INT16); + test_peek(result, "uint32", npy::data_type_t::UINT32); + test_peek(result, "int32", npy::data_type_t::INT32); + test_peek(result, "int32_big", npy::data_type_t::INT32, npy::endian_t::BIG); + test_peek(result, "uint64", npy::data_type_t::UINT64); + test_peek(result, "int64", npy::data_type_t::INT64); + test_peek(result, "float32", npy::data_type_t::FLOAT32); + test_peek(result, "float64", npy::data_type_t::FLOAT64); - return result; + return result; } \ No newline at end of file diff --git a/test/npy_read.cpp b/test/npy_read.cpp index 66cdfef..183edc8 100644 --- a/test/npy_read.cpp +++ b/test/npy_read.cpp @@ -1,26 +1,24 @@ -#include "libnpy_tests.h" #include "npy_read.h" +#include "libnpy_tests.h" +int test_npy_read() { + int result = EXIT_SUCCESS; -int test_npy_read() -{ - int result = EXIT_SUCCESS; - - test_read(result, "uint8"); - test_read(result, "uint8_fortran", true); - test_read(result, "int8"); - test_read(result, "uint16"); - test_read(result, "int16"); - test_read(result, "uint32"); - test_read(result, "int32"); - test_read(result, "int32_big"); - test_read_scalar(result, "int32_scalar"); - test_read_array(result, "int32_array"); - test_read(result, "uint64"); - test_read(result, "int64"); - test_read(result, "float32"); - test_read(result, "float64"); - test_read(result, "unicode"); + test_read(result, "uint8"); + test_read(result, "uint8_fortran", true); + test_read(result, "int8"); + test_read(result, "uint16"); + test_read(result, "int16"); + test_read(result, "uint32"); + test_read(result, "int32"); + test_read(result, "int32_big"); + test_read_scalar(result, "int32_scalar"); + test_read_array(result, "int32_array"); + test_read(result, "uint64"); + test_read(result, "int64"); + test_read(result, "float32"); + test_read(result, "float64"); + test_read(result, "unicode"); - return result; + return result; } \ No newline at end of file diff --git a/test/npy_read.h b/test/npy_read.h index 5c2f129..8650620 100644 --- a/test/npy_read.h +++ b/test/npy_read.h @@ -2,37 +2,37 @@ #define _NPY_READ_H_ #include "libnpy_tests.h" -#include "npy/tensor.h" #include "npy/npy.h" +#include "npy/tensor.h" template -void test_read(int &result, const std::string &name, bool fortran_order = false) -{ - npy::tensor expected = test::test_tensor({5, 2, 5}); - if (fortran_order) - { - expected = test::test_fortran_tensor(); - } +void test_read(int &result, const std::string &name, + bool fortran_order = false) { + npy::tensor expected = test::test_tensor({5, 2, 5}); + if (fortran_order) { + expected = test::test_fortran_tensor(); + } - npy::tensor actual = npy::load(test::asset_path(name + ".npy")); - test::assert_equal(expected, actual, result, "npy_read_" + name); + npy::tensor actual = + npy::load(test::asset_path(name + ".npy")); + test::assert_equal(expected, actual, result, "npy_read_" + name); } template -void test_read_scalar(int &result, const std::string &name) -{ - npy::tensor expected = test::test_tensor({}); - *expected.data() = static_cast(42); - npy::tensor actual = npy::load(test::asset_path(name + ".npy")); - test::assert_equal(expected, actual, result, "npy_read_" + name); +void test_read_scalar(int &result, const std::string &name) { + npy::tensor expected = test::test_tensor({}); + *expected.data() = static_cast(42); + npy::tensor actual = + npy::load(test::asset_path(name + ".npy")); + test::assert_equal(expected, actual, result, "npy_read_" + name); } template -void test_read_array(int& result, const std::string &name) -{ - npy::tensor expected = test::test_tensor({25}); - npy::tensor actual = npy::load(test::asset_path(name + ".npy")); - test::assert_equal(expected, actual, result, "npy_read_" + name); +void test_read_array(int &result, const std::string &name) { + npy::tensor expected = test::test_tensor({25}); + npy::tensor actual = + npy::load(test::asset_path(name + ".npy")); + test::assert_equal(expected, actual, result, "npy_read_" + name); } #endif \ No newline at end of file diff --git a/test/npy_write.cpp b/test/npy_write.cpp index 8cfe01c..ddc2d75 100644 --- a/test/npy_write.cpp +++ b/test/npy_write.cpp @@ -2,74 +2,74 @@ #include "npy/npy.h" #include "npy/tensor.h" -int test_npy_write() -{ - int result = EXIT_SUCCESS; +int test_npy_write() { + int result = EXIT_SUCCESS; - std::string expected, actual; + std::string expected, actual; - npy::endian_t endianness = npy::native_endian(); - std::string dtype = npy::to_dtype(npy::data_type_t::FLOAT32, npy::endian_t::BIG); + npy::endian_t endianness = npy::native_endian(); + std::string dtype = + npy::to_dtype(npy::data_type_t::FLOAT32, npy::endian_t::BIG); - expected = test::read_asset("uint8.npy"); - actual = test::npy_stream(); - test::assert_equal(expected, actual, result, "npy_write_uint8"); + expected = test::read_asset("uint8.npy"); + actual = test::npy_stream(); + test::assert_equal(expected, actual, result, "npy_write_uint8"); - expected = test::read_asset("uint8_fortran.npy"); - actual = test::npy_fortran_stream(); - test::assert_equal(expected, actual, result, "npy_write_uint8_fortran"); + expected = test::read_asset("uint8_fortran.npy"); + actual = test::npy_fortran_stream(); + test::assert_equal(expected, actual, result, "npy_write_uint8_fortran"); - expected = test::read_asset("int8.npy"); - actual = test::npy_stream(); - test::assert_equal(expected, actual, result, "npy_write_int8"); + expected = test::read_asset("int8.npy"); + actual = test::npy_stream(); + test::assert_equal(expected, actual, result, "npy_write_int8"); - expected = test::read_asset("uint16.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_uint16"); + expected = test::read_asset("uint16.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_uint16"); - expected = test::read_asset("int16.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_int16"); + expected = test::read_asset("int16.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_int16"); - expected = test::read_asset("uint32.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_uint32"); + expected = test::read_asset("uint32.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_uint32"); - expected = test::read_asset("int32.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_int32"); + expected = test::read_asset("int32.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_int32"); - expected = test::read_asset("int32_big.npy"); - actual = test::npy_stream(npy::endian_t::BIG); - test::assert_equal(expected, actual, result, "npy_write_int32_big"); + expected = test::read_asset("int32_big.npy"); + actual = test::npy_stream(npy::endian_t::BIG); + test::assert_equal(expected, actual, result, "npy_write_int32_big"); - expected = test::read_asset("int32_scalar.npy"); - actual = test::npy_scalar_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_int32_scalar"); + expected = test::read_asset("int32_scalar.npy"); + actual = test::npy_scalar_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_int32_scalar"); - expected = test::read_asset("int32_array.npy"); - actual = test::npy_array_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_int32_array"); + expected = test::read_asset("int32_array.npy"); + actual = test::npy_array_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_int32_array"); - expected = test::read_asset("uint64.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_uint64"); + expected = test::read_asset("uint64.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_uint64"); - expected = test::read_asset("int64.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_int64"); + expected = test::read_asset("int64.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_int64"); - expected = test::read_asset("float32.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_float32"); + expected = test::read_asset("float32.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_float32"); - expected = test::read_asset("float64.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_float64"); + expected = test::read_asset("float64.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_float64"); - expected = test::read_asset("unicode.npy"); - actual = test::npy_stream(npy::endian_t::LITTLE); - test::assert_equal(expected, actual, result, "npy_write_unicode"); + expected = test::read_asset("unicode.npy"); + actual = test::npy_stream(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_unicode"); - return result; + return result; }; diff --git a/test/npz_peek.cpp b/test/npz_peek.cpp index a07f5dc..1facf93 100644 --- a/test/npz_peek.cpp +++ b/test/npz_peek.cpp @@ -1,39 +1,45 @@ #include "libnpy_tests.h" #include "npy/npz.h" -namespace -{ -void _test(int &result, const std::string &filename, bool compressed) -{ - npy::header_info expected_color(npy::data_type_t::UINT8, npy::endian_t::NATIVE, false, {5, 5, 3}); - npy::header_info expected_depth(npy::data_type_t::FLOAT32, npy::endian_t::LITTLE, false, {5, 5}); - - - npy::inpzstream stream(test::asset_path(filename)); - const auto& keys = stream.keys(); - - test::assert_equal(keys[0], std::string("color.npy"), result, "npz_keys_incorrect"); - test::assert_equal(keys[1], std::string("depth.npy"), result, "npz_keys_incorrect"); - test::assert_equal(keys[2], std::string("unicode.npy"), result, "npz_keys_incorrect"); - test::assert_equal(false, stream.contains("not_there.npy"), result, "npz_contains_missing"); - test::assert_equal(true, stream.contains("color.npy"), result, "npz_contains_color"); - test::assert_equal(true, stream.contains("depth.npy"), result, "npz_contains_depth"); - - npy::header_info actual_color = stream.peek("color.npy"); - npy::header_info actual_depth = stream.peek("depth.npy"); - - std::string suffix = compressed ? "_compressed" : ""; - test::assert_equal(expected_color, actual_color, result, "npz_peek_color" + suffix); - test::assert_equal(expected_depth, actual_depth, result, "npz_peek_depth" + suffix); +namespace { +void _test(int &result, const std::string &filename, bool compressed) { + npy::header_info expected_color(npy::data_type_t::UINT8, + npy::endian_t::NATIVE, false, {5, 5, 3}); + npy::header_info expected_depth(npy::data_type_t::FLOAT32, + npy::endian_t::LITTLE, false, {5, 5}); + + npy::inpzstream stream(test::asset_path(filename)); + const auto &keys = stream.keys(); + + test::assert_equal(keys[0], std::string("color.npy"), result, + "npz_keys_incorrect"); + test::assert_equal(keys[1], std::string("depth.npy"), result, + "npz_keys_incorrect"); + test::assert_equal(keys[2], std::string("unicode.npy"), result, + "npz_keys_incorrect"); + test::assert_equal(false, stream.contains("not_there.npy"), result, + "npz_contains_missing"); + test::assert_equal(true, stream.contains("color.npy"), result, + "npz_contains_color"); + test::assert_equal(true, stream.contains("depth.npy"), result, + "npz_contains_depth"); + + npy::header_info actual_color = stream.peek("color.npy"); + npy::header_info actual_depth = stream.peek("depth.npy"); + + std::string suffix = compressed ? "_compressed" : ""; + test::assert_equal(expected_color, actual_color, result, + "npz_peek_color" + suffix); + test::assert_equal(expected_depth, actual_depth, result, + "npz_peek_depth" + suffix); } } // namespace -int test_npz_peek() -{ - int result = EXIT_SUCCESS; +int test_npz_peek() { + int result = EXIT_SUCCESS; - _test(result, "test.npz", false); - _test(result, "test_compressed.npz", true); + _test(result, "test.npz", false); + _test(result, "test_compressed.npz", true); - return result; + return result; } \ No newline at end of file diff --git a/test/npz_read.cpp b/test/npz_read.cpp index cc15783..0256f57 100644 --- a/test/npz_read.cpp +++ b/test/npz_read.cpp @@ -1,32 +1,75 @@ #include "libnpy_tests.h" +#include "npy/core.h" #include "npy/npz.h" -namespace -{ -void _test(int &result, const std::string &filename, bool compressed) -{ - auto expected_color = test::test_tensor({5, 5, 3}); - auto expected_depth = test::test_tensor({5, 5}); - auto expected_unicode = test::test_tensor({5, 2, 5}); - - npy::inpzstream stream(test::asset_path(filename)); - auto actual_color = stream.read("color.npy"); - auto actual_depth = stream.read("depth"); - auto actual_unicode = stream.read("unicode"); - - std::string suffix = compressed ? "_compressed" : ""; - test::assert_equal(expected_color, actual_color, result, "npz_read_color" + suffix); - test::assert_equal(expected_depth, actual_depth, result, "npz_read_depth" + suffix); - test::assert_equal(expected_unicode, actual_unicode, result, "npz_read_unicode" + suffix); +namespace { +void _test(int &result, const std::string &filename, bool compressed) { + auto expected_color = test::test_tensor({5, 5, 3}); + auto expected_depth = test::test_tensor({5, 5}); + auto expected_unicode = test::test_tensor({5, 2, 5}); + + npy::inpzstream stream(test::asset_path(filename)); + auto actual_color = stream.read("color.npy"); + auto actual_depth = stream.read("depth"); + auto actual_unicode = stream.read("unicode"); + + std::string suffix = compressed ? "_compressed" : ""; + test::assert_equal(expected_color, actual_color, result, + "npz_read_color" + suffix); + test::assert_equal(expected_depth, actual_depth, result, + "npz_read_depth" + suffix); + test::assert_equal(expected_unicode, actual_unicode, result, + "npz_read_unicode" + suffix); +} + +void _test_large(int &result, const std::string &filename, bool compressed) { + auto expected_int = test::test_tensor({200, 5, 1000}); + auto expected_float = test::test_tensor({1000, 5, 20, 10}); + + npy::inpzstream stream(test::asset_path(filename)); + auto actual_int = stream.read("test_int"); + auto actual_float = stream.read("test_float"); + + std::string suffix = compressed ? "_compressed" : ""; + test::assert_equal(expected_int, actual_int, result, + "npz_read_large_int" + suffix); + test::assert_equal(expected_float, actual_float, result, + "npz_read_large_float" + suffix); +} + +void _test_memory(int &result, const std::string &filename) { + std::ifstream input(test::asset_path(filename), + std::ios::in | std::ios::binary); + std::string contents((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + auto memory = std::make_shared(contents); + + auto expected_color = test::test_tensor({5, 5, 3}); + auto expected_depth = test::test_tensor({5, 5}); + auto expected_unicode = test::test_tensor({5, 2, 5}); + + npy::inpzstream stream(memory); + auto actual_color = stream.read("color.npy"); + auto actual_depth = stream.read("depth"); + auto actual_unicode = stream.read("unicode"); + + test::assert_equal(expected_color, actual_color, result, + "npz_read_color_memory"); + test::assert_equal(expected_depth, actual_depth, result, + "npz_read_depth_memory"); + test::assert_equal(expected_unicode, actual_unicode, result, + "npz_read_unicode_memory"); } } // namespace -int test_npz_read() -{ - int result = EXIT_SUCCESS; +int test_npz_read() { + int result = EXIT_SUCCESS; - _test(result, "test.npz", false); - _test(result, "test_compressed.npz", true); + _test(result, "test.npz", false); + _test(result, "test_compressed.npz", true); + _test_large(result, "test_large.npz", false); + _test_large(result, "test_large_compressed.npz", true); + _test_memory(result, "test.npz"); - return result; + return result; } \ No newline at end of file diff --git a/test/npz_write.cpp b/test/npz_write.cpp index 75f6d3c..cfc8760 100644 --- a/test/npz_write.cpp +++ b/test/npz_write.cpp @@ -1,51 +1,65 @@ -#include -#include -#include #include #include +#include +#include #include "libnpy_tests.h" +#include "npy/core.h" #include "npy/npz.h" -namespace -{ +namespace { const char *TEMP_NPZ = "temp.npz"; } -namespace -{ -void _test(int &result, npy::compression_method_t compression_method) -{ - std::string asset_name = "test.npz"; - std::string suffix = ""; - if (compression_method == npy::compression_method_t::DEFLATED) - { - asset_name = "test_compressed.npz"; - suffix = "_compressed"; - } - - std::string expected = test::read_asset(asset_name); - - { - npy::onpzstream npz(TEMP_NPZ, compression_method, npy::endian_t::LITTLE); - npz.write("color", test::test_tensor({5, 5, 3})); - npz.write("depth.npy", test::test_tensor({5, 5})); - npz.write("unicode.npy", test::test_tensor({5, 2, 5})); - } - - std::string actual = test::read_file(TEMP_NPZ); - test::assert_equal(expected, actual, result, "npz_write" + suffix); - - std::remove(TEMP_NPZ); +namespace { +void _test(int &result, npy::compression_method_t compression_method) { + std::string asset_name = "test.npz"; + std::string suffix = ""; + if (compression_method == npy::compression_method_t::DEFLATED) { + asset_name = "test_compressed.npz"; + suffix = "_compressed"; + } + + std::string expected = test::read_asset(asset_name); + + { + npy::onpzstream npz(TEMP_NPZ, compression_method, npy::endian_t::LITTLE); + npz.write("color", test::test_tensor({5, 5, 3})); + npz.write("depth.npy", test::test_tensor({5, 5})); + npz.write("unicode.npy", test::test_tensor({5, 2, 5})); + } + + std::string actual = test::read_file(TEMP_NPZ); + test::assert_equal(expected, actual, result, "npz_write" + suffix); + + std::remove(TEMP_NPZ); +} + +void _test_memory(int &result) { + std::string asset_name = "test.npz"; + + std::string expected = test::read_asset(asset_name); + auto memory = std::make_shared(); + + { + npy::onpzstream npz(memory, npy::compression_method_t::STORED, + npy::endian_t::LITTLE); + npz.write("color", test::test_tensor({5, 5, 3})); + npz.write("depth.npy", test::test_tensor({5, 5})); + npz.write("unicode.npy", test::test_tensor({5, 2, 5})); + } + + std::string actual = memory->str(); + test::assert_equal(expected, actual, result, "npz_write_memory"); } } // namespace -int test_npz_write() -{ - int result = EXIT_SUCCESS; +int test_npz_write() { + int result = EXIT_SUCCESS; - _test(result, npy::compression_method_t::STORED); - _test(result, npy::compression_method_t::DEFLATED); + _test(result, npy::compression_method_t::STORED); + _test(result, npy::compression_method_t::DEFLATED); + _test_memory(result); - return result; + return result; } \ No newline at end of file diff --git a/test/tensor.cpp b/test/tensor.cpp index 8351c23..a03fb59 100644 --- a/test/tensor.cpp +++ b/test/tensor.cpp @@ -4,52 +4,42 @@ #include "libnpy_tests.h" #include "npy/tensor.h" -namespace -{ +namespace { const char *TEMP_NPY = "temp.npy"; } -int test_tensor() -{ - int result = EXIT_SUCCESS; - - npy::tensor fortran({3, 4, 5}, true); - std::uint8_t value = 0; - for (auto i = 0; i < 3; ++i) - { - for (auto j = 0; j < 4; ++j) - { - for (auto k = 0; k < 5; ++k, ++value) - { - fortran(i, j, k) = value; - fortran({static_cast(i), - static_cast(j), - static_cast(k)}) = value; - } - } +int test_tensor() { + int result = EXIT_SUCCESS; + + npy::tensor fortran({3, 4, 5}, true); + std::uint8_t value = 0; + for (auto i = 0; i < 3; ++i) { + for (auto j = 0; j < 4; ++j) { + for (auto k = 0; k < 5; ++k, ++value) { + fortran(i, j, k) = value; + fortran({static_cast(i), static_cast(j), + static_cast(k)}) = value; + } } + } - fortran.save(TEMP_NPY); - - npy::tensor from_file(TEMP_NPY); - npy::tensor standard(from_file.shape(), false); - for (int i = 0; i < 3; ++i) - { - for (int j = 0; j < 4; ++j) - { - for (int k = 0; k < 5; ++k) - { - standard(i, j, k) = fortran(i, j, k); - } - } - } + fortran.save(TEMP_NPY); - for (std::uint8_t i = 0; i < 60; ++i) - { - test::assert_equal(i, standard.values()[i], result, "tensor read/write"); + npy::tensor from_file(TEMP_NPY); + npy::tensor standard(from_file.shape(), false); + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 4; ++j) { + for (int k = 0; k < 5; ++k) { + standard(i, j, k) = fortran(i, j, k); + } } + } + + for (std::uint8_t i = 0; i < 60; ++i) { + test::assert_equal(i, standard.values()[i], result, "tensor read/write"); + } - std::remove(TEMP_NPY); + std::remove(TEMP_NPY); - return result; + return result; };