Skip to content

Commit

Permalink
Update compression libs versions (#156)
Browse files Browse the repository at this point in the history
  • Loading branch information
byrnHDF authored Nov 18, 2024
1 parent 8864b05 commit c1e11d3
Show file tree
Hide file tree
Showing 28 changed files with 608 additions and 110 deletions.
178 changes: 178 additions & 0 deletions .github/workflows/check-releases.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
name: hdf5 plugins check compression releases

# Triggers the workflow on a call from another workflow
on:
workflow_call:

jobs:
check-compression-releases:
runs-on: ubuntu-latest
steps:
- name: Get bitshuffle release version
id: get-bitshuffle-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'kiyo-masui/bitshuffle'
latest: true

- name: Report bitshuffle release version
run: |
echo "BSHUF=${{ steps.get-bitshuffle-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get blosc release version
id: get-blosc-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'Blosc/c-blosc'
latest: true

- name: Report blosc release version
run: |
echo "BLOSC=${{ steps.get-blosc-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get blosc2 release version
id: get-blosc2-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'Blosc/c-blosc2'
latest: true

- name: Report blosc2 release version
run: |
echo "BLOSC2=${{ steps.get-blosc2-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
# - name: Get bzip2 release version
# id: get-bzip2-release
# uses: cardinalby/git-get-release-action@v1
# env:
# GITHUB_TOKEN: ${{ github.token }}
# with:
# repo: 'libarchive/bzip2'
# latest: true
#
# - name: Report bzip2 release version
# run: |
# echo "BZIP2=bzip2: ${{ steps.get-bzip2-release.outputs.tag_name }}" >> $GITHUB_OUTPUT

- name: Get fpzip release version
id: get-fpzip-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'LLNL/fpzip'
latest: true

- name: Report fpzip release version
run: |
echo "FPZIP=${{ steps.get-fpzip-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get libjpeg release version
id: get-libjpeg-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'libjpeg-turbo/libjpeg-turbo'
latest: true

- name: Report libjpeg release version
run: |
echo "JPEG=${{ steps.get-libjpeg-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get lz4 release version
id: get-lz4-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'lz4/lz4'
latest: true

- name: Report lz4 release version
run: |
echo "LZ4=${{ steps.get-lz4-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get zfp release version
id: get-zfp-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'LLNL/zfp'
latest: true

- name: Report zfp release version
run: |
echo "ZFP=${{ steps.get-zfp-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get zstd release version
id: get-zstd-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'facebook/zstd'
latest: true

- name: Report zstd release version
run: |
echo "ZSTD=${{ steps.get-zstd-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get zlib release version
id: get-zlib-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'madler/zlib'
latest: true

- name: Report zlib release version
run: |
echo "ZLIB=${{ steps.get-zlib-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: Get zlib-ng release version
id: get-zlib-ng-release
uses: cardinalby/git-get-release-action@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repo: 'zlib-ng/zlib-ng'
latest: true

- name: Report zlib-ng release version
run: |
echo "ZLIBNG=${{ steps.get-zlib-ng-release.outputs.tag_name }}" >> $GITHUB_OUTPUT
- name: create-json
id: create-json
uses: jsdaniell/[email protected]
with:
name: "versions-file.json"
json: '{"bitshuffle":"${{ steps.get-bitshuffle-release.outputs.tag_name }}"",
"blosc":"${{ steps.get-blosc-release.outputs.tag_name }}"
"blosc2":"${{ steps.get-blosc2-release.outputs.tag_name }}"
"fpzip":"${{ steps.get-fpzip-release.outputs.tag_name }}"
"libjpeg":"${{ steps.get-libjpeg-release.outputs.tag_name }}"
"lz4":"${{ steps.get-lz4-release.outputs.tag_name }}"
"zfp":"${{ steps.get-zfp-release.outputs.tag_name }}"
"zstd":"${{ steps.get-zstd-release.outputs.tag_name }}"
"zlib":"${{ steps.get-zlib-release.outputs.tag_name }}"
"zlib-ng":"${{ steps.get-zlib-ng-release.outputs.tag_name }}"
}'

# Save json report
- name: Save json report
uses: actions/upload-artifact@v4
with:
name: compression-versions
path: versions-file.json
if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`

3 changes: 3 additions & 0 deletions .github/workflows/daily-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,3 +121,6 @@ jobs:
use_environ: snapshots
if: ${{ ((needs.call-workflow-tarball.outputs.has_changes == 'true') || (needs.get-base-names.outputs.run-ignore == 'ignore')) && (needs.get-old-names.outputs.hdf5-name != needs.call-workflow-tarball.outputs.file_base) }}

call-workflow-versions:
uses: ./.github/workflows/check-releases.yml

2 changes: 1 addition & 1 deletion BLOSC/config/cmake/cacheinit.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ set (BLOSC_GIT_URL "https://github.com/Blosc/c-blosc.git" CACHE STRING "Use BLOS
set (BLOSC_GIT_BRANCH "master" CACHE STRING "" FORCE)

set (BLOSC_TGZ_ORIGPATH "https://github.com/Blosc/c-blosc/archive/refs/tags" CACHE STRING "Use PLUGINS from original location" FORCE)
set (BLOSC_TGZ_NAME "c-blosc-1.21.5.tar.gz" CACHE STRING "Use BLOSC from compressed file" FORCE)
set (BLOSC_TGZ_NAME "c-blosc-1.21.6.tar.gz" CACHE STRING "Use BLOSC from compressed file" FORCE)

set (BLOSC_PACKAGE_NAME "blosc" CACHE STRING "Name of BLOSC package" FORCE)
set (H5BLOSC_CPACK_ENABLE ON CACHE BOOL "Enable the CPACK include and components" FORCE)
Expand Down
2 changes: 1 addition & 1 deletion BLOSC2/config/cmake/cacheinit.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ set (BLOSC2_GIT_URL "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Use BL
set (BLOSC2_GIT_BRANCH "master" CACHE STRING "" FORCE)

set (BLOSC2_TGZ_ORIGPATH "https://github.com/Blosc/c-blosc2/archive/refs/tags" CACHE STRING "Use PLUGINS from original location" FORCE)
set (BLOSC2_TGZ_NAME "c-blosc2-1.21.5.tar.gz" CACHE STRING "Use BLOSC2 from compressed file" FORCE)
set (BLOSC2_TGZ_NAME "c-blosc2-2.15.1.tar.gz" CACHE STRING "Use BLOSC2 from compressed file" FORCE)

set (BLOSC2_PACKAGE_NAME "blosc2" CACHE STRING "Name of BLOSC2 package" FORCE)
set (H5BLOSC2_CPACK_ENABLE ON CACHE BOOL "Enable the CPACK include and components" FORCE)
Expand Down
4 changes: 2 additions & 2 deletions BSHUF/config/cmake/cacheinit.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ set (BUILD_BSHUF_LZ4_LIBRARY_SOURCE ON CACHE BOOL "build the lz4 library within
set (LZ4_GIT_URL "https://github.com/lz4/lz4.git" CACHE STRING "Use LZ4 from GitHub repository" FORCE)
set (LZ4_GIT_BRANCH "dev" CACHE STRING "" FORCE)

set (LZ4_TGZ_ORIGPATH "https://github.com/lz4/lz4/releases/download/v1.9.4" CACHE STRING "Use PLUGINS from original location" FORCE)
set (LZ4_TGZ_NAME "lz4-1.9.4.tar.gz" CACHE STRING "Use LZ4 from compressed file" FORCE)
set (LZ4_TGZ_ORIGPATH "https://github.com/lz4/lz4/releases/download/v1.10.0" CACHE STRING "Use PLUGINS from original location" FORCE)
set (LZ4_TGZ_NAME "lz4-1.10.0.tar.gz" CACHE STRING "Use LZ4 from compressed file" FORCE)

set (LZ4_PACKAGE_NAME "lz4" CACHE STRING "Name of LZ4 package" FORCE)

Expand Down
8 changes: 4 additions & 4 deletions CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@
"BUILD_BSHUF_LZ4_LIBRARY_SOURCE": "ON",
"BSHUF_TGZ_NAME": {"type": "STRING", "value": "bitshuffle-0.5.1.tar.gz"},
"BSHUF_PACKAGE_NAME": {"type": "STRING", "value": "bshuf"},
"BLOSC_TGZ_NAME": {"type": "STRING", "value": "c-blosc-1.21.5.tar.gz"},
"BLOSC_TGZ_NAME": {"type": "STRING", "value": "c-blosc-1.21.6.tar.gz"},
"BLOSC_PACKAGE_NAME": {"type": "STRING", "value": "blosc"},
"BLOSC_ZLIB_TGZ_NAME": {"type": "STRING", "value": "zlib-1.3.tar.gz"},
"BLOSC_ZLIB_PACKAGE_NAME": {"type": "STRING", "value": "zlib"},
"BLOSC2_TGZ_NAME": {"type": "STRING", "value": "c-blosc2-2.14.4.tar.gz"},
"BLOSC2_TGZ_NAME": {"type": "STRING", "value": "c-blosc2-2.15.1.tar.gz"},
"BLOSC2_PACKAGE_NAME": {"type": "STRING", "value": "blosc2"},
"BLOSC2_ZLIB_TGZ_NAME": {"type": "STRING", "value": "zlib-1.3.tar.gz"},
"BLOSC2_ZLIB_PACKAGE_NAME": {"type": "STRING", "value": "zlib"},
Expand All @@ -45,15 +45,15 @@
"JPEG_TGZ_NAME": {"type": "STRING", "value": "jpegsrc.v9e.tar.gz"},
"JPEG_PACKAGE_NAME": {"type": "STRING", "value": "jpeg"},
"BUILD_LZ4_LIBRARY_SOURCE": "ON",
"LZ4_TGZ_NAME": {"type": "STRING", "value": "lz4-1.9.4.tar.gz"},
"LZ4_TGZ_NAME": {"type": "STRING", "value": "lz4-1.10.0.tar.gz"},
"LZ4_PACKAGE_NAME": {"type": "STRING", "value": "lz4"},
"LZF_TGZ_NAME": {"type": "STRING", "value": "liblzf-3.6.tar.gz"},
"LZF_PACKAGE_NAME": {"type": "STRING", "value": "lzf"},
"SZ_TGZ_NAME": {"type": "STRING", "value": "SZ-2.1.12.5.tar.gz"},
"SZ_PACKAGE_NAME": {"type": "STRING", "value": "SZ"},
"ZFP_TGZ_NAME": {"type": "STRING", "value": "zfp-1.0.0.tar.gz"},
"ZFP_PACKAGE_NAME": {"type": "STRING", "value": "zfp"},
"ZSTD_TGZ_NAME": {"type": "STRING", "value": "zstd-1.5.5.tar.gz"},
"ZSTD_TGZ_NAME": {"type": "STRING", "value": "zstd-1.5.6.tar.gz"},
"ZSTD_PACKAGE_NAME": {"type": "STRING", "value": "zstd"}
}
},
Expand Down
56 changes: 26 additions & 30 deletions LZ4/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -83,36 +83,33 @@ message (STATUS "H5LZ4 link libs: ${H5PL_LINK_LIBS}")
#-----------------------------------------------------------------------------
configure_file (${H5LZ4_RESOURCES_DIR}/config.h.in ${H5LZ4_BINARY_DIR}/lz4_config.h @ONLY)

INCLUDE_DIRECTORIES (${HDF5_INCLUDE_DIR})

option (BUILD_LZ4_LIBRARY_SOURCE "build the lz4 library within the plugin" ON)
if (NOT BUILD_LZ4_LIBRARY_SOURCE)
include (ExternalProject)
set (H5PL_ALLOW_EXTERNAL_SUPPORT "NO" CACHE STRING "Allow External Library Building (NO GIT TGZ)")
set_property (CACHE H5PL_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO GIT TGZ)
if (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ")
option (LZ4_USE_EXTERNAL "Use External Library Building for LZ4" 1)
if (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT")
set (LZ4_URL ${LZ4_GIT_URL} CACHE STRING "Path to lz4 git repository")
set (LZ4_BRANCH ${LZ4_GIT_BRANCH})
elseif (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ")
if (NOT H5PL_COMP_TGZPATH)
set (H5PL_COMP_TGZPATH ${H5LZ4_SOURCE_DIR})
endif ()
set (LZ4_URL ${H5PL_COMP_TGZPATH}/${LZ4_TGZ_NAME})
else ()
set (LZ4_USE_EXTERNAL 0)
include (ExternalProject)
set (H5PL_ALLOW_EXTERNAL_SUPPORT "NO" CACHE STRING "Allow External Library Building (NO GIT TGZ)")
set_property (CACHE H5PL_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO GIT TGZ)
if (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ")
option (LZ4_USE_EXTERNAL "Use External Library Building for LZ4" 1)
if (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT")
set (LZ4_URL ${LZ4_GIT_URL} CACHE STRING "Path to lz4 git repository")
set (LZ4_BRANCH ${LZ4_GIT_BRANCH})
elseif (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ")
if (NOT H5PL_COMP_TGZPATH)
set (H5PL_COMP_TGZPATH ${H5LZ4_SOURCE_DIR})
endif ()
set (LZ4_URL ${H5PL_COMP_TGZPATH}/${LZ4_TGZ_NAME})
else ()
set (LZ4_USE_EXTERNAL 0)
endif ()
#-----------------------------------------------------------------------------
# lz4 Library Settings
#-----------------------------------------------------------------------------
endif ()
#-----------------------------------------------------------------------------
# lz4 Library Settings
#-----------------------------------------------------------------------------
if (NOT H5_LZ4_HEADER)
if (NOT LZ4_USE_EXTERNAL)
find_package (LZ4 NAMES ${LZ4_PACKAGE_NAME})
if (NOT LZ4_FOUND)
find_package (LZ4) # Legacy find
if (NOT LZ4_FOUND)
message (FATAL_ERROR " lz4 is Required for ${H5LZ4_PACKAGE_NAME} library")
if (LZ4_FOUND)
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${LZ4_LIBRARIES})
endif ()
endif ()
endif ()
Expand All @@ -132,13 +129,12 @@ if (NOT BUILD_LZ4_LIBRARY_SOURCE)
message (FATAL_ERROR " LZ4 is Required for ${H5LZ4_PACKAGE_NAME} library")
endif ()
endif ()
message (STATUS "lz4 found: INC=${LZ4_INCLUDE_DIR} TOOLS=${LZ4_LIBRARIES}")
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${LZ4_LIBRARIES})

if (LZ4_FOUND)
PACKAGE_LZ4_LIBRARY (${H5PL_ALLOW_EXTERNAL_SUPPORT})
endif ()
else ()
# This project is being called from within another and LZ4 is already configured
set (H5_HAVE_LZ4_H 1)
endif ()
message (STATUS "lz4 found: INC=${LZ4_INCLUDE_DIR} TOOLS=${LZ4_LIBRARIES}")
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${LZ4_LIBRARIES})
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
Expand Down
Loading

0 comments on commit c1e11d3

Please sign in to comment.