try zst

anndata-hdf5/Cargo.toml

@@ -13,7 +13,8 @@ homepage = "https://github.com/kaizhang/anndata-rs"
 [dependencies]
 anndata = { workspace = true }
 anyhow = "1.0"
-hdf5 = { package = "hdf5-metno", version = "0.9", features = ["lzf"] }
+hdf5 = { package = "hdf5-metno", version = "0.9", features = ["blosc"] }
+blosc-src = { version = "0.3.0", features = ["zstd"] }
 hdf5-sys = { package = "hdf5-metno-sys", version = "0.10", features = ["static", "zlib", "threadsafe"] }
 libz-sys = { version = "1", features = ["libc"], default-features = false }
 ndarray = { version = "0.16" }

anndata-hdf5/src/lib.rs

@@ -126,9 +126,9 @@ fn new_dataset<T: BackendData>(
     builder = if let Some(compression) = config.compression {
         match compression {
             Compression::Gzip(lvl) => builder.deflate(lvl),
-            Compression::Lzf => match dtype {
-                ScalarType::String => builder.deflate(1),
-                _ => builder.lzf(),
+            Compression::Zst(lvl) => match dtype {
+                ScalarType::String => builder.deflate(3),
+                _ => builder.blosc_zstd(lvl, hdf5::filters::BloscShuffle::Byte),
             }
         }
     } else {

anndata/src/backend.rs

@@ -12,7 +12,7 @@ use serde::Deserialize;
 #[derive(Debug, Copy, Clone)]
 pub enum Compression {
     Gzip(u8),
-    Lzf,
+    Zst(u8),
 }
 
 #[derive(Debug, Clone)]
@@ -24,7 +24,7 @@ pub struct WriteConfig {
 impl Default for WriteConfig {
     fn default() -> Self {
         Self {
-            compression: Some(Compression::Lzf),
+            compression: Some(Compression::Zst(3)),
             block_size: None,
         }
     }

python/pyproject.toml

@@ -26,6 +26,7 @@ dynamic = ["version"]
 
 dependencies = [
     'anndata >= 0.8.0, < 0.11.0',
+    'hdf5plugin >= 5.0.0',
     "numpy>=1.16",
     "pandas",
     "pyarrow",

python/tests/test_concat.py

@@ -1,6 +1,7 @@
 from anndata_rs import AnnData, read, concat
 
 import pytest
+import hdf5plugin
 import anndata as ad
 import numpy as np
 import pandas as pd