Skip to content

Commit

Permalink
Add zlib-ng
Browse files Browse the repository at this point in the history
  • Loading branch information
data-man committed Dec 18, 2021
1 parent 1e55f37 commit e9ec44f
Show file tree
Hide file tree
Showing 104 changed files with 20,361 additions and 8 deletions.
56 changes: 49 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ else
endif


DEFINES += $(addprefix -I$(SOURCE_PATH),. zstd/lib zstd/lib/common brotli/include xpack/common libcsc xz xz/api xz/check xz/common xz/lz xz/lzma xz/rangecoder)
DEFINES += $(addprefix -I$(SOURCE_PATH),. zstd/lib zstd/lib/common brotli/include xpack/common libcsc xz xz/api xz/check xz/common xz/lz xz/lzma xz/rangecoder zlib-ng zlib-ng/arch)
DEFINES += -DHAVE_CONFIG_H -DFL2_SINGLETHREAD
CODE_FLAGS += -Wno-unknown-pragmas -Wno-sign-compare -Wno-conversion
OPT_FLAGS ?= -fomit-frame-pointer -fstrict-aliasing -ffast-math
Expand Down Expand Up @@ -106,9 +106,45 @@ UCL_FILES = ucl/alloc.o ucl/n2b_99.o ucl/n2b_d.o ucl/n2b_ds.o ucl/n2b_to.o ucl/n
UCL_FILES += ucl/n2d_to.o ucl/n2e_99.o ucl/n2e_d.o ucl/n2e_ds.o ucl/n2e_to.o ucl/ucl_crc.o ucl/ucl_init.o
UCL_FILES += ucl/ucl_ptr.o ucl/ucl_str.o ucl/ucl_util.o

ZLIB_FILES = zlib/adler32.o zlib/compress.o zlib/crc32.o zlib/deflate.o zlib/gzclose.o zlib/gzlib.o zlib/gzread.o
ZLIB_FILES += zlib/gzwrite.o zlib/infback.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o zlib/trees.o
ZLIB_FILES += zlib/uncompr.o zlib/zutil.o
ifeq "$(DONT_BUILD_ZLIB)" "1"
DEFINES += -DBENCH_REMOVE_ZLIB
DONT_BUILD_SLZ = 1
else
ZLIB_FILES = zlib/adler32.o zlib/compress.o zlib/crc32.o zlib/deflate.o zlib/gzclose.o zlib/gzlib.o zlib/gzread.o
ZLIB_FILES += zlib/gzwrite.o zlib/infback.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o zlib/trees.o
ZLIB_FILES += zlib/uncompr.o zlib/zutil.o
endif

ifeq "$(DONT_BUILD_ZLIB_NG)" "1"
DEFINES += -DBENCH_REMOVE_ZLIB_NG
else
ZLIB_NG_FILES += zlib-ng/adler32.o zlib-ng/chunkset.o zlib-ng/compare258.o zlib-ng/compress.o zlib-ng/crc32.o zlib-ng/crc32_comb.o zlib-ng/crc32_fold.o
ZLIB_NG_FILES += zlib-ng/deflate.o zlib-ng/deflate_fast.o zlib-ng/deflate_huff.o zlib-ng/deflate_medium.o zlib-ng/deflate_quick.o
ZLIB_NG_FILES += zlib-ng/deflate_rle.o zlib-ng/deflate_slow.o zlib-ng/deflate_stored.o zlib-ng/functable.o
# ZLIB_NG_FILES += zlib-ng/gzlib.o zlib-ng/gzread.o zlib-ng/gzwrite.o
ZLIB_NG_FILES += zlib-ng/infback.o zlib-ng/inffast.o zlib-ng/inflate.o zlib-ng/inftrees.o zlib-ng/insert_string.o zlib-ng/insert_string_roll.o
ZLIB_NG_FILES += zlib-ng/slide_hash.o zlib-ng/trees.o zlib-ng/uncompr.o zlib-ng/zutil.o

ifeq ($(shell uname -p),powerpc)
ZLIB_NG_FILES += zlib-ng/arch/power/adler32_power8.o
ZLIB_NG_FILES += zlib-ng/arch/power/power.o
ZLIB_NG_FILES += zlib-ng/arch/power/slide_hash_power8.o
endif

ifeq ($(shell uname -m),x86_64)
ZLIB_NG_FILES += zlib-ng/arch/x86/adler32_avx.o
ZLIB_NG_FILES += zlib-ng/arch/x86/adler32_ssse3.o
ZLIB_NG_FILES += zlib-ng/arch/x86/chunkset_avx.o
ZLIB_NG_FILES += zlib-ng/arch/x86/chunkset_sse.o
ZLIB_NG_FILES += zlib-ng/arch/x86/compare258_avx.o
ZLIB_NG_FILES += zlib-ng/arch/x86/compare258_sse.o
ZLIB_NG_FILES += zlib-ng/arch/x86/crc32_fold_pclmulqdq.o
ZLIB_NG_FILES += zlib-ng/arch/x86/insert_string_sse.o
# ZLIB_NG_FILES += zlib-ng/arch/x86/slide_hash_avx.o
ZLIB_NG_FILES += zlib-ng/arch/x86/slide_hash_sse.o
ZLIB_NG_FILES += zlib-ng/arch/x86/x86.o
endif
endif

LZMAT_FILES = lzmat/lzmat_dec.o lzmat/lzmat_enc.o

Expand Down Expand Up @@ -176,10 +212,16 @@ LIBDEFLATE_FILES += libdeflate/deflate_decompress.o libdeflate/gzip_compress.o l
LIBDEFLATE_FILES += libdeflate/x86/cpu_features.o libdeflate/arm/cpu_features.o libdeflate/zlib_compress.o libdeflate/zlib_decompress.o

MISC_FILES = crush/crush.o shrinker/shrinker.o fastlz/fastlz.o pithy/pithy.o lzjb/lzjb2010.o wflz/wfLZ.o
MISC_FILES += lzlib/lzlib.o blosclz/blosclz.o blosclz/fastcopy.o slz/slz.o
MISC_FILES += lzlib/lzlib.o blosclz/blosclz.o blosclz/fastcopy.o

LZBENCH_FILES = _lzbench/lzbench.o _lzbench/compressors.o _lzbench/csc_codec.o

ifeq "$(DONT_BUILD_SLZ)" "1"
DEFINES += -DBENCH_REMOVE_SLZ
else
MISC_FILES += slz/slz.o
endif

ifeq "$(DONT_BUILD_BZIP2)" "1"
DEFINES += -DBENCH_REMOVE_BZIP2
else
Expand Down Expand Up @@ -352,7 +394,7 @@ $(NVCOMP_CPP_OBJ): %.cpp.o: %.cpp

_lzbench/lzbench.o: _lzbench/lzbench.cpp _lzbench/lzbench.h

lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) $(NVCOMP_FILES) $(LZBENCH_FILES)
lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(ZLIB_NG_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) $(NVCOMP_FILES) $(LZBENCH_FILES)
$(CXX) $^ -o $@ $(LDFLAGS)
@echo Linked GCC_VERSION=$(GCC_VERSION) CLANG_VERSION=$(CLANG_VERSION) COMPILER=$(COMPILER)

Expand All @@ -369,4 +411,4 @@ lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_F
$(CXX) $(CFLAGS) $< -c -o $@

clean:
rm -rf lzbench lzbench.exe *.o _lzbench/*.o bzip2/*.o fast-lzma2/*.o slz/*.o zstd/lib/*.o zstd/lib/*.a zstd/lib/common/*.o zstd/lib/compress/*.o zstd/lib/decompress/*.o zstd/lib/dictBuilder/*.o lzsse/lzsse2/*.o lzsse/lzsse4/*.o lzsse/lzsse8/*.o lzfse/*.o xpack/lib/*.o blosclz/*.o gipfeli/*.o xz/*.o xz/common/*.o xz/check/*.o xz/lzma/*.o xz/lz/*.o xz/rangecoder/*.o liblzg/*.o lzlib/*.o brieflz/*.o brotli/common/*.o brotli/enc/*.o brotli/dec/*.o libcsc/*.o wflz/*.o lzjb/*.o lzma/*.o density/buffers/*.o density/algorithms/*.o density/algorithms/cheetah/core/*.o density/algorithms/*.o density/algorithms/lion/forms/*.o density/algorithms/lion/core/*.o density/algorithms/chameleon/core/*.o density/*.o density/structure/*.o pithy/*.o glza/*.o libzling/*.o yappy/*.o shrinker/*.o fastlz/*.o ucl/*.o zlib/*.o lzham/*.o lzmat/*.o lizard/*.o lz4/*.o crush/*.o lzf/*.o lzrw/*.o lzo/*.o snappy/*.o quicklz/*.o tornado/*.o libdeflate/*.o libdeflate/x86/*.o libdeflate/arm/*.o nakamichi/*.o nvcomp/*.o
rm -rf lzbench lzbench.exe *.o _lzbench/*.o bzip2/*.o fast-lzma2/*.o slz/*.o zstd/lib/*.o zstd/lib/*.a zstd/lib/common/*.o zstd/lib/compress/*.o zstd/lib/decompress/*.o zstd/lib/dictBuilder/*.o lzsse/lzsse2/*.o lzsse/lzsse4/*.o lzsse/lzsse8/*.o lzfse/*.o xpack/lib/*.o blosclz/*.o gipfeli/*.o xz/*.o xz/common/*.o xz/check/*.o xz/lzma/*.o xz/lz/*.o xz/rangecoder/*.o liblzg/*.o lzlib/*.o brieflz/*.o brotli/common/*.o brotli/enc/*.o brotli/dec/*.o libcsc/*.o wflz/*.o lzjb/*.o lzma/*.o density/buffers/*.o density/algorithms/*.o density/algorithms/cheetah/core/*.o density/algorithms/*.o density/algorithms/lion/forms/*.o density/algorithms/lion/core/*.o density/algorithms/chameleon/core/*.o density/*.o density/structure/*.o pithy/*.o glza/*.o libzling/*.o yappy/*.o shrinker/*.o fastlz/*.o ucl/*.o zlib/*.o zlib-ng/*.o zlib-ng/arch/arm/*.o zlib-ng/arch/power/*.o zlib-ng/arch/s390/*.o zlib-ng/arch/x86/*.o lzham/*.o lzmat/*.o lizard/*.o lz4/*.o crush/*.o lzf/*.o lzrw/*.o lzo/*.o snappy/*.o quicklz/*.o tornado/*.o libdeflate/*.o libdeflate/x86/*.o libdeflate/arm/*.o nakamichi/*.o nvcomp/*.o
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ see the [CompFuzz Results](https://github.com/nemequ/compfuzz/wiki/Results) page
- [yalz77 2015-09-19](https://github.com/ivan-tkatchev/yalz77)
- [yappy 2014-03-22](https://encode.su/threads/2825-Yappy-(working)-compressor) - WARNING: fails to decompress properly on ARM
- [zlib 1.2.11](http://zlib.net)
- [zlib-ng 2.1.0-devel](https://github.com/zlib-ng/zlib-ng)
- [zling 2018-10-12](https://github.com/richox/libzling) - according to the author using libzling in a production environment is not a good idea
- [zstd 1.5.0](https://github.com/facebook/zstd)
- [nvcomp 1.2.3](https://github.com/NVIDIA/nvcomp) - If CUDA is available.
Expand Down
25 changes: 25 additions & 0 deletions _lzbench/compressors.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1523,6 +1523,31 @@ int64_t lzbench_zlib_decompress(char *inbuf, size_t insize, char *outbuf, size_t



#ifndef BENCH_REMOVE_ZLIB_NG
#include "zlib-ng/zlib-ng.h"

int64_t lzbench_zlib_ng_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*)
{
uLongf zcomplen = insize;
int err = zng_compress2((uint8_t*)outbuf, &zcomplen, (uint8_t*)inbuf, insize, level);
if (err != Z_OK)
return 0;
return zcomplen;
}

int64_t lzbench_zlib_ng_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*)
{
uLongf zdecomplen = outsize;
int err = zng_uncompress((uint8_t*)outbuf, &zdecomplen, (uint8_t*)inbuf, insize);
if (err != Z_OK)
return 0;
return outsize;
}

#endif



#if !defined(BENCH_REMOVE_SLZ) && !defined(BENCH_REMOVE_ZLIB)
extern "C"
{
Expand Down
9 changes: 9 additions & 0 deletions _lzbench/compressors.h
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,15 @@ int64_t lzbench_return_0(char *inbuf, size_t insize, char *outbuf, size_t outsiz
#endif


#ifndef BENCH_REMOVE_ZLIB_NG
int64_t lzbench_zlib_ng_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);
int64_t lzbench_zlib_ng_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);
#else
#define lzbench_zlib_ng_compress NULL
#define lzbench_zlib_ng_decompress NULL
#endif


#ifndef BENCH_REMOVE_ZLING
int64_t lzbench_zling_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);
int64_t lzbench_zling_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);
Expand Down
3 changes: 2 additions & 1 deletion _lzbench/lzbench.h
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ typedef struct



#define LZBENCH_COMPRESSOR_COUNT 73
#define LZBENCH_COMPRESSOR_COUNT 74

static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
{
Expand Down Expand Up @@ -203,6 +203,7 @@ static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
{ "yalz77", "2015-09-19", 1, 12, 0, 0, lzbench_yalz77_compress, lzbench_yalz77_decompress, NULL, NULL },
{ "yappy", "2014-03-22", 0, 99, 0, 0, lzbench_yappy_compress, lzbench_yappy_decompress, lzbench_yappy_init, NULL },
{ "zlib", "1.2.11", 1, 9, 0, 0, lzbench_zlib_compress, lzbench_zlib_decompress, NULL, NULL },
{ "zlib-ng", "2.1.0.devel", 1, 9, 0, 0, lzbench_zlib_ng_compress, lzbench_zlib_ng_decompress, NULL, NULL },
{ "zling", "2018-10-12", 0, 4, 0, 0, lzbench_zling_compress, lzbench_zling_decompress, NULL, NULL },
{ "zstd", "1.5.0", 1, 22, 0, 0, lzbench_zstd_compress, lzbench_zstd_decompress, lzbench_zstd_init, lzbench_zstd_deinit },
{ "zstd_fast", "1.5.0", -5, -1, 0, 0, lzbench_zstd_compress, lzbench_zstd_decompress, lzbench_zstd_init, lzbench_zstd_deinit },
Expand Down
116 changes: 116 additions & 0 deletions zlib-ng/adler32.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/* adler32.c -- compute the Adler-32 checksum of a data stream
* Copyright (C) 1995-2011, 2016 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/

#include "zbuild.h"
#include "zutil.h"
#include "functable.h"
#include "adler32_p.h"

/* ========================================================================= */
Z_INTERNAL uint32_t adler32_c(uint32_t adler, const unsigned char *buf, size_t len) {
uint32_t sum2;
unsigned n;

/* split Adler-32 into component sums */
sum2 = (adler >> 16) & 0xffff;
adler &= 0xffff;

/* in case user likes doing a byte at a time, keep it fast */
if (UNLIKELY(len == 1))
return adler32_len_1(adler, buf, sum2);

/* initial Adler-32 value (deferred check for len == 1 speed) */
if (UNLIKELY(buf == NULL))
return 1L;

/* in case short lengths are provided, keep it somewhat fast */
if (UNLIKELY(len < 16))
return adler32_len_16(adler, buf, len, sum2);

/* do length NMAX blocks -- requires just one modulo operation */
while (len >= NMAX) {
len -= NMAX;
#ifdef UNROLL_MORE
n = NMAX / 16; /* NMAX is divisible by 16 */
#else
n = NMAX / 8; /* NMAX is divisible by 8 */
#endif
do {
#ifdef UNROLL_MORE
DO16(adler, sum2, buf); /* 16 sums unrolled */
buf += 16;
#else
DO8(adler, sum2, buf, 0); /* 8 sums unrolled */
buf += 8;
#endif
} while (--n);
adler %= BASE;
sum2 %= BASE;
}

/* do remaining bytes (less than NMAX, still just one modulo) */
return adler32_len_64(adler, buf, len, sum2);
}

#ifdef ZLIB_COMPAT
unsigned long Z_EXPORT PREFIX(adler32_z)(unsigned long adler, const unsigned char *buf, size_t len) {
return (unsigned long)functable.adler32((uint32_t)adler, buf, len);
}
#else
uint32_t Z_EXPORT PREFIX(adler32_z)(uint32_t adler, const unsigned char *buf, size_t len) {
return functable.adler32(adler, buf, len);
}
#endif

/* ========================================================================= */
#ifdef ZLIB_COMPAT
unsigned long Z_EXPORT PREFIX(adler32)(unsigned long adler, const unsigned char *buf, unsigned int len) {
return (unsigned long)functable.adler32((uint32_t)adler, buf, len);
}
#else
uint32_t Z_EXPORT PREFIX(adler32)(uint32_t adler, const unsigned char *buf, uint32_t len) {
return functable.adler32(adler, buf, len);
}
#endif

/* ========================================================================= */
static uint32_t adler32_combine_(uint32_t adler1, uint32_t adler2, z_off64_t len2) {
uint32_t sum1;
uint32_t sum2;
unsigned rem;

/* for negative len, return invalid adler32 as a clue for debugging */
if (len2 < 0)
return 0xffffffff;

/* the derivation of this formula is left as an exercise for the reader */
len2 %= BASE; /* assumes len2 >= 0 */
rem = (unsigned)len2;
sum1 = adler1 & 0xffff;
sum2 = rem * sum1;
sum2 %= BASE;
sum1 += (adler2 & 0xffff) + BASE - 1;
sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem;
if (sum1 >= BASE) sum1 -= BASE;
if (sum1 >= BASE) sum1 -= BASE;
if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1);
if (sum2 >= BASE) sum2 -= BASE;
return sum1 | (sum2 << 16);
}

/* ========================================================================= */
#ifdef ZLIB_COMPAT
unsigned long Z_EXPORT PREFIX(adler32_combine)(unsigned long adler1, unsigned long adler2, z_off_t len2) {
return (unsigned long)adler32_combine_((uint32_t)adler1, (uint32_t)adler2, len2);
}

unsigned long Z_EXPORT PREFIX4(adler32_combine)(unsigned long adler1, unsigned long adler2, z_off64_t len2) {
return (unsigned long)adler32_combine_((uint32_t)adler1, (uint32_t)adler2, len2);
}
#else
uint32_t Z_EXPORT PREFIX4(adler32_combine)(uint32_t adler1, uint32_t adler2, z_off64_t len2) {
return adler32_combine_(adler1, adler2, len2);
}
#endif
58 changes: 58 additions & 0 deletions zlib-ng/adler32_p.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/* adler32_p.h -- Private inline functions and macros shared with
* different computation of the Adler-32 checksum
* of a data stream.
* Copyright (C) 1995-2011, 2016 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/

#ifndef ADLER32_P_H
#define ADLER32_P_H

#define BASE 65521U /* largest prime smaller than 65536 */
#define NMAX 5552
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */

#define DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);}
#define DO2(sum1, sum2, buf, i) {DO1(sum1, sum2, buf, i); DO1(sum1, sum2, buf, i+1);}
#define DO4(sum1, sum2, buf, i) {DO2(sum1, sum2, buf, i); DO2(sum1, sum2, buf, i+2);}
#define DO8(sum1, sum2, buf, i) {DO4(sum1, sum2, buf, i); DO4(sum1, sum2, buf, i+4);}
#define DO16(sum1, sum2, buf) {DO8(sum1, sum2, buf, 0); DO8(sum1, sum2, buf, 8);}

static inline uint32_t adler32_len_1(uint32_t adler, const unsigned char *buf, uint32_t sum2) {
adler += buf[0];
adler %= BASE;
sum2 += adler;
sum2 %= BASE;
return adler | (sum2 << 16);
}

static inline uint32_t adler32_len_16(uint32_t adler, const unsigned char *buf, size_t len, uint32_t sum2) {
while (len) {
--len;
adler += *buf++;
sum2 += adler;
}
adler %= BASE;
sum2 %= BASE; /* only added so many BASE's */
/* return recombined sums */
return adler | (sum2 << 16);
}

static inline uint32_t adler32_len_64(uint32_t adler, const unsigned char *buf, size_t len, uint32_t sum2) {
#ifdef UNROLL_MORE
while (len >= 16) {
len -= 16;
DO16(adler, sum2, buf);
buf += 16;
#else
while (len >= 8) {
len -= 8;
DO8(adler, sum2, buf, 0);
buf += 8;
#endif
}
/* Process tail (len < 16). */
return adler32_len_16(adler, buf, len, sum2);
}

#endif /* ADLER32_P_H */
Loading

0 comments on commit e9ec44f

Please sign in to comment.