diff options
Diffstat (limited to 'tests')
36 files changed, 1259 insertions, 409 deletions
diff --git a/tests/.gitignore b/tests/.gitignore index fcb865d6..311a8b5e 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -67,3 +67,6 @@ speedTest.pid *.exe *.out *.app + +# Specific exclusions +!golden-decompression/*.zst diff --git a/tests/Makefile b/tests/Makefile index 778c7d67..ed3692a2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -20,43 +20,45 @@ # zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode # ########################################################################## -LIBZSTD = ../lib - -ZSTD_LEGACY_SUPPORT ?= 0 +ZSTD_LEGACY_SUPPORT ?= 5 +export ZSTD_LEGACY_SUPPORT DEBUGLEVEL ?= 2 export DEBUGLEVEL # transmit value to sub-makefiles -include $(LIBZSTD)/libzstd.mk +LIBZSTD_MK_DIR := ../lib +include $(LIBZSTD_MK_DIR)/libzstd.mk -ZSTDDIR = $(LIBZSTD) PRGDIR = ../programs PYTHON ?= python3 TESTARTEFACT := versionsTest DEBUGFLAGS += -g -Wno-c++-compat -CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ - -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ +CPPFLAGS += -I$(LIB_SRCDIR) -I$(LIB_SRCDIR)/common -I$(LIB_SRCDIR)/compress -I$(LIB_SRCDIR)/legacy \ + -I$(LIB_SRCDIR)/dictBuilder -I$(LIB_SRCDIR)/deprecated -I$(PRGDIR) \ -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1 ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES)) ZSTDCOMP_FILES := $(sort $(ZSTD_COMPRESS_FILES)) ZSTDDECOMP_FILES := $(sort $(ZSTD_DECOMPRESS_FILES)) -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) +ZSTDLEGACY_FILES := $(sort $(wildcard $(LIB_SRCDIR)/legacy/*.c)) +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZSTDLEGACY_FILES) ZDICT_FILES := $(sort $(ZSTD_DICTBUILDER_FILES)) ZSTD_F1 := $(sort $(wildcard $(ZSTD_FILES))) -ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1)) -ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1)) -ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2)) -ZSTD_OBJ4 := $(ZSTD_OBJ3:.c=.o) -ZSTD_OBJECTS := $(ZSTD_OBJ4:.S=.o) - -ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1)) -ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1)) -ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2)) -ZSTDMT_OBJ4 := $(ZSTDMT_OBJ3:.c=.o) -ZSTDMT_OBJECTS := $(ZSTDMT_OBJ4:.S=.o) +ZSTD_OBJ1 := $(subst $(LIB_SRCDIR)/common/,zstdm_,$(ZSTD_F1)) +ZSTD_OBJ2 := $(subst $(LIB_SRCDIR)/compress/,zstdc_,$(ZSTD_OBJ1)) +ZSTD_OBJ3 := $(subst $(LIB_SRCDIR)/decompress/,zstdd_,$(ZSTD_OBJ2)) +ZSTD_OBJ4 := $(subst $(LIB_SRCDIR)/legacy/,zstdl_,$(ZSTD_OBJ3)) +ZSTD_OBJ5 := $(ZSTD_OBJ4:.c=.o) +ZSTD_OBJECTS := $(ZSTD_OBJ5:.S=.o) + +ZSTDMT_OBJ1 := $(subst $(LIB_SRCDIR)/common/,zstdmt_m_,$(ZSTD_F1)) +ZSTDMT_OBJ2 := $(subst $(LIB_SRCDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1)) +ZSTDMT_OBJ3 := $(subst $(LIB_SRCDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2)) +ZSTDMT_OBJ4 := $(subst $(LIB_SRCDIR)/legacy/,zstdmt_l_,$(ZSTDMT_OBJ3)) +ZSTDMT_OBJ5 := $(ZSTDMT_OBJ4:.c=.o) +ZSTDMT_OBJECTS := $(ZSTDMT_OBJ5:.S=.o) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -100,40 +102,46 @@ zstd zstd32 zstd-nolegacy zstd-dll: .PHONY: libzstd libzstd : - $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS+="$(DEBUGFLAGS)" + $(MAKE) -C $(LIB_SRCDIR) libzstd MOREFLAGS+="$(DEBUGFLAGS)" %-dll : libzstd -%-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd +%-dll : LDFLAGS += -L$(LIB_BINDIR) -lzstd -$(ZSTDDIR)/libzstd.a : - $(MAKE) -C $(ZSTDDIR) libzstd.a +$(LIB_BINDIR)/libzstd.a : + $(MAKE) -C $(LIB_SRCDIR) libzstd.a -zstdm_%.o : $(ZSTDDIR)/common/%.c +zstdm_%.o : $(LIB_SRCDIR)/common/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ -zstdc_%.o : $(ZSTDDIR)/compress/%.c +zstdc_%.o : $(LIB_SRCDIR)/compress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ -zstdd_%.o : $(ZSTDDIR)/decompress/%.c +zstdd_%.o : $(LIB_SRCDIR)/decompress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ -zstdd_%.o : $(ZSTDDIR)/decompress/%.S +zstdd_%.o : $(LIB_SRCDIR)/decompress/%.S $(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@ +zstdl_%.o : $(LIB_SRCDIR)/legacy/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP) -zstdmt_m_%.o : $(ZSTDDIR)/common/%.c +zstdmt_m_%.o : $(LIB_SRCDIR)/common/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ -zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c +zstdmt_c_%.o : $(LIB_SRCDIR)/compress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ -zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c +zstdmt_d_%.o : $(LIB_SRCDIR)/decompress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ -zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S +zstdmt_d_%.o : $(LIB_SRCDIR)/decompress/%.S $(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@ +zstdmt_l_%.o : $(LIB_SRCDIR)/legacy/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + FULLBENCHS := fullbench fullbench32 CLEAN += $(FULLBENCHS) fullbench32: CPPFLAGS += -m32 @@ -146,12 +154,12 @@ $(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR CLEAN += fullbench-lib fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c +fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(LIB_SRCDIR)/libzstd.a fullbench.c $(LINK.c) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c -# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll +# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(LIB_SRCDIR)/dll/libzstd.dll $(LINK.c) $^ $(LDLIBS) -o $@$(EXT) CLEAN += fuzzer fuzzer32 @@ -165,7 +173,7 @@ fuzzer32 : $(ZSTD_FILES) $(LINK.c) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library -fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c +fuzzer-dll : $(LIB_SRCDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) CLEAN += zstreamtest zstreamtest32 @@ -196,17 +204,17 @@ zstreamtest_ubsan : $(ZSTREAMFILES) $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library -zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll +zstreamtest-dll : $(LIB_SRCDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) CLEAN += paramgrill paramgrill : DEBUGFLAGS = # turn off debug for speed measurements paramgrill : LDLIBS += -lm -paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c +paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c paramgrill.c CLEAN += datagen -datagen : $(PRGDIR)/datagen.c datagencli.c +datagen : $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c loremOut.c datagencli.c $(LINK.c) $^ -o $@$(EXT) CLEAN += roundTripCrash @@ -224,15 +232,15 @@ CLEAN += invalidDictionaries invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c CLEAN += legacy -legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4 -legacy : $(ZSTD_FILES) $(sort $(wildcard $(ZSTDDIR)/legacy/*.c)) legacy.c +legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4 +legacy : $(ZSTD_FILES) legacy.c CLEAN += decodecorpus decodecorpus : LDLIBS += -lm decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c CLEAN += poolTests -poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c +poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(LIB_SRCDIR)/common/pool.c $(LIB_SRCDIR)/common/threading.c $(LIB_SRCDIR)/common/zstd_common.c $(LIB_SRCDIR)/common/error_private.c $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) .PHONY: versionsTest @@ -245,14 +253,15 @@ automated_benchmarking: clean # make checkTag : check that release tag corresponds to release version CLEAN += checkTag -checkTag.o : $(ZSTDDIR)/zstd.h +checkTag.o : $(LIB_SRCDIR)/zstd.h .PHONY: clean clean: - $(MAKE) -C $(ZSTDDIR) clean + $(MAKE) -C $(LIB_SRCDIR) clean $(MAKE) -C $(PRGDIR) clean - $(RM) -fR $(TESTARTEFACT) - $(RM) -rf tmp* # some test directories are named tmp* + $(MAKE) -C fuzz clean + $(RM) -R $(TESTARTEFACT) + $(RM) -r tmp* # some test directories are named tmp* $(RM) $(CLEAN) core *.o *.tmp result* *.gcda dictionary *.zst \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench-dll$(EXT) fuzzer-dll$(EXT) zstreamtest-dll$(EXT) @@ -263,7 +272,7 @@ clean: # valgrind tests validated only for some posix platforms #---------------------------------------------------------------------------------- UNAME := $(shell uname) -ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX)) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX CYGWIN_NT)) HOST_OS = POSIX .PHONY: test-valgrind @@ -313,7 +322,7 @@ check: shortest fuzztest: test-fuzzer test-zstream test-decodecorpus .PHONY: test -test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests +test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus ifeq ($(QEMU_SYS),) test: test-pool endif diff --git a/tests/cli-tests/basic/args.sh b/tests/cli-tests/basic/args.sh new file mode 100755 index 00000000..94331014 --- /dev/null +++ b/tests/cli-tests/basic/args.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +println "+ zstd --blah" >&2 +zstd --blah +println "+ zstd -xz" >&2 +zstd -xz +println "+ zstd --adapt=min=1,maxx=2 file.txt" >&2 +zstd --adapt=min=1,maxx=2 file.txt +println "+ zstd --train-cover=k=48,d=8,steps32 file.txt" >&2 +zstd --train-cover=k=48,d=8,steps32 file.txt diff --git a/tests/cli-tests/basic/args.sh.exit b/tests/cli-tests/basic/args.sh.exit new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/tests/cli-tests/basic/args.sh.exit @@ -0,0 +1 @@ +1 diff --git a/tests/cli-tests/basic/args.sh.stderr.glob b/tests/cli-tests/basic/args.sh.stderr.glob new file mode 100644 index 00000000..df275471 --- /dev/null +++ b/tests/cli-tests/basic/args.sh.stderr.glob @@ -0,0 +1,28 @@ ++ zstd --blah +Incorrect parameter: --blah +... +Usage: zstd * + +Options: +... ++ zstd -xz +Incorrect parameter: -x +... +Usage: zstd * + +Options: +... ++ zstd --adapt=min=1,maxx=2 file.txt +Incorrect parameter: --adapt=min=1,maxx=2 +... +Usage: zstd * + +Options: +... ++ zstd --train-cover=k=48,d=8,steps32 file.txt +Incorrect parameter: --train-cover=k=48,d=8,steps32 +... +Usage: zstd * + +Options: +... diff --git a/tests/cli-tests/decompression/detectErrors.sh b/tests/cli-tests/decompression/detectErrors.sh new file mode 100755 index 00000000..300cde36 --- /dev/null +++ b/tests/cli-tests/decompression/detectErrors.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression-errors/" + +for file in "$GOLDEN_DIR"/*; do + zstd -t $file && die "should have detected an error" +done +exit 0 + diff --git a/tests/cli-tests/file-handling/directory-mirror.sh b/tests/cli-tests/file-handling/directory-mirror.sh new file mode 100755 index 00000000..b2f70b59 --- /dev/null +++ b/tests/cli-tests/file-handling/directory-mirror.sh @@ -0,0 +1,49 @@ +#!/bin/sh +set -e + +# setup +mkdir -p src/.hidden src/dir +mkdir mid dst + +echo "file1" > src/file1 +echo "file2" > src/.file2 +echo "file3" > src/.hidden/.file3 +echo "file4" > src/dir/.file4 + +# relative paths +zstd -q -r --output-dir-mirror mid/ src/ +zstd -q -d -r --output-dir-mirror dst/ mid/src/ + +diff --brief --recursive --new-file src/ dst/mid/src/ + +# reset +rm -rf mid dst +mkdir mid dst + +# from inside the directory +(cd src; zstd -q -r --output-dir-mirror ../mid/ ./) +(cd mid; zstd -q -d -r --output-dir-mirror ../dst/ ./) + +diff --brief --recursive --new-file src/ dst/ + +# reset +rm -rf mid dst +mkdir mid dst + +# absolute paths +export BASE_PATH="$(pwd)" + +zstd -q -r --output-dir-mirror mid/ "${BASE_PATH}/src/" +zstd -q -d -r --output-dir-mirror dst/ "${BASE_PATH}/mid/${BASE_PATH}/src/" + +diff --brief --recursive --new-file src/ "dst/${BASE_PATH}/mid/${BASE_PATH}/src/" + +# reset +rm -rf mid dst +mkdir mid dst + +# dots +zstd -q -r --output-dir-mirror mid/ ./src/./ +zstd -q -d -r --output-dir-mirror dst/ ./mid/./src/./ + +diff --brief --recursive --new-file src/ dst/mid/src/ diff --git a/tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact b/tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact diff --git a/tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact b/tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact diff --git a/tests/datagencli.c b/tests/datagencli.c index 09ec5e9a..56616bef 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -8,122 +8,141 @@ * You may select, at your option, one of the above-listed licenses. */ - /*-************************************ -* Dependencies -**************************************/ -#include "util.h" /* Compiler options */ -#include <stdio.h> /* fprintf, stderr */ -#include "datagen.h" /* RDG_generate */ - + * Dependencies + **************************************/ +#include <stdio.h> /* fprintf, stderr */ +#include "datagen.h" /* RDG_generate */ +#include "loremOut.h" /* LOREM_genOut */ +#include "util.h" /* Compiler options */ /*-************************************ -* Constants -**************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) + * Constants + **************************************/ +#define KB *(1 << 10) +#define MB *(1 << 20) +#define GB *(1U << 30) #define SIZE_DEFAULT ((64 KB) + 1) #define SEED_DEFAULT 0 -#define COMPRESSIBILITY_DEFAULT 50 - +#define COMPRESSIBILITY_DEFAULT 9999 /*-************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + * Macros + **************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } static unsigned displayLevel = 2; - /*-******************************************************* -* Command line -*********************************************************/ + * Command line + *********************************************************/ static int usage(const char* programName) { - DISPLAY( "Compressible data generator\n"); - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); - DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", - COMPRESSIBILITY_DEFAULT); - DISPLAY( " -h : display help and exit\n"); + DISPLAY("Compressible data generator\n"); + DISPLAY("Usage :\n"); + DISPLAY(" %s [args]\n", programName); + DISPLAY("\n"); + DISPLAY("Arguments :\n"); + DISPLAY(" -g# : generate # data (default:%i)\n", SIZE_DEFAULT); + DISPLAY(" -s# : Select seed (default:%i)\n", SEED_DEFAULT); + DISPLAY(" -P# : Select compressibility in %% (range [0-100])\n"); + DISPLAY(" -h : display help and exit\n"); return 0; } - int main(int argc, const char** argv) { - unsigned probaU32 = COMPRESSIBILITY_DEFAULT; - double litProba = 0.0; - U64 size = SIZE_DEFAULT; - U32 seed = SEED_DEFAULT; + unsigned probaU32 = COMPRESSIBILITY_DEFAULT; + double litProba = 0.0; + U64 size = SIZE_DEFAULT; + U32 seed = SEED_DEFAULT; const char* const programName = argv[0]; int argNb; - for(argNb=1; argNb<argc; argNb++) { + for (argNb = 1; argNb < argc; argNb++) { const char* argument = argv[argNb]; - if(!argument) continue; /* Protection if argument empty */ + if (!argument) + continue; /* Protection if argument empty */ /* Handle commands. Aggregated commands are allowed */ - if (*argument=='-') { + if (*argument == '-') { argument++; - while (*argument!=0) { - switch(*argument) - { - case 'h': - return usage(programName); - case 'g': - argument++; - size=0; - while ((*argument>='0') && (*argument<='9')) - size *= 10, size += *argument++ - '0'; - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - seed *= 10, seed += *argument++ - '0'; - break; - case 'P': - argument++; - probaU32 = 0; - while ((*argument>='0') && (*argument<='9')) - probaU32 *= 10, probaU32 += *argument++ - '0'; - if (probaU32>100) probaU32 = 100; - break; - case 'L': /* hidden argument : Literal distribution probability */ - argument++; - litProba=0.; - while ((*argument>='0') && (*argument<='9')) - litProba *= 10, litProba += *argument++ - '0'; - if (litProba>100.) litProba=100.; - litProba /= 100.; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: - return usage(programName); + while (*argument != 0) { + switch (*argument) { + case 'h': + return usage(programName); + case 'g': + argument++; + size = 0; + while ((*argument >= '0') && (*argument <= '9')) + size *= 10, size += (U64)(*argument++ - '0'); + if (*argument == 'K') { + size <<= 10; + argument++; + } + if (*argument == 'M') { + size <<= 20; + argument++; + } + if (*argument == 'G') { + size <<= 30; + argument++; + } + if (*argument == 'B') { + argument++; + } + break; + case 's': + argument++; + seed = 0; + while ((*argument >= '0') && (*argument <= '9')) + seed *= 10, seed += (U32)(*argument++ - '0'); + break; + case 'P': + argument++; + probaU32 = 0; + while ((*argument >= '0') && (*argument <= '9')) + probaU32 *= 10, + probaU32 += (U32)(*argument++ - '0'); + if (probaU32 > 100) + probaU32 = 100; + break; + case 'L': /* hidden argument : Literal distribution + probability */ + argument++; + litProba = 0.; + while ((*argument >= '0') && (*argument <= '9')) + litProba *= 10, litProba += *argument++ - '0'; + if (litProba > 100.) + litProba = 100.; + litProba /= 100.; + break; + case 'v': + displayLevel = 4; + argument++; + break; + default: + return usage(programName); } - } } } /* for(argNb=1; argNb<argc; argNb++) */ + } + } + } /* for(argNb=1; argNb<argc; argNb++) */ DISPLAYLEVEL(4, "Compressible data Generator \n"); - if (probaU32!=COMPRESSIBILITY_DEFAULT) - DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32); DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed); - RDG_genStdout(size, (double)probaU32/100, litProba, seed); + if (probaU32 != COMPRESSIBILITY_DEFAULT) { + DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32); + RDG_genStdout(size, (double)probaU32 / 100, litProba, seed); + } else { + LOREM_genOut(size, seed); + } + DISPLAYLEVEL(3, "\n"); return 0; diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index e48eccd6..1abc7df8 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -732,7 +732,7 @@ generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, } } while (((!info.useDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); - { BYTE* const dictEnd = info.dictContent + info.dictContentSize; + { BYTE* const dictEnd = ZSTD_maybeNullPtrAdd(info.dictContent, info.dictContentSize); size_t j; for (j = 0; j < matchLen; j++) { if ((U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) { @@ -825,7 +825,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, /* Sequences Header */ if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); - if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; + if (nbSeq < 128) *op++ = (BYTE)nbSeq; else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; diff --git a/tests/fullbench.c b/tests/fullbench.c index 41bd26d0..c8f0c0af 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -138,18 +138,24 @@ static size_t local_ZSTD_decompress(const void* src, size_t srcSize, return ZSTD_decompress(dst, dstSize, buff2, g_cSize); } -static ZSTD_DCtx* g_zdc = NULL; +static ZSTD_DCtx* g_zdc = NULL; /* will be initialized within benchMem */ +static size_t local_ZSTD_decompressDCtx(const void* src, size_t srcSize, + void* dst, size_t dstSize, + void* buff2) +{ + (void)src; (void)srcSize; + return ZSTD_decompressDCtx(g_zdc, dst, dstSize, buff2, g_cSize); +} #ifndef ZSTD_DLL_IMPORT -typedef enum { - not_streaming = 0, - is_streaming = 1 -} streaming_operation; -extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize, void* dst, size_t dstCapacity, const streaming_operation streaming); + +extern size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeLiteralsBlock(g_zdc, buff2, g_cSize, dst, dstSize, not_streaming); + return ZSTD_decodeLiteralsBlock_wrapper(g_zdc, buff2, g_cSize, dst, dstSize); } static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) @@ -453,6 +459,9 @@ static int benchMem(unsigned benchNb, case 3: benchFunction = local_ZSTD_compress_freshCCtx; benchName = "compress_freshCCtx"; break; + case 4: + benchFunction = local_ZSTD_decompressDCtx; benchName = "decompressDCtx"; + break; #ifndef ZSTD_DLL_IMPORT case 11: benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue"; @@ -552,6 +561,9 @@ static int benchMem(unsigned benchNb, case 3: payload = &cparams; break; + case 4: + g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel); + break; #ifndef ZSTD_DLL_IMPORT case 11: payload = &cparams; @@ -606,7 +618,7 @@ static int benchMem(unsigned benchNb, ip += ZSTD_blockHeaderSize; /* skip block header */ ZSTD_decompressBegin(g_zdc); CONTROL(iend > ip); - ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip), dstBuff, dstBuffSize, not_streaming); /* skip literal segment */ + ip += ZSTD_decodeLiteralsBlock_wrapper(g_zdc, ip, (size_t)(iend-ip), dstBuff, dstBuffSize); /* skip literal segment */ g_cSize = (size_t)(iend-ip); memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 525e396b..430f6df1 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -24,13 +24,12 @@ else endif CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/ -LIBZSTD = ../../lib +LIBZSTD_MK_DIR = ../../lib DEBUGLEVEL ?= 2 ZSTD_LEGACY_SUPPORT ?= 1 -include $(LIBZSTD)/libzstd.mk +include $(LIBZSTD_MK_DIR)/libzstd.mk -ZSTDDIR = ../../lib PRGDIR = ../../programs CONTRIBDIR = ../../contrib @@ -38,8 +37,8 @@ DEFAULT_SEQ_PROD_DIR = $(CONTRIBDIR)/externalSequenceProducer DEFAULT_SEQ_PROD_SRC = $(DEFAULT_SEQ_PROD_DIR)/sequence_producer.c THIRD_PARTY_SEQ_PROD_OBJ ?= -FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ - -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \ +FUZZ_CPPFLAGS := -I$(LIB_SRCDIR) -I$(LIB_SRCDIR)/common -I$(LIB_SRCDIR)/compress \ + -I$(LIB_SRCDIR)/dictBuilder -I$(LIB_SRCDIR)/deprecated -I$(LIB_SRCDIR)/legacy \ -I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(DEFAULT_SEQ_PROD_DIR) \ -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS) FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -78,11 +77,11 @@ FUZZ_SRC := \ $(DEFAULT_SEQ_PROD_SRC) FUZZ_SRC := $(sort $(wildcard $(FUZZ_SRC))) -FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC)) -FUZZ_D_OBJ2 := $(subst $(ZSTDDIR)/compress/,d_lib_compress_,$(FUZZ_D_OBJ1)) -FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2)) -FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3)) -FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4)) +FUZZ_D_OBJ1 := $(subst $(LIB_SRCDIR)/common/,d_lib_common_,$(FUZZ_SRC)) +FUZZ_D_OBJ2 := $(subst $(LIB_SRCDIR)/compress/,d_lib_compress_,$(FUZZ_D_OBJ1)) +FUZZ_D_OBJ3 := $(subst $(LIB_SRCDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2)) +FUZZ_D_OBJ4 := $(subst $(LIB_SRCDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3)) +FUZZ_D_OBJ5 := $(subst $(LIB_SRCDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4)) FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5)) FUZZ_D_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,d_default_seq_prod_,$(FUZZ_D_OBJ6)) FUZZ_D_OBJ8 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ7)) @@ -90,11 +89,11 @@ FUZZ_D_OBJ9 := $(FUZZ_D_OBJ8:.c=.o) FUZZ_D_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_D_OBJ9) FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ10:.S=.o) -FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC)) -FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1)) -FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2)) -FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3)) -FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4)) +FUZZ_RT_OBJ1 := $(subst $(LIB_SRCDIR)/common/,rt_lib_common_,$(FUZZ_SRC)) +FUZZ_RT_OBJ2 := $(subst $(LIB_SRCDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1)) +FUZZ_RT_OBJ3 := $(subst $(LIB_SRCDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2)) +FUZZ_RT_OBJ4 := $(subst $(LIB_SRCDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3)) +FUZZ_RT_OBJ5 := $(subst $(LIB_SRCDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4)) FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5)) FUZZ_RT_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,rt_default_seq_prod_,$(FUZZ_RT_OBJ6)) FUZZ_RT_OBJ8 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ7)) @@ -125,26 +124,28 @@ FUZZ_TARGETS := \ sequence_compression_api \ seekable_roundtrip \ huf_round_trip \ - huf_decompress + huf_decompress \ + decompress_cross_format \ + generate_sequences all: libregression.a $(FUZZ_TARGETS) -rt_lib_common_%.o: $(ZSTDDIR)/common/%.c +rt_lib_common_%.o: $(LIB_SRCDIR)/common/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ -rt_lib_compress_%.o: $(ZSTDDIR)/compress/%.c +rt_lib_compress_%.o: $(LIB_SRCDIR)/compress/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ -rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c +rt_lib_decompress_%.o: $(LIB_SRCDIR)/decompress/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ -rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S +rt_lib_decompress_%.o: $(LIB_SRCDIR)/decompress/%.S $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_ASFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ -rt_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c +rt_lib_dictBuilder_%.o: $(LIB_SRCDIR)/dictBuilder/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ -rt_lib_legacy_%.o: $(ZSTDDIR)/legacy/%.c +rt_lib_legacy_%.o: $(LIB_SRCDIR)/legacy/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ rt_prg_%.o: $(PRGDIR)/%.c @@ -156,22 +157,22 @@ rt_fuzz_%.o: %.c rt_default_seq_prod_%.o: $(DEFAULT_SEQ_PROD_DIR)/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ -d_lib_common_%.o: $(ZSTDDIR)/common/%.c +d_lib_common_%.o: $(LIB_SRCDIR)/common/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ -d_lib_compress_%.o: $(ZSTDDIR)/compress/%.c +d_lib_compress_%.o: $(LIB_SRCDIR)/compress/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ -d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c +d_lib_decompress_%.o: $(LIB_SRCDIR)/decompress/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ -d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S +d_lib_decompress_%.o: $(LIB_SRCDIR)/decompress/%.S $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_ASFLAGS) $< -c -o $@ -d_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c +d_lib_dictBuilder_%.o: $(LIB_SRCDIR)/dictBuilder/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ -d_lib_legacy_%.o: $(ZSTDDIR)/legacy/%.c +d_lib_legacy_%.o: $(LIB_SRCDIR)/legacy/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ d_prg_%.o: $(PRGDIR)/%.c @@ -240,6 +241,12 @@ huf_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_huf_round_trip.o huf_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_huf_decompress.o $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_huf_decompress.o $(LIB_FUZZING_ENGINE) -o $@ +decompress_cross_format: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_cross_format.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_cross_format.o $(LIB_FUZZING_ENGINE) -o $@ + +generate_sequences: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_generate_sequences.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_generate_sequences.o $(LIB_FUZZING_ENGINE) -o $@ + libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o $(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o @@ -257,7 +264,7 @@ corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS)) seedcorpora: $(patsubst %,corpora/%_seed_corpus.zip,$(FUZZ_TARGETS)) regressiontest: corpora - CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all + CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all --debug=$(DEBUGLEVEL) $(PYTHON) ./fuzz.py regression all clean: diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 2a9bd457..e2196e83 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -117,3 +117,45 @@ CC=clang CXX=clang++ ./fuzz.py build all --enable-msan ## Fuzzing a custom sequence producer plugin Sequence producer plugin authors can use the zstd fuzzers to stress-test their code. See the documentation in `fuzz_third_party_seq_prod.h` for details. + +## Adding a new fuzzer +There are several steps involved in adding a new fuzzer harness. + +### Build your harness +1. Create a new your fuzzer harness `tests/fuzz/your_harness.c`. + +2. Add your harness to the Makefile + + 2.1 Follow [this example](https://github.com/facebook/zstd/blob/e124e39301381de8f323436a3e4c46539747ba24/tests/fuzz/Makefile#L216) if your fuzzer requires both compression and decompression symbols (prefix `rt_`). If your fuzzer only requires decompression symbols, follow [this example](https://github.com/facebook/zstd/blob/6a0052a409e2604bd40354b76b86272b712edd7d/tests/fuzz/Makefile#L194) (prefix `d_`). + + 2.2 Add your target to [`FUZZ_TARGETS`](https://github.com/facebook/zstd/blob/6a0052a409e2604bd40354b76b86272b712edd7d/tests/fuzz/Makefile#L108). + +3. Add your harness to [`fuzz.py`](https://github.com/facebook/zstd/blob/6a0052a409e2604bd40354b76b86272b712edd7d/tests/fuzz/fuzz.py#L48). + +### Generate seed data +Follow the instructions above to generate seed data: +``` +make -C ../tests decodecorpus +./fuzz.py gen your_harness +``` + +### Run the harness +Follow the instructions above to run your harness and fix any crashes: +``` +./fuzz.py build your_harness --enable-fuzzer --enable-asan --enable-ubsan --cc clang --cxx clang++ +./fuzz.py libfuzzer your_harness +``` + +### Minimize and zip the corpus +After running the fuzzer for a while, you will have a large corpus at `tests/fuzz/corpora/your_harness*`. +This corpus must be minimized and zipped before uploading to GitHub for regression testing: +``` +./fuzz.py minimize your_harness +./fuzz.py zip your_harness +``` + +### Upload the zip file to GitHub +The previous step should produce a `.zip` file containing the corpus for your new harness. +This corpus must be uploaded to GitHub here: https://github.com/facebook/zstd/releases/tag/fuzz-corpora + + diff --git a/tests/fuzz/decompress_cross_format.c b/tests/fuzz/decompress_cross_format.c new file mode 100644 index 00000000..da10702a --- /dev/null +++ b/tests/fuzz/decompress_cross_format.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +// This fuzz target validates decompression of magicless-format compressed data. + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "fuzz_data_producer.h" + +static ZSTD_DCtx *dctx = NULL; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + // Give a random portion of src data to the producer, to use for parameter generation. + // The rest will be interpreted as magicless compressed data. + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t magiclessSize = FUZZ_dataProducer_reserveDataPrefix(producer); + const uint8_t* const magiclessSrc = src; + size_t const dstSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + uint8_t* const standardDst = (uint8_t*)FUZZ_malloc(dstSize); + uint8_t* const magiclessDst = (uint8_t*)FUZZ_malloc(dstSize); + + // Create standard-format src from magicless-format src + const uint32_t zstd_magic = ZSTD_MAGICNUMBER; + size_t standardSize = sizeof(zstd_magic) + magiclessSize; + uint8_t* const standardSrc = (uint8_t*)FUZZ_malloc(standardSize); + memcpy(standardSrc, &zstd_magic, sizeof(zstd_magic)); // assume fuzzing on little-endian machine + memcpy(standardSrc + sizeof(zstd_magic), magiclessSrc, magiclessSize); + + // Truncate to a single frame + { + const size_t standardFrameCompressedSize = ZSTD_findFrameCompressedSize(standardSrc, standardSize); + if (ZSTD_isError(standardFrameCompressedSize)) { + goto cleanup_and_return; + } + standardSize = standardFrameCompressedSize; + magiclessSize = standardFrameCompressedSize - sizeof(zstd_magic); + } + + // Create DCtx if needed + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + // Test one-shot decompression + { + FUZZ_ZASSERT(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); + FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1)); + const size_t standardRet = ZSTD_decompressDCtx( + dctx, standardDst, dstSize, standardSrc, standardSize); + + FUZZ_ZASSERT(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); + FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless)); + const size_t magiclessRet = ZSTD_decompressDCtx( + dctx, magiclessDst, dstSize, magiclessSrc, magiclessSize); + + // Standard accepts => magicless should accept + if (!ZSTD_isError(standardRet)) FUZZ_ZASSERT(magiclessRet); + + // Magicless accepts => standard should accept + // NOTE: this is nice-to-have, please disable this check if it is difficult to satisfy. + if (!ZSTD_isError(magiclessRet)) FUZZ_ZASSERT(standardRet); + + // If both accept, decompressed size and data should match + if (!ZSTD_isError(standardRet) && !ZSTD_isError(magiclessRet)) { + FUZZ_ASSERT(standardRet == magiclessRet); + if (standardRet > 0) { + FUZZ_ASSERT( + memcmp(standardDst, magiclessDst, standardRet) == 0 + ); + } + } + } + + // Test streaming decompression + { + ZSTD_inBuffer standardIn = { standardSrc, standardSize, 0 }; + ZSTD_inBuffer magiclessIn = { magiclessSrc, magiclessSize, 0 }; + ZSTD_outBuffer standardOut = { standardDst, dstSize, 0 }; + ZSTD_outBuffer magiclessOut = { magiclessDst, dstSize, 0 }; + + FUZZ_ZASSERT(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); + FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1)); + const size_t standardRet = ZSTD_decompressStream(dctx, &standardOut, &standardIn); + + FUZZ_ZASSERT(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); + FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless)); + const size_t magiclessRet = ZSTD_decompressStream(dctx, &magiclessOut, &magiclessIn); + + // Standard accepts => magicless should accept + if (standardRet == 0) FUZZ_ASSERT(magiclessRet == 0); + + // Magicless accepts => standard should accept + // NOTE: this is nice-to-have, please disable this check if it is difficult to satisfy. + if (magiclessRet == 0) FUZZ_ASSERT(standardRet == 0); + + // If both accept, decompressed size and data should match + if (standardRet == 0 && magiclessRet == 0) { + FUZZ_ASSERT(standardOut.pos == magiclessOut.pos); + if (standardOut.pos > 0) { + FUZZ_ASSERT( + memcmp(standardOut.dst, magiclessOut.dst, standardOut.pos) == 0 + ); + } + } + } + +cleanup_and_return: +#ifndef STATEFUL_FUZZING + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + free(standardSrc); + free(standardDst); + free(magiclessDst); + FUZZ_dataProducer_free(producer); + return 0; +} diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index 06fdf24e..0470fbf5 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -23,13 +23,13 @@ #include "fuzz_data_producer.h" #include "fuzz_third_party_seq_prod.h" -static ZSTD_CCtx *cctx = NULL; -static ZSTD_DCtx *dctx = NULL; +static ZSTD_CCtx* cctx = NULL; +static ZSTD_DCtx* dctx = NULL; -static size_t roundTripTest(void *result, size_t resultCapacity, - void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize, - FUZZ_dataProducer_t *producer) +static size_t roundTripTest(void* result, size_t resultCapacity, + void* compressed, size_t compressedCapacity, + const void* src, size_t srcSize, + FUZZ_dataProducer_t* producer) { ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer); diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index 8e0a9eaa..d59df926 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -65,6 +65,8 @@ TARGET_INFO = { 'seekable_roundtrip': TargetInfo(InputType.RAW_DATA), 'huf_round_trip': TargetInfo(InputType.RAW_DATA), 'huf_decompress': TargetInfo(InputType.RAW_DATA), + 'decompress_cross_format': TargetInfo(InputType.RAW_DATA), + 'generate_sequences': TargetInfo(InputType.RAW_DATA), } TARGETS = list(TARGET_INFO.keys()) ALL_TARGETS = TARGETS + ['all'] @@ -250,10 +252,10 @@ def build_parser(args): action='store_true', help='Enable UBSAN') parser.add_argument( - '--enable-ubsan-pointer-overflow', + '--disable-ubsan-pointer-overflow', dest='ubsan_pointer_overflow', - action='store_true', - help='Enable UBSAN pointer overflow check (known failure)') + action='store_false', + help='Disable UBSAN pointer overflow check (known failure)') parser.add_argument( '--enable-msan', dest='msan', action='store_true', help='Enable MSAN') parser.add_argument( @@ -383,8 +385,6 @@ def build_parser(args): raise RuntimeError('MSAN may not be used with any other sanitizers') if args.msan_track_origins and not args.msan: raise RuntimeError('--enable-msan-track-origins requires MSAN') - if args.ubsan_pointer_overflow and not args.ubsan: - raise RuntimeError('--enable-ubsan-pointer-overflow requires UBSAN') if args.sanitize_recover and not args.sanitize: raise RuntimeError('--enable-sanitize-recover but no sanitizers used') @@ -407,7 +407,12 @@ def build(args): cxxflags = shlex.split(args.cxxflags) mflags = shlex.split(args.mflags) # Flags to be added to both cflags and cxxflags - common_flags = [] + common_flags = [ + '-Werror', + '-Wno-error=declaration-after-statement', + '-Wno-error=c++-compat', + '-Wno-error=deprecated' # C files are sometimes compiled with CXX + ] cppflags += [ '-DDEBUGLEVEL={}'.format(args.debug), @@ -494,6 +499,7 @@ def build(args): subprocess.check_call(clean_cmd) build_cmd = [ 'make', + '-j', cc_str, cxx_str, cppflags_str, diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index bf846b68..056de3ee 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -28,12 +28,12 @@ void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer) { free(producer); } uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t min, uint32_t max) { - FUZZ_ASSERT(min <= max); - uint32_t range = max - min; uint32_t rolling = range; uint32_t result = 0; + FUZZ_ASSERT(min <= max); + while (rolling > 0 && producer->size > 0) { uint8_t next = *(producer->data + producer->size - 1); producer->size -= 1; @@ -79,11 +79,11 @@ int FUZZ_dataProducer_empty(FUZZ_dataProducer_t *producer) { size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) { - newSize = newSize > producer->size ? producer->size : newSize; + const size_t effectiveNewSize = newSize > producer->size ? producer->size : newSize; - size_t remaining = producer->size - newSize; + size_t remaining = producer->size - effectiveNewSize; producer->data = producer->data + remaining; - producer->size = newSize; + producer->size = effectiveNewSize; return remaining; } diff --git a/tests/fuzz/fuzz_third_party_seq_prod.h b/tests/fuzz/fuzz_third_party_seq_prod.h index f04ad31a..f0771e47 100644 --- a/tests/fuzz/fuzz_third_party_seq_prod.h +++ b/tests/fuzz/fuzz_third_party_seq_prod.h @@ -52,7 +52,7 @@ extern "C" { size_t FUZZ_seqProdSetup(void); /* The fuzzer will call this function after each test-case. It should free - * resources aquired by FUZZ_seqProdSetup() to prevent leaks across test-cases. + * resources acquired by FUZZ_seqProdSetup() to prevent leaks across test-cases. * * The fuzzer will assert() that the return value is zero. To signal an error, * please return a non-zero value. */ @@ -72,7 +72,7 @@ size_t FUZZ_seqProdTearDown(void); void* FUZZ_createSeqProdState(void); /* The fuzzer will call this function after each test-case. It should free any - * resources aquired by FUZZ_createSeqProdState(). + * resources acquired by FUZZ_createSeqProdState(). * * The fuzzer will assert() that the return value is zero. To signal an error, * please return a non-zero value. */ diff --git a/tests/fuzz/generate_sequences.c b/tests/fuzz/generate_sequences.c new file mode 100644 index 00000000..1cc57e84 --- /dev/null +++ b/tests/fuzz/generate_sequences.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdint.h> +#include <string.h> +#include <stdlib.h> + +#include "fuzz_data_producer.h" +#include "fuzz_helpers.h" +#include "zstd_helpers.h" + +/** + * This fuzz target ensures that ZSTD_generateSequences() does not crash and + * if it succeeds that ZSTD_compressSequences() round trips. + */ + +static void testRoundTrip(ZSTD_CCtx* cctx, ZSTD_Sequence const* seqs, size_t nbSeqs, const void* src, size_t srcSize) { + /* Compress the sequences with block delimiters */ + const size_t compressBound = ZSTD_compressBound(srcSize); + void* dst = FUZZ_malloc(compressBound); + FUZZ_ASSERT(dst); + + size_t compressedSize = ZSTD_compressSequences(cctx, dst, compressBound, seqs, nbSeqs, src, srcSize); + FUZZ_ZASSERT(compressedSize); + + void* decompressed = FUZZ_malloc(srcSize); + FUZZ_ASSERT(srcSize == 0 || decompressed); + size_t decompressedSize = ZSTD_decompress(decompressed, srcSize, dst, compressedSize); + FUZZ_ZASSERT(decompressedSize); + FUZZ_ASSERT(decompressedSize == srcSize); + if (srcSize != 0) { + FUZZ_ASSERT(!memcmp(src, decompressed, srcSize)); + } + + free(decompressed); + free(dst); +} + +int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + + const size_t seqsCapacity = FUZZ_dataProducer_uint32Range(producer, 0, 2 * ZSTD_sequenceBound(size)); + ZSTD_Sequence* seqs = (ZSTD_Sequence*)FUZZ_malloc(sizeof(ZSTD_Sequence) * seqsCapacity); + FUZZ_ASSERT(seqsCapacity == 0 || seqs); + + FUZZ_setRandomParameters(cctx, size, producer); + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 0)); + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0)); + + const size_t nbSeqs = ZSTD_generateSequences(cctx, seqs, seqsCapacity, data, size); + if (ZSTD_isError(nbSeqs)) { + /* Allowed to error if the destination is too small */ + if (ZSTD_getErrorCode(nbSeqs) == ZSTD_error_dstSize_tooSmall) { + FUZZ_ASSERT(seqsCapacity < ZSTD_sequenceBound(size)); + } + } else { + /* Ensure we round trip with and without block delimiters*/ + + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters)); + testRoundTrip(cctx, seqs, nbSeqs, data, size); + + const size_t nbMergedSeqs = ZSTD_mergeBlockDelimiters(seqs, nbSeqs); + FUZZ_ASSERT(nbMergedSeqs <= nbSeqs); + FUZZ_ZASSERT(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only)); + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters)); + testRoundTrip(cctx, seqs, nbMergedSeqs, data, size); + } + + free(seqs); + ZSTD_freeCCtx(cctx); + FUZZ_dataProducer_free(producer); + return 0; +} diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c index 550c65d8..26e2b6af 100644 --- a/tests/fuzz/regression_driver.c +++ b/tests/fuzz/regression_driver.c @@ -44,11 +44,12 @@ int main(int argc, char const **argv) { fprintf(stderr, "WARNING: No files passed to %s\n", argv[0]); for (i = 0; i < files->tableSize; ++i) { char const *fileName = files->fileNames[i]; - DEBUGLOG(3, "Running %s", fileName); size_t const fileSize = UTIL_getFileSize(fileName); size_t readSize; FILE *file; + DEBUGLOG(3, "Running %s", fileName); + /* Check that it is a regular file, and that the fileSize is valid. * If it is not a regular file, then it may have been deleted since we * constructed the list, so just skip it, but return an error exit code. diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c index ede7080e..ec0106c1 100644 --- a/tests/fuzz/sequence_compression_api.c +++ b/tests/fuzz/sequence_compression_api.c @@ -116,7 +116,7 @@ static size_t decodeSequences(void* dst, size_t nbSequences, } } for (; j < matchLength; ++j) { - op[j] = op[j - generatedSequences[i].offset]; + op[j] = op[(ptrdiff_t)(j - generatedSequences[i].offset)]; } op += j; FUZZ_ASSERT(generatedSequences[i].matchLength == j + k); diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index ce5f9f09..0dc9e5b7 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -16,6 +16,9 @@ #include <stddef.h> #include <stdlib.h> #include <stdio.h> + +#define ZSTD_STATIC_LINKING_ONLY + #include "fuzz_helpers.h" #include "zstd.h" #include "fuzz_data_producer.h" @@ -34,11 +37,18 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_ASSERT(dctx); } - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); - void *rBuf = FUZZ_malloc(bufSize); - - ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); - free(rBuf); + { + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + void *rBuf = FUZZ_malloc(bufSize); + size_t const dSize = ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); + if (!ZSTD_isError(dSize)) { + /* If decompression was successful, the content size from the frame header(s) should be valid. */ + unsigned long long const expectedSize = ZSTD_findDecompressedSize(src, size); + FUZZ_ASSERT(expectedSize != ZSTD_CONTENTSIZE_ERROR); + FUZZ_ASSERT(expectedSize == ZSTD_CONTENTSIZE_UNKNOWN || expectedSize == dSize); + } + free(rBuf); + } FUZZ_dataProducer_free(producer); diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 8b123197..660092e6 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -27,7 +27,7 @@ static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_t srcSize, int hasSmallBlocks) +static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_t srcSize, int hasSmallBlocks, int maxBlockSize) { size_t margin = ZSTD_decompressionMargin(compressed, cSize); if (!hasSmallBlocks) { @@ -37,7 +37,12 @@ static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_ ZSTD_frameHeader zfh; size_t marginM; FUZZ_ZASSERT(ZSTD_getFrameHeader(&zfh, compressed, cSize)); - marginM = ZSTD_DECOMPRESSION_MARGIN(srcSize, zfh.blockSizeMax); + if (maxBlockSize == 0) { + maxBlockSize = zfh.blockSizeMax; + } else { + maxBlockSize = MIN(maxBlockSize, (int)zfh.blockSizeMax); + } + marginM = ZSTD_DECOMPRESSION_MARGIN(srcSize, maxBlockSize); if (marginM < margin) margin = marginM; } @@ -52,12 +57,14 @@ static size_t roundTripTest(void *result, size_t resultCapacity, size_t cSize; size_t dSize; int targetCBlockSize = 0; + int maxBlockSize = 0; if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { size_t const remainingBytes = FUZZ_dataProducer_remainingBytes(producer); FUZZ_setRandomParameters(cctx, srcSize, producer); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); FUZZ_ZASSERT(cSize); FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetCBlockSize, &targetCBlockSize)); + FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_maxBlockSize, &maxBlockSize)); // Compress a second time and check for determinism { size_t const cSize0 = cSize; @@ -83,13 +90,16 @@ static size_t roundTripTest(void *result, size_t resultCapacity, FUZZ_ASSERT(XXH64(compressed, cSize, 0) == hash0); } } + if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { + FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, maxBlockSize)); + } dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); FUZZ_ZASSERT(dSize); FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, result, dSize), "Corruption!"); { - size_t margin = getDecompressionMargin(compressed, cSize, srcSize, targetCBlockSize); + size_t margin = getDecompressionMargin(compressed, cSize, srcSize, targetCBlockSize, maxBlockSize); size_t const outputSize = srcSize + margin; char* const output = (char*)FUZZ_malloc(outputSize); char* const input = output + outputSize - cSize; diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 7d277a85..6e340c81 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -63,6 +63,8 @@ static size_t compress(uint8_t *dst, size_t capacity, size_t dstSize = 0; ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); FUZZ_setRandomParameters(cctx, srcSize, producer); + int maxBlockSize; + FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_maxBlockSize, &maxBlockSize)); while (srcSize > 0) { ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer); @@ -93,6 +95,8 @@ static size_t compress(uint8_t *dst, size_t capacity, if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) { size_t const remaining = in.size - in.pos; FUZZ_setRandomParameters(cctx, remaining, producer); + /* Always use the same maxBlockSize */ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, maxBlockSize)); } mode = -1; } @@ -132,6 +136,23 @@ static size_t compress(uint8_t *dst, size_t capacity, return dstSize; } +static size_t decompress(void* dst, size_t dstCapacity, void const* src, size_t srcSize, FUZZ_dataProducer_t* producer) +{ + ZSTD_inBuffer in = {src, srcSize, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + int maxBlockSize; + FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_maxBlockSize, &maxBlockSize)); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { + FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, maxBlockSize)); + } + while (in.pos < in.size) { + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + FUZZ_ZASSERT(ret); + FUZZ_ASSERT(ret == 0); + } + return out.pos; +} + int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { FUZZ_SEQ_PROD_SETUP(); @@ -163,8 +184,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const cSize = compress(cBuf, neededBufSize, src, size, producer); - size_t const rSize = - ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); + size_t const rSize = decompress(rBuf, neededBufSize, cBuf, cSize, producer); FUZZ_ZASSERT(rSize); FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!"); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 07ddfefd..f7bdae90 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -328,7 +328,7 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, if (seqs[i].offset != 0) { for (j = 0; j < seqs[i].matchLength; ++j) - dst[j] = dst[j - seqs[i].offset]; + dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)]; dst += seqs[i].matchLength; src += seqs[i].matchLength; size -= seqs[i].matchLength; @@ -376,7 +376,7 @@ static int threadPoolTests(void) { RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0); - DISPLAYLEVEL(3, "thread pool test : threadPool re-use roundtrips: "); + DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: "); { ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); @@ -531,7 +531,7 @@ static void test_decompressBound(unsigned tnb) CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 ); } CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); - CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000LLU /* 4 GB */ ); + CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ ); ZSTD_freeCCtx(cctx); free(outBuffer); } @@ -953,6 +953,25 @@ static int basicUnitTests(U32 const seed, double compressibility) ZSTD_freeCCtx(cctx); } + DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K", testNb++); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048)); + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048)); + + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSize); + CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)); + + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024)); + CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize))); + + ZSTD_freeDCtx(dctx); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); @@ -1100,6 +1119,9 @@ static int basicUnitTests(U32 const seed, double compressibility) size_t const srcSize1 = kWindowSize / 2; size_t const srcSize2 = kWindowSize * 10; + CHECK(cctx!=NULL); + CHECK(dctx!=NULL); + CHECK(dict!=NULL); if (CNBuffSize < dictSize) goto _output_error; RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); @@ -1121,6 +1143,7 @@ static int basicUnitTests(U32 const seed, double compressibility) cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); CHECK_Z(cSize); CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); /* Streaming decompression to catch out of bounds offsets. */ { @@ -1134,24 +1157,22 @@ static int basicUnitTests(U32 const seed, double compressibility) CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); /* Round trip once with a dictionary. */ CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); - { - ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; + { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); cSize = out.pos; } CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); - { - ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; + + { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); cSize = out.pos; } /* Streaming decompression to catch out of bounds offsets. */ - { - ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + { ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); CHECK_Z(dSize); @@ -1353,7 +1374,7 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++); + DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer; @@ -1506,14 +1527,14 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++); + DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ - CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ + CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ { size_t const inSize = 2* 128 KB; size_t const outSize = ZSTD_compressBound(inSize); CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); @@ -1808,7 +1829,7 @@ static int basicUnitTests(U32 const seed, double compressibility) params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; for (cnb = 0; cnb < nbCompressions; ++cnb) { DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions); - CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ + CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); } ZSTD_freeCCtx(cctx); @@ -2407,6 +2428,14 @@ static int basicUnitTests(U32 const seed, double compressibility) } } DISPLAYLEVEL(3, "OK \n"); +#if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ + && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ + && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) /* Note : these tests should be replaced by proper regression tests, * but existing ones do not focus on small data + dictionary + all levels. */ @@ -2505,6 +2534,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(4, "compression efficiency tests OK \n"); } +#endif ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); @@ -3656,11 +3686,13 @@ static int basicUnitTests(U32 const seed, double compressibility) /* Test with block delimiters roundtrip */ seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); + CHECK_Z(seqsSize); FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters); assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); /* Test no block delimiters roundtrip */ seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); + CHECK_Z(seqsSize); FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters); assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); @@ -3669,6 +3701,31 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++); + { + const size_t seqsCapacity = 10; + const size_t srcSize = 150 KB; + const BYTE* src = (BYTE*)CNBuffer; + + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence)); + + if (seqs == NULL) goto _output_error; + if (cctx == NULL) goto _output_error; + /* Populate src with random data */ + RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); + + /* Test with block delimiters roundtrip */ + { + size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize); + if (!ZSTD_isError(seqsSize)) goto _output_error; + } + + ZSTD_freeCCtx(cctx); + free(seqs); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); { const size_t srcSize = 500 KB; diff --git a/tests/golden-decompression-errors/.gitignore b/tests/golden-decompression-errors/.gitignore new file mode 100644 index 00000000..574b3750 --- /dev/null +++ b/tests/golden-decompression-errors/.gitignore @@ -0,0 +1 @@ +!*.zst diff --git a/tests/golden-decompression-errors/off0.bin.zst b/tests/golden-decompression-errors/off0.bin.zst Binary files differnew file mode 100644 index 00000000..13493fb3 --- /dev/null +++ b/tests/golden-decompression-errors/off0.bin.zst diff --git a/tests/golden-decompression-errors/zeroSeq_extraneous.zst b/tests/golden-decompression-errors/zeroSeq_extraneous.zst Binary files differnew file mode 100644 index 00000000..0953be34 --- /dev/null +++ b/tests/golden-decompression-errors/zeroSeq_extraneous.zst diff --git a/tests/golden-decompression/block-128k.zst b/tests/golden-decompression/block-128k.zst Binary files differnew file mode 100644 index 00000000..cdaeae39 --- /dev/null +++ b/tests/golden-decompression/block-128k.zst diff --git a/tests/golden-decompression/zeroSeq_2B.zst b/tests/golden-decompression/zeroSeq_2B.zst Binary files differnew file mode 100644 index 00000000..f9f3520a --- /dev/null +++ b/tests/golden-decompression/zeroSeq_2B.zst diff --git a/tests/loremOut.c b/tests/loremOut.c new file mode 100644 index 00000000..9fb48b1c --- /dev/null +++ b/tests/loremOut.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Implementation notes: + * Generates a stream of Lorem ipsum paragraphs to stdout, + * up to the requested size, which can be very large (> 4 GB). + * Note that, beyond 1 paragraph, this generator produces + * a different content than LOREM_genBuffer (even when using same seed). + */ + +#include "loremOut.h" +#include <assert.h> +#include <stdio.h> +#include "lorem.h" /* LOREM_genBlock */ +#include "platform.h" /* Compiler options, SET_BINARY_MODE */ + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define LOREM_BLOCKSIZE (1 << 10) +void LOREM_genOut(unsigned long long size, unsigned seed) +{ + char buff[LOREM_BLOCKSIZE] = { 0 }; + unsigned long long total = 0; + size_t genBlockSize = (size_t)MIN(size, LOREM_BLOCKSIZE); + + /* init */ + SET_BINARY_MODE(stdout); + + /* Generate Ipsum text, one paragraph at a time */ + while (total < size) { + size_t generated = + LOREM_genBlock(buff, genBlockSize, seed++, total == 0, 0); + assert(generated <= genBlockSize); + total += generated; + assert(total <= size); + fwrite(buff, + 1, + generated, + stdout); /* note: should check potential write error */ + if (size - total < genBlockSize) + genBlockSize = (size_t)(size - total); + } + assert(total == size); +} diff --git a/tests/loremOut.h b/tests/loremOut.h new file mode 100644 index 00000000..3a32e116 --- /dev/null +++ b/tests/loremOut.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* LOREM_genOut(): + * Generate @size bytes of compressible data using lorem ipsum generator into + * stdout. + */ +void LOREM_genOut(unsigned long long size, unsigned seed); diff --git a/tests/playTests.sh b/tests/playTests.sh index 5f595f61..e2a0694f 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1,6 +1,7 @@ #!/bin/sh -set -e +set -e # exit immediately on error +# set -x # print commands before execution (debug) unset ZSTD_CLEVEL unset ZSTD_NBTHREADS @@ -16,18 +17,18 @@ datagen() { } zstd() { - if [ -z "$EXEC_PREFIX" ]; then + if [ -z "$EXE_PREFIX" ]; then "$ZSTD_BIN" "$@" else - "$EXEC_PREFIX" "$ZSTD_BIN" "$@" + "$EXE_PREFIX" "$ZSTD_BIN" "$@" fi } sudoZstd() { - if [ -z "$EXEC_PREFIX" ]; then + if [ -z "$EXE_PREFIX" ]; then sudo "$ZSTD_BIN" "$@" else - sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@" + sudo "$EXE_PREFIX" "$ZSTD_BIN" "$@" fi } @@ -91,7 +92,13 @@ fi SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) PRGDIR="$SCRIPT_DIR/../programs" TESTDIR="$SCRIPT_DIR/../tests" -UNAME=$(uname) +UNAME=${UNAME:-$(uname)} +GREP=${GREP:-grep} + +case "$UNAME" in + SunOS) DIFF=${DIFF:-gdiff} ;; + *) DIFF=${DIFF:-diff} ;; +esac detectedTerminal=false if [ -t 0 ] && [ -t 1 ] @@ -151,11 +158,6 @@ assertSamePermissions() { [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" } -DIFF="diff" -case "$UNAME" in - SunOS) DIFF="gdiff" ;; -esac - # check if ZSTD_BIN is defined. if not, use the default value if [ -z "${ZSTD_BIN}" ]; then @@ -177,7 +179,7 @@ fi [ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" -if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' +if echo hello | zstd -v -T2 2>&1 > $INTOVOID | $GREP -q 'multi-threading is disabled' then hasMT="" else @@ -232,12 +234,23 @@ unset ZSTD_CLEVEL println "test : compress to stdout" zstd tmp -c > tmpCompressed zstd tmp --stdout > tmpCompressed # long command format -println "test : compress to named file" + +println "test : compress to named file (-o)" rm -f tmpCompressed zstd tmp -o tmpCompressed test -f tmpCompressed # file must be created + println "test : force write, correct order" zstd tmp -fo tmpCompressed + +println "test : -c + -o : last one wins" +rm -f tmpOut +zstd tmp -c > tmpCompressed -o tmpOut +test -f tmpOut # file must be created +rm -f tmpCompressed +zstd tmp -o tmpOut -c > tmpCompressed +test -f tmpCompressed # file must be created + println "test : forgotten argument" cp tmp tmp2 zstd tmp2 -fo && die "-o must be followed by filename " @@ -253,8 +266,8 @@ println "test : null-length file roundtrip" println -n '' | zstd - --stdout | zstd -d --stdout println "test : ensure small file doesn't add 3-bytes null block" datagen -g1 > tmp1 -zstd tmp1 -c | wc -c | grep "14" -zstd < tmp1 | wc -c | grep "14" +zstd tmp1 -c | wc -c | $GREP "14" +zstd < tmp1 | wc -c | $GREP "14" println "test : decompress file with wrong suffix (must fail)" zstd -d tmpCompressed && die "wrong suffix error not detected!" zstd -df tmp && die "should have refused : wrong extension" @@ -291,9 +304,9 @@ println "test: --no-progress flag" zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress println "test: --progress flag" -zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes" -zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes" -zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes" +zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' +zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' +zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' rm -f tmpro tmpro.zst println "test: overwrite input file (must fail)" zstd tmp -fo tmp && die "zstd compression overwrote the input file" @@ -320,10 +333,55 @@ zstd -d -f tmp.zst --no-check if [ "$isWindows" = false ] && [ "$UNAME" != "AIX" ]; then if [ -n "$(which readelf)" ]; then println "test: check if binary has executable stack (#2963)" - readelf -lW "$ZSTD_BIN" | grep 'GNU_STACK .* RW ' || die "zstd binary has executable stack!" + readelf -lW "$ZSTD_BIN" | $GREP 'GNU_STACK .* RW ' || die "zstd binary has executable stack!" fi fi +println "\n===> multiple_thread test " + +datagen > tmp +println "test : single-thread " +zstd --fast --single-thread tmp -o tmpMT0 +println "test : one worker thread (default)" +zstd --fast -T1 tmp -o tmpMT1 +println "test : two worker threads " +zstd --fast -T2 tmp -o tmpMT2 +println "test : 16-thread " +zstd --fast -T16 tmp -o tmpMT3 +println "test : 127-thread " +zstd --fast -T127 tmp -o tmpMT4 +println "test : 128-thread " +zstd --fast -T128 tmp -o tmpMT5 +println "test : max allowed numeric value is 4294967295 " +zstd --fast -4294967295 tmp -o tmpMT6 +println "test : numeric value overflows 32-bit unsigned int " +zstd --fast -4294967296 tmp -o tmptest9 && die "max allowed numeric value is 4294967295" + +datagen > tmp +println "test : basic compression " +zstd -f tmp # trivial compression case, creates tmp.zst +println "test : basic decompression" +zstd -d -f -T1 tmp.zst +println "note : decompression does not support -T mode, but execution support" +rm -rf tmpMT* + +println "\n===> --fast_argument test " +datagen > tmp +println "test : basic compression " +zstd -f tmp # trivial compression case, creates tmp.zst +println "test: --fast=1" +zstd --fast=1 -f tmp +println "test: --fast=99" +zstd --fast=99 -f tmp +println "test: Invalid value -- negative number" +zstd --fast=-1 -f tmp && die "error: Invalid value -- negative number" +println "test: Invalid value -- zero" +zstd --fast=0 -f tmp && die "error: Invalid value -- 0 number" +println "test: max allowed numeric argument of --fast is 4294967295" +zstd --fast=4294967295 -f tmp +println "test: numeric value overflows 32-bit unsigned int " +zstd --fast=4294967296 -f tmp && die "max allowed argument of --fast is 4294967295" + println "\n===> --exclude-compressed flag" rm -rf precompressedFilterTestDir mkdir -p precompressedFilterTestDir @@ -352,6 +410,19 @@ zstd --long --rm -r precompressedFilterTestDir # Files should get compressed again without the --exclude-compressed flag. test -f precompressedFilterTestDir/input.5.zst.zst test -f precompressedFilterTestDir/input.6.zst.zst + +# Test some other compressed file extensions +datagen $size > precompressedFilterTestDir/input.flac +datagen $size > precompressedFilterTestDir/input.mov +datagen $size > precompressedFilterTestDir/input.mp3 +zstd --exclude-compressed --long --rm -r precompressedFilterTestDir +test ! -f precompressedFilterTestDir/input.flac.zst +test ! -f precompressedFilterTestDir/input.mov.zst +test ! -f precompressedFilterTestDir/input.mp3.zst +zstd --long --rm -r precompressedFilterTestDir +test -f precompressedFilterTestDir/input.flac.zst +test -f precompressedFilterTestDir/input.mov.zst +test -f precompressedFilterTestDir/input.mp3.zst rm -rf precompressedFilterTestDir println "Test completed" @@ -392,6 +463,8 @@ println "test: --rm is disabled when output is stdout" test -f tmp zstd --rm tmp -c > $INTOVOID test -f tmp # tmp shall still be there +zstd --rm tmp --stdout > $INTOVOID +test -f tmp # tmp shall still be there zstd -f --rm tmp -c > $INTOVOID test -f tmp # tmp shall still be there zstd -f tmp -c > $INTOVOID --rm @@ -409,13 +482,28 @@ zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt test -f tmp test -f tmp2 zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate" - +println "test: --rm is active with -o when single input" +rm -f tmp2.zst +zstd --rm tmp2 -o tmp2.zst +test -f tmp2.zst +test ! -f tmp2 +println "test: -c followed by -o => -o wins, so --rm remains active" # (#3719) +rm tmp2.zst +cp tmp tmp2 +zstd --rm tmp2 -c > $INTOVOID -o tmp2.zst +test ! -f tmp2 +println "test: -o followed by -c => -c wins, so --rm is disabled" # (#3719) +rm tmp3.zst +cp tmp tmp2 +zstd -v --rm tmp2 -o tmp2.zst -c > tmp3.zst +test -f tmp2 +test -f tmp3.zst println "test : should quietly not remove non-regular file" println hello > tmp zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" -grep "Refusing to remove non-regular file" tmplog && die +$GREP "Refusing to remove non-regular file" tmplog && die rm -f tmplog -zstd tmp -f -o "$INTOVOID" 2>&1 | grep "Refusing to remove non-regular file" && die +zstd tmp -f -o "$INTOVOID" 2>&1 | $GREP "Refusing to remove non-regular file" && die println "test : --rm on stdin" println a | zstd --rm > $INTOVOID # --rm should remain silent rm -f tmp @@ -444,6 +532,11 @@ $DIFF -s tmp1 tmp touch tmp_empty zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst" $DIFF -s tmp2 tmp_empty + +zstd -t "$TESTDIR/golden-decompression/zeroSeq_2B.zst" + +zstd -t "$TESTDIR/golden-decompression-errors/zeroSeq_extraneous.zst" && die "invalid Sequences section should have been detected" + rm -f tmp* println "\n===> compress multiple files" @@ -610,7 +703,7 @@ if [ -n "$DEVNULLRIGHTS" ] ; then zstd tmp -f -o tmp.zst sudoZstd -d tmp.zst -c > $INTOVOID sudoZstd -d tmp.zst -o $INTOVOID - ls -las $INTOVOID | grep "rw-rw-rw-" + ls -las $INTOVOID | $GREP "rw-rw-rw-" fi if [ -n "$READFROMBLOCKDEVICE" ] ; then @@ -620,7 +713,7 @@ if [ -n "$READFROMBLOCKDEVICE" ] ; then println "\n===> checking that zstd can read from a block device" datagen -g65536 > tmp.img sudo losetup -fP tmp.img - LOOP_DEV=$(losetup -a | grep 'tmp\.img' | cut -f1 -d:) + LOOP_DEV=$(losetup -a | $GREP 'tmp\.img' | cut -f1 -d:) [ -z "$LOOP_DEV" ] && die "failed to get loopback device" sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" sudoZstd -f $LOOP_DEV -c > tmp.img.zst @@ -769,13 +862,13 @@ println "\n===> --[no-]content-size tests" datagen > tmp_contentsize zstd -f tmp_contentsize -zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" +zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" zstd -f --no-content-size tmp_contentsize -zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die +zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die zstd -f --content-size tmp_contentsize -zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" +zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" zstd -f --content-size --no-content-size tmp_contentsize -zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die +zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die rm -rf tmp* println "test : show-default-cparams regular" @@ -795,8 +888,7 @@ rm -rf tmp* println "test : show compression parameters in verbose mode" datagen > tmp zstd -vv tmp 2>&1 | \ -grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\ -slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+" +$GREP -q -- "--zstd=wlog=[0-9]*,clog=[0-9]*,hlog=[0-9]*,slog=[0-9]*,mml=[0-9]*,tlen=[0-9]*,strat=[0-9]*" rm -rf tmp* println "\n===> Advanced compression parameters " @@ -1093,8 +1185,8 @@ println "- Test --memory for dictionary compression" datagen -g12M -P90 > tmpCorpusHighCompress zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr -cat zstTrainWithMemLimitStdErr | grep "setting manual memory limit for dictionary training data at 5 MB" -cat zstTrainWithMemLimitStdErr | grep "Training samples set too large (12 MB); training on 5 MB only..." +cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB" +cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..." rm zstTrainWithMemLimitStdErr println "\n===> fastCover dictionary builder : advanced options " @@ -1380,16 +1472,16 @@ println "\n===> suffix list test" ! zstd -d tmp.abc 2> tmplg if [ $GZIPMODE -ne 1 ]; then - grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" + $GREP ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" fi if [ $LZMAMODE -ne 1 ]; then - grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" - grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" + $GREP ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" + $GREP ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" fi if [ $LZ4MODE -ne 1 ]; then - grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" + $GREP ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" fi touch tmp1 @@ -1518,7 +1610,7 @@ datagen > tmp2 datagen > tmp3 zstd tmp* zstd -l ./*.zst -zstd -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header +zstd -lv ./*.zst | $GREP "Decompressed Size:" # check that decompressed size is present in header zstd --list ./*.zst zstd --list -v ./*.zst @@ -1561,13 +1653,13 @@ datagen -g0 > tmp5 zstd tmp5 zstd -l tmp5.zst zstd -l tmp5* && die "-l must fail on non-zstd file" -zstd -lv tmp5.zst | grep "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header +zstd -lv tmp5.zst | $GREP "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header zstd -lv tmp5* && die "-l must fail on non-zstd file" println "\n===> zstd --list/-l test with no content size field " datagen -g513K | zstd > tmp6.zst zstd -l tmp6.zst -zstd -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" +zstd -lv tmp6.zst | $GREP "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" println "\n===> zstd --list/-l test with no checksum " zstd -f --no-check tmp1 @@ -1602,22 +1694,24 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" -println "\n===> zstd long distance matching with optimal parser compressed size tests " -optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) -longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) -optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) -longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) -optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) -longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) -if [ "$longCSize16" -gt "$optCSize16" ]; then - echo using --long on compression level 16 should not cause compressed size regression - exit 1 -elif [ "$longCSize19" -gt "$optCSize19" ]; then - echo using --long on compression level 19 should not cause compressed size regression - exit 1 -elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then - echo using --long on compression level 19 with wLog=23 should not cause compressed size regression - exit 1 +if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then + println "\n===> zstd long distance matching with optimal parser compressed size tests " + optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) + longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) + optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) + longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) + optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) + longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) + if [ "$longCSize16" -gt "$optCSize16" ]; then + echo using --long on compression level 16 should not cause compressed size regression + exit 1 + elif [ "$longCSize19" -gt "$optCSize19" ]; then + echo using --long on compression level 19 should not cause compressed size regression + exit 1 + elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then + echo using --long on compression level 19 with wLog=23 should not cause compressed size regression + exit 1 + fi fi println "\n===> zstd asyncio tests " @@ -1708,9 +1802,15 @@ zstd --patch-from=tmp_dict -r tmp_dir && die rm -rf tmp* println "\n===> patch-from long mode trigger larger file test" -datagen -g5000000 > tmp_dict -datagen -g5000000 > tmp_patch -zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" +if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -eq "1" ]; then + # if binary tree strategies are excluded, the threshold is different + datagen -g10000000 > tmp_dict + datagen -g10000000 > tmp_patch +else + datagen -g5000000 > tmp_dict + datagen -g5000000 > tmp_patch +fi +zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | $GREP "long mode automatically triggered" rm -rf tmp* println "\n===> patch-from very large dictionary and file test" diff --git a/tests/regression/results.csv b/tests/regression/results.csv index d072c0d8..fc3fbe7c 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -11,10 +11,10 @@ silesia.tar, level 6, compress silesia.tar, level 7, compress simple, 4579828 silesia.tar, level 9, compress simple, 4555448 silesia.tar, level 13, compress simple, 4502956 -silesia.tar, level 16, compress simple, 4360546 -silesia.tar, level 19, compress simple, 4265911 +silesia.tar, level 16, compress simple, 4360385 +silesia.tar, level 19, compress simple, 4260939 silesia.tar, uncompressed literals, compress simple, 4854086 -silesia.tar, uncompressed literals optimal, compress simple, 4265911 +silesia.tar, uncompressed literals optimal, compress simple, 4260939 silesia.tar, huffman literals, compress simple, 6179047 github.tar, level -5, compress simple, 52115 github.tar, level -3, compress simple, 45678 @@ -29,9 +29,9 @@ github.tar, level 7, compress github.tar, level 9, compress simple, 36723 github.tar, level 13, compress simple, 35501 github.tar, level 16, compress simple, 40466 -github.tar, level 19, compress simple, 32276 +github.tar, level 19, compress simple, 32262 github.tar, uncompressed literals, compress simple, 38831 -github.tar, uncompressed literals optimal, compress simple, 32276 +github.tar, uncompressed literals optimal, compress simple, 32262 github.tar, huffman literals, compress simple, 42560 silesia, level -5, compress cctx, 6857372 silesia, level -3, compress cctx, 6503412 @@ -45,8 +45,8 @@ silesia, level 6, compress silesia, level 7, compress cctx, 4570271 silesia, level 9, compress cctx, 4545850 silesia, level 13, compress cctx, 4493990 -silesia, level 16, compress cctx, 4360041 -silesia, level 19, compress cctx, 4296055 +silesia, level 16, compress cctx, 4359652 +silesia, level 19, compress cctx, 4266582 silesia, long distance mode, compress cctx, 4842075 silesia, multithreaded, compress cctx, 4842075 silesia, multithreaded long distance mode, compress cctx, 4842075 @@ -55,7 +55,7 @@ silesia, small hash log, compress silesia, small chain log, compress cctx, 4912197 silesia, explicit params, compress cctx, 4794318 silesia, uncompressed literals, compress cctx, 4842075 -silesia, uncompressed literals optimal, compress cctx, 4296055 +silesia, uncompressed literals optimal, compress cctx, 4266582 silesia, huffman literals, compress cctx, 6172202 silesia, multithreaded with advanced params, compress cctx, 4842075 github, level -5, compress cctx, 204407 @@ -83,9 +83,9 @@ github, level 9 with dict, compress github, level 13, compress cctx, 132878 github, level 13 with dict, compress cctx, 39948 github, level 16, compress cctx, 133209 -github, level 16 with dict, compress cctx, 37568 +github, level 16 with dict, compress cctx, 37892 github, level 19, compress cctx, 132879 -github, level 19 with dict, compress cctx, 37567 +github, level 19 with dict, compress cctx, 37906 github, long distance mode, compress cctx, 141069 github, multithreaded, compress cctx, 141069 github, multithreaded long distance mode, compress cctx, 141069 @@ -109,8 +109,8 @@ silesia, level 6, zstdcli, silesia, level 7, zstdcli, 4570319 silesia, level 9, zstdcli, 4545898 silesia, level 13, zstdcli, 4494038 -silesia, level 16, zstdcli, 4360089 -silesia, level 19, zstdcli, 4296103 +silesia, level 16, zstdcli, 4359700 +silesia, level 19, zstdcli, 4266630 silesia, long distance mode, zstdcli, 4833785 silesia, multithreaded, zstdcli, 4842123 silesia, multithreaded long distance mode, zstdcli, 4833785 @@ -119,7 +119,7 @@ silesia, small hash log, zstdcli, silesia, small chain log, zstdcli, 4912245 silesia, explicit params, zstdcli, 4795840 silesia, uncompressed literals, zstdcli, 5120614 -silesia, uncompressed literals optimal, zstdcli, 4319566 +silesia, uncompressed literals optimal, zstdcli, 4316928 silesia, huffman literals, zstdcli, 5321417 silesia, multithreaded with advanced params, zstdcli, 5120614 silesia.tar, level -5, zstdcli, 6862049 @@ -134,8 +134,8 @@ silesia.tar, level 6, zstdcli, silesia.tar, level 7, zstdcli, 4581791 silesia.tar, level 9, zstdcli, 4555452 silesia.tar, level 13, zstdcli, 4502960 -silesia.tar, level 16, zstdcli, 4360550 -silesia.tar, level 19, zstdcli, 4265915 +silesia.tar, level 16, zstdcli, 4360389 +silesia.tar, level 19, zstdcli, 4260943 silesia.tar, no source size, zstdcli, 4854160 silesia.tar, long distance mode, zstdcli, 4845745 silesia.tar, multithreaded, zstdcli, 4854164 @@ -145,7 +145,7 @@ silesia.tar, small hash log, zstdcli, silesia.tar, small chain log, zstdcli, 4917022 silesia.tar, explicit params, zstdcli, 4821112 silesia.tar, uncompressed literals, zstdcli, 5122571 -silesia.tar, uncompressed literals optimal, zstdcli, 4310145 +silesia.tar, uncompressed literals optimal, zstdcli, 4308455 silesia.tar, huffman literals, zstdcli, 5342074 silesia.tar, multithreaded with advanced params, zstdcli, 5122571 github, level -5, zstdcli, 206407 @@ -173,9 +173,9 @@ github, level 9 with dict, zstdcli, github, level 13, zstdcli, 134878 github, level 13 with dict, zstdcli, 41900 github, level 16, zstdcli, 135209 -github, level 16 with dict, zstdcli, 39577 +github, level 16 with dict, zstdcli, 39902 github, level 19, zstdcli, 134879 -github, level 19 with dict, zstdcli, 39576 +github, level 19 with dict, zstdcli, 39916 github, long distance mode, zstdcli, 138332 github, multithreaded, zstdcli, 138332 github, multithreaded long distance mode, zstdcli, 138332 @@ -212,9 +212,9 @@ github.tar, level 9 with dict, zstdcli, github.tar, level 13, zstdcli, 35505 github.tar, level 13 with dict, zstdcli, 37134 github.tar, level 16, zstdcli, 40470 -github.tar, level 16 with dict, zstdcli, 33378 -github.tar, level 19, zstdcli, 32280 -github.tar, level 19 with dict, zstdcli, 32716 +github.tar, level 16 with dict, zstdcli, 33379 +github.tar, level 19, zstdcli, 32266 +github.tar, level 19 with dict, zstdcli, 32705 github.tar, no source size, zstdcli, 38832 github.tar, no source size with dict, zstdcli, 38004 github.tar, long distance mode, zstdcli, 40236 @@ -225,7 +225,7 @@ github.tar, small hash log, zstdcli, github.tar, small chain log, zstdcli, 41673 github.tar, explicit params, zstdcli, 41385 github.tar, uncompressed literals, zstdcli, 41529 -github.tar, uncompressed literals optimal, zstdcli, 35401 +github.tar, uncompressed literals optimal, zstdcli, 35360 github.tar, huffman literals, zstdcli, 38857 github.tar, multithreaded with advanced params, zstdcli, 41529 silesia, level -5, advanced one pass, 6857372 @@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass, 4505658 silesia, level 12 row 2, advanced one pass, 4503429 silesia, level 13, advanced one pass, 4493990 -silesia, level 16, advanced one pass, 4360041 -silesia, level 19, advanced one pass, 4296055 +silesia, level 16, advanced one pass, 4359652 +silesia, level 19, advanced one pass, 4266582 silesia, no source size, advanced one pass, 4842075 silesia, long distance mode, advanced one pass, 4833710 silesia, multithreaded, advanced one pass, 4842075 @@ -259,7 +259,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass, 4912197 silesia, explicit params, advanced one pass, 4795840 silesia, uncompressed literals, advanced one pass, 5120566 -silesia, uncompressed literals optimal, advanced one pass, 4319518 +silesia, uncompressed literals optimal, advanced one pass, 4316880 silesia, huffman literals, advanced one pass, 5321369 silesia, multithreaded with advanced params, advanced one pass, 5120566 silesia.tar, level -5, advanced one pass, 6861055 @@ -282,8 +282,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass, 4514517 silesia.tar, level 12 row 2, advanced one pass, 4514007 silesia.tar, level 13, advanced one pass, 4502956 -silesia.tar, level 16, advanced one pass, 4360546 -silesia.tar, level 19, advanced one pass, 4265911 +silesia.tar, level 16, advanced one pass, 4360385 +silesia.tar, level 19, advanced one pass, 4260939 silesia.tar, no source size, advanced one pass, 4854086 silesia.tar, long distance mode, advanced one pass, 4840452 silesia.tar, multithreaded, advanced one pass, 4854160 @@ -293,7 +293,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass, 4917041 silesia.tar, explicit params, advanced one pass, 4807274 silesia.tar, uncompressed literals, advanced one pass, 5122473 -silesia.tar, uncompressed literals optimal, advanced one pass, 4310141 +silesia.tar, uncompressed literals optimal, advanced one pass, 4308451 silesia.tar, huffman literals, advanced one pass, 5341705 silesia.tar, multithreaded with advanced params, advanced one pass, 5122567 github, level -5, advanced one pass, 204407 @@ -397,17 +397,17 @@ github, level 13 with dict dds, advanced github, level 13 with dict copy, advanced one pass, 39948 github, level 13 with dict load, advanced one pass, 42624 github, level 16, advanced one pass, 133209 -github, level 16 with dict, advanced one pass, 37577 -github, level 16 with dict dms, advanced one pass, 37577 -github, level 16 with dict dds, advanced one pass, 37577 -github, level 16 with dict copy, advanced one pass, 37568 -github, level 16 with dict load, advanced one pass, 42338 +github, level 16 with dict, advanced one pass, 37902 +github, level 16 with dict dms, advanced one pass, 37902 +github, level 16 with dict dds, advanced one pass, 37902 +github, level 16 with dict copy, advanced one pass, 37892 +github, level 16 with dict load, advanced one pass, 42402 github, level 19, advanced one pass, 132879 -github, level 19 with dict, advanced one pass, 37576 -github, level 19 with dict dms, advanced one pass, 37576 -github, level 19 with dict dds, advanced one pass, 37576 -github, level 19 with dict copy, advanced one pass, 37567 -github, level 19 with dict load, advanced one pass, 39613 +github, level 19 with dict, advanced one pass, 37916 +github, level 19 with dict dms, advanced one pass, 37916 +github, level 19 with dict dds, advanced one pass, 37916 +github, level 19 with dict copy, advanced one pass, 37906 +github, level 19 with dict load, advanced one pass, 39770 github, no source size, advanced one pass, 136332 github, no source size with dict, advanced one pass, 41148 github, long distance mode, advanced one pass, 136332 @@ -522,17 +522,17 @@ github.tar, level 13 with dict dds, advanced github.tar, level 13 with dict copy, advanced one pass, 37130 github.tar, level 13 with dict load, advanced one pass, 36010 github.tar, level 16, advanced one pass, 40466 -github.tar, level 16 with dict, advanced one pass, 33374 -github.tar, level 16 with dict dms, advanced one pass, 33206 -github.tar, level 16 with dict dds, advanced one pass, 33206 -github.tar, level 16 with dict copy, advanced one pass, 33374 +github.tar, level 16 with dict, advanced one pass, 33375 +github.tar, level 16 with dict dms, advanced one pass, 33207 +github.tar, level 16 with dict dds, advanced one pass, 33207 +github.tar, level 16 with dict copy, advanced one pass, 33375 github.tar, level 16 with dict load, advanced one pass, 39081 -github.tar, level 19, advanced one pass, 32276 -github.tar, level 19 with dict, advanced one pass, 32712 -github.tar, level 19 with dict dms, advanced one pass, 32555 -github.tar, level 19 with dict dds, advanced one pass, 32555 -github.tar, level 19 with dict copy, advanced one pass, 32712 -github.tar, level 19 with dict load, advanced one pass, 32479 +github.tar, level 19, advanced one pass, 32262 +github.tar, level 19 with dict, advanced one pass, 32701 +github.tar, level 19 with dict dms, advanced one pass, 32565 +github.tar, level 19 with dict dds, advanced one pass, 32565 +github.tar, level 19 with dict copy, advanced one pass, 32701 +github.tar, level 19 with dict load, advanced one pass, 32428 github.tar, no source size, advanced one pass, 38831 github.tar, no source size with dict, advanced one pass, 37995 github.tar, long distance mode, advanced one pass, 40252 @@ -543,7 +543,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass, 41669 github.tar, explicit params, advanced one pass, 41385 github.tar, uncompressed literals, advanced one pass, 41525 -github.tar, uncompressed literals optimal, advanced one pass, 35397 +github.tar, uncompressed literals optimal, advanced one pass, 35356 github.tar, huffman literals, advanced one pass, 38853 github.tar, multithreaded with advanced params, advanced one pass, 41525 silesia, level -5, advanced one pass small out, 6857372 @@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass small out, 4505658 silesia, level 12 row 2, advanced one pass small out, 4503429 silesia, level 13, advanced one pass small out, 4493990 -silesia, level 16, advanced one pass small out, 4360041 -silesia, level 19, advanced one pass small out, 4296055 +silesia, level 16, advanced one pass small out, 4359652 +silesia, level 19, advanced one pass small out, 4266582 silesia, no source size, advanced one pass small out, 4842075 silesia, long distance mode, advanced one pass small out, 4833710 silesia, multithreaded, advanced one pass small out, 4842075 @@ -577,7 +577,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass small out, 4912197 silesia, explicit params, advanced one pass small out, 4795840 silesia, uncompressed literals, advanced one pass small out, 5120566 -silesia, uncompressed literals optimal, advanced one pass small out, 4319518 +silesia, uncompressed literals optimal, advanced one pass small out, 4316880 silesia, huffman literals, advanced one pass small out, 5321369 silesia, multithreaded with advanced params, advanced one pass small out, 5120566 silesia.tar, level -5, advanced one pass small out, 6861055 @@ -600,8 +600,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass small out, 4514517 silesia.tar, level 12 row 2, advanced one pass small out, 4514007 silesia.tar, level 13, advanced one pass small out, 4502956 -silesia.tar, level 16, advanced one pass small out, 4360546 -silesia.tar, level 19, advanced one pass small out, 4265911 +silesia.tar, level 16, advanced one pass small out, 4360385 +silesia.tar, level 19, advanced one pass small out, 4260939 silesia.tar, no source size, advanced one pass small out, 4854086 silesia.tar, long distance mode, advanced one pass small out, 4840452 silesia.tar, multithreaded, advanced one pass small out, 4854160 @@ -611,7 +611,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass small out, 4917041 silesia.tar, explicit params, advanced one pass small out, 4807274 silesia.tar, uncompressed literals, advanced one pass small out, 5122473 -silesia.tar, uncompressed literals optimal, advanced one pass small out, 4310141 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4308451 silesia.tar, huffman literals, advanced one pass small out, 5341705 silesia.tar, multithreaded with advanced params, advanced one pass small out, 5122567 github, level -5, advanced one pass small out, 204407 @@ -715,17 +715,17 @@ github, level 13 with dict dds, advanced github, level 13 with dict copy, advanced one pass small out, 39948 github, level 13 with dict load, advanced one pass small out, 42624 github, level 16, advanced one pass small out, 133209 -github, level 16 with dict, advanced one pass small out, 37577 -github, level 16 with dict dms, advanced one pass small out, 37577 -github, level 16 with dict dds, advanced one pass small out, 37577 -github, level 16 with dict copy, advanced one pass small out, 37568 -github, level 16 with dict load, advanced one pass small out, 42338 +github, level 16 with dict, advanced one pass small out, 37902 +github, level 16 with dict dms, advanced one pass small out, 37902 +github, level 16 with dict dds, advanced one pass small out, 37902 +github, level 16 with dict copy, advanced one pass small out, 37892 +github, level 16 with dict load, advanced one pass small out, 42402 github, level 19, advanced one pass small out, 132879 -github, level 19 with dict, advanced one pass small out, 37576 -github, level 19 with dict dms, advanced one pass small out, 37576 -github, level 19 with dict dds, advanced one pass small out, 37576 -github, level 19 with dict copy, advanced one pass small out, 37567 -github, level 19 with dict load, advanced one pass small out, 39613 +github, level 19 with dict, advanced one pass small out, 37916 +github, level 19 with dict dms, advanced one pass small out, 37916 +github, level 19 with dict dds, advanced one pass small out, 37916 +github, level 19 with dict copy, advanced one pass small out, 37906 +github, level 19 with dict load, advanced one pass small out, 39770 github, no source size, advanced one pass small out, 136332 github, no source size with dict, advanced one pass small out, 41148 github, long distance mode, advanced one pass small out, 136332 @@ -840,17 +840,17 @@ github.tar, level 13 with dict dds, advanced github.tar, level 13 with dict copy, advanced one pass small out, 37130 github.tar, level 13 with dict load, advanced one pass small out, 36010 github.tar, level 16, advanced one pass small out, 40466 -github.tar, level 16 with dict, advanced one pass small out, 33374 -github.tar, level 16 with dict dms, advanced one pass small out, 33206 -github.tar, level 16 with dict dds, advanced one pass small out, 33206 -github.tar, level 16 with dict copy, advanced one pass small out, 33374 +github.tar, level 16 with dict, advanced one pass small out, 33375 +github.tar, level 16 with dict dms, advanced one pass small out, 33207 +github.tar, level 16 with dict dds, advanced one pass small out, 33207 +github.tar, level 16 with dict copy, advanced one pass small out, 33375 github.tar, level 16 with dict load, advanced one pass small out, 39081 -github.tar, level 19, advanced one pass small out, 32276 -github.tar, level 19 with dict, advanced one pass small out, 32712 -github.tar, level 19 with dict dms, advanced one pass small out, 32555 -github.tar, level 19 with dict dds, advanced one pass small out, 32555 -github.tar, level 19 with dict copy, advanced one pass small out, 32712 -github.tar, level 19 with dict load, advanced one pass small out, 32479 +github.tar, level 19, advanced one pass small out, 32262 +github.tar, level 19 with dict, advanced one pass small out, 32701 +github.tar, level 19 with dict dms, advanced one pass small out, 32565 +github.tar, level 19 with dict dds, advanced one pass small out, 32565 +github.tar, level 19 with dict copy, advanced one pass small out, 32701 +github.tar, level 19 with dict load, advanced one pass small out, 32428 github.tar, no source size, advanced one pass small out, 38831 github.tar, no source size with dict, advanced one pass small out, 37995 github.tar, long distance mode, advanced one pass small out, 40252 @@ -861,7 +861,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass small out, 41669 github.tar, explicit params, advanced one pass small out, 41385 github.tar, uncompressed literals, advanced one pass small out, 41525 -github.tar, uncompressed literals optimal, advanced one pass small out, 35397 +github.tar, uncompressed literals optimal, advanced one pass small out, 35356 github.tar, huffman literals, advanced one pass small out, 38853 github.tar, multithreaded with advanced params, advanced one pass small out, 41525 silesia, level -5, advanced streaming, 6854744 @@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced streaming, 4505658 silesia, level 12 row 2, advanced streaming, 4503429 silesia, level 13, advanced streaming, 4493990 -silesia, level 16, advanced streaming, 4360041 -silesia, level 19, advanced streaming, 4296055 +silesia, level 16, advanced streaming, 4359652 +silesia, level 19, advanced streaming, 4266582 silesia, no source size, advanced streaming, 4842039 silesia, long distance mode, advanced streaming, 4833710 silesia, multithreaded, advanced streaming, 4842075 @@ -895,7 +895,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced streaming, 4912197 silesia, explicit params, advanced streaming, 4795857 silesia, uncompressed literals, advanced streaming, 5120566 -silesia, uncompressed literals optimal, advanced streaming, 4319518 +silesia, uncompressed literals optimal, advanced streaming, 4316880 silesia, huffman literals, advanced streaming, 5321370 silesia, multithreaded with advanced params, advanced streaming, 5120566 silesia.tar, level -5, advanced streaming, 6856523 @@ -918,8 +918,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced streaming, 4514514 silesia.tar, level 12 row 2, advanced streaming, 4514003 silesia.tar, level 13, advanced streaming, 4502956 -silesia.tar, level 16, advanced streaming, 4360546 -silesia.tar, level 19, advanced streaming, 4265911 +silesia.tar, level 16, advanced streaming, 4360385 +silesia.tar, level 19, advanced streaming, 4260939 silesia.tar, no source size, advanced streaming, 4859267 silesia.tar, long distance mode, advanced streaming, 4840452 silesia.tar, multithreaded, advanced streaming, 4854160 @@ -929,7 +929,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced streaming, 4917021 silesia.tar, explicit params, advanced streaming, 4807288 silesia.tar, uncompressed literals, advanced streaming, 5127423 -silesia.tar, uncompressed literals optimal, advanced streaming, 4310141 +silesia.tar, uncompressed literals optimal, advanced streaming, 4308451 silesia.tar, huffman literals, advanced streaming, 5341712 silesia.tar, multithreaded with advanced params, advanced streaming, 5122567 github, level -5, advanced streaming, 204407 @@ -1033,17 +1033,17 @@ github, level 13 with dict dds, advanced github, level 13 with dict copy, advanced streaming, 39948 github, level 13 with dict load, advanced streaming, 42624 github, level 16, advanced streaming, 133209 -github, level 16 with dict, advanced streaming, 37577 -github, level 16 with dict dms, advanced streaming, 37577 -github, level 16 with dict dds, advanced streaming, 37577 -github, level 16 with dict copy, advanced streaming, 37568 -github, level 16 with dict load, advanced streaming, 42338 +github, level 16 with dict, advanced streaming, 37902 +github, level 16 with dict dms, advanced streaming, 37902 +github, level 16 with dict dds, advanced streaming, 37902 +github, level 16 with dict copy, advanced streaming, 37892 +github, level 16 with dict load, advanced streaming, 42402 github, level 19, advanced streaming, 132879 -github, level 19 with dict, advanced streaming, 37576 -github, level 19 with dict dms, advanced streaming, 37576 -github, level 19 with dict dds, advanced streaming, 37576 -github, level 19 with dict copy, advanced streaming, 37567 -github, level 19 with dict load, advanced streaming, 39613 +github, level 19 with dict, advanced streaming, 37916 +github, level 19 with dict dms, advanced streaming, 37916 +github, level 19 with dict dds, advanced streaming, 37916 +github, level 19 with dict copy, advanced streaming, 37906 +github, level 19 with dict load, advanced streaming, 39770 github, no source size, advanced streaming, 136332 github, no source size with dict, advanced streaming, 41148 github, long distance mode, advanced streaming, 136332 @@ -1158,17 +1158,17 @@ github.tar, level 13 with dict dds, advanced github.tar, level 13 with dict copy, advanced streaming, 37130 github.tar, level 13 with dict load, advanced streaming, 36010 github.tar, level 16, advanced streaming, 40466 -github.tar, level 16 with dict, advanced streaming, 33374 -github.tar, level 16 with dict dms, advanced streaming, 33206 -github.tar, level 16 with dict dds, advanced streaming, 33206 -github.tar, level 16 with dict copy, advanced streaming, 33374 +github.tar, level 16 with dict, advanced streaming, 33375 +github.tar, level 16 with dict dms, advanced streaming, 33207 +github.tar, level 16 with dict dds, advanced streaming, 33207 +github.tar, level 16 with dict copy, advanced streaming, 33375 github.tar, level 16 with dict load, advanced streaming, 39081 -github.tar, level 19, advanced streaming, 32276 -github.tar, level 19 with dict, advanced streaming, 32712 -github.tar, level 19 with dict dms, advanced streaming, 32555 -github.tar, level 19 with dict dds, advanced streaming, 32555 -github.tar, level 19 with dict copy, advanced streaming, 32712 -github.tar, level 19 with dict load, advanced streaming, 32479 +github.tar, level 19, advanced streaming, 32262 +github.tar, level 19 with dict, advanced streaming, 32701 +github.tar, level 19 with dict dms, advanced streaming, 32565 +github.tar, level 19 with dict dds, advanced streaming, 32565 +github.tar, level 19 with dict copy, advanced streaming, 32701 +github.tar, level 19 with dict load, advanced streaming, 32428 github.tar, no source size, advanced streaming, 38828 github.tar, no source size with dict, advanced streaming, 38000 github.tar, long distance mode, advanced streaming, 40252 @@ -1179,7 +1179,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced streaming, 41669 github.tar, explicit params, advanced streaming, 41385 github.tar, uncompressed literals, advanced streaming, 41525 -github.tar, uncompressed literals optimal, advanced streaming, 35397 +github.tar, uncompressed literals optimal, advanced streaming, 35356 github.tar, huffman literals, advanced streaming, 38853 github.tar, multithreaded with advanced params, advanced streaming, 41525 silesia, level -5, old streaming, 6854744 @@ -1194,11 +1194,11 @@ silesia, level 6, old stre silesia, level 7, old streaming, 4570271 silesia, level 9, old streaming, 4545850 silesia, level 13, old streaming, 4493990 -silesia, level 16, old streaming, 4360041 -silesia, level 19, old streaming, 4296055 +silesia, level 16, old streaming, 4359652 +silesia, level 19, old streaming, 4266582 silesia, no source size, old streaming, 4842039 silesia, uncompressed literals, old streaming, 4842075 -silesia, uncompressed literals optimal, old streaming, 4296055 +silesia, uncompressed literals optimal, old streaming, 4266582 silesia, huffman literals, old streaming, 6172207 silesia.tar, level -5, old streaming, 6856523 silesia.tar, level -3, old streaming, 6505954 @@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming, 4579823 silesia.tar, level 9, old streaming, 4555445 silesia.tar, level 13, old streaming, 4502956 -silesia.tar, level 16, old streaming, 4360546 -silesia.tar, level 19, old streaming, 4265911 +silesia.tar, level 16, old streaming, 4360385 +silesia.tar, level 19, old streaming, 4260939 silesia.tar, no source size, old streaming, 4859267 silesia.tar, uncompressed literals, old streaming, 4859271 -silesia.tar, uncompressed literals optimal, old streaming, 4265911 +silesia.tar, uncompressed literals optimal, old streaming, 4260939 silesia.tar, huffman literals, old streaming, 6179056 github, level -5, old streaming, 204407 github, level -5 with dict, old streaming, 45832 @@ -1243,9 +1243,9 @@ github, level 9 with dict, old stre github, level 13, old streaming, 132878 github, level 13 with dict, old streaming, 39900 github, level 16, old streaming, 133209 -github, level 16 with dict, old streaming, 37577 +github, level 16 with dict, old streaming, 37902 github, level 19, old streaming, 132879 -github, level 19 with dict, old streaming, 37576 +github, level 19 with dict, old streaming, 37916 github, no source size, old streaming, 140599 github, no source size with dict, old streaming, 40654 github, uncompressed literals, old streaming, 136332 @@ -1276,13 +1276,13 @@ github.tar, level 9 with dict, old stre github.tar, level 13, old streaming, 35501 github.tar, level 13 with dict, old streaming, 37130 github.tar, level 16, old streaming, 40466 -github.tar, level 16 with dict, old streaming, 33374 -github.tar, level 19, old streaming, 32276 -github.tar, level 19 with dict, old streaming, 32712 +github.tar, level 16 with dict, old streaming, 33375 +github.tar, level 19, old streaming, 32262 +github.tar, level 19 with dict, old streaming, 32701 github.tar, no source size, old streaming, 38828 github.tar, no source size with dict, old streaming, 38000 github.tar, uncompressed literals, old streaming, 38831 -github.tar, uncompressed literals optimal, old streaming, 32276 +github.tar, uncompressed literals optimal, old streaming, 32262 github.tar, huffman literals, old streaming, 42560 silesia, level -5, old streaming advanced, 6854744 silesia, level -3, old streaming advanced, 6503319 @@ -1296,8 +1296,8 @@ silesia, level 6, old stre silesia, level 7, old streaming advanced, 4570271 silesia, level 9, old streaming advanced, 4545850 silesia, level 13, old streaming advanced, 4493990 -silesia, level 16, old streaming advanced, 4360041 -silesia, level 19, old streaming advanced, 4296055 +silesia, level 16, old streaming advanced, 4359652 +silesia, level 19, old streaming advanced, 4266582 silesia, no source size, old streaming advanced, 4842039 silesia, long distance mode, old streaming advanced, 4842075 silesia, multithreaded, old streaming advanced, 4842075 @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre silesia, small chain log, old streaming advanced, 4912197 silesia, explicit params, old streaming advanced, 4795857 silesia, uncompressed literals, old streaming advanced, 4842075 -silesia, uncompressed literals optimal, old streaming advanced, 4296055 +silesia, uncompressed literals optimal, old streaming advanced, 4266582 silesia, huffman literals, old streaming advanced, 6172207 silesia, multithreaded with advanced params, old streaming advanced, 4842075 silesia.tar, level -5, old streaming advanced, 6856523 @@ -1322,8 +1322,8 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming advanced, 4579823 silesia.tar, level 9, old streaming advanced, 4555445 silesia.tar, level 13, old streaming advanced, 4502956 -silesia.tar, level 16, old streaming advanced, 4360546 -silesia.tar, level 19, old streaming advanced, 4265911 +silesia.tar, level 16, old streaming advanced, 4360385 +silesia.tar, level 19, old streaming advanced, 4260939 silesia.tar, no source size, old streaming advanced, 4859267 silesia.tar, long distance mode, old streaming advanced, 4859271 silesia.tar, multithreaded, old streaming advanced, 4859271 @@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre silesia.tar, small chain log, old streaming advanced, 4917021 silesia.tar, explicit params, old streaming advanced, 4807288 silesia.tar, uncompressed literals, old streaming advanced, 4859271 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4265911 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4260939 silesia.tar, huffman literals, old streaming advanced, 6179056 silesia.tar, multithreaded with advanced params, old streaming advanced, 4859271 github, level -5, old streaming advanced, 213265 @@ -1361,9 +1361,9 @@ github, level 9 with dict, old stre github, level 13, old streaming advanced, 138676 github, level 13 with dict, old streaming advanced, 39725 github, level 16, old streaming advanced, 138575 -github, level 16 with dict, old streaming advanced, 40789 +github, level 16 with dict, old streaming advanced, 40804 github, level 19, old streaming advanced, 132879 -github, level 19 with dict, old streaming advanced, 37576 +github, level 19 with dict, old streaming advanced, 37916 github, no source size, old streaming advanced, 140599 github, no source size with dict, old streaming advanced, 40608 github, long distance mode, old streaming advanced, 141104 @@ -1403,8 +1403,8 @@ github.tar, level 13, old stre github.tar, level 13 with dict, old streaming advanced, 35807 github.tar, level 16, old streaming advanced, 40466 github.tar, level 16 with dict, old streaming advanced, 38578 -github.tar, level 19, old streaming advanced, 32276 -github.tar, level 19 with dict, old streaming advanced, 32704 +github.tar, level 19, old streaming advanced, 32262 +github.tar, level 19 with dict, old streaming advanced, 32678 github.tar, no source size, old streaming advanced, 38828 github.tar, no source size with dict, old streaming advanced, 38015 github.tar, long distance mode, old streaming advanced, 38831 @@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre github.tar, small chain log, old streaming advanced, 41669 github.tar, explicit params, old streaming advanced, 41385 github.tar, uncompressed literals, old streaming advanced, 38831 -github.tar, uncompressed literals optimal, old streaming advanced, 32276 +github.tar, uncompressed literals optimal, old streaming advanced, 32262 github.tar, huffman literals, old streaming advanced, 42560 github.tar, multithreaded with advanced params, old streaming advanced, 38831 github, level -5 with dict, old streaming cdict, 45832 @@ -1430,8 +1430,8 @@ github, level 6 with dict, old stre github, level 7 with dict, old streaming cdict, 38765 github, level 9 with dict, old streaming cdict, 39439 github, level 13 with dict, old streaming cdict, 39900 -github, level 16 with dict, old streaming cdict, 37577 -github, level 19 with dict, old streaming cdict, 37576 +github, level 16 with dict, old streaming cdict, 37902 +github, level 19 with dict, old streaming cdict, 37916 github, no source size with dict, old streaming cdict, 40654 github.tar, level -5 with dict, old streaming cdict, 51286 github.tar, level -3 with dict, old streaming cdict, 45147 @@ -1446,7 +1446,7 @@ github.tar, level 7 with dict, old stre github.tar, level 9 with dict, old streaming cdict, 36322 github.tar, level 13 with dict, old streaming cdict, 36010 github.tar, level 16 with dict, old streaming cdict, 39081 -github.tar, level 19 with dict, old streaming cdict, 32479 +github.tar, level 19 with dict, old streaming cdict, 32428 github.tar, no source size with dict, old streaming cdict, 38000 github, level -5 with dict, old streaming advanced cdict, 46708 github, level -3 with dict, old streaming advanced cdict, 45476 @@ -1460,8 +1460,8 @@ github, level 6 with dict, old stre github, level 7 with dict, old streaming advanced cdict, 38875 github, level 9 with dict, old streaming advanced cdict, 38941 github, level 13 with dict, old streaming advanced cdict, 39725 -github, level 16 with dict, old streaming advanced cdict, 40789 -github, level 19 with dict, old streaming advanced cdict, 37576 +github, level 16 with dict, old streaming advanced cdict, 40804 +github, level 19 with dict, old streaming advanced cdict, 37916 github, no source size with dict, old streaming advanced cdict, 40608 github.tar, level -5 with dict, old streaming advanced cdict, 50791 github.tar, level -3 with dict, old streaming advanced cdict, 44926 @@ -1476,5 +1476,5 @@ github.tar, level 7 with dict, old stre github.tar, level 9 with dict, old streaming advanced cdict, 36241 github.tar, level 13 with dict, old streaming advanced cdict, 35807 github.tar, level 16 with dict, old streaming advanced cdict, 38578 -github.tar, level 19 with dict, old streaming advanced cdict, 32704 +github.tar, level 19 with dict, old streaming advanced cdict, 32678 github.tar, no source size with dict, old streaming advanced cdict, 38015 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 14c4af82..e0ee4c3e 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -408,8 +408,8 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(3, "OK \n"); - /* Re-use without init */ - DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++); + /* Reuse without init */ + DISPLAYLEVEL(3, "test%3i : decompress again without init (reuse previous settings): ", testNb++); outBuff.pos = 0; { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2); if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ @@ -653,8 +653,8 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); } } - /* Compression state re-use scenario */ - DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++); + /* Compression state reuse scenario */ + DISPLAYLEVEL(3, "test%3i : context reuse : ", testNb++); ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); if (zc==NULL) goto _output_error; /* memory allocation issue */ @@ -722,6 +722,67 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2KB : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t singlePassSize, streamingSize, streaming2KSize; + + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize); + CHECK_Z(cSize); + ZSTD_freeCCtx(cctx); + } + + CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBufferSize, compressedBuffer, cSize)); + singlePassSize = ZSTD_sizeof_DCtx(dctx); + CHECK_Z(singlePassSize); + + inBuff.src = compressedBuffer; + inBuff.size = cSize; + + outBuff.dst = decodedBuffer; + outBuff.size = decodedBufferSize; + + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048)); + inBuff.pos = 0; + outBuff.pos = 0; + { + size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff); + CHECK_Z(r); + CHECK(r != 0, "Entire frame must be decompressed"); + } + streaming2KSize = ZSTD_sizeof_DCtx(dctx); + CHECK_Z(streaming2KSize); + + CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); + inBuff.pos = 0; + outBuff.pos = 0; + { + size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff); + CHECK_Z(r); + CHECK(r != 0, "Entire frame must be decompressed"); + } + streamingSize = ZSTD_sizeof_DCtx(dctx); + CHECK_Z(streamingSize); + + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024)); + inBuff.pos = 0; + outBuff.pos = 0; + CHECK(!ZSTD_isError(ZSTD_decompressStream(dctx, &outBuff, &inBuff)), "decompression must fail"); + + CHECK(streamingSize < singlePassSize + (1 << 18) + 3 * ZSTD_BLOCKSIZE_MAX, "Streaming doesn't use the right amount of memory"); + CHECK(streamingSize != streaming2KSize + 3 * (ZSTD_BLOCKSIZE_MAX - 2048), "ZSTD_d_blockSizeMax didn't save the right amount of memory"); + DISPLAYLEVEL(3, "| %zu | %zu | %zu | ", singlePassSize, streaming2KSize, streamingSize); + + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + /* Decompression with ZSTD_d_stableOutBuffer */ cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1); CHECK_Z(cSize); @@ -1859,7 +1920,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++); { size_t const dstBufSize = ZSTD_compressBound(CNBufferSize); - BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize)); + BYTE* const dstBuf = (BYTE*)malloc(dstBufSize); size_t const checkBufSize = CNBufferSize; BYTE* const checkBuf = (BYTE*)malloc(checkBufSize); int enableFallback; @@ -2295,6 +2356,102 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx: ", testNb++); + { + size_t const dstBufSize = ZSTD_compressBound(CNBufferSize); + BYTE* const dstBuf = (BYTE*)malloc(dstBufSize); + size_t const checkBufSize = CNBufferSize; + BYTE* const checkBuf = (BYTE*)malloc(checkBufSize); + ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + ZSTD_CCtx* staticCCtx; + void* cctxBuf; + EMF_testCase seqProdState; + + CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_validateSequences, 1)); + CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableSeqProducerFallback, 0)); + ZSTD_CCtxParams_registerSequenceProducer(params, &seqProdState, zstreamSequenceProducer); + + { + size_t const cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); + cctxBuf = malloc(cctxSize); + staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize); + ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params); + } + + // Check that compression with external sequence producer succeeds when expected + seqProdState = EMF_LOTS_OF_SEQS; + { + size_t dResult; + size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize); + CHECK(ZSTD_isError(cResult), "EMF: Compression error: %s", ZSTD_getErrorName(cResult)); + dResult = ZSTD_decompress(checkBuf, checkBufSize, dstBuf, cResult); + CHECK(ZSTD_isError(dResult), "EMF: Decompression error: %s", ZSTD_getErrorName(dResult)); + CHECK(dResult != CNBufferSize, "EMF: Corruption!"); + CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!"); + } + + // Check that compression with external sequence producer fails when expected + seqProdState = EMF_BIG_ERROR; + { + size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize); + CHECK(!ZSTD_isError(cResult), "EMF: Should have raised an error!"); + CHECK( + ZSTD_getErrorCode(cResult) != ZSTD_error_sequenceProducer_failed, + "EMF: Wrong error code: %s", ZSTD_getErrorName(cResult) + ); + } + + free(dstBuf); + free(checkBuf); + free(cctxBuf); + ZSTD_freeCCtxParams(params); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Decoder should reject invalid frame header on legacy frames: ", testNb++); + { + const unsigned char compressed[] = { 0x26,0xb5,0x2f,0xfd,0x50,0x91,0xfd,0xd8,0xb5 }; + const size_t compressedSize = 9; + size_t const dSize = ZSTD_decompress(NULL, 0, compressed, compressedSize); + CHECK(!ZSTD_isError(dSize), "must reject when legacy frame header is invalid"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Test single-shot fallback for magicless mode: ", testNb++); + { + // Aquire resources + size_t const srcSize = COMPRESSIBLE_NOISE_LENGTH; + void* src = malloc(srcSize); + size_t const dstSize = ZSTD_compressBound(srcSize); + void* dst = malloc(dstSize); + size_t const valSize = srcSize; + void* val = malloc(valSize); + ZSTD_inBuffer inBuf = { dst, dstSize, 0 }; + ZSTD_outBuffer outBuf = { val, valSize, 0 }; + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + CHECK(!src || !dst || !val || !dctx || !cctx, "memory allocation failure"); + + // Write test data for decompression to dst + RDG_genBuffer(src, srcSize, compressibility, 0.0, 0xdeadbeef); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless)); + CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); + + // Run decompression + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless)); + CHECK_Z(ZSTD_decompressStream(dctx, &outBuf, &inBuf)); + + // Validate + CHECK(outBuf.pos != srcSize, "decompressed size must match"); + CHECK(memcmp(src, val, srcSize) != 0, "decompressed data must match"); + + // Cleanup + free(src); free(dst); free(val); + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); @@ -2845,6 +3002,13 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) ); + /* Set max block size parameters */ + if (FUZ_rand(&lseed) & 1) { + int maxBlockSize = (int)(FUZ_rand(&lseed) % ZSTD_BLOCKSIZE_MAX); + maxBlockSize = MAX(1024, maxBlockSize); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_maxBlockSize, maxBlockSize, opaqueAPI) ); + } + /* Apply parameters */ if (opaqueAPI) { DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb); @@ -2976,6 +3140,13 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, if (FUZ_rand(&lseed) & 1) { CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1)); } + if (FUZ_rand(&lseed) & 1) { + int maxBlockSize; + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_maxBlockSize, &maxBlockSize)); + CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, maxBlockSize)); + } else { + CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, 0)); + } { size_t decompressionResult = 1; ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; |