aboutsummaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/LICENSE2
-rw-r--r--lib/Makefile110
-rw-r--r--lib/README.md58
-rw-r--r--lib/dll/example/Makefile2
-rw-r--r--lib/dll/example/README.md20
-rw-r--r--lib/liblz4-dll.rc.in2
-rw-r--r--lib/liblz4.pc.in6
-rw-r--r--lib/lz4.c617
-rw-r--r--lib/lz4.h160
-rw-r--r--lib/lz4file.c311
-rw-r--r--lib/lz4file.h93
-rw-r--r--lib/lz4frame.c663
-rw-r--r--lib/lz4frame.h175
-rw-r--r--lib/lz4frame_static.h2
-rw-r--r--lib/lz4hc.c238
-rw-r--r--lib/lz4hc.h28
16 files changed, 1745 insertions, 742 deletions
diff --git a/lib/LICENSE b/lib/LICENSE
index 74c2cdd7..48849169 100644
--- a/lib/LICENSE
+++ b/lib/LICENSE
@@ -1,5 +1,5 @@
LZ4 Library
-Copyright (c) 2011-2016, Yann Collet
+Copyright (c) 2011-2020, Yann Collet
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
diff --git a/lib/Makefile b/lib/Makefile
index c12949bb..06503cb2 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,6 +1,6 @@
# ################################################################
# LZ4 library - Makefile
-# Copyright (C) Yann Collet 2011-2016
+# Copyright (C) Yann Collet 2011-2020
# All rights reserved.
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
@@ -31,11 +31,12 @@
# - LZ4 source repository : https://github.com/lz4/lz4
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
+SED = sed
# Version numbers
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_MAJOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_MINOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_PATCH_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
@@ -46,12 +47,13 @@ BUILD_SHARED:=yes
BUILD_STATIC:=yes
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
+CPPFLAGS+= $(MOREFLAGS)
CFLAGS ?= -O3
DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wundef -Wpointer-arith -Wstrict-aliasing=1
-CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CFLAGS += $(DEBUGFLAGS)
+FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
SRCFILES := $(sort $(wildcard *.c))
@@ -74,27 +76,33 @@ endif
.PHONY: default
default: lib-release
+# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1
+$(V)$(VERBOSE).SILENT:
+
lib-release: DEBUGFLAGS :=
lib-release: lib
+.PHONY: lib
lib: liblz4.a liblz4
+.PHONY: all
all: lib
+.PHONY: all32
all32: CFLAGS+=-m32
all32: all
liblz4.a: $(SRCFILES)
ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
@echo compiling static library
- $(Q)$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
- $(Q)$(AR) rcs $@ *.o
+ $(COMPILE.c) $^
+ $(AR) rcs $@ *.o
endif
ifeq ($(WINBASED),yes)
liblz4-dll.rc: liblz4-dll.rc.in
@echo creating library resource
- $(Q)sed -e 's|@LIBLZ4@|$(LIBLZ4)|' \
+ $(SED) -e 's|@LIBLZ4@|$(LIBLZ4)|' \
-e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
-e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
-e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
@@ -104,31 +112,30 @@ liblz4-dll.o: liblz4-dll.rc
$(WINDRES) -i liblz4-dll.rc -o liblz4-dll.o
$(LIBLZ4): $(SRCFILES) liblz4-dll.o
-else
+ @echo compiling dynamic library $(LIBVER)
+ $(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/$@.dll -Wl,--out-implib,dll/$(LIBLZ4_EXP)
+
+else # not windows
+
$(LIBLZ4): $(SRCFILES)
-endif
-ifeq ($(BUILD_SHARED),yes) # can be disabled on command line
@echo compiling dynamic library $(LIBVER)
- ifeq ($(WINBASED),yes)
- $(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/$@.dll -Wl,--out-implib,dll/$(LIBLZ4_EXP)
- else
- $(Q)$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
+ $(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
@echo creating versioned links
- $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT)
- endif
+ $(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR)
+ $(LN_SF) $@ liblz4.$(SHARED_EXT)
+
endif
-ifeq (,$(filter MINGW%,$(TARGET_OS)))
+.PHONY: liblz4
liblz4: $(LIBLZ4)
-endif
+.PHONY: clean
clean:
ifeq ($(WINBASED),yes)
- $(Q)$(RM) *.rc
+ $(RM) *.rc
endif
- $(Q)$(RM) core *.o liblz4.pc dll/$(LIBLZ4).dll dll/$(LIBLZ4_EXP)
- $(Q)$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
+ $(RM) core *.o liblz4.pc dll/$(LIBLZ4).dll dll/$(LIBLZ4_EXP)
+ $(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
@echo Cleaning library completed
#-----------------------------------------------------------------------------
@@ -164,54 +171,55 @@ pkgconfigdir ?= $(PKGCONFIGDIR)
liblz4.pc: liblz4.pc.in Makefile
@echo creating pkgconfig
- $(Q)sed -e 's|@PREFIX@|$(prefix)|' \
+ $(SED) -e 's|@PREFIX@|$(prefix)|' \
-e 's|@LIBDIR@|$(libdir)|' \
-e 's|@INCLUDEDIR@|$(includedir)|' \
-e 's|@VERSION@|$(LIBVER)|' \
+ -e 's|=${prefix}/|=$${prefix}/|' \
$< >$@
install: lib liblz4.pc
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
- $(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
- @echo Installing libraries
+ $(INSTALL_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
+ $(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
+ @echo Installing libraries in $(DESTDIR)$(libdir)
ifeq ($(BUILD_STATIC),yes)
- $(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
- $(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
+ $(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
+ $(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
endif
ifeq ($(BUILD_SHARED),yes)
-# Traditionnally, one installs the DLLs in the bin directory as programs
+# Traditionally, one installs the DLLs in the bin directory as programs
# search them first in their directory. This allows to not pollute system
# directories (like c:/windows/system32), nor modify the PATH variable.
ifeq ($(WINBASED),yes)
- $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
- $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4_EXP) $(DESTDIR)$(libdir)
+ $(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
+ $(INSTALL_PROGRAM) dll/$(LIBLZ4_EXP) $(DESTDIR)$(libdir)
else
- $(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
- $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+ $(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
+ $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+ $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
endif
endif
- @echo Installing headers in $(includedir)
- $(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
- $(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
- $(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
+ @echo Installing headers in $(DESTDIR)$(includedir)
+ $(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
+ $(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
+ $(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
@echo lz4 libraries installed
uninstall:
- $(Q)$(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
+ $(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
ifeq (WINBASED,1)
- $(Q)$(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
- $(Q)$(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP)
+ $(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
+ $(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP)
else
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
endif
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.a
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4hc.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
+ $(RM) $(DESTDIR)$(libdir)/liblz4.a
+ $(RM) $(DESTDIR)$(includedir)/lz4.h
+ $(RM) $(DESTDIR)$(includedir)/lz4hc.h
+ $(RM) $(DESTDIR)$(includedir)/lz4frame.h
+ $(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
@echo lz4 libraries successfully uninstalled
endif
diff --git a/lib/README.md b/lib/README.md
index e2af868f..08d1cef2 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -2,16 +2,20 @@ LZ4 - Library Files
================================
The `/lib` directory contains many files, but depending on project's objectives,
-not all of them are necessary.
+not all of them are required.
+Limited systems may want to reduce the nb of source files to include
+as a way to reduce binary size and dependencies.
-#### Minimal LZ4 build
+Capabilities are added at the "level" granularity, detailed below.
+
+#### Level 1 : Minimal LZ4 build
The minimum required is **`lz4.c`** and **`lz4.h`**,
which provides the fast compression and decompression algorithms.
They generate and decode data using the [LZ4 block format].
-#### High Compression variant
+#### Level 2 : High Compression variant
For more compression ratio at the cost of compression speed,
the High Compression variant called **lz4hc** is available.
@@ -20,7 +24,7 @@ This variant also compresses data using the [LZ4 block format],
and depends on regular `lib/lz4.*` source files.
-#### Frame support, for interoperability
+#### Level 3 : Frame support, for interoperability
In order to produce compressed data compatible with `lz4` command line utility,
it's necessary to use the [official interoperable frame format].
@@ -28,14 +32,29 @@ This format is generated and decoded automatically by the **lz4frame** library.
Its public API is described in `lib/lz4frame.h`.
In order to work properly, lz4frame needs all other modules present in `/lib`,
including, lz4 and lz4hc, and also **xxhash**.
-So it's necessary to include all `*.c` and `*.h` files present in `/lib`.
+So it's necessary to also include `xxhash.c` and `xxhash.h`.
+
+
+#### Level 4 : File compression operations
+
+As a helper around file operations,
+the library has been recently extended with `lz4file.c` and `lz4file.h`
+(still considered experimental at the time of this writing).
+These helpers allow opening, reading, writing, and closing files
+using transparent LZ4 compression / decompression.
+As a consequence, using `lz4file` adds a dependency on `<stdio.h>`.
+
+`lz4file` relies on `lz4frame` in order to produce compressed data
+conformant to the [LZ4 Frame format] specification.
+Consequently, to enable this capability,
+it's necessary to include all `*.c` and `*.h` files from `lib/` directory.
#### Advanced / Experimental API
Definitions which are not guaranteed to remain stable in future versions,
are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`.
-As the name strongly implies, these definitions should only be invoked
+As the name suggests, these definitions should only be invoked
in the context of static linking ***only***.
Otherwise, dependent application may fail on API or ABI break in the future.
The associated symbols are also not exposed by the dynamic library by default.
@@ -58,7 +77,7 @@ The following build macro can be selected to adjust source code behavior at comp
Set to 65535 by default, which is the maximum value supported by lz4 format.
Reducing maximum distance will reduce opportunities for LZ4 to find matches,
hence will produce a worse compression ratio.
- However, a smaller max distance can allow compatibility with specific decoders using limited memory budget.
+ Setting a smaller max distance could allow compatibility with specific decoders with limited memory budget.
This build macro only influences the compressed output of the compressor.
- `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning.
@@ -69,15 +88,11 @@ The following build macro can be selected to adjust source code behavior at comp
This build macro offers another project-specific method
by defining `LZ4_DISABLE_DEPRECATE_WARNINGS` before including the LZ4 header files.
-- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to <stdlib>'s `malloc`, `calloc` and `free`
- by user-defined functions, which must be called `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`.
- User functions must be available at link time.
-
- `LZ4_FORCE_SW_BITCOUNT` : by default, the compression algorithm tries to determine lengths
by using bitcount instructions, generally implemented as fast single instructions in many cpus.
In case the target cpus doesn't support it, or compiler intrinsic doesn't work, or feature bad performance,
it's possible to use an optimized software path instead.
- This is achieved by setting this build macros .
+ This is achieved by setting this build macros.
In most cases, it's not expected to be necessary,
but it can be legitimately considered for less common platforms.
@@ -85,6 +100,22 @@ The following build macro can be selected to adjust source code behavior at comp
passed as argument to become a compression state is suitably aligned.
This test can be disabled if it proves flaky, by setting this value to 0.
+- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to `<stdlib,h>`'s `malloc()`, `calloc()` and `free()`
+ by user-defined functions, which must be named `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`.
+ User functions must be available at link time.
+
+- `LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION` :
+ Remove support of dynamic memory allocation.
+ For more details, see description of this macro in `lib/lz4.c`.
+
+- `LZ4_FREESTANDING` : by setting this build macro to 1,
+ LZ4/HC removes dependencies on the C standard library,
+ including allocation functions and `memmove()`, `memcpy()`, and `memset()`.
+ This build macro is designed to help use LZ4/HC in restricted environments
+ (embedded, bootloader, etc).
+ For more details, see description of this macro in `lib/lz4.h`.
+
+
#### Amalgamation
@@ -101,7 +132,7 @@ All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`.
DLL can be created using MinGW+MSYS with the `make liblz4` command.
This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`.
-To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits:
+To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits:
```
make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT
```
@@ -127,6 +158,7 @@ Other files present in the directory are not source code. They are :
- `README.md` : this file
[official interoperable frame format]: ../doc/lz4_Frame_format.md
+[LZ4 Frame format]: ../doc/lz4_Frame_format.md
[LZ4 block format]: ../doc/lz4_Block_format.md
diff --git a/lib/dll/example/Makefile b/lib/dll/example/Makefile
index e9879568..eb8cc1e4 100644
--- a/lib/dll/example/Makefile
+++ b/lib/dll/example/Makefile
@@ -1,6 +1,6 @@
# ##########################################################################
# LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2016
+# Copyright (C) Yann Collet 2016-2020
#
# GPL v2 License
#
diff --git a/lib/dll/example/README.md b/lib/dll/example/README.md
index 223e4738..b93914ba 100644
--- a/lib/dll/example/README.md
+++ b/lib/dll/example/README.md
@@ -4,8 +4,8 @@ LZ4 Windows binary package
#### The package contents
- `lz4.exe` : Command Line Utility, supporting gzip-like arguments
-- `dll\liblz4.dll` : The DLL of LZ4 library
-- `dll\liblz4.lib` : The import library of LZ4 library for Visual C++
+- `dll\msys-lz4-1.dll` : The DLL of LZ4 library, compiled by msys
+- `dll\liblz4.dll.a` : The import library of LZ4 library for Visual C++
- `example\` : The example of usage of LZ4 library
- `include\` : Header files required with LZ4 library
- `static\liblz4_static.lib` : The static LZ4 library
@@ -35,15 +35,15 @@ Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`.
#### Using LZ4 DLL with gcc/MinGW
-The header files from `include\` and the dynamic library `dll\liblz4.dll`
+The header files from `include\` and the dynamic library `dll\msys-lz4-1.dll`
are required to compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses LZ4 consists of a single `test-dll.c`
-file it should be linked with `dll\liblz4.dll`. For example:
+file it should be linked with `dll\msys-lz4-1.dll`. For example:
```
- gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll
+ gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\msys-lz4-1.dll
```
-The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
+The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`.
#### The example of usage of static and dynamic LZ4 libraries with Visual C++
@@ -51,19 +51,19 @@ The compiled executable will require LZ4 DLL which is available at `dll\liblz4.d
Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a
dynamic LZ4 library from the `dll` directory. The solution works with Visual C++
2010 or newer. When one will open the solution with Visual C++ newer than 2010
-then the solution will upgraded to the current version.
+then the solution will be upgraded to the current version.
#### Using LZ4 DLL with Visual C++
-The header files from `include\` and the import library `dll\liblz4.lib`
+The header files from `include\` and the import library `dll\liblz4.dll.a`
are required to compile a project using Visual C++.
1. The header files should be added to `Additional Include Directories` that can
be found in project properties `C/C++` then `General`.
2. The import library has to be added to `Additional Dependencies` that can
be found in project properties `Linker` then `Input`.
- If one will provide only the name `liblz4.lib` without a full path to the library
+ If one will provide only the name `liblz4.dll.a` without a full path to the library
the directory has to be added to `Linker\General\Additional Library Directories`.
-The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
+The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`.
diff --git a/lib/liblz4-dll.rc.in b/lib/liblz4-dll.rc.in
index bf9adf56..e2d84b6e 100644
--- a/lib/liblz4-dll.rc.in
+++ b/lib/liblz4-dll.rc.in
@@ -22,7 +22,7 @@ BEGIN
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
VALUE "InternalName", "@LIBLZ4@"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "@LIBLZ4@.dll"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
diff --git a/lib/liblz4.pc.in b/lib/liblz4.pc.in
index cb31cd78..ed52214f 100644
--- a/lib/liblz4.pc.in
+++ b/lib/liblz4.pc.in
@@ -1,5 +1,5 @@
# LZ4 - Fast LZ compression algorithm
-# Copyright (C) 2011-2014, Yann Collet.
+# Copyright (C) 2011-2020, Yann Collet.
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
prefix=@PREFIX@
@@ -10,5 +10,5 @@ Name: lz4
Description: extremely fast lossless compression algorithm library
URL: http://www.lz4.org/
Version: @VERSION@
-Libs: -L@LIBDIR@ -llz4
-Cflags: -I@INCLUDEDIR@
+Libs: -L${libdir} -llz4
+Cflags: -I${includedir}
diff --git a/lib/lz4.c b/lib/lz4.c
index 9f5e9bfa..654bfdf3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-present, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -124,6 +124,7 @@
#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */
# include <intrin.h> /* only present in VS2005+ */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */
#endif /* _MSC_VER */
#ifndef LZ4_FORCE_INLINE
@@ -187,7 +188,27 @@
/*-************************************
* Memory routines
**************************************/
-#ifdef LZ4_USER_MEMORY_FUNCTIONS
+
+/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION :
+ * Disable relatively high-level LZ4/HC functions that use dynamic memory
+ * allocation functions (malloc(), calloc(), free()).
+ *
+ * Note that this is a compile-time switch. And since it disables
+ * public/stable LZ4 v1 API functions, we don't recommend using this
+ * symbol to generate a library for distribution.
+ *
+ * The following public functions are removed when this symbol is defined.
+ * - lz4 : LZ4_createStream, LZ4_freeStream,
+ * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated)
+ * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC,
+ * LZ4_createHC (deprecated), LZ4_freeHC (deprecated)
+ * - lz4frame, lz4file : All LZ4F_* functions
+ */
+#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
+# define ALLOC(s) lz4_error_memory_allocation_is_disabled
+# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled
+# define FREEMEM(p) lz4_error_memory_allocation_is_disabled
+#elif defined(LZ4_USER_MEMORY_FUNCTIONS)
/* memory management functions can be customized by user project.
* Below functions must exist somewhere in the Project
* and be available at link time */
@@ -204,8 +225,13 @@ void LZ4_free(void* p);
# define FREEMEM(p) free(p)
#endif
-#include <string.h> /* memset, memcpy */
-#define MEM_INIT(p,v,s) memset((p),(v),(s))
+#if ! LZ4_FREESTANDING
+# include <string.h> /* memset, memcpy */
+#endif
+#if !defined(LZ4_memset)
+# define LZ4_memset(p,v,s) memset((p),(v),(s))
+#endif
+#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s))
/*-************************************
@@ -316,10 +342,20 @@ typedef enum {
* memcpy() as if it were standard compliant, so it can inline it in freestanding
* environments. This is needed when decompressing the Linux Kernel, for example.
*/
-#if defined(__GNUC__) && (__GNUC__ >= 4)
-#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size)
-#else
-#define LZ4_memcpy(dst, src, size) memcpy(dst, src, size)
+#if !defined(LZ4_memcpy)
+# if defined(__GNUC__) && (__GNUC__ >= 4)
+# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size)
+# else
+# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size)
+# endif
+#endif
+
+#if !defined(LZ4_memmove)
+# if defined(__GNUC__) && (__GNUC__ >= 4)
+# define LZ4_memmove __builtin_memmove
+# else
+# define LZ4_memmove memmove
+# endif
#endif
static unsigned LZ4_isLittleEndian(void)
@@ -343,14 +379,14 @@ static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign;
+typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) LZ4_unalign;
-static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }
+static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign*)ptr)->u16; }
+static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign*)ptr)->u32; }
+static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalign*)ptr)->uArch; }
-static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
+static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign*)memPtr)->u16 = value; }
+static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign*)memPtr)->u32 = value; }
#else /* safe and portable access using memcpy() */
@@ -421,10 +457,12 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3};
#ifndef LZ4_FAST_DEC_LOOP
# if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64
# define LZ4_FAST_DEC_LOOP 1
+# elif defined(__aarch64__) && defined(__APPLE__)
+# define LZ4_FAST_DEC_LOOP 1
# elif defined(__aarch64__) && !defined(__clang__)
- /* On aarch64, we disable this optimization for clang because on certain
- * mobile chipsets, performance is reduced with clang. For information
- * refer to https://github.com/lz4/lz4/pull/707 */
+ /* On non-Apple aarch64, we disable this optimization for clang because
+ * on certain mobile chipsets, performance is reduced with clang. For
+ * more information refer to https://github.com/lz4/lz4/pull/707 */
# define LZ4_FAST_DEC_LOOP 1
# else
# define LZ4_FAST_DEC_LOOP 0
@@ -486,7 +524,14 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si
case 2:
LZ4_memcpy(v, srcPtr, 2);
LZ4_memcpy(&v[2], srcPtr, 2);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(push)
+# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */
+#endif
LZ4_memcpy(&v[4], v, 4);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(pop)
+#endif
break;
case 4:
LZ4_memcpy(v, srcPtr, 4);
@@ -515,9 +560,20 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
assert(val != 0);
if (LZ4_isLittleEndian()) {
if (sizeof(val) == 8) {
-# if defined(_MSC_VER) && (_MSC_VER >= 1800) && defined(_M_AMD64) && !defined(LZ4_FORCE_SW_BITCOUNT)
+# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT)
+/*-*************************************************************************************************
+* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11.
+* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics
+* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC.
+****************************************************************************************************/
+# if defined(__clang__) && (__clang_major__ < 10)
+ /* Avoid undefined clang-cl intrinsics issue.
+ * See https://github.com/lz4/lz4/pull/1017 for details. */
+ return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3;
+# else
/* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */
return (unsigned)_tzcnt_u64(val) >> 3;
+# endif
# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64(&r, (U64)val);
@@ -652,10 +708,10 @@ typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
* - usingExtDict : Like withPrefix64k, but the preceding content is somewhere
* else in memory, starting at ctx->dictionary with length
* ctx->dictSize.
- * - usingDictCtx : Like usingExtDict, but everything concerning the preceding
- * content is in a separate context, pointed to by
- * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table
- * entries in the current context that refer to positions
+ * - usingDictCtx : Everything concerning the preceding content is
+ * in a separate context, pointed to by ctx->dictCtx.
+ * ctx->dictionary, ctx->dictSize, and table entries
+ * in the current context that refer to positions
* preceding the beginning of the current compression are
* ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx
* ->dictSize describe the location and size of the preceding
@@ -672,12 +728,12 @@ typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; }
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
-int LZ4_sizeofState(void) { return LZ4_STREAMSIZE; }
+int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); }
-/*-************************************
-* Internal Definitions used in Tests
-**************************************/
+/*-****************************************
+* Internal Definitions, used only in Tests
+*******************************************/
#if defined (__cplusplus)
extern "C" {
#endif
@@ -687,7 +743,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
int compressedSize, int maxOutputSize,
const void* dictStart, size_t dictSize);
-
+int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest,
+ int compressedSize, int targetOutputSize, int dstCapacity,
+ const void* dictStart, size_t dictSize);
#if defined (__cplusplus)
}
#endif
@@ -827,9 +885,10 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx,
}
}
- /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster
- * than compressing without a gap. However, compressing with
- * currentOffset == 0 is faster still, so we preserve that case.
+ /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back,
+ * is faster than compressing without a gap.
+ * However, compressing with currentOffset == 0 is faster still,
+ * so we preserve that case.
*/
if (cctx->currentOffset != 0 && tableType == byU32) {
DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset");
@@ -853,7 +912,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated(
const char* const source,
char* const dest,
const int inputSize,
- int *inputConsumed, /* only written when outputDirective == fillOutput */
+ int* inputConsumed, /* only written when outputDirective == fillOutput */
const int maxOutputSize,
const limitedOutput_directive outputDirective,
const tableType_t tableType,
@@ -885,7 +944,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated(
/* the dictCtx currentOffset is indexed on the start of the dictionary,
* while a dictionary in the current context precedes the currentOffset */
- const BYTE* dictBase = !dictionary ? NULL : (dictDirective == usingDictCtx) ?
+ const BYTE* dictBase = (dictionary == NULL) ? NULL :
+ (dictDirective == usingDictCtx) ?
dictionary + dictSize - dictCtx->currentOffset :
dictionary + dictSize - startIndex;
@@ -981,10 +1041,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated(
match = base + matchIndex;
lowLimit = (const BYTE*)source;
}
- } else if (dictDirective==usingExtDict) {
+ } else if (dictDirective == usingExtDict) {
if (matchIndex < startIndex) {
DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex);
assert(startIndex - matchIndex >= MINMATCH);
+ assert(dictBase);
match = dictBase + matchIndex;
lowLimit = dictionary;
} else {
@@ -1048,7 +1109,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated(
_next_match:
/* at this stage, the following variables must be correctly set :
* - ip : at start of LZ operation
- * - match : at start of previous pattern occurence; can be within current prefix, or within extDict
+ * - match : at start of previous pattern occurrence; can be within current prefix, or within extDict
* - offset : if maybe_ext_memSegment==1 (constant)
* - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise
* - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
@@ -1173,6 +1234,7 @@ _next_match:
}
} else if (dictDirective==usingExtDict) {
if (matchIndex < startIndex) {
+ assert(dictBase);
match = dictBase + matchIndex;
lowLimit = dictionary; /* required for match length counter */
} else {
@@ -1355,7 +1417,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
{
int result;
#if (LZ4_HEAPMODE)
- LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
+ LZ4_stream_t* ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
if (ctxPtr == NULL) return 0;
#else
LZ4_stream_t ctx;
@@ -1420,15 +1482,17 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
* Streaming functions
********************************/
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_stream_t* LZ4_createStream(void)
{
LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
- LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */
+ LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal));
DEBUGLOG(4, "LZ4_createStream %p", lz4s);
if (lz4s == NULL) return NULL;
LZ4_initStream(lz4s, sizeof(*lz4s));
return lz4s;
}
+#endif
static size_t LZ4_stream_t_alignment(void)
{
@@ -1462,6 +1526,7 @@ void LZ4_resetStream_fast(LZ4_stream_t* ctx) {
LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
}
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
{
if (!LZ4_stream) return 0; /* support free on NULL */
@@ -1469,6 +1534,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
FREEMEM(LZ4_stream);
return (0);
}
+#endif
#define HASH_UNIT sizeof(reg_t)
@@ -1514,8 +1580,9 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
return (int)dict->dictSize;
}
-void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) {
- const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL :
+void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream)
+{
+ const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL :
&(dictionaryStream->internal_donotuse);
DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)",
@@ -1568,36 +1635,40 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream,
int acceleration)
{
const tableType_t tableType = byU32;
- LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
- const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+ LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse;
+ const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL;
- DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);
+ DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize);
- LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */
+ LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */
if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT;
if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX;
/* invalidate tiny dictionaries */
- if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */
- && (dictEnd != (const BYTE*)source) ) {
+ if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */
+ && (dictEnd != source) /* prefix mode */
+ && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */
+ && (streamPtr->dictCtx == NULL) /* usingDictCtx */
+ ) {
DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary);
+ /* remove dictionary existence from history, to employ faster prefix mode */
streamPtr->dictSize = 0;
streamPtr->dictionary = (const BYTE*)source;
- dictEnd = (const BYTE*)source;
+ dictEnd = source;
}
/* Check overlapping input/dictionary space */
- { const BYTE* sourceEnd = (const BYTE*) source + inputSize;
- if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) {
+ { const char* const sourceEnd = source + inputSize;
+ if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) {
streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
- streamPtr->dictionary = dictEnd - streamPtr->dictSize;
+ streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize;
}
}
/* prefix mode : source data follows dictionary */
- if (dictEnd == (const BYTE*)source) {
+ if (dictEnd == source) {
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
else
@@ -1623,7 +1694,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream,
} else {
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
}
- } else {
+ } else { /* small data <= 4 KB */
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
} else {
@@ -1661,21 +1732,25 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
/*! LZ4_saveDict() :
* If previously compressed data block is not guaranteed to remain available at its memory location,
* save it into a safer place (char* safeBuffer).
- * Note : you don't need to call LZ4_loadDict() afterwards,
- * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
- * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable,
+ * one can therefore call LZ4_compress_fast_continue() right after.
+ * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
*/
int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
{
LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
- const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
+
+ DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);
if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
if (safeBuffer == NULL) assert(dictSize == 0);
- if (dictSize > 0)
- memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
+ if (dictSize > 0) {
+ const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
+ assert(dict->dictionary);
+ LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize);
+ }
dict->dictionary = (const BYTE*)safeBuffer;
dict->dictSize = (U32)dictSize;
@@ -1689,39 +1764,163 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
* Decompression functions
********************************/
-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive;
#undef MIN
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+/* variant for decompress_unsafe()
+ * does not know end of input
+ * presumes input is well formed
+ * note : will consume at least one byte */
+size_t read_long_length_no_check(const BYTE** pp)
+{
+ size_t b, l = 0;
+ do { b = **pp; (*pp)++; l += b; } while (b==255);
+ DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1)
+ return l;
+}
+
+/* core decoder variant for LZ4_decompress_fast*()
+ * for legacy support only : these entry points are deprecated.
+ * - Presumes input is correctly formed (no defense vs malformed inputs)
+ * - Does not know input size (presume input buffer is "large enough")
+ * - Decompress a full block (only)
+ * @return : nb of bytes read from input.
+ * Note : this variant is not optimized for speed, just for maintenance.
+ * the goal is to remove support of decompress_fast*() variants by v2.0
+**/
+LZ4_FORCE_INLINE int
+LZ4_decompress_unsafe_generic(
+ const BYTE* const istart,
+ BYTE* const ostart,
+ int decompressedSize,
+
+ size_t prefixSize,
+ const BYTE* const dictStart, /* only if dict==usingExtDict */
+ const size_t dictSize /* note: =0 if dictStart==NULL */
+ )
+{
+ const BYTE* ip = istart;
+ BYTE* op = (BYTE*)ostart;
+ BYTE* const oend = ostart + decompressedSize;
+ const BYTE* const prefixStart = ostart - prefixSize;
+
+ DEBUGLOG(5, "LZ4_decompress_unsafe_generic");
+ if (dictStart == NULL) assert(dictSize == 0);
+
+ while (1) {
+ /* start new sequence */
+ unsigned token = *ip++;
+
+ /* literals */
+ { size_t ll = token >> ML_BITS;
+ if (ll==15) {
+ /* long literal length */
+ ll += read_long_length_no_check(&ip);
+ }
+ if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */
+ LZ4_memmove(op, ip, ll); /* support in-place decompression */
+ op += ll;
+ ip += ll;
+ if ((size_t)(oend-op) < MFLIMIT) {
+ if (op==oend) break; /* end of block */
+ DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op);
+ /* incorrect end of block :
+ * last match must start at least MFLIMIT==12 bytes before end of output block */
+ return -1;
+ } }
+
+ /* match */
+ { size_t ml = token & 15;
+ size_t const offset = LZ4_readLE16(ip);
+ ip+=2;
+
+ if (ml==15) {
+ /* long literal length */
+ ml += read_long_length_no_check(&ip);
+ }
+ ml += MINMATCH;
+
+ if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */
+
+ { const BYTE* match = op - offset;
+
+ /* out of range */
+ if (offset > (size_t)(op - prefixStart) + dictSize) {
+ DEBUGLOG(6, "offset out of range");
+ return -1;
+ }
+
+ /* check special case : extDict */
+ if (offset > (size_t)(op - prefixStart)) {
+ /* extDict scenario */
+ const BYTE* const dictEnd = dictStart + dictSize;
+ const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart));
+ size_t const extml = (size_t)(dictEnd - extMatch);
+ if (extml > ml) {
+ /* match entirely within extDict */
+ LZ4_memmove(op, extMatch, ml);
+ op += ml;
+ ml = 0;
+ } else {
+ /* match split between extDict & prefix */
+ LZ4_memmove(op, extMatch, extml);
+ op += extml;
+ ml -= extml;
+ }
+ match = prefixStart;
+ }
+
+ /* match copy - slow variant, supporting overlap copy */
+ { size_t u;
+ for (u=0; u<ml; u++) {
+ op[u] = match[u];
+ } } }
+ op += ml;
+ if ((size_t)(oend-op) < LASTLITERALS) {
+ DEBUGLOG(5, "invalid: match ends at distance %zi from end of block", oend-op);
+ /* incorrect end of block :
+ * last match must stop at least LASTLITERALS==5 bytes before end of output block */
+ return -1;
+ }
+ } /* match */
+ } /* main loop */
+ return (int)(ip - istart);
+}
+
+
/* Read the variable-length literal or match length.
*
- * ip - pointer to use as input.
- * lencheck - end ip. Return an error if ip advances >= lencheck.
- * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so.
- * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so.
- * error (output) - error code. Should be set to 0 before call.
- */
-typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error;
-LZ4_FORCE_INLINE unsigned
-read_variable_length(const BYTE**ip, const BYTE* lencheck,
- int loop_check, int initial_check,
- variable_length_error* error)
-{
- U32 length = 0;
- U32 s;
- if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
- *error = initial_error;
- return length;
+ * @ip : input pointer
+ * @ilimit : position after which if length is not decoded, the input is necessarily corrupted.
+ * @initial_check - check ip >= ipmax before start of loop. Returns initial_error if so.
+ * @error (output) - error code. Must be set to 0 before call.
+**/
+typedef size_t Rvl_t;
+static const Rvl_t rvl_error = (Rvl_t)(-1);
+LZ4_FORCE_INLINE Rvl_t
+read_variable_length(const BYTE** ip, const BYTE* ilimit,
+ int initial_check)
+{
+ Rvl_t s, length = 0;
+ assert(ip != NULL);
+ assert(*ip != NULL);
+ assert(ilimit != NULL);
+ if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */
+ return rvl_error;
}
do {
s = **ip;
(*ip)++;
length += s;
- if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
- *error = loop_error;
- return length;
+ if (unlikely((*ip) > ilimit)) { /* read limit reached */
+ return rvl_error;
+ }
+ /* accumulator overflow detection (32-bit mode only) */
+ if ((sizeof(length)<8) && unlikely(length > ((Rvl_t)(-1)/2)) ) {
+ return rvl_error;
}
} while (s==255);
@@ -1741,7 +1940,6 @@ LZ4_decompress_generic(
int srcSize,
int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */
- endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */
earlyEnd_directive partialDecoding, /* full, partial */
dict_directive dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */
@@ -1749,7 +1947,7 @@ LZ4_decompress_generic(
const size_t dictSize /* note : = 0 if noDict */
)
{
- if (src == NULL) { return -1; }
+ if ((src == NULL) || (outputSize < 0)) { return -1; }
{ const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
@@ -1760,13 +1958,12 @@ LZ4_decompress_generic(
const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize;
- const int safeDecode = (endOnInput==endOnInputSize);
- const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
+ const int checkOffset = (dictSize < (int)(64 KB));
/* Set up the "end" pointers for the shortcut. */
- const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
- const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
+ const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/;
+ const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/;
const BYTE* match;
size_t offset;
@@ -1778,83 +1975,70 @@ LZ4_decompress_generic(
/* Special cases */
assert(lowPrefix <= op);
- if ((endOnInput) && (unlikely(outputSize==0))) {
+ if (unlikely(outputSize==0)) {
/* Empty output buffer */
if (partialDecoding) return 0;
return ((srcSize==1) && (*ip==0)) ? 0 : -1;
}
- if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); }
- if ((endOnInput) && unlikely(srcSize==0)) { return -1; }
+ if (unlikely(srcSize==0)) { return -1; }
- /* Currently the fast loop shows a regression on qualcomm arm chips. */
+ /* LZ4_FAST_DEC_LOOP:
+ * designed for modern OoO performance cpus,
+ * where copying reliably 32-bytes is preferable to an unpredictable branch.
+ * note : fast loop may show a regression for some client arm chips. */
#if LZ4_FAST_DEC_LOOP
if ((oend - op) < FASTLOOP_SAFE_DISTANCE) {
DEBUGLOG(6, "skip fast decode loop");
goto safe_decode;
}
- /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */
+ /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */
while (1) {
/* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */
assert(oend - op >= FASTLOOP_SAFE_DISTANCE);
- if (endOnInput) { assert(ip < iend); }
+ assert(ip < iend);
token = *ip++;
length = token >> ML_BITS; /* literal length */
- assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
-
/* decode literal length */
if (length == RUN_MASK) {
- variable_length_error error = ok;
- length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error);
- if (error == initial_error) { goto _output_error; }
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
- if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
+ if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
+ if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
/* copy literals */
cpy = op+length;
LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
- if (endOnInput) { /* LZ4_decompress_safe() */
- if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
- LZ4_wildCopy32(op, ip, cpy);
- } else { /* LZ4_decompress_fast() */
- if (cpy>oend-8) { goto safe_literal_copy; }
- LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
- * it doesn't know input length, and only relies on end-of-block properties */
- }
+ if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
+ LZ4_wildCopy32(op, ip, cpy);
ip += length; op = cpy;
} else {
cpy = op+length;
- if (endOnInput) { /* LZ4_decompress_safe() */
- DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
- /* We don't need to check oend, since we check it once for each loop below */
- if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
- /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
- LZ4_memcpy(op, ip, 16);
- } else { /* LZ4_decompress_fast() */
- /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
- * it doesn't know input length, and relies on end-of-block properties */
- LZ4_memcpy(op, ip, 8);
- if (length > 8) { LZ4_memcpy(op+8, ip+8, 8); }
- }
+ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
+ /* We don't need to check oend, since we check it once for each loop below */
+ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
+ /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */
+ LZ4_memcpy(op, ip, 16);
ip += length; op = cpy;
}
/* get offset */
offset = LZ4_readLE16(ip); ip+=2;
match = op - offset;
- assert(match <= op);
+ assert(match <= op); /* overflow check */
/* get matchlength */
length = token & ML_MASK;
if (length == ML_MASK) {
- variable_length_error error = ok;
- if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
- length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error);
- if (error != ok) { goto _output_error; }
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
length += MINMATCH;
+ if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
goto safe_match_copy;
}
@@ -1864,7 +2048,7 @@ LZ4_decompress_generic(
goto safe_match_copy;
}
- /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */
+ /* Fastpath check: skip LZ4_wildCopy32 when true */
if ((dict == withPrefix64k) || (match >= lowPrefix)) {
if (offset >= 8) {
assert(match >= lowPrefix);
@@ -1881,6 +2065,7 @@ LZ4_decompress_generic(
if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
/* match starting within external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
+ assert(dictEnd != NULL);
if (unlikely(op+length > oend-LASTLITERALS)) {
if (partialDecoding) {
DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd");
@@ -1891,7 +2076,7 @@ LZ4_decompress_generic(
if (length <= (size_t)(lowPrefix-match)) {
/* match fits entirely within external dictionary : just copy */
- memmove(op, dictEnd - (lowPrefix-match), length);
+ LZ4_memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match stretches into both external dictionary and current block */
@@ -1927,11 +2112,10 @@ LZ4_decompress_generic(
/* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */
while (1) {
+ assert(ip < iend);
token = *ip++;
length = token >> ML_BITS; /* literal length */
- assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
-
/* A two-stage shortcut for the most common case:
* 1) If the literal length is 0..14, and there is enough space,
* enter the shortcut and copy 16 bytes on behalf of the literals
@@ -1941,11 +2125,11 @@ LZ4_decompress_generic(
* those 18 bytes earlier, upon entering the shortcut (in other words,
* there is a combined check for both stages).
*/
- if ( (endOnInput ? length != RUN_MASK : length <= 8)
+ if ( (length != RUN_MASK)
/* strictly "less than" on input, to re-enter the loop with at least one byte */
- && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {
+ && likely((ip < shortiend) & (op <= shortoend)) ) {
/* Copy the literals */
- LZ4_memcpy(op, ip, endOnInput ? 16 : 8);
+ LZ4_memcpy(op, ip, 16);
op += length; ip += length;
/* The second stage: prepare for match copying, decode full info.
@@ -1975,11 +2159,11 @@ LZ4_decompress_generic(
/* decode literal length */
if (length == RUN_MASK) {
- variable_length_error error = ok;
- length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error);
- if (error == initial_error) { goto _output_error; }
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
- if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
+ if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
+ if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
}
/* copy literals */
@@ -1988,9 +2172,7 @@ LZ4_decompress_generic(
safe_literal_copy:
#endif
LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
- if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
- {
+ if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) {
/* We've either hit the input parsing restriction or the output parsing restriction.
* In the normal scenario, decoding a full block, it must be the last sequence,
* otherwise it's an error (invalid input or dimensions).
@@ -2000,7 +2182,6 @@ LZ4_decompress_generic(
/* Since we are partial decoding we may be in this block because of the output parsing
* restriction, which is not valid since the output buffer is allowed to be undersized.
*/
- assert(endOnInput);
DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end")
DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length);
DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op));
@@ -2021,21 +2202,17 @@ LZ4_decompress_generic(
length = (size_t)(oend-op);
}
} else {
- /* We must be on the last sequence because of the parsing limitations so check
- * that we exactly regenerate the original size (must be exact when !endOnInput).
- */
- if ((!endOnInput) && (cpy != oend)) { goto _output_error; }
/* We must be on the last sequence (or invalid) because of the parsing limitations
* so check that we exactly consume the input and don't overrun the output buffer.
*/
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) {
+ if ((ip+length != iend) || (cpy > oend)) {
DEBUGLOG(6, "should have been last run of literals")
DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend);
DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend);
goto _output_error;
}
}
- memmove(op, ip, length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */
+ LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */
ip += length;
op += length;
/* Necessarily EOF when !partialDecoding.
@@ -2047,7 +2224,7 @@ LZ4_decompress_generic(
break;
}
} else {
- LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
+ LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */
ip += length; op = cpy;
}
@@ -2060,10 +2237,10 @@ LZ4_decompress_generic(
_copy_match:
if (length == ML_MASK) {
- variable_length_error error = ok;
- length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error);
- if (error != ok) goto _output_error;
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
+ if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
}
length += MINMATCH;
@@ -2073,6 +2250,7 @@ LZ4_decompress_generic(
if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
/* match starting within external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
+ assert(dictEnd != NULL);
if (unlikely(op+length > oend-LASTLITERALS)) {
if (partialDecoding) length = MIN(length, (size_t)(oend-op));
else goto _output_error; /* doesn't respect parsing restriction */
@@ -2080,7 +2258,7 @@ LZ4_decompress_generic(
if (length <= (size_t)(lowPrefix-match)) {
/* match fits entirely within external dictionary : just copy */
- memmove(op, dictEnd - (lowPrefix-match), length);
+ LZ4_memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match stretches into both external dictionary and current block */
@@ -2151,12 +2329,8 @@ LZ4_decompress_generic(
}
/* end of decoding */
- if (endOnInput) {
- DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst));
- return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
- } else {
- return (int) (((const char*)ip)-src); /* Nb of input bytes read */
- }
+ DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst));
+ return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
/* Overflow error detected */
_output_error:
@@ -2171,7 +2345,7 @@ LZ4_FORCE_O2
int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
- endOnInputSize, decode_full_block, noDict,
+ decode_full_block, noDict,
(BYTE*)dest, NULL, 0);
}
@@ -2180,16 +2354,17 @@ int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize,
{
dstCapacity = MIN(targetOutputSize, dstCapacity);
return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity,
- endOnInputSize, partial_decode,
+ partial_decode,
noDict, (BYTE*)dst, NULL, 0);
}
LZ4_FORCE_O2
int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, decode_full_block, withPrefix64k,
- (BYTE*)dest - 64 KB, NULL, 0);
+ DEBUGLOG(5, "LZ4_decompress_fast");
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ 0, NULL, 0);
}
/*===== Instantiate a few more decoding cases, used more than once. =====*/
@@ -2198,16 +2373,25 @@ LZ4_FORCE_O2 /* Exported, an obsolete API function. */
int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, withPrefix64k,
+ decode_full_block, withPrefix64k,
+ (BYTE*)dest - 64 KB, NULL, 0);
+}
+
+LZ4_FORCE_O2
+static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ partial_decode, withPrefix64k,
(BYTE*)dest - 64 KB, NULL, 0);
}
/* Another obsolete API function, paired with the previous one. */
int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
{
- /* LZ4_decompress_fast doesn't validate match offsets,
- * and thus serves well with any prefixed dictionary. */
- return LZ4_decompress_fast(source, dest, originalSize);
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ 64 KB, NULL, 0);
}
LZ4_FORCE_O2
@@ -2215,7 +2399,17 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i
size_t prefixSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, noDict,
+ decode_full_block, noDict,
+ (BYTE*)dest-prefixSize, NULL, 0);
+}
+
+LZ4_FORCE_O2
+static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity,
+ size_t prefixSize)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ partial_decode, noDict,
(BYTE*)dest-prefixSize, NULL, 0);
}
@@ -2225,7 +2419,18 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, usingExtDict,
+ decode_full_block, usingExtDict,
+ (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+}
+
+LZ4_FORCE_O2
+int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest,
+ int compressedSize, int targetOutputSize, int dstCapacity,
+ const void* dictStart, size_t dictSize)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ partial_decode, usingExtDict,
(BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
@@ -2233,9 +2438,9 @@ LZ4_FORCE_O2
static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,
const void* dictStart, size_t dictSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, decode_full_block, usingExtDict,
- (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ 0, (const BYTE*)dictStart, dictSize);
}
/* The "double dictionary" mode, for use with e.g. ring buffers: the first part
@@ -2247,26 +2452,17 @@ int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compresse
size_t prefixSize, const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, usingExtDict,
- (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
-}
-
-LZ4_FORCE_INLINE
-int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize,
- size_t prefixSize, const void* dictStart, size_t dictSize)
-{
- return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, decode_full_block, usingExtDict,
+ decode_full_block, usingExtDict,
(BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
}
/*===== streaming decompression functions =====*/
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
- LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
- LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */
- return lz4s;
+ LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal));
+ return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
}
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
@@ -2275,6 +2471,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
FREEMEM(LZ4_stream);
return 0;
}
+#endif
/*! LZ4_setStreamDecode() :
* Use this function to instruct where to find the dictionary.
@@ -2285,8 +2482,13 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
{
LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
- lz4sd->prefixSize = (size_t) dictSize;
- lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;
+ lz4sd->prefixSize = (size_t)dictSize;
+ if (dictSize) {
+ assert(dictionary != NULL);
+ lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;
+ } else {
+ lz4sd->prefixEnd = (const BYTE*) dictionary;
+ }
lz4sd->externalDict = NULL;
lz4sd->extDictSize = 0;
return 1;
@@ -2358,29 +2560,35 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
return result;
}
-LZ4_FORCE_O2
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
+LZ4_FORCE_O2 int
+LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode,
+ const char* source, char* dest, int originalSize)
{
- LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
+ LZ4_streamDecode_t_internal* const lz4sd =
+ (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse);
int result;
+
+ DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize);
assert(originalSize >= 0);
if (lz4sd->prefixSize == 0) {
+ DEBUGLOG(5, "first invocation : no prefix nor extDict");
assert(lz4sd->extDictSize == 0);
result = LZ4_decompress_fast(source, dest, originalSize);
if (result <= 0) return result;
lz4sd->prefixSize = (size_t)originalSize;
lz4sd->prefixEnd = (BYTE*)dest + originalSize;
} else if (lz4sd->prefixEnd == (BYTE*)dest) {
- if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0)
- result = LZ4_decompress_fast(source, dest, originalSize);
- else
- result = LZ4_decompress_fast_doubleDict(source, dest, originalSize,
- lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+ DEBUGLOG(5, "continue using existing prefix");
+ result = LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ lz4sd->prefixSize,
+ lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize += (size_t)originalSize;
lz4sd->prefixEnd += originalSize;
} else {
+ DEBUGLOG(5, "prefix becomes extDict");
lz4sd->extDictSize = lz4sd->prefixSize;
lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
result = LZ4_decompress_fast_extDict(source, dest, originalSize,
@@ -2416,10 +2624,27 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize);
}
+int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize)
+{
+ if (dictSize==0)
+ return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity);
+ if (dictStart+dictSize == dest) {
+ if (dictSize >= 64 KB - 1) {
+ return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity);
+ }
+ assert(dictSize >= 0);
+ return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize);
+ }
+ assert(dictSize >= 0);
+ return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize);
+}
+
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
if (dictSize==0 || dictStart+dictSize == dest)
- return LZ4_decompress_fast(source, dest, originalSize);
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ (size_t)dictSize, NULL, 0);
assert(dictSize >= 0);
return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize);
}
@@ -2471,7 +2696,7 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize,
/* Obsolete Streaming functions */
-int LZ4_sizeofStreamState(void) { return LZ4_STREAMSIZE; }
+int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); }
int LZ4_resetStreamState(void* state, char* inputBuffer)
{
@@ -2480,11 +2705,13 @@ int LZ4_resetStreamState(void* state, char* inputBuffer)
return 0;
}
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
void* LZ4_create (char* inputBuffer)
{
(void)inputBuffer;
return LZ4_createStream();
}
+#endif
char* LZ4_slideInputBuffer (void* state)
{
diff --git a/lib/lz4.h b/lib/lz4.h
index 7ab1e483..491c6087 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -1,7 +1,7 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
- * Copyright (C) 2011-present, Yann Collet.
+ * Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -97,36 +97,77 @@ extern "C" {
# define LZ4LIB_API LZ4LIB_VISIBILITY
#endif
+/*! LZ4_FREESTANDING :
+ * When this macro is set to 1, it enables "freestanding mode" that is
+ * suitable for typical freestanding environment which doesn't support
+ * standard C library.
+ *
+ * - LZ4_FREESTANDING is a compile-time switch.
+ * - It requires the following macros to be defined:
+ * LZ4_memcpy, LZ4_memmove, LZ4_memset.
+ * - It only enables LZ4/HC functions which don't use heap.
+ * All LZ4F_* functions are not supported.
+ * - See tests/freestanding.c to check its basic setup.
+ */
+#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1)
+# define LZ4_HEAPMODE 0
+# define LZ4HC_HEAPMODE 0
+# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1
+# if !defined(LZ4_memcpy)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'."
+# endif
+# if !defined(LZ4_memset)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memset'."
+# endif
+# if !defined(LZ4_memmove)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'."
+# endif
+#elif ! defined(LZ4_FREESTANDING)
+# define LZ4_FREESTANDING 0
+#endif
+
+
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
#define LZ4_QUOTE(str) #str
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
-#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
+#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */
-LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */
-LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */
+LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */
+LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */
/*-************************************
* Tuning parameter
**************************************/
+#define LZ4_MEMORY_USAGE_MIN 10
+#define LZ4_MEMORY_USAGE_DEFAULT 14
+#define LZ4_MEMORY_USAGE_MAX 20
+
/*!
* LZ4_MEMORY_USAGE :
- * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- * Increasing memory usage improves compression ratio.
- * Reduced memory usage may improve speed, thanks to better cache locality.
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; )
+ * Increasing memory usage improves compression ratio, at the cost of speed.
+ * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#ifndef LZ4_MEMORY_USAGE
-# define LZ4_MEMORY_USAGE 14
+# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
#endif
+#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN)
+# error "LZ4_MEMORY_USAGE is too small !"
+#endif
+
+#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX)
+# error "LZ4_MEMORY_USAGE is too large !"
+#endif
/*-************************************
* Simple Functions
@@ -270,8 +311,25 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS
***********************************************/
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
+/**
+ Note about RC_INVOKED
+
+ - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio).
+ https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros
+
+ - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars)
+ and reports warning "RC4011: identifier truncated".
+
+ - To eliminate the warning, we surround long preprocessor symbol with
+ "#if !defined(RC_INVOKED) ... #endif" block that means
+ "skip this block when rc.exe is trying to read it".
+*/
+#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
+#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
+#endif
/*! LZ4_resetStream_fast() : v1.9.0+
* Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
@@ -355,8 +413,12 @@ typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */
* creation / destruction of streaming decompression tracking context.
* A tracking context can be re-used multiple times.
*/
+#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
+#endif
/*! LZ4_setStreamDecode() :
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
@@ -406,7 +468,10 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
* save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
*/
-LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
+LZ4LIB_API int
+LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode,
+ const char* src, char* dst,
+ int srcSize, int dstCapacity);
/*! LZ4_decompress_*_usingDict() :
@@ -417,7 +482,16 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod
* Performance tip : Decompression speed can be substantially increased
* when dst == dictStart + dictSize.
*/
-LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
+LZ4LIB_API int
+LZ4_decompress_safe_usingDict(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize);
+
+LZ4LIB_API int
+LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
+ int compressedSize,
+ int targetOutputSize, int maxOutputSize,
+ const char* dictStart, int dictSize);
#endif /* LZ4_H_2983827168210 */
@@ -496,13 +570,15 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the completion of the first compression call on the stream.
*/
-LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
+LZ4LIB_STATIC_API void
+LZ4_attach_dictionary(LZ4_stream_t* workingStream,
+ const LZ4_stream_t* dictionaryStream);
/*! In-place compression and decompression
*
* It's possible to have input and output sharing the same buffer,
- * for highly contrained memory environments.
+ * for highly constrained memory environments.
* In both cases, it requires input to lay at the end of the buffer,
* and decompression to start at beginning of the buffer.
* Buffer size must feature some margin, hence be larger than final size.
@@ -592,38 +668,26 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
typedef unsigned int LZ4_u32;
#endif
+/*! LZ4_stream_t :
+ * Never ever use below internal definitions directly !
+ * These definitions are not API/ABI safe, and may change in future versions.
+ * If you need static allocation, declare or allocate an LZ4_stream_t object.
+**/
+
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
LZ4_u32 hashTable[LZ4_HASH_SIZE_U32];
- LZ4_u32 currentOffset;
- LZ4_u32 tableType;
const LZ4_byte* dictionary;
const LZ4_stream_t_internal* dictCtx;
+ LZ4_u32 currentOffset;
+ LZ4_u32 tableType;
LZ4_u32 dictSize;
+ /* Implicit padding to ensure structure is aligned */
};
-typedef struct {
- const LZ4_byte* externalDict;
- size_t extDictSize;
- const LZ4_byte* prefixEnd;
- size_t prefixSize;
-} LZ4_streamDecode_t_internal;
-
-
-/*! LZ4_stream_t :
- * Do not use below internal definitions directly !
- * Declare or allocate an LZ4_stream_t instead.
- * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
- * The structure definition can be convenient for static allocation
- * (on stack, or as part of larger structure).
- * Init this structure with LZ4_initStream() before first use.
- * note : only use this definition in association with static linking !
- * this definition is not API/ABI safe, and may change in future versions.
- */
-#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */
-#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*))
+#define LZ4_STREAM_MINSIZE ((1UL << LZ4_MEMORY_USAGE) + 32) /* static size, for inter-version compatibility */
union LZ4_stream_u {
- void* table[LZ4_STREAMSIZE_VOIDP];
+ char minStateSize[LZ4_STREAM_MINSIZE];
LZ4_stream_t_internal internal_donotuse;
}; /* previously typedef'd to LZ4_stream_t */
@@ -641,21 +705,25 @@ union LZ4_stream_u {
* In which case, the function will @return NULL.
* Note2: An LZ4_stream_t structure guarantees correct alignment and size.
* Note3: Before v1.9.0, use LZ4_resetStream() instead
- */
+**/
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
/*! LZ4_streamDecode_t :
- * information structure to track an LZ4 stream during decompression.
- * init this structure using LZ4_setStreamDecode() before first use.
- * note : only use in association with static linking !
- * this definition is not API/ABI safe,
- * and may change in a future version !
- */
-#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ )
-#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
+ * Never ever use below internal definitions directly !
+ * These definitions are not API/ABI safe, and may change in future versions.
+ * If you need static allocation, declare or allocate an LZ4_streamDecode_t object.
+**/
+typedef struct {
+ const LZ4_byte* externalDict;
+ const LZ4_byte* prefixEnd;
+ size_t extDictSize;
+ size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+
+#define LZ4_STREAMDECODE_MINSIZE 32
union LZ4_streamDecode_u {
- unsigned long long table[LZ4_STREAMDECODESIZE_U64];
+ char minStateSize[LZ4_STREAMDECODE_MINSIZE];
LZ4_streamDecode_t_internal internal_donotuse;
} ; /* previously typedef'd to LZ4_streamDecode_t */
diff --git a/lib/lz4file.c b/lib/lz4file.c
new file mode 100644
index 00000000..eaf9b170
--- /dev/null
+++ b/lib/lz4file.c
@@ -0,0 +1,311 @@
+/*
+ * LZ4 file library
+ * Copyright (C) 2022, Xiaomi Inc.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
+ */
+#include <stdlib.h>
+#include <string.h>
+#include "lz4.h"
+#include "lz4file.h"
+
+struct LZ4_readFile_s {
+ LZ4F_dctx* dctxPtr;
+ FILE* fp;
+ LZ4_byte* srcBuf;
+ size_t srcBufNext;
+ size_t srcBufSize;
+ size_t srcBufMaxSize;
+};
+
+struct LZ4_writeFile_s {
+ LZ4F_cctx* cctxPtr;
+ FILE* fp;
+ LZ4_byte* dstBuf;
+ size_t maxWriteSize;
+ size_t dstBufMaxSize;
+ LZ4F_errorCode_t errCode;
+};
+
+LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp)
+{
+ char buf[LZ4F_HEADER_SIZE_MAX];
+ size_t consumedSize;
+ LZ4F_errorCode_t ret;
+ LZ4F_frameInfo_t info;
+
+ if (fp == NULL || lz4fRead == NULL) {
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ *lz4fRead = (LZ4_readFile_t*)calloc(1, sizeof(LZ4_readFile_t));
+ if (*lz4fRead == NULL) {
+ return -LZ4F_ERROR_allocation_failed;
+ }
+
+ ret = LZ4F_createDecompressionContext(&(*lz4fRead)->dctxPtr, LZ4F_getVersion());
+ if (LZ4F_isError(ret)) {
+ free(*lz4fRead);
+ return ret;
+ }
+
+ (*lz4fRead)->fp = fp;
+ consumedSize = fread(buf, 1, sizeof(buf), (*lz4fRead)->fp);
+ if (consumedSize != sizeof(buf)) {
+ free(*lz4fRead);
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ ret = LZ4F_getFrameInfo((*lz4fRead)->dctxPtr, &info, buf, &consumedSize);
+ if (LZ4F_isError(ret)) {
+ LZ4F_freeDecompressionContext((*lz4fRead)->dctxPtr);
+ free(*lz4fRead);
+ return ret;
+ }
+
+ switch (info.blockSizeID) {
+ case LZ4F_default :
+ case LZ4F_max64KB :
+ (*lz4fRead)->srcBufMaxSize = 64 * 1024;
+ break;
+ case LZ4F_max256KB:
+ (*lz4fRead)->srcBufMaxSize = 256 * 1024;
+ break;
+ case LZ4F_max1MB:
+ (*lz4fRead)->srcBufMaxSize = 1 * 1024 * 1024;
+ break;
+ case LZ4F_max4MB:
+ (*lz4fRead)->srcBufMaxSize = 4 * 1024 * 1024;
+ break;
+ default:
+ LZ4F_freeDecompressionContext((*lz4fRead)->dctxPtr);
+ free(*lz4fRead);
+ return -LZ4F_ERROR_maxBlockSize_invalid;
+ }
+
+ (*lz4fRead)->srcBuf = (LZ4_byte*)malloc((*lz4fRead)->srcBufMaxSize);
+ if ((*lz4fRead)->srcBuf == NULL) {
+ LZ4F_freeDecompressionContext((*lz4fRead)->dctxPtr);
+ free(lz4fRead);
+ return -LZ4F_ERROR_allocation_failed;
+ }
+
+ (*lz4fRead)->srcBufSize = sizeof(buf) - consumedSize;
+ memcpy((*lz4fRead)->srcBuf, buf + consumedSize, (*lz4fRead)->srcBufSize);
+
+ return ret;
+}
+
+size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size)
+{
+ LZ4_byte* p = (LZ4_byte*)buf;
+ size_t next = 0;
+
+ if (lz4fRead == NULL || buf == NULL)
+ return -LZ4F_ERROR_GENERIC;
+
+ while (next < size) {
+ size_t srcsize = lz4fRead->srcBufSize - lz4fRead->srcBufNext;
+ size_t dstsize = size - next;
+ size_t ret;
+
+ if (srcsize == 0) {
+ ret = fread(lz4fRead->srcBuf, 1, lz4fRead->srcBufMaxSize, lz4fRead->fp);
+ if (ret > 0) {
+ lz4fRead->srcBufSize = ret;
+ srcsize = lz4fRead->srcBufSize;
+ lz4fRead->srcBufNext = 0;
+ }
+ else if (ret == 0) {
+ break;
+ }
+ else {
+ return -LZ4F_ERROR_GENERIC;
+ }
+ }
+
+ ret = LZ4F_decompress(lz4fRead->dctxPtr,
+ p, &dstsize,
+ lz4fRead->srcBuf + lz4fRead->srcBufNext,
+ &srcsize,
+ NULL);
+ if (LZ4F_isError(ret)) {
+ return ret;
+ }
+
+ lz4fRead->srcBufNext += srcsize;
+ next += dstsize;
+ p += dstsize;
+ }
+
+ return next;
+}
+
+LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead)
+{
+ if (lz4fRead == NULL)
+ return -LZ4F_ERROR_GENERIC;
+ LZ4F_freeDecompressionContext(lz4fRead->dctxPtr);
+ free(lz4fRead->srcBuf);
+ free(lz4fRead);
+ return LZ4F_OK_NoError;
+}
+
+LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr)
+{
+ LZ4_byte buf[LZ4F_HEADER_SIZE_MAX];
+ size_t ret;
+
+ if (fp == NULL || lz4fWrite == NULL)
+ return -LZ4F_ERROR_GENERIC;
+
+ *lz4fWrite = (LZ4_writeFile_t*)malloc(sizeof(LZ4_writeFile_t));
+ if (*lz4fWrite == NULL) {
+ return -LZ4F_ERROR_allocation_failed;
+ }
+ if (prefsPtr != NULL) {
+ switch (prefsPtr->frameInfo.blockSizeID) {
+ case LZ4F_default :
+ case LZ4F_max64KB :
+ (*lz4fWrite)->maxWriteSize = 64 * 1024;
+ break;
+ case LZ4F_max256KB:
+ (*lz4fWrite)->maxWriteSize = 256 * 1024;
+ break;
+ case LZ4F_max1MB:
+ (*lz4fWrite)->maxWriteSize = 1 * 1024 * 1024;
+ break;
+ case LZ4F_max4MB:
+ (*lz4fWrite)->maxWriteSize = 4 * 1024 * 1024;
+ break;
+ default:
+ free(lz4fWrite);
+ return -LZ4F_ERROR_maxBlockSize_invalid;
+ }
+ } else {
+ (*lz4fWrite)->maxWriteSize = 64 * 1024;
+ }
+
+ (*lz4fWrite)->dstBufMaxSize = LZ4F_compressBound((*lz4fWrite)->maxWriteSize, prefsPtr);
+ (*lz4fWrite)->dstBuf = (LZ4_byte*)malloc((*lz4fWrite)->dstBufMaxSize);
+ if ((*lz4fWrite)->dstBuf == NULL) {
+ free(*lz4fWrite);
+ return -LZ4F_ERROR_allocation_failed;
+ }
+
+ ret = LZ4F_createCompressionContext(&(*lz4fWrite)->cctxPtr, LZ4F_getVersion());
+ if (LZ4F_isError(ret)) {
+ free((*lz4fWrite)->dstBuf);
+ free(*lz4fWrite);
+ return ret;
+ }
+
+ ret = LZ4F_compressBegin((*lz4fWrite)->cctxPtr, buf, LZ4F_HEADER_SIZE_MAX, prefsPtr);
+ if (LZ4F_isError(ret)) {
+ LZ4F_freeCompressionContext((*lz4fWrite)->cctxPtr);
+ free((*lz4fWrite)->dstBuf);
+ free(*lz4fWrite);
+ return ret;
+ }
+
+ if (ret != fwrite(buf, 1, ret, fp)) {
+ LZ4F_freeCompressionContext((*lz4fWrite)->cctxPtr);
+ free((*lz4fWrite)->dstBuf);
+ free(*lz4fWrite);
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ (*lz4fWrite)->fp = fp;
+ (*lz4fWrite)->errCode = LZ4F_OK_NoError;
+ return LZ4F_OK_NoError;
+}
+
+size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, void* buf, size_t size)
+{
+ LZ4_byte* p = (LZ4_byte*)buf;
+ size_t remain = size;
+ size_t chunk;
+ size_t ret;
+
+ if (lz4fWrite == NULL || buf == NULL)
+ return -LZ4F_ERROR_GENERIC;
+ while (remain) {
+ if (remain > lz4fWrite->maxWriteSize)
+ chunk = lz4fWrite->maxWriteSize;
+ else
+ chunk = remain;
+
+ ret = LZ4F_compressUpdate(lz4fWrite->cctxPtr,
+ lz4fWrite->dstBuf, lz4fWrite->dstBufMaxSize,
+ p, chunk,
+ NULL);
+ if (LZ4F_isError(ret)) {
+ lz4fWrite->errCode = ret;
+ return ret;
+ }
+
+ if(ret != fwrite(lz4fWrite->dstBuf, 1, ret, lz4fWrite->fp)) {
+ lz4fWrite->errCode = -LZ4F_ERROR_GENERIC;
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ p += chunk;
+ remain -= chunk;
+ }
+
+ return size;
+}
+
+LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite)
+{
+ LZ4F_errorCode_t ret = LZ4F_OK_NoError;
+
+ if (lz4fWrite == NULL)
+ return -LZ4F_ERROR_GENERIC;
+
+ if (lz4fWrite->errCode == LZ4F_OK_NoError) {
+ ret = LZ4F_compressEnd(lz4fWrite->cctxPtr,
+ lz4fWrite->dstBuf, lz4fWrite->dstBufMaxSize,
+ NULL);
+ if (LZ4F_isError(ret)) {
+ goto out;
+ }
+
+ if (ret != fwrite(lz4fWrite->dstBuf, 1, ret, lz4fWrite->fp)) {
+ ret = -LZ4F_ERROR_GENERIC;
+ }
+ }
+
+out:
+ LZ4F_freeCompressionContext(lz4fWrite->cctxPtr);
+ free(lz4fWrite->dstBuf);
+ free(lz4fWrite);
+ return ret;
+}
diff --git a/lib/lz4file.h b/lib/lz4file.h
new file mode 100644
index 00000000..55271307
--- /dev/null
+++ b/lib/lz4file.h
@@ -0,0 +1,93 @@
+/*
+ LZ4 file library
+ Header File
+ Copyright (C) 2022, Xiaomi Inc.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 source repository : https://github.com/lz4/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef LZ4FILE_H
+#define LZ4FILE_H
+
+#include <stdio.h>
+#include "lz4frame_static.h"
+
+typedef struct LZ4_readFile_s LZ4_readFile_t;
+typedef struct LZ4_writeFile_s LZ4_writeFile_t;
+
+/*! LZ4F_readOpen() :
+ * Set read lz4file handle.
+ * `lz4f` will set a lz4file handle.
+ * `fp` must be the return value of the lz4 file opened by fopen.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp);
+
+/*! LZ4F_read() :
+ * Read lz4file content to buffer.
+ * `lz4f` must use LZ4_readOpen to set first.
+ * `buf` read data buffer.
+ * `size` read data buffer size.
+ */
+LZ4FLIB_STATIC_API size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size);
+
+/*! LZ4F_readClose() :
+ * Close lz4file handle.
+ * `lz4f` must use LZ4_readOpen to set first.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead);
+
+/*! LZ4F_writeOpen() :
+ * Set write lz4file handle.
+ * `lz4f` will set a lz4file handle.
+ * `fp` must be the return value of the lz4 file opened by fopen.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr);
+
+/*! LZ4F_write() :
+ * Write buffer to lz4file.
+ * `lz4f` must use LZ4F_writeOpen to set first.
+ * `buf` write data buffer.
+ * `size` write data buffer size.
+ */
+LZ4FLIB_STATIC_API size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, void* buf, size_t size);
+
+/*! LZ4F_writeClose() :
+ * Close lz4file handle.
+ * `lz4f` must use LZ4F_writeOpen to set first.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite);
+
+#endif /* LZ4FILE_H */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index ec02c92f..174f9ae4 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -45,7 +45,7 @@
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
@@ -63,6 +63,19 @@
/*-************************************
+* Library declarations
+**************************************/
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
+#define LZ4_STATIC_LINKING_ONLY
+#include "lz4.h"
+#define LZ4_HC_STATIC_LINKING_ONLY
+#include "lz4hc.h"
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
+
+/*-************************************
* Memory routines
**************************************/
/*
@@ -70,7 +83,13 @@
* malloc(), calloc() and free()
* towards another library or solution of their choice
* by modifying below section.
- */
+**/
+
+#include <string.h> /* memset, memcpy, memmove */
+#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
+# define MEM_INIT(p,v,s) memset((p),(v),(s))
+#endif
+
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
# include <stdlib.h> /* malloc, calloc, free */
# define ALLOC(s) malloc(s)
@@ -78,23 +97,42 @@
# define FREEMEM(p) free(p)
#endif
-#include <string.h> /* memset, memcpy, memmove */
-#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
-# define MEM_INIT(p,v,s) memset((p),(v),(s))
-#endif
+static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem)
+{
+ /* custom calloc defined : use it */
+ if (cmem.customCalloc != NULL) {
+ return cmem.customCalloc(cmem.opaqueState, s);
+ }
+ /* nothing defined : use default <stdlib.h>'s calloc() */
+ if (cmem.customAlloc == NULL) {
+ return ALLOC_AND_ZERO(s);
+ }
+ /* only custom alloc defined : use it, and combine it with memset() */
+ { void* const p = cmem.customAlloc(cmem.opaqueState, s);
+ if (p != NULL) MEM_INIT(p, 0, s);
+ return p;
+} }
+static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem)
+{
+ /* custom malloc defined : use it */
+ if (cmem.customAlloc != NULL) {
+ return cmem.customAlloc(cmem.opaqueState, s);
+ }
+ /* nothing defined : use default <stdlib.h>'s malloc() */
+ return ALLOC(s);
+}
-/*-************************************
-* Library declarations
-**************************************/
-#define LZ4F_STATIC_LINKING_ONLY
-#include "lz4frame.h"
-#define LZ4_STATIC_LINKING_ONLY
-#include "lz4.h"
-#define LZ4_HC_STATIC_LINKING_ONLY
-#include "lz4hc.h"
-#define XXH_STATIC_LINKING_ONLY
-#include "xxhash.h"
+static void LZ4F_free(void* p, LZ4F_CustomMem cmem)
+{
+ /* custom malloc defined : use it */
+ if (cmem.customFree != NULL) {
+ cmem.customFree(cmem.opaqueState, p);
+ return;
+ }
+ /* nothing defined : use default <stdlib.h>'s free() */
+ FREEMEM(p);
+}
/*-************************************
@@ -143,7 +181,7 @@ static int g_debuglog_enable = 1;
#endif
-/* unoptimized version; solves endianess & alignment issues */
+/* unoptimized version; solves endianness & alignment issues */
static U32 LZ4F_readLE32 (const void* src)
{
const BYTE* const srcPtr = (const BYTE*)src;
@@ -206,8 +244,6 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
#define _4BITS 0x0F
#define _8BITS 0xFF
-#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
-#define LZ4F_MAGICNUMBER 0x184D2204U
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
@@ -220,22 +256,27 @@ static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checks
/*-************************************
* Structures and local types
**************************************/
+
+typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t;
+
typedef struct LZ4F_cctx_s
{
+ LZ4F_CustomMem cmem;
LZ4F_preferences_t prefs;
U32 version;
U32 cStage;
const LZ4F_CDict* cdict;
size_t maxBlockSize;
size_t maxBufferSize;
- BYTE* tmpBuff;
- BYTE* tmpIn;
- size_t tmpInSize;
+ BYTE* tmpBuff; /* internal buffer, for streaming */
+ BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */
+ size_t tmpInSize; /* amount of data to compress after tmpIn */
U64 totalInSize;
XXH32_state_t xxh;
void* lz4CtxPtr;
U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+ LZ4F_blockCompression_t blockCompression;
} LZ4F_cctx_t;
@@ -264,27 +305,33 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);
}
-static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
+static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code)
{
/* A compilation error here means sizeof(ptrdiff_t) is not large enough */
LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
}
+#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e)
+
+#define RETURN_ERROR_IF(c,e) if (c) RETURN_ERROR(e)
+
+#define FORWARD_IF_ERROR(r) if (LZ4F_isError(r)) return (r)
+
unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
-size_t LZ4F_getBlockSize(unsigned blockSizeID)
+size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID)
{
static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB)
- return err0r(LZ4F_ERROR_maxBlockSize_invalid);
- blockSizeID -= LZ4F_max64KB;
- return blockSizes[blockSizeID];
-}
+ RETURN_ERROR(maxBlockSize_invalid);
+ { int const blockSizeIdx = (int)blockSizeID - (int)LZ4F_max64KB;
+ return blockSizes[blockSizeIdx];
+} }
/*-************************************
* Private functions
@@ -397,21 +444,20 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
MEM_INIT(&options, 0, sizeof(options));
options.stableSrc = 1;
- if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
- return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+ RETURN_ERROR_IF(dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs), dstMaxSize_tooSmall);
{ size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
- if (LZ4F_isError(headerSize)) return headerSize;
+ FORWARD_IF_ERROR(headerSize);
dstPtr += headerSize; /* header size */ }
assert(dstEnd >= dstPtr);
{ size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);
- if (LZ4F_isError(cSize)) return cSize;
+ FORWARD_IF_ERROR(cSize);
dstPtr += cSize; }
assert(dstEnd >= dstPtr);
{ size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */
- if (LZ4F_isError(tailSize)) return tailSize;
+ FORWARD_IF_ERROR(tailSize);
dstPtr += tailSize; }
assert(dstEnd >= dstStart);
@@ -432,27 +478,26 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
{
size_t result;
#if (LZ4F_HEAPMODE)
- LZ4F_cctx_t *cctxPtr;
+ LZ4F_cctx_t* cctxPtr;
result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
- if (LZ4F_isError(result)) return result;
+ FORWARD_IF_ERROR(result);
#else
LZ4F_cctx_t cctx;
LZ4_stream_t lz4ctx;
- LZ4F_cctx_t *cctxPtr = &cctx;
+ LZ4F_cctx_t* const cctxPtr = &cctx;
- DEBUGLOG(4, "LZ4F_compressFrame");
MEM_INIT(&cctx, 0, sizeof(cctx));
cctx.version = LZ4F_VERSION;
cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
- if (preferencesPtr == NULL ||
- preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
- {
+ if ( preferencesPtr == NULL
+ || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) {
LZ4_initStream(&lz4ctx, sizeof(lz4ctx));
cctxPtr->lz4CtxPtr = &lz4ctx;
cctxPtr->lz4CtxAlloc = 1;
cctxPtr->lz4CtxState = 1;
}
#endif
+ DEBUGLOG(4, "LZ4F_compressFrame");
result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
srcBuffer, srcSize,
@@ -461,10 +506,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
#if (LZ4F_HEAPMODE)
LZ4F_freeCompressionContext(cctxPtr);
#else
- if (preferencesPtr != NULL &&
- preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
- {
- FREEMEM(cctxPtr->lz4CtxPtr);
+ if ( preferencesPtr != NULL
+ && preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) {
+ LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
}
#endif
return result;
@@ -476,30 +520,31 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
*****************************************************/
struct LZ4F_CDict_s {
+ LZ4F_CustomMem cmem;
void* dictContent;
LZ4_stream_t* fastCtx;
LZ4_streamHC_t* HCCtx;
}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
-/*! LZ4F_createCDict() :
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
- * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
- * @return : digested dictionary for compression, or NULL if failed */
-LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
+LZ4F_CDict*
+LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize)
{
const char* dictStart = (const char*)dictBuffer;
- LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
- DEBUGLOG(4, "LZ4F_createCDict");
+ LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem);
+ DEBUGLOG(4, "LZ4F_createCDict_advanced");
if (!cdict) return NULL;
+ cdict->cmem = cmem;
if (dictSize > 64 KB) {
dictStart += dictSize - 64 KB;
dictSize = 64 KB;
}
- cdict->dictContent = ALLOC(dictSize);
- cdict->fastCtx = LZ4_createStream();
- cdict->HCCtx = LZ4_createStreamHC();
+ cdict->dictContent = LZ4F_malloc(dictSize, cmem);
+ cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem);
+ if (cdict->fastCtx)
+ LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t));
+ cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem);
+ if (cdict->HCCtx)
+ LZ4_initStream(cdict->HCCtx, sizeof(LZ4_streamHC_t));
if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
LZ4F_freeCDict(cdict);
return NULL;
@@ -511,13 +556,25 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
return cdict;
}
+/*! LZ4F_createCDict() :
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict
+ * @return : digested dictionary for compression, or NULL if failed */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
+{
+ DEBUGLOG(4, "LZ4F_createCDict");
+ return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize);
+}
+
void LZ4F_freeCDict(LZ4F_CDict* cdict)
{
if (cdict==NULL) return; /* support free on NULL */
- FREEMEM(cdict->dictContent);
- LZ4_freeStream(cdict->fastCtx);
- LZ4_freeStreamHC(cdict->HCCtx);
- FREEMEM(cdict);
+ LZ4F_free(cdict->dictContent, cdict->cmem);
+ LZ4F_free(cdict->fastCtx, cdict->cmem);
+ LZ4F_free(cdict->HCCtx, cdict->cmem);
+ LZ4F_free(cdict, cdict->cmem);
}
@@ -525,6 +582,20 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
* Advanced compression functions
***********************************/
+LZ4F_cctx*
+LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
+{
+ LZ4F_cctx* const cctxPtr =
+ (LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem);
+ if (cctxPtr==NULL) return NULL;
+
+ cctxPtr->cmem = customMem;
+ cctxPtr->version = version;
+ cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */
+
+ return cctxPtr;
+}
+
/*! LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
@@ -532,17 +603,16 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
* The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
* Object can release its memory using LZ4F_freeCompressionContext();
- */
-LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version)
+**/
+LZ4F_errorCode_t
+LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version)
{
- LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
- if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
-
- cctxPtr->version = version;
- cctxPtr->cStage = 0; /* Next stage : init stream */
-
- *LZ4F_compressionContextPtr = cctxPtr;
+ assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */
+ /* in case it nonetheless happen in production */
+ RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null);
+ *LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version);
+ RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed);
return LZ4F_OK_NoError;
}
@@ -550,11 +620,10 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionConte
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr)
{
if (cctxPtr != NULL) { /* support free on NULL */
- FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */
- FREEMEM(cctxPtr->tmpBuff);
- FREEMEM(cctxPtr);
+ LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */
+ LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
+ LZ4F_free(cctxPtr, cctxPtr->cmem);
}
-
return LZ4F_OK_NoError;
}
@@ -588,11 +657,21 @@ static void LZ4F_initStream(void* ctx,
}
}
+static int ctxTypeID_to_size(int ctxTypeID) {
+ switch(ctxTypeID) {
+ case 1:
+ return LZ4_sizeofState();
+ case 2:
+ return LZ4_sizeofStateHC();
+ default:
+ return 0;
+ }
+}
/*! LZ4F_compressBegin_usingCDict() :
- * init streaming compression and writes frame header into dstBuffer.
- * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
- * @return : number of bytes written into dstBuffer for the header
+ * init streaming compression AND writes frame header into @dstBuffer.
+ * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * @return : number of bytes written into @dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
@@ -600,41 +679,46 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
const LZ4F_CDict* cdict,
const LZ4F_preferences_t* preferencesPtr)
{
- LZ4F_preferences_t prefNull;
+ LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
- BYTE* headerStart;
- if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- MEM_INIT(&prefNull, 0, sizeof(prefNull));
+ RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall);
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
cctxPtr->prefs = *preferencesPtr;
- /* Ctx Management */
+ /* cctx Management */
{ U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
- if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
- FREEMEM(cctxPtr->lz4CtxPtr);
+ int requiredSize = ctxTypeID_to_size(ctxTypeID);
+ int allocatedSize = ctxTypeID_to_size(cctxPtr->lz4CtxAlloc);
+ if (allocatedSize < requiredSize) {
+ /* not enough space allocated */
+ LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- cctxPtr->lz4CtxPtr = LZ4_createStream();
+ /* must take ownership of memory allocation,
+ * in order to respect custom allocator contract */
+ cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctxPtr->cmem);
+ if (cctxPtr->lz4CtxPtr)
+ LZ4_initStream(cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
} else {
- cctxPtr->lz4CtxPtr = LZ4_createStreamHC();
+ cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctxPtr->cmem);
+ if (cctxPtr->lz4CtxPtr)
+ LZ4_initStreamHC(cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
}
- if (cctxPtr->lz4CtxPtr == NULL)
- return err0r(LZ4F_ERROR_allocation_failed);
+ RETURN_ERROR_IF(cctxPtr->lz4CtxPtr == NULL, allocation_failed);
cctxPtr->lz4CtxAlloc = ctxTypeID;
cctxPtr->lz4CtxState = ctxTypeID;
} else if (cctxPtr->lz4CtxState != ctxTypeID) {
- /* otherwise, a sufficient buffer is allocated, but we need to
- * reset it to the correct context type */
+ /* otherwise, a sufficient buffer is already allocated,
+ * but we need to reset it to the correct context type */
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t));
+ LZ4_initStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
} else {
- LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
- LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ LZ4_initStreamHC((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
}
cctxPtr->lz4CtxState = ctxTypeID;
- }
- }
+ } }
/* Buffer Management */
if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
@@ -647,9 +731,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
if (cctxPtr->maxBufferSize < requiredBuffSize) {
cctxPtr->maxBufferSize = 0;
- FREEMEM(cctxPtr->tmpBuff);
- cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
- if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
+ cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem);
+ RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed);
cctxPtr->maxBufferSize = requiredBuffSize;
} }
cctxPtr->tmpIn = cctxPtr->tmpBuff;
@@ -669,31 +753,32 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
/* Magic Number */
LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
dstPtr += 4;
- headerStart = dstPtr;
-
- /* FLG Byte */
- *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
- + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
- + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
- + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
- + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
- + (cctxPtr->prefs.frameInfo.dictID > 0) );
- /* BD Byte */
- *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
- /* Optional Frame content size field */
- if (cctxPtr->prefs.frameInfo.contentSize) {
- LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
- dstPtr += 8;
- cctxPtr->totalInSize = 0;
- }
- /* Optional dictionary ID field */
- if (cctxPtr->prefs.frameInfo.dictID) {
- LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
- dstPtr += 4;
+ { BYTE* const headerStart = dstPtr;
+
+ /* FLG Byte */
+ *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
+ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
+ + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
+ + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+ + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
+ + (cctxPtr->prefs.frameInfo.dictID > 0) );
+ /* BD Byte */
+ *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
+ /* Optional Frame content size field */
+ if (cctxPtr->prefs.frameInfo.contentSize) {
+ LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
+ dstPtr += 8;
+ cctxPtr->totalInSize = 0;
+ }
+ /* Optional dictionary ID field */
+ if (cctxPtr->prefs.frameInfo.dictID) {
+ LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
+ dstPtr += 4;
+ }
+ /* Header CRC Byte */
+ *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
+ dstPtr++;
}
- /* Header CRC Byte */
- *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
- dstPtr++;
cctxPtr->cStage = 1; /* header written, now request input data block */
return (size_t)(dstPtr - dstStart);
@@ -701,9 +786,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
/*! LZ4F_compressBegin() :
- * init streaming compression and writes frame header into dstBuffer.
- * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
- * preferencesPtr can be NULL, in which case default parameters are selected.
+ * init streaming compression AND writes frame header into @dstBuffer.
+ * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * @preferencesPtr can be NULL, in which case default parameters are selected.
* @return : number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
*/
@@ -744,11 +829,13 @@ static size_t LZ4F_makeBlock(void* dst,
LZ4F_blockChecksum_t crcFlag)
{
BYTE* const cSizePtr = (BYTE*)dst;
- U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
- (int)(srcSize), (int)(srcSize-1),
- level, cdict);
- if (cSize == 0) { /* compression failed */
- DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+ U32 cSize;
+ assert(compress != NULL);
+ cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
+ (int)(srcSize), (int)(srcSize-1),
+ level, cdict);
+
+ if (cSize == 0 || cSize >= srcSize) {
cSize = (U32)srcSize;
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
memcpy(cSizePtr+BHSize, src, srcSize);
@@ -766,6 +853,7 @@ static size_t LZ4F_makeBlock(void* dst,
static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
int const acceleration = (level < 0) ? -level + 1 : 1;
+ DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize);
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
if (cdict) {
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
@@ -778,6 +866,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
{
int const acceleration = (level < 0) ? -level + 1 : 1;
(void)cdict; /* init once at beginning of frame */
+ DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize);
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
}
@@ -796,8 +885,15 @@ static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst,
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
}
-static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
+static int LZ4F_doNotCompressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
+{
+ (void)ctx; (void)src; (void)dst; (void)srcSize; (void)dstCapacity; (void)level; (void)cdict;
+ return 0;
+}
+
+static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level, LZ4F_blockCompression_t compressMode)
{
+ if (compressMode == LZ4B_UNCOMPRESSED) return LZ4F_doNotCompressBlock;
if (level < LZ4HC_CLEVEL_MIN) {
if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
return LZ4F_compressBlock_continue;
@@ -806,6 +902,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
return LZ4F_compressBlockHC_continue;
}
+/* Save history (up to 64KB) into @tmpBuff */
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
{
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
@@ -815,38 +912,57 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
-/*! LZ4F_compressUpdate() :
+static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
+
+
+ /*! LZ4F_compressUpdateImpl() :
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
- * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If the block compression does not match the compression of the previous block, the old data is flushed
+ * and operations continue with the new compression mode.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
* or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
*/
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
- void* dstBuffer, size_t dstCapacity,
+static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
const void* srcBuffer, size_t srcSize,
- const LZ4F_compressOptions_t* compressOptionsPtr)
-{
- LZ4F_compressOptions_t cOptionsNull;
+ const LZ4F_compressOptions_t* compressOptionsPtr,
+ LZ4F_blockCompression_t blockCompression)
+ {
size_t const blockSize = cctxPtr->maxBlockSize;
const BYTE* srcPtr = (const BYTE*)srcBuffer;
const BYTE* const srcEnd = srcPtr + srcSize;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
- compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
-
+ compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, blockCompression);
+ size_t bytesWritten;
DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
- if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
+ RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
- return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
- if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
+ RETURN_ERROR(dstMaxSize_tooSmall);
+
+ if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
+ RETURN_ERROR(dstMaxSize_tooSmall);
+
+ /* flush currently written block, to continue with new block compression */
+ if (cctxPtr->blockCompression != blockCompression) {
+ bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+ dstPtr += bytesWritten;
+ cctxPtr->blockCompression = blockCompression;
+ }
+
+ if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
/* complete tmp buffer */
if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
+ assert(blockSize > cctxPtr->tmpInSize);
if (sizeToCopy > srcSize) {
/* add src to tmpIn buffer */
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
@@ -864,11 +980,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag);
-
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
- }
- }
+ } }
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
/* compress full blocks */
@@ -882,33 +996,38 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
- /* compress remaining input < blockSize */
+ /* autoFlush : remaining input (< blockSize) is compressed */
lastBlockCompressed = fromSrcBuffer;
dstPtr += LZ4F_makeBlock(dstPtr,
srcPtr, (size_t)(srcEnd - srcPtr),
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag);
- srcPtr = srcEnd;
+ srcPtr = srcEnd;
}
- /* preserve dictionary if necessary */
+ /* preserve dictionary within @tmpBuff whenever necessary */
if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
+ /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */
+ assert(blockCompression == LZ4B_COMPRESSED);
if (compressOptionsPtr->stableSrc) {
- cctxPtr->tmpIn = cctxPtr->tmpBuff;
+ cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
} else {
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
- if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);
+ assert(0 <= realDictSize && realDictSize <= 64 KB);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
}
/* keep tmpIn within limits */
- if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
- && !(cctxPtr->prefs.autoFlush))
+ if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */
+ && (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */
{
+ /* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
+ * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+ assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
}
/* some input data left, necessarily < blockSize */
@@ -926,6 +1045,53 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
return (size_t)(dstPtr - dstStart);
}
+/*! LZ4F_compressUpdate() :
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr)
+{
+ return LZ4F_compressUpdateImpl(cctxPtr,
+ dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ compressOptionsPtr, LZ4B_COMPRESSED);
+}
+
+/*! LZ4F_compressUpdate() :
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
+ * This is only supported when LZ4F_blockIndependent is used
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr) {
+ RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid);
+ return LZ4F_compressUpdateImpl(cctxPtr,
+ dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ compressOptionsPtr, LZ4B_UNCOMPRESSED);
+}
+
/*! LZ4F_flush() :
* When compressed data must be sent immediately, without waiting for a block to be filled,
@@ -944,13 +1110,12 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
compressFunc_t compress;
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
- if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
- if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize))
- return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- (void)compressOptionsPtr; /* not yet useful */
+ RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);
+ RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall);
+ (void)compressOptionsPtr; /* not useful (yet) */
/* select compression function */
- compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
+ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, cctxPtr->blockCompression);
/* compress tmp buffer */
dstPtr += LZ4F_makeBlock(dstPtr,
@@ -992,19 +1157,19 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity);
- if (LZ4F_isError(flushSize)) return flushSize;
+ FORWARD_IF_ERROR(flushSize);
dstPtr += flushSize;
assert(flushSize <= dstCapacity);
dstCapacity -= flushSize;
- if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+ RETURN_ERROR_IF(dstCapacity < 4, dstMaxSize_tooSmall);
LZ4F_writeLE32(dstPtr, 0);
dstPtr += 4; /* endMark */
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
- if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+ RETURN_ERROR_IF(dstCapacity < 8, dstMaxSize_tooSmall);
DEBUGLOG(5,"Writing 32-bit content checksum");
LZ4F_writeLE32(dstPtr, xxh);
dstPtr+=4; /* content Checksum */
@@ -1015,7 +1180,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
if (cctxPtr->prefs.frameInfo.contentSize) {
if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
- return err0r(LZ4F_ERROR_frameSize_wrong);
+ RETURN_ERROR(frameSize_wrong);
}
return (size_t)(dstPtr - dstStart);
@@ -1039,6 +1204,7 @@ typedef enum {
} dStage_t;
struct LZ4F_dctx_s {
+ LZ4F_CustomMem cmem;
LZ4F_frameInfo_t frameInfo;
U32 version;
dStage_t dStage;
@@ -1056,26 +1222,37 @@ struct LZ4F_dctx_s {
size_t tmpOutStart;
XXH32_state_t xxh;
XXH32_state_t blockChecksum;
+ int skipChecksum;
BYTE header[LZ4F_HEADER_SIZE_MAX];
}; /* typedef'd to LZ4F_dctx in lz4frame.h */
+LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
+{
+ LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem);
+ if (dctx == NULL) return NULL;
+
+ dctx->cmem = customMem;
+ dctx->version = version;
+ return dctx;
+}
+
/*! LZ4F_createDecompressionContext() :
* Create a decompressionContext object, which will track all decompression operations.
* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
* Object can later be released using LZ4F_freeDecompressionContext().
* @return : if != 0, there was an error during context creation.
*/
-LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
+LZ4F_errorCode_t
+LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
{
- LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
- if (dctx == NULL) { /* failed allocation */
- *LZ4F_decompressionContextPtr = NULL;
- return err0r(LZ4F_ERROR_allocation_failed);
- }
+ assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */
+ RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */
- dctx->version = versionNumber;
- *LZ4F_decompressionContextPtr = dctx;
+ *LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber);
+ if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */
+ RETURN_ERROR(allocation_failed);
+ }
return LZ4F_OK_NoError;
}
@@ -1084,9 +1261,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
LZ4F_errorCode_t result = LZ4F_OK_NoError;
if (dctx != NULL) { /* can accept NULL input, like free() */
result = (LZ4F_errorCode_t)dctx->dStage;
- FREEMEM(dctx->tmpIn);
- FREEMEM(dctx->tmpOutBuffer);
- FREEMEM(dctx);
+ LZ4F_free(dctx->tmpIn, dctx->cmem);
+ LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
+ LZ4F_free(dctx, dctx->cmem);
}
return result;
}
@@ -1099,6 +1276,7 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
dctx->dStage = dstage_getFrameHeader;
dctx->dict = NULL;
dctx->dictSize = 0;
+ dctx->skipChecksum = 0;
}
@@ -1118,7 +1296,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
DEBUGLOG(5, "LZ4F_decodeHeader");
/* need to decode header to get frameInfo */
- if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
+ RETURN_ERROR_IF(srcSize < minFHSize, frameHeader_incomplete); /* minimal frame header size */
MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
/* special case : skippable frames */
@@ -1132,14 +1310,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
} else {
dctx->dStage = dstage_getSFrameSize;
return 4;
- }
- }
+ } }
/* control magic number */
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) {
DEBUGLOG(4, "frame header error : unknown magic number");
- return err0r(LZ4F_ERROR_frameType_unknown);
+ RETURN_ERROR(frameType_unknown);
}
#endif
dctx->frameInfo.frameType = LZ4F_frame;
@@ -1153,8 +1330,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
contentChecksumFlag = (FLG>>2) & _1BIT;
dictIDFlag = FLG & _1BIT;
/* validate */
- if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
- if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
+ if (((FLG>>1)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
+ if (version != 1) RETURN_ERROR(headerVersion_wrong); /* Version Number, only supported value */
}
/* Frame Header Size */
@@ -1173,17 +1350,16 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
{ U32 const BD = srcPtr[5];
blockSizeID = (BD>>4) & _3BITS;
/* validate */
- if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
- if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */
- if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+ if (((BD>>7)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
+ if (blockSizeID < 4) RETURN_ERROR(maxBlockSize_invalid); /* 4-7 only supported values for the time being */
+ if (((BD>>0)&_4BITS) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bits */
}
/* check header */
assert(frameHeaderSize > 5);
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
{ BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
- if (HC != srcPtr[frameHeaderSize-1])
- return err0r(LZ4F_ERROR_headerChecksum_invalid);
+ RETURN_ERROR_IF(HC != srcPtr[frameHeaderSize-1], headerChecksum_invalid);
}
#endif
@@ -1192,10 +1368,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
- dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
+ dctx->maxBlockSize = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID);
if (contentSizeFlag)
- dctx->frameRemainingSize =
- dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+ dctx->frameRemainingSize = dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
if (dictIDFlag)
dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
@@ -1211,11 +1386,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
*/
size_t LZ4F_headerSize(const void* src, size_t srcSize)
{
- if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong);
+ RETURN_ERROR_IF(src == NULL, srcPtr_wrong);
/* minimal srcSize to determine header size */
if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH)
- return err0r(LZ4F_ERROR_frameHeader_incomplete);
+ RETURN_ERROR(frameHeader_incomplete);
/* special case : skippable frames */
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
@@ -1224,7 +1399,7 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize)
/* control magic number */
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
- return err0r(LZ4F_ERROR_frameType_unknown);
+ RETURN_ERROR(frameType_unknown);
#endif
/* Frame Header Size */
@@ -1266,13 +1441,13 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
if (dctx->dStage == dstage_storeFrameHeader) {
/* frame decoding already started, in the middle of header => automatic fail */
*srcSizePtr = 0;
- return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
+ RETURN_ERROR(frameDecoding_alreadyStarted);
} else {
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
if (*srcSizePtr < hSize) {
*srcSizePtr=0;
- return err0r(LZ4F_ERROR_frameHeader_incomplete);
+ RETURN_ERROR(frameHeader_incomplete);
}
{ size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
@@ -1290,16 +1465,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
/* LZ4F_updateDict() :
* only used for LZ4F_blockLinked mode
- * Condition : dstPtr != NULL
+ * Condition : @dstPtr != NULL
*/
static void LZ4F_updateDict(LZ4F_dctx* dctx,
const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
unsigned withinTmp)
{
assert(dstPtr != NULL);
- if (dctx->dictSize==0) {
- dctx->dict = (const BYTE*)dstPtr; /* priority to prefix mode */
- }
+ if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* will lead to prefix mode */
assert(dctx->dict != NULL);
if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */
@@ -1362,7 +1535,6 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx,
}
-
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate compressed data in srcBuffer.
* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
@@ -1406,6 +1578,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
*srcSizePtr = 0;
*dstSizePtr = 0;
assert(dctx != NULL);
+ dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */
/* behaves as a state machine */
@@ -1418,7 +1591,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
DEBUGLOG(6, "dstage_getFrameHeader");
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */
- if (LZ4F_isError(hSize)) return hSize;
+ FORWARD_IF_ERROR(hSize);
srcPtr += hSize;
break;
}
@@ -1440,9 +1613,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
- { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */
- if (LZ4F_isError(hSize)) return hSize;
- }
+ FORWARD_IF_ERROR( LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget) ); /* will update dStage appropriately */
break;
case dstage_init:
@@ -1453,14 +1624,12 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
+ ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0);
if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
- FREEMEM(dctx->tmpIn);
- dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */);
- if (dctx->tmpIn == NULL)
- return err0r(LZ4F_ERROR_allocation_failed);
- FREEMEM(dctx->tmpOutBuffer);
- dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
- if (dctx->tmpOutBuffer== NULL)
- return err0r(LZ4F_ERROR_allocation_failed);
+ LZ4F_free(dctx->tmpIn, dctx->cmem);
+ dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem);
+ RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed);
+ LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
+ dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem);
+ RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed);
dctx->maxBufferSize = bufferNeeded;
} }
dctx->tmpInSize = 0;
@@ -1509,7 +1678,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
break;
}
if (nextCBlockSize > dctx->maxBlockSize) {
- return err0r(LZ4F_ERROR_maxBlockSize_invalid);
+ RETURN_ERROR(maxBlockSize_invalid);
}
if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
/* next block is uncompressed */
@@ -1540,11 +1709,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
memcpy(dstPtr, srcPtr, sizeToCopy);
- if (dctx->frameInfo.blockChecksumFlag) {
- (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+ if (!dctx->skipChecksum) {
+ if (dctx->frameInfo.blockChecksumFlag) {
+ (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+ }
+ if (dctx->frameInfo.contentChecksumFlag)
+ (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
}
- if (dctx->frameInfo.contentChecksumFlag)
- (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= sizeToCopy;
@@ -1590,14 +1761,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
}
crcSrc = dctx->header;
}
- { U32 const readCRC = LZ4F_readLE32(crcSrc);
+ if (!dctx->skipChecksum) {
+ U32 const readCRC = LZ4F_readLE32(crcSrc);
U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
DEBUGLOG(6, "compare block checksum");
if (readCRC != calcCRC) {
DEBUGLOG(4, "incorrect block checksum: %08X != %08X",
readCRC, calcCRC);
- return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ RETURN_ERROR(blockChecksum_invalid);
}
#else
(void)readCRC;
@@ -1637,37 +1809,44 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
}
/* At this stage, input is large enough to decode a block */
+
+ /* First, decode and control block checksum if it exists */
if (dctx->frameInfo.blockChecksumFlag) {
+ assert(dctx->tmpInTarget >= 4);
dctx->tmpInTarget -= 4;
assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
{ U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- if (readBlockCrc != calcBlockCrc)
- return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ RETURN_ERROR_IF(readBlockCrc != calcBlockCrc, blockChecksum_invalid);
#else
(void)readBlockCrc;
(void)calcBlockCrc;
#endif
} }
- if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
+ /* decode directly into destination buffer if there is enough room */
+ if ( ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize)
+ /* unless the dictionary is stored in tmpOut:
+ * in which case it's faster to decode within tmpOut
+ * to benefit from prefix speedup */
+ && !(dctx->dict!= NULL && (const BYTE*)dctx->dict + dctx->dictSize == dctx->tmpOut) )
+ {
const char* dict = (const char*)dctx->dict;
size_t dictSize = dctx->dictSize;
int decodedSize;
assert(dstPtr != NULL);
if (dict && dictSize > 1 GB) {
- /* the dictSize param is an int, avoid truncation / sign issues */
+ /* overflow control : dctx->dictSize is an int, avoid truncation / sign issues */
dict += dictSize - 64 KB;
dictSize = 64 KB;
}
- /* enough capacity in `dst` to decompress directly there */
decodedSize = LZ4_decompress_safe_usingDict(
(const char*)selectedIn, (char*)dstPtr,
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
dict, (int)dictSize);
- if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
- if (dctx->frameInfo.contentChecksumFlag)
+ RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
+ if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum))
XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= (size_t)decodedSize;
@@ -1678,25 +1857,27 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
}
dstPtr += decodedSize;
- dctx->dStage = dstage_getBlockHeader;
+ dctx->dStage = dstage_getBlockHeader; /* end of block, let's get another one */
break;
}
/* not enough place into dst : decode into tmpOut */
- /* ensure enough place for tmpOut */
+
+ /* manage dictionary */
if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
if (dctx->dict == dctx->tmpOutBuffer) {
+ /* truncate dictionary to 64 KB if too big */
if (dctx->dictSize > 128 KB) {
memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
dctx->dictSize = 64 KB;
}
dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
- } else { /* dict not within tmp */
+ } else { /* dict not within tmpOut */
size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
} }
- /* Decode block */
+ /* Decode block into tmpOut */
{ const char* dict = (const char*)dctx->dict;
size_t dictSize = dctx->dictSize;
int decodedSize;
@@ -1709,9 +1890,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
(const char*)selectedIn, (char*)dctx->tmpOut,
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
dict, (int)dictSize);
- if (decodedSize < 0) /* decompression failed */
- return err0r(LZ4F_ERROR_decompressionFailed);
- if (dctx->frameInfo.contentChecksumFlag)
+ RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
+ if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum)
XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= (size_t)decodedSize;
@@ -1744,8 +1924,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
break;
case dstage_getSuffix:
- if (dctx->frameRemainingSize)
- return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
+ RETURN_ERROR_IF(dctx->frameRemainingSize, frameSize_wrong); /* incorrect frame size decoded */
if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */
nextSrcSizeHint = 0;
LZ4F_resetDecompressionContext(dctx);
@@ -1777,20 +1956,20 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
} /* if (dctx->dStage == dstage_storeSuffix) */
/* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
- { U32 const readCRC = LZ4F_readLE32(selectedIn);
+ if (!dctx->skipChecksum) {
+ U32 const readCRC = LZ4F_readLE32(selectedIn);
U32 const resultCRC = XXH32_digest(&(dctx->xxh));
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- if (readCRC != resultCRC)
- return err0r(LZ4F_ERROR_contentChecksum_invalid);
+ RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid);
#else
(void)readCRC;
(void)resultCRC;
#endif
- nextSrcSizeHint = 0;
- LZ4F_resetDecompressionContext(dctx);
- doAnotherStage = 0;
- break;
}
+ nextSrcSizeHint = 0;
+ LZ4F_resetDecompressionContext(dctx);
+ doAnotherStage = 0;
+ break;
case dstage_getSFrameSize:
if ((srcEnd - srcPtr) >= 4) {
@@ -1841,7 +2020,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
} /* switch (dctx->dStage) */
} /* while (doAnotherStage) */
- /* preserve history within tmp whenever necessary */
+ /* preserve history within tmpOut whenever necessary */
LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */
&& (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 4573317e..1bdf6c4f 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -1,7 +1,7 @@
/*
- LZ4 auto-framing library
+ LZ4F - LZ4-Frame library
Header File
- Copyright (C) 2011-2017, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -39,7 +39,7 @@
* LZ4F also offers streaming capabilities.
*
* lz4.h is not required when using lz4frame.h,
- * except to extract common constant such as LZ4_VERSION_NUMBER.
+ * except to extract common constants such as LZ4_VERSION_NUMBER.
* */
#ifndef LZ4F_H_09782039843
@@ -54,12 +54,12 @@ extern "C" {
/**
- Introduction
-
- lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
- lz4frame.h provides frame compression functions that take care
- of encoding standard metadata alongside LZ4-compressed blocks.
-*/
+ * Introduction
+ *
+ * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md .
+ * LZ4 Frames are compatible with `lz4` CLI,
+ * and designed to be interoperable with any system.
+**/
/*-***************************************************************
* Compiler specifics
@@ -210,7 +210,7 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */
* Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
* `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
* Note : this result is only usable with LZ4F_compressFrame().
- * It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
+ * It may also be relevant to LZ4F_compressUpdate() _only if_ no flush() operation is ever performed.
*/
LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
@@ -230,7 +230,7 @@ LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
* Advanced compression functions
*************************************/
typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */
-typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */
+typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with older APIs, prefer using LZ4F_cctx */
typedef struct {
unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */
@@ -243,20 +243,27 @@ typedef struct {
LZ4FLIB_API unsigned LZ4F_getVersion(void);
/*! LZ4F_createCompressionContext() :
- * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
- * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
- * The function will provide a pointer to a fully allocated LZ4F_cctx object.
- * If @return != zero, there was an error during context creation.
- * Object can release its memory using LZ4F_freeCompressionContext();
- */
+ * The first thing to do is to create a compressionContext object,
+ * which will keep track of operation state during streaming compression.
+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version,
+ * and a pointer to LZ4F_cctx*, to write the resulting pointer into.
+ * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
+ * The function provides a pointer to a fully allocated LZ4F_cctx object.
+ * @cctxPtr MUST be != NULL.
+ * If @return != zero, context creation failed.
+ * A created compression context can be employed multiple times for consecutive streaming operations.
+ * Once all streaming compression jobs are completed,
+ * the state object can be released using LZ4F_freeCompressionContext().
+ * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored.
+ * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing).
+**/
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/*---- Compression ----*/
-#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */
+#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected parameters */
#define LZ4F_HEADER_SIZE_MAX 19
/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */
@@ -301,8 +308,9 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
* Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
* This value is provided by LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- * LZ4F_compressUpdate() doesn't guarantee error recovery.
- * When an error occurs, compression context must be freed or resized.
+ * After an error, the state is left in a UB state, and must be re-initialized or freed.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
* `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
@@ -347,8 +355,12 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */
typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */
typedef struct {
- unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
- unsigned reserved[3]; /* must be set to zero for forward compatibility */
+ unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified between invocations.
+ * This optimization skips storage operations in tmp buffers. */
+ unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time.
+ * Setting this option to 1 once disables all checksums for the rest of the frame. */
+ unsigned reserved1; /* must be set to zero for forward compatibility */
+ unsigned reserved0; /* idem */
} LZ4F_decompressOptions_t;
@@ -356,9 +368,10 @@ typedef struct {
/*! LZ4F_createDecompressionContext() :
* Create an LZ4F_dctx object, to track all decompression operations.
- * The version provided MUST be LZ4F_VERSION.
- * The function provides a pointer to an allocated and initialized LZ4F_dctx object.
- * The result is an errorCode, which can be tested using LZ4F_isError().
+ * @version provided MUST be LZ4F_VERSION.
+ * @dctxPtr MUST be valid.
+ * The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object.
+ * The @return is an errorCode, which can be tested using LZ4F_isError().
* dctx memory can be released using LZ4F_freeDecompressionContext();
* Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
@@ -371,6 +384,8 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
* Streaming decompression functions
*************************************/
+#define LZ4F_MAGICNUMBER 0x184D2204U
+#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5
/*! LZ4F_headerSize() : v1.9.0+
@@ -386,7 +401,7 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize);
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (max blockSize, dictID, etc.).
- * Its usage is optional: user can call LZ4F_decompress() directly.
+ * Its usage is optional: user can also invoke LZ4F_decompress() directly.
*
* Extracted information will fill an existing LZ4F_frameInfo_t structure.
* This can be useful for allocation and dictionary identification purposes.
@@ -427,9 +442,10 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize);
* note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
* note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
-LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
- LZ4F_frameInfo_t* frameInfoPtr,
- const void* srcBuffer, size_t* srcSizePtr);
+LZ4FLIB_API size_t
+LZ4F_getFrameInfo(LZ4F_dctx* dctx,
+ LZ4F_frameInfo_t* frameInfoPtr,
+ const void* srcBuffer, size_t* srcSizePtr);
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate data compressed in `srcBuffer`.
@@ -462,10 +478,11 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
-LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const LZ4F_decompressOptions_t* dOptPtr);
+LZ4FLIB_API size_t
+LZ4F_decompress(LZ4F_dctx* dctx,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const LZ4F_decompressOptions_t* dOptPtr);
/*! LZ4F_resetDecompressionContext() : added in v1.8.0
@@ -529,6 +546,8 @@ extern "C" {
ITEM(ERROR_headerChecksum_invalid) \
ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_frameDecoding_alreadyStarted) \
+ ITEM(ERROR_compressionState_uninitialized) \
+ ITEM(ERROR_parameter_null) \
ITEM(ERROR_maxCode)
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
@@ -539,7 +558,31 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM)
LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
-LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);
+
+/*! LZ4F_getBlockSize() :
+ * Return, in scalar format (size_t),
+ * the maximum block size associated with blockSizeID.
+**/
+LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID);
+
+/*! LZ4F_uncompressedUpdate() :
+ * LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ * Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ * no compression is done for this operation
+ * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ * After an error, the state is left in a UB state, and must be re-initialized or freed.
+ * If previously a compressed block was written, buffered data is flushed
+ * before appending uncompressed data is continued.
+ * This is only supported when LZ4F_blockIndependent is used
+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+LZ4FLIB_STATIC_API size_t
+LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* cOptPtr);
/**********************************
* Bulk processing dictionary API
@@ -583,12 +626,12 @@ LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict);
* but it's not recommended, as it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer.
* or an error code if it fails (can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
- LZ4F_cctx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* preferencesPtr);
+LZ4FLIB_STATIC_API size_t
+LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr);
/*! LZ4F_compressBegin_usingCDict() :
@@ -598,23 +641,49 @@ LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
* however, it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer for the header,
* or an error code (which can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
- LZ4F_cctx* cctx,
- void* dstBuffer, size_t dstCapacity,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* prefsPtr);
+LZ4FLIB_STATIC_API size_t
+LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* prefsPtr);
/*! LZ4F_decompress_usingDict() :
* Same as LZ4F_decompress(), using a predefined dictionary.
* Dictionary is used "in place", without any preprocessing.
- * It must remain accessible throughout the entire frame decoding. */
-LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
- LZ4F_dctx* dctxPtr,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const void* dict, size_t dictSize,
- const LZ4F_decompressOptions_t* decompressOptionsPtr);
+** It must remain accessible throughout the entire frame decoding. */
+LZ4FLIB_STATIC_API size_t
+LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+
+/*! Custom memory allocation :
+ * These prototypes make it possible to pass custom allocation/free functions.
+ * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below.
+ * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
+ */
+typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size);
+typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size);
+typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address);
+typedef struct {
+ LZ4F_AllocFunction customAlloc;
+ LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */
+ LZ4F_FreeFunction customFree;
+ void* opaqueState;
+} LZ4F_CustomMem;
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
+
+LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version);
+LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version);
+LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict_advanced(LZ4F_CustomMem customMem, const void* dictBuffer, size_t dictSize);
+
#if defined (__cplusplus)
}
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index 925a2c5c..2b44a631 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -1,7 +1,7 @@
/*
LZ4 auto-framing library
Header File for static linking only
- Copyright (C) 2011-2016, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 77c9f430..b21ad6bb 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
- Copyright (C) 2011-2017, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -42,7 +42,7 @@
* Select how default compression function will allocate workplace memory,
* in stack (0:fastest), or in heap (1:requires malloc()).
* Since workplace is rather large, heap mode is recommended.
- */
+**/
#ifndef LZ4HC_HEAPMODE
# define LZ4HC_HEAPMODE 1
#endif
@@ -99,18 +99,20 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start)
{
- uptrval startingOffset = (uptrval)(hc4->end - hc4->base);
- if (startingOffset > 1 GB) {
+ size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart);
+ size_t newStartingOffset = bufferSize + hc4->dictLimit;
+ assert(newStartingOffset >= bufferSize); /* check overflow */
+ if (newStartingOffset > 1 GB) {
LZ4HC_clearTables(hc4);
- startingOffset = 0;
+ newStartingOffset = 0;
}
- startingOffset += 64 KB;
- hc4->nextToUpdate = (U32) startingOffset;
- hc4->base = start - startingOffset;
+ newStartingOffset += 64 KB;
+ hc4->nextToUpdate = (U32)newStartingOffset;
+ hc4->prefixStart = start;
hc4->end = start;
- hc4->dictBase = start - startingOffset;
- hc4->dictLimit = (U32) startingOffset;
- hc4->lowLimit = (U32) startingOffset;
+ hc4->dictStart = start;
+ hc4->dictLimit = (U32)newStartingOffset;
+ hc4->lowLimit = (U32)newStartingOffset;
}
@@ -119,12 +121,15 @@ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
{
U16* const chainTable = hc4->chainTable;
U32* const hashTable = hc4->hashTable;
- const BYTE* const base = hc4->base;
- U32 const target = (U32)(ip - base);
+ const BYTE* const prefixPtr = hc4->prefixStart;
+ U32 const prefixIdx = hc4->dictLimit;
+ U32 const target = (U32)(ip - prefixPtr) + prefixIdx;
U32 idx = hc4->nextToUpdate;
+ assert(ip >= prefixPtr);
+ assert(target >= prefixIdx);
while (idx < target) {
- U32 const h = LZ4HC_hashPtr(base+idx);
+ U32 const h = LZ4HC_hashPtr(prefixPtr+idx-prefixIdx);
size_t delta = idx - hashTable[h];
if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX;
DELTANEXTU16(chainTable, idx) = (U16)delta;
@@ -193,15 +198,14 @@ LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
BYTE const byte = (BYTE)(pattern >> bitOffset);
if (*ip != byte) break;
ip ++; bitOffset -= 8;
- }
- }
+ } }
return (unsigned)(ip - iStart);
}
/* LZ4HC_reverseCountPattern() :
* pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!)
- * read using natural platform endianess */
+ * read using natural platform endianness */
static unsigned
LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
{
@@ -211,7 +215,7 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
if (LZ4_read32(ip-4) != pattern) break;
ip -= 4;
}
- { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */
+ { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianness */
while (likely(ip>iLow)) {
if (ip[-1] != *bytePtr) break;
ip--; bytePtr--;
@@ -234,28 +238,28 @@ typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e;
LZ4_FORCE_INLINE int
LZ4HC_InsertAndGetWiderMatch (
- LZ4HC_CCtx_internal* hc4,
- const BYTE* const ip,
- const BYTE* const iLowLimit,
- const BYTE* const iHighLimit,
- int longest,
- const BYTE** matchpos,
- const BYTE** startpos,
- const int maxNbAttempts,
- const int patternAnalysis,
- const int chainSwap,
- const dictCtx_directive dict,
- const HCfavor_e favorDecSpeed)
+ LZ4HC_CCtx_internal* const hc4,
+ const BYTE* const ip,
+ const BYTE* const iLowLimit, const BYTE* const iHighLimit,
+ int longest,
+ const BYTE** matchpos,
+ const BYTE** startpos,
+ const int maxNbAttempts,
+ const int patternAnalysis, const int chainSwap,
+ const dictCtx_directive dict,
+ const HCfavor_e favorDecSpeed)
{
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx;
- const BYTE* const base = hc4->base;
- const U32 dictLimit = hc4->dictLimit;
- const BYTE* const lowPrefixPtr = base + dictLimit;
- const U32 ipIndex = (U32)(ip - base);
- const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
- const BYTE* const dictBase = hc4->dictBase;
+ const BYTE* const prefixPtr = hc4->prefixStart;
+ const U32 prefixIdx = hc4->dictLimit;
+ const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx;
+ const int withinStartDistance = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex);
+ const U32 lowestMatchIndex = (withinStartDistance) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
+ const BYTE* const dictStart = hc4->dictStart;
+ const U32 dictIdx = hc4->lowLimit;
+ const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx;
int const lookBackLength = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
U32 matchChainPos = 0;
@@ -277,14 +281,13 @@ LZ4HC_InsertAndGetWiderMatch (
assert(matchIndex < ipIndex);
if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
/* do nothing */
- } else if (matchIndex >= dictLimit) { /* within current Prefix */
- const BYTE* const matchPtr = base + matchIndex;
- assert(matchPtr >= lowPrefixPtr);
+ } else if (matchIndex >= prefixIdx) { /* within current Prefix */
+ const BYTE* const matchPtr = prefixPtr + matchIndex - prefixIdx;
assert(matchPtr < ip);
assert(longest >= 1);
if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
if (LZ4_read32(matchPtr) == pattern) {
- int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
+ int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, prefixPtr) : 0;
matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
matchLength -= back;
if (matchLength > longest) {
@@ -293,24 +296,25 @@ LZ4HC_InsertAndGetWiderMatch (
*startpos = ip + back;
} } }
} else { /* lowestMatchIndex <= matchIndex < dictLimit */
- const BYTE* const matchPtr = dictBase + matchIndex;
- if (LZ4_read32(matchPtr) == pattern) {
- const BYTE* const dictStart = dictBase + hc4->lowLimit;
+ const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx);
+ assert(matchIndex >= dictIdx);
+ if ( likely(matchIndex <= prefixIdx - 4)
+ && (LZ4_read32(matchPtr) == pattern) ) {
int back = 0;
- const BYTE* vLimit = ip + (dictLimit - matchIndex);
+ const BYTE* vLimit = ip + (prefixIdx - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
- matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit);
+ matchLength += LZ4_count(ip+matchLength, prefixPtr, iHighLimit);
back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;
matchLength -= back;
if (matchLength > longest) {
longest = matchLength;
- *matchpos = base + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */
+ *matchpos = prefixPtr - prefixIdx + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */
*startpos = ip + back;
} } }
- if (chainSwap && matchLength==longest) { /* better match => select a better chain */
+ if (chainSwap && matchLength==longest) { /* better match => select a better chain */
assert(lookBackLength==0); /* search forward only */
if (matchIndex + (U32)longest <= ipIndex) {
int const kTrigger = 4;
@@ -326,8 +330,7 @@ LZ4HC_InsertAndGetWiderMatch (
distanceToNextMatch = candidateDist;
matchChainPos = (U32)pos;
accel = 1 << kTrigger;
- }
- }
+ } }
if (distanceToNextMatch > 1) {
if (distanceToNextMatch > matchIndex) break; /* avoid overflow */
matchIndex -= distanceToNextMatch;
@@ -347,23 +350,24 @@ LZ4HC_InsertAndGetWiderMatch (
repeat = rep_not;
} }
if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex)
- && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) {
- const int extDict = matchCandidateIdx < dictLimit;
- const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx;
+ && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) {
+ const int extDict = matchCandidateIdx < prefixIdx;
+ const BYTE* const matchPtr = (extDict ? dictStart - dictIdx : prefixPtr - prefixIdx) + matchCandidateIdx;
if (LZ4_read32(matchPtr) == pattern) { /* good candidate */
- const BYTE* const dictStart = dictBase + hc4->lowLimit;
- const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit;
+ const BYTE* const iLimit = extDict ? dictEnd : iHighLimit;
size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern);
if (extDict && matchPtr + forwardPatternLength == iLimit) {
U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern);
- forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern);
+ forwardPatternLength += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern);
}
- { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr;
+ { const BYTE* const lowestMatchPtr = extDict ? dictStart : prefixPtr;
size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
size_t currentSegmentLength;
- if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) {
+ if (!extDict
+ && matchPtr - backLength == prefixPtr
+ && dictIdx < prefixIdx) {
U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern);
- backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern);
+ backLength += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern);
}
/* Limit backLength not go further than lowestMatchIndex */
backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex);
@@ -373,28 +377,28 @@ LZ4HC_InsertAndGetWiderMatch (
if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */
&& (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */
- if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex))
+ if (LZ4HC_protectDictEnd(prefixIdx, newMatchIndex))
matchIndex = newMatchIndex;
else {
/* Can only happen if started in the prefix */
- assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
- matchIndex = dictLimit;
+ assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict);
+ matchIndex = prefixIdx;
}
} else {
U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
- if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) {
- assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
- matchIndex = dictLimit;
+ if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) {
+ assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict);
+ matchIndex = prefixIdx;
} else {
matchIndex = newMatchIndex;
if (lookBackLength==0) { /* no back possible */
size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
if ((size_t)longest < maxML) {
- assert(base + matchIndex != ip);
- if ((size_t)(ip - base) - matchIndex > LZ4_DISTANCE_MAX) break;
+ assert(prefixPtr - prefixIdx + matchIndex != ip);
+ if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break;
assert(maxML < 2 GB);
longest = (int)maxML;
- *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
+ *matchpos = prefixPtr - prefixIdx + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
*startpos = ip;
}
{ U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
@@ -413,12 +417,12 @@ LZ4HC_InsertAndGetWiderMatch (
if ( dict == usingDictCtxHc
&& nbAttempts > 0
&& ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) {
- size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base);
+ size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit;
U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
assert(dictEndOffset <= 1 GB);
matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset;
while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) {
- const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
+ const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex;
if (LZ4_read32(matchPtr) == pattern) {
int mlt;
@@ -426,11 +430,11 @@ LZ4HC_InsertAndGetWiderMatch (
const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
- back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+ back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0;
mlt -= back;
if (mlt > longest) {
longest = mlt;
- *matchpos = base + matchIndex + back;
+ *matchpos = prefixPtr - prefixIdx + matchIndex + back;
*startpos = ip + back;
} }
@@ -442,13 +446,13 @@ LZ4HC_InsertAndGetWiderMatch (
return longest;
}
-LZ4_FORCE_INLINE
-int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
- const BYTE* const ip, const BYTE* const iLimit,
- const BYTE** matchpos,
- const int maxNbAttempts,
- const int patternAnalysis,
- const dictCtx_directive dict)
+LZ4_FORCE_INLINE int
+LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
+ const BYTE* const ip, const BYTE* const iLimit,
+ const BYTE** matchpos,
+ const int maxNbAttempts,
+ const int patternAnalysis,
+ const dictCtx_directive dict)
{
const BYTE* uselessPtr = ip;
/* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
@@ -751,7 +755,7 @@ _last_literals:
} else {
*op++ = (BYTE)(lastRunSize << ML_BITS);
}
- memcpy(op, anchor, lastRunSize);
+ LZ4_memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
@@ -884,13 +888,13 @@ LZ4HC_compress_generic_dictCtx (
limitedOutput_directive limit
)
{
- const size_t position = (size_t)(ctx->end - ctx->base) - ctx->lowLimit;
+ const size_t position = (size_t)(ctx->end - ctx->prefixStart) + (ctx->dictLimit - ctx->lowLimit);
assert(ctx->dictCtx != NULL);
if (position >= 64 KB) {
ctx->dictCtx = NULL;
return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
} else if (position == 0 && *srcSizePtr > 4 KB) {
- memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
+ LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
LZ4HC_setExternalDict(ctx, (const BYTE *)src);
ctx->compressionLevel = (short)cLevel;
return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
@@ -953,13 +957,15 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
+ int cSize;
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
+ if (statePtr==NULL) return 0;
#else
LZ4_streamHC_t state;
LZ4_streamHC_t* const statePtr = &state;
#endif
- int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+ cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
FREEMEM(statePtr);
#endif
@@ -982,6 +988,7 @@ int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* s
* Streaming Functions
**************************************/
/* allocation */
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_streamHC_t* LZ4_createStreamHC(void)
{
LZ4_streamHC_t* const state =
@@ -998,13 +1005,12 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr)
FREEMEM(LZ4_streamHCPtr);
return 0;
}
+#endif
LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size)
{
LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer;
- /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
- LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE);
DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size);
/* check conditions */
if (buffer == NULL) return NULL;
@@ -1030,9 +1036,13 @@ void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
if (LZ4_streamHCPtr->internal_donotuse.dirty) {
LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));
} else {
- /* preserve end - base : can trigger clearTable's threshold */
- LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
- LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ /* preserve end - prefixStart : can trigger clearTable's threshold */
+ if (LZ4_streamHCPtr->internal_donotuse.end != NULL) {
+ LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.prefixStart;
+ } else {
+ assert(LZ4_streamHCPtr->internal_donotuse.prefixStart == NULL);
+ }
+ LZ4_streamHCPtr->internal_donotuse.prefixStart = NULL;
LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
}
LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
@@ -1083,14 +1093,14 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
{
DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
- if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4)
+ if (ctxPtr->end >= ctxPtr->prefixStart + 4)
LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
- ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
- ctxPtr->dictBase = ctxPtr->base;
- ctxPtr->base = newBlock - ctxPtr->dictLimit;
+ ctxPtr->dictStart = ctxPtr->prefixStart;
+ ctxPtr->dictLimit += (U32)(ctxPtr->end - ctxPtr->prefixStart);
+ ctxPtr->prefixStart = newBlock;
ctxPtr->end = newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
@@ -1109,11 +1119,11 @@ LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
LZ4_streamHCPtr, src, *srcSizePtr, limit);
assert(ctxPtr != NULL);
/* auto-init if forgotten */
- if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src);
+ if (ctxPtr->prefixStart == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src);
/* Check overflow */
- if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
- size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
+ if ((size_t)(ctxPtr->end - ctxPtr->prefixStart) + ctxPtr->dictLimit > 2 GB) {
+ size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->prefixStart);
if (dictSize > 64 KB) dictSize = 64 KB;
LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
}
@@ -1124,13 +1134,16 @@ LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
/* Check overlapping input/dictionary space */
{ const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr;
- const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
- const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
+ const BYTE* const dictBegin = ctxPtr->dictStart;
+ const BYTE* const dictEnd = ctxPtr->dictStart + (ctxPtr->dictLimit - ctxPtr->lowLimit);
if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) {
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
- ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
- if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
- } }
+ ctxPtr->lowLimit += (U32)(sourceEnd - ctxPtr->dictStart);
+ ctxPtr->dictStart += (U32)(sourceEnd - ctxPtr->dictStart);
+ if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) {
+ ctxPtr->lowLimit = ctxPtr->dictLimit;
+ ctxPtr->dictStart = ctxPtr->prefixStart;
+ } } }
return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit);
}
@@ -1158,7 +1171,7 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
- int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+ int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart);
DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
assert(prefixSize >= 0);
if (dictSize > 64 KB) dictSize = 64 KB;
@@ -1166,12 +1179,13 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
if (dictSize > prefixSize) dictSize = prefixSize;
if (safeBuffer == NULL) assert(dictSize == 0);
if (dictSize > 0)
- memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
- { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base);
+ LZ4_memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
+ { U32 const endIndex = (U32)(streamPtr->end - streamPtr->prefixStart) + streamPtr->dictLimit;
streamPtr->end = (const BYTE*)safeBuffer + dictSize;
- streamPtr->base = streamPtr->end - endIndex;
+ streamPtr->prefixStart = streamPtr->end - dictSize;
streamPtr->dictLimit = endIndex - (U32)dictSize;
streamPtr->lowLimit = endIndex - (U32)dictSize;
+ streamPtr->dictStart = streamPtr->prefixStart;
if (streamPtr->nextToUpdate < streamPtr->dictLimit)
streamPtr->nextToUpdate = streamPtr->dictLimit;
}
@@ -1199,7 +1213,7 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src,
/* Deprecated streaming functions */
-int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
+int LZ4_sizeofStreamStateHC(void) { return sizeof(LZ4_streamHC_t); }
/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t)
* @return : 0 on success, !=0 if error */
@@ -1211,6 +1225,7 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
return 0;
}
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
void* LZ4_createHC (const char* inputBuffer)
{
LZ4_streamHC_t* const hc4 = LZ4_createStreamHC();
@@ -1225,6 +1240,7 @@ int LZ4_freeHC (void* LZ4HC_Data)
FREEMEM(LZ4HC_Data);
return 0;
}
+#endif
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
{
@@ -1238,11 +1254,11 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
- LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data;
- const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit;
+ LZ4_streamHC_t* const ctx = (LZ4_streamHC_t*)LZ4HC_Data;
+ const BYTE* bufferStart = ctx->internal_donotuse.prefixStart - ctx->internal_donotuse.dictLimit + ctx->internal_donotuse.lowLimit;
LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel);
/* avoid const char * -> char * conversion warning :( */
- return (char *)(uptrval)bufferStart;
+ return (char*)(uptrval)bufferStart;
}
@@ -1325,7 +1341,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
{
int retval = 0;
#define TRAILING_LITERALS 3
-#ifdef LZ4HC_HEAPMODE
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS));
#else
LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */
@@ -1343,7 +1359,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
const BYTE* ovref = NULL;
/* init */
-#ifdef LZ4HC_HEAPMODE
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
if (opt == NULL) goto _return_label;
#endif
DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity);
@@ -1575,7 +1591,7 @@ _last_literals:
} else {
*op++ = (BYTE)(lastRunSize << ML_BITS);
}
- memcpy(op, anchor, lastRunSize);
+ LZ4_memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
@@ -1608,7 +1624,7 @@ if (limit == fillOutput) {
goto _last_literals;
}
_return_label:
-#ifdef LZ4HC_HEAPMODE
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
FREEMEM(opt);
#endif
return retval;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 3d441fb6..e937acfe 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -1,7 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
- Copyright (C) 2011-2017, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -198,14 +198,17 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
+/* Never ever use these definitions directly !
+ * Declare or allocate an LZ4_streamHC_t instead.
+**/
typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
struct LZ4HC_CCtx_internal
{
LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE];
LZ4_u16 chainTable[LZ4HC_MAXD];
const LZ4_byte* end; /* next block here to continue on current prefix */
- const LZ4_byte* base; /* All index relative to this position */
- const LZ4_byte* dictBase; /* alternate base for extDict */
+ const LZ4_byte* prefixStart; /* Indexes relative to this position */
+ const LZ4_byte* dictStart; /* alternate reference for extDict */
LZ4_u32 dictLimit; /* below that point, need extDict */
LZ4_u32 lowLimit; /* below that point, no more dict */
LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */
@@ -216,20 +219,15 @@ struct LZ4HC_CCtx_internal
const LZ4HC_CCtx_internal* dictCtx;
};
-
-/* Do not use these definitions directly !
- * Declare or allocate an LZ4_streamHC_t instead.
- */
-#define LZ4_STREAMHCSIZE 262200 /* static size, for inter-version compatibility */
-#define LZ4_STREAMHCSIZE_VOIDP (LZ4_STREAMHCSIZE / sizeof(void*))
+#define LZ4_STREAMHC_MINSIZE 262200 /* static size, for inter-version compatibility */
union LZ4_streamHC_u {
- void* table[LZ4_STREAMHCSIZE_VOIDP];
+ char minStateSize[LZ4_STREAMHC_MINSIZE];
LZ4HC_CCtx_internal internal_donotuse;
}; /* previously typedef'd to LZ4_streamHC_t */
/* LZ4_streamHC_t :
* This structure allows static allocation of LZ4 HC streaming state.
- * This can be used to allocate statically, on state, or as part of a larger structure.
+ * This can be used to allocate statically on stack, or as part of a larger structure.
*
* Such state **must** be initialized using LZ4_initStreamHC() before first use.
*
@@ -244,7 +242,7 @@ union LZ4_streamHC_u {
* Required before first use of a statically allocated LZ4_streamHC_t.
* Before v1.9.0 : use LZ4_resetStreamHC() instead
*/
-LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size);
+LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC(void* buffer, size_t size);
/*-************************************
@@ -272,9 +270,11 @@ LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_comp
* LZ4_slideInputBufferHC() will truncate the history of the stream, rather
* than preserve a window-sized chunk of history.
*/
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data);
+#endif
+LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void);
@@ -305,7 +305,7 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL
* They should not be linked from DLL,
* as there is no guarantee of API stability yet.
* Prototypes will be promoted to "stable" status
- * after successfull usage in real-life scenarios.
+ * after successful usage in real-life scenarios.
***************************************************/
#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */
#ifndef LZ4_HC_SLO_098092834
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-08 08:08:13 +0000