aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYann Collet <Cyan4973@users.noreply.github.com>2022-07-29 20:56:22 +0200
committerGitHub <noreply@github.com>2022-07-29 20:56:22 +0200
commit5c8bb8739eb5e503a02a1976d1ddcadb049a041d (patch)
tree0cfdde4daa06923e62f050802434f1e5f4b7fe98
parentfb46a7fe8cd99bb9b8d9fc95d8fa6c238593a740 (diff)
parent4f4d09a0d1604da4a93d05716e586d73231c2e63 (diff)
downloadlz4-5c8bb8739eb5e503a02a1976d1ddcadb049a041d.tar.gz
Merge pull request #1121 from lz4/decBench
Implement decoder-only benchmark
-rw-r--r--lib/lz4.h21
-rw-r--r--lib/lz4frame.h75
-rw-r--r--programs/bench.c426
-rw-r--r--programs/bench.h26
-rw-r--r--programs/lz4cli.c13
-rw-r--r--tests/Makefile3
-rw-r--r--tests/fullbench.c3
7 files changed, 344 insertions, 223 deletions
diff --git a/lib/lz4.h b/lib/lz4.h
index 7081e76c..fee8890e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -417,7 +417,10 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
* save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
*/
-LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
+LZ4LIB_API int
+LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode,
+ const char* src, char* dst,
+ int srcSize, int dstCapacity);
/*! LZ4_decompress_*_usingDict() :
@@ -428,9 +431,17 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod
* Performance tip : Decompression speed can be substantially increased
* when dst == dictStart + dictSize.
*/
-LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
+LZ4LIB_API int
+LZ4_decompress_safe_usingDict(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize);
+
+LZ4LIB_API int
+LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
+ int compressedSize,
+ int targetOutputSize, int maxOutputSize,
+ const char* dictStart, int dictSize);
-LZ4LIB_API int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize, const char* dictStart, int dictSize);
#endif /* LZ4_H_2983827168210 */
@@ -508,7 +519,9 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the completion of the first compression call on the stream.
*/
-LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
+LZ4LIB_STATIC_API void
+LZ4_attach_dictionary(LZ4_stream_t* workingStream,
+ const LZ4_stream_t* dictionaryStream);
/*! In-place compression and decompression
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index a864ef9a..1c2d0dd9 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -54,12 +54,12 @@ extern "C" {
/**
- Introduction
-
- lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
- lz4frame.h provides frame compression functions that take care
- of encoding standard metadata alongside LZ4-compressed blocks.
-*/
+ * Introduction
+ *
+ * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md .
+ * LZ4 Frames are compatible with `lz4` CLI,
+ * and designed to be interoperable with any system.
+**/
/*-***************************************************************
* Compiler specifics
@@ -397,7 +397,7 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize);
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (max blockSize, dictID, etc.).
- * Its usage is optional: user can call LZ4F_decompress() directly.
+ * Its usage is optional: user can also invoke LZ4F_decompress() directly.
*
* Extracted information will fill an existing LZ4F_frameInfo_t structure.
* This can be useful for allocation and dictionary identification purposes.
@@ -438,9 +438,10 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize);
* note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
* note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
-LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
- LZ4F_frameInfo_t* frameInfoPtr,
- const void* srcBuffer, size_t* srcSizePtr);
+LZ4FLIB_API size_t
+LZ4F_getFrameInfo(LZ4F_dctx* dctx,
+ LZ4F_frameInfo_t* frameInfoPtr,
+ const void* srcBuffer, size_t* srcSizePtr);
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate data compressed in `srcBuffer`.
@@ -473,10 +474,11 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
-LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const LZ4F_decompressOptions_t* dOptPtr);
+LZ4FLIB_API size_t
+LZ4F_decompress(LZ4F_dctx* dctx,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const LZ4F_decompressOptions_t* dOptPtr);
/*! LZ4F_resetDecompressionContext() : added in v1.8.0
@@ -572,10 +574,11 @@ LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID);
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
*/
-LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
- void* dstBuffer, size_t dstCapacity,
- const void* srcBuffer, size_t srcSize,
- const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_STATIC_API size_t
+LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* cOptPtr);
/**********************************
* Bulk processing dictionary API
@@ -619,12 +622,12 @@ LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict);
* but it's not recommended, as it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer.
* or an error code if it fails (can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
- LZ4F_cctx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* preferencesPtr);
+LZ4FLIB_STATIC_API size_t
+LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr);
/*! LZ4F_compressBegin_usingCDict() :
@@ -634,23 +637,23 @@ LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
* however, it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer for the header,
* or an error code (which can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
- LZ4F_cctx* cctx,
- void* dstBuffer, size_t dstCapacity,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* prefsPtr);
+LZ4FLIB_STATIC_API size_t
+LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* prefsPtr);
/*! LZ4F_decompress_usingDict() :
* Same as LZ4F_decompress(), using a predefined dictionary.
* Dictionary is used "in place", without any preprocessing.
- * It must remain accessible throughout the entire frame decoding. */
-LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
- LZ4F_dctx* dctxPtr,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const void* dict, size_t dictSize,
- const LZ4F_decompressOptions_t* decompressOptionsPtr);
+** It must remain accessible throughout the entire frame decoding. */
+LZ4FLIB_STATIC_API size_t
+LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr);
/*! Custom memory allocation :
diff --git a/programs/bench.c b/programs/bench.c
index 603705c0..7836717f 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -51,10 +51,98 @@
#include "lz4.h"
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
+#include "lz4frame.h" /* LZ4F_decompress */
/* *************************************
-* Compression parameters and functions
+* Constants
+***************************************/
+#ifndef LZ4_GIT_COMMIT_STRING
+# define LZ4_GIT_COMMIT_STRING ""
+#else
+# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT)
+#endif
+
+#define NBSECONDS 3
+#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
+#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */
+#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
+#define COOLPERIOD_SEC 10
+#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define LZ4_MAX_DICT_SIZE (64 KB)
+
+static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+
+static U32 g_compressibilityDefault = 50;
+
+
+/* *************************************
+* console display
+***************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
+ { g_time = clock(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stdout); } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+
+/* *************************************
+* DEBUG and error conditions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define END_PROCESS(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
+
+#define LZ4_isError(errcode) (errcode==0)
+
+
+/* *************************************
+* Benchmark Parameters
+***************************************/
+static U32 g_nbSeconds = NBSECONDS;
+static size_t g_blockSize = 0;
+int g_additionalParam = 0;
+int g_benchSeparately = 0;
+int g_decodeOnly = 0;
+
+void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
+
+void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+
+void BMK_setNbSeconds(unsigned nbSeconds)
+{
+ g_nbSeconds = nbSeconds;
+ DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
+}
+
+void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; }
+
+void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); }
+
+void BMK_setDecodeOnlyMode(int set) { g_decodeOnly = (set!=0); }
+
+
+/* *************************************
+ * Compression state management
***************************************/
struct compressionParameters
@@ -79,8 +167,8 @@ struct compressionParameters
const struct compressionParameters* pThis);
};
-static void LZ4_compressInitNoStream(
- struct compressionParameters* pThis)
+static void
+LZ4_compressInitNoStream(struct compressionParameters* pThis)
{
pThis->LZ4_stream = NULL;
pThis->LZ4_dictStream = NULL;
@@ -88,8 +176,8 @@ static void LZ4_compressInitNoStream(
pThis->LZ4_dictStreamHC = NULL;
}
-static void LZ4_compressInitStream(
- struct compressionParameters* pThis)
+static void
+LZ4_compressInitStream(struct compressionParameters* pThis)
{
pThis->LZ4_stream = LZ4_createStream();
pThis->LZ4_dictStream = LZ4_createStream();
@@ -98,8 +186,8 @@ static void LZ4_compressInitStream(
LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize);
}
-static void LZ4_compressInitStreamHC(
- struct compressionParameters* pThis)
+static void
+LZ4_compressInitStreamHC(struct compressionParameters* pThis)
{
pThis->LZ4_stream = NULL;
pThis->LZ4_dictStream = NULL;
@@ -108,83 +196,84 @@ static void LZ4_compressInitStreamHC(
LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize);
}
-static void LZ4_compressResetNoStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressResetNoStream(const struct compressionParameters* pThis)
{
(void)pThis;
}
-static void LZ4_compressResetStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressResetStream(const struct compressionParameters* pThis)
{
LZ4_resetStream_fast(pThis->LZ4_stream);
LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream);
}
-static void LZ4_compressResetStreamHC(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressResetStreamHC(const struct compressionParameters* pThis)
{
LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel);
LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC);
}
-static int LZ4_compressBlockNoStream(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockNoStream(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
}
-static int LZ4_compressBlockNoStreamHC(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockNoStreamHC(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel);
}
-static int LZ4_compressBlockStream(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockStream(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration);
}
-static int LZ4_compressBlockStreamHC(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockStreamHC(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize);
}
-static void LZ4_compressCleanupNoStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressCleanupNoStream(const struct compressionParameters* pThis)
{
(void)pThis;
}
-static void LZ4_compressCleanupStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressCleanupStream(const struct compressionParameters* pThis)
{
LZ4_freeStream(pThis->LZ4_stream);
LZ4_freeStream(pThis->LZ4_dictStream);
}
-static void LZ4_compressCleanupStreamHC(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressCleanupStreamHC(const struct compressionParameters* pThis)
{
LZ4_freeStreamHC(pThis->LZ4_streamHC);
LZ4_freeStreamHC(pThis->LZ4_dictStreamHC);
}
-static void LZ4_buildCompressionParameters(
- struct compressionParameters* pParams,
- int cLevel, const char* dictBuf, int dictSize)
+static void
+LZ4_buildCompressionParameters(struct compressionParameters* pParams,
+ int cLevel,
+ const char* dictBuf, int dictSize)
{
pParams->cLevel = cLevel;
pParams->dictBuf = dictBuf;
@@ -215,90 +304,32 @@ static void LZ4_buildCompressionParameters(
}
}
-#define LZ4_isError(errcode) (errcode==0)
-
-
-/* *************************************
-* Constants
-***************************************/
-#ifndef LZ4_GIT_COMMIT_STRING
-# define LZ4_GIT_COMMIT_STRING ""
-#else
-# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT)
-#endif
-
-#define NBSECONDS 3
-#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
-#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */
-#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
-#define COOLPERIOD_SEC 10
-#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */
-
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
-
-#define LZ4_MAX_DICT_SIZE (64 KB)
-
-static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
-
-static U32 g_compressibilityDefault = 50;
-
-
-/* *************************************
-* console display
-***************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
-
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
- { g_time = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_time = 0;
-
-
-/* *************************************
-* Exceptions
-***************************************/
-#ifndef DEBUG
-# define DEBUG 0
-#endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...) \
-{ \
- DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
- DISPLAYLEVEL(1, "Error %i : ", error); \
- DISPLAYLEVEL(1, __VA_ARGS__); \
- DISPLAYLEVEL(1, "\n"); \
- exit(error); \
-}
-
-
-/* *************************************
-* Benchmark Parameters
-***************************************/
-static U32 g_nbSeconds = NBSECONDS;
-static size_t g_blockSize = 0;
-int g_additionalParam = 0;
-int g_benchSeparately = 0;
-void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
+typedef int (*DecFunction_f)(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize);
-void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+static LZ4F_dctx* g_dctx = NULL;
-void BMK_setNbSeconds(unsigned nbSeconds)
+static int
+LZ4F_decompress_binding(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize)
{
- g_nbSeconds = nbSeconds;
- DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
+ size_t dstSize = (size_t)dstCapacity;
+ size_t readSize = (size_t)srcSize;
+ size_t const decStatus = LZ4F_decompress(g_dctx,
+ dst, &dstSize,
+ src, &readSize,
+ NULL /* dOptPtr */);
+ if ( (decStatus == 0) /* decompression successful */
+ && ((int)readSize==srcSize) /* consume all input */ )
+ return (int)dstSize;
+ /* else, error */
+ return -1;
+ (void)dictStart; (void)dictSize; /* not compatible with dictionary yet */
}
-void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; }
-
-void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); }
-
/* ********************************************************
* Bench functions
@@ -321,24 +352,32 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, U32 nbFiles,
const char* dictBuf, int dictSize)
{
- size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
- U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ size_t const blockSize = (g_blockSize>=32 && !g_decodeOnly ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+ U32 const maxNbBlocks = (U32)((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
- size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
+ size_t const maxCompressedSize = (size_t)LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
- void* const resultBuffer = malloc(srcSize);
+ size_t const decMultiplier = g_decodeOnly ? 255 : 1;
+ size_t const maxInSize = (size_t)LZ4_MAX_INPUT_SIZE / decMultiplier;
+ size_t const maxDecSize = srcSize < maxInSize ? srcSize * decMultiplier : LZ4_MAX_INPUT_SIZE;
+ void* const resultBuffer = malloc(maxDecSize);
U32 nbBlocks;
struct compressionParameters compP;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable)
- EXM_THROW(31, "allocation error : not enough memory");
+ END_PROCESS(31, "allocation error : not enough memory");
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
/* init */
LZ4_buildCompressionParameters(&compP, cLevel, dictBuf, dictSize);
compP.initFunction(&compP);
+ if (g_dctx==NULL) {
+ LZ4F_createDecompressionContext(&g_dctx, LZ4F_VERSION);
+ if (g_dctx==NULL)
+ END_PROCESS(1, "allocation error - decompression state");
+ }
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
@@ -351,6 +390,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
+ size_t const resMaxSize = thisBlockSize * decMultiplier;
+ size_t const resCapa = (thisBlockSize < maxInSize) ? resMaxSize : LZ4_MAX_INPUT_SIZE;
blockTable[nbBlocks].srcPtr = srcPtr;
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
@@ -358,29 +399,37 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
- resPtr += thisBlockSize;
+ resPtr += resCapa;
remaining -= thisBlockSize;
} } }
/* warmimg up memory */
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+ /* decode-only mode : copy input to @compressedBuffer */
+ if (g_decodeOnly) {
+ U32 blockNb;
+ for (blockNb=0; blockNb < nbBlocks; blockNb++) {
+ memcpy(blockTable[blockNb].cPtr, blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize);
+ blockTable[blockNb].cSize = blockTable[blockNb].srcSize;
+ } }
+
/* Bench */
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
- UTIL_time_t coolTime;
+ UTIL_time_t coolTime = UTIL_getTime();
U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1; /* conservative initial compression speed estimate */
U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1; /* conservative initial decode speed estimate */
U64 totalCTime=0, totalDTime=0;
- U32 cCompleted=0, dCompleted=0;
+ U32 cCompleted=(g_decodeOnly==1), dCompleted=0;
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
U32 markNb = 0;
- size_t cSize = 0;
+ size_t cSize = srcSize;
+ size_t totalRSize = srcSize;
double ratio = 0.;
- coolTime = UTIL_getTime();
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted || !dCompleted) {
/* overheat protection */
@@ -391,8 +440,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
/* Compression */
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
- if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)totalRSize);
+ if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase compressed buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick();
@@ -408,7 +457,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
&compP,
blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr,
(int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom);
- if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed");
+ if (LZ4_isError(rSize)) END_PROCESS(1, "LZ4 compression failed");
blockTable[blockNb].cSize = rSize;
} }
{ U64 const clockSpan = UTIL_clockSpanNano(clockStart);
@@ -423,17 +472,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
totalCTime += clockSpan;
cCompleted = totalCTime>maxTime;
- } }
-
- cSize = 0;
- { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
- cSize += !cSize; /* avoid div by 0 */
- ratio = (double)srcSize / (double)cSize;
- markNb = (markNb+1) % NB_MARKS;
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- ((double)srcSize / fastestC) * 1000 );
-
+ }
+
+ cSize = 0;
+ { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
+ cSize += !cSize; /* avoid div by 0 */
+ ratio = (double)totalRSize / (double)cSize;
+ markNb = (markNb+1) % NB_MARKS;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+ marks[markNb], displayName,
+ (U32)totalRSize, (U32)cSize, ratio,
+ ((double)totalRSize / fastestC) * 1000 );
+ }
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#if 1
/* Decompression */
@@ -443,17 +493,30 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_waitForNextTick();
if (!dCompleted) {
+ const DecFunction_f decFunction = g_decodeOnly ?
+ LZ4F_decompress_binding : LZ4_decompress_safe_usingDict;
+ const char* const decString = g_decodeOnly ?
+ "LZ4F_decompress" : "LZ4_decompress_safe_usingDict";
UTIL_time_t const clockStart = UTIL_getTime();
U32 nbLoops;
+
for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- int const regenSize = LZ4_decompress_safe_usingDict(
+ size_t const inMaxSize = (size_t)INT_MAX / decMultiplier;
+ size_t const resCapa = (blockTable[blockNb].srcSize < inMaxSize) ?
+ blockTable[blockNb].srcSize * decMultiplier :
+ INT_MAX;
+ int const regenSize = decFunction(
blockTable[blockNb].cPtr, blockTable[blockNb].resPtr,
- (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize,
+ (int)blockTable[blockNb].cSize, (int)resCapa,
dictBuf, dictSize);
if (regenSize < 0) {
- DISPLAY("LZ4_decompress_safe_usingDict() failed on block %u \n", blockNb);
+ DISPLAY("%s() failed on block %u of size %u \n",
+ decString, blockNb, (unsigned)blockTable[blockNb].srcSize);
+ if (g_decodeOnly)
+ DISPLAY("Is input using LZ4 Frame format ? \n");
+ END_PROCESS(2, "error during decoding");
break;
}
blockTable[blockNb].resSize = (size_t)regenSize;
@@ -472,14 +535,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
dCompleted = totalDTime > (DECOMP_MULT*maxTime);
} }
+ if (g_decodeOnly) {
+ unsigned u;
+ totalRSize = 0;
+ for (u=0; u<nbBlocks; u++) totalRSize += blockTable[u].resSize;
+ }
markNb = (markNb+1) % NB_MARKS;
+ ratio = (double)totalRSize / (double)cSize;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- ((double)srcSize / fastestC) * 1000,
- ((double)srcSize / fastestD) * 1000);
-
- /* CRC Checking */
- { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+ marks[markNb], displayName,
+ (U32)totalRSize, (U32)cSize, ratio,
+ ((double)totalRSize / fastestC) * 1000,
+ ((double)totalRSize / fastestD) * 1000);
+
+ /* CRC Checking (not possible in decode-only mode)*/
+ if (!g_decodeOnly) {
+ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
@@ -594,21 +665,21 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
continue;
}
f = fopen(fileNamesTable[n], "rb");
- if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+ if (f==NULL) END_PROCESS(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */
fileSize = bufferSize-pos;
nbFiles=n;
}
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
- if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ if (readSize != (size_t)fileSize) END_PROCESS(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
}
- if (totalSize == 0) EXM_THROW(12, "no data to bench");
+ if (totalSize == 0) END_PROCESS(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
@@ -621,11 +692,11 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
- if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+ if (!fileSizes) END_PROCESS(12, "not enough memory for fileSizes");
/* Memory allocation & restrictions */
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
- if (benchedSize==0) EXM_THROW(12, "not enough memory");
+ if (benchedSize==0) END_PROCESS(12, "not enough memory");
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize > LZ4_MAX_INPUT_SIZE) {
benchedSize = LZ4_MAX_INPUT_SIZE;
@@ -635,7 +706,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
}
srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */
- if (!srcBuffer) EXM_THROW(12, "not enough memory");
+ if (!srcBuffer) END_PROCESS(12, "not enough memory");
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
@@ -663,7 +734,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
void* const srcBuffer = malloc(benchedSize);
/* Memory allocation */
- if (!srcBuffer) EXM_THROW(21, "not enough memory");
+ if (!srcBuffer) END_PROCESS(21, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
@@ -677,7 +748,8 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
}
-int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
+static int
+BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
int cLevel, int cLevelLast,
const char* dictBuf, int dictSize)
{
@@ -685,7 +757,6 @@ int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
for (fileNb=0; fileNb<nbFiles; fileNb++)
BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast, dictBuf, dictSize);
@@ -700,45 +771,54 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
{
double const compressibility = (double)g_compressibilityDefault / 100;
char* dictBuf = NULL;
- int dictSize = 0;
+ size_t dictSize = 0;
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
+ if (g_decodeOnly) {
+ DISPLAYLEVEL(2, "Benchmark Decompression (only) of LZ4 Frame \n");
+ cLevelLast = cLevel;
+ }
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+ if (cLevelLast > cLevel)
+ DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (dictFileName) {
FILE* dictFile = NULL;
- U64 dictFileSize = UTIL_getFileSize(dictFileName);
- if (!dictFileSize) EXM_THROW(25, "Dictionary error : could not stat dictionary file");
+ U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+ if (!dictFileSize)
+ END_PROCESS(25, "Dictionary error : could not stat dictionary file");
+ if (g_decodeOnly)
+ END_PROCESS(26, "Error : LZ4 Frame decoder mode not compatible with dictionary yet");
dictFile = fopen(dictFileName, "rb");
- if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+ if (!dictFile)
+ END_PROCESS(25, "Dictionary error : could not open dictionary file");
if (dictFileSize > LZ4_MAX_DICT_SIZE) {
dictSize = LZ4_MAX_DICT_SIZE;
- if (UTIL_fseek(dictFile, dictFileSize - dictSize, SEEK_SET))
- EXM_THROW(25, "Dictionary error : could not seek dictionary file");
+ if (UTIL_fseek(dictFile, (long)(dictFileSize - dictSize), SEEK_SET))
+ END_PROCESS(25, "Dictionary error : could not seek dictionary file");
} else {
- dictSize = (int)dictFileSize;
+ dictSize = (size_t)dictFileSize;
}
- dictBuf = (char *)malloc(dictSize);
- if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+ dictBuf = (char*)malloc(dictSize);
+ if (!dictBuf) END_PROCESS(25, "Allocation error : not enough memory");
- if (fread(dictBuf, 1, dictSize, dictFile) != (size_t)dictSize)
- EXM_THROW(25, "Dictionary error : could not read dictionary file");
+ if (fread(dictBuf, 1, dictSize, dictFile) != dictSize)
+ END_PROCESS(25, "Dictionary error : could not read dictionary file");
fclose(dictFile);
}
if (nbFiles == 0)
- BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, dictSize);
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, (int)dictSize);
else {
if (g_benchSeparately)
- BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
+ BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize);
else
- BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
+ BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize);
}
free(dictBuf);
diff --git a/programs/bench.h b/programs/bench.h
index 642d4966..9b0f667d 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -25,15 +25,29 @@
#include <stddef.h>
+/* BMK_benchFiles() :
+ * Benchmark all files provided through array @fileNamesTable.
+ * All files must be valid, otherwise benchmark fails.
+ * Roundtrip measurements are done for each file individually, but
+ * unless BMK_setBenchSeparately() is set, all results are agglomerated.
+ * The method benchmarks all compression levels from @cLevelStart to @cLevelLast,
+ * both inclusive, providing one result per compression level.
+ * If @cLevelLast <= @cLevelStart, BMK_benchFiles() benchmarks @cLevelStart only.
+ * @dictFileName is optional, it's possible to provide NULL.
+ * When provided, compression and decompression use the specified file as dictionary.
+ * Only one dictionary can be provided, in which case it's applied to all benchmarked files.
+**/
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
- int cLevel, int cLevelLast,
+ int cLevelStart, int cLevelLast,
const char* dictFileName);
/* Set Parameters */
-void BMK_setNbSeconds(unsigned nbLoops);
-void BMK_setBlockSize(size_t blockSize);
-void BMK_setAdditionalParam(int additionalParam);
-void BMK_setNotificationLevel(unsigned level);
-void BMK_setBenchSeparately(int separate);
+void BMK_setNbSeconds(unsigned nbSeconds); /* minimum benchmark duration, in seconds, for both compression and decompression */
+void BMK_setBlockSize(size_t blockSize); /* Internally cut input file(s) into independent blocks of specified size */
+void BMK_setNotificationLevel(unsigned level); /* Influence verbosity level */
+void BMK_setBenchSeparately(int separate); /* When providing multiple files, output one result per file */
+void BMK_setDecodeOnlyMode(int set); /* v1.9.4+: set benchmark mode to decode only */
+
+void BMK_setAdditionalParam(int additionalParam); /* hidden param, influence output format, for python parsing */
#endif /* BENCH_H_125623623633 */
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 254a6ce5..42132b9f 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -377,8 +377,12 @@ int main(int argc, const char** argv)
if (argument[1]=='-') {
if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; }
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
- if ((!strcmp(argument, "--decompress"))
- || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
+ if ( (!strcmp(argument, "--decompress"))
+ || (!strcmp(argument, "--uncompress"))) {
+ if (mode != om_bench) mode = om_decompress;
+ BMK_setDecodeOnlyMode(1);
+ continue;
+ }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; }
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; }
@@ -478,7 +482,10 @@ int main(int argc, const char** argv)
case 'l': legacy_format = 1; blockSize = 8 MB; break;
/* Decoding */
- case 'd': mode = om_decompress; break;
+ case 'd':
+ if (mode != om_bench) mode = om_decompress;
+ BMK_setDecodeOnlyMode(1);
+ break;
/* Force stdout, even if stdout==console */
case 'c':
diff --git a/tests/Makefile b/tests/Makefile
index 469db9ab..d804f251 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -476,6 +476,9 @@ test-lz4-testmode: FPREFIX = tmp-ltm
test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----"
$(LZ4) -bi0
+ $(DATAGEN) > $(FPREFIX)
+ $(LZ4) -f $(FPREFIX) -c > $(FPREFIX).lz4
+ $(LZ4) -bdi0 $(FPREFIX).lz4 # test benchmark decode-only mode
@echo "\n ---- test mode ----"
! $(DATAGEN) | $(LZ4) -t
! $(DATAGEN) | $(LZ4) -tf
diff --git a/tests/fullbench.c b/tests/fullbench.c
index ec20dcb3..9c139961 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -397,7 +397,8 @@ static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcS
size_t outRemaining = maxOutSize - outPos;
for (;;) {
- size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
+ size_t const sizeHint =
+ LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
assert(!LZ4F_isError(sizeHint));
inPos += inSize;