aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/dev-short-tests.yml5
-rw-r--r--build/cmake/tests/CMakeLists.txt6
-rw-r--r--lib/common/cpu.h36
-rw-r--r--lib/compress/zstd_lazy.c18
-rw-r--r--lib/decompress/zstd_decompress.c9
-rw-r--r--programs/fileio.c5
-rw-r--r--tests/.gitignore3
-rw-r--r--tests/Makefile1
-rw-r--r--tests/fuzz/simple_decompress.c8
-rw-r--r--tests/golden-decompression-errors/.gitignore1
-rw-r--r--tests/golden-decompression-errors/zeroSeq_extraneous.zstbin0 -> 27 bytes
-rw-r--r--tests/zstreamtest.c9
12 files changed, 84 insertions, 17 deletions
diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 9c36386a..b2aaff89 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -272,14 +272,15 @@ jobs:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # tag=v2.0.0
- - name: Build
+ - name: Build & Test
working-directory: ${{env.GITHUB_WORKSPACE}}
run: |
cd build\cmake
mkdir build
cd build
- cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} ..
+ cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZSTREAM_FLAGS=-T30s -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_FULLBENCH_FLAGS=-i0 ..
cmake.exe --build .
+ ctest.exe -V -C Debug
msbuild-visual-studio:
strategy:
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 3ead0701..56104a4e 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -61,7 +61,7 @@ if (NOT MSVC)
target_compile_options(fullbench PRIVATE "-Wno-deprecated-declarations")
endif()
target_link_libraries(fullbench libzstd_static)
-add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})
+add_test(NAME fullbench COMMAND "$<TARGET_FILE:fullbench>" ${ZSTD_FULLBENCH_FLAGS})
#
# fuzzer
@@ -73,7 +73,7 @@ endif()
target_link_libraries(fuzzer libzstd_static)
AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
"Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
-add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
+add_test(NAME fuzzer COMMAND "$<TARGET_FILE:fuzzer>" ${ZSTD_FUZZER_FLAGS})
# Disable the timeout since the run time is too long for the default timeout of
# 1500 seconds and varies considerably between low-end and high-end CPUs.
# set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
@@ -88,7 +88,7 @@ endif()
target_link_libraries(zstreamtest libzstd_static)
AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
"Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
-add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
+add_test(NAME zstreamtest COMMAND "$<TARGET_FILE:zstreamtest>" ${ZSTD_ZSTREAM_FLAGS})
#
# playTests.sh
diff --git a/lib/common/cpu.h b/lib/common/cpu.h
index 8bc34a36..0e684d9a 100644
--- a/lib/common/cpu.h
+++ b/lib/common/cpu.h
@@ -35,6 +35,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
U32 f7b = 0;
U32 f7c = 0;
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+#if !defined(__clang__)
int reg[4];
__cpuid((int*)reg, 0);
{
@@ -50,6 +51,41 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
f7c = (U32)reg[2];
}
}
+#else
+ /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in
+ * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs
+ * to due to being a reserved register. So in that case, do the `cpuid`
+ * ourselves. Clang supports inline assembly anyway.
+ */
+ U32 n;
+ __asm__(
+ "pushq %%rbx\n\t"
+ "cpuid\n\t"
+ "popq %%rbx\n\t"
+ : "=a"(n)
+ : "a"(0)
+ : "rcx", "rdx");
+ if (n >= 1) {
+ U32 f1a;
+ __asm__(
+ "pushq %%rbx\n\t"
+ "cpuid\n\t"
+ "popq %%rbx\n\t"
+ : "=a"(f1a), "=c"(f1c), "=d"(f1d)
+ : "a"(1)
+ :);
+ }
+ if (n >= 7) {
+ __asm__(
+ "pushq %%rbx\n\t"
+ "cpuid\n\t"
+ "movq %%rbx, %%rax\n\t"
+ "popq %%rbx"
+ : "=a"(f7b), "=c"(f7c)
+ : "a"(7), "c"(0)
+ : "rdx");
+ }
+#endif
#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
/* The following block like the normal cpuid branch below, but gcc
* reserves ebx for use of its pic register so we must specially
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 3aba83c6..67dd55fd 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1123,18 +1123,18 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
/* The high-level approach of the SIMD row based match finder is as follows:
* - Figure out where to insert the new entry:
- * - Generate a hash from a byte along with an additional 1-byte "short hash". The additional byte is our "tag"
- * - The hashTable is effectively split into groups or "rows" of 16 or 32 entries of U32, and the hash determines
+ * - Generate a hash for current input posistion and split it into a one byte of tag and `rowHashLog` bits of index.
+ * - The hash is salted by a value that changes on every contex reset, so when the same table is used
+ * we will avoid collisions that would otherwise slow us down by intorducing phantom matches.
+ * - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines
* which row to insert into.
- * - Determine the correct position within the row to insert the entry into. Each row of 16 or 32 can
- * be considered as a circular buffer with a "head" index that resides in the tagTable.
- * - Also insert the "tag" into the equivalent row and position in the tagTable.
- * - Note: The tagTable has 17 or 33 1-byte entries per row, due to 16 or 32 tags, and 1 "head" entry.
- * The 17 or 33 entry rows are spaced out to occur every 32 or 64 bytes, respectively,
- * for alignment/performance reasons, leaving some bytes unused.
- * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte "short hash" and
+ * - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can
+ * be considered as a circular buffer with a "head" index that resides in the tagTable (overall 16 or 32 bytes
+ * per row).
+ * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte tag calculated for the position and
* generate a bitfield that we can cycle through to check the collisions in the hash table.
* - Pick the longest match.
+ * - Insert the tag into the equivalent row and position in the tagTable.
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 17305908..42636d57 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1093,6 +1093,15 @@ size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
+ {
+ unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize);
+ RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!");
+ if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+ RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected,
+ "Frame header size does not match decoded size!");
+ }
+ }
+
assert(decodedSize <= dstCapacity);
dst = (BYTE*)dst + decodedSize;
dstCapacity -= decodedSize;
diff --git a/programs/fileio.c b/programs/fileio.c
index 6fd55d9a..e3012a71 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -2441,9 +2441,10 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
U64 frameSize = 0;
IOJob_t *writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
- /* display last 20 characters only */
+ /* display last 20 characters only when not --verbose */
{ size_t const srcFileLength = strlen(srcFileName);
- if (srcFileLength>20) srcFileName += srcFileLength-20;
+ if ((srcFileLength>20) && (g_display_prefs.displayLevel<3))
+ srcFileName += srcFileLength-20;
}
ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only);
diff --git a/tests/.gitignore b/tests/.gitignore
index fcb865d6..311a8b5e 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -67,3 +67,6 @@ speedTest.pid
*.exe
*.out
*.app
+
+# Specific exclusions
+!golden-decompression/*.zst
diff --git a/tests/Makefile b/tests/Makefile
index 3550b7a9..ed3692a2 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -21,6 +21,7 @@
# ##########################################################################
ZSTD_LEGACY_SUPPORT ?= 5
+export ZSTD_LEGACY_SUPPORT
DEBUGLEVEL ?= 2
export DEBUGLEVEL # transmit value to sub-makefiles
diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c
index ce5f9f09..0ee61902 100644
--- a/tests/fuzz/simple_decompress.c
+++ b/tests/fuzz/simple_decompress.c
@@ -37,7 +37,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size);
void *rBuf = FUZZ_malloc(bufSize);
- ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size);
+ size_t const dSize = ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size);
+ if (!ZSTD_isError(dSize)) {
+ /* If decompression was successful, the content size from the frame header(s) should be valid. */
+ size_t const expectedSize = ZSTD_findDecompressedSize(src, size);
+ FUZZ_ASSERT(expectedSize != ZSTD_CONTENTSIZE_ERROR);
+ FUZZ_ASSERT(expectedSize == ZSTD_CONTENTSIZE_UNKNOWN || expectedSize == dSize);
+ }
free(rBuf);
FUZZ_dataProducer_free(producer);
diff --git a/tests/golden-decompression-errors/.gitignore b/tests/golden-decompression-errors/.gitignore
new file mode 100644
index 00000000..574b3750
--- /dev/null
+++ b/tests/golden-decompression-errors/.gitignore
@@ -0,0 +1 @@
+!*.zst
diff --git a/tests/golden-decompression-errors/zeroSeq_extraneous.zst b/tests/golden-decompression-errors/zeroSeq_extraneous.zst
new file mode 100644
index 00000000..0953be34
--- /dev/null
+++ b/tests/golden-decompression-errors/zeroSeq_extraneous.zst
Binary files differ
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 82aaf3db..7cc4068b 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2408,6 +2408,15 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
}
DISPLAYLEVEL(3, "OK \n");
+ DISPLAYLEVEL(3, "test%3i : Decoder should reject invalid frame header on legacy frames: ", testNb++);
+ {
+ const unsigned char compressed[] = { 0x26,0xb5,0x2f,0xfd,0x50,0x91,0xfd,0xd8,0xb5 };
+ const size_t compressedSize = 9;
+ size_t const dSize = ZSTD_decompress(NULL, 0, compressed, compressedSize);
+ CHECK(!ZSTD_isError(dSize), "must reject when legacy frame header is invalid");
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
_end:
FUZ_freeDictionary(dictionary);
ZSTD_freeCStream(zc);
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-21 01:50:40 +0000