diff options
| author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 04:44:13 +0000 |
|---|---|---|
| committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 04:44:13 +0000 |
| commit | b3e19fb66dc3149597f4b130a92b86c155cce30a (patch) | |
| tree | fb5f91a0b6ecc9dc0229fad0f71e4c8fdf97d5a9 | |
| parent | 4e031b3a08a06e36fa74d66192dac5250ad69131 (diff) | |
| parent | 246d740ca4f773cbb896c83aa38ef2012e3b6fbf (diff) | |
| download | libfuzzer-sys-android14-mainline-appsearch-release.tar.gz | |
Snap for 10453563 from 246d740ca4f773cbb896c83aa38ef2012e3b6fbf to mainline-appsearch-releaseaml_ase_341510000aml_ase_341410000aml_ase_341310010aml_ase_341113000aml_ase_340913000android14-mainline-appsearch-release
Change-Id: I7b177dbc83e4a54110ff6d1a6dce248a559b70f3
47 files changed, 1208 insertions, 494 deletions
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json index 4d6cfd1..f360df2 100644 --- a/.cargo_vcs_info.json +++ b/.cargo_vcs_info.json @@ -1,5 +1,6 @@ { "git": { - "sha1": "a89115ac1105fa0c7c7d9cb5cdb479af36031aff" - } -} + "sha1": "e07c487220402f5bec2151de2856a7caff0639c6" + }, + "path_in_vcs": "" +}
\ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 47e2bf2..5888af1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,5 +20,7 @@ jobs: rustup component add rustfmt --toolchain stable cargo +stable fmt --all -- --check + - run: cargo install cargo-fuzz + - name: Run tests run: ./ci/script.sh @@ -1,2 +1,3 @@ target Cargo.lock +corpus @@ -39,7 +39,7 @@ rust_library_rlib { host_supported: true, crate_name: "libfuzzer_sys", cargo_env_compat: true, - cargo_pkg_version: "0.4.2", + cargo_pkg_version: "0.4.6", srcs: ["src/lib.rs"], edition: "2018", features: ["arbitrary-derive"], @@ -47,4 +47,10 @@ rust_library_rlib { "libarbitrary", "libonce_cell", ], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + product_available: true, + vendor_available: true, } diff --git a/CHANGELOG.md b/CHANGELOG.md index 77be1fd..0fba42c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,70 @@ Released YYYY-MM-DD. -------------------------------------------------------------------------------- +## 0.4.6 + +Released 2023-01-26. + +### Fixed + +* Fixed a potential name conflict in functions generated by the `fuzz_target!` + macro. +* Fixed potential stale builds when updating custom libfuzzers to link against. + +-------------------------------------------------------------------------------- + +## 0.4.5 + +Released 2022-10-18. + +### Added + +* Added the ability to tell libfuzzer whether to keep files inputs in the corpus + or not. See the `Corpus` type and extended documentation for the + `fuzz_target!` macro for details. + +### Changed + +* Ensured that there is always at least one inline-never frame on the stack per + fuzz target. This helps prevent oss-fuzz from "deduplicating" different bugs + from different fuzz targets into the same bug. + +-------------------------------------------------------------------------------- + +## 0.4.4 + +Released 2022-09-01. + +### Changed + +* Updated to `libFuzzer` commit `df90d22` (`release/15.x`). +* LLVM 16's [upcoming change][llvm_cxx17] to build requirements to C++17 + necessitate reflecting those changes. (`libFuzzer` updates contain C++14 code) +* Drastically reduce build times by using parallel C++ compilation jobs +* Updated `rand` dependency from 0.8.3 to 0.8.5 +* Updated `flate2` dependency from 1.0.20 to 1.0.24 + +[llvm_cxx17]: https://llvm.org/docs/ReleaseNotes.html#update-on-required-toolchains-to-build-llvm + +-------------------------------------------------------------------------------- + +## 0.4.3 + +Released 2020-03-03. + +### Changed + +* Updated to `libFuzzer` commit `60e32a1`. + +### Fixed + +* Fixed an issue where the `fuzz_target!` macro would sometimes expand to + versions of itself that were not `$crate` prefixed and would result in "error: + cannot find macro `fuzz_target` in this scope" if the caller didn't import the + macro but used the qualified version of it instead. + +-------------------------------------------------------------------------------- + ## 0.4.2 Released 2020-05-26. @@ -3,34 +3,37 @@ # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies +# to registry (e.g., crates.io) dependencies. # -# If you believe there's an error in this file please file an -# issue against the rust-lang/cargo repository. If you're -# editing this file be aware that the upstream Cargo.toml -# will likely look very different (and much more reasonable) +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "libfuzzer-sys" -version = "0.4.2" +version = "0.4.6" authors = ["The rust-fuzz Project Developers"] description = "A wrapper around LLVM's libFuzzer runtime." readme = "./README.md" license = "MIT/Apache-2.0/NCSA" repository = "https://github.com/rust-fuzz/libfuzzer" + [dependencies.arbitrary] version = "1" [dependencies.once_cell] version = "1" + [dev-dependencies.flate2] -version = "1.0.20" +version = "1.0.24" [dev-dependencies.rand] -version = "0.8.3" +version = "0.8.5" + [build-dependencies.cc] version = "1.0" +features = ["parallel"] [features] arbitrary-derive = ["arbitrary/derive"] diff --git a/Cargo.toml.orig b/Cargo.toml.orig index 72285b9..1552302 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -6,25 +6,25 @@ license = "MIT/Apache-2.0/NCSA" name = "libfuzzer-sys" readme = "./README.md" repository = "https://github.com/rust-fuzz/libfuzzer" -version = "0.4.2" +version = "0.4.6" [dependencies] arbitrary = "1" once_cell = "1" [build-dependencies] -cc = "1.0" +cc = { version = "1.0", features = ["parallel"] } [features] arbitrary-derive = ["arbitrary/derive"] [workspace] members = [ - "./example", - "./example_arbitrary", - "./example_mutator", + "./example/fuzz", + "./example_arbitrary/fuzz", + "./example_mutator/fuzz", ] [dev-dependencies] -flate2 = "1.0.20" -rand = "0.8.3" +flate2 = "1.0.24" +rand = "0.8.5" @@ -1,3 +1,7 @@ +# This project was upgraded with external_updater. +# Usage: tools/external_updater/updater.sh update rust/crates/libfuzzer-sys +# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md + name: "libfuzzer-sys" description: "A wrapper around LLVM\'s libFuzzer runtime." third_party { @@ -7,13 +11,13 @@ third_party { } url { type: ARCHIVE - value: "https://static.crates.io/crates/libfuzzer-sys/libfuzzer-sys-0.4.2.crate" + value: "https://static.crates.io/crates/libfuzzer-sys/libfuzzer-sys-0.4.6.crate" } - version: "0.4.2" + version: "0.4.6" license_type: NOTICE last_upgrade_date { - year: 2021 - month: 6 - day: 21 + year: 2023 + month: 2 + day: 16 } } @@ -1,10 +1,15 @@ fn main() { + println!("cargo:rerun-if-env-changed=CUSTOM_LIBFUZZER_PATH"); if let Ok(custom) = ::std::env::var("CUSTOM_LIBFUZZER_PATH") { + println!("cargo:rerun-if-changed={custom}"); + let custom_lib_path = ::std::path::PathBuf::from(&custom); let custom_lib_dir = custom_lib_path.parent().unwrap().to_string_lossy(); let custom_lib_name = custom_lib_path.file_stem().unwrap().to_string_lossy(); - let custom_lib_name = custom_lib_name.trim_start_matches("lib"); + let custom_lib_name = custom_lib_name + .strip_prefix("lib") + .unwrap_or(custom_lib_name.as_ref()); println!("cargo:rustc-link-search=native={}", custom_lib_dir); println!("cargo:rustc-link-lib=static={}", custom_lib_name); @@ -26,7 +31,7 @@ fn main() { println!("cargo:rerun-if-changed={}", source.display()); build.file(source.to_str().unwrap()); } - build.flag("-std=c++11"); + build.flag("-std=c++17"); build.flag("-fno-omit-frame-pointer"); build.flag("-w"); build.cpp(true); diff --git a/ci/script.sh b/ci/script.sh index 91fad80..59462d4 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -8,53 +8,26 @@ export CARGO_TARGET_DIR=$(pwd)/target cargo test --doc pushd ./example -cargo rustc \ - --release \ - -- \ - -Cpasses='sancov' \ - -Cllvm-args=-sanitizer-coverage-level=3 \ - -Cllvm-args=-sanitizer-coverage-trace-compares \ - -Cllvm-args=-sanitizer-coverage-inline-8bit-counters \ - -Cllvm-args=-sanitizer-coverage-stack-depth \ - -Cllvm-args=-sanitizer-coverage-trace-geps \ - -Cllvm-args=-sanitizer-coverage-prune-blocks=0 \ - -Zsanitizer=address -(! $CARGO_TARGET_DIR/release/example -runs=100000) +cargo fuzz build +cargo fuzz build --dev +(! cargo fuzz run bananas -- -runs=100000) popd pushd ./example_arbitrary -cargo rustc \ - --release \ - -- \ - -Cpasses='sancov' \ - -Cllvm-args=-sanitizer-coverage-level=3 \ - -Cllvm-args=-sanitizer-coverage-trace-compares \ - -Cllvm-args=-sanitizer-coverage-inline-8bit-counters \ - -Cllvm-args=-sanitizer-coverage-stack-depth \ - -Cllvm-args=-sanitizer-coverage-trace-geps \ - -Cllvm-args=-sanitizer-coverage-prune-blocks=0 \ - -Zsanitizer=address -(! $CARGO_TARGET_DIR/release/example_arbitrary -runs=10000000) +cargo fuzz build +cargo fuzz build --dev +(! cargo fuzz run rgb -- -runs=10000000) RUST_LIBFUZZER_DEBUG_PATH=$(pwd)/debug_output \ - $CARGO_TARGET_DIR/release/example_arbitrary \ - $(ls ./crash-* | head -n 1) + cargo fuzz run rgb \ + $(ls ./fuzz/artifacts/rgb/crash-* | head -n 1) cat $(pwd)/debug_output grep -q Rgb $(pwd)/debug_output popd pushd ./example_mutator -cargo rustc \ - --release \ - -- \ - -Cpasses='sancov' \ - -Cllvm-args=-sanitizer-coverage-level=3 \ - -Cllvm-args=-sanitizer-coverage-trace-compares \ - -Cllvm-args=-sanitizer-coverage-inline-8bit-counters \ - -Cllvm-args=-sanitizer-coverage-stack-depth \ - -Cllvm-args=-sanitizer-coverage-trace-geps \ - -Cllvm-args=-sanitizer-coverage-prune-blocks=0 \ - -Zsanitizer=address -(! $CARGO_TARGET_DIR/release/example_mutator -runs=10000000) +cargo fuzz build +cargo fuzz build --dev +(! cargo fuzz run boom -- -runs=10000000) popd echo "All good!" diff --git a/libfuzzer/CMakeLists.txt b/libfuzzer/CMakeLists.txt index 3201ed2..a9a10f7 100644 --- a/libfuzzer/CMakeLists.txt +++ b/libfuzzer/CMakeLists.txt @@ -6,6 +6,8 @@ set(LIBFUZZER_SOURCES FuzzerExtFunctionsWeak.cpp FuzzerExtFunctionsWindows.cpp FuzzerExtraCounters.cpp + FuzzerExtraCountersDarwin.cpp + FuzzerExtraCountersWindows.cpp FuzzerFork.cpp FuzzerIO.cpp FuzzerIOPosix.cpp @@ -64,18 +66,19 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ LIBFUZZER_CFLAGS) elseif(TARGET cxx-headers OR HAVE_LIBCXX) # libFuzzer uses C++ standard library headers. + list(APPEND LIBFUZZER_CFLAGS ${COMPILER_RT_CXX_CFLAGS}) set(LIBFUZZER_DEPS cxx-headers) endif() append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS) if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage") - list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters) + list(APPEND LIBFUZZER_CFLAGS -fsanitize-coverage=0) endif() if(MSVC) # Silence warnings by turning off exceptions in MSVC headers and avoid an - # error by unecessarily defining thread_local when it isn't even used on + # error by unnecessarily defining thread_local when it isn't even used on # Windows. list(APPEND LIBFUZZER_CFLAGS -D_HAS_EXCEPTIONS=0) else() @@ -136,15 +139,15 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH AND COMPILER_RT_LIBCXXABI_PATH) macro(partially_link_libcxx name dir arch) - if(${arch} MATCHES "i386") - set(EMULATION_ARGUMENT "-m" "elf_i386") - else() - set(EMULATION_ARGUMENT "") + get_target_flags_for_arch(${arch} target_cflags) + if(CMAKE_CXX_COMPILER_ID MATCHES Clang) + get_compiler_rt_target(${arch} target) + set(target_cflags --target=${target} ${target_cflags}) endif() set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir") file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir}) add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD - COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o + COMMAND ${CMAKE_CXX_COMPILER} ${target_cflags} -Wl,--whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" -Wl,--no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o @@ -160,7 +163,8 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DLIBCXXABI_ENABLE_EXCEPTIONS=OFF - -DLIBCXX_ABI_NAMESPACE=__Fuzzer) + -DLIBCXX_ABI_NAMESPACE=__Fuzzer + -DLIBCXX_ENABLE_EXCEPTIONS=OFF) target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build) target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) diff --git a/libfuzzer/FuzzerBuiltinsMsvc.h b/libfuzzer/FuzzerBuiltinsMsvc.h index ab191b6..421dee7 100644 --- a/libfuzzer/FuzzerBuiltinsMsvc.h +++ b/libfuzzer/FuzzerBuiltinsMsvc.h @@ -41,7 +41,8 @@ inline uint32_t Clzll(uint64_t X) { #if !defined(_M_ARM) && !defined(_M_X64) // Scan the high 32 bits. if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32))) - return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. + return static_cast<int>( + 63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. // Scan the low 32 bits. if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X))) return static_cast<int>(63 - LeadZeroIdx); diff --git a/libfuzzer/FuzzerCommand.h b/libfuzzer/FuzzerCommand.h index 8730886..f653fe3 100644 --- a/libfuzzer/FuzzerCommand.h +++ b/libfuzzer/FuzzerCommand.h @@ -33,7 +33,7 @@ public: Command() : CombinedOutAndErr(false) {} - explicit Command(const Vector<std::string> &ArgsToAdd) + explicit Command(const std::vector<std::string> &ArgsToAdd) : Args(ArgsToAdd), CombinedOutAndErr(false) {} explicit Command(const Command &Other) @@ -58,7 +58,7 @@ public: // Gets all of the current command line arguments, **including** those after // "-ignore-remaining-args=1". - const Vector<std::string> &getArguments() const { return Args; } + const std::vector<std::string> &getArguments() const { return Args; } // Adds the given argument before "-ignore_remaining_args=1", or at the end // if that flag isn't present. @@ -68,7 +68,7 @@ public: // Adds all given arguments before "-ignore_remaining_args=1", or at the end // if that flag isn't present. - void addArguments(const Vector<std::string> &ArgsToAdd) { + void addArguments(const std::vector<std::string> &ArgsToAdd) { Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end()); } @@ -155,16 +155,16 @@ private: Command(Command &&Other) = delete; Command &operator=(Command &&Other) = delete; - Vector<std::string>::iterator endMutableArgs() { + std::vector<std::string>::iterator endMutableArgs() { return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); } - Vector<std::string>::const_iterator endMutableArgs() const { + std::vector<std::string>::const_iterator endMutableArgs() const { return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); } // The command arguments. Args[0] is the command name. - Vector<std::string> Args; + std::vector<std::string> Args; // True indicates stderr is redirected to stdout. bool CombinedOutAndErr; diff --git a/libfuzzer/FuzzerCorpus.h b/libfuzzer/FuzzerCorpus.h index f8c1260..e01891e 100644 --- a/libfuzzer/FuzzerCorpus.h +++ b/libfuzzer/FuzzerCorpus.h @@ -39,13 +39,13 @@ struct InputInfo { bool MayDeleteFile = false; bool Reduced = false; bool HasFocusFunction = false; - Vector<uint32_t> UniqFeatureSet; - Vector<uint8_t> DataFlowTraceForFocusFunction; + std::vector<uint32_t> UniqFeatureSet; + std::vector<uint8_t> DataFlowTraceForFocusFunction; // Power schedule. bool NeedsEnergyUpdate = false; double Energy = 0.0; double SumIncidence = 0.0; - Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; + std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; // Delete feature Idx and its frequency from FeatureFreqs. bool DeleteFeatureFreq(uint32_t Idx) { @@ -209,7 +209,7 @@ public: InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, bool HasFocusFunction, bool NeverReduce, std::chrono::microseconds TimeOfUnit, - const Vector<uint32_t> &FeatureSet, + const std::vector<uint32_t> &FeatureSet, const DataFlowTrace &DFT, const InputInfo *BaseII) { assert(!U.empty()); if (FeatureDebug) @@ -258,7 +258,7 @@ public: } // Debug-only - void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) { + void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) { if (!FeatureDebug) return; Printf("{"); for (uint32_t Feature: FeatureSet) @@ -284,7 +284,8 @@ public: } } - void Replace(InputInfo *II, const Unit &U) { + void Replace(InputInfo *II, const Unit &U, + std::chrono::microseconds TimeOfUnit) { assert(II->U.size() > U.size()); Hashes.erase(Sha1ToString(II->Sha1)); DeleteFile(*II); @@ -292,6 +293,7 @@ public: Hashes.insert(Sha1ToString(II->Sha1)); II->U = U; II->Reduced = true; + II->TimeOfUnit = TimeOfUnit; DistributionNeedsUpdate = true; } @@ -325,7 +327,8 @@ public: const auto &II = *Inputs[i]; Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i, Sha1ToString(II.Sha1).c_str(), II.U.size(), - II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction); + II.NumExecutedMutations, II.NumSuccessfullMutations, + II.HasFocusFunction); } } @@ -563,11 +566,11 @@ private: } std::piecewise_constant_distribution<double> CorpusDistribution; - Vector<double> Intervals; - Vector<double> Weights; + std::vector<double> Intervals; + std::vector<double> Weights; std::unordered_set<std::string> Hashes; - Vector<InputInfo*> Inputs; + std::vector<InputInfo *> Inputs; size_t NumAddedFeatures = 0; size_t NumUpdatedFeatures = 0; @@ -577,7 +580,7 @@ private: bool DistributionNeedsUpdate = true; uint16_t FreqOfMostAbundantRareFeature = 0; uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {}; - Vector<uint32_t> RareFeatures; + std::vector<uint32_t> RareFeatures; std::string OutputCorpus; }; diff --git a/libfuzzer/FuzzerDataFlowTrace.cpp b/libfuzzer/FuzzerDataFlowTrace.cpp index 23d4225..2f9a4d2 100644 --- a/libfuzzer/FuzzerDataFlowTrace.cpp +++ b/libfuzzer/FuzzerDataFlowTrace.cpp @@ -37,7 +37,7 @@ bool BlockCoverage::AppendCoverage(const std::string &S) { // Coverage lines have this form: // CN X Y Z T // where N is the number of the function, T is the total number of instrumented -// BBs, and X,Y,Z, if present, are the indecies of covered BB. +// BBs, and X,Y,Z, if present, are the indices of covered BB. // BB #0, which is the entry block, is not explicitly listed. bool BlockCoverage::AppendCoverage(std::istream &IN) { std::string L; @@ -52,7 +52,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { continue; } if (L[0] != 'C') continue; - Vector<uint32_t> CoveredBlocks; + std::vector<uint32_t> CoveredBlocks; while (true) { uint32_t BB = 0; SS >> BB; @@ -68,7 +68,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { auto It = Functions.find(FunctionId); auto &Counters = It == Functions.end() - ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)}) + ? Functions.insert({FunctionId, std::vector<uint32_t>(NumBlocks)}) .first->second : It->second; @@ -86,8 +86,8 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { // * any uncovered function gets weight 0. // * a function with lots of uncovered blocks gets bigger weight. // * a function with a less frequently executed code gets bigger weight. -Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { - Vector<double> Res(NumFunctions); +std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { + std::vector<double> Res(NumFunctions); for (auto It : Functions) { auto FunctionID = It.first; auto Counters = It.second; @@ -104,7 +104,7 @@ Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { } void DataFlowTrace::ReadCoverage(const std::string &DirPath) { - Vector<SizedFile> Files; + std::vector<SizedFile> Files; GetSizedFilesFromDir(DirPath, &Files); for (auto &SF : Files) { auto Name = Basename(SF.File); @@ -115,16 +115,16 @@ void DataFlowTrace::ReadCoverage(const std::string &DirPath) { } } -static void DFTStringAppendToVector(Vector<uint8_t> *DFT, +static void DFTStringAppendToVector(std::vector<uint8_t> *DFT, const std::string &DFTString) { assert(DFT->size() == DFTString.size()); for (size_t I = 0, Len = DFT->size(); I < Len; I++) (*DFT)[I] = DFTString[I] == '1'; } -// converts a string of '0' and '1' into a Vector<uint8_t> -static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) { - Vector<uint8_t> DFT(DFTString.size()); +// converts a string of '0' and '1' into a std::vector<uint8_t> +static std::vector<uint8_t> DFTStringToVector(const std::string &DFTString) { + std::vector<uint8_t> DFT(DFTString.size()); DFTStringAppendToVector(&DFT, DFTString); return DFT; } @@ -159,14 +159,14 @@ static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, } bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, - Vector<SizedFile> &CorporaFiles, Random &Rand) { + std::vector<SizedFile> &CorporaFiles, Random &Rand) { if (DirPath.empty()) return false; Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); - Vector<SizedFile> Files; + std::vector<SizedFile> Files; GetSizedFilesFromDir(DirPath, &Files); std::string L; size_t FocusFuncIdx = SIZE_MAX; - Vector<std::string> FunctionNames; + std::vector<std::string> FunctionNames; // Collect the hashes of the corpus files. for (auto &SF : CorporaFiles) @@ -191,7 +191,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, // * chooses a random function according to the weights. ReadCoverage(DirPath); auto Weights = Coverage.FunctionWeights(NumFunctions); - Vector<double> Intervals(NumFunctions + 1); + std::vector<double> Intervals(NumFunctions + 1); std::iota(Intervals.begin(), Intervals.end(), 0); auto Distribution = std::piecewise_constant_distribution<double>( Intervals.begin(), Intervals.end(), Weights.begin()); @@ -247,7 +247,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, } int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, - const Vector<SizedFile> &CorporaFiles) { + const std::vector<SizedFile> &CorporaFiles) { Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); if (CorporaFiles.empty()) { @@ -265,7 +265,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, // we then request tags in [0,Size/2) and [Size/2, Size), and so on. // Function number => DFT. auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File))); - std::unordered_map<size_t, Vector<uint8_t>> DFTMap; + std::unordered_map<size_t, std::vector<uint8_t>> DFTMap; std::unordered_set<std::string> Cov; Command Cmd; Cmd.addArgument(DFTBinary); diff --git a/libfuzzer/FuzzerDataFlowTrace.h b/libfuzzer/FuzzerDataFlowTrace.h index 07c03bb..054dce1 100644 --- a/libfuzzer/FuzzerDataFlowTrace.h +++ b/libfuzzer/FuzzerDataFlowTrace.h @@ -39,7 +39,7 @@ namespace fuzzer { int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, - const Vector<SizedFile> &CorporaFiles); + const std::vector<SizedFile> &CorporaFiles); class BlockCoverage { public: @@ -77,11 +77,11 @@ public: return Result; } - Vector<double> FunctionWeights(size_t NumFunctions) const; + std::vector<double> FunctionWeights(size_t NumFunctions) const; void clear() { Functions.clear(); } private: - typedef Vector<uint32_t> CoverageVector; + typedef std::vector<uint32_t> CoverageVector; uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { uint32_t Res = 0; @@ -117,9 +117,9 @@ class DataFlowTrace { public: void ReadCoverage(const std::string &DirPath); bool Init(const std::string &DirPath, std::string *FocusFunction, - Vector<SizedFile> &CorporaFiles, Random &Rand); + std::vector<SizedFile> &CorporaFiles, Random &Rand); void Clear() { Traces.clear(); } - const Vector<uint8_t> *Get(const std::string &InputSha1) const { + const std::vector<uint8_t> *Get(const std::string &InputSha1) const { auto It = Traces.find(InputSha1); if (It != Traces.end()) return &It->second; @@ -128,9 +128,9 @@ class DataFlowTrace { private: // Input's sha1 => DFT for the FocusFunction. - std::unordered_map<std::string, Vector<uint8_t> > Traces; - BlockCoverage Coverage; - std::unordered_set<std::string> CorporaHashes; + std::unordered_map<std::string, std::vector<uint8_t>> Traces; + BlockCoverage Coverage; + std::unordered_set<std::string> CorporaHashes; }; } // namespace fuzzer diff --git a/libfuzzer/FuzzerDefs.h b/libfuzzer/FuzzerDefs.h index 1a2752a..db1f74a 100644 --- a/libfuzzer/FuzzerDefs.h +++ b/libfuzzer/FuzzerDefs.h @@ -38,28 +38,8 @@ struct ExternalFunctions; // Global interface to functions that may or may not be available. extern ExternalFunctions *EF; -// We are using a custom allocator to give a different symbol name to STL -// containers in order to avoid ODR violations. -template<typename T> - class fuzzer_allocator: public std::allocator<T> { - public: - fuzzer_allocator() = default; - - template<class U> - fuzzer_allocator(const fuzzer_allocator<U>&) {} - - template<class Other> - struct rebind { typedef fuzzer_allocator<Other> other; }; - }; - -template<typename T> -using Vector = std::vector<T, fuzzer_allocator<T>>; - -template<typename T> -using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>; - -typedef Vector<uint8_t> Unit; -typedef Vector<Unit> UnitVector; +typedef std::vector<uint8_t> Unit; +typedef std::vector<Unit> UnitVector; typedef int (*UserCallback)(const uint8_t *Data, size_t Size); int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); diff --git a/libfuzzer/FuzzerDictionary.h b/libfuzzer/FuzzerDictionary.h index db55907..48f063c 100644 --- a/libfuzzer/FuzzerDictionary.h +++ b/libfuzzer/FuzzerDictionary.h @@ -52,10 +52,13 @@ class DictionaryEntry { public: DictionaryEntry() {} DictionaryEntry(Word W) : W(W) {} - DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {} + DictionaryEntry(Word W, size_t PositionHint) + : W(W), PositionHint(PositionHint) {} const Word &GetW() const { return W; } - bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); } + bool HasPositionHint() const { + return PositionHint != std::numeric_limits<size_t>::max(); + } size_t GetPositionHint() const { assert(HasPositionHint()); return PositionHint; @@ -108,12 +111,12 @@ private: }; // Parses one dictionary entry. -// If successful, write the enty to Unit and returns true, +// If successful, writes the entry to Unit and returns true, // otherwise returns false. bool ParseOneDictionaryEntry(const std::string &Str, Unit *U); // Parses the dictionary file, fills Units, returns true iff all lines // were parsed successfully. -bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units); +bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units); } // namespace fuzzer diff --git a/libfuzzer/FuzzerDriver.cpp b/libfuzzer/FuzzerDriver.cpp index ceaa907..6b007f2 100644 --- a/libfuzzer/FuzzerDriver.cpp +++ b/libfuzzer/FuzzerDriver.cpp @@ -86,7 +86,7 @@ static const FlagDescription FlagDescriptions [] { static const size_t kNumFlags = sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); -static Vector<std::string> *Inputs; +static std::vector<std::string> *Inputs; static std::string *ProgName; static void PrintHelp() { @@ -187,7 +187,7 @@ static bool ParseOneFlag(const char *Param) { } // We don't use any library to minimize dependencies. -static void ParseFlags(const Vector<std::string> &Args, +static void ParseFlags(const std::vector<std::string> &Args, const ExternalFunctions *EF) { for (size_t F = 0; F < kNumFlags; F++) { if (FlagDescriptions[F].IntFlag) @@ -206,7 +206,7 @@ static void ParseFlags(const Vector<std::string> &Args, "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); } - Inputs = new Vector<std::string>; + Inputs = new std::vector<std::string>; for (size_t A = 1; A < Args.size(); A++) { if (ParseOneFlag(Args[A].c_str())) { if (Flags.ignore_remaining_args) @@ -272,7 +272,7 @@ static void ValidateDirectoryExists(const std::string &Path, exit(1); } -std::string CloneArgsWithoutX(const Vector<std::string> &Args, +std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X1, const char *X2) { std::string Cmd; for (auto &S : Args) { @@ -283,18 +283,19 @@ std::string CloneArgsWithoutX(const Vector<std::string> &Args, return Cmd; } -static int RunInMultipleProcesses(const Vector<std::string> &Args, +static int RunInMultipleProcesses(const std::vector<std::string> &Args, unsigned NumWorkers, unsigned NumJobs) { std::atomic<unsigned> Counter(0); std::atomic<bool> HasErrors(false); Command Cmd(Args); Cmd.removeFlag("jobs"); Cmd.removeFlag("workers"); - Vector<std::thread> V; + std::vector<std::thread> V; std::thread Pulse(PulseThread); Pulse.detach(); for (unsigned i = 0; i < NumWorkers; i++) - V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, + &HasErrors)); for (auto &T : V) T.join(); return HasErrors ? 1 : 0; @@ -348,8 +349,8 @@ static std::string GetDedupTokenFromCmdOutput(const std::string &S) { return S.substr(Beg, End - Beg); } -int CleanseCrashInput(const Vector<std::string> &Args, - const FuzzingOptions &Options) { +int CleanseCrashInput(const std::vector<std::string> &Args, + const FuzzingOptions &Options) { if (Inputs->size() != 1 || !Flags.exact_artifact_path) { Printf("ERROR: -cleanse_crash should be given one input file and" " -exact_artifact_path\n"); @@ -372,7 +373,7 @@ int CleanseCrashInput(const Vector<std::string> &Args, auto U = FileToVector(CurrentFilePath); size_t Size = U.size(); - const Vector<uint8_t> ReplacementBytes = {' ', 0xff}; + const std::vector<uint8_t> ReplacementBytes = {' ', 0xff}; for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { bool Changed = false; for (size_t Idx = 0; Idx < Size; Idx++) { @@ -403,7 +404,7 @@ int CleanseCrashInput(const Vector<std::string> &Args, return 0; } -int MinimizeCrashInput(const Vector<std::string> &Args, +int MinimizeCrashInput(const std::vector<std::string> &Args, const FuzzingOptions &Options) { if (Inputs->size() != 1) { Printf("ERROR: -minimize_crash should be given one input file\n"); @@ -503,14 +504,15 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { return 0; } -void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, - const Vector<std::string> &Corpora, const char *CFPathOrNull) { +void Merge(Fuzzer *F, FuzzingOptions &Options, + const std::vector<std::string> &Args, + const std::vector<std::string> &Corpora, const char *CFPathOrNull) { if (Corpora.size() < 2) { Printf("INFO: Merge requires two or more corpus dirs\n"); exit(0); } - Vector<SizedFile> OldCorpus, NewCorpus; + std::vector<SizedFile> OldCorpus, NewCorpus; GetSizedFilesFromDir(Corpora[0], &OldCorpus); for (size_t i = 1; i < Corpora.size(); i++) GetSizedFilesFromDir(Corpora[i], &NewCorpus); @@ -518,10 +520,10 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, std::sort(NewCorpus.begin(), NewCorpus.end()); std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); - Vector<std::string> NewFiles; - Set<uint32_t> NewFeatures, NewCov; + std::vector<std::string> NewFiles; + std::set<uint32_t> NewFeatures, NewCov; CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, - {}, &NewCov, CFPath, true); + {}, &NewCov, CFPath, true, Flags.set_cover_merge); for (auto &Path : NewFiles) F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); // We are done, delete the control file if it was a temporary one. @@ -531,17 +533,17 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, exit(0); } -int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, - UnitVector& Corpus) { +int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict, + UnitVector &Corpus) { Printf("Started dictionary minimization (up to %d tests)\n", Dict.size() * Corpus.size() * 2); // Scores and usage count for each dictionary unit. - Vector<int> Scores(Dict.size()); - Vector<int> Usages(Dict.size()); + std::vector<int> Scores(Dict.size()); + std::vector<int> Usages(Dict.size()); - Vector<size_t> InitialFeatures; - Vector<size_t> ModifiedFeatures; + std::vector<size_t> InitialFeatures; + std::vector<size_t> ModifiedFeatures; for (auto &C : Corpus) { // Get coverage for the testcase without modifications. F->ExecuteCallback(C.data(), C.size()); @@ -551,7 +553,7 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, }); for (size_t i = 0; i < Dict.size(); ++i) { - Vector<uint8_t> Data = C; + std::vector<uint8_t> Data = C; auto StartPos = std::search(Data.begin(), Data.end(), Dict[i].begin(), Dict[i].end()); // Skip dictionary unit, if the testcase does not contain it. @@ -597,9 +599,9 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, return 0; } -Vector<std::string> ParseSeedInuts(const char *seed_inputs) { +std::vector<std::string> ParseSeedInuts(const char *seed_inputs) { // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file - Vector<std::string> Files; + std::vector<std::string> Files; if (!seed_inputs) return Files; std::string SeedInputs; if (Flags.seed_inputs[0] == '@') @@ -620,9 +622,10 @@ Vector<std::string> ParseSeedInuts(const char *seed_inputs) { return Files; } -static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, - const Vector<std::string> &ExtraSeedFiles) { - Vector<SizedFile> SizedFiles; +static std::vector<SizedFile> +ReadCorpora(const std::vector<std::string> &CorpusDirs, + const std::vector<std::string> &ExtraSeedFiles) { + std::vector<SizedFile> SizedFiles; size_t LastNumFiles = 0; for (auto &Dir : CorpusDirs) { GetSizedFilesFromDir(Dir, &SizedFiles); @@ -645,7 +648,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { EF->LLVMFuzzerInitialize(argc, argv); if (EF->__msan_scoped_disable_interceptor_checks) EF->__msan_scoped_disable_interceptor_checks(); - const Vector<std::string> Args(*argv, *argv + *argc); + const std::vector<std::string> Args(*argv, *argv + *argc); assert(!Args.empty()); ProgName = new std::string(Args[0]); if (Argv0 != *ProgName) { @@ -734,7 +737,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { ValidateDirectoryExists(DirName(Options.ExactArtifactPath), Flags.create_missing_dirs); } - Vector<Unit> Dictionary; + std::vector<Unit> Dictionary; if (Flags.dict) if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) return 1; @@ -794,7 +797,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); - if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { + if (Flags.collect_data_flow && !Flags.fork && + !(Flags.merge || Flags.set_cover_merge)) { if (RunIndividualFiles) return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, ReadCorpora({}, *Inputs)); @@ -866,10 +870,11 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } + Options.ForkCorpusGroups = Flags.fork_corpus_groups; if (Flags.fork) FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); - if (Flags.merge) + if (Flags.merge || Flags.set_cover_merge) Merge(F, Options, Args, *Inputs, Flags.merge_control_file); if (Flags.merge_inner) { @@ -877,7 +882,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Options.MaxLen == 0) F->SetMaxInputLen(kDefaultMaxMergeLen); assert(Flags.merge_control_file); - F->CrashResistantMergeInternalStep(Flags.merge_control_file); + F->CrashResistantMergeInternalStep(Flags.merge_control_file, + !strncmp(Flags.merge_inner, "2", 1)); exit(0); } diff --git a/libfuzzer/FuzzerExtraCounters.cpp b/libfuzzer/FuzzerExtraCounters.cpp index 04f569a..54ecbf7 100644 --- a/libfuzzer/FuzzerExtraCounters.cpp +++ b/libfuzzer/FuzzerExtraCounters.cpp @@ -31,12 +31,4 @@ void ClearExtraCounters() { // hand-written memset, don't asan-ify. } // namespace fuzzer -#else -// TODO: implement for other platforms. -namespace fuzzer { -uint8_t *ExtraCountersBegin() { return nullptr; } -uint8_t *ExtraCountersEnd() { return nullptr; } -void ClearExtraCounters() {} -} // namespace fuzzer - #endif diff --git a/libfuzzer/FuzzerExtraCountersDarwin.cpp b/libfuzzer/FuzzerExtraCountersDarwin.cpp new file mode 100644 index 0000000..2321ba8 --- /dev/null +++ b/libfuzzer/FuzzerExtraCountersDarwin.cpp @@ -0,0 +1,22 @@ +//===- FuzzerExtraCountersDarwin.cpp - Extra coverage counters for Darwin -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code for Darwin. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include <cstdint> + +#if LIBFUZZER_APPLE + +namespace fuzzer { +uint8_t *ExtraCountersBegin() { return nullptr; } +uint8_t *ExtraCountersEnd() { return nullptr; } +void ClearExtraCounters() {} +} // namespace fuzzer + +#endif diff --git a/libfuzzer/FuzzerExtraCountersWindows.cpp b/libfuzzer/FuzzerExtraCountersWindows.cpp new file mode 100644 index 0000000..102f5fe --- /dev/null +++ b/libfuzzer/FuzzerExtraCountersWindows.cpp @@ -0,0 +1,80 @@ +//===- FuzzerExtraCountersWindows.cpp - Extra coverage counters for Win32 -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code for Windows. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include <cstdint> + +#if LIBFUZZER_WINDOWS +#include <windows.h> + +namespace fuzzer { + +// +// The __start___libfuzzer_extra_counters variable is align 16, size 16 to +// ensure the padding between it and the next variable in this section (either +// __libfuzzer_extra_counters or __stop___libfuzzer_extra_counters) will be +// located at (__start___libfuzzer_extra_counters + +// sizeof(__start___libfuzzer_extra_counters)). Otherwise, the calculation of +// (stop - (start + sizeof(start))) might be skewed. +// +// The section name, __libfuzzer_extra_countaaa ends with "aaa", so it sorts +// before __libfuzzer_extra_counters alphabetically. We want the start symbol to +// be placed in the section just before the user supplied counters (if present). +// +#pragma section(".data$__libfuzzer_extra_countaaa") +ATTRIBUTE_ALIGNED(16) +__declspec(allocate(".data$__libfuzzer_extra_countaaa")) uint8_t + __start___libfuzzer_extra_counters[16] = {0}; + +// +// Example of what the user-supplied counters should look like. First, the +// pragma to create the section name. It will fall alphabetically between +// ".data$__libfuzzer_extra_countaaa" and ".data$__libfuzzer_extra_countzzz". +// Next, the declspec to allocate the variable inside the specified section. +// Finally, some array, struct, whatever that is used to track the counter data. +// The size of this variable is computed at runtime by finding the difference of +// __stop___libfuzzer_extra_counters and __start___libfuzzer_extra_counters + +// sizeof(__start___libfuzzer_extra_counters). +// + +// +// #pragma section(".data$__libfuzzer_extra_counters") +// __declspec(allocate(".data$__libfuzzer_extra_counters")) +// uint8_t any_name_variable[64 * 1024]; +// + +// +// Here, the section name, __libfuzzer_extra_countzzz ends with "zzz", so it +// sorts after __libfuzzer_extra_counters alphabetically. We want the stop +// symbol to be placed in the section just after the user supplied counters (if +// present). Align to 1 so there isn't any padding placed between this and the +// previous variable. +// +#pragma section(".data$__libfuzzer_extra_countzzz") +ATTRIBUTE_ALIGNED(1) +__declspec(allocate(".data$__libfuzzer_extra_countzzz")) uint8_t + __stop___libfuzzer_extra_counters = 0; + +uint8_t *ExtraCountersBegin() { + return __start___libfuzzer_extra_counters + + sizeof(__start___libfuzzer_extra_counters); +} + +uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; } + +ATTRIBUTE_NO_SANITIZE_ALL +void ClearExtraCounters() { + uint8_t *Beg = ExtraCountersBegin(); + SecureZeroMemory(Beg, ExtraCountersEnd() - Beg); +} + +} // namespace fuzzer + +#endif diff --git a/libfuzzer/FuzzerFlags.def b/libfuzzer/FuzzerFlags.def index ab31da0..1181534 100644 --- a/libfuzzer/FuzzerFlags.def +++ b/libfuzzer/FuzzerFlags.def @@ -58,12 +58,21 @@ FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " FUZZER_FLAG_INT(help, 0, "Print help.") FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " "in a subprocess") +FUZZER_FLAG_INT(fork_corpus_groups, 0, "For fork mode, enable the corpus-group " + "strategy, The main corpus will be grouped according to size, " + "and each sub-process will randomly select seeds from different " + "groups as the sub-corpus.") FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " "merged into the 1-st corpus. Only interesting units will be taken. " "This flag can be used to minimize a corpus.") +FUZZER_FLAG_INT(set_cover_merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " + "merged into the 1-st corpus. Same as the 'merge' flag, but uses the " + "standard greedy algorithm for the set cover problem to " + "compute an approximation of the minimum set of testcases that " + "provide the same coverage as the initial corpora") FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") FUZZER_FLAG_STRING(merge_inner, "internal flag") FUZZER_FLAG_STRING(merge_control_file, diff --git a/libfuzzer/FuzzerFork.cpp b/libfuzzer/FuzzerFork.cpp index 5134a5d..d59d513 100644 --- a/libfuzzer/FuzzerFork.cpp +++ b/libfuzzer/FuzzerFork.cpp @@ -86,18 +86,21 @@ struct FuzzJob { }; struct GlobalEnv { - Vector<std::string> Args; - Vector<std::string> CorpusDirs; + std::vector<std::string> Args; + std::vector<std::string> CorpusDirs; std::string MainCorpusDir; std::string TempDir; std::string DFTDir; std::string DataFlowBinary; - Set<uint32_t> Features, Cov; - Set<std::string> FilesWithDFT; - Vector<std::string> Files; + std::set<uint32_t> Features, Cov; + std::set<std::string> FilesWithDFT; + std::vector<std::string> Files; + std::vector<std::size_t> FilesSizes; Random *Rand; std::chrono::system_clock::time_point ProcessStartTime; int Verbosity = 0; + int Group = 0; + int NumCorpuses = 8; size_t NumTimeouts = 0; size_t NumOOMs = 0; @@ -136,10 +139,24 @@ struct GlobalEnv { if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { auto Time1 = std::chrono::system_clock::now(); - for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + if (Group) { // whether to group the corpus. + size_t AverageCorpusSize = Files.size() / NumCorpuses + 1; + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize; + for (size_t i = 0; i < CorpusSubsetSize; i++) { + size_t RandNum = (*Rand)(AverageCorpusSize); + size_t Index = RandNum + StartIndex; + Index = Index < Files.size() ? Index + : Rand->SkewTowardsLast(Files.size()); + auto &SF = Files[Index]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + } else { + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } } auto Time2 = std::chrono::system_clock::now(); auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count(); @@ -183,7 +200,7 @@ struct GlobalEnv { auto Stats = ParseFinalStatsFromLog(Job->LogPath); NumRuns += Stats.number_of_executed_units; - Vector<SizedFile> TempFiles, MergeCandidates; + std::vector<SizedFile> TempFiles, MergeCandidates; // Read all newly created inputs and their feature sets. // Choose only those inputs that have new features. GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); @@ -193,7 +210,7 @@ struct GlobalEnv { FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); auto FeatureBytes = FileToVector(FeatureFile, 0, false); assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); - Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); for (auto Ft : NewFeatures) { if (!Features.count(Ft)) { @@ -211,15 +228,27 @@ struct GlobalEnv { if (MergeCandidates.empty()) return; - Vector<std::string> FilesToAdd; - Set<uint32_t> NewFeatures, NewCov; + std::vector<std::string> FilesToAdd; + std::set<uint32_t> NewFeatures, NewCov; + bool IsSetCoverMerge = + !Job->Cmd.getFlagValue("set_cover_merge").compare("1"); CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, - &NewFeatures, Cov, &NewCov, Job->CFPath, false); + &NewFeatures, Cov, &NewCov, Job->CFPath, false, + IsSetCoverMerge); for (auto &Path : FilesToAdd) { auto U = FileToVector(Path); auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); WriteToFile(U, NewPath); - Files.push_back(NewPath); + if (Group) { // Insert the queue according to the size of the seed. + size_t UnitSize = U.size(); + auto Idx = + std::upper_bound(FilesSizes.begin(), FilesSizes.end(), UnitSize) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + Idx, UnitSize); + Files.insert(Files.begin() + Idx, NewPath); + } else { + Files.push_back(NewPath); + } } Features.insert(NewFeatures.begin(), NewFeatures.end()); Cov.insert(NewCov.begin(), NewCov.end()); @@ -228,10 +257,8 @@ struct GlobalEnv { if (TPC.PcIsFuncEntry(TE)) PrintPC(" NEW_FUNC: %p %F %L\n", "", TPC.GetNextInstructionPc(TE->PC)); - } - void CollectDFT(const std::string &InputPath) { if (DataFlowBinary.empty()) return; if (!FilesWithDFT.insert(InputPath).second) return; @@ -283,8 +310,8 @@ void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { // This is just a skeleton of an experimental -fork=1 feature. void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, - const Vector<std::string> &Args, - const Vector<std::string> &CorpusDirs, int NumJobs) { + const std::vector<std::string> &Args, + const std::vector<std::string> &CorpusDirs, int NumJobs) { Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); GlobalEnv Env; @@ -294,8 +321,9 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.Verbosity = Options.Verbosity; Env.ProcessStartTime = std::chrono::system_clock::now(); Env.DataFlowBinary = Options.CollectDataFlow; + Env.Group = Options.ForkCorpusGroups; - Vector<SizedFile> SeedFiles; + std::vector<SizedFile> SeedFiles; for (auto &Dir : CorpusDirs) GetSizedFilesFromDir(Dir, &SeedFiles); std::sort(SeedFiles.begin(), SeedFiles.end()); @@ -316,13 +344,20 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.Files.push_back(File.File); } else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); - Set<uint32_t> NewFeatures, NewCov; + std::set<uint32_t> NewFeatures, NewCov; CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features, - &NewFeatures, Env.Cov, &NewCov, CFPath, false); + &NewFeatures, Env.Cov, &NewCov, CFPath, + /*Verbose=*/false, /*IsSetCoverMerge=*/false); Env.Features.insert(NewFeatures.begin(), NewFeatures.end()); Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } + + if (Env.Group) { + for (auto &path : Env.Files) + Env.FilesSizes.push_back(FileSize(path)); + } + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); @@ -337,8 +372,10 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, WriteToFile(Unit({1}), Env.StopFile()); }; + size_t MergeCycle = 20; + size_t JobExecuted = 0; size_t JobId = 1; - Vector<std::thread> Threads; + std::vector<std::thread> Threads; for (int t = 0; t < NumJobs; t++) { Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); FuzzQ.Push(Env.CreateNewJob(JobId++)); @@ -358,7 +395,46 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.RunOneMergeJob(Job.get()); - // Continue if our crash is one of the ignorred ones. + // merge the corpus . + JobExecuted++; + if (Env.Group && JobExecuted >= MergeCycle) { + std::vector<SizedFile> CurrentSeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &CurrentSeedFiles); + std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end()); + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + std::set<uint32_t> TmpNewFeatures, TmpNewCov; + std::set<uint32_t> TmpFeatures, TmpCov; + Env.Files.clear(); + Env.FilesSizes.clear(); + CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files, + TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov, + CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false); + for (auto &path : Env.Files) + Env.FilesSizes.push_back(FileSize(path)); + RemoveFile(CFPath); + JobExecuted = 0; + MergeCycle += 5; + } + + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() < 2000) + Env.NumCorpuses = 12; + else if (Env.Files.size() < 6000) + Env.NumCorpuses = 20; + else if (Env.Files.size() < 12000) + Env.NumCorpuses = 32; + else if (Env.Files.size() < 16000) + Env.NumCorpuses = 40; + else if (Env.Files.size() < 24000) + Env.NumCorpuses = 60; + else + Env.NumCorpuses = 80; + + // Continue if our crash is one of the ignored ones. if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) Env.NumTimeouts++; else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) diff --git a/libfuzzer/FuzzerFork.h b/libfuzzer/FuzzerFork.h index b29a43e..fc3e9d6 100644 --- a/libfuzzer/FuzzerFork.h +++ b/libfuzzer/FuzzerFork.h @@ -17,8 +17,8 @@ namespace fuzzer { void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, - const Vector<std::string> &Args, - const Vector<std::string> &CorpusDirs, int NumJobs); + const std::vector<std::string> &Args, + const std::vector<std::string> &CorpusDirs, int NumJobs); } // namespace fuzzer #endif // LLVM_FUZZER_FORK_H diff --git a/libfuzzer/FuzzerIO.cpp b/libfuzzer/FuzzerIO.cpp index 7f149ac..0a58c53 100644 --- a/libfuzzer/FuzzerIO.cpp +++ b/libfuzzer/FuzzerIO.cpp @@ -23,6 +23,14 @@ namespace fuzzer { static FILE *OutputFile = stderr; +FILE *GetOutputFile() { + return OutputFile; +} + +void SetOutputFile(FILE *NewOutputFile) { + OutputFile = NewOutputFile; +} + long GetEpoch(const std::string &Path) { struct stat St; if (stat(Path.c_str(), &St)) @@ -90,11 +98,11 @@ void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path) { fclose(Out); } -void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch, +void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch, size_t MaxSize, bool ExitOnError, - Vector<std::string> *VPaths) { + std::vector<std::string> *VPaths) { long E = Epoch ? *Epoch : 0; - Vector<std::string> Files; + std::vector<std::string> Files; ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); size_t NumLoaded = 0; for (size_t i = 0; i < Files.size(); i++) { @@ -112,8 +120,8 @@ void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch, } } -void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) { - Vector<std::string> Files; +void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V) { + std::vector<std::string> Files; ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); for (auto &File : Files) if (size_t Size = FileSize(File)) diff --git a/libfuzzer/FuzzerIO.h b/libfuzzer/FuzzerIO.h index bde1826..401afa0 100644 --- a/libfuzzer/FuzzerIO.h +++ b/libfuzzer/FuzzerIO.h @@ -32,9 +32,9 @@ void WriteToFile(const Unit &U, const std::string &Path); void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path); void AppendToFile(const std::string &Data, const std::string &Path); -void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch, +void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch, size_t MaxSize, bool ExitOnError, - Vector<std::string> *VPaths = 0); + std::vector<std::string> *VPaths = 0); // Returns "Dir/FileName" or equivalent for the current OS. std::string DirPlusFile(const std::string &DirPath, @@ -54,6 +54,10 @@ void DupAndCloseStderr(); void CloseStdout(); +// For testing. +FILE *GetOutputFile(); +void SetOutputFile(FILE *NewOutputFile); + void Printf(const char *Fmt, ...); void VPrintf(bool Verbose, const char *Fmt, ...); @@ -66,7 +70,7 @@ bool IsDirectory(const std::string &Path); size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir); + std::vector<std::string> *V, bool TopDir); bool MkDirRecursive(const std::string &Dir); void RmDirRecursive(const std::string &Dir); @@ -85,7 +89,7 @@ struct SizedFile { bool operator<(const SizedFile &B) const { return Size < B.Size; } }; -void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); +void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V); char GetSeparator(); bool IsSeparator(char C); diff --git a/libfuzzer/FuzzerIOPosix.cpp b/libfuzzer/FuzzerIOPosix.cpp index 4706a40..3700fb0 100644 --- a/libfuzzer/FuzzerIOPosix.cpp +++ b/libfuzzer/FuzzerIOPosix.cpp @@ -53,7 +53,7 @@ std::string Basename(const std::string &Path) { } void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir) { + std::vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; @@ -78,7 +78,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } - void IterateDirRecursive(const std::string &Dir, void (*DirPreCallback)(const std::string &Dir), void (*DirPostCallback)(const std::string &Dir), diff --git a/libfuzzer/FuzzerIOWindows.cpp b/libfuzzer/FuzzerIOWindows.cpp index 61ad35e..6771fc1 100644 --- a/libfuzzer/FuzzerIOWindows.cpp +++ b/libfuzzer/FuzzerIOWindows.cpp @@ -111,7 +111,7 @@ size_t FileSize(const std::string &Path) { } void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir) { + std::vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; @@ -159,7 +159,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } - void IterateDirRecursive(const std::string &Dir, void (*DirPreCallback)(const std::string &Dir), void (*DirPostCallback)(const std::string &Dir), @@ -297,9 +296,8 @@ static size_t ParseServerAndShare(const std::string &FileName, return Pos - Offset; } -// Parse the given Ref string from the position Offset, to exactly match the given -// string Patt. -// Returns number of characters considered if successful. +// Parse the given Ref string from the position Offset, to exactly match the +// given string Patt. Returns number of characters considered if successful. static size_t ParseCustomString(const std::string &Ref, size_t Offset, const char *Patt) { size_t Len = strlen(Patt); diff --git a/libfuzzer/FuzzerInterceptors.cpp b/libfuzzer/FuzzerInterceptors.cpp index b877986..d5b0a42 100644 --- a/libfuzzer/FuzzerInterceptors.cpp +++ b/libfuzzer/FuzzerInterceptors.cpp @@ -25,6 +25,7 @@ } #include <cassert> +#include <cstddef> // for size_t #include <cstdint> #include <dlfcn.h> // for dlsym() diff --git a/libfuzzer/FuzzerInternal.h b/libfuzzer/FuzzerInternal.h index 37c8a01..31f54ea 100644 --- a/libfuzzer/FuzzerInternal.h +++ b/libfuzzer/FuzzerInternal.h @@ -35,8 +35,8 @@ public: Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options); ~Fuzzer(); - void Loop(Vector<SizedFile> &CorporaFiles); - void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles); + void Loop(std::vector<SizedFile> &CorporaFiles); + void ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles); void MinimizeCrashLoop(const Unit &U); void RereadOutputCorpus(size_t MaxSize); @@ -65,15 +65,19 @@ public: static void StaticFileSizeExceedCallback(); static void StaticGracefulExitCallback(); - void ExecuteCallback(const uint8_t *Data, size_t Size); + // Executes the target callback on {Data, Size} once. + // Returns false if the input was rejected by the target (target returned -1), + // and true otherwise. + bool ExecuteCallback(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, InputInfo *II = nullptr, bool ForceAddToCorpus = false, bool *FoundUniqFeatures = nullptr); void TPCUpdateObservedPCs(); // Merge Corpora[1:] into Corpora[0]. - void Merge(const Vector<std::string> &Corpora); - void CrashResistantMergeInternalStep(const std::string &ControlFilePath); + void Merge(const std::vector<std::string> &Corpora); + void CrashResistantMergeInternalStep(const std::string &ControlFilePath, + bool IsSetCoverMerge); MutationDispatcher &GetMD() { return MD; } void PrintFinalStats(); void SetMaxInputLen(size_t MaxInputLen); @@ -141,7 +145,7 @@ private: size_t MaxMutationLen = 0; size_t TmpMaxMutationLen = 0; - Vector<uint32_t> UniqFeatureSetTmp; + std::vector<uint32_t> UniqFeatureSetTmp; // Need to know our own thread. static thread_local bool IsMyThread; diff --git a/libfuzzer/FuzzerLoop.cpp b/libfuzzer/FuzzerLoop.cpp index 86a78ab..f095757 100644 --- a/libfuzzer/FuzzerLoop.cpp +++ b/libfuzzer/FuzzerLoop.cpp @@ -388,7 +388,7 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { - static auto *PCsSet = new Set<uintptr_t>; + static auto *PCsSet = new std::set<uintptr_t>; auto HandlePC = [&](const TracePC::PCTableEntry *TE) { if (!PCsSet->insert(TE->PC).second) return; @@ -413,8 +413,8 @@ void Fuzzer::CheckExitOnSrcPosOrItem() { void Fuzzer::RereadOutputCorpus(size_t MaxSize) { if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return; - Vector<Unit> AdditionalCorpus; - Vector<std::string> AdditionalCorpusPaths; + std::vector<Unit> AdditionalCorpus; + std::vector<std::string> AdditionalCorpusPaths; ReadDirToVectorOfUnits( Options.OutputCorpus.c_str(), &AdditionalCorpus, &EpochOfLastReadOfOutputCorpus, MaxSize, @@ -457,7 +457,7 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { static void WriteFeatureSetToFile(const std::string &FeaturesDir, const std::string &FileName, - const Vector<uint32_t> &FeatureSet) { + const std::vector<uint32_t> &FeatureSet) { if (FeaturesDir.empty() || FeatureSet.empty()) return; WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()), FeatureSet.size() * sizeof(FeatureSet[0]), @@ -511,7 +511,7 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, // Largest input length should be INT_MAX. assert(Size < std::numeric_limits<uint32_t>::max()); - ExecuteCallback(Data, Size); + if(!ExecuteCallback(Data, Size)) return false; auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime); UniqFeatureSetTmp.clear(); @@ -548,7 +548,7 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && II->U.size() > Size) { auto OldFeaturesFile = Sha1ToString(II->Sha1); - Corpus.Replace(II, {Data, Data + Size}); + Corpus.Replace(II, {Data, Data + Size}, TimeOfUnit); RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, Sha1ToString(II->Sha1)); return true; @@ -586,7 +586,7 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { // This method is not inlined because it would cause a test to fail where it // is part of the stack unwinding. See D97975 for details. -ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data, +ATTRIBUTE_NOINLINE bool Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; @@ -602,23 +602,24 @@ ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data, if (CurrentUnitData && CurrentUnitData != Data) memcpy(CurrentUnitData, Data, Size); CurrentUnitSize = Size; + int CBRes = 0; { ScopedEnableMsanInterceptorChecks S; AllocTracer.Start(Options.TraceMalloc); UnitStartTime = system_clock::now(); TPC.ResetMaps(); RunningUserCallback = true; - int Res = CB(DataCopy, Size); + CBRes = CB(DataCopy, Size); RunningUserCallback = false; UnitStopTime = system_clock::now(); - (void)Res; - assert(Res == 0); + assert(CBRes == 0 || CBRes == -1); HasMoreMallocsThanFrees = AllocTracer.Stop(); } if (!LooseMemeq(DataCopy, Data, Size)) CrashOnOverwrittenData(); CurrentUnitSize = 0; delete[] DataCopy; + return CBRes == 0; } std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { @@ -784,7 +785,7 @@ void Fuzzer::PurgeAllocator() { LastAllocatorPurgeAttemptTime = system_clock::now(); } -void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { +void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) { const size_t kMaxSaneLen = 1 << 20; const size_t kMinDefaultLen = 4096; size_t MaxSize = 0; @@ -843,13 +844,20 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { } if (Corpus.empty() && Options.MaxNumberOfRuns) { - Printf("ERROR: no interesting inputs were found. " - "Is the code instrumented for coverage? Exiting.\n"); - exit(1); + Printf("WARNING: no interesting inputs were found so far. " + "Is the code instrumented for coverage?\n" + "This may also happen if the target rejected all inputs we tried so " + "far\n"); + // The remaining logic requires that the corpus is not empty, + // so we add one fake input to the in-memory corpus. + Corpus.AddToCorpus({'\n'}, /*NumFeatures=*/1, /*MayDeleteFile=*/true, + /*HasFocusFunction=*/false, /*NeverReduce=*/false, + /*TimeOfUnit=*/duration_cast<microseconds>(0s), {0}, DFT, + /*BaseII*/ nullptr); } } -void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { +void Fuzzer::Loop(std::vector<SizedFile> &CorporaFiles) { auto FocusFunctionOrAuto = Options.FocusFunction; DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, MD.GetRand()); diff --git a/libfuzzer/FuzzerMerge.cpp b/libfuzzer/FuzzerMerge.cpp index 162453c..24bd119 100644 --- a/libfuzzer/FuzzerMerge.cpp +++ b/libfuzzer/FuzzerMerge.cpp @@ -77,8 +77,8 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; - Vector<uint32_t> TmpFeatures; - Set<uint32_t> PCs; + std::vector<uint32_t> TmpFeatures; + std::set<uint32_t> PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; @@ -132,15 +132,16 @@ size_t Merger::ApproximateMemoryConsumption() const { // Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. -size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, - Vector<std::string> *NewFiles) { +size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles) { NewFiles->clear(); NewFeatures->clear(); NewCov->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - Set<uint32_t> AllFeatures = InitialFeatures; + std::set<uint32_t> AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { @@ -150,7 +151,7 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; - Vector<uint32_t> Tmp; + std::vector<uint32_t> Tmp; std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); Cur.swap(Tmp); @@ -188,15 +189,16 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, return NewFeatures->size(); } -Set<uint32_t> Merger::AllFeatures() const { - Set<uint32_t> S; +std::set<uint32_t> Merger::AllFeatures() const { + std::set<uint32_t> S; for (auto &File : Files) S.insert(File.Features.begin(), File.Features.end()); return S; } // Inner process. May crash if the target crashes. -void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { +void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath, + bool IsSetCoverMerge) { Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); Merger M; std::ifstream IF(CFPath); @@ -212,11 +214,11 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { M.Files.size() - M.FirstNotProcessedFile); std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); - Set<size_t> AllFeatures; + std::set<size_t> AllFeatures; auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) { this->PrintStats(Where, "\n", 0, AllFeatures.size()); }; - Set<const TracePC::PCTableEntry *> AllPCs; + std::set<const TracePC::PCTableEntry *> AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); @@ -234,13 +236,14 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { // Collect coverage. We are iterating over the files in this order: // * First, files in the initial corpus ordered by size, smallest first. // * Then, all other files, smallest first. - // So it makes no sense to record all features for all files, instead we - // only record features that were not seen before. - Set<size_t> UniqFeatures; - TPC.CollectFeatures([&](size_t Feature) { - if (AllFeatures.insert(Feature).second) - UniqFeatures.insert(Feature); - }); + std::set<size_t> Features; + if (IsSetCoverMerge) + TPC.CollectFeatures([&](size_t Feature) { Features.insert(Feature); }); + else + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) + Features.insert(Feature); + }); TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) @@ -249,7 +252,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { PrintStatsWrapper("LOADED"); // Write the post-run marker and the coverage. OF << "FT " << i; - for (size_t F : UniqFeatures) + for (size_t F : Features) OF << " " << F; OF << "\n"; OF << "COV " << i; @@ -263,15 +266,137 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { PrintStatsWrapper("DONE "); } -static size_t WriteNewControlFile(const std::string &CFPath, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - const Vector<MergeFileInfo> &KnownFiles) { +// Merges all corpora into the first corpus. A file is added into +// the first corpus only if it adds new features. Unlike `Merger::Merge`, +// this implementation calculates an approximation of the minimum set +// of corpora files, that cover all known features (set cover problem). +// Generally, this means that files with more features are preferred for +// merge into the first corpus. When two files have the same number of +// features, the smaller one is preferred. +size_t Merger::SetCoverMerge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles) { + assert(NumFilesInFirstCorpus <= Files.size()); + NewFiles->clear(); + NewFeatures->clear(); + NewCov->clear(); + std::set<uint32_t> AllFeatures; + // 1 << 21 - 1 is the maximum feature index. + // See 'kFeatureSetSize' in 'FuzzerCorpus.h'. + const uint32_t kFeatureSetSize = 1 << 21; + std::vector<bool> Covered(kFeatureSetSize, false); + size_t NumCovered = 0; + + std::set<uint32_t> ExistingFeatures = InitialFeatures; + for (size_t i = 0; i < NumFilesInFirstCorpus; ++i) + ExistingFeatures.insert(Files[i].Features.begin(), Files[i].Features.end()); + + // Mark the existing features as covered. + for (const auto &F : ExistingFeatures) { + if (!Covered[F % kFeatureSetSize]) { + ++NumCovered; + Covered[F % kFeatureSetSize] = true; + } + // Calculate an underestimation of the set of covered features + // since the `Covered` bitvector is smaller than the feature range. + AllFeatures.insert(F % kFeatureSetSize); + } + + std::set<size_t> RemainingFiles; + for (size_t i = NumFilesInFirstCorpus; i < Files.size(); ++i) { + // Construct an incremental sequence which represent the + // indices to all files (excluding those in the initial corpus). + // RemainingFiles = range(NumFilesInFirstCorpus..Files.size()). + RemainingFiles.insert(i); + // Insert this file's unique features to all features. + for (const auto &F : Files[i].Features) + AllFeatures.insert(F % kFeatureSetSize); + } + + // Integrate files into Covered until set is complete. + while (NumCovered != AllFeatures.size()) { + // Index to file with largest number of unique features. + size_t MaxFeaturesIndex = NumFilesInFirstCorpus; + // Indices to remove from RemainingFiles. + std::set<size_t> RemoveIndices; + // Running max unique feature count. + // Updated upon finding a file with more features. + size_t MaxNumFeatures = 0; + + // Iterate over all files not yet integrated into Covered, + // to find the file which has the largest number of + // features that are not already in Covered. + for (const auto &i : RemainingFiles) { + const auto &File = Files[i]; + size_t CurrentUnique = 0; + // Count number of features in this file + // which are not yet in Covered. + for (const auto &F : File.Features) + if (!Covered[F % kFeatureSetSize]) + ++CurrentUnique; + + if (CurrentUnique == 0) { + // All features in this file are already in Covered: skip next time. + RemoveIndices.insert(i); + } else if (CurrentUnique > MaxNumFeatures || + (CurrentUnique == MaxNumFeatures && + File.Size < Files[MaxFeaturesIndex].Size)) { + // Update the max features file based on unique features + // Break ties by selecting smaller files. + MaxNumFeatures = CurrentUnique; + MaxFeaturesIndex = i; + } + } + // Must be a valid index/ + assert(MaxFeaturesIndex < Files.size()); + // Remove any feature-less files found. + for (const auto &i : RemoveIndices) + RemainingFiles.erase(i); + if (MaxNumFeatures == 0) { + // Did not find a file that adds unique features. + // This means that we should have no remaining files. + assert(RemainingFiles.size() == 0); + assert(NumCovered == AllFeatures.size()); + break; + } + + // MaxFeaturesIndex must be an element of Remaining. + assert(RemainingFiles.find(MaxFeaturesIndex) != RemainingFiles.end()); + // Remove the file with the most features from Remaining. + RemainingFiles.erase(MaxFeaturesIndex); + const auto &MaxFeatureFile = Files[MaxFeaturesIndex]; + // Add the features of the max feature file to Covered. + for (const auto &F : MaxFeatureFile.Features) { + if (!Covered[F % kFeatureSetSize]) { + ++NumCovered; + Covered[F % kFeatureSetSize] = true; + NewFeatures->insert(F); + } + } + // Add the index to this file to the result. + NewFiles->push_back(MaxFeatureFile.Name); + // Update NewCov with the additional coverage + // that MaxFeatureFile provides. + for (const auto &C : MaxFeatureFile.Cov) + if (InitialCov.find(C) == InitialCov.end()) + NewCov->insert(C); + } + + return NewFeatures->size(); +} + +static size_t +WriteNewControlFile(const std::string &CFPath, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + const std::vector<MergeFileInfo> &KnownFiles) { std::unordered_set<std::string> FilesToSkip; for (auto &SF: KnownFiles) FilesToSkip.insert(SF.Name); - Vector<std::string> FilesToUse; + std::vector<std::string> FilesToUse; auto MaybeUseFile = [=, &FilesToUse](std::string Name) { if (FilesToSkip.find(Name) == FilesToSkip.end()) FilesToUse.push_back(Name); @@ -299,19 +424,19 @@ static size_t WriteNewControlFile(const std::string &CFPath, } // Outer process. Does not call the target code and thus should not fail. -void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - Vector<std::string> *NewFiles, - const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, - Set<uint32_t> *NewCov, - const std::string &CFPath, - bool V /*Verbose*/) { +void CrashResistantMerge(const std::vector<std::string> &Args, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + std::vector<std::string> *NewFiles, + const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, const std::string &CFPath, + bool V, /*Verbose*/ + bool IsSetCoverMerge) { if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; - Vector<MergeFileInfo> KnownFiles; + std::vector<MergeFileInfo> KnownFiles; if (FileSize(CFPath)) { VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", CFPath.c_str()); @@ -363,6 +488,7 @@ void CrashResistantMerge(const Vector<std::string> &Args, // Every inner process should execute at least one input. Command BaseCmd(Args); BaseCmd.removeFlag("merge"); + BaseCmd.removeFlag("set_cover_merge"); BaseCmd.removeFlag("fork"); BaseCmd.removeFlag("collect_data_flow"); for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { @@ -370,14 +496,16 @@ void CrashResistantMerge(const Vector<std::string> &Args, VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); - Cmd.addFlag("merge_inner", "1"); + // If we are going to use the set cover implementation for + // minimization add the merge_inner=2 internal flag. + Cmd.addFlag("merge_inner", IsSetCoverMerge ? "2" : "1"); if (!V) { Cmd.setOutputFile(getDevNull()); Cmd.combineOutAndErr(); } auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); + VPrintf(V, "MERGE-OUTER: successful in %zd attempt(s)\n", Attempt); break; } } @@ -395,7 +523,10 @@ void CrashResistantMerge(const Vector<std::string> &Args, M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end()); - M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + if (IsSetCoverMerge) + M.SetCoverMerge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + else + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " "%zd new coverage edges\n", NewFiles->size(), NewFeatures->size(), NewCov->size()); diff --git a/libfuzzer/FuzzerMerge.h b/libfuzzer/FuzzerMerge.h index e0c6bc5..42f798e 100644 --- a/libfuzzer/FuzzerMerge.h +++ b/libfuzzer/FuzzerMerge.h @@ -41,6 +41,7 @@ #define LLVM_FUZZER_MERGE_H #include "FuzzerDefs.h" +#include "FuzzerIO.h" #include <istream> #include <ostream> @@ -52,11 +53,11 @@ namespace fuzzer { struct MergeFileInfo { std::string Name; size_t Size = 0; - Vector<uint32_t> Features, Cov; + std::vector<uint32_t> Features, Cov; }; struct Merger { - Vector<MergeFileInfo> Files; + std::vector<MergeFileInfo> Files; size_t NumFilesInFirstCorpus = 0; size_t FirstNotProcessedFile = 0; std::string LastFailure; @@ -64,23 +65,28 @@ struct Merger { bool Parse(std::istream &IS, bool ParseCoverage); bool Parse(const std::string &Str, bool ParseCoverage); void ParseOrExit(std::istream &IS, bool ParseCoverage); - size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, - Vector<std::string> *NewFiles); + size_t Merge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles); + size_t SetCoverMerge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles); size_t ApproximateMemoryConsumption() const; - Set<uint32_t> AllFeatures() const; + std::set<uint32_t> AllFeatures() const; }; -void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - Vector<std::string> *NewFiles, - const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, - Set<uint32_t> *NewCov, - const std::string &CFPath, - bool Verbose); +void CrashResistantMerge(const std::vector<std::string> &Args, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + std::vector<std::string> *NewFiles, + const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, const std::string &CFPath, + bool Verbose, bool IsSetCoverMerge); } // namespace fuzzer diff --git a/libfuzzer/FuzzerMutate.cpp b/libfuzzer/FuzzerMutate.cpp index 4650f1b..d663900 100644 --- a/libfuzzer/FuzzerMutate.cpp +++ b/libfuzzer/FuzzerMutate.cpp @@ -485,7 +485,7 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { } void MutationDispatcher::PrintRecommendedDictionary() { - Vector<DictionaryEntry> V; + std::vector<DictionaryEntry> V; for (auto &DE : PersistentAutoDictionary) if (!ManualDictionary.ContainsWord(DE.GetW())) V.push_back(DE); @@ -540,7 +540,7 @@ size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size, // Mutates Data in place, returns new size. size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - Vector<Mutator> &Mutators) { + std::vector<Mutator> &Mutators) { assert(MaxSize > 0); // Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize), // in which case they will return 0. @@ -562,7 +562,7 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, // Mask represents the set of Data bytes that are worth mutating. size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, - const Vector<uint8_t> &Mask) { + const std::vector<uint8_t> &Mask) { size_t MaskedSize = std::min(Size, Mask.size()); // * Copy the worthy bytes into a temporary array T // * Mutate T diff --git a/libfuzzer/FuzzerMutate.h b/libfuzzer/FuzzerMutate.h index fd37191..97704e2 100644 --- a/libfuzzer/FuzzerMutate.h +++ b/libfuzzer/FuzzerMutate.h @@ -77,7 +77,7 @@ public: /// that have '1' in Mask. /// Mask.size() should be >= Size. size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, - const Vector<uint8_t> &Mask); + const std::vector<uint8_t> &Mask); /// Applies one of the default mutations. Provided as a service /// to mutation authors. @@ -104,7 +104,7 @@ public: size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - Vector<Mutator> &Mutators); + std::vector<Mutator> &Mutators); size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, size_t ToSize, size_t MaxToSize); @@ -133,22 +133,22 @@ public: // entries that led to successful discoveries in the past mutations. Dictionary PersistentAutoDictionary; - Vector<DictionaryEntry *> CurrentDictionaryEntrySequence; + std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; const Unit *CrossOverWith = nullptr; - Vector<uint8_t> MutateInPlaceHere; - Vector<uint8_t> MutateWithMaskTemp; + std::vector<uint8_t> MutateInPlaceHere; + std::vector<uint8_t> MutateWithMaskTemp; // CustomCrossOver needs its own buffer as a custom implementation may call // LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere. - Vector<uint8_t> CustomCrossOverInPlaceHere; + std::vector<uint8_t> CustomCrossOverInPlaceHere; - Vector<Mutator> Mutators; - Vector<Mutator> DefaultMutators; - Vector<Mutator> CurrentMutatorSequence; + std::vector<Mutator> Mutators; + std::vector<Mutator> DefaultMutators; + std::vector<Mutator> CurrentMutatorSequence; }; } // namespace fuzzer diff --git a/libfuzzer/FuzzerOptions.h b/libfuzzer/FuzzerOptions.h index d0c285a..72e2561 100644 --- a/libfuzzer/FuzzerOptions.h +++ b/libfuzzer/FuzzerOptions.h @@ -47,6 +47,7 @@ struct FuzzingOptions { int ReportSlowUnits = 10; bool OnlyASCII = false; bool Entropic = true; + bool ForkCorpusGroups = false; size_t EntropicFeatureFrequencyThreshold = 0xFF; size_t EntropicNumberOfRarestFeatures = 100; bool EntropicScalePerExecTime = false; diff --git a/libfuzzer/FuzzerTracePC.cpp b/libfuzzer/FuzzerTracePC.cpp index d808b9b..f12f7aa 100644 --- a/libfuzzer/FuzzerTracePC.cpp +++ b/libfuzzer/FuzzerTracePC.cpp @@ -133,13 +133,14 @@ inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { // so we return (pc-2) in that case in order to be safe. // For A32 mode we return (pc-4) because all instructions are 32 bit long. return (PC - 3) & (~1); -#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__) - // PCs are always 4 byte aligned. - return PC - 4; #elif defined(__sparc__) || defined(__mips__) return PC - 8; -#else +#elif defined(__riscv__) + return PC - 2; +#elif defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) return PC - 1; +#else + return PC - 4; #endif } @@ -157,7 +158,7 @@ ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { } void TracePC::UpdateObservedPCs() { - Vector<uintptr_t> CoveredFuncs; + std::vector<uintptr_t> CoveredFuncs; auto ObservePC = [&](const PCTableEntry *TE) { if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", @@ -300,8 +301,8 @@ void TracePC::PrintCoverage(bool PrintAllCounters) { FunctionStr = FunctionStr.substr(3); std::string LineStr = DescribePC("%l", VisualizePC); size_t NumEdges = Last - First; - Vector<uintptr_t> UncoveredPCs; - Vector<uintptr_t> CoveredPCs; + std::vector<uintptr_t> UncoveredPCs; + std::vector<uintptr_t> CoveredPCs; for (auto TE = First; TE < Last; TE++) if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); @@ -391,6 +392,7 @@ void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); } +ATTRIBUTE_NO_SANITIZE_MEMORY static size_t InternalStrnlen(const char *S, size_t MaxLen) { size_t Len = 0; for (; Len < MaxLen && S[Len]; Len++) {} @@ -398,7 +400,8 @@ static size_t InternalStrnlen(const char *S, size_t MaxLen) { } // Finds min of (strlen(S1), strlen(S2)). -// Needed bacause one of these strings may actually be non-zero terminated. +// Needed because one of these strings may actually be non-zero terminated. +ATTRIBUTE_NO_SANITIZE_MEMORY static size_t InternalStrnlen2(const char *S1, const char *S2) { size_t Len = 0; for (; S1[Len] && S2[Len]; Len++) {} diff --git a/libfuzzer/FuzzerTracePC.h b/libfuzzer/FuzzerTracePC.h index a937329..af1f9d8 100644 --- a/libfuzzer/FuzzerTracePC.h +++ b/libfuzzer/FuzzerTracePC.h @@ -169,7 +169,7 @@ private: size_t NumPCTables; size_t NumPCsInPCTables; - Set<const PCTableEntry*> ObservedPCs; + std::set<const PCTableEntry *> ObservedPCs; std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter. uint8_t *FocusFunctionCounterPtr = nullptr; diff --git a/libfuzzer/FuzzerUtil.cpp b/libfuzzer/FuzzerUtil.cpp index 0518549..aeab70f 100644 --- a/libfuzzer/FuzzerUtil.cpp +++ b/libfuzzer/FuzzerUtil.cpp @@ -43,7 +43,7 @@ void PrintASCIIByte(uint8_t Byte) { else if (Byte >= 32 && Byte < 127) Printf("%c", Byte); else - Printf("\\x%02x", Byte); + Printf("\\%03o", Byte); } void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) { @@ -124,7 +124,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { return true; } -bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) { +bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) { if (Text.empty()) { Printf("ParseDictionaryFile: file does not exist or is empty\n"); return false; diff --git a/libfuzzer/FuzzerUtil.h b/libfuzzer/FuzzerUtil.h index a188a7b..71d4909 100644 --- a/libfuzzer/FuzzerUtil.h +++ b/libfuzzer/FuzzerUtil.h @@ -66,10 +66,10 @@ int CloseProcessPipe(FILE *F); const void *SearchMemory(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); -std::string CloneArgsWithoutX(const Vector<std::string> &Args, +std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X1, const char *X2); -inline std::string CloneArgsWithoutX(const Vector<std::string> &Args, +inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X) { return CloneArgsWithoutX(Args, X, X); } diff --git a/libfuzzer/FuzzerUtilFuchsia.cpp b/libfuzzer/FuzzerUtilFuchsia.cpp index 5034b4a..d80b80c 100644 --- a/libfuzzer/FuzzerUtilFuchsia.cpp +++ b/libfuzzer/FuzzerUtilFuchsia.cpp @@ -52,6 +52,12 @@ void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); namespace { +// The signal handler thread uses Zircon exceptions to resume crashed threads +// into libFuzzer's POSIX signal handlers. The associated event is used to +// signal when the thread is running, and when it should stop. +std::thread SignalHandler; +zx_handle_t SignalHandlerEvent = ZX_HANDLE_INVALID; + // Helper function to handle Zircon syscall failures. void ExitOnErr(zx_status_t Status, const char *Syscall) { if (Status != ZX_OK) { @@ -68,23 +74,6 @@ void AlarmHandler(int Seconds) { } } -// CFAOffset is used to reference the stack pointer before entering the -// trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping -// to the trampoline we copy all the registers onto the stack. We need to make -// sure that the new stack has enough space to store all the registers. -// -// The trampoline holds CFI information regarding the registers stored in the -// stack, which is then used by the unwinder to restore them. -#if defined(__x86_64__) -// In x86_64 the crashing function might also be using the red zone (128 bytes -// on top of their rsp). -constexpr size_t CFAOffset = 128 + sizeof(zx_thread_state_general_regs_t); -#elif defined(__aarch64__) -// In aarch64 we need to always have the stack pointer aligned to 16 bytes, so we -// make sure that we are keeping that same alignment. -constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(uintptr_t)16; -#endif - // For the crash handler, we need to call Fuzzer::StaticCrashSignalCallback // without POSIX signal handlers. To achieve this, we use an assembly function // to add the necessary CFI unwinding information and a C function to bridge @@ -163,10 +152,10 @@ constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(u // Produces an assembler immediate operand for the named or numbered register. // This operand contains the offset of the register relative to the CFA. -#define ASM_OPERAND_REG(reg) \ - [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg) - CFAOffset), -#define ASM_OPERAND_NUM(num) \ - [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num]) - CFAOffset), +#define ASM_OPERAND_REG(reg) \ + [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg)), +#define ASM_OPERAND_NUM(num) \ + [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num])), // Trampoline to bridge from the assembly below to the static C++ crash // callback. @@ -178,62 +167,57 @@ static void StaticCrashHandler() { } } -// Creates the trampoline with the necessary CFI information to unwind through -// to the crashing call stack: -// * Defining the CFA so that it points to the stack pointer at the point -// of crash. -// * Storing all registers at the point of crash in the stack and refer to them -// via CFI information (relative to the CFA). -// * Setting the return column so the unwinder knows how to continue unwinding. -// * (x86_64) making sure rsp is aligned before calling StaticCrashHandler. -// * Calling StaticCrashHandler that will trigger the unwinder. +// This trampoline function has the necessary CFI information to unwind +// and get a backtrace: +// * The stack contains a copy of all the registers at the point of crash, +// the code has CFI directives specifying how to restore them. +// * A call to StaticCrashHandler, which will print the stacktrace and exit +// the fuzzer, generating a crash artifact. // // The __attribute__((used)) is necessary because the function // is never called; it's just a container around the assembly to allow it to // use operands for compile-time computed constants. __attribute__((used)) void MakeTrampoline() { - __asm__(".cfi_endproc\n" - ".pushsection .text.CrashTrampolineAsm\n" - ".type CrashTrampolineAsm,STT_FUNC\n" -"CrashTrampolineAsm:\n" - ".cfi_startproc simple\n" - ".cfi_signal_frame\n" + __asm__( + ".cfi_endproc\n" + ".pushsection .text.CrashTrampolineAsm\n" + ".type CrashTrampolineAsm,STT_FUNC\n" + "CrashTrampolineAsm:\n" + ".cfi_startproc simple\n" + ".cfi_signal_frame\n" #if defined(__x86_64__) - ".cfi_return_column rip\n" - ".cfi_def_cfa rsp, %c[CFAOffset]\n" - FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) - "mov %%rsp, %%rbp\n" - ".cfi_def_cfa_register rbp\n" - "andq $-16, %%rsp\n" - "call %c[StaticCrashHandler]\n" - "ud2\n" + ".cfi_return_column rip\n" + ".cfi_def_cfa rsp, 0\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + "call %c[StaticCrashHandler]\n" + "ud2\n" #elif defined(__aarch64__) - ".cfi_return_column 33\n" - ".cfi_def_cfa sp, %c[CFAOffset]\n" - FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) - ".cfi_offset 33, %c[pc]\n" - ".cfi_offset 30, %c[lr]\n" - "bl %c[StaticCrashHandler]\n" - "brk 1\n" + ".cfi_return_column 33\n" + ".cfi_def_cfa sp, 0\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + ".cfi_offset 33, %c[pc]\n" + ".cfi_offset 30, %c[lr]\n" + "bl %c[StaticCrashHandler]\n" + "brk 1\n" #else #error "Unsupported architecture for fuzzing on Fuchsia" #endif - ".cfi_endproc\n" - ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n" - ".popsection\n" - ".cfi_startproc\n" - : // No outputs - : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM) + ".cfi_endproc\n" + ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n" + ".popsection\n" + ".cfi_startproc\n" + : // No outputs + : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM) #if defined(__aarch64__) - ASM_OPERAND_REG(pc) - ASM_OPERAND_REG(lr) + ASM_OPERAND_REG(pc) ASM_OPERAND_REG(lr) #endif - [StaticCrashHandler] "i" (StaticCrashHandler), - [CFAOffset] "i" (CFAOffset)); + [StaticCrashHandler] "i"(StaticCrashHandler)); } -void CrashHandler(zx_handle_t *Event) { +void CrashHandler() { + assert(SignalHandlerEvent != ZX_HANDLE_INVALID); + // This structure is used to ensure we close handles to objects we create in // this handler. struct ScopedHandle { @@ -251,16 +235,30 @@ void CrashHandler(zx_handle_t *Event) { Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle), "_zx_task_create_exception_channel"); - ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0), + ExitOnErr(_zx_object_signal(SignalHandlerEvent, 0, ZX_USER_SIGNAL_0), "_zx_object_signal"); // This thread lives as long as the process in order to keep handling // crashes. In practice, the first crashed thread to reach the end of the // StaticCrashHandler will end the process. while (true) { - ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE, - ZX_TIME_INFINITE, nullptr), - "_zx_object_wait_one"); + zx_wait_item_t WaitItems[] = { + { + .handle = SignalHandlerEvent, + .waitfor = ZX_SIGNAL_HANDLE_CLOSED, + .pending = 0, + }, + { + .handle = Channel.Handle, + .waitfor = ZX_CHANNEL_READABLE | ZX_CHANNEL_PEER_CLOSED, + .pending = 0, + }, + }; + auto Status = _zx_object_wait_many( + WaitItems, sizeof(WaitItems) / sizeof(WaitItems[0]), ZX_TIME_INFINITE); + if (Status != ZX_OK || (WaitItems[1].pending & ZX_CHANNEL_READABLE) == 0) { + break; + } zx_exception_info_t ExceptionInfo; ScopedHandle Exception; @@ -296,14 +294,17 @@ void CrashHandler(zx_handle_t *Event) { // onto the stack and jump into a trampoline with CFI instructions on how // to restore it. #if defined(__x86_64__) - uintptr_t StackPtr = GeneralRegisters.rsp - CFAOffset; + uintptr_t StackPtr = + (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) & + -(uintptr_t)16; __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, sizeof(GeneralRegisters)); GeneralRegisters.rsp = StackPtr; GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); #elif defined(__aarch64__) - uintptr_t StackPtr = GeneralRegisters.sp - CFAOffset; + uintptr_t StackPtr = + (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16; __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, sizeof(GeneralRegisters)); GeneralRegisters.sp = StackPtr; @@ -327,6 +328,13 @@ void CrashHandler(zx_handle_t *Event) { } } +void StopSignalHandler() { + _zx_handle_close(SignalHandlerEvent); + if (SignalHandler.joinable()) { + SignalHandler.join(); + } +} + } // namespace // Platform specific functions. @@ -356,16 +364,14 @@ void SetSignalHandler(const FuzzingOptions &Options) { return; // Set up the crash handler and wait until it is ready before proceeding. - zx_handle_t Event; - ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create"); + ExitOnErr(_zx_event_create(0, &SignalHandlerEvent), "_zx_event_create"); - std::thread T(CrashHandler, &Event); - zx_status_t Status = - _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr); - _zx_handle_close(Event); + SignalHandler = std::thread(CrashHandler); + zx_status_t Status = _zx_object_wait_one(SignalHandlerEvent, ZX_USER_SIGNAL_0, + ZX_TIME_INFINITE, nullptr); ExitOnErr(Status, "_zx_object_wait_one"); - T.detach(); + std::atexit(StopSignalHandler); } void SleepSeconds(int Seconds) { diff --git a/libfuzzer/FuzzerUtilWindows.cpp b/libfuzzer/FuzzerUtilWindows.cpp index 1a54bb5..3598758 100644 --- a/libfuzzer/FuzzerUtilWindows.cpp +++ b/libfuzzer/FuzzerUtilWindows.cpp @@ -204,7 +204,7 @@ const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, } std::string DisassembleCmd(const std::string &FileName) { - Vector<std::string> command_vector; + std::vector<std::string> command_vector; command_vector.push_back("dumpbin /summary > nul"); if (ExecuteCommand(Command(command_vector)) == 0) return "dumpbin /disasm " + FileName; diff --git a/libfuzzer/dataflow/DataFlow.cpp b/libfuzzer/dataflow/DataFlow.cpp index 78b3f9a..7e5f041 100644 --- a/libfuzzer/dataflow/DataFlow.cpp +++ b/libfuzzer/dataflow/DataFlow.cpp @@ -17,11 +17,9 @@ // and also provides basic-block coverage for every input. // // Build: -// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow -mllvm -// -dfsan-fast-16-labels and -O2. +// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow and -O2. // 2. Compile DataFlowCallbacks.cpp with -O2 -fPIC. // 3. Build the fuzz target with -g -fsanitize=dataflow -// -mllvm -dfsan-fast-16-labels // -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp // 4. Link those together with -fsanitize=dataflow // @@ -82,7 +80,7 @@ static inline bool BlockIsEntry(size_t BlockIdx) { return __dft.PCsBeg[BlockIdx * 2 + 1] & PCFLAG_FUNC_ENTRY; } -const int kNumLabels = 16; +const int kNumLabels = 8; // Prints all instrumented functions. static int PrintFunctions() { @@ -92,8 +90,8 @@ static int PrintFunctions() { // We'll need to make a proper in-process symbolizer work with DFSan. FILE *Pipe = popen("sed 's/(+/ /g; s/).*//g' " "| llvm-symbolizer " - "| grep 'dfs\\$' " - "| sed 's/dfs\\$//g' " + "| grep '\\.dfsan' " + "| sed 's/\\.dfsan//g' " "| c++filt", "w"); for (size_t I = 0; I < __dft.NumGuards; I++) { diff --git a/libfuzzer/tests/CMakeLists.txt b/libfuzzer/tests/CMakeLists.txt index 5b3e906..10fcfba 100644 --- a/libfuzzer/tests/CMakeLists.txt +++ b/libfuzzer/tests/CMakeLists.txt @@ -33,7 +33,8 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH AND COMPILER_RT_LIBCXXABI_PATH) - list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++) + list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++ -fno-exceptions) + list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -nostdlib++ -fno-exceptions) endif() if ("-fvisibility=hidden" IN_LIST LIBFUZZER_CFLAGS) @@ -73,7 +74,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch} SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE} RUNTIME ${LIBFUZZER_TEST_RUNTIME} - DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} + DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS} LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS}) set_target_properties(FuzzerUnitTests PROPERTIES @@ -83,7 +84,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) generate_compiler_rt_tests(FuzzedDataProviderTestObjects FuzzedDataProviderUnitTests "FuzzerUtils-${arch}-Test" ${arch} SOURCES FuzzedDataProviderUnittest.cpp ${COMPILER_RT_GTEST_SOURCE} - DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h + DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS} LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS}) set_target_properties(FuzzedDataProviderUnitTests PROPERTIES diff --git a/libfuzzer/tests/FuzzerUnittest.cpp b/libfuzzer/tests/FuzzerUnittest.cpp index 974a01f..5315a8e 100644 --- a/libfuzzer/tests/FuzzerUnittest.cpp +++ b/libfuzzer/tests/FuzzerUnittest.cpp @@ -24,7 +24,8 @@ using namespace fuzzer; // For now, have LLVMFuzzerTestOneInput just to make it link. -// Later we may want to make unittests that actually call LLVMFuzzerTestOneInput. +// Later we may want to make unittests that actually call +// LLVMFuzzerTestOneInput. extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { abort(); } @@ -88,7 +89,7 @@ TEST(Fuzzer, CrossOver) { { 0, 5, 6, 7, 1, 2 } }; for (size_t Len = 1; Len < 8; Len++) { - Set<Unit> FoundUnits, ExpectedUnitsWitThisLength; + std::set<Unit> FoundUnits, ExpectedUnitsWitThisLength; for (int Iter = 0; Iter < 3000; Iter++) { C.resize(Len); size_t NewSize = MD->CrossOver(A.data(), A.size(), B.data(), B.size(), @@ -242,7 +243,8 @@ void TestInsertRepeatedBytes(Mutator M, int NumIter) { } TEST(FuzzerMutate, InsertRepeatedBytes1) { - TestInsertRepeatedBytes(&MutationDispatcher::Mutate_InsertRepeatedBytes, 10000); + TestInsertRepeatedBytes(&MutationDispatcher::Mutate_InsertRepeatedBytes, + 10000); } TEST(FuzzerMutate, InsertRepeatedBytes2) { TestInsertRepeatedBytes(&MutationDispatcher::Mutate, 300000); @@ -557,7 +559,7 @@ TEST(FuzzerDictionary, ParseOneDictionaryEntry) { } TEST(FuzzerDictionary, ParseDictionaryFile) { - Vector<Unit> Units; + std::vector<Unit> Units; EXPECT_FALSE(ParseDictionaryFile("zzz\n", &Units)); EXPECT_FALSE(ParseDictionaryFile("", &Units)); EXPECT_TRUE(ParseDictionaryFile("\n", &Units)); @@ -569,11 +571,11 @@ TEST(FuzzerDictionary, ParseDictionaryFile) { EXPECT_TRUE(ParseDictionaryFile(" #zzzz\n", &Units)); EXPECT_EQ(Units.size(), 0U); EXPECT_TRUE(ParseDictionaryFile(" #zzzz\naaa=\"aa\"", &Units)); - EXPECT_EQ(Units, Vector<Unit>({Unit({'a', 'a'})})); + EXPECT_EQ(Units, std::vector<Unit>({Unit({'a', 'a'})})); EXPECT_TRUE( ParseDictionaryFile(" #zzzz\naaa=\"aa\"\n\nabc=\"abc\"", &Units)); EXPECT_EQ(Units, - Vector<Unit>({Unit({'a', 'a'}), Unit({'a', 'b', 'c'})})); + std::vector<Unit>({Unit({'a', 'a'}), Unit({'a', 'b', 'c'})})); } TEST(FuzzerUtil, Base64) { @@ -589,6 +591,42 @@ TEST(FuzzerUtil, Base64) { EXPECT_EQ("YWJjeHl6", Base64({'a', 'b', 'c', 'x', 'y', 'z'})); } +#ifdef __GLIBC__ +class PrintfCapture { + public: + PrintfCapture() { + OldOutputFile = GetOutputFile(); + SetOutputFile(open_memstream(&Buffer, &Size)); + } + ~PrintfCapture() { + fclose(GetOutputFile()); + SetOutputFile(OldOutputFile); + free(Buffer); + } + std::string str() { return std::string(Buffer, Size); } + + private: + char *Buffer; + size_t Size; + FILE *OldOutputFile; +}; + +TEST(FuzzerUtil, PrintASCII) { + auto f = [](const char *Str, const char *PrintAfter = "") { + PrintfCapture Capture; + PrintASCII(reinterpret_cast<const uint8_t*>(Str), strlen(Str), PrintAfter); + return Capture.str(); + }; + EXPECT_EQ("hello", f("hello")); + EXPECT_EQ("c:\\\\", f("c:\\")); + EXPECT_EQ("\\\"hi\\\"", f("\"hi\"")); + EXPECT_EQ("\\011a", f("\ta")); + EXPECT_EQ("\\0111", f("\t1")); + EXPECT_EQ("hello\\012", f("hello\n")); + EXPECT_EQ("hello\n", f("hello", "\n")); +} +#endif + TEST(Corpus, Distribution) { DataFlowTrace DFT; Random Rand(0); @@ -604,7 +642,7 @@ TEST(Corpus, Distribution) { /*FeatureSet*/ {}, DFT, /*BaseII*/ nullptr); - Vector<size_t> Hist(N); + std::vector<size_t> Hist(N); for (size_t i = 0; i < N * TriesPerUnit; i++) { Hist[C->ChooseUnitIdxToMutate(Rand)]++; } @@ -614,19 +652,60 @@ TEST(Corpus, Distribution) { } } -template <typename T> void EQ(const Vector<T> &A, const Vector<T> &B) { +TEST(Corpus, Replace) { + DataFlowTrace DFT; + struct EntropicOptions Entropic = {false, 0xFF, 100, false}; + std::unique_ptr<InputCorpus> C( + new InputCorpus(/*OutputCorpus*/ "", Entropic)); + InputInfo *FirstII = + C->AddToCorpus(Unit{0x01, 0x00}, /*NumFeatures*/ 1, + /*MayDeleteFile*/ false, /*HasFocusFunction*/ false, + /*ForceAddToCorpus*/ false, + /*TimeOfUnit*/ std::chrono::microseconds(1234), + /*FeatureSet*/ {}, DFT, + /*BaseII*/ nullptr); + InputInfo *SecondII = + C->AddToCorpus(Unit{0x02}, /*NumFeatures*/ 1, + /*MayDeleteFile*/ false, /*HasFocusFunction*/ false, + /*ForceAddToCorpus*/ false, + /*TimeOfUnit*/ std::chrono::microseconds(5678), + /*FeatureSet*/ {}, DFT, + /*BaseII*/ nullptr); + Unit ReplacedU = Unit{0x03}; + + C->Replace(FirstII, ReplacedU, + /*TimeOfUnit*/ std::chrono::microseconds(321)); + + // FirstII should be replaced. + EXPECT_EQ(FirstII->U, Unit{0x03}); + EXPECT_EQ(FirstII->Reduced, true); + EXPECT_EQ(FirstII->TimeOfUnit, std::chrono::microseconds(321)); + std::vector<uint8_t> ExpectedSha1(kSHA1NumBytes); + ComputeSHA1(ReplacedU.data(), ReplacedU.size(), ExpectedSha1.data()); + std::vector<uint8_t> IISha1(FirstII->Sha1, FirstII->Sha1 + kSHA1NumBytes); + EXPECT_EQ(IISha1, ExpectedSha1); + + // SecondII should not be replaced. + EXPECT_EQ(SecondII->U, Unit{0x02}); + EXPECT_EQ(SecondII->Reduced, false); + EXPECT_EQ(SecondII->TimeOfUnit, std::chrono::microseconds(5678)); +} + +template <typename T> +void EQ(const std::vector<T> &A, const std::vector<T> &B) { EXPECT_EQ(A, B); } -template <typename T> void EQ(const Set<T> &A, const Vector<T> &B) { - EXPECT_EQ(A, Set<T>(B.begin(), B.end())); +template <typename T> void EQ(const std::set<T> &A, const std::vector<T> &B) { + EXPECT_EQ(A, std::set<T>(B.begin(), B.end())); } -void EQ(const Vector<MergeFileInfo> &A, const Vector<std::string> &B) { - Set<std::string> a; +void EQ(const std::vector<MergeFileInfo> &A, + const std::vector<std::string> &B) { + std::set<std::string> a; for (const auto &File : A) a.insert(File.Name); - Set<std::string> b(B.begin(), B.end()); + std::set<std::string> b(B.begin(), B.end()); EXPECT_EQ(a, b); } @@ -746,9 +825,9 @@ TEST(Merger, Parse) { TEST(Merger, Merge) { Merger M; - Set<uint32_t> Features, NewFeatures; - Set<uint32_t> Cov, NewCov; - Vector<std::string> NewFiles; + std::set<uint32_t> Features, NewFeatures; + std::set<uint32_t> Cov, NewCov; + std::vector<std::string> NewFiles; // Adds new files and features EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n" @@ -861,6 +940,137 @@ TEST(Merger, Merge) { TRACED_EQ(NewFeatures, {1, 2, 3}); } +TEST(Merger, SetCoverMerge) { + Merger M; + std::set<uint32_t> Features, NewFeatures; + std::set<uint32_t> Cov, NewCov; + std::vector<std::string> NewFiles; + + // Adds new files and features + EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n" + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "FT 2 6 1 3\n" + "", + true)); + EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), + 6U); + TRACED_EQ(M.Files, {"A", "B", "C"}); + TRACED_EQ(NewFiles, {"A", "B"}); + TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6}); + + // Doesn't return features or files in the initial corpus. + EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n" + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "FT 2 6 1 3\n" + "", + true)); + EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), + 3U); + TRACED_EQ(M.Files, {"A", "B", "C"}); + TRACED_EQ(NewFiles, {"B"}); + TRACED_EQ(NewFeatures, {4, 5, 6}); + + // No new features, so no new files + EXPECT_TRUE(M.Parse("3\n2\nA\nB\nC\n" + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "FT 2 6 1 3\n" + "", + true)); + EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), + 0U); + TRACED_EQ(M.Files, {"A", "B", "C"}); + TRACED_EQ(NewFiles, {}); + TRACED_EQ(NewFeatures, {}); + + // Can pass initial features and coverage. + Features = {1, 2, 3}; + Cov = {}; + EXPECT_TRUE(M.Parse("2\n0\nA\nB\n" + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6\n" + "", + true)); + EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), + 3U); + TRACED_EQ(M.Files, {"A", "B"}); + TRACED_EQ(NewFiles, {"B"}); + TRACED_EQ(NewFeatures, {4, 5, 6}); + Features.clear(); + Cov.clear(); + + // Prefer files with a lot of features first (C has 4 features) + // Then prefer B over A due to the smaller size. After choosing C and B, + // A and D have no new features to contribute. + EXPECT_TRUE(M.Parse("4\n0\nA\nB\nC\nD\n" + "STARTED 0 2000\n" + "FT 0 3 5 6\n" + "STARTED 1 1000\n" + "FT 1 4 5 6 \n" + "STARTED 2 1000\n" + "FT 2 1 2 3 4 \n" + "STARTED 3 500\n" + "FT 3 1 \n" + "", + true)); + EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), + 6U); + TRACED_EQ(M.Files, {"A", "B", "C", "D"}); + TRACED_EQ(NewFiles, {"C", "B"}); + TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6}); + + // Only 1 file covers all features. + EXPECT_TRUE(M.Parse("4\n1\nA\nB\nC\nD\n" + "STARTED 0 2000\n" + "FT 0 4 5 6 7 8\n" + "STARTED 1 1100\n" + "FT 1 1 2 3 \n" + "STARTED 2 1100\n" + "FT 2 2 3 \n" + "STARTED 3 1000\n" + "FT 3 1 \n" + "", + true)); + EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), + 3U); + TRACED_EQ(M.Files, {"A", "B", "C", "D"}); + TRACED_EQ(NewFiles, {"B"}); + TRACED_EQ(NewFeatures, {1, 2, 3}); + + // A Feature has a value greater than (1 << 21) and hence + // there are collisions in the underlying `covered features` + // bitvector. + EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n" + "STARTED 0 2000\n" + "FT 0 1 2 3\n" + "STARTED 1 1000\n" + "FT 1 3 4 5 \n" + "STARTED 2 1000\n" + "FT 2 3 2097153 \n" // Last feature is (2^21 + 1). + "", + true)); + EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), + 5U); + TRACED_EQ(M.Files, {"A", "B", "C"}); + // File 'C' is not added because it's last feature is considered + // covered due to collision with feature 1. + TRACED_EQ(NewFiles, {"B", "A"}); + TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5}); +} + #undef TRACED_EQ TEST(DFT, BlockCoverage) { @@ -968,7 +1178,7 @@ TEST(Fuzzer, ForEachNonZeroByte) { 0, 0, 0, 0, 0, 0, 0, 8, 9, 9, 9, 9, 9, 9, 9, 9, }; - typedef Vector<std::pair<size_t, uint8_t> > Vec; + typedef std::vector<std::pair<size_t, uint8_t>> Vec; Vec Res, Expected; auto CB = [&](size_t FirstFeature, size_t Idx, uint8_t V) { Res.push_back({FirstFeature + Idx, V}); @@ -993,7 +1203,7 @@ TEST(Fuzzer, ForEachNonZeroByte) { // FuzzerCommand unit tests. The arguments in the two helper methods below must // match. -static void makeCommandArgs(Vector<std::string> *ArgsToAdd) { +static void makeCommandArgs(std::vector<std::string> *ArgsToAdd) { assert(ArgsToAdd); ArgsToAdd->clear(); ArgsToAdd->push_back("foo"); @@ -1029,7 +1239,7 @@ TEST(FuzzerCommand, Create) { EXPECT_EQ(CmdLine, ""); // Explicit constructor - Vector<std::string> ArgsToAdd; + std::vector<std::string> ArgsToAdd; makeCommandArgs(&ArgsToAdd); Command InitializedCmd(ArgsToAdd); @@ -1061,7 +1271,7 @@ TEST(FuzzerCommand, Create) { } TEST(FuzzerCommand, ModifyArguments) { - Vector<std::string> ArgsToAdd; + std::vector<std::string> ArgsToAdd; makeCommandArgs(&ArgsToAdd); Command Cmd; std::string CmdLine; @@ -1084,7 +1294,7 @@ TEST(FuzzerCommand, ModifyArguments) { } TEST(FuzzerCommand, ModifyFlags) { - Vector<std::string> ArgsToAdd; + std::vector<std::string> ArgsToAdd; makeCommandArgs(&ArgsToAdd); Command Cmd(ArgsToAdd); std::string Value, CmdLine; @@ -1116,7 +1326,7 @@ TEST(FuzzerCommand, ModifyFlags) { } TEST(FuzzerCommand, SetOutput) { - Vector<std::string> ArgsToAdd; + std::vector<std::string> ArgsToAdd; makeCommandArgs(&ArgsToAdd); Command Cmd(ArgsToAdd); std::string CmdLine; @@ -1196,7 +1406,8 @@ TEST(Entropic, ComputeEnergy) { struct EntropicOptions Entropic = {true, 0xFF, 100, false}; std::unique_ptr<InputCorpus> C(new InputCorpus("", Entropic)); std::unique_ptr<InputInfo> II(new InputInfo()); - Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs = {{1, 3}, {2, 3}, {3, 3}}; + std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs = { + {1, 3}, {2, 3}, {3, 3}}; II->FeatureFreqs = FeatureFreqs; II->NumExecutedMutations = 0; II->UpdateEnergy(4, false, std::chrono::microseconds(0)); @@ -14,10 +14,41 @@ pub use arbitrary; use once_cell::sync::OnceCell; +/// Indicates whether the input should be kept in the corpus or rejected. This +/// should be returned by your fuzz target. If your fuzz target does not return +/// a value (i.e., returns `()`), then the input will be kept in the corpus. +#[derive(Debug)] +pub enum Corpus { + /// Keep the input in the corpus. + Keep, + + /// Reject the input and do not keep it in the corpus. + Reject, +} + +impl From<()> for Corpus { + fn from(_: ()) -> Self { + Self::Keep + } +} + +impl Corpus { + #[doc(hidden)] + /// Convert this Corpus result into the [integer codes used by + /// `libFuzzer`](https://llvm.org/docs/LibFuzzer.html#rejecting-unwanted-inputs). + /// This is -1 for reject, 0 for keep. + pub fn to_libfuzzer_code(self) -> i32 { + match self { + Corpus::Keep => 0, + Corpus::Reject => -1, + } + } +} + extern "C" { // We do not actually cross the FFI bound here. #[allow(improper_ctypes)] - fn rust_fuzzer_test_input(input: &[u8]); + fn rust_fuzzer_test_input(input: &[u8]) -> i32; fn LLVMFuzzerMutate(data: *mut u8, size: usize, max_size: usize) -> usize; } @@ -27,14 +58,17 @@ extern "C" { pub fn test_input_wrap(data: *const u8, size: usize) -> i32 { let test_input = ::std::panic::catch_unwind(|| unsafe { let data_slice = ::std::slice::from_raw_parts(data, size); - rust_fuzzer_test_input(data_slice); + rust_fuzzer_test_input(data_slice) }); - if test_input.err().is_some() { - // hopefully the custom panic hook will be called before and abort the - // process before the stack frames are unwinded. - ::std::process::abort(); + + match test_input { + Ok(i) => i, + Err(_) => { + // hopefully the custom panic hook will be called before and abort the + // process before the stack frames are unwinded. + ::std::process::abort(); + } } - 0 } #[doc(hidden)] @@ -86,6 +120,39 @@ pub fn initialize(_argc: *const isize, _argv: *const *const *const u8) -> isize /// # mod my_crate { pub fn parse(_: &[u8]) -> Result<(), ()> { unimplemented!() } } /// ``` /// +/// ## Rejecting Inputs +/// +/// It may be desirable to reject some inputs, i.e. to not add them to the +/// corpus. +/// +/// For example, when fuzzing an API consisting of parsing and other logic, +/// one may want to allow only those inputs into the corpus that parse +/// successfully. To indicate whether an input should be kept in or rejected +/// from the corpus, return either [Corpus::Keep] or [Corpus::Reject] from your +/// fuzz target. The default behavior (e.g. if `()` is returned) is to keep the +/// input in the corpus. +/// +/// For example: +/// +/// ```no_run +/// #![no_main] +/// +/// use libfuzzer_sys::{Corpus, fuzz_target}; +/// +/// fuzz_target!(|input: String| -> Corpus { +/// let parts: Vec<&str> = input.splitn(2, '=').collect(); +/// if parts.len() != 2 { +/// return Corpus::Reject; +/// } +/// +/// let key = parts[0]; +/// let value = parts[1]; +/// let _result: Result<_, _> = my_crate::parse(key, value); +/// Corpus::Keep +/// }); +/// # mod my_crate { pub fn parse(_key: &str, _value: &str) -> Result<(), ()> { unimplemented!() } } +/// ``` +/// /// ## Arbitrary Input Types /// /// The input is a `&[u8]` slice by default, but you can take arbitrary input @@ -136,74 +203,107 @@ pub fn initialize(_argc: *const isize, _argv: *const *const *const u8) -> isize #[macro_export] macro_rules! fuzz_target { (|$bytes:ident| $body:block) => { - /// Auto-generated function - #[no_mangle] - pub extern "C" fn rust_fuzzer_test_input($bytes: &[u8]) { - // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug - // formatting of the input to that file. This is only intended for - // `cargo fuzz`'s use! + const _: () = { + /// Auto-generated function + #[no_mangle] + pub extern "C" fn rust_fuzzer_test_input(bytes: &[u8]) -> i32 { + // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug + // formatting of the input to that file. This is only intended for + // `cargo fuzz`'s use! - // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization. - if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() { - use std::io::Write; - let mut file = std::fs::File::create(path) - .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file"); - writeln!(&mut file, "{:?}", $bytes) - .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file"); - return; + // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization. + if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() { + use std::io::Write; + let mut file = std::fs::File::create(path) + .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file"); + writeln!(&mut file, "{:?}", bytes) + .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file"); + return 0; + } + + __libfuzzer_sys_run(bytes); + 0 } - $body - } + // Split out the actual fuzzer into a separate function which is + // tagged as never being inlined. This ensures that if the fuzzer + // panics there's at least one stack frame which is named uniquely + // according to this specific fuzzer that this is embedded within. + // + // Systems like oss-fuzz try to deduplicate crashes and without this + // panics in separate fuzzers can accidentally appear the same + // because each fuzzer will have a function called + // `rust_fuzzer_test_input`. By using a normal Rust function here + // it's named something like `the_fuzzer_name::_::__libfuzzer_sys_run` which should + // ideally help prevent oss-fuzz from deduplicate fuzz bugs across + // distinct targets accidentally. + #[inline(never)] + fn __libfuzzer_sys_run($bytes: &[u8]) { + $body + } + }; }; (|$data:ident: &[u8]| $body:block) => { - fuzz_target!(|$data| $body); + $crate::fuzz_target!(|$data| $body); }; (|$data:ident: $dty: ty| $body:block) => { - /// Auto-generated function - #[no_mangle] - pub extern "C" fn rust_fuzzer_test_input(bytes: &[u8]) { - use $crate::arbitrary::{Arbitrary, Unstructured}; + $crate::fuzz_target!(|$data: $dty| -> () $body); + }; - // Early exit if we don't have enough bytes for the `Arbitrary` - // implementation. This helps the fuzzer avoid exploring all the - // different not-enough-input-bytes paths inside the `Arbitrary` - // implementation. Additionally, it exits faster, letting the fuzzer - // get to longer inputs that actually lead to interesting executions - // quicker. - if bytes.len() < <$dty as Arbitrary>::size_hint(0).0 { - return; - } + (|$data:ident: $dty: ty| -> $rty: ty $body:block) => { + const _: () = { + /// Auto-generated function + #[no_mangle] + pub extern "C" fn rust_fuzzer_test_input(bytes: &[u8]) -> i32 { + use $crate::arbitrary::{Arbitrary, Unstructured}; - let mut u = Unstructured::new(bytes); - let data = <$dty as Arbitrary>::arbitrary_take_rest(u); + // Early exit if we don't have enough bytes for the `Arbitrary` + // implementation. This helps the fuzzer avoid exploring all the + // different not-enough-input-bytes paths inside the `Arbitrary` + // implementation. Additionally, it exits faster, letting the fuzzer + // get to longer inputs that actually lead to interesting executions + // quicker. + if bytes.len() < <$dty as Arbitrary>::size_hint(0).0 { + return -1; + } - // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug - // formatting of the input to that file. This is only intended for - // `cargo fuzz`'s use! + let mut u = Unstructured::new(bytes); + let data = <$dty as Arbitrary>::arbitrary_take_rest(u); - // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization. - if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() { - use std::io::Write; - let mut file = std::fs::File::create(path) - .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file"); - (match data { - Ok(data) => writeln!(&mut file, "{:#?}", data), - Err(err) => writeln!(&mut file, "Arbitrary Error: {}", err), - }) - .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file"); - return; - } + // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug + // formatting of the input to that file. This is only intended for + // `cargo fuzz`'s use! + + // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization. + if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() { + use std::io::Write; + let mut file = std::fs::File::create(path) + .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file"); + (match data { + Ok(data) => writeln!(&mut file, "{:#?}", data), + Err(err) => writeln!(&mut file, "Arbitrary Error: {}", err), + }) + .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file"); + return -1; + } - let $data = match data { - Ok(d) => d, - Err(_) => return, - }; + let data = match data { + Ok(d) => d, + Err(_) => return -1, + }; - $body - } + let result = ::libfuzzer_sys::Corpus::from(__libfuzzer_sys_run(data)); + result.to_libfuzzer_code() + } + + // See above for why this is split to a separate function. + #[inline(never)] + fn __libfuzzer_sys_run($data: $dty) -> $rty { + $body + } + }; }; } |