From d1325ea49959305aec6fa54768f6d694aae3e4b7 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Thu, 1 Apr 2021 17:09:16 -0700 Subject: Upgrade rust/crates/libfuzzer-sys to 0.4.0 Test: make Change-Id: I0411b0aba37380808fc84ad2dfcde2f57560e789 --- libfuzzer/CMakeLists.txt | 8 ++-- libfuzzer/FuzzerCorpus.h | 62 ++++++++++++++++++++++--- libfuzzer/FuzzerDataFlowTrace.cpp | 2 +- libfuzzer/FuzzerDriver.cpp | 87 ++++++++++++++++++++++++++++++------ libfuzzer/FuzzerExtFunctionsWeak.cpp | 2 +- libfuzzer/FuzzerExtraCounters.cpp | 2 +- libfuzzer/FuzzerFlags.def | 37 ++++++++++++++- libfuzzer/FuzzerFork.cpp | 14 +++--- libfuzzer/FuzzerIO.cpp | 45 +++++++++++++++++++ libfuzzer/FuzzerIO.h | 6 +++ libfuzzer/FuzzerIOPosix.cpp | 8 +++- libfuzzer/FuzzerIOWindows.cpp | 14 +++++- libfuzzer/FuzzerInterceptors.cpp | 20 ++++++++- libfuzzer/FuzzerInternal.h | 4 +- libfuzzer/FuzzerLoop.cpp | 69 +++++++++++++++++++++++----- libfuzzer/FuzzerMutate.cpp | 40 ++++++++++++----- libfuzzer/FuzzerMutate.h | 14 +++--- libfuzzer/FuzzerOptions.h | 9 +++- libfuzzer/FuzzerPlatform.h | 18 +------- libfuzzer/FuzzerTracePC.cpp | 34 ++++++++++---- libfuzzer/FuzzerTracePC.h | 6 ++- libfuzzer/FuzzerUtil.h | 6 +++ libfuzzer/FuzzerUtilFuchsia.cpp | 19 +------- libfuzzer/FuzzerUtilLinux.cpp | 2 +- libfuzzer/FuzzerUtilPosix.cpp | 4 +- libfuzzer/FuzzerUtilWindows.cpp | 14 ++++-- libfuzzer/afl/afl_driver.cpp | 10 ----- libfuzzer/dataflow/DataFlow.cpp | 6 ++- libfuzzer/tests/CMakeLists.txt | 4 ++ libfuzzer/tests/FuzzerUnittest.cpp | 20 +++++---- 30 files changed, 444 insertions(+), 142 deletions(-) (limited to 'libfuzzer') diff --git a/libfuzzer/CMakeLists.txt b/libfuzzer/CMakeLists.txt index b98c271..3201ed2 100644 --- a/libfuzzer/CMakeLists.txt +++ b/libfuzzer/CMakeLists.txt @@ -55,15 +55,15 @@ CHECK_CXX_SOURCE_COMPILES(" } " HAS_THREAD_LOCAL) -set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +set(LIBFUZZER_CFLAGS ${COMPILER_RT_COMMON_CFLAGS}) if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH AND COMPILER_RT_LIBCXXABI_PATH) - list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer) - # Remove -stdlib= which is unused when passing -nostdinc++. - string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + list(APPEND LIBFUZZER_CFLAGS -D_LIBCPP_ABI_VERSION=Fuzzer) + append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ LIBFUZZER_CFLAGS) elseif(TARGET cxx-headers OR HAVE_LIBCXX) + # libFuzzer uses C++ standard library headers. set(LIBFUZZER_DEPS cxx-headers) endif() diff --git a/libfuzzer/FuzzerCorpus.h b/libfuzzer/FuzzerCorpus.h index 54d1e09..daea4f5 100644 --- a/libfuzzer/FuzzerCorpus.h +++ b/libfuzzer/FuzzerCorpus.h @@ -18,6 +18,7 @@ #include "FuzzerSHA1.h" #include "FuzzerTracePC.h" #include +#include #include #include #include @@ -26,6 +27,7 @@ namespace fuzzer { struct InputInfo { Unit U; // The actual input data. + std::chrono::microseconds TimeOfUnit; uint8_t Sha1[kSHA1NumBytes]; // Checksum. // Number of features that this input has and no smaller input has. size_t NumFeatures = 0; @@ -33,6 +35,7 @@ struct InputInfo { // Stats. size_t NumExecutedMutations = 0; size_t NumSuccessfullMutations = 0; + bool NeverReduce = false; bool MayDeleteFile = false; bool Reduced = false; bool HasFocusFunction = false; @@ -61,11 +64,15 @@ struct InputInfo { } // Assign more energy to a high-entropy seed, i.e., that reveals more - // information about the globally rare features in the neighborhood - // of the seed. Since we do not know the entropy of a seed that has - // never been executed we assign fresh seeds maximum entropy and - // let II->Energy approach the true entropy from above. - void UpdateEnergy(size_t GlobalNumberOfFeatures) { + // information about the globally rare features in the neighborhood of the + // seed. Since we do not know the entropy of a seed that has never been + // executed we assign fresh seeds maximum entropy and let II->Energy approach + // the true entropy from above. If ScalePerExecTime is true, the computed + // entropy is scaled based on how fast this input executes compared to the + // average execution time of inputs. The faster an input executes, the more + // energy gets assigned to the input. + void UpdateEnergy(size_t GlobalNumberOfFeatures, bool ScalePerExecTime, + std::chrono::microseconds AverageUnitExecutionTime) { Energy = 0.0; SumIncidence = 0; @@ -88,6 +95,27 @@ struct InputInfo { // Normalize. if (SumIncidence != 0) Energy = (Energy / SumIncidence) + logl(SumIncidence); + + if (ScalePerExecTime) { + // Scaling to favor inputs with lower execution time. + uint32_t PerfScore = 100; + if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 10) + PerfScore = 10; + else if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 4) + PerfScore = 25; + else if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 2) + PerfScore = 50; + else if (TimeOfUnit.count() * 3 > AverageUnitExecutionTime.count() * 4) + PerfScore = 75; + else if (TimeOfUnit.count() * 4 < AverageUnitExecutionTime.count()) + PerfScore = 300; + else if (TimeOfUnit.count() * 3 < AverageUnitExecutionTime.count()) + PerfScore = 200; + else if (TimeOfUnit.count() * 2 < AverageUnitExecutionTime.count()) + PerfScore = 150; + + Energy *= PerfScore; + } } // Increment the frequency of the feature Idx. @@ -120,6 +148,7 @@ struct EntropicOptions { bool Enabled; size_t NumberOfRarestFeatures; size_t FeatureFrequencyThreshold; + bool ScalePerExecTime; }; class InputCorpus { @@ -177,7 +206,8 @@ public: bool empty() const { return Inputs.empty(); } const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, - bool HasFocusFunction, + bool HasFocusFunction, bool NeverReduce, + std::chrono::microseconds TimeOfUnit, const Vector &FeatureSet, const DataFlowTrace &DFT, const InputInfo *BaseII) { assert(!U.empty()); @@ -187,6 +217,8 @@ public: InputInfo &II = *Inputs.back(); II.U = U; II.NumFeatures = NumFeatures; + II.NeverReduce = NeverReduce; + II.TimeOfUnit = TimeOfUnit; II.MayDeleteFile = MayDeleteFile; II.UniqFeatureSet = FeatureSet; II.HasFocusFunction = HasFocusFunction; @@ -268,6 +300,15 @@ public: return II; } + InputInfo &ChooseUnitToCrossOverWith(Random &Rand, bool UniformDist) { + if (!UniformDist) { + return ChooseUnitToMutate(Rand); + } + InputInfo &II = *Inputs[Rand(Inputs.size())]; + assert(!II.U.empty()); + return II; + } + // Returns an index of random unit from the corpus to mutate. size_t ChooseUnitIdxToMutate(Random &Rand) { UpdateCorpusDistribution(Rand); @@ -460,12 +501,19 @@ private: Weights.resize(N); std::iota(Intervals.begin(), Intervals.end(), 0); + std::chrono::microseconds AverageUnitExecutionTime(0); + for (auto II : Inputs) { + AverageUnitExecutionTime += II->TimeOfUnit; + } + AverageUnitExecutionTime /= N; + bool VanillaSchedule = true; if (Entropic.Enabled) { for (auto II : Inputs) { if (II->NeedsEnergyUpdate && II->Energy != 0.0) { II->NeedsEnergyUpdate = false; - II->UpdateEnergy(RareFeatures.size()); + II->UpdateEnergy(RareFeatures.size(), Entropic.ScalePerExecTime, + AverageUnitExecutionTime); } } diff --git a/libfuzzer/FuzzerDataFlowTrace.cpp b/libfuzzer/FuzzerDataFlowTrace.cpp index 48df8e6..0e9cdf7 100644 --- a/libfuzzer/FuzzerDataFlowTrace.cpp +++ b/libfuzzer/FuzzerDataFlowTrace.cpp @@ -253,7 +253,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, return 1; } - static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0"; + static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0"; putenv(DFSanEnv); MkDir(DirPath); for (auto &F : CorporaFiles) { diff --git a/libfuzzer/FuzzerDriver.cpp b/libfuzzer/FuzzerDriver.cpp index 00a33a4..447cafc 100644 --- a/libfuzzer/FuzzerDriver.cpp +++ b/libfuzzer/FuzzerDriver.cpp @@ -250,6 +250,28 @@ static void WorkerThread(const Command &BaseCmd, std::atomic *Counter, } } +static void ValidateDirectoryExists(const std::string &Path, + bool CreateDirectory) { + if (Path.empty()) { + Printf("ERROR: Provided directory path is an empty string\n"); + exit(1); + } + + if (IsDirectory(Path)) + return; + + if (CreateDirectory) { + if (!MkDirRecursive(Path)) { + Printf("ERROR: Failed to create directory \"%s\"\n", Path.c_str()); + exit(1); + } + return; + } + + Printf("ERROR: The required directory \"%s\" does not exist\n", Path.c_str()); + exit(1); +} + std::string CloneArgsWithoutX(const Vector &Args, const char *X1, const char *X2) { std::string Cmd; @@ -299,7 +321,12 @@ int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { if (MaxLen && MaxLen < U.size()) U.resize(MaxLen); F->ExecuteCallback(U.data(), U.size()); - F->TryDetectingAMemoryLeak(U.data(), U.size(), true); + if (Flags.print_full_coverage) { + // Leak detection is not needed when collecting full coverage data. + F->TPCUpdateObservedPCs(); + } else { + F->TryDetectingAMemoryLeak(U.data(), U.size(), true); + } return 0; } @@ -649,6 +676,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.Verbosity = Flags.verbosity; Options.MaxLen = Flags.max_len; Options.LenControl = Flags.len_control; + Options.KeepSeed = Flags.keep_seed; Options.UnitTimeoutSec = Flags.timeout; Options.ErrorExitCode = Flags.error_exitcode; Options.TimeoutExitCode = Flags.timeout_exitcode; @@ -657,6 +685,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.IgnoreCrashes = Flags.ignore_crashes; Options.MaxTotalTimeSec = Flags.max_total_time; Options.DoCrossOver = Flags.cross_over; + Options.CrossOverUniformDist = Flags.cross_over_uniform_dist; Options.MutateDepth = Flags.mutate_depth; Options.ReduceDepth = Flags.reduce_depth; Options.UseCounters = Flags.use_counters; @@ -678,13 +707,33 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.MallocLimitMb = Options.RssLimitMb; if (Flags.runs >= 0) Options.MaxNumberOfRuns = Flags.runs; - if (!Inputs->empty() && !Flags.minimize_crash_internal_step) - Options.OutputCorpus = (*Inputs)[0]; + if (!Inputs->empty() && !Flags.minimize_crash_internal_step) { + // Ensure output corpus assumed to be the first arbitrary argument input + // is not a path to an existing file. + std::string OutputCorpusDir = (*Inputs)[0]; + if (!IsFile(OutputCorpusDir)) { + Options.OutputCorpus = OutputCorpusDir; + ValidateDirectoryExists(Options.OutputCorpus, Flags.create_missing_dirs); + } + } Options.ReportSlowUnits = Flags.report_slow_units; - if (Flags.artifact_prefix) + if (Flags.artifact_prefix) { Options.ArtifactPrefix = Flags.artifact_prefix; - if (Flags.exact_artifact_path) + + // Since the prefix could be a full path to a file name prefix, assume + // that if the path ends with the platform's separator that a directory + // is desired + std::string ArtifactPathDir = Options.ArtifactPrefix; + if (!IsSeparator(ArtifactPathDir[ArtifactPathDir.length() - 1])) { + ArtifactPathDir = DirName(ArtifactPathDir); + } + ValidateDirectoryExists(ArtifactPathDir, Flags.create_missing_dirs); + } + if (Flags.exact_artifact_path) { Options.ExactArtifactPath = Flags.exact_artifact_path; + ValidateDirectoryExists(DirName(Options.ExactArtifactPath), + Flags.create_missing_dirs); + } Vector Dictionary; if (Flags.dict) if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) @@ -699,6 +748,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.PrintFinalStats = Flags.print_final_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; + Options.PrintFullCoverage = Flags.print_full_coverage; if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; if (Flags.exit_on_item) @@ -707,8 +757,12 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.FocusFunction = Flags.focus_function; if (Flags.data_flow_trace) Options.DataFlowTrace = Flags.data_flow_trace; - if (Flags.features_dir) + if (Flags.features_dir) { Options.FeaturesDir = Flags.features_dir; + ValidateDirectoryExists(Options.FeaturesDir, Flags.create_missing_dirs); + } + if (Flags.mutation_graph_file) + Options.MutationGraphFile = Flags.mutation_graph_file; if (Flags.collect_data_flow) Options.CollectDataFlow = Flags.collect_data_flow; if (Flags.stop_file) @@ -718,21 +772,19 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { (size_t)Flags.entropic_feature_frequency_threshold; Options.EntropicNumberOfRarestFeatures = (size_t)Flags.entropic_number_of_rarest_features; - if (Options.Entropic) { - if (!Options.FocusFunction.empty()) { - Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot " - "be used together.\n"); - exit(1); - } + Options.EntropicScalePerExecTime = Flags.entropic_scale_per_exec_time; + if (!Options.FocusFunction.empty()) + Options.Entropic = false; // FocusFunction overrides entropic scheduling. + if (Options.Entropic) Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", Options.EntropicFeatureFrequencyThreshold, Options.EntropicNumberOfRarestFeatures); - } struct EntropicOptions Entropic; Entropic.Enabled = Options.Entropic; Entropic.FeatureFrequencyThreshold = Options.EntropicFeatureFrequencyThreshold; Entropic.NumberOfRarestFeatures = Options.EntropicNumberOfRarestFeatures; + Entropic.ScalePerExecTime = Options.EntropicScalePerExecTime; unsigned Seed = Flags.seed; // Initialize Seed. @@ -767,6 +819,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { #endif // LIBFUZZER_EMSCRIPTEN Options.HandleAbrt = Flags.handle_abrt; + Options.HandleAlrm = !Flags.minimize_crash; Options.HandleBus = Flags.handle_bus; Options.HandleFpe = Flags.handle_fpe; Options.HandleIll = Flags.handle_ill; @@ -776,6 +829,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.HandleXfsz = Flags.handle_xfsz; Options.HandleUsr1 = Flags.handle_usr1; Options.HandleUsr2 = Flags.handle_usr2; + Options.HandleWinExcept = Flags.handle_winexcept; + SetSignalHandler(Options); std::atexit(Fuzzer::StaticExitCallback); @@ -858,6 +913,12 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); // Don't let F destroy itself. } +extern "C" ATTRIBUTE_INTERFACE int +LLVMFuzzerRunDriver(int *argc, char ***argv, + int (*UserCb)(const uint8_t *Data, size_t Size)) { + return FuzzerDriver(argc, argv, UserCb); +} + // Storage for global ExternalFunctions object. ExternalFunctions *EF = nullptr; diff --git a/libfuzzer/FuzzerExtFunctionsWeak.cpp b/libfuzzer/FuzzerExtFunctionsWeak.cpp index 24ddc57..3ef758d 100644 --- a/libfuzzer/FuzzerExtFunctionsWeak.cpp +++ b/libfuzzer/FuzzerExtFunctionsWeak.cpp @@ -13,7 +13,7 @@ //===----------------------------------------------------------------------===// #include "FuzzerPlatform.h" #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA || \ - LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN + LIBFUZZER_FREEBSD || LIBFUZZER_EMSCRIPTEN #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" diff --git a/libfuzzer/FuzzerExtraCounters.cpp b/libfuzzer/FuzzerExtraCounters.cpp index d36beba..04f569a 100644 --- a/libfuzzer/FuzzerExtraCounters.cpp +++ b/libfuzzer/FuzzerExtraCounters.cpp @@ -12,7 +12,7 @@ #include #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ - LIBFUZZER_OPENBSD || LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN + LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN __attribute__((weak)) extern uint8_t __start___libfuzzer_extra_counters; __attribute__((weak)) extern uint8_t __stop___libfuzzer_extra_counters; diff --git a/libfuzzer/FuzzerFlags.def b/libfuzzer/FuzzerFlags.def index 832224a..ab31da0 100644 --- a/libfuzzer/FuzzerFlags.def +++ b/libfuzzer/FuzzerFlags.def @@ -23,7 +23,21 @@ FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " "to use as an additional seed corpus. Alternatively, an \"@\" followed by " "the name of a file containing the comma-separated list.") +FUZZER_FLAG_INT(keep_seed, 0, "If 1, keep seed inputs in the corpus even if " + "they do not produce new coverage. When used with |reduce_inputs==1|, the " + "seed inputs will never be reduced. This option can be useful when seeds are" + "not properly formed for the fuzz target but still have useful snippets.") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") +FUZZER_FLAG_INT(cross_over_uniform_dist, 0, "Experimental. If 1, use a " + "uniform probability distribution when choosing inputs to cross over with. " + "Some of the inputs in the corpus may never get chosen for mutation " + "depending on the input mutation scheduling policy. With this flag, all " + "inputs, regardless of the input mutation scheduling policy, can be chosen " + "as an input to cross over with. This can be particularly useful with " + "|keep_seed==1|; all the initial seed inputs, even though they do not " + "increase coverage because they are not properly formed, will still be " + "chosen as an input to cross over with.") + FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. " @@ -74,6 +88,11 @@ FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on di "Every time a new input is added to the corpus, a corresponding file in the features_dir" " is created containing the unique features of that input." " Features are stored in binary format.") +FUZZER_FLAG_STRING(mutation_graph_file, "Saves a graph (in DOT format) to" + " mutation_graph_file. The graph contains a vertex for each input that has" + " unique coverage; directed edges are provided between parents and children" + " where the child has unique coverage, and are recorded with the type of" + " mutation that caused the child.") FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") FUZZER_FLAG_INT(use_memmem, 1, "Use hints from intercepting memmem, strstr, etc") @@ -113,6 +132,8 @@ FUZZER_FLAG_INT(print_corpus_stats, 0, "If 1, print statistics on corpus elements at exit.") FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" " at exit.") +FUZZER_FLAG_INT(print_full_coverage, 0, "If 1, print full coverage information " + "(all branches) as text at exit.") FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") @@ -124,6 +145,8 @@ FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.") FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") +FUZZER_FLAG_INT(handle_winexcept, 1, "If 1, try to intercept uncaught Windows " + "Visual C++ Exceptions.") FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " "if 2, close stderr; if 3, close both. " "Be careful, this will also close e.g. stderr of asan.") @@ -152,8 +175,9 @@ FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " FUZZER_FLAG_STRING(focus_function, "Experimental. " "Fuzzing will focus on inputs that trigger calls to this function. " "If -focus_function=auto and -data_flow_trace is used, libFuzzer " - "will choose the focus functions automatically.") -FUZZER_FLAG_INT(entropic, 0, "Experimental. Enables entropic power schedule.") + "will choose the focus functions automatically. Disables -entropic when " + "specified.") +FUZZER_FLAG_INT(entropic, 1, "Enables entropic power schedule.") FUZZER_FLAG_INT(entropic_feature_frequency_threshold, 0xFF, "Experimental. If " "entropic is enabled, all features which are observed less often than " "the specified value are considered as rare.") @@ -161,9 +185,18 @@ FUZZER_FLAG_INT(entropic_number_of_rarest_features, 100, "Experimental. If " "entropic is enabled, we keep track of the frequencies only for the " "Top-X least abundant features (union features that are considered as " "rare).") +FUZZER_FLAG_INT(entropic_scale_per_exec_time, 0, "Experimental. If 1, " + "the Entropic power schedule gets scaled based on the input execution " + "time. Inputs with lower execution time get scheduled more (up to 30x). " + "Note that, if 1, fuzzer stops from being deterministic even if a " + "non-zero random seed is given.") FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") FUZZER_FLAG_STRING(collect_data_flow, "Experimental: collect the data flow trace") + +FUZZER_FLAG_INT(create_missing_dirs, 0, "Automatically attempt to create " + "directories for arguments that would normally expect them to already " + "exist (i.e. artifact_prefix, exact_artifact_path, features_dir, corpus)") diff --git a/libfuzzer/FuzzerFork.cpp b/libfuzzer/FuzzerFork.cpp index d9e6b79..84725d2 100644 --- a/libfuzzer/FuzzerFork.cpp +++ b/libfuzzer/FuzzerFork.cpp @@ -309,11 +309,15 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, else Env.MainCorpusDir = CorpusDirs[0]; - auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); - CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, - {}, &Env.Cov, - CFPath, false); - RemoveFile(CFPath); + if (Options.KeepSeed) { + for (auto &File : SeedFiles) + Env.Files.push_back(File.File); + } else { + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, CFPath, false); + RemoveFile(CFPath); + } Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); diff --git a/libfuzzer/FuzzerIO.cpp b/libfuzzer/FuzzerIO.cpp index cbb1dbe..54a7219 100644 --- a/libfuzzer/FuzzerIO.cpp +++ b/libfuzzer/FuzzerIO.cpp @@ -77,6 +77,19 @@ void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { fclose(Out); } +void AppendToFile(const std::string &Data, const std::string &Path) { + AppendToFile(reinterpret_cast(Data.data()), Data.size(), + Path); +} + +void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path) { + FILE *Out = fopen(Path.c_str(), "a"); + if (!Out) + return; + fwrite(Data, sizeof(Data[0]), Size, Out); + fclose(Out); +} + void ReadDirToVectorOfUnits(const char *Path, Vector *V, long *Epoch, size_t MaxSize, bool ExitOnError) { long E = Epoch ? *Epoch : 0; @@ -144,6 +157,38 @@ void VPrintf(bool Verbose, const char *Fmt, ...) { fflush(OutputFile); } +static bool MkDirRecursiveInner(const std::string &Leaf) { + // Prevent chance of potential infinite recursion + if (Leaf == ".") + return true; + + const std::string &Dir = DirName(Leaf); + + if (IsDirectory(Dir)) { + MkDir(Leaf); + return IsDirectory(Leaf); + } + + bool ret = MkDirRecursiveInner(Dir); + if (!ret) { + // Give up early if a previous MkDir failed + return ret; + } + + MkDir(Leaf); + return IsDirectory(Leaf); +} + +bool MkDirRecursive(const std::string &Dir) { + if (Dir.empty()) + return false; + + if (IsDirectory(Dir)) + return true; + + return MkDirRecursiveInner(Dir); +} + void RmDirRecursive(const std::string &Dir) { IterateDirRecursive( Dir, [](const std::string &Path) {}, diff --git a/libfuzzer/FuzzerIO.h b/libfuzzer/FuzzerIO.h index 6e4368b..abd2511 100644 --- a/libfuzzer/FuzzerIO.h +++ b/libfuzzer/FuzzerIO.h @@ -29,6 +29,9 @@ void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path); void WriteToFile(const std::string &Data, const std::string &Path); void WriteToFile(const Unit &U, const std::string &Path); +void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path); +void AppendToFile(const std::string &Data, const std::string &Path); + void ReadDirToVectorOfUnits(const char *Path, Vector *V, long *Epoch, size_t MaxSize, bool ExitOnError); @@ -58,11 +61,13 @@ void RawPrint(const char *Str); // Platform specific functions: bool IsFile(const std::string &Path); +bool IsDirectory(const std::string &Path); size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector *V, bool TopDir); +bool MkDirRecursive(const std::string &Dir); void RmDirRecursive(const std::string &Dir); // Iterate files and dirs inside Dir, recursively. @@ -82,6 +87,7 @@ struct SizedFile { void GetSizedFilesFromDir(const std::string &Dir, Vector *V); char GetSeparator(); +bool IsSeparator(char C); // Similar to the basename utility: returns the file name w/o the dir prefix. std::string Basename(const std::string &Path); diff --git a/libfuzzer/FuzzerIOPosix.cpp b/libfuzzer/FuzzerIOPosix.cpp index aac85b0..4706a40 100644 --- a/libfuzzer/FuzzerIOPosix.cpp +++ b/libfuzzer/FuzzerIOPosix.cpp @@ -31,7 +31,7 @@ bool IsFile(const std::string &Path) { return S_ISREG(St.st_mode); } -static bool IsDirectory(const std::string &Path) { +bool IsDirectory(const std::string &Path) { struct stat St; if (stat(Path.c_str(), &St)) return false; @@ -104,6 +104,10 @@ char GetSeparator() { return '/'; } +bool IsSeparator(char C) { + return C == '/'; +} + FILE* OpenFile(int Fd, const char* Mode) { return fdopen(Fd, Mode); } @@ -155,7 +159,7 @@ bool IsInterestingCoverageFile(const std::string &FileName) { } void RawPrint(const char *Str) { - write(2, Str, strlen(Str)); + (void)write(2, Str, strlen(Str)); } void MkDir(const std::string &Path) { diff --git a/libfuzzer/FuzzerIOWindows.cpp b/libfuzzer/FuzzerIOWindows.cpp index 651283a..61ad35e 100644 --- a/libfuzzer/FuzzerIOWindows.cpp +++ b/libfuzzer/FuzzerIOWindows.cpp @@ -76,6 +76,18 @@ static bool IsDir(DWORD FileAttrs) { return FileAttrs & FILE_ATTRIBUTE_DIRECTORY; } +bool IsDirectory(const std::string &Path) { + DWORD Att = GetFileAttributesA(Path.c_str()); + + if (Att == INVALID_FILE_ATTRIBUTES) { + Printf("GetFileAttributesA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), GetLastError()); + return false; + } + + return IsDir(Att); +} + std::string Basename(const std::string &Path) { size_t Pos = Path.find_last_of("/\\"); if (Pos == std::string::npos) return Path; @@ -227,7 +239,7 @@ intptr_t GetHandleFromFd(int fd) { return _get_osfhandle(fd); } -static bool IsSeparator(char C) { +bool IsSeparator(char C) { return C == '\\' || C == '/'; } diff --git a/libfuzzer/FuzzerInterceptors.cpp b/libfuzzer/FuzzerInterceptors.cpp index a1a6478..b877986 100644 --- a/libfuzzer/FuzzerInterceptors.cpp +++ b/libfuzzer/FuzzerInterceptors.cpp @@ -27,7 +27,6 @@ #include #include #include // for dlsym() -#include static void *getFuncAddr(const char *name, uintptr_t wrapper_addr) { void *addr = dlsym(RTLD_NEXT, name); @@ -119,6 +118,25 @@ static char *internal_strstr(const char *haystack, const char *needle) { extern "C" { +// Weak hooks forward-declared to avoid dependency on +// . +void __sanitizer_weak_hook_memcmp(void *called_pc, const void *s1, + const void *s2, size_t n, int result); +void __sanitizer_weak_hook_strncmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result); +void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result); +void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1, + const char *s2, int result); +void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, + const char *s2, int result); +void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, + const char *s2, char *result); +void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, + const char *s2, char *result); +void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1, + const void *s2, size_t len2, void *result); + DEFINE_REAL(int, bcmp, const void *, const void *, size_t) DEFINE_REAL(int, memcmp, const void *, const void *, size_t) DEFINE_REAL(int, strncmp, const char *, const char *, size_t) diff --git a/libfuzzer/FuzzerInternal.h b/libfuzzer/FuzzerInternal.h index 31096ce..37c8a01 100644 --- a/libfuzzer/FuzzerInternal.h +++ b/libfuzzer/FuzzerInternal.h @@ -67,7 +67,9 @@ public: void ExecuteCallback(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, - InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + InputInfo *II = nullptr, bool ForceAddToCorpus = false, + bool *FoundUniqFeatures = nullptr); + void TPCUpdateObservedPCs(); // Merge Corpora[1:] into Corpora[0]. void Merge(const Vector &Corpora); diff --git a/libfuzzer/FuzzerLoop.cpp b/libfuzzer/FuzzerLoop.cpp index 02db6d2..6e3bf44 100644 --- a/libfuzzer/FuzzerLoop.cpp +++ b/libfuzzer/FuzzerLoop.cpp @@ -354,8 +354,10 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units, } void Fuzzer::PrintFinalStats() { + if (Options.PrintFullCoverage) + TPC.PrintCoverage(/*PrintAllCounters=*/true); if (Options.PrintCoverage) - TPC.PrintCoverage(); + TPC.PrintCoverage(/*PrintAllCounters=*/false); if (Options.PrintCorpusStats) Corpus.PrintStats(); if (!Options.PrintFinalStats) @@ -463,12 +465,45 @@ static void RenameFeatureSetFile(const std::string &FeaturesDir, DirPlusFile(FeaturesDir, NewFile)); } +static void WriteEdgeToMutationGraphFile(const std::string &MutationGraphFile, + const InputInfo *II, + const InputInfo *BaseII, + const std::string &MS) { + if (MutationGraphFile.empty()) + return; + + std::string Sha1 = Sha1ToString(II->Sha1); + + std::string OutputString; + + // Add a new vertex. + OutputString.append("\""); + OutputString.append(Sha1); + OutputString.append("\"\n"); + + // Add a new edge if there is base input. + if (BaseII) { + std::string BaseSha1 = Sha1ToString(BaseII->Sha1); + OutputString.append("\""); + OutputString.append(BaseSha1); + OutputString.append("\" -> \""); + OutputString.append(Sha1); + OutputString.append("\" [label=\""); + OutputString.append(MS); + OutputString.append("\"];\n"); + } + + AppendToFile(OutputString, MutationGraphFile); +} + bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, - InputInfo *II, bool *FoundUniqFeatures) { + InputInfo *II, bool ForceAddToCorpus, + bool *FoundUniqFeatures) { if (!Size) return false; ExecuteCallback(Data, Size); + auto TimeOfUnit = duration_cast(UnitStopTime - UnitStartTime); UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; @@ -478,7 +513,7 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, UniqFeatureSetTmp.push_back(Feature); if (Options.Entropic) Corpus.UpdateFeatureFrequency(II, Feature); - if (Options.ReduceInputs && II) + if (Options.ReduceInputs && II && !II->NeverReduce) if (std::binary_search(II->UniqFeatureSet.begin(), II->UniqFeatureSet.end(), Feature)) FoundUniqFeaturesOfII++; @@ -487,13 +522,16 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, *FoundUniqFeatures = FoundUniqFeaturesOfII; PrintPulseAndReportSlowInput(Data, Size); size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; - if (NumNewFeatures) { + if (NumNewFeatures || ForceAddToCorpus) { TPC.UpdateObservedPCs(); - auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, - MayDeleteFile, TPC.ObservedFocusFunction(), - UniqFeatureSetTmp, DFT, II); + auto NewII = + Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, + TPC.ObservedFocusFunction(), ForceAddToCorpus, + TimeOfUnit, UniqFeatureSetTmp, DFT, II); WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1), NewII->UniqFeatureSet); + WriteEdgeToMutationGraphFile(Options.MutationGraphFile, NewII, II, + MD.MutationSequence()); return true; } if (II && FoundUniqFeaturesOfII && @@ -509,6 +547,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, return false; } +void Fuzzer::TPCUpdateObservedPCs() { TPC.UpdateObservedPCs(); } + size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const { assert(InFuzzingThread()); *Data = CurrentUnitData; @@ -600,7 +640,7 @@ void Fuzzer::PrintStatusForNewUnit(const Unit &U, const char *Text) { PrintStats(Text, ""); if (Options.Verbosity) { Printf(" L: %zd/%zd ", U.size(), Corpus.MaxInputSize()); - MD.PrintMutationSequence(); + MD.PrintMutationSequence(Options.Verbosity >= 2); Printf("\n"); } } @@ -664,8 +704,11 @@ void Fuzzer::MutateAndTestOne() { MD.StartMutationSequence(); auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); - if (Options.DoCrossOver) - MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U); + if (Options.DoCrossOver) { + auto &CrossOverII = Corpus.ChooseUnitToCrossOverWith( + MD.GetRand(), Options.CrossOverUniformDist); + MD.SetCrossOverWith(&CrossOverII.U); + } const auto &U = II.U; memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); assert(CurrentUnitData); @@ -700,7 +743,7 @@ void Fuzzer::MutateAndTestOne() { bool FoundUniqFeatures = false; bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, - &FoundUniqFeatures); + /*ForceAddToCorpus*/ false, &FoundUniqFeatures); TryDetectingAMemoryLeak(CurrentUnitData, Size, /*DuringInitialCorpusExecution*/ false); if (NewCov) { @@ -768,7 +811,9 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector &CorporaFiles) { for (auto &SF : CorporaFiles) { auto U = FileToVector(SF.File, MaxInputLen, /*ExitOnError=*/false); assert(U.size() <= MaxInputLen); - RunOne(U.data(), U.size()); + RunOne(U.data(), U.size(), /*MayDeleteFile*/ false, /*II*/ nullptr, + /*ForceAddToCorpus*/ Options.KeepSeed, + /*FoundUniqFeatures*/ nullptr); CheckExitOnSrcPosOrItem(); TryDetectingAMemoryLeak(U.data(), U.size(), /*DuringInitialCorpusExecution*/ true); diff --git a/libfuzzer/FuzzerMutate.cpp b/libfuzzer/FuzzerMutate.cpp index 29541ea..cf34a9f 100644 --- a/libfuzzer/FuzzerMutate.cpp +++ b/libfuzzer/FuzzerMutate.cpp @@ -18,6 +18,7 @@ namespace fuzzer { const size_t Dictionary::kMaxDictSize; +static const size_t kMaxMutationsToPrint = 10; static void PrintASCII(const Word &W, const char *PrintAfter) { PrintASCII(W.data(), W.size(), PrintAfter); @@ -425,26 +426,26 @@ size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, if (!CrossOverWith) return 0; const Unit &O = *CrossOverWith; if (O.empty()) return 0; - MutateInPlaceHere.resize(MaxSize); - auto &U = MutateInPlaceHere; size_t NewSize = 0; switch(Rand(3)) { case 0: - NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size()); + MutateInPlaceHere.resize(MaxSize); + NewSize = CrossOver(Data, Size, O.data(), O.size(), + MutateInPlaceHere.data(), MaxSize); + memcpy(Data, MutateInPlaceHere.data(), NewSize); break; case 1: - NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize); + NewSize = InsertPartOf(O.data(), O.size(), Data, Size, MaxSize); if (!NewSize) - NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); + NewSize = CopyPartOf(O.data(), O.size(), Data, Size); break; case 2: - NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); + NewSize = CopyPartOf(O.data(), O.size(), Data, Size); break; default: assert(0); } assert(NewSize > 0 && "CrossOver returned empty unit"); assert(NewSize <= MaxSize && "CrossOver returned overisized unit"); - memcpy(Data, U.data(), NewSize); return NewSize; } @@ -481,19 +482,34 @@ void MutationDispatcher::PrintRecommendedDictionary() { Printf("###### End of recommended dictionary. ######\n"); } -void MutationDispatcher::PrintMutationSequence() { +void MutationDispatcher::PrintMutationSequence(bool Verbose) { Printf("MS: %zd ", CurrentMutatorSequence.size()); - for (auto M : CurrentMutatorSequence) - Printf("%s-", M.Name); + size_t EntriesToPrint = + Verbose ? CurrentMutatorSequence.size() + : std::min(kMaxMutationsToPrint, CurrentMutatorSequence.size()); + for (size_t i = 0; i < EntriesToPrint; i++) + Printf("%s-", CurrentMutatorSequence[i].Name); if (!CurrentDictionaryEntrySequence.empty()) { Printf(" DE: "); - for (auto DE : CurrentDictionaryEntrySequence) { + EntriesToPrint = Verbose ? CurrentDictionaryEntrySequence.size() + : std::min(kMaxMutationsToPrint, + CurrentDictionaryEntrySequence.size()); + for (size_t i = 0; i < EntriesToPrint; i++) { Printf("\""); - PrintASCII(DE->GetW(), "\"-"); + PrintASCII(CurrentDictionaryEntrySequence[i]->GetW(), "\"-"); } } } +std::string MutationDispatcher::MutationSequence() { + std::string MS; + for (auto M : CurrentMutatorSequence) { + MS += M.Name; + MS += "-"; + } + return MS; +} + size_t MutationDispatcher::Mutate(uint8_t *Data, size_t Size, size_t MaxSize) { return MutateImpl(Data, Size, MaxSize, Mutators); } diff --git a/libfuzzer/FuzzerMutate.h b/libfuzzer/FuzzerMutate.h index 6cbce80..fd37191 100644 --- a/libfuzzer/FuzzerMutate.h +++ b/libfuzzer/FuzzerMutate.h @@ -24,8 +24,11 @@ public: ~MutationDispatcher() {} /// Indicate that we are about to start a new sequence of mutations. void StartMutationSequence(); - /// Print the current sequence of mutations. - void PrintMutationSequence(); + /// Print the current sequence of mutations. Only prints the full sequence + /// when Verbose is true. + void PrintMutationSequence(bool Verbose = true); + /// Return the current sequence of mutations. + std::string MutationSequence(); /// Indicate that the current sequence of mutations was successful. void RecordSuccessfulMutationSequence(); /// Mutates data by invoking user-provided mutator. @@ -40,9 +43,9 @@ public: size_t Mutate_InsertByte(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by inserting several repeated bytes. size_t Mutate_InsertRepeatedBytes(uint8_t *Data, size_t Size, size_t MaxSize); - /// Mutates data by chanding one byte. + /// Mutates data by changing one byte. size_t Mutate_ChangeByte(uint8_t *Data, size_t Size, size_t MaxSize); - /// Mutates data by chanding one bit. + /// Mutates data by changing one bit. size_t Mutate_ChangeBit(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by copying/inserting a part of data into a different place. size_t Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize); @@ -126,9 +129,6 @@ public: // Dictionary provided by the user via -dict=DICT_FILE. Dictionary ManualDictionary; - // Temporary dictionary modified by the fuzzer itself, - // recreated periodically. - Dictionary TempAutoDictionary; // Persistent dictionary modified by the fuzzer, consists of // entries that led to successful discoveries in the past mutations. Dictionary PersistentAutoDictionary; diff --git a/libfuzzer/FuzzerOptions.h b/libfuzzer/FuzzerOptions.h index 9d975bd..d0c285a 100644 --- a/libfuzzer/FuzzerOptions.h +++ b/libfuzzer/FuzzerOptions.h @@ -18,6 +18,7 @@ struct FuzzingOptions { int Verbosity = 1; size_t MaxLen = 0; size_t LenControl = 1000; + bool KeepSeed = false; int UnitTimeoutSec = 300; int TimeoutExitCode = 70; int OOMExitCode = 71; @@ -30,6 +31,7 @@ struct FuzzingOptions { int RssLimitMb = 0; int MallocLimitMb = 0; bool DoCrossOver = true; + bool CrossOverUniformDist = false; int MutateDepth = 5; bool ReduceDepth = false; bool UseCounters = false; @@ -44,9 +46,10 @@ struct FuzzingOptions { size_t MaxNumberOfRuns = -1L; int ReportSlowUnits = 10; bool OnlyASCII = false; - bool Entropic = false; + bool Entropic = true; size_t EntropicFeatureFrequencyThreshold = 0xFF; size_t EntropicNumberOfRarestFeatures = 100; + bool EntropicScalePerExecTime = false; std::string OutputCorpus; std::string ArtifactPrefix = "./"; std::string ExactArtifactPath; @@ -56,6 +59,7 @@ struct FuzzingOptions { std::string DataFlowTrace; std::string CollectDataFlow; std::string FeaturesDir; + std::string MutationGraphFile; std::string StopFile; bool SaveArtifacts = true; bool PrintNEW = true; // Print a status line when new units are found; @@ -64,11 +68,13 @@ struct FuzzingOptions { bool PrintFinalStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; + bool PrintFullCoverage = false; bool DumpCoverage = false; bool DetectLeaks = true; int PurgeAllocatorIntervalSec = 1; int TraceMalloc = 0; bool HandleAbrt = false; + bool HandleAlrm = false; bool HandleBus = false; bool HandleFpe = false; bool HandleIll = false; @@ -78,6 +84,7 @@ struct FuzzingOptions { bool HandleXfsz = false; bool HandleUsr1 = false; bool HandleUsr2 = false; + bool HandleWinExcept = false; }; } // namespace fuzzer diff --git a/libfuzzer/FuzzerPlatform.h b/libfuzzer/FuzzerPlatform.h index 8befdb8..1602e67 100644 --- a/libfuzzer/FuzzerPlatform.h +++ b/libfuzzer/FuzzerPlatform.h @@ -18,7 +18,6 @@ #define LIBFUZZER_LINUX 1 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #define LIBFUZZER_WINDOWS 0 #define LIBFUZZER_EMSCRIPTEN 0 #elif __APPLE__ @@ -27,7 +26,6 @@ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #define LIBFUZZER_WINDOWS 0 #define LIBFUZZER_EMSCRIPTEN 0 #elif __NetBSD__ @@ -36,7 +34,6 @@ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_NETBSD 1 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #define LIBFUZZER_WINDOWS 0 #define LIBFUZZER_EMSCRIPTEN 0 #elif __FreeBSD__ @@ -45,16 +42,6 @@ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 1 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif __OpenBSD__ -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 1 #define LIBFUZZER_WINDOWS 0 #define LIBFUZZER_EMSCRIPTEN 0 #elif _WIN32 @@ -63,7 +50,6 @@ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #define LIBFUZZER_WINDOWS 1 #define LIBFUZZER_EMSCRIPTEN 0 #elif __Fuchsia__ @@ -72,7 +58,6 @@ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #define LIBFUZZER_WINDOWS 0 #define LIBFUZZER_EMSCRIPTEN 0 #elif __EMSCRIPTEN__ @@ -81,7 +66,6 @@ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #define LIBFUZZER_WINDOWS 0 #define LIBFUZZER_EMSCRIPTEN 1 #else @@ -101,7 +85,7 @@ #define LIBFUZZER_POSIX \ (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \ - LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) + LIBFUZZER_FREEBSD || LIBFUZZER_EMSCRIPTEN) #ifdef __x86_64 #if __has_attribute(target) diff --git a/libfuzzer/FuzzerTracePC.cpp b/libfuzzer/FuzzerTracePC.cpp index b2ca769..91e94d8 100644 --- a/libfuzzer/FuzzerTracePC.cpp +++ b/libfuzzer/FuzzerTracePC.cpp @@ -269,7 +269,7 @@ bool TracePC::ObservedFocusFunction() { return FocusFunctionCounterPtr && *FocusFunctionCounterPtr; } -void TracePC::PrintCoverage() { +void TracePC::PrintCoverage(bool PrintAllCounters) { if (!EF->__sanitizer_symbolize_pc || !EF->__sanitizer_get_module_and_offset_for_pc) { Printf("INFO: __sanitizer_symbolize_pc or " @@ -277,7 +277,7 @@ void TracePC::PrintCoverage() { " not printing coverage\n"); return; } - Printf("COVERAGE:\n"); + Printf(PrintAllCounters ? "FULL COVERAGE:\n" : "COVERAGE:\n"); auto CoveredFunctionCallback = [&](const PCTableEntry *First, const PCTableEntry *Last, uintptr_t Counter) { @@ -292,17 +292,33 @@ void TracePC::PrintCoverage() { std::string LineStr = DescribePC("%l", VisualizePC); size_t NumEdges = Last - First; Vector UncoveredPCs; + Vector CoveredPCs; for (auto TE = First; TE < Last; TE++) if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); - Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); - Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); - Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(), - LineStr.c_str()); - if (Counter) + else + CoveredPCs.push_back(TE->PC); + + if (PrintAllCounters) { + Printf("U"); for (auto PC : UncoveredPCs) - Printf(" UNCOVERED_PC: %s\n", - DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); + Printf(DescribePC(" %l", GetNextInstructionPc(PC)).c_str()); + Printf("\n"); + + Printf("C"); + for (auto PC : CoveredPCs) + Printf(DescribePC(" %l", GetNextInstructionPc(PC)).c_str()); + Printf("\n"); + } else { + Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); + Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); + Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(), + LineStr.c_str()); + if (Counter) + for (auto PC : UncoveredPCs) + Printf(" UNCOVERED_PC: %s\n", + DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); + } }; IterateCoveredFunctions(CoveredFunctionCallback); diff --git a/libfuzzer/FuzzerTracePC.h b/libfuzzer/FuzzerTracePC.h index 501f3b5..0090923 100644 --- a/libfuzzer/FuzzerTracePC.h +++ b/libfuzzer/FuzzerTracePC.h @@ -94,7 +94,7 @@ class TracePC { void PrintModuleInfo(); - void PrintCoverage(); + void PrintCoverage(bool PrintAllCounters); template void IterateCoveredFunctions(CallBack CB); @@ -194,10 +194,12 @@ size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, // Iterate by Step bytes at a time. for (; P < End; P += Step) - if (LargeType Bundle = *reinterpret_cast(P)) + if (LargeType Bundle = *reinterpret_cast(P)) { + Bundle = HostToLE(Bundle); for (size_t I = 0; I < Step; I++, Bundle >>= 8) if (uint8_t V = Bundle & 0xff) Handle8bitCounter(FirstFeature, P - Begin + I, V); + } // Iterate by 1 byte until the end. for (; P < End; P++) diff --git a/libfuzzer/FuzzerUtil.h b/libfuzzer/FuzzerUtil.h index 4ae3583..e90be08 100644 --- a/libfuzzer/FuzzerUtil.h +++ b/libfuzzer/FuzzerUtil.h @@ -106,6 +106,12 @@ inline uint8_t *RoundDownByPage(uint8_t *P) { return reinterpret_cast(X); } +#if __BYTE_ORDER == __LITTLE_ENDIAN +template T HostToLE(T X) { return X; } +#else +template T HostToLE(T X) { return Bswap(X); } +#endif + } // namespace fuzzer #endif // LLVM_FUZZER_UTIL_H diff --git a/libfuzzer/FuzzerUtilFuchsia.cpp b/libfuzzer/FuzzerUtilFuchsia.cpp index 190fb78..af43946 100644 --- a/libfuzzer/FuzzerUtilFuchsia.cpp +++ b/libfuzzer/FuzzerUtilFuchsia.cpp @@ -68,17 +68,6 @@ void AlarmHandler(int Seconds) { } } -void InterruptHandler() { - fd_set readfds; - // Ctrl-C sends ETX in Zircon. - do { - FD_ZERO(&readfds); - FD_SET(STDIN_FILENO, &readfds); - select(STDIN_FILENO + 1, &readfds, nullptr, nullptr, nullptr); - } while(!FD_ISSET(STDIN_FILENO, &readfds) || getchar() != 0x03); - Fuzzer::StaticInterruptCallback(); -} - // CFAOffset is used to reference the stack pointer before entering the // trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping // to the trampoline we copy all the registers onto the stack. We need to make @@ -354,16 +343,12 @@ void SetSignalHandler(const FuzzingOptions &Options) { Printf("%s", Buf); // Set up alarm handler if needed. - if (Options.UnitTimeoutSec > 0) { + if (Options.HandleAlrm && Options.UnitTimeoutSec > 0) { std::thread T(AlarmHandler, Options.UnitTimeoutSec / 2 + 1); T.detach(); } - // Set up interrupt handler if needed. - if (Options.HandleInt || Options.HandleTerm) { - std::thread T(InterruptHandler); - T.detach(); - } + // Options.HandleInt and Options.HandleTerm are not supported on Fuchsia // Early exit if no crash handler needed. if (!Options.HandleSegv && !Options.HandleBus && !Options.HandleIll && diff --git a/libfuzzer/FuzzerUtilLinux.cpp b/libfuzzer/FuzzerUtilLinux.cpp index 95490b9..981f9a8 100644 --- a/libfuzzer/FuzzerUtilLinux.cpp +++ b/libfuzzer/FuzzerUtilLinux.cpp @@ -9,7 +9,7 @@ //===----------------------------------------------------------------------===// #include "FuzzerPlatform.h" #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ - LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN + LIBFUZZER_EMSCRIPTEN #include "FuzzerCommand.h" #include diff --git a/libfuzzer/FuzzerUtilPosix.cpp b/libfuzzer/FuzzerUtilPosix.cpp index fc57b72..afb7334 100644 --- a/libfuzzer/FuzzerUtilPosix.cpp +++ b/libfuzzer/FuzzerUtilPosix.cpp @@ -113,7 +113,7 @@ void SetTimer(int Seconds) { void SetSignalHandler(const FuzzingOptions& Options) { // setitimer is not implemented in emscripten. - if (Options.UnitTimeoutSec > 0 && !LIBFUZZER_EMSCRIPTEN) + if (Options.HandleAlrm && Options.UnitTimeoutSec > 0 && !LIBFUZZER_EMSCRIPTEN) SetTimer(Options.UnitTimeoutSec / 2 + 1); if (Options.HandleInt) SetSigaction(SIGINT, InterruptHandler); @@ -148,7 +148,7 @@ size_t GetPeakRSSMb() { if (getrusage(RUSAGE_SELF, &usage)) return 0; if (LIBFUZZER_LINUX || LIBFUZZER_FREEBSD || LIBFUZZER_NETBSD || - LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) { + LIBFUZZER_EMSCRIPTEN) { // ru_maxrss is in KiB return usage.ru_maxrss >> 10; } else if (LIBFUZZER_APPLE) { diff --git a/libfuzzer/FuzzerUtilWindows.cpp b/libfuzzer/FuzzerUtilWindows.cpp index 6c693e3..1a54bb5 100644 --- a/libfuzzer/FuzzerUtilWindows.cpp +++ b/libfuzzer/FuzzerUtilWindows.cpp @@ -60,7 +60,15 @@ static LONG CALLBACK ExceptionHandler(PEXCEPTION_POINTERS ExceptionInfo) { if (HandlerOpt->HandleFpe) Fuzzer::StaticCrashSignalCallback(); break; - // TODO: handle (Options.HandleXfsz) + // This is an undocumented exception code corresponding to a Visual C++ + // Exception. + // + // See: https://devblogs.microsoft.com/oldnewthing/20100730-00/?p=13273 + case 0xE06D7363: + if (HandlerOpt->HandleWinExcept) + Fuzzer::StaticCrashSignalCallback(); + break; + // TODO: Handle (Options.HandleXfsz) } return EXCEPTION_CONTINUE_SEARCH; } @@ -115,7 +123,7 @@ static void CrashHandler(int) { Fuzzer::StaticCrashSignalCallback(); } void SetSignalHandler(const FuzzingOptions& Options) { HandlerOpt = &Options; - if (Options.UnitTimeoutSec > 0) + if (Options.HandleAlrm && Options.UnitTimeoutSec > 0) Timer.SetTimer(Options.UnitTimeoutSec / 2 + 1); if (Options.HandleInt || Options.HandleTerm) @@ -127,7 +135,7 @@ void SetSignalHandler(const FuzzingOptions& Options) { } if (Options.HandleSegv || Options.HandleBus || Options.HandleIll || - Options.HandleFpe) + Options.HandleFpe || Options.HandleWinExcept) SetUnhandledExceptionFilter(ExceptionHandler); if (Options.HandleAbrt) diff --git a/libfuzzer/afl/afl_driver.cpp b/libfuzzer/afl/afl_driver.cpp index 457f180..52aede7 100644 --- a/libfuzzer/afl/afl_driver.cpp +++ b/libfuzzer/afl/afl_driver.cpp @@ -60,31 +60,21 @@ If 1, close stdout at startup. If 2 close stderr; if 3 close both. #define LIBFUZZER_APPLE 0 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #elif __APPLE__ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_APPLE 1 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #elif __NetBSD__ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_APPLE 0 #define LIBFUZZER_NETBSD 1 #define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 #elif __FreeBSD__ #define LIBFUZZER_LINUX 0 #define LIBFUZZER_APPLE 0 #define LIBFUZZER_NETBSD 0 #define LIBFUZZER_FREEBSD 1 -#define LIBFUZZER_OPENBSD 0 -#elif __OpenBSD__ -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 1 #else #error "Support for your platform has not been implemented" #endif diff --git a/libfuzzer/dataflow/DataFlow.cpp b/libfuzzer/dataflow/DataFlow.cpp index 8bf4e25..78b3f9a 100644 --- a/libfuzzer/dataflow/DataFlow.cpp +++ b/libfuzzer/dataflow/DataFlow.cpp @@ -17,9 +17,11 @@ // and also provides basic-block coverage for every input. // // Build: -// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow and -O2. +// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow -mllvm +// -dfsan-fast-16-labels and -O2. // 2. Compile DataFlowCallbacks.cpp with -O2 -fPIC. // 3. Build the fuzz target with -g -fsanitize=dataflow +// -mllvm -dfsan-fast-16-labels // -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp // 4. Link those together with -fsanitize=dataflow // @@ -36,7 +38,7 @@ // Run: // # Collect data flow and coverage for INPUT_FILE // # write to OUTPUT_FILE (default: stdout) -// export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0 +// export DFSAN_OPTIONS=warn_unimplemented=0 // ./a.out INPUT_FILE [OUTPUT_FILE] // // # Print all instrumented functions. llvm-symbolizer must be present in PATH diff --git a/libfuzzer/tests/CMakeLists.txt b/libfuzzer/tests/CMakeLists.txt index cfb039c..5b3e906 100644 --- a/libfuzzer/tests/CMakeLists.txt +++ b/libfuzzer/tests/CMakeLists.txt @@ -23,6 +23,10 @@ list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS --driver-mode=g++) if(WIN32) list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -Wl,-defaultlib:libcmt,-defaultlib:oldnames) else() + if (APPLE) + list(APPEND LIBFUZZER_UNITTEST_CFLAGS -isysroot ${DARWIN_osx_SYSROOT}) + list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -isysroot ${DARWIN_osx_SYSROOT}) + endif() list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lpthread) endif() diff --git a/libfuzzer/tests/FuzzerUnittest.cpp b/libfuzzer/tests/FuzzerUnittest.cpp index 0e9435a..d2b5cbb 100644 --- a/libfuzzer/tests/FuzzerUnittest.cpp +++ b/libfuzzer/tests/FuzzerUnittest.cpp @@ -592,13 +592,17 @@ TEST(FuzzerUtil, Base64) { TEST(Corpus, Distribution) { DataFlowTrace DFT; Random Rand(0); - struct EntropicOptions Entropic = {false, 0xFF, 100}; + struct EntropicOptions Entropic = {false, 0xFF, 100, false}; std::unique_ptr C(new InputCorpus("", Entropic)); size_t N = 10; size_t TriesPerUnit = 1<<16; for (size_t i = 0; i < N; i++) - C->AddToCorpus(Unit{static_cast(i)}, 1, false, false, {}, DFT, - nullptr); + C->AddToCorpus(Unit{static_cast(i)}, /*NumFeatures*/ 1, + /*MayDeleteFile*/ false, /*HasFocusFunction*/ false, + /*ForceAddToCorpus*/ false, + /*TimeOfUnit*/ std::chrono::microseconds(0), + /*FeatureSet*/ {}, DFT, + /*BaseII*/ nullptr); Vector Hist(N); for (size_t i = 0; i < N * TriesPerUnit; i++) { @@ -1056,7 +1060,7 @@ TEST(Entropic, UpdateFrequency) { const size_t FeatIdx1 = 0, FeatIdx2 = 42, FeatIdx3 = 12, FeatIdx4 = 26; size_t Index; // Create input corpus with default entropic configuration - struct EntropicOptions Entropic = {true, 0xFF, 100}; + struct EntropicOptions Entropic = {true, 0xFF, 100, false}; std::unique_ptr C(new InputCorpus("", Entropic)); std::unique_ptr II(new InputInfo()); @@ -1093,23 +1097,23 @@ double SubAndSquare(double X, double Y) { TEST(Entropic, ComputeEnergy) { const double Precision = 0.01; - struct EntropicOptions Entropic = {true, 0xFF, 100}; + struct EntropicOptions Entropic = {true, 0xFF, 100, false}; std::unique_ptr C(new InputCorpus("", Entropic)); std::unique_ptr II(new InputInfo()); Vector> FeatureFreqs = {{1, 3}, {2, 3}, {3, 3}}; II->FeatureFreqs = FeatureFreqs; II->NumExecutedMutations = 0; - II->UpdateEnergy(4); + II->UpdateEnergy(4, false, std::chrono::microseconds(0)); EXPECT_LT(SubAndSquare(II->Energy, 1.450805), Precision); II->NumExecutedMutations = 9; - II->UpdateEnergy(5); + II->UpdateEnergy(5, false, std::chrono::microseconds(0)); EXPECT_LT(SubAndSquare(II->Energy, 1.525496), Precision); II->FeatureFreqs[0].second++; II->FeatureFreqs.push_back(std::pair(42, 6)); II->NumExecutedMutations = 20; - II->UpdateEnergy(10); + II->UpdateEnergy(10, false, std::chrono::microseconds(0)); EXPECT_LT(SubAndSquare(II->Energy, 1.792831), Precision); } -- cgit v1.2.3