aboutsummaryrefslogtreecommitdiff
path: root/libfuzzer/FuzzerFork.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'libfuzzer/FuzzerFork.cpp')
-rw-r--r--libfuzzer/FuzzerFork.cpp124
1 files changed, 100 insertions, 24 deletions
diff --git a/libfuzzer/FuzzerFork.cpp b/libfuzzer/FuzzerFork.cpp
index 5134a5d..d59d513 100644
--- a/libfuzzer/FuzzerFork.cpp
+++ b/libfuzzer/FuzzerFork.cpp
@@ -86,18 +86,21 @@ struct FuzzJob {
};
struct GlobalEnv {
- Vector<std::string> Args;
- Vector<std::string> CorpusDirs;
+ std::vector<std::string> Args;
+ std::vector<std::string> CorpusDirs;
std::string MainCorpusDir;
std::string TempDir;
std::string DFTDir;
std::string DataFlowBinary;
- Set<uint32_t> Features, Cov;
- Set<std::string> FilesWithDFT;
- Vector<std::string> Files;
+ std::set<uint32_t> Features, Cov;
+ std::set<std::string> FilesWithDFT;
+ std::vector<std::string> Files;
+ std::vector<std::size_t> FilesSizes;
Random *Rand;
std::chrono::system_clock::time_point ProcessStartTime;
int Verbosity = 0;
+ int Group = 0;
+ int NumCorpuses = 8;
size_t NumTimeouts = 0;
size_t NumOOMs = 0;
@@ -136,10 +139,24 @@ struct GlobalEnv {
if (size_t CorpusSubsetSize =
std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) {
auto Time1 = std::chrono::system_clock::now();
- for (size_t i = 0; i < CorpusSubsetSize; i++) {
- auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
- Seeds += (Seeds.empty() ? "" : ",") + SF;
- CollectDFT(SF);
+ if (Group) { // whether to group the corpus.
+ size_t AverageCorpusSize = Files.size() / NumCorpuses + 1;
+ size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize;
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ size_t RandNum = (*Rand)(AverageCorpusSize);
+ size_t Index = RandNum + StartIndex;
+ Index = Index < Files.size() ? Index
+ : Rand->SkewTowardsLast(Files.size());
+ auto &SF = Files[Index];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
+ } else {
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
}
auto Time2 = std::chrono::system_clock::now();
auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
@@ -183,7 +200,7 @@ struct GlobalEnv {
auto Stats = ParseFinalStatsFromLog(Job->LogPath);
NumRuns += Stats.number_of_executed_units;
- Vector<SizedFile> TempFiles, MergeCandidates;
+ std::vector<SizedFile> TempFiles, MergeCandidates;
// Read all newly created inputs and their feature sets.
// Choose only those inputs that have new features.
GetSizedFilesFromDir(Job->CorpusDir, &TempFiles);
@@ -193,7 +210,7 @@ struct GlobalEnv {
FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir);
auto FeatureBytes = FileToVector(FeatureFile, 0, false);
assert((FeatureBytes.size() % sizeof(uint32_t)) == 0);
- Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
+ std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size());
for (auto Ft : NewFeatures) {
if (!Features.count(Ft)) {
@@ -211,15 +228,27 @@ struct GlobalEnv {
if (MergeCandidates.empty()) return;
- Vector<std::string> FilesToAdd;
- Set<uint32_t> NewFeatures, NewCov;
+ std::vector<std::string> FilesToAdd;
+ std::set<uint32_t> NewFeatures, NewCov;
+ bool IsSetCoverMerge =
+ !Job->Cmd.getFlagValue("set_cover_merge").compare("1");
CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features,
- &NewFeatures, Cov, &NewCov, Job->CFPath, false);
+ &NewFeatures, Cov, &NewCov, Job->CFPath, false,
+ IsSetCoverMerge);
for (auto &Path : FilesToAdd) {
auto U = FileToVector(Path);
auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
WriteToFile(U, NewPath);
- Files.push_back(NewPath);
+ if (Group) { // Insert the queue according to the size of the seed.
+ size_t UnitSize = U.size();
+ auto Idx =
+ std::upper_bound(FilesSizes.begin(), FilesSizes.end(), UnitSize) -
+ FilesSizes.begin();
+ FilesSizes.insert(FilesSizes.begin() + Idx, UnitSize);
+ Files.insert(Files.begin() + Idx, NewPath);
+ } else {
+ Files.push_back(NewPath);
+ }
}
Features.insert(NewFeatures.begin(), NewFeatures.end());
Cov.insert(NewCov.begin(), NewCov.end());
@@ -228,10 +257,8 @@ struct GlobalEnv {
if (TPC.PcIsFuncEntry(TE))
PrintPC(" NEW_FUNC: %p %F %L\n", "",
TPC.GetNextInstructionPc(TE->PC));
-
}
-
void CollectDFT(const std::string &InputPath) {
if (DataFlowBinary.empty()) return;
if (!FilesWithDFT.insert(InputPath).second) return;
@@ -283,8 +310,8 @@ void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) {
// This is just a skeleton of an experimental -fork=1 feature.
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &CorpusDirs, int NumJobs) {
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &CorpusDirs, int NumJobs) {
Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
GlobalEnv Env;
@@ -294,8 +321,9 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
Env.Verbosity = Options.Verbosity;
Env.ProcessStartTime = std::chrono::system_clock::now();
Env.DataFlowBinary = Options.CollectDataFlow;
+ Env.Group = Options.ForkCorpusGroups;
- Vector<SizedFile> SeedFiles;
+ std::vector<SizedFile> SeedFiles;
for (auto &Dir : CorpusDirs)
GetSizedFilesFromDir(Dir, &SeedFiles);
std::sort(SeedFiles.begin(), SeedFiles.end());
@@ -316,13 +344,20 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
Env.Files.push_back(File.File);
} else {
auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
- Set<uint32_t> NewFeatures, NewCov;
+ std::set<uint32_t> NewFeatures, NewCov;
CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features,
- &NewFeatures, Env.Cov, &NewCov, CFPath, false);
+ &NewFeatures, Env.Cov, &NewCov, CFPath,
+ /*Verbose=*/false, /*IsSetCoverMerge=*/false);
Env.Features.insert(NewFeatures.begin(), NewFeatures.end());
Env.Cov.insert(NewFeatures.begin(), NewFeatures.end());
RemoveFile(CFPath);
}
+
+ if (Env.Group) {
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ }
+
Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
Env.Files.size(), Env.TempDir.c_str());
@@ -337,8 +372,10 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
WriteToFile(Unit({1}), Env.StopFile());
};
+ size_t MergeCycle = 20;
+ size_t JobExecuted = 0;
size_t JobId = 1;
- Vector<std::thread> Threads;
+ std::vector<std::thread> Threads;
for (int t = 0; t < NumJobs; t++) {
Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ));
FuzzQ.Push(Env.CreateNewJob(JobId++));
@@ -358,7 +395,46 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
Env.RunOneMergeJob(Job.get());
- // Continue if our crash is one of the ignorred ones.
+ // merge the corpus .
+ JobExecuted++;
+ if (Env.Group && JobExecuted >= MergeCycle) {
+ std::vector<SizedFile> CurrentSeedFiles;
+ for (auto &Dir : CorpusDirs)
+ GetSizedFilesFromDir(Dir, &CurrentSeedFiles);
+ std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end());
+
+ auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
+ std::set<uint32_t> TmpNewFeatures, TmpNewCov;
+ std::set<uint32_t> TmpFeatures, TmpCov;
+ Env.Files.clear();
+ Env.FilesSizes.clear();
+ CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files,
+ TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov,
+ CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false);
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ RemoveFile(CFPath);
+ JobExecuted = 0;
+ MergeCycle += 5;
+ }
+
+ // Since the number of corpus seeds will gradually increase, in order to
+ // control the number in each group to be about three times the number of
+ // seeds selected each time, the number of groups is dynamically adjusted.
+ if (Env.Files.size() < 2000)
+ Env.NumCorpuses = 12;
+ else if (Env.Files.size() < 6000)
+ Env.NumCorpuses = 20;
+ else if (Env.Files.size() < 12000)
+ Env.NumCorpuses = 32;
+ else if (Env.Files.size() < 16000)
+ Env.NumCorpuses = 40;
+ else if (Env.Files.size() < 24000)
+ Env.NumCorpuses = 60;
+ else
+ Env.NumCorpuses = 80;
+
+ // Continue if our crash is one of the ignored ones.
if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
Env.NumTimeouts++;
else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-15 19:01:17 +0000