corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1365994 - Update libfuzzer r=decoder

Browse Source 
MozReview-Commit-ID: GCwIIZd5PTS
***
1365994 - update moz.build for libfuzzer

MozReview-Commit-ID: IxbLnviJC08
This commit is contained in:
rforbes
2017-05-18 10:23:05 -07:00
parent 5c5e3b8e4c
commit 5ebee3ca5d
44 changed files with 2843 additions and 1038 deletions
Download Patch File Download Diff File Expand all files Collapse all files

339
tools/fuzzing/libfuzzer/FuzzerLoop.cpp
View File

@@ -9,16 +9,17 @@
// Fuzzer's main loop.
//===----------------------------------------------------------------------===//
#include "FuzzerInternal.h"
#include "FuzzerCorpus.h"
#include "FuzzerInternal.h"
#include "FuzzerIO.h"
#include "FuzzerMutate.h"
#include "FuzzerTracePC.h"
#include "FuzzerRandom.h"
#include "FuzzerShmem.h"
#include "FuzzerTracePC.h"
#include <algorithm>
#include <cstring>
#include <set>
#include <memory>
#include <set>
#if defined(__has_include)
#if __has_include(<sanitizer / coverage_interface.h>)
@@ -42,73 +43,11 @@ static const size_t kMaxUnitSizeToPrint = 256;
thread_local bool Fuzzer::IsMyThread;
static void MissingExternalApiFunction(const char *FnName) {
Printf("ERROR: %s is not defined. Exiting.\n"
"Did you use -fsanitize-coverage=... to build your code?\n",
FnName);
exit(1);
}
#define CHECK_EXTERNAL_FUNCTION(fn) \
do { \
if (!(EF->fn)) \
MissingExternalApiFunction(#fn); \
} while (false)
SharedMemoryRegion SMR;
// Only one Fuzzer per process.
static Fuzzer *F;
void Fuzzer::ResetEdgeCoverage() {
CHECK_EXTERNAL_FUNCTION(__sanitizer_reset_coverage);
EF->__sanitizer_reset_coverage();
}
void Fuzzer::ResetCounters() {
if (Options.UseCounters)
EF->__sanitizer_update_counter_bitset_and_clear_counters(0);
}
void Fuzzer::PrepareCounters(Fuzzer::Coverage *C) {
if (Options.UseCounters) {
size_t NumCounters = EF->__sanitizer_get_number_of_counters();
C->CounterBitmap.resize(NumCounters);
}
}
// Records data to a maximum coverage tracker. Returns true if additional
// coverage was discovered.
bool Fuzzer::RecordMaxCoverage(Fuzzer::Coverage *C) {
bool Res = false;
uint64_t NewBlockCoverage = EF->__sanitizer_get_total_unique_coverage();
if (NewBlockCoverage > C->BlockCoverage) {
Res = true;
C->BlockCoverage = NewBlockCoverage;
}
if (Options.UseIndirCalls &&
EF->__sanitizer_get_total_unique_caller_callee_pairs) {
uint64_t NewCallerCalleeCoverage =
EF->__sanitizer_get_total_unique_caller_callee_pairs();
if (NewCallerCalleeCoverage > C->CallerCalleeCoverage) {
Res = true;
C->CallerCalleeCoverage = NewCallerCalleeCoverage;
}
}
if (Options.UseCounters) {
uint64_t CounterDelta =
EF->__sanitizer_update_counter_bitset_and_clear_counters(
C->CounterBitmap.data());
if (CounterDelta > 0) {
Res = true;
C->CounterBitmapBits += CounterDelta;
}
}
return Res;
}
// Leak detection is expensive, so we first check if there were more mallocs
// than frees (using the sanitizer malloc hooks) and only then try to call lsan.
struct MallocFreeTracer {
@@ -137,14 +76,18 @@ struct MallocFreeTracer {
static MallocFreeTracer AllocTracer;
ATTRIBUTE_NO_SANITIZE_MEMORY
void MallocHook(const volatile void *ptr, size_t size) {
size_t N = AllocTracer.Mallocs++;
F->HandleMalloc(size);
if (int TraceLevel = AllocTracer.TraceLevel) {
Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
if (TraceLevel >= 2 && EF)
EF->__sanitizer_print_stack_trace();
}
}
ATTRIBUTE_NO_SANITIZE_MEMORY
void FreeHook(const volatile void *ptr) {
size_t N = AllocTracer.Frees++;
if (int TraceLevel = AllocTracer.TraceLevel) {
@@ -154,15 +97,30 @@ void FreeHook(const volatile void *ptr) {
}
}
// Crash on a single malloc that exceeds the rss limit.
void Fuzzer::HandleMalloc(size_t Size) {
if (!Options.RssLimitMb || (Size >> 20) < (size_t)Options.RssLimitMb)
return;
Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
Size);
Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
if (EF->__sanitizer_print_stack_trace)
EF->__sanitizer_print_stack_trace();
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
_Exit(Options.ErrorExitCode); // Stop right now.
}
Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
FuzzingOptions Options)
: CB(CB), Corpus(Corpus), MD(MD), Options(Options) {
SetDeathCallback();
if (EF->__sanitizer_set_death_callback)
EF->__sanitizer_set_death_callback(StaticDeathCallback);
InitializeTraceState();
assert(!F);
F = this;
TPC.ResetMaps();
ResetCoverage();
IsMyThread = true;
if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks)
EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook);
@@ -176,6 +134,8 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus);
MaxInputLen = MaxMutationLen = Options.MaxLen;
AllocateCurrentUnitData();
CurrentUnitSize = 0;
memset(BaseSha1, 0, sizeof(BaseSha1));
}
Fuzzer::~Fuzzer() { }
@@ -185,33 +145,12 @@ void Fuzzer::AllocateCurrentUnitData() {
CurrentUnitData = new uint8_t[MaxInputLen];
}
void Fuzzer::SetDeathCallback() {
CHECK_EXTERNAL_FUNCTION(__sanitizer_set_death_callback);
EF->__sanitizer_set_death_callback(StaticDeathCallback);
}
void Fuzzer::StaticDeathCallback() {
assert(F);
F->DeathCallback();
}
static void WarnOnUnsuccessfullMerge(bool DoWarn) {
if (!DoWarn) return;
Printf(
"***\n"
"***\n"
"***\n"
"*** NOTE: merge did not succeed due to a failure on one of the inputs.\n"
"*** You will need to filter out crashes from the corpus, e.g. like this:\n"
"*** for f in WITH_CRASHES/*; do ./fuzzer $f && cp $f NO_CRASHES; done\n"
"*** Future versions may have crash-resistant merge, stay tuned.\n"
"***\n"
"***\n"
"***\n");
}
void Fuzzer::DumpCurrentUnit(const char *Prefix) {
WarnOnUnsuccessfullMerge(InMergeMode);
if (!CurrentUnitData) return; // Happens when running individual inputs.
MD.PrintMutationSequence();
Printf("; base unit: %s\n", Sha1ToString(BaseSha1).c_str());
@@ -245,8 +184,13 @@ void Fuzzer::StaticInterruptCallback() {
F->InterruptCallback();
}
void Fuzzer::StaticFileSizeExceedCallback() {
Printf("==%lu== ERROR: libFuzzer: file size exceeded\n", GetPid());
exit(1);
}
void Fuzzer::CrashCallback() {
Printf("==%d== ERROR: libFuzzer: deadly signal\n", GetPid());
Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
if (EF->__sanitizer_print_stack_trace)
EF->__sanitizer_print_stack_trace();
Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
@@ -255,11 +199,11 @@ void Fuzzer::CrashCallback() {
Printf("SUMMARY: libFuzzer: deadly signal\n");
DumpCurrentUnit("crash-");
PrintFinalStats();
exit(Options.ErrorExitCode);
_Exit(Options.ErrorExitCode); // Stop right now.
}
void Fuzzer::InterruptCallback() {
Printf("==%d== libFuzzer: run interrupted; exiting\n", GetPid());
Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
PrintFinalStats();
_Exit(0); // Stop right now, don't perform any at-exit actions.
}
@@ -267,8 +211,11 @@ void Fuzzer::InterruptCallback() {
NO_SANITIZE_MEMORY
void Fuzzer::AlarmCallback() {
assert(Options.UnitTimeoutSec > 0);
// In Windows Alarm callback is executed by a different thread.
#if !LIBFUZZER_WINDOWS
if (!InFuzzingThread()) return;
if (!CurrentUnitSize)
#endif
if (!RunningCB)
return; // We have not started running units yet.
size_t Seconds =
duration_cast<seconds>(system_clock::now() - UnitStartTime).count();
@@ -281,7 +228,7 @@ void Fuzzer::AlarmCallback() {
Printf(" and the timeout value is %d (use -timeout=N to change)\n",
Options.UnitTimeoutSec);
DumpCurrentUnit("timeout-");
Printf("==%d== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
Seconds);
if (EF->__sanitizer_print_stack_trace)
EF->__sanitizer_print_stack_trace();
@@ -293,11 +240,11 @@ void Fuzzer::AlarmCallback() {
void Fuzzer::RssLimitCallback() {
Printf(
"==%d== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
"==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
if (EF->__sanitizer_print_memory_profile)
EF->__sanitizer_print_memory_profile(95);
EF->__sanitizer_print_memory_profile(95, 8);
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
@@ -306,32 +253,13 @@ void Fuzzer::RssLimitCallback() {
void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
size_t ExecPerSec = execPerSec();
if (Options.OutputCSV) {
static bool csvHeaderPrinted = false;
if (!csvHeaderPrinted) {
csvHeaderPrinted = true;
Printf("runs,block_cov,bits,cc_cov,corpus,execs_per_sec,tbms,reason\n");
}
Printf("%zd,%zd,%zd,%zd,%zd,%zd,%s\n", TotalNumberOfRuns,
MaxCoverage.BlockCoverage, MaxCoverage.CounterBitmapBits,
MaxCoverage.CallerCalleeCoverage, Corpus.size(), ExecPerSec, Where);
}
if (!Options.Verbosity)
return;
Printf("#%zd\t%s", TotalNumberOfRuns, Where);
if (MaxCoverage.BlockCoverage)
Printf(" cov: %zd", MaxCoverage.BlockCoverage);
if (size_t N = MaxCoverage.VPMap.GetNumBitsSinceLastMerge())
Printf(" vp: %zd", N);
if (size_t N = TPC.GetTotalPCCoverage())
Printf(" cov: %zd", N);
if (auto TB = MaxCoverage.CounterBitmapBits)
Printf(" bits: %zd", TB);
if (size_t N = Corpus.NumFeatures())
Printf( " ft: %zd", N);
if (MaxCoverage.CallerCalleeCoverage)
Printf(" indir: %zd", MaxCoverage.CallerCalleeCoverage);
if (!Corpus.empty()) {
Printf(" corp: %zd", Corpus.NumActiveUnits());
if (size_t N = Corpus.SizeInBytes()) {
@@ -354,6 +282,8 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
void Fuzzer::PrintFinalStats() {
if (Options.PrintCoverage)
TPC.PrintCoverage();
if (Options.DumpCoverage)
TPC.DumpCoverage();
if (Options.PrintCorpusStats)
Corpus.PrintStats();
if (!Options.PrintFinalStats) return;
@@ -428,7 +358,7 @@ void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
}
void Fuzzer::ShuffleCorpus(UnitVector *V) {
std::random_shuffle(V->begin(), V->end(), MD.GetRand());
std::shuffle(V->begin(), V->end(), MD.GetRand());
if (Options.PreferSmall)
std::stable_sort(V->begin(), V->end(), [](const Unit &A, const Unit &B) {
return A.size() < B.size();
@@ -448,8 +378,6 @@ void Fuzzer::ShuffleAndMinimize(UnitVector *InitialCorpus) {
if (size_t NumFeatures = RunOne(U)) {
CheckExitOnSrcPosOrItem();
Corpus.AddToCorpus(U, NumFeatures);
if (Options.Verbosity >= 2)
Printf("NEW0: %zd L %zd\n", MaxCoverage.BlockCoverage, U.size());
}
TryDetectingAMemoryLeak(U.data(), U.size(),
/*DuringInitialCorpusExecution*/ true);
@@ -468,16 +396,11 @@ size_t Fuzzer::RunOne(const uint8_t *Data, size_t Size) {
ExecuteCallback(Data, Size);
size_t Res = 0;
if (size_t NumFeatures = TPC.FinalizeTrace(&Corpus, Size, Options.Shrink))
Res = NumFeatures;
if (!TPC.UsingTracePcGuard()) {
if (TPC.UpdateValueProfileMap(&MaxCoverage.VPMap))
Res = 1;
if (!Res && RecordMaxCoverage(&MaxCoverage))
Res = 1;
}
size_t NumUpdatesBefore = Corpus.NumFeatureUpdates();
TPC.CollectFeatures([&](size_t Feature) {
Corpus.AddFeature(Feature, Size, Options.Shrink);
});
size_t NumUpdatesAfter = Corpus.NumFeatureUpdates();
auto TimeOfUnit =
duration_cast<seconds>(UnitStopTime - UnitStartTime).count();
@@ -490,7 +413,7 @@ size_t Fuzzer::RunOne(const uint8_t *Data, size_t Size) {
Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-");
}
return Res;
return NumUpdatesAfter - NumUpdatesBefore;
}
size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const {
@@ -499,8 +422,28 @@ size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const {
return CurrentUnitSize;
}
void Fuzzer::CrashOnOverwrittenData() {
Printf("==%d== ERROR: libFuzzer: fuzz target overwrites it's const input\n",
GetPid());
DumpCurrentUnit("crash-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
_Exit(Options.ErrorExitCode); // Stop right now.
}
// Compare two arrays, but not all bytes if the arrays are large.
static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) {
const size_t Limit = 64;
if (Size <= 64)
return !memcmp(A, B, Size);
// Compare first and last Limit/2 bytes.
return !memcmp(A, B, Limit / 2) &&
!memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2);
}
void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
assert(InFuzzingThread());
if (SMR.IsClient())
SMR.WriteByteArray(Data, Size);
// We copy the contents of Unit into a separate heap buffer
// so that we reliably find buffer overflows in it.
uint8_t *DataCopy = new uint8_t[Size];
@@ -510,13 +453,16 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
CurrentUnitSize = Size;
AllocTracer.Start(Options.TraceMalloc);
UnitStartTime = system_clock::now();
ResetCounters(); // Reset coverage right before the callback.
TPC.ResetMaps();
RunningCB = true;
int Res = CB(DataCopy, Size);
RunningCB = false;
UnitStopTime = system_clock::now();
(void)Res;
assert(Res == 0);
HasMoreMallocsThanFrees = AllocTracer.Stop();
if (!LooseMemeq(DataCopy, Data, Size))
CrashOnOverwrittenData();
CurrentUnitSize = 0;
delete[] DataCopy;
}
@@ -565,76 +511,6 @@ void Fuzzer::ReportNewCoverage(InputInfo *II, const Unit &U) {
TPC.PrintNewPCs();
}
// Finds minimal number of units in 'Extra' that add coverage to 'Initial'.
// We do it by actually executing the units, sometimes more than once,
// because we may be using different coverage-like signals and the only
// common thing between them is that we can say "this unit found new stuff".
UnitVector Fuzzer::FindExtraUnits(const UnitVector &Initial,
const UnitVector &Extra) {
UnitVector Res = Extra;
UnitVector Tmp;
size_t OldSize = Res.size();
for (int Iter = 0; Iter < 10; Iter++) {
ShuffleCorpus(&Res);
TPC.ResetMaps();
Corpus.ResetFeatureSet();
ResetCoverage();
for (auto &U : Initial) {
TPC.ResetMaps();
RunOne(U);
}
Tmp.clear();
for (auto &U : Res) {
TPC.ResetMaps();
if (RunOne(U))
Tmp.push_back(U);
}
char Stat[7] = "MIN ";
Stat[3] = '0' + Iter;
PrintStats(Stat, "\n", Tmp.size());
size_t NewSize = Tmp.size();
assert(NewSize <= OldSize);
Res.swap(Tmp);
if (NewSize + 5 >= OldSize)
break;
OldSize = NewSize;
}
return Res;
}
void Fuzzer::Merge(const std::vector<std::string> &Corpora) {
if (Corpora.size() <= 1) {
Printf("Merge requires two or more corpus dirs\n");
return;
}
InMergeMode = true;
std::vector<std::string> ExtraCorpora(Corpora.begin() + 1, Corpora.end());
assert(MaxInputLen > 0);
UnitVector Initial, Extra;
ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen, true);
for (auto &C : ExtraCorpora)
ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen, true);
if (!Initial.empty()) {
Printf("=== Minimizing the initial corpus of %zd units\n", Initial.size());
Initial = FindExtraUnits({}, Initial);
}
Printf("=== Merging extra %zd units\n", Extra.size());
auto Res = FindExtraUnits(Initial, Extra);
for (auto &U: Res)
WriteToOutputCorpus(U);
Printf("=== Merge: written %zd units\n", Res.size());
}
// Tries detecting a memory leak on the particular input that we have just
// executed before calling this function.
void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
@@ -674,6 +550,19 @@ void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
}
}
static size_t ComputeMutationLen(size_t MaxInputSize, size_t MaxMutationLen,
Random &Rand) {
assert(MaxInputSize <= MaxMutationLen);
if (MaxInputSize == MaxMutationLen) return MaxMutationLen;
size_t Result = MaxInputSize;
size_t R = Rand.Rand();
if ((R % (1U << 7)) == 0)
Result++;
if ((R % (1U << 15)) == 0)
Result += 10 + Result / 2;
return Min(Result, MaxMutationLen);
}
void Fuzzer::MutateAndTestOne() {
MD.StartMutationSequence();
@@ -687,13 +576,19 @@ void Fuzzer::MutateAndTestOne() {
assert(MaxMutationLen > 0);
size_t CurrentMaxMutationLen =
Options.ExperimentalLenControl
? ComputeMutationLen(Corpus.MaxInputSize(), MaxMutationLen,
MD.GetRand())
: MaxMutationLen;
for (int i = 0; i < Options.MutateDepth; i++) {
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
break;
size_t NewSize = 0;
NewSize = MD.Mutate(CurrentUnitData, Size, MaxMutationLen);
NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen);
assert(NewSize > 0 && "Mutator returned empty unit");
assert(NewSize <= MaxMutationLen && "Mutator return overisized unit");
assert(NewSize <= CurrentMaxMutationLen && "Mutator return overisized unit");
Size = NewSize;
if (i == 0)
StartTraceRecording();
@@ -710,13 +605,8 @@ void Fuzzer::MutateAndTestOne() {
}
}
void Fuzzer::ResetCoverage() {
ResetEdgeCoverage();
MaxCoverage.Reset();
PrepareCounters(&MaxCoverage);
}
void Fuzzer::Loop() {
TPC.InitializePrintNewPCs();
system_clock::time_point LastCorpusReload = system_clock::now();
if (Options.DoCrossOver)
MD.SetCorpus(&Corpus);
@@ -739,7 +629,7 @@ void Fuzzer::Loop() {
}
void Fuzzer::MinimizeCrashLoop(const Unit &U) {
if (U.size() <= 2) return;
if (U.size() <= 1) return;
while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) {
MD.StartMutationSequence();
memcpy(CurrentUnitData, U.data(), U.size());
@@ -753,6 +643,29 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) {
}
}
void Fuzzer::AnnounceOutput(const uint8_t *Data, size_t Size) {
if (SMR.IsServer()) {
SMR.WriteByteArray(Data, Size);
} else if (SMR.IsClient()) {
SMR.PostClient();
SMR.WaitServer();
size_t OtherSize = SMR.ReadByteArraySize();
uint8_t *OtherData = SMR.GetByteArray();
if (Size != OtherSize || memcmp(Data, OtherData, Size) != 0) {
size_t i = 0;
for (i = 0; i < Min(Size, OtherSize); i++)
if (Data[i] != OtherData[i])
break;
Printf("==%lu== ERROR: libFuzzer: equivalence-mismatch. Sizes: %zd %zd; "
"offset %zd\n", GetPid(), Size, OtherSize, i);
DumpCurrentUnit("mismatch-");
Printf("SUMMARY: libFuzzer: equivalence-mismatch\n");
PrintFinalStats();
_Exit(Options.ErrorExitCode);
}
}
}
} // namespace fuzzer
extern "C" {
@@ -761,4 +674,10 @@ size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
assert(fuzzer::F);
return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize);
}
// Experimental
void LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) {
assert(fuzzer::F);
fuzzer::F->AnnounceOutput(Data, Size);
}
} // extern "C"