Upgrade libprotobuf-mutator to 750dfa3b67e02587166d8246543a39ca943ab5ad

Change-Id: Ifab97ae848fd6140236a40cb1b5a4671e91b9067

11 files changed, 192 insertions, 66 deletions

diff --git a/.travis.yml b/.travis.yml
index 731b74a..92a1ebe 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,6 +43,12 @@ matrix:
     - env: BUILD_TYPE=Debug CC_COMPILER=gcc-${GCC_VERSION} CXX_COMPILER=g++-${GCC_VERSION}
       addons: *gcc
 
+    - env:
+      install:
+      before_script:
+      script:
+        - travis_retry wget --quiet -O - https://raw.githubusercontent.com/cpplint/cpplint/master/cpplint.py | python - --recursive src examples
+
 install:
   - mkdir -p deps && cd deps
   - travis_retry wget --no-check-certificate --quiet -O - https://cmake.org/files/v3.12/cmake-3.12.3-Linux-x86_64.tar.gz | tar --strip-components=1 -xz
@@ -50,7 +56,6 @@ install:
   - cd -
 
 before_script:
-  - travis_retry wget --quiet -O - https://raw.githubusercontent.com/cpplint/cpplint/master/cpplint.py | python - --recursive src examples
   - mkdir -p build && cd build
   - rm -rf *
   - cmake .. -GNinja -DLIB_PROTO_MUTATOR_WITH_ASAN=ON -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON -DCMAKE_C_COMPILER=${CC_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=/usr
@@ -59,4 +64,4 @@ script:
   - export ASAN_OPTIONS=detect_leaks=0
   - ninja
   - ninja check
-  - DESTDIR="/tmp/testing/" ninja install
+  - DESTDIR="/tmp/testing/" ninja install
\ No newline at end of file
diff --git a/METADATA b/METADATA
index ca1c076..4cf4256 100644
--- a/METADATA
+++ b/METADATA
@@ -5,11 +5,11 @@ third_party {
     type: GIT
     value: "https://github.com/google/libprotobuf-mutator"
   }
-  version: "7a2ed51a6b682a83e345ff49fc4cfd7ca47550db"
+  version: "750dfa3b67e02587166d8246543a39ca943ab5ad"
   license_type: NOTICE
   last_upgrade_date {
     year: 2020
-    month: 5
-    day: 28
+    month: 7
+    day: 29
   }
 }
diff --git a/README.md b/README.md
index 4e16fb6..ac3c247 100644
--- a/README.md
+++ b/README.md
@@ -98,18 +98,23 @@ PostProcessorRegistration can be used to avoid such issue and guide your fuzzer
 code. It registers callback which will be called for each message of particular type after each mutation.
 
 ```
-DEFINE_PROTO_FUZZER(const MyMessageType& input) {
-  static PostProcessorRegistration reg = {
-      [](MyMessageType* message, unsigned int seed) {
-        TweakMyMessage(message, seed);
-      }};
+static protobuf_mutator::libfuzzer::PostProcessorRegistration<MyMessageType> reg = {
+    [](MyMessageType* message, unsigned int seed) {
+      TweakMyMessage(message, seed);
+    }};
 
+DEFINE_PROTO_FUZZER(const MyMessageType& input) {
   // Code which needs to be fuzzed.
   ConsumeMyMessageType(input);
 }
 ```
 Optional: Use seed if callback uses random numbers. It may help later with debugging.
 
+Important: Callbacks should be deterministic and avoid modifying good messages.
+Callbacks are called for both: mutator generated and user provided inputs, like
+corpus or bug reproducer. So if callback performs unnecessary transformation it
+may corrupt the reproducer so it stops triggering the bug.
+
 Note: You can add callback for any nested message and you can add multiple callbacks for
 the same message type.
 ```
diff --git a/cmake/external/googletest.cmake b/cmake/external/googletest.cmake
index fe71115..3c3f8cf 100644
--- a/cmake/external/googletest.cmake
+++ b/cmake/external/googletest.cmake
@@ -18,7 +18,7 @@ set(GTEST_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/${GTEST_TARGET})
 set(GTEST_INCLUDE_DIRS ${GTEST_INSTALL_DIR}/include)
 include_directories(${GTEST_INCLUDE_DIRS})
 
-set(GTEST_LIBRARIES gtest)
+set(GTEST_LIBRARIES gtest gmock)
 set(GTEST_MAIN_LIBRARIES gtest_main)
 set(GTEST_BOTH_LIBRARIES ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES})
 
diff --git a/examples/libfuzzer/libfuzzer_bin_example.cc b/examples/libfuzzer/libfuzzer_bin_example.cc
index e5665c4..963b522 100644
--- a/examples/libfuzzer/libfuzzer_bin_example.cc
+++ b/examples/libfuzzer/libfuzzer_bin_example.cc
@@ -21,32 +21,30 @@
 
 protobuf_mutator::protobuf::LogSilencer log_silincer;
 
-DEFINE_BINARY_PROTO_FUZZER(const libfuzzer_example::Msg& message) {
-  static PostProcessorRegistration reg = {
-      [](libfuzzer_example::Msg* message, unsigned int seed) {
-        if (seed % 2) {
-          message->set_optional_uint64(
-              std::hash<std::string>{}(message->optional_string()));
-        }
-
-        if (message->has_any()) {
-          auto* any = message->mutable_any();
-
-          // Guide mutator to usefull 'Any' types.
-          static const char* const expected_types[] = {
-              "type.googleapis.com/google.protobuf.DescriptorProto",
-              "type.googleapis.com/google.protobuf.FileDescriptorProto",
-          };
-
-          if (!std::count(std::begin(expected_types), std::end(expected_types),
-                          any->type_url())) {
-            const size_t num =
-                (std::end(expected_types) - std::begin(expected_types));
-            any->set_type_url(expected_types[seed % num]);
-          }
+protobuf_mutator::libfuzzer::PostProcessorRegistration<libfuzzer_example::Msg>
+    reg = {[](libfuzzer_example::Msg* message, unsigned int seed) {
+      message->set_optional_uint64(
+          std::hash<std::string>{}(message->optional_string()));
+
+      if (message->has_any()) {
+        auto* any = message->mutable_any();
+
+        // Guide mutator to usefull 'Any' types.
+        static const char* const expected_types[] = {
+            "type.googleapis.com/google.protobuf.DescriptorProto",
+            "type.googleapis.com/google.protobuf.FileDescriptorProto",
+        };
+
+        if (!std::count(std::begin(expected_types), std::end(expected_types),
+                        any->type_url())) {
+          const size_t num =
+              (std::end(expected_types) - std::begin(expected_types));
+          any->set_type_url(expected_types[seed % num]);
         }
-      }};
+      }
+    }};
 
+DEFINE_BINARY_PROTO_FUZZER(const libfuzzer_example::Msg& message) {
   protobuf_mutator::protobuf::FileDescriptorProto file;
 
   // Emulate a bug.
diff --git a/examples/libfuzzer/libfuzzer_example.cc b/examples/libfuzzer/libfuzzer_example.cc
index 2f2b8d4..aa65125 100644
--- a/examples/libfuzzer/libfuzzer_example.cc
+++ b/examples/libfuzzer/libfuzzer_example.cc
@@ -21,32 +21,30 @@
 
 protobuf_mutator::protobuf::LogSilencer log_silincer;
 
-DEFINE_PROTO_FUZZER(const libfuzzer_example::Msg& message) {
-  static PostProcessorRegistration reg = {
-      [](libfuzzer_example::Msg* message, unsigned int seed) {
-        if (seed % 2) {
-          message->set_optional_uint64(
-              std::hash<std::string>{}(message->optional_string()));
-        }
-
-        if (message->has_any()) {
-          auto* any = message->mutable_any();
-
-          // Guide mutator to usefull 'Any' types.
-          static const char* const expected_types[] = {
-              "type.googleapis.com/google.protobuf.DescriptorProto",
-              "type.googleapis.com/google.protobuf.FileDescriptorProto",
-          };
-
-          if (!std::count(std::begin(expected_types), std::end(expected_types),
-                          any->type_url())) {
-            const size_t num =
-                (std::end(expected_types) - std::begin(expected_types));
-            any->set_type_url(expected_types[seed % num]);
-          }
+protobuf_mutator::libfuzzer::PostProcessorRegistration<libfuzzer_example::Msg>
+    reg = {[](libfuzzer_example::Msg* message, unsigned int seed) {
+      message->set_optional_uint64(
+          std::hash<std::string>{}(message->optional_string()));
+
+      if (message->has_any()) {
+        auto* any = message->mutable_any();
+
+        // Guide mutator to usefull 'Any' types.
+        static const char* const expected_types[] = {
+            "type.googleapis.com/google.protobuf.DescriptorProto",
+            "type.googleapis.com/google.protobuf.FileDescriptorProto",
+        };
+
+        if (!std::count(std::begin(expected_types), std::end(expected_types),
+                        any->type_url())) {
+          const size_t num =
+              (std::end(expected_types) - std::begin(expected_types));
+          any->set_type_url(expected_types[seed % num]);
         }
-      }};
+      }
+    }};
 
+DEFINE_PROTO_FUZZER(const libfuzzer_example::Msg& message) {
   protobuf_mutator::protobuf::FileDescriptorProto file;
 
   // Emulate a bug.
diff --git a/port/gtest.h b/port/gtest.h
index 485aa98..60d50b8 100644
--- a/port/gtest.h
+++ b/port/gtest.h
@@ -15,6 +15,7 @@
 #ifndef PORT_GTEST_H_
 #define PORT_GTEST_H_
 
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 #endif  // PORT_GTEST_H_
diff --git a/src/libfuzzer/libfuzzer_macro.cc b/src/libfuzzer/libfuzzer_macro.cc
index b2a5302..4e506cb 100644
--- a/src/libfuzzer/libfuzzer_macro.cc
+++ b/src/libfuzzer/libfuzzer_macro.cc
@@ -15,6 +15,8 @@
 #include "src/libfuzzer/libfuzzer_macro.h"
 
 #include <algorithm>
+#include <memory>
+#include <vector>
 
 #include "src/binary_format.h"
 #include "src/libfuzzer/libfuzzer_mutator.h"
@@ -91,6 +93,35 @@ class BinaryOutputWriter : public OutputWriter {
   }
 };
 
+class LastMutationCache {
+ public:
+  void Store(const uint8_t* data, size_t size, protobuf::Message* message) {
+    if (!message_) message_.reset(message->New());
+    message->GetReflection()->Swap(message, message_.get());
+    data_.assign(data, data + size);
+  }
+
+  bool LoadIfSame(const uint8_t* data, size_t size,
+                  protobuf::Message* message) {
+    if (!message_ || size != data_.size() ||
+        !std::equal(data_.begin(), data_.end(), data))
+      return false;
+
+    message->GetReflection()->Swap(message, message_.get());
+    message_.reset();
+    return true;
+  }
+
+ private:
+  std::vector<uint8_t> data_;
+  std::unique_ptr<protobuf::Message> message_;
+};
+
+LastMutationCache* GetCache() {
+  static LastMutationCache cache;
+  return &cache;
+}
+
 Mutator* GetMutator() {
   static Mutator mutator;
   return &mutator;
@@ -111,6 +142,7 @@ size_t MutateMessage(unsigned int seed, const InputReader& input,
   GetMutator()->Mutate(message, max_size);
   if (size_t new_size = output->Write(*message)) {
     assert(new_size <= output->size());
+    GetCache()->Store(output->data(), new_size, message);
     return new_size;
   }
   return 0;
@@ -127,6 +159,7 @@ size_t CrossOverMessages(unsigned int seed, const InputReader& input1,
   GetMutator()->CrossOver(*message2, message1, max_size);
   if (size_t new_size = output->Write(*message1)) {
     assert(new_size <= output->size());
+    GetCache()->Store(output->data(), new_size, message1);
     return new_size;
   }
   return 0;
@@ -189,8 +222,13 @@ size_t CustomProtoCrossOver(bool binary, const uint8_t* data1, size_t size1,
 
 bool LoadProtoInput(bool binary, const uint8_t* data, size_t size,
                     protobuf::Message* input) {
-  return binary ? ParseBinaryMessage(data, size, input)
-                : ParseTextMessage(data, size, input);
+  if (GetCache()->LoadIfSame(data, size, input)) return true;
+  auto result = binary ? ParseBinaryMessage(data, size, input)
+                       : ParseTextMessage(data, size, input);
+  if (!result) return false;
+  GetMutator()->Seed(size);
+  GetMutator()->Fix(input);
+  return true;
 }
 
 void RegisterPostProcessor(
diff --git a/src/libfuzzer/libfuzzer_test.cc b/src/libfuzzer/libfuzzer_test.cc
index 180a02c..f19eb12 100644
--- a/src/libfuzzer/libfuzzer_test.cc
+++ b/src/libfuzzer/libfuzzer_test.cc
@@ -13,16 +13,85 @@
 // limitations under the License.
 
 #include "port/gtest.h"
+#include "port/protobuf.h"
 #include "src/libfuzzer/libfuzzer_macro.h"
 #include "src/mutator_test_proto2.pb.h"
 
-static bool reached = false;
+using protobuf_mutator::protobuf::util::MessageDifferencer;
+using ::testing::_;
+using ::testing::AllOf;
+using ::testing::DoAll;
+using ::testing::Ref;
+using ::testing::SaveArg;
+using ::testing::SaveArgPointee;
+using ::testing::StrictMock;
 
-DEFINE_PROTO_FUZZER(const protobuf_mutator::Msg::EmptyMessage& message) {
-  reached = true;
+static class MockFuzzer* mock_fuzzer;
+
+class MockFuzzer {
+ public:
+  MockFuzzer() { mock_fuzzer = this; }
+  ~MockFuzzer() { mock_fuzzer = nullptr; }
+  MOCK_METHOD(void, PostProcess,
+              (protobuf_mutator::Msg * message, unsigned int seed));
+  MOCK_METHOD(void, TestOneInput, (const protobuf_mutator::Msg& message));
+};
+
+protobuf_mutator::libfuzzer::PostProcessorRegistration<protobuf_mutator::Msg>
+    reg = {[](protobuf_mutator::Msg* message, unsigned int seed) {
+      mock_fuzzer->PostProcess(message, seed);
+    }};
+
+DEFINE_TEXT_PROTO_FUZZER(const protobuf_mutator::Msg& message) {
+  mock_fuzzer->TestOneInput(message);
 }
 
-TEST(LibFuzzerTest, Basic) {
+MATCHER_P(IsMessageEq, msg, "") {
+  return MessageDifferencer::Equals(arg, msg.get());
+}
+MATCHER(IsInitialized, "") { return arg.IsInitialized(); }
+
+TEST(LibFuzzerTest, LLVMFuzzerTestOneInput) {
+  unsigned int seed = 0;
+  testing::StrictMock<MockFuzzer> mock;
+  protobuf_mutator::Msg msg;
+  EXPECT_CALL(mock, PostProcess(_, _))
+      .WillOnce(DoAll(SaveArgPointee<0>(&msg), SaveArg<1>(&seed)));
+  EXPECT_CALL(
+      mock, TestOneInput(AllOf(IsMessageEq(std::cref(msg)), IsInitialized())));
+  LLVMFuzzerTestOneInput((const uint8_t*)"", 0);
+
+  EXPECT_CALL(mock, PostProcess(_, seed)).WillOnce(SaveArgPointee<0>(&msg));
+  EXPECT_CALL(
+      mock, TestOneInput(AllOf(IsMessageEq(std::cref(msg)), IsInitialized())));
   LLVMFuzzerTestOneInput((const uint8_t*)"", 0);
-  EXPECT_TRUE(reached);
+}
+
+TEST(LibFuzzerTest, LLVMFuzzerCustomMutator) {
+  testing::StrictMock<MockFuzzer> mock;
+  protobuf_mutator::Msg msg;
+  EXPECT_CALL(mock, PostProcess(_, _)).WillOnce(SaveArgPointee<0>(&msg));
+  EXPECT_CALL(
+      mock, TestOneInput(AllOf(IsMessageEq(std::cref(msg)), IsInitialized())));
+
+  uint8_t buff[1024] = {};
+  size_t size = LLVMFuzzerCustomMutator(buff, 0, sizeof(buff), 5);
+  ASSERT_GT(size, 0U);
+  LLVMFuzzerTestOneInput(buff, size);
+}
+
+TEST(LibFuzzerTest, LLVMFuzzerCustomCrossOver) {
+  testing::StrictMock<MockFuzzer> mock;
+  protobuf_mutator::Msg msg;
+  EXPECT_CALL(mock, PostProcess(_, _)).WillOnce(SaveArgPointee<0>(&msg));
+  EXPECT_CALL(
+      mock, TestOneInput(AllOf(IsMessageEq(std::cref(msg)), IsInitialized())));
+
+  uint8_t buff1[1024] = {};
+  uint8_t buff2[1024] = {};
+  uint8_t buff3[1024] = {};
+  size_t size =
+      LLVMFuzzerCustomCrossOver(buff1, 0, buff2, 0, buff3, sizeof(buff3), 6);
+  ASSERT_GT(size, 0U);
+  LLVMFuzzerTestOneInput(buff3, size);
 }
diff --git a/src/mutator.cc b/src/mutator.cc
index b29ea8f..fb5542b 100644
--- a/src/mutator.cc
+++ b/src/mutator.cc
@@ -625,6 +625,15 @@ struct CreateField : public FieldFunction<CreateField> {
 
 void Mutator::Seed(uint32_t value) { random_.seed(value); }
 
+void Mutator::Fix(Message* message) {
+  UnpackedAny any;
+  UnpackAny(*message, &any);
+
+  PostProcessing(keep_initialized_, post_processors_, any, &random_)
+      .Run(message, kMaxInitializeDepth);
+  assert(IsInitialized(*message));
+}
+
 void Mutator::Mutate(Message* message, size_t max_size_hint) {
   UnpackedAny any;
   UnpackAny(*message, &any);
diff --git a/src/mutator.h b/src/mutator.h
index 57c7c47..afb6940 100644
--- a/src/mutator.h
+++ b/src/mutator.h
@@ -60,6 +60,9 @@ class Mutator {
   void CrossOver(const protobuf::Message& message1, protobuf::Message* message2,
                  size_t max_size_hint);
 
+  // Makes message initialized and calls post processors to make it valid.
+  void Fix(protobuf::Message* message);
+
   // Callback to postprocess mutations.
   // Implementation should use seed to initialize random number generators.
   using PostProcess =