Reland "[Zucchini] (raw) Apply fuzzer"

This is a reland of f4a598ff5adfe27f8153bd36984ee9cb549f99e9

Windows cannot resolve #!/usr/bin/env python depending on how it is
configured. To fix this explicitly use python in the subprocess call.

Interestingly, the Tryjobs didn't catch this and only the official
build waterfall does...

Original change's description:
> [Zucchini] (raw) Apply fuzzer
>
> This is part of a series of Fuzzers to be added to Zucchini for
> security review. This tests the raw data patch application logic
> exercising the patch reader and apply process. It only covers ~20%
> of code in 100000 executions as the bulk of the remaining code is
> associated with the much more complex and expensive to fuzz reference
> related code.
>
> With the supplied seed corpus the fuzzer reaches approximately 11000
> execs/s.
>
> This found a couple bugs which are fixed in:
> https://chromium-review.googlesource.com/c/chromium/src/+/1028575
>
>
> Bug: 835341
> Change-Id: Idc1d862bfaa6eb6313f39e10536f4750c05ab863
> Reviewed-on: https://chromium-review.googlesource.com/1028570
> Commit-Queue: Calder Kitagawa <ckitagawa@google.com>
> Reviewed-by: Samuel Huang <huangs@chromium.org>
> Reviewed-by: Greg Thompson <grt@chromium.org>
> Reviewed-by: Max Moroz <mmoroz@chromium.org>
> Reviewed-by: Jonathan Metzman <metzman@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#557185}

Bug: 835341
Change-Id: I24e94dd0c2035d84c84636f0a0a30756ae7f0c36
Reviewed-on: https://chromium-review.googlesource.com/1052567
Commit-Queue: Calder Kitagawa <ckitagawa@google.com>
Reviewed-by: Samuel Huang <huangs@chromium.org>
Cr-Commit-Position: refs/heads/master@{#557286}
NOKEYCHECK=True
GitOrigin-RevId: 8e7c08d3d11c61d08ad05d3ebc283aa2d6bf7c91

8 files changed, 296 insertions, 0 deletions

diff --git a/fuzzers/BUILD.gn b/fuzzers/BUILD.gn
index 2e489fd..66c3d18 100644
--- a/fuzzers/BUILD.gn
+++ b/fuzzers/BUILD.gn
@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import("//testing/libfuzzer/fuzzer_test.gni")
+import("//third_party/protobuf/proto_library.gni")
 
 # To download the corpus for local fuzzing use:
 # gsutil -m rsync \
@@ -28,3 +29,56 @@ fuzzer_test("zucchini_patch_fuzzer") {
   ]
   seed_corpus = "testdata/patch_fuzzer"
 }
+
+proto_library("zucchini_file_pair_proto") {
+  sources = [
+    "file_pair.proto",
+  ]
+}
+
+# Ensure protoc is available.
+if (current_toolchain == host_toolchain) {
+  action("zucchini_raw_apply_seed") {
+    script = "generate_fuzzer_data.py"
+
+    args = [
+      "--raw",
+      "old_eventlog_provider.dll",  # <old_file>
+      "new_eventlog_provider.dll",  # <new_file>
+      "eventlog_provider.patch",  # <patch_file> (temporary)
+
+      # <output_dir>
+      rebase_path("$target_gen_dir/testdata/raw_apply_fuzzer", root_build_dir),
+    ]
+
+    # Files depended upon.
+    sources = [
+      "create_seed_file_pair.py",
+      "testdata/new_eventlog_provider.dll",
+      "testdata/old_eventlog_provider.dll",
+    ]
+
+    # Outputs: necessary for validation.
+    outputs = [
+      "$target_gen_dir/testdata/raw_apply_fuzzer/seed_proto.bin",
+    ]
+    deps = [
+      "//components/zucchini:zucchini",
+      "//third_party/protobuf:protoc",
+    ]
+  }
+
+  fuzzer_test("zucchini_raw_apply_fuzzer") {
+    sources = [
+      "raw_apply_fuzzer.cc",
+    ]
+    deps = [
+      ":zucchini_file_pair_proto",
+      "//base",
+      "//components/zucchini:zucchini_lib",
+      "//third_party/libprotobuf-mutator",
+    ]
+    seed_corpus = "$target_gen_dir/testdata/raw_apply_fuzzer"
+    seed_corpus_deps = [ ":zucchini_raw_apply_seed" ]
+  }
+}
diff --git a/fuzzers/create_seed_file_pair.py b/fuzzers/create_seed_file_pair.py
new file mode 100755
index 0000000..a44db7b
--- /dev/null
+++ b/fuzzers/create_seed_file_pair.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Create binary protobuf encoding for fuzzer seeds.
+
+This script is used to generate binary encoded protobuf seeds for fuzzers
+related to Zucchini-gen and -apply, which take pairs of files are arguments. The
+binary protobuf format is faster to parse so it is the preferred method for
+encoding the seeds. For gen related fuzzers this should only need to be run
+once. For any apply related fuzzers this should be rerun whenever the patch
+format is changed.
+"""
+
+import argparse
+import logging
+import os
+import subprocess
+import sys
+
+ABS_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__)))
+PROTO_DEFINITION_FILE = 'file_pair.proto'
+OUTPUT_FORMAT = b'old_file: "{}"\nnew_or_patch_file: "{}"'
+
+def parse_args():
+  """Parse commandline args."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument('protoc_path', help='Path to protoc.')
+  parser.add_argument('old_file', help='Old file to generate/apply patch.')
+  parser.add_argument('new_or_patch_file',
+                      help='New file to generate or patch to apply.')
+  parser.add_argument('output_file', help='File to write binary protobuf to.')
+  return parser.parse_args()
+
+
+def read_to_proto_escaped_string(filename):
+  """Reads a file and converts it to hex escape sequences."""
+  with open(filename, 'rb') as f:
+    # Note that string_escape escapes all non-ASCII printable characters
+    # excluding ", which needs to be manually escaped.
+    return f.read().encode('string_escape').replace('"', '\\"')
+
+
+def main():
+  args = parse_args()
+  # Create an ASCII string representing a protobuf.
+  content = OUTPUT_FORMAT.format(read_to_proto_escaped_string(args.old_file),
+                                 read_to_proto_escaped_string(
+                                     args.new_or_patch_file))
+
+  # Encode the ASCII protobuf as a binary protobuf.
+  ps = subprocess.Popen([args.protoc_path, '--proto_path=%s' % ABS_PATH,
+                         '--encode=zucchini.fuzzers.FilePair',
+                         os.path.join(ABS_PATH, PROTO_DEFINITION_FILE)],
+                        stdin=subprocess.PIPE,
+                        stdout=subprocess.PIPE)
+  # Write the string to the subprocess. Single line IO is fine as protoc returns
+  # a string.
+  output = ps.communicate(input=content)
+  ps.wait()
+  if ps.returncode:
+    logging.error('Binary protobuf encoding failed.')
+    return ps.returncode
+
+  # Write stdout of the subprocess for protoc to the |output_file|.
+  with open(args.output_file, 'wb') as f:
+    f.write(output[0])
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/fuzzers/file_pair.proto b/fuzzers/file_pair.proto
new file mode 100644
index 0000000..2216381
--- /dev/null
+++ b/fuzzers/file_pair.proto
@@ -0,0 +1,15 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto2";
+
+package zucchini.fuzzers;
+
+// NEXT_TAG = 3
+message FilePair {
+  // File to generate patch from or apply patch to.
+  required bytes old_file = 1;
+  // New file to generate patch or the patch to apply.
+  required bytes new_or_patch_file = 2;
+}
diff --git a/fuzzers/generate_fuzzer_data.py b/fuzzers/generate_fuzzer_data.py
new file mode 100755
index 0000000..55e1edc
--- /dev/null
+++ b/fuzzers/generate_fuzzer_data.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script for generating new binary protobuf seeds for fuzzers.
+
+Currently supports creating a single seed binary protobuf of the form
+zucchini.fuzzer.FilePair.
+"""
+
+import argparse
+import hashlib
+import logging
+import os
+import platform
+import subprocess
+import sys
+
+ABS_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__)))
+ABS_TESTDATA_PATH = os.path.join(ABS_PATH, 'testdata')
+
+def parse_args():
+  """Parses arguments from command-line."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--raw', help='Whether to use Raw Zucchini.',
+                      action='store_true')
+  parser.add_argument('old_file', help='Old file to generate/apply patch.')
+  parser.add_argument('new_file', help='New file to generate patch from.')
+  parser.add_argument('patch_file', help='Patch filename to use.')
+  parser.add_argument('output_dir',
+                      help='Directory to write binary protobuf to.')
+  return parser.parse_args()
+
+
+def gen(old_file, new_file, patch_file, output_dir, is_raw, is_win):
+  """Generates a new patch and binary encodes a protobuf pair."""
+  # Create output directory if missing.
+  if not os.path.exists(output_dir):
+    os.makedirs(output_dir)
+
+  # Handle Windows executable names.
+  zucchini = 'zucchini'
+  protoc = 'protoc'
+  # NOTE: Windows can't find Python via #! so we have to help it.
+  python = sys.executable
+  if not python:
+    python = 'python'
+  if is_win:
+    if not python.lower().endswith('.exe'):
+      python += '.exe'
+    zucchini += '.exe'
+    protoc += '.exe'
+
+  zuc_cmd = [os.path.abspath(zucchini), '-gen']
+  if is_raw:
+    zuc_cmd.append('-raw')
+  # Generate a new patch.
+  ret = subprocess.call(zuc_cmd + [old_file, new_file, patch_file],
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE)
+  if ret:
+    logging.error('Patch generation failed for ({}, {})'.format(old_file,
+                                                                new_file))
+    return ret
+  # Binary encode the protobuf pair.
+  ret = subprocess.call([python,
+                         os.path.join(ABS_PATH, 'create_seed_file_pair.py'),
+                         os.path.abspath(protoc), old_file, patch_file,
+                         os.path.join(output_dir, 'seed_proto.bin')],
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE)
+  os.remove(patch_file)
+  return ret
+
+
+def main():
+  args = parse_args()
+  return gen(os.path.join(ABS_TESTDATA_PATH, args.old_file),
+             os.path.join(ABS_TESTDATA_PATH, args.new_file),
+             os.path.join(ABS_TESTDATA_PATH, args.patch_file),
+             os.path.abspath(args.output_dir),
+             args.raw,
+             platform.system() == 'Windows')
+
+
+if __name__ == '__main__':
+  sys.exit(main())
+
diff --git a/fuzzers/raw_apply_fuzzer.cc b/fuzzers/raw_apply_fuzzer.cc
new file mode 100644
index 0000000..da3230a
--- /dev/null
+++ b/fuzzers/raw_apply_fuzzer.cc
@@ -0,0 +1,59 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <iostream>
+#include <vector>
+
+#include "base/environment.h"
+#include "base/logging.h"
+#include "components/zucchini/buffer_view.h"
+#include "components/zucchini/fuzzers/file_pair.pb.h"
+#include "components/zucchini/patch_reader.h"
+#include "components/zucchini/zucchini.h"
+#include "testing/libfuzzer/proto/lpm_interface.h"
+
+struct Environment {
+  Environment() {
+    logging::SetMinLogLevel(3);  // Disable console spamming.
+  }
+};
+
+Environment* env = new Environment();
+
+DEFINE_BINARY_PROTO_FUZZER(const zucchini::fuzzers::FilePair& file_pair) {
+  // Dump code for debugging.
+  if (base::Environment::Create()->HasVar("LPM_DUMP_NATIVE_INPUT")) {
+    std::cout << "Old File: " << file_pair.old_file() << std::endl
+              << "Patch File: " << file_pair.new_or_patch_file() << std::endl;
+  }
+
+  // Prepare data.
+  zucchini::ConstBufferView old_image(
+      reinterpret_cast<const uint8_t*>(file_pair.old_file().data()),
+      file_pair.old_file().size());
+  zucchini::ConstBufferView patch_file(
+      reinterpret_cast<const uint8_t*>(file_pair.new_or_patch_file().data()),
+      file_pair.new_or_patch_file().size());
+
+  // Generate a patch reader.
+  auto patch_reader = zucchini::EnsemblePatchReader::Create(patch_file);
+  // Abort if the patch can't be read.
+  if (!patch_reader.has_value())
+    return;
+
+  // Create the underlying new file.
+  size_t new_size = patch_reader->header().new_size;
+  // Reject unreasonably large "new" files that fuzzed patch may specify.
+  if (new_size > 64 * 1024)
+    return;
+  std::vector<uint8_t> new_data(new_size);
+  zucchini::MutableBufferView new_image(new_data.data(), new_size);
+
+  // Fuzz target.
+  zucchini::Apply(old_image, *patch_reader, new_image);
+  // No need to check whether output exist, or if so, whether it's valid.
+}
diff --git a/fuzzers/testdata/.gitignore b/fuzzers/testdata/.gitignore
new file mode 100644
index 0000000..d345889
--- /dev/null
+++ b/fuzzers/testdata/.gitignore
@@ -0,0 +1,4 @@
+# Exclude testdata binaries.
+*.bin
+*.dll
+*.patch
diff --git a/fuzzers/testdata/new_eventlog_provider.dll.sha1 b/fuzzers/testdata/new_eventlog_provider.dll.sha1
new file mode 100644
index 0000000..bbf56f9
--- /dev/null
+++ b/fuzzers/testdata/new_eventlog_provider.dll.sha1
@@ -0,0 +1 @@
+89ce67035d2d2dae33cb2d98d4762e955b93df95
\ No newline at end of file
diff --git a/fuzzers/testdata/old_eventlog_provider.dll.sha1 b/fuzzers/testdata/old_eventlog_provider.dll.sha1
new file mode 100644
index 0000000..5daf440
--- /dev/null
+++ b/fuzzers/testdata/old_eventlog_provider.dll.sha1
@@ -0,0 +1 @@
+c80fdce994ba043956e192f650d894555460ff9b
\ No newline at end of file