[cifuzz] Fuzz in cifuzz-base (#6142)

Fixes: #5926

20 files changed, 249 insertions, 185 deletions

diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile
index 3f2c70316..3ac07158e 100755
--- a/infra/base-images/base-runner/Dockerfile
+++ b/infra/base-images/base-runner/Dockerfile
@@ -50,7 +50,8 @@ RUN apt-get update && apt-get install -y \
     wget \
     zip --no-install-recommends
 
-RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage /opt/code_coverage && \
+ENV CODE_COVERAGE_SRC=/opt/code_coverage
+RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage $CODE_COVERAGE_SRC && \
     cd /opt/code_coverage && \
     git checkout edba4873b5e8a390e977a64c522db2df18a8b27d && \
     pip3 install wheel && \
diff --git a/infra/base-images/base-runner/coverage_helper b/infra/base-images/base-runner/coverage_helper
index 22c9cb5d6..4d29ceac8 100755
--- a/infra/base-images/base-runner/coverage_helper
+++ b/infra/base-images/base-runner/coverage_helper
@@ -14,4 +14,4 @@
 # limitations under the License.
 #
 ################################################################################
-python3 /opt/code_coverage/coverage_utils.py $@
+python3 $CODE_COVERAGE_SRC/coverage_utils.py $@
diff --git a/infra/build_fuzzers.Dockerfile b/infra/build_fuzzers.Dockerfile
index 6e8adf958..77a34ae40 100644
--- a/infra/build_fuzzers.Dockerfile
+++ b/infra/build_fuzzers.Dockerfile
@@ -13,7 +13,8 @@
 # limitations under the License.
 #
 ################################################################################
-# Docker image to run the CIFuzz action build_fuzzers in.
+# Docker image to run fuzzers for CIFuzz (the run_fuzzers action on GitHub
+# actions).
 
 FROM gcr.io/oss-fuzz-base/cifuzz-base
 
diff --git a/infra/ci/requirements.txt b/infra/ci/requirements.txt
index f0a8be0b5..ab17b712c 100644
--- a/infra/ci/requirements.txt
+++ b/infra/ci/requirements.txt
@@ -6,3 +6,5 @@ pytest==6.2.1
 pytest-xdist==2.2.0
 PyYAML==5.4
 yapf==0.30.0
+# Needed for cifuzz tests.
+Jinja2==2.10
diff --git a/infra/cifuzz/base_runner_utils.py b/infra/cifuzz/base_runner_utils.py
new file mode 100644
index 000000000..246375481
--- /dev/null
+++ b/infra/cifuzz/base_runner_utils.py
@@ -0,0 +1,33 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for scripts from gcr.io/oss-fuzz-base/base-runner."""
+
+import os
+
+import config_utils
+
+
+def get_env(config, workspace):
+  """Returns a dictionary containing the current environment with additional env
+  vars set to values needed to run a fuzzer."""
+  env = os.environ.copy()
+  env['SANITIZER'] = config.sanitizer
+  env['FUZZING_LANGUAGE'] = config.language
+  env['OUT'] = workspace.out
+  env['CIFUZZ'] = 'True'
+  env['FUZZING_ENGINE'] = config_utils.DEFAULT_ENGINE
+  env['ARCHITECTURE'] = config_utils.DEFAULT_ARCHITECTURE
+  # Do this so we don't fail in tests.
+  env['FUZZER_ARGS'] = '-rss_limit_mb=2560 -timeout=25'
+  return env
diff --git a/infra/cifuzz/build_fuzzers.py b/infra/cifuzz/build_fuzzers.py
index 970004061..0b8f9afa5 100644
--- a/infra/cifuzz/build_fuzzers.py
+++ b/infra/cifuzz/build_fuzzers.py
@@ -19,6 +19,7 @@ import os
 import sys
 
 import affected_fuzz_targets
+import base_runner_utils
 import clusterfuzz_deployment
 import continuous_integration
 import docker
@@ -27,6 +28,7 @@ import workspace_utils
 # pylint: disable=wrong-import-position,import-error
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import helper
+import utils
 
 logging.basicConfig(
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
@@ -180,22 +182,16 @@ def build_fuzzers(config):
   return builder.build()
 
 
-def check_fuzzer_build(workspace,
-                       sanitizer,
-                       language,
-                       allowed_broken_targets_percentage=None):
+def check_fuzzer_build(config):
   """Checks the integrity of the built fuzzers.
 
   Args:
-    workspace: The workspace used by CIFuzz.
-    sanitizer: The sanitizer the fuzzers are built with.
-    language: The programming language the fuzzers are written in.
-    allowed_broken_targets_percentage (optional): A custom percentage of broken
-        targets to allow.
+    config: The config object.
 
   Returns:
     True if fuzzers pass OSS-Fuzz's build check.
   """
+  workspace = workspace_utils.Workspace(config)
   if not os.path.exists(workspace.out):
     logging.error('Invalid out directory: %s.', workspace.out)
     return False
@@ -203,21 +199,13 @@ def check_fuzzer_build(workspace,
     logging.error('No fuzzers found in out directory: %s.', workspace.out)
     return False
 
-  docker_args, _ = docker.get_base_docker_run_args(workspace, sanitizer,
-                                                   language)
-  if allowed_broken_targets_percentage is not None:
-    docker_args += [
-        '-e',
-        ('ALLOWED_BROKEN_TARGETS_PERCENTAGE=' +
-         allowed_broken_targets_percentage)
-    ]
+  env = base_runner_utils.get_env(config, workspace)
+  if config.allowed_broken_targets_percentage is not None:
+    env['ALLOWED_BROKEN_TARGETS_PERCENTAGE'] = (
+        config.allowed_broken_targets_percentage)
 
-  docker_args.extend(['-t', docker.BASE_RUNNER_TAG, 'test_all.py'])
-  result = helper.docker_run(docker_args)
-  if not result:
-    logging.error('Check fuzzer build failed.')
-    return False
-  return True
+  _, _, retcode = utils.execute('test_all.py', env=env)
+  return retcode == 0
 
 
 def _get_docker_build_fuzzers_args_not_container(host_repo_path):
diff --git a/infra/cifuzz/build_fuzzers_entrypoint.py b/infra/cifuzz/build_fuzzers_entrypoint.py
index 5aed2e615..c128ae937 100644
--- a/infra/cifuzz/build_fuzzers_entrypoint.py
+++ b/infra/cifuzz/build_fuzzers_entrypoint.py
@@ -17,7 +17,6 @@ import sys
 
 import build_fuzzers
 import config_utils
-import workspace_utils
 
 # pylint: disable=c-extension-no-member
 # pylint gets confused because of the relative import of cifuzz.
@@ -47,14 +46,7 @@ def build_fuzzers_entrypoint():
     # If we've gotten to this point and we don't need to do bad_build_check,
     # then the build has succeeded.
     returncode = 0
-  # yapf: disable
-  elif build_fuzzers.check_fuzzer_build(
-      workspace_utils.Workspace(config),
-      config.sanitizer,
-      config.language,
-      allowed_broken_targets_percentage=config.allowed_broken_targets_percentage
-  ):
-    # yapf: enable
+  elif build_fuzzers.check_fuzzer_build(config):
     returncode = 0
 
   return returncode
diff --git a/infra/cifuzz/build_fuzzers_test.py b/infra/cifuzz/build_fuzzers_test.py
index e9a2182d7..e2d65002b 100644
--- a/infra/cifuzz/build_fuzzers_test.py
+++ b/infra/cifuzz/build_fuzzers_test.py
@@ -266,44 +266,45 @@ class CheckFuzzerBuildTest(unittest.TestCase):
   def setUp(self):
     self.temp_dir_obj = tempfile.TemporaryDirectory()
     workspace_path = os.path.join(self.temp_dir_obj.name, 'workspace')
+    self.config = test_helpers.create_build_config(
+        oss_fuzz_project_name=EXAMPLE_PROJECT,
+        sanitizer=self.SANITIZER,
+        language=self.LANGUAGE,
+        workspace=workspace_path,
+        pr_ref='refs/pull/1757/merge')
     self.workspace = test_helpers.create_workspace(workspace_path)
     shutil.copytree(TEST_DATA_PATH, workspace_path)
+    test_helpers.patch_environ(self, runner=True)
 
   def tearDown(self):
     self.temp_dir_obj.cleanup()
 
   def test_correct_fuzzer_build(self):
     """Checks check_fuzzer_build function returns True for valid fuzzers."""
-    self.assertTrue(
-        build_fuzzers.check_fuzzer_build(self.workspace, self.SANITIZER,
-                                         self.LANGUAGE))
+    self.assertTrue(build_fuzzers.check_fuzzer_build(self.config))
 
   def test_not_a_valid_path(self):
     """Tests that False is returned when a nonexistent path is given."""
-    workspace = test_helpers.create_workspace('not/a/valid/path')
-    self.assertFalse(
-        build_fuzzers.check_fuzzer_build(workspace, self.SANITIZER,
-                                         self.LANGUAGE))
+    self.config.workspace = 'not/a/valid/path'
+    self.assertFalse(build_fuzzers.check_fuzzer_build(self.config))
 
   def test_no_valid_fuzzers(self):
     """Tests that False is returned when an empty directory is given."""
     with tempfile.TemporaryDirectory() as tmp_dir:
-      workspace = test_helpers.create_workspace(tmp_dir)
-    self.assertFalse(
-        build_fuzzers.check_fuzzer_build(workspace, self.SANITIZER,
-                                         self.LANGUAGE))
+      self.config.workspace = tmp_dir
+      os.mkdir(os.path.join(self.config.workspace, 'build-out'))
+      self.assertFalse(build_fuzzers.check_fuzzer_build(self.config))
 
-  @mock.patch('helper.docker_run')
-  def test_allow_broken_fuzz_targets_percentage(self, mocked_docker_run):
+  @mock.patch('utils.execute', return_value=(None, None, 0))
+  def test_allow_broken_fuzz_targets_percentage(self, mocked_execute):
     """Tests that ALLOWED_BROKEN_TARGETS_PERCENTAGE is set when running
     docker if passed to check_fuzzer_build."""
-    mocked_docker_run.return_value = 0
-    build_fuzzers.check_fuzzer_build(self.workspace,
-                                     self.SANITIZER,
-                                     self.LANGUAGE,
-                                     allowed_broken_targets_percentage='0')
-    self.assertIn('-e ALLOWED_BROKEN_TARGETS_PERCENTAGE=0',
-                  ' '.join(mocked_docker_run.call_args[0][0]))
+    percentage = '0'
+    self.config.allowed_broken_targets_percentage = percentage
+    build_fuzzers.check_fuzzer_build(self.config)
+    self.assertEqual(
+        mocked_execute.call_args[1]['env']['ALLOWED_BROKEN_TARGETS_PERCENTAGE'],
+        percentage)
 
 
 @unittest.skip('Test is too long to be run with presubmit.')
diff --git a/infra/cifuzz/cifuzz-base/Dockerfile b/infra/cifuzz/cifuzz-base/Dockerfile
index 001e46ff5..d5ea93e21 100644
--- a/infra/cifuzz/cifuzz-base/Dockerfile
+++ b/infra/cifuzz/cifuzz-base/Dockerfile
@@ -14,17 +14,11 @@
 #
 ################################################################################
 
-# Don't bother with a slimmer base image.
-# When we pull base-builder to build project builder image we need to pull
-# ubuntu:16.04 anyway. So in the long run we probably would waste time if
-# we pulled something like alpine here instead.
-FROM ubuntu:16.04
+FROM gcr.io/oss-fuzz-base/base-runner
 
 RUN apt-get update && \
-    apt-get install ca-certificates wget git-core --no-install-recommends -y && \
     wget https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce-cli_20.10.5~3-0~ubuntu-xenial_amd64.deb -O /tmp/docker-ce.deb && \
-    dpkg -i /tmp/docker-ce.deb && rm /tmp/docker-ce.deb && \
-    apt-get remove wget -y --purge
+    dpkg -i /tmp/docker-ce.deb && rm /tmp/docker-ce.deb
 
 # Install newer Python from base-builder.
 COPY --from=gcr.io/oss-fuzz-base/base-builder /usr/local/bin/python3 /usr/local/bin/python3
diff --git a/infra/cifuzz/cifuzz_end_to_end_test.py b/infra/cifuzz/cifuzz_end_to_end_test.py
index 2250874ae..2a4234faf 100644
--- a/infra/cifuzz/cifuzz_end_to_end_test.py
+++ b/infra/cifuzz/cifuzz_end_to_end_test.py
@@ -13,7 +13,6 @@
 # limitations under the License.
 """End-to-End tests for CIFuzz."""
 import os
-import tempfile
 import unittest
 
 import run_cifuzz
@@ -34,13 +33,14 @@ class EndToEndTest(unittest.TestCase):
   """End-to-End tests for CIFuzz."""
 
   def setUp(self):
-    test_helpers.patch_environ(self)
+    test_helpers.patch_environ(self, runner=True)
 
   def test_simple(self):
     """Simple end-to-end test using run_cifuzz.main()."""
     os.environ['REPOSITORY'] = 'external-project'
     os.environ['PROJECT_SRC_PATH'] = EXTERNAL_PROJECT_PATH
-    with tempfile.TemporaryDirectory() as temp_dir:
+
+    with test_helpers.docker_temp_dir() as temp_dir:
       os.environ['WORKSPACE'] = temp_dir
-      # TODO(metzman): Verify the crash, affected fuzzers and other things.
+      # TODO(metzman): Verify the crash, affected fuzzers, and other things.
       self.assertEqual(run_cifuzz.main(), 1)
diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py
index cac076212..8ca3eec14 100644
--- a/infra/cifuzz/config_utils.py
+++ b/infra/cifuzz/config_utils.py
@@ -29,6 +29,10 @@ import constants
 RUN_FUZZERS_MODES = ['batch', 'ci', 'coverage']
 SANITIZERS = ['address', 'memory', 'undefined', 'coverage']
 
+# TODO(metzman): Set these on config objects so there's one source of truth.
+DEFAULT_ENGINE = 'libfuzzer'
+DEFAULT_ARCHITECTURE = 'x86_64'
+
 # This module deals a lot with env variables. Many of these will be set by users
 # and others beyond CIFuzz's control. Thus, you should be careful about using
 # the environment.py helpers for getting env vars, since it can cause values
diff --git a/infra/cifuzz/docker.py b/infra/cifuzz/docker.py
index e20e5ca00..d0bad5d05 100644
--- a/infra/cifuzz/docker.py
+++ b/infra/cifuzz/docker.py
@@ -23,10 +23,10 @@ import constants
 import utils
 
 BASE_BUILDER_TAG = 'gcr.io/oss-fuzz-base/base-builder'
-BASE_RUNNER_TAG = 'gcr.io/oss-fuzz-base/base-runner'
 MSAN_LIBS_BUILDER_TAG = 'gcr.io/oss-fuzz-base/msan-libs-builder'
 PROJECT_TAG_PREFIX = 'gcr.io/oss-fuzz/'
 
+# Default fuzz configuration.
 _DEFAULT_DOCKER_RUN_ARGS = [
     '--cap-add', 'SYS_PTRACE', '-e',
     'FUZZING_ENGINE=' + constants.DEFAULT_ENGINE, '-e',
diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py
index e972b55b5..14a63a1ed 100644
--- a/infra/cifuzz/fuzz_target.py
+++ b/infra/cifuzz/fuzz_target.py
@@ -21,8 +21,7 @@ import stat
 import subprocess
 import sys
 
-import docker
-
+import base_runner_utils
 # pylint: disable=wrong-import-position,import-error
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import utils
@@ -92,31 +91,25 @@ class FuzzTarget:  # pylint: disable=too-many-instance-attributes
       FuzzResult namedtuple with stacktrace and testcase if applicable.
     """
     logging.info('Running fuzzer: %s.', self.target_name)
-    command, _ = docker.get_base_docker_run_command(self.workspace,
-                                                    self.config.sanitizer,
-                                                    self.config.language)
+    env = base_runner_utils.get_env(self.config, self.workspace)
+    # TODO(metzman): Is this needed?
+    env['RUN_FUZZER_MODE'] = 'interactive'
 
-    # If corpus can be downloaded use it for fuzzing.
+    # If corpus can be downloaded, use it for fuzzing.
     self.latest_corpus_path = self.clusterfuzz_deployment.download_corpus(
         self.target_name)
-    command += docker.get_docker_env_vars({
-        'CORPUS_DIR': self.latest_corpus_path,
-        'RUN_FUZZER_MODE': 'interactive'
-    })
-
-    command += [docker.BASE_RUNNER_TAG, 'bash', '-c']
+    env['CORPUS_DIR'] = self.latest_corpus_path
 
     options = LIBFUZZER_OPTIONS.copy() + [
         f'-max_total_time={self.duration}',
         # Make sure libFuzzer artifact files don't pollute $OUT.
         f'-artifact_prefix={self.workspace.artifacts}/'
     ]
-    options = ' '.join(options)
-    run_fuzzer_command = f'run_fuzzer {self.target_name} {options}'
-    command.append(run_fuzzer_command)
+    command = ['run_fuzzer', self.target_name] + options
 
-    logging.info('Running command: %s', ' '.join(command))
+    logging.info('Running command: %s', command)
     process = subprocess.Popen(command,
+                               env=env,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
 
@@ -192,20 +185,14 @@ class FuzzTarget:  # pylint: disable=too-many-instance-attributes
 
     os.chmod(target_path, stat.S_IRWXO)
 
-    command, container = docker.get_base_docker_run_command(
-        self.workspace, self.config.sanitizer, self.config.language)
-    if container:
-      command += docker.get_docker_env_vars({'TESTCASE': testcase})
-    else:
-      command += ['-v', f'{testcase}:/testcase']
-
-    command += [
-        '-t', docker.BASE_RUNNER_TAG, 'reproduce', self.target_name, '-runs=100'
-    ]
+    env = base_runner_utils.get_env(self.config, self.workspace)
+    env['TESTCASE'] = testcase
+    command = ['reproduce', self.target_name, '-runs=100']
 
     logging.info('Running reproduce command: %s.', ' '.join(command))
     for _ in range(REPRODUCE_ATTEMPTS):
-      _, _, returncode = utils.execute(command)
+      _, _, returncode = utils.execute(command, env=env)
+
       if returncode != 0:
         logging.info('Reproduce command returned: %s. Reproducible on %s.',
                      returncode, target_path)
diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py
index 1ff01e266..268d3a64d 100644
--- a/infra/cifuzz/fuzz_target_test.py
+++ b/infra/cifuzz/fuzz_target_test.py
@@ -71,6 +71,7 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase):
     """Sets up example fuzz target to test is_reproducible method."""
     self.fuzz_target_name = 'fuzz-target'
     deployment = _create_deployment()
+    self.config = deployment.config
     self.workspace = deployment.workspace
     self.fuzz_target_path = os.path.join(self.workspace.out,
                                          self.fuzz_target_name)
@@ -80,6 +81,8 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase):
                                          self.workspace, deployment,
                                          deployment.config)
 
+    test_helpers.patch_environ(self, empty=True)
+
   def test_reproducible(self, _):
     """Tests that is_reproducible returns True if crash is detected and that
     is_reproducible uses the correct command to reproduce a crash."""
@@ -88,16 +91,18 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase):
     with mock.patch('utils.execute', side_effect=all_repro) as mocked_execute:
       result = self.target.is_reproducible(self.testcase_path,
                                            self.fuzz_target_path)
-      mocked_execute.assert_called_once_with([
-          'docker', 'run', '--rm', '--privileged', '--cap-add', 'SYS_PTRACE',
-          '-e', 'FUZZING_ENGINE=libfuzzer', '-e', 'ARCHITECTURE=x86_64', '-e',
-          'CIFUZZ=True', '-e', 'SANITIZER=' + self.target.config.sanitizer,
-          '-e', 'FUZZING_LANGUAGE=' + self.target.config.language, '-e',
-          'OUT=' + self.workspace.out, '--volumes-from', 'container', '-e',
-          'TESTCASE=' + self.testcase_path, '-t',
-          'gcr.io/oss-fuzz-base/base-runner', 'reproduce',
-          self.fuzz_target_name, '-runs=100'
-      ])
+      expected_command = ['reproduce', 'fuzz-target', '-runs=100']
+      expected_env = {
+          'SANITIZER': self.config.sanitizer,
+          'FUZZING_LANGUAGE': 'c++',
+          'OUT': self.workspace.out,
+          'CIFUZZ': 'True',
+          'FUZZING_ENGINE': 'libfuzzer',
+          'ARCHITECTURE': 'x86_64',
+          'TESTCASE': self.testcase_path,
+          'FUZZER_ARGS': '-rss_limit_mb=2560 -timeout=25'
+      }
+      mocked_execute.assert_called_once_with(expected_command, env=expected_env)
       self.assertTrue(result)
       self.assertEqual(1, mocked_execute.call_count)
 
diff --git a/infra/cifuzz/generate_coverage_report.py b/infra/cifuzz/generate_coverage_report.py
index 1d392c15a..2bfbe51d6 100644
--- a/infra/cifuzz/generate_coverage_report.py
+++ b/infra/cifuzz/generate_coverage_report.py
@@ -14,25 +14,19 @@
 """Module for generating coverage reports."""
 import os
 
-import helper
-import docker
+import base_runner_utils
+import utils
 
 
-def run_coverage_command(workspace, config):
+def run_coverage_command(config, workspace):
   """Runs the coverage command in base-runner to generate a coverage report."""
-  docker_args, _ = docker.get_base_docker_run_args(workspace, config.sanitizer,
-                                                   config.language)
-  env_mapping = {
-      'COVERAGE_EXTRA_ARGS': '',
-      'HTTP_PORT': '',
-      'CORPUS_DIR': workspace.corpora,
-      'COVERAGE_OUTPUT_DIR': workspace.coverage_report
-  }
-  docker_args += docker.get_docker_env_vars(env_mapping)
-
-  docker_args += ['-t', docker.BASE_RUNNER_TAG, 'coverage']
-
-  return helper.docker_run(docker_args)
+  env = base_runner_utils.get_env(config, workspace)
+  env['HTTP_PORT'] = ''
+  env['COVERAGE_EXTRA_ARGS'] = ''
+  env['CORPUS_DIR'] = workspace.corpora
+  env['COVERAGE_OUTPUT_DIR'] = workspace.coverage_report
+  command = 'coverage'
+  return utils.execute(command, env=env)
 
 
 def download_corpora(fuzz_target_paths, clusterfuzz_deployment):
@@ -47,5 +41,5 @@ def generate_coverage_report(fuzz_target_paths, workspace,
                              clusterfuzz_deployment, config):
   """Generates a coverage report using Clang's source based coverage."""
   download_corpora(fuzz_target_paths, clusterfuzz_deployment)
-  run_coverage_command(workspace, config)
+  run_coverage_command(config, workspace)
   clusterfuzz_deployment.upload_coverage()
diff --git a/infra/cifuzz/generate_coverage_report_test.py b/infra/cifuzz/generate_coverage_report_test.py
index c34939f0f..bed494308 100644
--- a/infra/cifuzz/generate_coverage_report_test.py
+++ b/infra/cifuzz/generate_coverage_report_test.py
@@ -27,26 +27,31 @@ SANITIZER = 'coverage'
 class TestRunCoverageCommand(unittest.TestCase):
   """Tests run_coverage_command"""
 
-  @mock.patch('helper.docker_run')
-  def test_run_coverage_command(self, mocked_docker_run):  # pylint: disable=no-self-use
+  def setUp(self):
+    test_helpers.patch_environ(self, empty=True)
+
+  @mock.patch('utils.execute')
+  def test_run_coverage_command(self, mocked_execute):  # pylint: disable=no-self-use
     """Tests that run_coverage_command works as intended."""
     config = test_helpers.create_run_config(oss_fuzz_project_name=PROJECT,
                                             sanitizer=SANITIZER)
     workspace = test_helpers.create_workspace()
-    expected_docker_args = [
-        '--cap-add', 'SYS_PTRACE', '-e', 'FUZZING_ENGINE=libfuzzer', '-e',
-        'ARCHITECTURE=x86_64', '-e', 'CIFUZZ=True', '-e',
-        f'SANITIZER={SANITIZER}', '-e', 'FUZZING_LANGUAGE=c++', '-e',
-        'OUT=/workspace/build-out', '-v',
-        f'{workspace.workspace}:{workspace.workspace}', '-e',
-        'COVERAGE_EXTRA_ARGS=', '-e', 'HTTP_PORT=', '-e',
-        f'CORPUS_DIR={workspace.corpora}', '-e',
-        f'COVERAGE_OUTPUT_DIR={workspace.coverage_report}', '-t',
-        'gcr.io/oss-fuzz-base/base-runner', 'coverage'
-    ]
-
-    generate_coverage_report.run_coverage_command(workspace, config)
-    mocked_docker_run.assert_called_with(expected_docker_args)
+    generate_coverage_report.run_coverage_command(config, workspace)
+    expected_command = 'coverage'
+    expected_env = {
+        'SANITIZER': config.sanitizer,
+        'FUZZING_LANGUAGE': config.language,
+        'OUT': workspace.out,
+        'CIFUZZ': 'True',
+        'FUZZING_ENGINE': 'libfuzzer',
+        'ARCHITECTURE': 'x86_64',
+        'FUZZER_ARGS': '-rss_limit_mb=2560 -timeout=25',
+        'HTTP_PORT': '',
+        'COVERAGE_EXTRA_ARGS': '',
+        'CORPUS_DIR': workspace.corpora,
+        'COVERAGE_OUTPUT_DIR': workspace.coverage_report
+    }
+    mocked_execute.assert_called_with(expected_command, env=expected_env)
 
 
 class DownloadCorporaTest(unittest.TestCase):
diff --git a/infra/cifuzz/run_fuzzers_test.py b/infra/cifuzz/run_fuzzers_test.py
index e91ddbaf6..5b79ac5b4 100644
--- a/infra/cifuzz/run_fuzzers_test.py
+++ b/infra/cifuzz/run_fuzzers_test.py
@@ -23,7 +23,6 @@ from unittest import mock
 import parameterized
 from pyfakefs import fake_filesystem_unittest
 
-import docker
 import build_fuzzers
 import fuzz_target
 import run_fuzzers
@@ -59,6 +58,10 @@ class RunFuzzerIntegrationTestMixin:  # pylint: disable=too-few-public-methods,i
   FUZZER_DIR = None
   FUZZER = None
 
+  def setUp(self):
+    """Patch the environ so that we can execute runner scripts."""
+    test_helpers.patch_environ(self, runner=True)
+
   def _test_run_with_sanitizer(self, fuzzer_dir, sanitizer):
     """Calls run_fuzzers on fuzzer_dir and |sanitizer| and asserts
     the run succeeded and that no bug was found."""
@@ -341,7 +344,7 @@ class CoverageReportIntegrationTest(unittest.TestCase):
   SANITIZER = 'coverage'
 
   def setUp(self):
-    test_helpers.patch_environ(self)
+    test_helpers.patch_environ(self, runner=True)
 
   @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client'
               '.upload_artifact',
@@ -350,45 +353,56 @@ class CoverageReportIntegrationTest(unittest.TestCase):
     """Tests generation of coverage reports end-to-end, from building to
     generation."""
 
-    with tempfile.TemporaryDirectory() as workspace:
-      try:
-        # Do coverage build.
-        build_config = test_helpers.create_build_config(
-            oss_fuzz_project_name=EXAMPLE_PROJECT,
-            project_repo_name='oss-fuzz',
-            workspace=workspace,
-            commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523',
-            base_commit='da0746452433dc18bae699e355a9821285d863c8',
-            sanitizer=self.SANITIZER,
-            is_github=True)
-        self.assertTrue(build_fuzzers.build_fuzzers(build_config))
-
-        # Generate report.
-        run_config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS,
-                                                    workspace=workspace,
-                                                    sanitizer=self.SANITIZER,
-                                                    run_fuzzers_mode='coverage',
-                                                    is_github=True)
-        result = run_fuzzers.run_fuzzers(run_config)
-        self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND)
-        expected_summary_path = os.path.join(
-            TEST_DATA_PATH, 'example_coverage_report_summary.json')
-        with open(expected_summary_path) as file_handle:
-          expected_summary = json.loads(file_handle.read())
-        actual_summary_path = os.path.join(workspace, 'cifuzz-coverage',
+    with test_helpers.docker_temp_dir() as temp_dir:
+      shared = os.path.join(temp_dir, 'shared')
+      os.mkdir(shared)
+      copy_command = ('cp -r /opt/code_coverage /shared && '
+                      'cp $(which llvm-profdata) /shared && '
+                      'cp $(which llvm-cov) /shared')
+      assert helper.docker_run([
+          '-v', f'{shared}:/shared', 'gcr.io/oss-fuzz-base/base-runner', 'bash',
+          '-c', copy_command
+      ])
+
+      os.environ['CODE_COVERAGE_SRC'] = os.path.join(shared, 'code_coverage')
+      os.environ['PATH'] += os.pathsep + shared
+      # Do coverage build.
+      build_config = test_helpers.create_build_config(
+          oss_fuzz_project_name=EXAMPLE_PROJECT,
+          project_repo_name='oss-fuzz',
+          workspace=temp_dir,
+          commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523',
+          base_commit='da0746452433dc18bae699e355a9821285d863c8',
+          sanitizer=self.SANITIZER,
+          is_github=True)
+      self.assertTrue(build_fuzzers.build_fuzzers(build_config))
+
+      # TODO(metzman): Get rid of this here and make 'compile' do this.
+      chmod_command = ('chmod -R +r /out && '
+                       'find /out -type d -exec chmod +x {} +')
+
+      assert helper.docker_run([
+          '-v', f'{os.path.join(temp_dir, "build-out")}:/out',
+          'gcr.io/oss-fuzz-base/base-builder', 'bash', '-c', chmod_command
+      ])
+
+      # Generate report.
+      run_config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS,
+                                                  workspace=temp_dir,
+                                                  sanitizer=self.SANITIZER,
+                                                  run_fuzzers_mode='coverage',
+                                                  is_github=True)
+      result = run_fuzzers.run_fuzzers(run_config)
+      self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND)
+      expected_summary_path = os.path.join(
+          TEST_DATA_PATH, 'example_coverage_report_summary.json')
+      with open(expected_summary_path) as file_handle:
+        expected_summary = json.loads(file_handle.read())
+        actual_summary_path = os.path.join(temp_dir, 'cifuzz-coverage',
                                            'report', 'linux', 'summary.json')
-        with open(actual_summary_path) as file_handle:
-          actual_summary = json.loads(file_handle.read())
-        self.assertEqual(expected_summary, actual_summary)
-      finally:
-        # If we don't do this, there will be an exception when the temporary
-        # directory is deleted because there are files there that are only
-        # writeable by root.
-        if os.listdir(workspace):
-          helper.docker_run([
-              '-v', f'{workspace}:/workspace', '-t', docker.BASE_RUNNER_TAG,
-              '/bin/bash', '-c', 'rm -rf /workspace/*'
-          ])
+      with open(actual_summary_path) as file_handle:
+        actual_summary = json.loads(file_handle.read())
+      self.assertEqual(expected_summary, actual_summary)
 
 
 @unittest.skipIf(not os.getenv('INTEGRATION_TESTS'),
diff --git a/infra/cifuzz/test_helpers.py b/infra/cifuzz/test_helpers.py
index 6fefba95b..85b5a8a67 100644
--- a/infra/cifuzz/test_helpers.py
+++ b/infra/cifuzz/test_helpers.py
@@ -15,13 +15,21 @@
 
 import contextlib
 import os
+import sys
 import shutil
 import tempfile
 from unittest import mock
 
 import config_utils
+import docker
 import workspace_utils
 
+INFRA_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# pylint: disable=wrong-import-position,import-error
+sys.path.append(INFRA_DIR)
+
+import helper
+
 
 @mock.patch('config_utils._is_dry_run', return_value=True)
 @mock.patch('config_utils.GenericCiEnvironment.project_src_path',
@@ -57,14 +65,32 @@ def create_workspace(workspace_path='/workspace'):
   return workspace_utils.Workspace(config)
 
 
-def patch_environ(testcase_obj, env=None):
-  """Patch environment."""
+def patch_environ(testcase_obj, env=None, empty=False, runner=False):
+  """Patch environment. |testcase_obj| is the unittest.TestCase that contains
+  tests. |env|, if specified, is a dictionary of environment variables to start
+  from. If |empty| is True then the new patched environment will be empty. If
+  |runner| is True then the necessary environment variables will be set to run
+  the scripts from base-runner."""
   if env is None:
     env = {}
 
   patcher = mock.patch.dict(os.environ, env)
   testcase_obj.addCleanup(patcher.stop)
   patcher.start()
+  if empty:
+    for key in os.environ.copy():
+      del os.environ[key]
+
+  if runner:
+    # Add the scripts for base-runner to the path since the wont be in
+    # /usr/local/bin on host machines during testing.
+    base_runner_dir = os.path.join(INFRA_DIR, 'base-images', 'base-runner')
+    os.environ['PATH'] = (os.environ.get('PATH', '') + os.pathsep +
+                          base_runner_dir)
+    if 'GOPATH' not in os.environ:
+      # A GOPATH must be set or else the coverage script fails, even for getting
+      # the coverage of non-Go programs.
+      os.environ['GOPATH'] = '/root/go'
 
 
 @contextlib.contextmanager
@@ -74,3 +100,17 @@ def temp_dir_copy(directory):
     temp_copy_path = os.path.join(temp_dir, os.path.basename(directory))
     shutil.copytree(directory, temp_copy_path)
     yield temp_copy_path
+
+
+@contextlib.contextmanager
+def docker_temp_dir():
+  """Returns a temporary a directory that is useful for use with docker. On
+  cleanup this contextmanager uses docker to delete the directory's contents so
+  that if anything is owned by root it can be deleted (which
+  tempfile.TemporaryDirectory() cannot do) by non-root users."""
+  with tempfile.TemporaryDirectory() as temp_dir:
+    yield temp_dir
+    helper.docker_run([
+        '-v', f'{temp_dir}:/temp_dir', '-t', docker.BASE_BUILDER_TAG,
+        '/bin/bash', '-c', 'rm -rf /temp_dir/*'
+    ])
diff --git a/infra/run_fuzzers.Dockerfile b/infra/run_fuzzers.Dockerfile
index aab7b24fb..8c8d7bb1b 100644
--- a/infra/run_fuzzers.Dockerfile
+++ b/infra/run_fuzzers.Dockerfile
@@ -13,7 +13,8 @@
 # limitations under the License.
 #
 ################################################################################
-# Docker image to run the CIFuzz action run_fuzzers in.
+# Docker image for running fuzzers on CIFuzz (the run_fuzzers action on GitHub
+# actions).
 
 FROM gcr.io/oss-fuzz-base/cifuzz-base
 
diff --git a/infra/utils.py b/infra/utils.py
index 1f814b27e..d991a285e 100644
--- a/infra/utils.py
+++ b/infra/utils.py
@@ -38,16 +38,17 @@ def chdir_to_root():
     os.chdir(helper.OSS_FUZZ_DIR)
 
 
-def execute(command, location=None, check_result=False):
-  """ Runs a shell command in the specified directory location.
+def execute(command, env=None, location=None, check_result=False):
+  """Runs a shell command in the specified directory location.
 
   Args:
     command: The command as a list to be run.
+    env: (optional) an environment to pass to Popen to run the command in.
     location: The directory the command is run in.
     check_result: Should an exception be thrown on failed command.
 
   Returns:
-    stdout, stderr, error code.
+    stdout, stderr, return code.
 
   Raises:
     RuntimeError: running a command resulted in an error.
@@ -58,7 +59,8 @@ def execute(command, location=None, check_result=False):
   process = subprocess.Popen(command,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
-                             cwd=location)
+                             cwd=location,
+                             env=env)
   out, err = process.communicate()
   out = out.decode('utf-8', errors='ignore')
   err = err.decode('utf-8', errors='ignore')