[CIFuzz] Add support for external projects to helper.py (#6141)

Add support to helper.py for external projects for all subcommands except `download_corpora`.
External users just need to specify `build_integration_path`, `project_src_path` and `external`.

Also do a big refactor to pass a project object as part of args instead of a string containing the name of the project or its
path.
Related: #6125

6 files changed, 418 insertions, 180 deletions

diff --git a/infra/bisector.py b/infra/bisector.py
index a9df522a2..9afdc7c50 100644
--- a/infra/bisector.py
+++ b/infra/bisector.py
@@ -105,7 +105,7 @@ def main():
                                                 architecture=args.architecture)
 
   result = bisect(args.type, args.old_commit, args.new_commit,
-                  args.testcase_path, args.fuzz_target, build_data)
+                  args.test_case_path, args.fuzz_target, build_data)
   if not result.commit:
     logging.error('No error was found in commit range %s:%s', args.old_commit,
                   args.new_commit)
@@ -142,12 +142,15 @@ def _check_for_crash(project_name, fuzz_target, testcase_path):
     return utils.execute(command + args)
 
   logging.info('Checking for crash')
-  out, err, return_code = helper.reproduce_impl(project_name,
-                                                fuzz_target,
-                                                False, [], [],
-                                                testcase_path,
-                                                run_function=docker_run,
-                                                err_result=(None, None, None))
+  out, err, return_code = helper.reproduce_impl(
+      project=helper.Project(project_name),
+      fuzzer_name=fuzz_target,
+      valgrind=False,
+      env_to_add=[],
+      fuzzer_args=[],
+      testcase_path=testcase_path,
+      run_function=docker_run,
+      err_result=(None, None, None))
   if return_code is None:
     return None
 
diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py
index 798010b7b..4768275e1 100644
--- a/infra/build_specified_commit.py
+++ b/infra/build_specified_commit.py
@@ -147,7 +147,7 @@ def copy_src_from_docker(project_name, host_dir):
 @retry.wrap(_IMAGE_BUILD_TRIES, 2)
 def _build_image_with_retries(project_name):
   """Build image with retries."""
-  return helper.build_image_impl(project_name)
+  return helper.build_image_impl(helper.Project(project_name))
 
 
 def get_required_post_checkout_steps(dockerfile_path):
@@ -223,7 +223,8 @@ def build_fuzzers_from_commit(commit,
           post_checkout_step,
       ])
 
-    result = helper.build_fuzzers_impl(project_name=build_data.project_name,
+    project = helper.Project(build_data.project_name)
+    result = helper.build_fuzzers_impl(project=project,
                                        clean=True,
                                        engine=build_data.engine,
                                        sanitizer=build_data.sanitizer,
diff --git a/infra/build_specified_commit_test.py b/infra/build_specified_commit_test.py
index 7aa27509f..42adae5d0 100644
--- a/infra/build_specified_commit_test.py
+++ b/infra/build_specified_commit_test.py
@@ -61,7 +61,8 @@ class BuildImageIntegrationTest(unittest.TestCase):
       build_specified_commit.build_fuzzers_from_commit(test_repo.old_commit,
                                                        test_repo_manager,
                                                        host_src_dir, build_data)
-      old_result = helper.reproduce_impl(project_name=test_repo.project_name,
+      project = helper.Project(test_repo.project_name)
+      old_result = helper.reproduce_impl(project=project,
                                          fuzzer_name=test_repo.fuzz_target,
                                          valgrind=False,
                                          env_to_add=[],
@@ -70,9 +71,12 @@ class BuildImageIntegrationTest(unittest.TestCase):
       build_specified_commit.build_fuzzers_from_commit(test_repo.project_name,
                                                        test_repo_manager,
                                                        host_src_dir, build_data)
-      new_result = helper.reproduce_impl(test_repo.project_name,
-                                         test_repo.fuzz_target, False, [], [],
-                                         test_repo.testcase_path)
+      new_result = helper.reproduce_impl(project=project,
+                                         fuzzer_name=test_repo.fuzz_target,
+                                         valgrind=False,
+                                         env_to_add=[],
+                                         fuzzer_args=[],
+                                         testcase_path=test_repo.testcase_path)
       self.assertNotEqual(new_result, old_result)
 
   def test_detect_main_repo_from_commit(self):
diff --git a/infra/helper.py b/infra/helper.py
index 9ae49954b..246c9e775 100755
--- a/infra/helper.py
+++ b/infra/helper.py
@@ -63,9 +63,70 @@ LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust']
 
 WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
 
+# TODO(jonathanmetzman): Enforce this.
+DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH = '.cifuzz'
+
 # pylint: disable=too-many-lines
 
 
+class Project:
+  """Class representing a project that is in OSS-Fuzz or an external project
+  (ClusterFuzzLite user)."""
+
+  def __init__(self,
+               project_name_or_path,
+               is_external=False,
+               build_integration_path=DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH):
+    self.is_external = is_external
+    if self.is_external:
+      self.name = os.path.basename(os.path.abspath(project_name_or_path))
+      self.path = project_name_or_path
+      self.build_integration_path = os.path.join(self.path,
+                                                 build_integration_path)
+    else:
+      self.name = project_name_or_path
+      self.path = os.path.join(OSS_FUZZ_DIR, 'projects', self.name)
+      self.build_integration_path = self.path
+
+  @property
+  def dockerfile_path(self):
+    """Returns path to the project Dockerfile."""
+    return os.path.join(self.path, 'Dockerfile')
+
+  @property
+  def language(self):
+    """Returns project language."""
+    if self.is_external:
+      # TODO(metzman): Handle this properly.
+      return 'c++'
+
+    project_yaml_path = os.path.join(self.path, 'project.yaml')
+    with open(project_yaml_path) as file_handle:
+      content = file_handle.read()
+      for line in content.splitlines():
+        match = PROJECT_LANGUAGE_REGEX.match(line)
+        if match:
+          return match.group(1)
+
+    logging.warning('Language not specified in project.yaml.')
+    return None
+
+  @property
+  def out(self):
+    """Returns the out dir for the project. Creates it if needed."""
+    return _get_out_dir(self.name)
+
+  @property
+  def work(self):
+    """Returns the out dir for the project. Creates it if needed."""
+    return _get_project_build_subdir(self.name, 'work')
+
+  @property
+  def corpus(self):
+    """Returns the out dir for the project. Creates it if needed."""
+    return _get_project_build_subdir(self.name, 'corpus')
+
+
 def main():  # pylint: disable=too-many-branches,too-many-return-statements
   """Gets subcommand from program arguments and does it. Returns 0 on success 1
   on error."""
@@ -121,10 +182,37 @@ def bool_to_retcode(boolean):
 
 
 def parse_args(parser, args=None):
-  """Parses |args| using |parser| and returns parsed args."""
+  """Parses |args| using |parser| and returns parsed args. Also changes
+  |args.build_integration_path| to have correct default behavior."""
   # Use default argument None for args so that in production, argparse does its
   # normal behavior, but unittesting is easier.
-  return parser.parse_args(args)
+  parsed_args = parser.parse_args(args)
+  project = getattr(parsed_args, 'project', None)
+  if not project:
+    return parsed_args
+
+  # Use hacky method for extracting attributes so that ShellTest works.
+  # TODO(metzman): Fix this.
+  is_external = getattr(parsed_args, 'is_external', False)
+  build_integration_path = getattr(parsed_args, 'build_integration_path', False)
+
+  parsed_args.project = Project(parsed_args.project, is_external,
+                                build_integration_path)
+  return parsed_args
+
+
+def _add_external_project_args(parser):
+  parser.add_argument('--build-integration-path',
+                      help=('Path to the build integration for non-OSS-Fuzz '
+                            'projects.'),
+                      default=DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH)
+
+  parser.add_argument(
+      '--external',
+      help='Is project external?',
+      default=False,
+      action='store_true',
+  )
 
 
 def get_parser():  # pylint: disable=too-many-statements
@@ -134,11 +222,12 @@ def get_parser():  # pylint: disable=too-many-statements
 
   generate_parser = subparsers.add_parser(
       'generate', help='Generate files for new project.')
-  generate_parser.add_argument('project_name')
+  generate_parser.add_argument('project')
+  _add_external_project_args(generate_parser)
 
   build_image_parser = subparsers.add_parser('build_image',
                                              help='Build an image.')
-  build_image_parser.add_argument('project_name')
+  build_image_parser.add_argument('project')
   build_image_parser.add_argument('--pull',
                                   action='store_true',
                                   help='Pull latest base image.')
@@ -149,6 +238,7 @@ def get_parser():  # pylint: disable=too-many-statements
   build_image_parser.add_argument('--no-pull',
                                   action='store_true',
                                   help='Do not pull latest base image.')
+  _add_external_project_args(build_image_parser)
 
   build_fuzzers_parser = subparsers.add_parser(
       'build_fuzzers', help='Build fuzzers for a project.')
@@ -156,7 +246,8 @@ def get_parser():  # pylint: disable=too-many-statements
   _add_engine_args(build_fuzzers_parser)
   _add_sanitizer_args(build_fuzzers_parser)
   _add_environment_args(build_fuzzers_parser)
-  build_fuzzers_parser.add_argument('project_name')
+  _add_external_project_args(build_fuzzers_parser)
+  build_fuzzers_parser.add_argument('project')
   build_fuzzers_parser.add_argument('source_path',
                                     help='path of local source',
                                     nargs='?')
@@ -185,19 +276,23 @@ def get_parser():  # pylint: disable=too-many-statements
       check_build_parser,
       choices=['address', 'memory', 'undefined', 'dataflow', 'thread'])
   _add_environment_args(check_build_parser)
-  check_build_parser.add_argument('project_name', help='name of the project')
+  check_build_parser.add_argument('project',
+                                  help='name of the project or path (external)')
   check_build_parser.add_argument('fuzzer_name',
                                   help='name of the fuzzer',
                                   nargs='?')
+  _add_external_project_args(check_build_parser)
 
   run_fuzzer_parser = subparsers.add_parser(
       'run_fuzzer', help='Run a fuzzer in the emulated fuzzing environment.')
   _add_engine_args(run_fuzzer_parser)
   _add_sanitizer_args(run_fuzzer_parser)
   _add_environment_args(run_fuzzer_parser)
+  _add_external_project_args(run_fuzzer_parser)
   run_fuzzer_parser.add_argument(
       '--corpus-dir', help='directory to store corpus for the fuzz target')
-  run_fuzzer_parser.add_argument('project_name', help='name of the project')
+  run_fuzzer_parser.add_argument('project',
+                                 help='name of the project or path (external)')
   run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer')
   run_fuzzer_parser.add_argument('fuzzer_args',
                                  help='arguments to pass to the fuzzer',
@@ -221,35 +316,40 @@ def get_parser():  # pylint: disable=too-many-statements
   coverage_parser.add_argument('--corpus-dir',
                                help='specify location of corpus'
                                ' to be used (requires --fuzz-target argument)')
-  coverage_parser.add_argument('project_name', help='name of the project')
+  coverage_parser.add_argument('project',
+                               help='name of the project or path (external)')
   coverage_parser.add_argument('extra_args',
                                help='additional arguments to '
                                'pass to llvm-cov utility.',
                                nargs='*')
+  _add_external_project_args(coverage_parser)
 
   download_corpora_parser = subparsers.add_parser(
       'download_corpora', help='Download all corpora for a project.')
   download_corpora_parser.add_argument('--fuzz-target',
                                        help='specify name of a fuzz target')
-  download_corpora_parser.add_argument('project_name',
-                                       help='name of the project')
+  download_corpora_parser.add_argument(
+      'project', help='name of the project or path (external)')
 
   reproduce_parser = subparsers.add_parser('reproduce',
                                            help='Reproduce a crash.')
   reproduce_parser.add_argument('--valgrind',
                                 action='store_true',
                                 help='run with valgrind')
-  reproduce_parser.add_argument('project_name', help='name of the project')
+  reproduce_parser.add_argument('project',
+                                help='name of the project or path (external)')
   reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
   reproduce_parser.add_argument('testcase_path', help='path of local testcase')
   reproduce_parser.add_argument('fuzzer_args',
                                 help='arguments to pass to the fuzzer',
                                 nargs=argparse.REMAINDER)
   _add_environment_args(reproduce_parser)
+  _add_external_project_args(reproduce_parser)
 
   shell_parser = subparsers.add_parser(
       'shell', help='Run /bin/bash within the builder container.')
-  shell_parser.add_argument('project_name', help='name of the project')
+  shell_parser.add_argument('project',
+                            help='name of the project or path (external)')
   shell_parser.add_argument('source_path',
                             help='path of local source',
                             nargs='?')
@@ -257,6 +357,7 @@ def get_parser():  # pylint: disable=too-many-statements
   _add_engine_args(shell_parser)
   _add_sanitizer_args(shell_parser)
   _add_environment_args(shell_parser)
+  _add_external_project_args(shell_parser)
 
   subparsers.add_parser('pull_images', help='Pull base images.')
   return parser
@@ -267,19 +368,19 @@ def is_base_image(image_name):
   return os.path.exists(os.path.join('infra', 'base-images', image_name))
 
 
-def check_project_exists(project_name):
+def check_project_exists(project):
   """Checks if a project exists."""
-  if not os.path.exists(_get_project_dir(project_name)):
-    logging.error('%s does not exist.', project_name)
+  if not os.path.exists(project.path):
+    logging.error('%s does not exist.', project.name)
     return False
 
   return True
 
 
-def _check_fuzzer_exists(project_name, fuzzer_name):
+def _check_fuzzer_exists(project, fuzzer_name):
   """Checks if a fuzzer exists."""
   command = ['docker', 'run', '--rm']
-  command.extend(['-v', '%s:/out' % _get_out_dir(project_name)])
+  command.extend(['-v', '%s:/out' % project.out])
   command.append('ubuntu:16.04')
 
   command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])
@@ -304,56 +405,20 @@ def _get_command_string(command):
   return ' '.join(pipes.quote(part) for part in command)
 
 
-def _get_project_dir(project_name):
-  """Returns path to the project."""
-  return os.path.join(OSS_FUZZ_DIR, 'projects', project_name)
-
-
-def get_dockerfile_path(project_name):
-  """Returns path to the project Dockerfile."""
-  return os.path.join(_get_project_dir(project_name), 'Dockerfile')
-
-
-def _get_project_build_subdir(project_name, subdir_name):
-  """Creates the |subdir_name| subdirectory of the |project_name| subdirectory
-  in |BUILD_DIR| and returns its path."""
-  directory = os.path.join(BUILD_DIR, subdir_name, project_name)
+def _get_project_build_subdir(project, subdir_name):
+  """Creates the |subdir_name| subdirectory of the |project| subdirectory in
+  |BUILD_DIR| and returns its path."""
+  directory = os.path.join(BUILD_DIR, subdir_name, project)
   if not os.path.exists(directory):
     os.makedirs(directory)
 
   return directory
 
 
-def _get_corpus_dir(project_name=''):
-  """Creates and returns path to /corpus directory for the given project (if
-  specified)."""
-  return _get_project_build_subdir(project_name, 'corpus')
-
-
-def _get_out_dir(project_name=''):
+def _get_out_dir(project=''):
   """Creates and returns path to /out directory for the given project (if
   specified)."""
-  return _get_project_build_subdir(project_name, 'out')
-
-
-def _get_work_dir(project_name=''):
-  """Creates and returns path to /work directory for the given project (if
-  specified)."""
-  return _get_project_build_subdir(project_name, 'work')
-
-
-def _get_project_language(project_name):
-  """Returns project language."""
-  project_yaml_path = os.path.join(OSS_FUZZ_DIR, 'projects', project_name,
-                                   'project.yaml')
-  with open(project_yaml_path) as file_handle:
-    content = file_handle.read()
-    for line in content.splitlines():
-      match = PROJECT_LANGUAGE_REGEX.match(line)
-      if match:
-        return match.group(1)
-
-  return None
+  return _get_project_build_subdir(project, 'out')
 
 
 def _add_architecture_args(parser, choices=('x86_64', 'i386')):
@@ -386,17 +451,27 @@ def _add_environment_args(parser):
                       help="set environment variable e.g. VAR=value")
 
 
-def build_image_impl(image_name, cache=True, pull=False):
+def build_image_impl(project, cache=True, pull=False):
   """Builds image."""
-  proj_is_base_image = is_base_image(image_name)
-  if proj_is_base_image:
+  image_name = project.name
+
+  if is_base_image(image_name):
     image_project = 'oss-fuzz-base'
-    dockerfile_dir = os.path.join('infra', 'base-images', image_name)
-  else:
+    docker_build_dir = os.path.join(OSS_FUZZ_DIR, 'infra', 'base-images',
+                                    image_name)
+    docker_file_path = None
+  elif project.is_external:
+    # External projects need to use the repo root as the build directory.
+    docker_file_path = os.path.join(project.build_integration_path,
+                                    'Dockerfile')
+    docker_build_dir = project.path
     image_project = 'oss-fuzz'
-    if not check_project_exists(image_name):
+  else:
+    if not check_project_exists(project):
       return False
-    dockerfile_dir = os.path.join('projects', image_name)
+    docker_file_path = None
+    docker_build_dir = project.path
+    image_project = 'oss-fuzz'
 
   if pull and not pull_images():
     return False
@@ -405,9 +480,13 @@ def build_image_impl(image_name, cache=True, pull=False):
   if not cache:
     build_args.append('--no-cache')
 
-  build_args += [
-      '-t', 'gcr.io/%s/%s' % (image_project, image_name), dockerfile_dir
-  ]
+  build_args += ['-t', 'gcr.io/%s/%s' % (image_project, image_name)]
+  if docker_file_path:
+    build_args += [
+        '--file',
+        docker_file_path,
+    ]
+  build_args.append(docker_build_dir)
   return docker_build(build_args)
 
 
@@ -432,14 +511,12 @@ def workdir_from_lines(lines, default='/src'):
   return default
 
 
-def _workdir_from_dockerfile(project_name):
+def _workdir_from_dockerfile(project):
   """Parses WORKDIR from the Dockerfile for the given project."""
-  dockerfile_path = get_dockerfile_path(project_name)
-
-  with open(dockerfile_path) as file_handle:
+  with open(project.dockerfile_path) as file_handle:
     lines = file_handle.readlines()
 
-  return workdir_from_lines(lines, default=os.path.join('/src', project_name))
+  return workdir_from_lines(lines, default=os.path.join('/src', project.name))
 
 
 def docker_run(run_args, print_output=True):
@@ -514,14 +591,14 @@ def build_image(args):
     logging.error('Using cached base images...')
 
   # If build_image is called explicitly, don't use cache.
-  if build_image_impl(args.project_name, cache=args.cache, pull=pull):
+  if build_image_impl(args.project, cache=args.cache, pull=pull):
     return True
 
   return False
 
 
 def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
-    project_name,
+    project,
     clean,
     engine,
     sanitizer,
@@ -530,29 +607,23 @@ def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,to
     source_path,
     mount_path=None):
   """Builds fuzzers."""
-  if not build_image_impl(project_name):
+  if not build_image_impl(project):
     return False
 
-  project_out_dir = _get_out_dir(project_name)
-  project_work_dir = _get_work_dir(project_name)
-  project_language = _get_project_language(project_name)
-  if not project_language:
-    logging.warning('Language not specified in project.yaml. Build may fail.')
-
   if clean:
     logging.info('Cleaning existing build artifacts.')
 
     # Clean old and possibly conflicting artifacts in project's out directory.
     docker_run([
         '-v',
-        '%s:/out' % project_out_dir, '-t',
-        'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /out/*'
+        '%s:/out' % project.out, '-t',
+        'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /out/*'
     ])
 
     docker_run([
         '-v',
-        '%s:/work' % project_work_dir, '-t',
-        'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /work/*'
+        '%s:/work' % project.work, '-t',
+        'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /work/*'
     ])
 
   else:
@@ -565,8 +636,8 @@ def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,to
 
   _add_oss_fuzz_ci_if_needed(env)
 
-  if project_language:
-    env.append('FUZZING_LANGUAGE=' + project_language)
+  if project.language:
+    env.append('FUZZING_LANGUAGE=' + project.language)
 
   if env_to_add:
     env += env_to_add
@@ -575,14 +646,14 @@ def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,to
   if sanitizer == 'memory':
     docker_run([
         '-v',
-        '%s:/work' % project_work_dir, 'gcr.io/oss-fuzz-base/msan-libs-builder',
+        '%s:/work' % project.work, 'gcr.io/oss-fuzz-base/msan-libs-builder',
         'bash', '-c', 'cp -r /msan /work'
     ])
     env.append('MSAN_LIBS_PATH=' + '/work/msan')
 
   command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env)
   if source_path:
-    workdir = _workdir_from_dockerfile(project_name)
+    workdir = _workdir_from_dockerfile(project)
     if mount_path:
       command += [
           '-v',
@@ -600,9 +671,9 @@ def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,to
 
   command += [
       '-v',
-      '%s:/out' % project_out_dir, '-v',
-      '%s:/work' % project_work_dir, '-t',
-      'gcr.io/oss-fuzz/%s' % project_name
+      '%s:/out' % project.out, '-v',
+      '%s:/work' % project.work, '-t',
+      'gcr.io/oss-fuzz/%s' % project.name
   ]
 
   result = docker_run(command)
@@ -612,21 +683,19 @@ def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,to
 
   # Patch MSan builds to use instrumented shared libraries.
   if sanitizer == 'memory':
-    docker_run([
-        '-v',
-        '%s:/out' % project_out_dir, '-v',
-        '%s:/work' % project_work_dir
-    ] + _env_to_docker_args(env) + [
-        'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', 'patch_build.py',
-        '/out'
-    ])
+    docker_run(
+        ['-v', '%s:/out' % project.out, '-v',
+         '%s:/work' % project.work] + _env_to_docker_args(env) + [
+             'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder',
+             'patch_build.py', '/out'
+         ])
 
   return True
 
 
 def build_fuzzers(args):
   """Builds fuzzers."""
-  return build_fuzzers_impl(args.project_name,
+  return build_fuzzers_impl(args.project,
                             args.clean,
                             args.engine,
                             args.sanitizer,
@@ -645,15 +714,15 @@ def _add_oss_fuzz_ci_if_needed(env):
 
 def check_build(args):
   """Checks that fuzzers in the container execute without errors."""
-  if not check_project_exists(args.project_name):
+  if not check_project_exists(args.project):
     return False
 
   if (args.fuzzer_name and
-      not _check_fuzzer_exists(args.project_name, args.fuzzer_name)):
+      not _check_fuzzer_exists(args.project, args.fuzzer_name)):
     return False
 
-  fuzzing_language = _get_project_language(args.project_name)
-  if fuzzing_language is None:
+  fuzzing_language = args.project.language
+  if not fuzzing_language:
     logging.warning(
         'Language not specified in project.yaml. Defaulting to C++.')
     fuzzing_language = 'c++'
@@ -670,8 +739,7 @@ def check_build(args):
 
   run_args = _env_to_docker_args(env) + [
       '-v',
-      '%s:/out' % _get_out_dir(args.project_name), '-t',
-      'gcr.io/oss-fuzz-base/base-runner'
+      '%s:/out' % args.project.out, '-t', 'gcr.io/oss-fuzz-base/base-runner'
   ]
 
   if args.fuzzer_name:
@@ -688,10 +756,10 @@ def check_build(args):
   return result
 
 
-def _get_fuzz_targets(project_name):
+def _get_fuzz_targets(project):
   """Returns names of fuzz targest build in the project's /out directory."""
   fuzz_targets = []
-  for name in os.listdir(_get_out_dir(project_name)):
+  for name in os.listdir(project.out):
     if name.startswith('afl-'):
       continue
     if name.startswith('jazzer_'):
@@ -699,7 +767,7 @@ def _get_fuzz_targets(project_name):
     if name == 'llvm-symbolizer':
       continue
 
-    path = os.path.join(_get_out_dir(project_name), name)
+    path = os.path.join(project.out, name)
     # Python and JVM fuzz targets are only executable for the root user, so
     # we can't use os.access.
     if os.path.isfile(path) and (os.stat(path).st_mode & 0o111):
@@ -750,7 +818,7 @@ def _get_latest_corpus(project_name, fuzz_target, base_corpus_dir):
 
 def download_corpora(args):
   """Downloads most recent corpora from GCS for the given project."""
-  if not check_project_exists(args.project_name):
+  if not check_project_exists(args.project):
     return False
 
   try:
@@ -764,22 +832,20 @@ def download_corpora(args):
   if args.fuzz_target:
     fuzz_targets = [args.fuzz_target]
   else:
-    fuzz_targets = _get_fuzz_targets(args.project_name)
+    fuzz_targets = _get_fuzz_targets(args.project)
 
-  corpus_dir = _get_corpus_dir(args.project_name)
-  if not os.path.exists(corpus_dir):
-    os.makedirs(corpus_dir)
+  corpus_dir = args.project.corpus
 
   def _download_for_single_target(fuzz_target):
     try:
-      _get_latest_corpus(args.project_name, fuzz_target, corpus_dir)
+      _get_latest_corpus(args.project, fuzz_target, corpus_dir)
       return True
     except Exception as error:  # pylint:disable=broad-except
       logging.error('Corpus download for %s failed: %s.', fuzz_target,
                     str(error))
       return False
 
-  logging.info('Downloading corpora for %s project to %s.', args.project_name,
+  logging.info('Downloading corpora for %s project to %s.', args.project.name,
                corpus_dir)
   thread_pool = ThreadPool()
   return all(thread_pool.map(_download_for_single_target, fuzz_targets))
@@ -793,24 +859,24 @@ def coverage(args):
         '--fuzz-target')
     return False
 
-  if not check_project_exists(args.project_name):
+  if not check_project_exists(args.project):
     return False
 
-  project_language = _get_project_language(args.project_name)
-  if project_language not in LANGUAGES_WITH_COVERAGE_SUPPORT:
+  if args.project.language not in LANGUAGES_WITH_COVERAGE_SUPPORT:
     logging.error(
         'Project is written in %s, coverage for it is not supported yet.',
-        project_language)
+        args.project.language)
     return False
 
-  if not args.no_corpus_download and not args.corpus_dir:
+  if (not args.no_corpus_download and not args.corpus_dir and
+      not args.project.is_external):
     if not download_corpora(args):
       return False
 
   env = [
       'FUZZING_ENGINE=libfuzzer',
-      'FUZZING_LANGUAGE=%s' % project_language,
-      'PROJECT=%s' % args.project_name,
+      'FUZZING_LANGUAGE=%s' % args.project.language,
+      'PROJECT=%s' % args.project.name,
       'SANITIZER=coverage',
       'HTTP_PORT=%s' % args.port,
       'COVERAGE_EXTRA_ARGS=%s' % ' '.join(args.extra_args),
@@ -832,11 +898,11 @@ def coverage(args):
     corpus_dir = os.path.realpath(args.corpus_dir)
     run_args.extend(['-v', '%s:/corpus/%s' % (corpus_dir, args.fuzz_target)])
   else:
-    run_args.extend(['-v', '%s:/corpus' % _get_corpus_dir(args.project_name)])
+    run_args.extend(['-v', '%s:/corpus' % args.project.corpus])
 
   run_args.extend([
       '-v',
-      '%s:/out' % _get_out_dir(args.project_name),
+      '%s:/out' % args.project.out,
       '-t',
       'gcr.io/oss-fuzz-base/base-runner',
   ])
@@ -856,10 +922,10 @@ def coverage(args):
 
 def run_fuzzer(args):
   """Runs a fuzzer in the container."""
-  if not check_project_exists(args.project_name):
+  if not check_project_exists(args.project):
     return False
 
-  if not _check_fuzzer_exists(args.project_name, args.fuzzer_name):
+  if not _check_fuzzer_exists(args.project, args.fuzzer_name):
     return False
 
   env = [
@@ -886,7 +952,7 @@ def run_fuzzer(args):
 
   run_args.extend([
       '-v',
-      '%s:/out' % _get_out_dir(args.project_name),
+      '%s:/out' % args.project.out,
       '-t',
       'gcr.io/oss-fuzz-base/base-runner',
       'run_fuzzer',
@@ -898,12 +964,12 @@ def run_fuzzer(args):
 
 def reproduce(args):
   """Reproduces a specific test case from a specific project."""
-  return reproduce_impl(args.project_name, args.fuzzer_name, args.valgrind,
-                        args.e, args.fuzzer_args, args.testcase_path)
+  return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e,
+                        args.fuzzer_args, args.testcase_path)
 
 
 def reproduce_impl(  # pylint: disable=too-many-arguments
-    project_name,
+    project,
     fuzzer_name,
     valgrind,
     env_to_add,
@@ -912,10 +978,10 @@ def reproduce_impl(  # pylint: disable=too-many-arguments
     run_function=docker_run,
     err_result=False):
   """Reproduces a testcase in the container."""
-  if not check_project_exists(project_name):
+  if not check_project_exists(project):
     return err_result
 
-  if not _check_fuzzer_exists(project_name, fuzzer_name):
+  if not _check_fuzzer_exists(project, fuzzer_name):
     return err_result
 
   debugger = ''
@@ -934,7 +1000,7 @@ def reproduce_impl(  # pylint: disable=too-many-arguments
 
   run_args = _env_to_docker_args(env) + [
       '-v',
-      '%s:/out' % _get_out_dir(project_name),
+      '%s:/out' % project.out,
       '-v',
       '%s:/testcase' % _get_absolute_path(testcase_path),
       '-t',
@@ -947,51 +1013,85 @@ def reproduce_impl(  # pylint: disable=too-many-arguments
   return run_function(run_args)
 
 
-def generate(args):
-  """Generates empty project files."""
-  if len(args.project_name) > MAX_PROJECT_NAME_LENGTH:
+def _validate_project_name(project_name):
+  """Validates |project_name| is a valid OSS-Fuzz project name."""
+  if len(project_name) > MAX_PROJECT_NAME_LENGTH:
     logging.error(
         'Project name needs to be less than or equal to %d characters.',
         MAX_PROJECT_NAME_LENGTH)
     return False
 
-  if not VALID_PROJECT_NAME_REGEX.match(args.project_name):
-    logging.error('Invalid project name.')
+  if not VALID_PROJECT_NAME_REGEX.match(project_name):
+    logging.info('Invalid project name: %s.', project_name)
     return False
 
-  directory = os.path.join('projects', args.project_name)
+  return True
+
 
+def _create_build_integration_directory(directory):
+  """Returns True on successful creation of a build integration directory.
+  Suitable for OSS-Fuzz and external projects."""
   try:
-    os.mkdir(directory)
+    os.makedirs(directory)
   except OSError as error:
     if error.errno != errno.EEXIST:
       raise
     logging.error('%s already exists.', directory)
     return False
+  return True
 
-  logging.info('Writing new files to %s.', directory)
 
-  template_args = {
-      'project_name': args.project_name,
-      'year': datetime.datetime.now().year
-  }
-  with open(os.path.join(directory, 'project.yaml'), 'w') as file_handle:
-    file_handle.write(templates.PROJECT_YAML_TEMPLATE % template_args)
+def _template_project_file(filename, template, template_args, directory):
+  """Templates |template| using |template_args| and writes the result to
+  |directory|/|filename|. Sets the file to executable if |filename| is
+  build.sh."""
+  file_path = os.path.join(directory, filename)
+  with open(file_path, 'w') as file_handle:
+    file_handle.write(template % template_args)
 
-  with open(os.path.join(directory, 'Dockerfile'), 'w') as file_handle:
-    file_handle.write(templates.DOCKER_TEMPLATE % template_args)
+  if filename == 'build.sh':
+    os.chmod(file_path, 0o755)
 
-  build_sh_path = os.path.join(directory, 'build.sh')
-  with open(build_sh_path, 'w') as file_handle:
-    file_handle.write(templates.BUILD_TEMPLATE % template_args)
 
-  os.chmod(build_sh_path, 0o755)
+def generate(args):
+  """Generates empty project files."""
+  return _generate_impl(args.project)
+
+
+def _get_current_datetime():
+  """Returns this year. Needed for mocking."""
+  return datetime.datetime.now()
+
+
+def _generate_impl(project):
+  """Implementation of generate(). Useful for testing."""
+  if project.is_external:
+    # External project.
+    project_templates = templates.EXTERNAL_TEMPLATES
+  else:
+    # Internal project.
+    if not _validate_project_name(project.name):
+      return False
+    project_templates = templates.TEMPLATES
+
+  directory = project.build_integration_path
+  if not _create_build_integration_directory(directory):
+    return False
+
+  logging.info('Writing new files to: %s.', directory)
+
+  template_args = {
+      'project_name': project.name,
+      'year': _get_current_datetime().year
+  }
+  for filename, template in project_templates.items():
+    _template_project_file(filename, template, template_args, directory)
   return True
 
 
 def shell(args):
   """Runs a shell within a docker image."""
-  if not build_image_impl(args.project_name):
+  if not build_image_impl(args.project):
     return False
 
   env = [
@@ -1000,18 +1100,18 @@ def shell(args):
       'ARCHITECTURE=' + args.architecture,
   ]
 
-  if args.project_name != 'base-runner-debug':
-    env.append('FUZZING_LANGUAGE=' + _get_project_language(args.project_name))
+  if args.project.name != 'base-runner-debug':
+    env.append('FUZZING_LANGUAGE=' + args.project.language)
 
   if args.e:
     env += args.e
 
-  if is_base_image(args.project_name):
+  if is_base_image(args.project.name):
     image_project = 'oss-fuzz-base'
     out_dir = _get_out_dir()
   else:
     image_project = 'oss-fuzz'
-    out_dir = _get_out_dir(args.project_name)
+    out_dir = args.project.out
 
   run_args = _env_to_docker_args(env)
   if args.source_path:
@@ -1023,8 +1123,8 @@ def shell(args):
   run_args.extend([
       '-v',
       '%s:/out' % out_dir, '-v',
-      '%s:/work' % _get_work_dir(args.project_name), '-t',
-      'gcr.io/%s/%s' % (image_project, args.project_name), '/bin/bash'
+      '%s:/work' % args.project.work, '-t',
+      'gcr.io/%s/%s' % (image_project, args.project.name), '/bin/bash'
   ])
 
   docker_run(run_args)
diff --git a/infra/helper_test.py b/infra/helper_test.py
index 7ebbc3adf..cb9e6a659 100644
--- a/infra/helper_test.py
+++ b/infra/helper_test.py
@@ -13,18 +13,25 @@
 # limitations under the License.
 """Tests for helper.py"""
 
+import datetime
+import os
 import unittest
 from unittest import mock
 
+from pyfakefs import fake_filesystem_unittest
+
 import helper
+import templates
+
+# pylint: disable=no-self-use,protected-access
 
 
-class TestShell(unittest.TestCase):
+class ShellTest(unittest.TestCase):
   """Tests 'shell' command."""
 
   @mock.patch('helper.docker_run')
   @mock.patch('helper.build_image_impl')
-  def test_base_runner_debug(self, mocked_build_image_impl, _):
+  def test_base_runner_debug(self, _, __):
     """Tests that shell base-runner-debug works as intended."""
     image_name = 'base-runner-debug'
     unparsed_args = ['shell', image_name]
@@ -32,5 +39,95 @@ class TestShell(unittest.TestCase):
     args = helper.parse_args(parser, unparsed_args)
     args.sanitizer = 'address'
     result = helper.shell(args)
-    mocked_build_image_impl.assert_called_with(image_name)
     self.assertTrue(result)
+
+
+class BuildImageImplTest(unittest.TestCase):
+  """Tests for build_image_impl."""
+
+  @mock.patch('helper.docker_build')
+  def test_no_cache(self, mocked_docker_build):
+    """Tests that cache=False is handled properly."""
+    image_name = 'base-image'
+    helper.build_image_impl(helper.Project(image_name), cache=False)
+    self.assertIn('--no-cache', mocked_docker_build.call_args_list[0][0][0])
+
+  @mock.patch('helper.docker_build')
+  @mock.patch('helper.pull_images')
+  def test_pull(self, mocked_pull_images, _):
+    """Tests that pull=True is handled properly."""
+    image_name = 'base-image'
+    helper.build_image_impl(helper.Project(image_name), pull=True)
+    mocked_pull_images.assert_called_with()
+
+  @mock.patch('helper.docker_build')
+  def test_base_image(self, mocked_docker_build):
+    """Tests that build_image_impl works as intended with a base-image."""
+    image_name = 'base-image'
+    helper.build_image_impl(helper.Project(image_name))
+    mocked_docker_build.assert_called_with([
+        '-t', 'gcr.io/oss-fuzz-base/base-image',
+        os.path.join(helper.OSS_FUZZ_DIR, 'infra/base-images/base-image')
+    ])
+
+  @mock.patch('helper.docker_build')
+  def test_oss_fuzz_project(self, mocked_docker_build):
+    """Tests that build_image_impl works as intended with an OSS-Fuzz
+    project."""
+    project_name = 'example'
+    helper.build_image_impl(helper.Project(project_name))
+    mocked_docker_build.assert_called_with([
+        '-t', 'gcr.io/oss-fuzz/example',
+        os.path.join(helper.OSS_FUZZ_DIR, 'projects/example')
+    ])
+
+  @mock.patch('helper.docker_build')
+  def test_external_project(self, mocked_docker_build):
+    """Tests that build_image_impl works as intended with a non-OSS-Fuzz
+    project."""
+    project_src_path = '/example'
+    build_integration_path = 'build-integration'
+    project = helper.Project(project_src_path,
+                             is_external=True,
+                             build_integration_path=build_integration_path)
+    helper.build_image_impl(project)
+    mocked_docker_build.assert_called_with([
+        '-t', 'gcr.io/oss-fuzz/example', '--file',
+        '/example/build-integration/Dockerfile', project_src_path
+    ])
+
+
+class GenerateImplTest(fake_filesystem_unittest.TestCase):
+  """Tests for _generate_impl."""
+  PROJECT_NAME = 'newfakeproject'
+
+  def setUp(self):
+    self.setUpPyfakefs()
+    self.fs.add_real_directory(helper.OSS_FUZZ_DIR)
+
+  def _verify_templated_files(self, template_dict, directory):
+    template_args = {'project_name': self.PROJECT_NAME, 'year': 2021}
+    for filename, template in template_dict.items():
+      file_path = os.path.join(directory, filename)
+      with open(file_path, 'r') as file_handle:
+        contents = file_handle.read()
+      self.assertEqual(contents, template % template_args)
+
+  @mock.patch('helper._get_current_datetime',
+              return_value=datetime.datetime(year=2021, month=1, day=1))
+  def test_generate_oss_fuzz_project(self, _):
+    """Tests that the correct files are generated for an OSS-Fuzz project."""
+    helper._generate_impl(helper.Project(self.PROJECT_NAME))
+    self._verify_templated_files(
+        templates.TEMPLATES,
+        os.path.join(helper.OSS_FUZZ_DIR, 'projects', self.PROJECT_NAME))
+
+  def test_generate_external_project(self):
+    """Tests that the correct files are generated for a non-OSS-Fuzz project."""
+    build_integration_path = '/newfakeproject/build-integration'
+    helper._generate_impl(
+        helper.Project('/newfakeproject/',
+                       is_external=True,
+                       build_integration_path=build_integration_path))
+    self._verify_templated_files(templates.EXTERNAL_TEMPLATES,
+                                 build_integration_path)
diff --git a/infra/templates.py b/infra/templates.py
index f16da924f..66e3e78c2 100755
--- a/infra/templates.py
+++ b/infra/templates.py
@@ -46,6 +46,14 @@ WORKDIR %(project_name)s
 COPY build.sh $SRC/
 """
 
+EXTERNAL_DOCKER_TEMPLATE = """\
+FROM gcr.io/oss-fuzz-base/base-builder
+RUN apt-get update && apt-get install -y make autoconf automake libtool
+RUN git clone --depth 1 <git_url> %(project_name)s     # or use other version control
+WORKDIR %(project_name)s
+COPY build.sh $SRC/
+"""
+
 BUILD_TEMPLATE = """\
 #!/bin/bash -eu
 # Copyright %(year)d Google LLC
@@ -76,3 +84,28 @@ BUILD_TEMPLATE = """\
 #     /path/to/name_of_fuzzer.cc -o $OUT/name_of_fuzzer \\
 #     $LIB_FUZZING_ENGINE /path/to/library.a
 """
+
+EXTERNAL_BUILD_TEMPLATE = """\
+# build project
+# e.g.
+# ./autogen.sh
+# ./configure
+# make -j$(nproc) all
+
+# build fuzzers
+# e.g.
+# $CXX $CXXFLAGS -std=c++11 -Iinclude \\
+#     /path/to/name_of_fuzzer.cc -o $OUT/name_of_fuzzer \\
+#     $LIB_FUZZING_ENGINE /path/to/library.a
+"""
+
+TEMPLATES = {
+    'build.sh': BUILD_TEMPLATE,
+    'Dockerfile': DOCKER_TEMPLATE,
+    'project.yaml': PROJECT_YAML_TEMPLATE
+}
+
+EXTERNAL_TEMPLATES = {
+    'build.sh': EXTERNAL_BUILD_TEMPLATE,
+    'Dockerfile': EXTERNAL_DOCKER_TEMPLATE
+}