From 571dbfb39f1b13b121693dfae880158d06b92169 Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Thu, 6 Aug 2020 14:33:08 -0700 Subject: Reland 55d9a81. (#4269) The issue with 55d9a81 that caused it to be reverted by 3370f8f was probably due to the build trigger on GCB being stale. I've updated it so this commit should stick. --- .dockerignore | 10 ++++ .../build_fuzzers/build_fuzzers_entrypoint.py | 3 +- infra/cifuzz/cifuzz-base/Dockerfile | 2 +- infra/cifuzz/cifuzz.py | 67 ++++++++++++++-------- 4 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..61ac0b13e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +out +.git + +# Copied from .gitignore. +.vscode/ +*.pyc +/build/ +*~ +.DS_Store +*.swp diff --git a/infra/cifuzz/actions/build_fuzzers/build_fuzzers_entrypoint.py b/infra/cifuzz/actions/build_fuzzers/build_fuzzers_entrypoint.py index 74cd2d495..f46e6390e 100644 --- a/infra/cifuzz/actions/build_fuzzers/build_fuzzers_entrypoint.py +++ b/infra/cifuzz/actions/build_fuzzers/build_fuzzers_entrypoint.py @@ -97,7 +97,8 @@ def main(): out_dir = os.path.join(workspace, 'out') if cifuzz.check_fuzzer_build(out_dir, sanitizer=sanitizer): - return 0 + returncode = 0 + return returncode diff --git a/infra/cifuzz/cifuzz-base/Dockerfile b/infra/cifuzz/cifuzz-base/Dockerfile index ecb889a01..3d887b4d5 100644 --- a/infra/cifuzz/cifuzz-base/Dockerfile +++ b/infra/cifuzz/cifuzz-base/Dockerfile @@ -34,4 +34,4 @@ RUN add-apt-repository \ RUN apt-get update && apt-get install docker-ce docker-ce-cli containerd.io -y ENV OSS_FUZZ_ROOT=/opt/oss-fuzz -RUN git clone https://github.com/google/oss-fuzz.git ${OSS_FUZZ_ROOT} +ADD . ${OSS_FUZZ_ROOT} diff --git a/infra/cifuzz/cifuzz.py b/infra/cifuzz/cifuzz.py index 1d3d0e2f7..e20dc45ac 100644 --- a/infra/cifuzz/cifuzz.py +++ b/infra/cifuzz/cifuzz.py @@ -79,6 +79,20 @@ logging.basicConfig( level=logging.DEBUG) +def checkout_specified_commit(build_repo_manager, pr_ref, commit_sha): + """Checks out the specified commit or pull request using + build_repo_manager.""" + try: + if pr_ref: + build_repo_manager.checkout_pr(pr_ref) + else: + build_repo_manager.checkout_commit(commit_sha) + except (RuntimeError, ValueError): + logging.error( + 'Can not check out requested state %s. ' + 'Using current repo state', pr_ref or commit_sha) + + # pylint: disable=too-many-arguments # pylint: disable=too-many-locals def build_fuzzers(project_name, @@ -109,34 +123,40 @@ def build_fuzzers(project_name, logging.info("Using %s sanitizer.", sanitizer) - git_workspace = os.path.join(workspace, 'storage') - os.makedirs(git_workspace, exist_ok=True) out_dir = os.path.join(workspace, 'out') os.makedirs(out_dir, exist_ok=True) - # Detect repo information. - inferred_url, oss_fuzz_repo_path = build_specified_commit.detect_main_repo( - project_name, repo_name=project_repo_name) - if not inferred_url or not oss_fuzz_repo_path: + # Build Fuzzers using docker run. + inferred_url, project_builder_repo_path = ( + build_specified_commit.detect_main_repo(project_name, + repo_name=project_repo_name)) + if not inferred_url or not project_builder_repo_path: logging.error('Could not detect repo from project %s.', project_name) return False - src_in_docker = os.path.dirname(oss_fuzz_repo_path) - oss_fuzz_repo_name = os.path.basename(oss_fuzz_repo_path) + project_repo_name = os.path.basename(project_builder_repo_path) + src_in_project_builder = os.path.dirname(project_builder_repo_path) + + manual_src_path = os.getenv('MANUAL_SRC_PATH') + if manual_src_path: + if not os.path.exists(manual_src_path): + logging.error( + 'MANUAL_SRC_PATH: %s does not exist. ' + 'Are you mounting it correctly?', manual_src_path) + return False + # This is the path taken outside of GitHub actions. + git_workspace = os.path.dirname(manual_src_path) + else: + git_workspace = os.path.join(workspace, 'storage') + os.makedirs(git_workspace, exist_ok=True) # Checkout projects repo in the shared volume. build_repo_manager = repo_manager.RepoManager(inferred_url, git_workspace, - repo_name=oss_fuzz_repo_name) - try: - if pr_ref: - build_repo_manager.checkout_pr(pr_ref) - else: - build_repo_manager.checkout_commit(commit_sha) - except (RuntimeError, ValueError): - logging.error('Can not check out requested state %s.', pr_ref or commit_sha) - logging.error('Using current repo state.') + repo_name=project_repo_name) + + if not manual_src_path: + checkout_specified_commit(build_repo_manager, pr_ref, commit_sha) - # Build Fuzzers using docker run. command = [ '--cap-add', 'SYS_PTRACE', @@ -153,13 +173,14 @@ def build_fuzzers(project_name, if container: command += ['-e', 'OUT=' + out_dir, '--volumes-from', container] bash_command = 'rm -rf {0} && cp -r {1} {2} && compile'.format( - os.path.join(src_in_docker, oss_fuzz_repo_name, '*'), - os.path.join(git_workspace, oss_fuzz_repo_name), src_in_docker) + os.path.join(src_in_project_builder, project_repo_name, '*'), + os.path.join(git_workspace, project_repo_name), src_in_project_builder) else: command += [ '-e', 'OUT=' + '/out', '-v', - '%s:%s' % (os.path.join(git_workspace, oss_fuzz_repo_name), - os.path.join(src_in_docker, oss_fuzz_repo_name)), '-v', + '%s:%s' % (os.path.join(git_workspace, project_repo_name), + os.path.join(src_in_project_builder, project_repo_name)), + '-v', '%s:%s' % (out_dir, '/out') ] bash_command = 'compile' @@ -175,7 +196,7 @@ def build_fuzzers(project_name, return False remove_unaffected_fuzzers(project_name, out_dir, build_repo_manager.get_git_diff(), - oss_fuzz_repo_path) + project_builder_repo_path) return True -- cgit v1.2.3