diff options
| -rw-r--r-- | infra/bisector.py | 42 | ||||
| -rw-r--r-- | infra/build_specified_commit.py | 55 | ||||
| -rw-r--r-- | infra/repo_manager.py | 2 |
3 files changed, 92 insertions, 7 deletions
diff --git a/infra/bisector.py b/infra/bisector.py index 39c35479c..1e762fa06 100644 --- a/infra/bisector.py +++ b/infra/bisector.py @@ -32,6 +32,9 @@ This is done with the following steps: import argparse import collections +import datetime +from distutils import spawn +import json import logging import os import tempfile @@ -94,8 +97,33 @@ def main(): return 0 +def _load_base_builder_repo(): + """Get base-image digests.""" + gcloud_path = spawn.find_executable('gcloud') + if not gcloud_path: + logging.warning('gcloud not found in PATH.') + return None + + result, _, _ = utils.execute([ + gcloud_path, 'container', 'images', 'list-tags', + 'gcr.io/oss-fuzz-base/base-builder', '--format=json' + ], + check_result=True) + result = json.loads(result) + + repo = build_specified_commit.BaseBuilderRepo() + for image in result: + timestamp = datetime.datetime.fromisoformat( + image['timestamp']['datetime']).astimezone(datetime.timezone.utc) + repo.add_digest(timestamp, image['digest']) + + return repo + + def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals """Perform the bisect.""" + base_builder_repo = _load_base_builder_repo() + with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) @@ -115,7 +143,11 @@ def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) if not build_specified_commit.build_fuzzers_from_commit( - commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data): + commit_list[new_idx], + bisect_repo_manager, + host_src_dir, + build_data, + base_builder_repo=base_builder_repo): raise RuntimeError('Failed to build new_commit') expected_error_code = helper.reproduce_impl(build_data.project_name, @@ -130,7 +162,7 @@ def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # bisect_repo_manager, host_src_dir, build_data, - ): + base_builder_repo=base_builder_repo): raise RuntimeError('Failed to build old_commit') if expected_error_code == helper.reproduce_impl(build_data.project_name, @@ -146,7 +178,11 @@ def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # logging.info('Testing against %s (idx=%d)', commit_list[curr_idx], curr_idx) if not build_specified_commit.build_fuzzers_from_commit( - commit_list[curr_idx], bisect_repo_manager, host_src_dir, build_data): + commit_list[curr_idx], + bisect_repo_manager, + host_src_dir, + build_data, + base_builder_repo=base_builder_repo): # Treat build failures as if we couldn't repo. # TODO(ochang): retry nearby commits? old_idx = curr_idx diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py index b0f7b5b29..0a7a5259c 100644 --- a/infra/build_specified_commit.py +++ b/infra/build_specified_commit.py @@ -17,6 +17,7 @@ This module is allows each of the OSS Fuzz projects fuzzers to be built from a specific point in time. This feature can be used for implementations like continuious integration fuzzing and bisection to find errors """ +import bisect import os import collections import logging @@ -33,6 +34,27 @@ BuildData = collections.namedtuple( _GIT_DIR_MARKER = 'gitdir: ' +class BaseBuilderRepo: + """Repo of base-builder images.""" + + def __init__(self): + self.timestamps = [] + self.digests = [] + + def add_digest(self, timestamp, digest): + """Add a digest.""" + self.timestamps.append(timestamp) + self.digests.append(digest) + + def find_digest(self, timestamp): + """Find the latest image before the given timestamp.""" + index = bisect.bisect_right(self.timestamps, timestamp) + if index > 0: + return self.digests[index - 1] + + raise ValueError('Failed to find suitable base-builder.') + + def _make_gitdirs_relative(src_dir): """Make gitdirs relative.""" for root_dir, _, files in os.walk(src_dir): @@ -60,7 +82,23 @@ def _make_gitdirs_relative(src_dir): new_lines.append(line) with open(file_path, 'w') as handle: - handle.write('\n'.join(new_lines)) + handle.write(''.join(new_lines)) + + +def _replace_base_builder_digest(dockerfile_path, digest): + """Replace the base-builder digest in a Dockerfile.""" + with open(dockerfile_path) as handle: + lines = handle.readlines() + + new_lines = [] + for line in lines: + if line.strip().startswith('FROM'): + line = 'FROM gcr.io/oss-fuzz-base/base-builder@' + digest + + new_lines.append(line) + + with open(dockerfile_path, 'w') as handle: + handle.write(''.join(new_lines)) def copy_src_from_docker(project_name, host_dir): @@ -87,14 +125,18 @@ def copy_src_from_docker(project_name, host_dir): return src_dir -def build_fuzzers_from_commit(commit, build_repo_manager, host_src_path, - build_data): +def build_fuzzers_from_commit(commit, + build_repo_manager, + host_src_path, + build_data, + base_builder_repo=None): """Builds a OSS-Fuzz fuzzer at a specific commit SHA. Args: commit: The commit SHA to build the fuzzers at. build_repo_manager: The OSS-Fuzz project's repo manager to be built at. build_data: A struct containing project build information. + base_builder_repo: A BaseBuilderRepo. Returns: 0 on successful build or error code on failure. """ @@ -138,6 +180,13 @@ def build_fuzzers_from_commit(commit, build_repo_manager, host_src_path, oss_fuzz_repo_manager.git(['checkout', oss_fuzz_commit, projects_dir], check_result=True) + # Also use the closest base-builder we can find. + if base_builder_repo: + base_builder_digest = base_builder_repo.find_digest(commit_date) + logging.info('Using base-builder with digest %s.', base_builder_digest) + _replace_base_builder_digest(os.path.join(projects_dir, 'Dockerfile'), + base_builder_digest) + # Rebuild image and re-copy src dir since things in /src could have changed. if not helper.build_image_impl(build_data.project_name): raise RuntimeError('Failed to rebuild image.') diff --git a/infra/repo_manager.py b/infra/repo_manager.py index 238cb3e7b..946559849 100644 --- a/infra/repo_manager.py +++ b/infra/repo_manager.py @@ -84,7 +84,7 @@ class BaseRepoManager: """ out, _, _ = self.git(['show', '-s', '--format=%ct', commit], check_result=True) - return datetime.datetime.fromtimestamp(int(out)) + return datetime.datetime.fromtimestamp(int(out), tz=datetime.timezone.utc) def get_git_diff(self): """Gets a list of files that have changed from the repo head. |