diff options
Diffstat (limited to 'infra/cifuzz/continuous_integration.py')
| -rw-r--r-- | infra/cifuzz/continuous_integration.py | 169 |
1 files changed, 136 insertions, 33 deletions
diff --git a/infra/cifuzz/continuous_integration.py b/infra/cifuzz/continuous_integration.py index b2e8af28e..47c4a7cbf 100644 --- a/infra/cifuzz/continuous_integration.py +++ b/infra/cifuzz/continuous_integration.py @@ -21,16 +21,21 @@ import logging # pylint: disable=wrong-import-position,import-error sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import build_specified_commit +import docker import helper import repo_manager import retry import utils +import workspace_utils # pylint: disable=too-few-public-methods BuildPreparationResult = collections.namedtuple( 'BuildPreparationResult', ['success', 'image_repo_path', 'repo_manager']) +_IMAGE_BUILD_TRIES = 3 +_IMAGE_BUILD_BACKOFF = 2 + def fix_git_repo_for_diff(repo_manager_obj): """Fixes git repos cloned by the "checkout" action so that diffing works on @@ -47,16 +52,34 @@ class BaseCi: def __init__(self, config): self.config = config + self.workspace = workspace_utils.Workspace(config) + + def repo_dir(self): + """Returns the source repo path, if it has been checked out. None is + returned otherwise.""" + if not os.path.exists(self.workspace.repo_storage): + return None + + # Note: this assumes there is only one repo checked out here. + listing = os.listdir(self.workspace.repo_storage) + if len(listing) != 1: + raise RuntimeError('Invalid repo storage.') + + repo_path = os.path.join(self.workspace.repo_storage, listing[0]) + if not os.path.isdir(repo_path): + raise RuntimeError('Repo is not a directory.') + + return repo_path def prepare_for_fuzzer_build(self): """Builds the fuzzer builder image and gets the source code we need to fuzz.""" - raise NotImplementedError('Children must implement this method.') + raise NotImplementedError('Child class must implement method.') def get_diff_base(self): """Returns the base to diff against with git to get the change under test.""" - raise NotImplementedError('Children must implement this method.') + raise NotImplementedError('Child class must implement method.') def get_changed_code_under_test(self, repo_manager_obj): """Returns the changed files that need to be tested.""" @@ -65,10 +88,38 @@ class BaseCi: logging.info('Diffing against %s.', base) return repo_manager_obj.get_git_diff(base) + def get_build_command(self, host_repo_path, image_repo_path): + """Returns the command for building the project that is run inside the + project builder container.""" + raise NotImplementedError('Child class must implement method.') + + +def get_build_command(): + """Returns the command to build the project inside the project builder + container.""" + return 'compile' + + +def get_replace_repo_and_build_command(host_repo_path, image_repo_path): + """Returns the command to replace the repo located at |image_repo_path| with + |host_repo_path| and build the project inside the project builder + container.""" + rm_path = os.path.join(image_repo_path, '*') + image_src_path = os.path.dirname(image_repo_path) + build_command = get_build_command() + command = (f'cd / && rm -rf {rm_path} && cp -r {host_repo_path} ' + f'{image_src_path} && cd - && {build_command}') + return command + def get_ci(config): """Determines what kind of CI is being used and returns the object representing that system.""" + + if config.platform == config.Platform.EXTERNAL_GENERIC_CI: + # Non-OSS-Fuzz projects must bring their own source and their own build + # integration (which is relative to that source). + return ExternalGeneric(config) if config.platform == config.Platform.EXTERNAL_GITHUB: # Non-OSS-Fuzz projects must bring their own source and their own build # integration (which is relative to that source). @@ -127,27 +178,35 @@ class InternalGithub(GithubCiMixin, BaseCi): assert self.config.pr_ref or self.config.commit_sha # detect_main_repo builds the image as a side effect. inferred_url, image_repo_path = (build_specified_commit.detect_main_repo( - self.config.project_name, repo_name=self.config.project_repo_name)) + self.config.oss_fuzz_project_name, + repo_name=self.config.project_repo_name)) if not inferred_url or not image_repo_path: - logging.error('Could not detect repo from project %s.', - self.config.project_name) - return BuildPreparationResult(False, None, None) + logging.error('Could not detect repo.') + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) - git_workspace = os.path.join(self.config.workspace, 'storage') - os.makedirs(git_workspace, exist_ok=True) + os.makedirs(self.workspace.repo_storage, exist_ok=True) # Use the same name used in the docker image so we can overwrite it. image_repo_name = os.path.basename(image_repo_path) # Checkout project's repo in the shared volume. - manager = repo_manager.clone_repo_and_get_manager(inferred_url, - git_workspace, - repo_name=image_repo_name) + manager = repo_manager.clone_repo_and_get_manager( + inferred_url, self.workspace.repo_storage, repo_name=image_repo_name) checkout_specified_commit(manager, self.config.pr_ref, self.config.commit_sha) - return BuildPreparationResult(True, image_repo_path, manager) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager) + + def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use + """Returns the command for building the project that is run inside the + project builder container. Command also replaces |image_repo_path| with + |host_repo_path|.""" + return get_replace_repo_and_build_command(host_repo_path, image_repo_path) class InternalGeneric(BaseCi): @@ -162,35 +221,71 @@ class InternalGeneric(BaseCi): logging.info('Building OSS-Fuzz project.') # detect_main_repo builds the image as a side effect. _, image_repo_path = (build_specified_commit.detect_main_repo( - self.config.project_name, repo_name=self.config.project_repo_name)) + self.config.oss_fuzz_project_name, + repo_name=self.config.project_repo_name)) if not image_repo_path: - logging.error('Could not detect repo from project %s.', - self.config.project_name) - return BuildPreparationResult(False, None, None) + logging.error('Could not detect repo.') + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) manager = repo_manager.RepoManager(self.config.project_src_path) - return BuildPreparationResult(True, image_repo_path, manager) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager) def get_diff_base(self): return 'origin...' - -_IMAGE_BUILD_TRIES = 3 -_IMAGE_BUILD_BACKOFF = 2 + def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use + """Returns the command for building the project that is run inside the + project builder container. Command also replaces |image_repo_path| with + |host_repo_path|.""" + return get_replace_repo_and_build_command(host_repo_path, image_repo_path) @retry.wrap(_IMAGE_BUILD_TRIES, _IMAGE_BUILD_BACKOFF) -def build_external_project_docker_image(project_name, project_src, - build_integration_path): +def build_external_project_docker_image(project_src, build_integration_path): """Builds the project builder image for an external (non-OSS-Fuzz) project. Returns True on success.""" dockerfile_path = os.path.join(build_integration_path, 'Dockerfile') - tag = 'gcr.io/oss-fuzz/{project_name}'.format(project_name=project_name) - command = ['-t', tag, '-f', dockerfile_path, project_src] + command = [ + '-t', docker.EXTERNAL_PROJECT_IMAGE, '-f', dockerfile_path, project_src + ] return helper.docker_build(command) +class ExternalGeneric(BaseCi): + """CI implementation for generic CI for external (non-OSS-Fuzz) projects.""" + + def get_diff_base(self): + return 'origin...' + + def prepare_for_fuzzer_build(self): + logging.info('ExternalGeneric: preparing for fuzzer build.') + manager = repo_manager.RepoManager(self.config.project_src_path) + build_integration_abs_path = os.path.join( + manager.repo_dir, self.config.build_integration_path) + if not build_external_project_docker_image(manager.repo_dir, + build_integration_abs_path): + logging.error('Failed to build external project: %s.', + self.config.oss_fuzz_project_name) + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) + + image_repo_path = os.path.join('/src', self.config.project_repo_name) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager) + + def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use + """Returns the command for building the project that is run inside the + project builder container.""" + return get_build_command() + + class ExternalGithub(GithubCiMixin, BaseCi): """Class representing CI for a non-OSS-Fuzz project on Github Actions.""" @@ -200,24 +295,32 @@ class ExternalGithub(GithubCiMixin, BaseCi): projects are expected to bring their own source code to CIFuzz. Returns True on success.""" logging.info('Building external project.') - git_workspace = os.path.join(self.config.workspace, 'storage') - os.makedirs(git_workspace, exist_ok=True) + os.makedirs(self.workspace.repo_storage, exist_ok=True) # Checkout before building, so we don't need to rely on copying the source # into the image. # TODO(metzman): Figure out if we want second copy at all. manager = repo_manager.clone_repo_and_get_manager( self.config.git_url, - git_workspace, + self.workspace.repo_storage, repo_name=self.config.project_repo_name) checkout_specified_commit(manager, self.config.pr_ref, self.config.commit_sha) - build_integration_path = os.path.join(manager.repo_dir, - self.config.build_integration_path) - if not build_external_project_docker_image( - self.config.project_name, manager.repo_dir, build_integration_path): + build_integration_abs_path = os.path.join( + manager.repo_dir, self.config.build_integration_path) + if not build_external_project_docker_image(manager.repo_dir, + build_integration_abs_path): logging.error('Failed to build external project.') - return BuildPreparationResult(False, None, None) + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) image_repo_path = os.path.join('/src', self.config.project_repo_name) - return BuildPreparationResult(True, image_repo_path, manager) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager) + + def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use + """Returns the command for building the project that is run inside the + project builder container.""" + return get_build_command() |