diff options
Diffstat (limited to 'infra/cifuzz')
-rw-r--r-- | infra/cifuzz/clusterfuzz_deployment.py | 25 | ||||
-rw-r--r-- | infra/cifuzz/clusterfuzz_deployment_test.py | 29 | ||||
-rw-r--r-- | infra/cifuzz/config_utils.py | 4 | ||||
-rw-r--r-- | infra/cifuzz/external-actions/build_fuzzers/action.yml | 13 | ||||
-rw-r--r-- | infra/cifuzz/external-actions/run_fuzzers/action.yml | 16 | ||||
-rw-r--r-- | infra/cifuzz/filestore/__init__.py | 20 | ||||
-rw-r--r-- | infra/cifuzz/filestore/git/__init__.py | 149 | ||||
-rw-r--r-- | infra/cifuzz/filestore/git/git_test.py | 122 | ||||
-rw-r--r-- | infra/cifuzz/filestore/github_actions/__init__.py | 32 | ||||
-rw-r--r-- | infra/cifuzz/filestore/github_actions/github_actions_test.py | 110 | ||||
-rw-r--r-- | infra/cifuzz/filestore_utils.py | 8 |
11 files changed, 457 insertions, 71 deletions
diff --git a/infra/cifuzz/clusterfuzz_deployment.py b/infra/cifuzz/clusterfuzz_deployment.py index ef09723de..694905bd0 100644 --- a/infra/cifuzz/clusterfuzz_deployment.py +++ b/infra/cifuzz/clusterfuzz_deployment.py @@ -89,8 +89,7 @@ class BaseClusterFuzzDeployment: class ClusterFuzzLite(BaseClusterFuzzDeployment): """Class representing a deployment of ClusterFuzzLite.""" - BASE_BUILD_NAME = 'build-' - COVERAGE_NAME = 'coverage' + COVERAGE_NAME = 'latest' def __init__(self, config, workspace): super().__init__(config, workspace) @@ -110,8 +109,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): try: logging.info('Downloading latest build.') - if self.filestore.download_latest_build(build_name, - self.workspace.clusterfuzz_build): + if self.filestore.download_build(build_name, + self.workspace.clusterfuzz_build): logging.info('Done downloading latest build.') return self.workspace.clusterfuzz_build except Exception as err: # pylint: disable=broad-except @@ -133,15 +132,15 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): return corpus_dir def _get_build_name(self): - return self.BASE_BUILD_NAME + self.config.sanitizer + return self.config.sanitizer + '-latest' def _get_corpus_name(self, target_name): # pylint: disable=no-self-use """Returns the name of the corpus artifact.""" - return 'corpus-{target_name}'.format(target_name=target_name) + return target_name def _get_crashes_artifact_name(self): # pylint: disable=no-self-use """Returns the name of the crashes artifact.""" - return 'crashes' + return 'current' def upload_corpus(self, target_name): """Upload the corpus produced by |target_name|.""" @@ -149,7 +148,7 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): logging.info('Uploading corpus in %s for %s.', corpus_dir, target_name) name = self._get_corpus_name(target_name) try: - self.filestore.upload_directory(name, corpus_dir) + self.filestore.upload_corpus(name, corpus_dir) logging.info('Done uploading corpus.') except Exception as error: # pylint: disable=broad-except logging.error('Failed to upload corpus for target: %s. Error: %s.', @@ -160,7 +159,7 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): logging.info('Uploading latest build in %s.', self.workspace.out) build_name = self._get_build_name() try: - result = self.filestore.upload_directory(build_name, self.workspace.out) + result = self.filestore.upload_build(build_name, self.workspace.out) logging.info('Done uploading latest build.') return result except Exception as error: # pylint: disable=broad-except @@ -177,16 +176,16 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): logging.info('Uploading crashes in %s.', self.workspace.artifacts) try: - self.filestore.upload_directory(crashes_artifact_name, - self.workspace.artifacts) + self.filestore.upload_crashes(crashes_artifact_name, + self.workspace.artifacts) logging.info('Done uploading crashes.') except Exception as error: # pylint: disable=broad-except logging.error('Failed to upload crashes. Error: %s', error) def upload_coverage(self): """Uploads the coverage report to the filestore.""" - self.filestore.upload_directory(self.COVERAGE_NAME, - self.workspace.coverage_report) + self.filestore.upload_coverage(self.COVERAGE_NAME, + self.workspace.coverage_report) def get_coverage(self, repo_path): """Returns the project coverage object for the project.""" diff --git a/infra/cifuzz/clusterfuzz_deployment_test.py b/infra/cifuzz/clusterfuzz_deployment_test.py index bf10e0541..c896d86eb 100644 --- a/infra/cifuzz/clusterfuzz_deployment_test.py +++ b/infra/cifuzz/clusterfuzz_deployment_test.py @@ -143,7 +143,7 @@ class ClusterFuzzLiteTest(fake_filesystem_unittest.TestCase): expected_corpus_dir = os.path.join(WORKSPACE, 'cifuzz-corpus', EXAMPLE_FUZZER) self.assertEqual(result, expected_corpus_dir) - mocked_download_corpus.assert_called_with('corpus-example_crash_fuzzer', + mocked_download_corpus.assert_called_with('example_crash_fuzzer', expected_corpus_dir) @mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus', @@ -156,33 +156,30 @@ class ClusterFuzzLiteTest(fake_filesystem_unittest.TestCase): '/workspace/cifuzz-corpus/example_crash_fuzzer') self.assertEqual(os.listdir(corpus_path), []) - @mock.patch( - 'filestore.github_actions.GithubActionsFilestore.download_latest_build', - return_value=True) - def test_download_latest_build(self, mocked_download_latest_build): + @mock.patch('filestore.github_actions.GithubActionsFilestore.download_build', + return_value=True) + def test_download_latest_build(self, mocked_download_build): """Tests that downloading the latest build works as intended under normal circumstances.""" self.assertEqual(self.deployment.download_latest_build(), EXPECTED_LATEST_BUILD_PATH) - expected_artifact_name = 'build-address' - mocked_download_latest_build.assert_called_with(expected_artifact_name, - EXPECTED_LATEST_BUILD_PATH) + expected_artifact_name = 'address-latest' + mocked_download_build.assert_called_with(expected_artifact_name, + EXPECTED_LATEST_BUILD_PATH) - @mock.patch( - 'filestore.github_actions.GithubActionsFilestore.download_latest_build', - side_effect=Exception) + @mock.patch('filestore.github_actions.GithubActionsFilestore.download_build', + side_effect=Exception) def test_download_latest_build_fail(self, _): """Tests that download_latest_build returns None when it fails to download a build.""" self.assertIsNone(self.deployment.download_latest_build()) - @mock.patch('filestore.github_actions.GithubActionsFilestore.' - 'upload_directory') - def test_upload_latest_build(self, mocked_upload_directory): + @mock.patch('filestore.github_actions.GithubActionsFilestore.' 'upload_build') + def test_upload_latest_build(self, mocked_upload_build): """Tests that upload_latest_build works as intended.""" self.deployment.upload_latest_build() - mocked_upload_directory.assert_called_with('build-address', - '/workspace/build-out') + mocked_upload_build.assert_called_with('address-latest', + '/workspace/build-out') class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase): diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index 68dcde6a0..c60c3fc11 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -107,6 +107,10 @@ class BaseConfig: self.low_disk_space = environment.get('LOW_DISK_SPACE', False) self.github_token = os.environ.get('GITHUB_TOKEN') + self.git_store_repo = os.environ.get('GIT_STORE_REPO') + self.git_store_branch = os.environ.get('GIT_STORE_BRANCH') + self.git_store_branch_coverage = os.environ.get('GIT_STORE_BRANCH_COVERAGE', + self.git_store_branch) @property def is_internal(self): diff --git a/infra/cifuzz/external-actions/build_fuzzers/action.yml b/infra/cifuzz/external-actions/build_fuzzers/action.yml index 34adc08a7..c20442491 100644 --- a/infra/cifuzz/external-actions/build_fuzzers/action.yml +++ b/infra/cifuzz/external-actions/build_fuzzers/action.yml @@ -25,6 +25,19 @@ inputs: description: "Whether or not OSS-Fuzz's check for bad builds should be done." required: false default: true + storage-repo: + description: | + The git repo to use for storing certain artifacts from fuzzing. + required: false + storage-repo-branch: + description: | + The branch of the git repo to use for storing certain artifacts from + fuzzing. + required: false + storage-repo-branch-coverage: + description: | + The branch of the git repo to use for storing coverage reports. + required: false runs: using: 'docker' image: '../../../build_fuzzers.Dockerfile' diff --git a/infra/cifuzz/external-actions/run_fuzzers/action.yml b/infra/cifuzz/external-actions/run_fuzzers/action.yml index 24b93325c..f1ef5da3a 100644 --- a/infra/cifuzz/external-actions/run_fuzzers/action.yml +++ b/infra/cifuzz/external-actions/run_fuzzers/action.yml @@ -35,6 +35,19 @@ inputs: TODO(https://github.com/google/oss-fuzz/pull/5841#discussion_r639393361): Document locking this down. required: true + storage-repo: + description: | + The git repo to use for storing certain artifacts from fuzzing. + required: false + storage-repo-branch: + description: | + The branch of the git repo to use for storing certain artifacts from + fuzzing. + required: false + storage-repo-branch-coverage: + description: | + The branch of the git repo to use for storing coverage reports. + required: false runs: using: 'docker' image: '../../../run_fuzzers.Dockerfile' @@ -48,3 +61,6 @@ runs: BUILD_INTEGRATION_PATH: ${{ inputs.build-integration-path }} GITHUB_TOKEN: ${{ inputs.github-token }} LOW_DISK_SPACE: 'True' + GIT_STORE_REPO: ${{ inputs.storage-repo }} + GIT_STORE_BRANCH: ${{ inputs.storage-repo-branch }} + GIT_STORE_BRANCH_COVERAGE: ${{ inputs.storage-repo-branch-coverage }} diff --git a/infra/cifuzz/filestore/__init__.py b/infra/cifuzz/filestore/__init__.py index ab240c832..49999d081 100644 --- a/infra/cifuzz/filestore/__init__.py +++ b/infra/cifuzz/filestore/__init__.py @@ -25,16 +25,28 @@ class BaseFilestore: def __init__(self, config): self.config = config - def upload_directory(self, name, directory): - """Uploads the |directory| to |name|.""" + def upload_crashes(self, name, directory): + """Uploads the crashes at |directory| to |name|.""" + raise NotImplementedError('Child class must implement method.') + + def upload_corpus(self, name, directory): + """Uploads the corpus at |directory| to |name|.""" + raise NotImplementedError('Child class must implement method.') + + def upload_build(self, name, directory): + """Uploads the build at |directory| to |name|.""" + raise NotImplementedError('Child class must implement method.') + + def upload_coverage(self, name, directory): + """Uploads the coverage report at |directory| to |name|.""" raise NotImplementedError('Child class must implement method.') def download_corpus(self, name, dst_directory): """Downloads the corpus located at |name| to |dst_directory|.""" raise NotImplementedError('Child class must implement method.') - def download_latest_build(self, name, dst_directory): - """Downloads the latest build with |name| to |dst_directory|.""" + def download_build(self, name, dst_directory): + """Downloads the build with |name| to |dst_directory|.""" raise NotImplementedError('Child class must implement method.') def download_coverage(self, dst_directory): diff --git a/infra/cifuzz/filestore/git/__init__.py b/infra/cifuzz/filestore/git/__init__.py new file mode 100644 index 000000000..5281c10a1 --- /dev/null +++ b/infra/cifuzz/filestore/git/__init__.py @@ -0,0 +1,149 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for a git based filestore.""" + +from distutils import dir_util +import logging +import os +import shutil +import subprocess +import tempfile + +import filestore + +import retry + +_PUSH_RETRIES = 3 +_PUSH_BACKOFF = 1 +_GIT_EMAIL = 'cifuzz@clusterfuzz.com' +_GIT_NAME = 'CIFuzz' +_CORPUS_DIR = 'corpus' +_COVERAGE_DIR = 'coverage' + + +def git_runner(repo_path): + """Returns a gits runner for the repo_path.""" + + def func(*args): + return subprocess.check_call(('git', '-C', repo_path) + args) + + return func + + +# pylint: disable=unused-argument,no-self-use +class GitFilestore(filestore.BaseFilestore): + """Generic git filestore. This still relies on another filestore provided by + the CI for larger artifacts or artifacts which make sense to be included as + the result of a workflow run.""" + + def __init__(self, config, ci_filestore): + super().__init__(config) + self.repo_path = tempfile.mkdtemp() + self._git = git_runner(self.repo_path) + self._clone(self.config.git_store_repo) + + self._ci_filestore = ci_filestore + + def __del__(self): + shutil.rmtree(self.repo_path) + + def _clone(self, repo_url): + """Clones repo URL.""" + self._git('clone', repo_url, '.') + self._git('config', '--local', 'user.email', _GIT_EMAIL) + self._git('config', '--local', 'user.name', _GIT_NAME) + + def _reset_git(self, branch): + """Resets the git repo.""" + self._git('fetch', 'origin') + try: + self._git('checkout', '-B', branch, 'origin/' + branch) + self._git('reset', '--hard', 'HEAD') + except subprocess.CalledProcessError: + self._git('checkout', '--orphan', branch) + + self._git('clean', '-fxd') + + # pylint: disable=too-many-arguments + @retry.wrap(_PUSH_RETRIES, _PUSH_BACKOFF) + def _upload_to_git(self, + message, + branch, + upload_path, + local_path, + replace=False): + """Uploads a directory to git. If `replace` is True, then existing contents in + the upload_path is deleted.""" + self._reset_git(branch) + + full_repo_path = os.path.join(self.repo_path, upload_path) + if replace and os.path.exists(full_repo_path): + shutil.rmtree(full_repo_path) + + dir_util.copy_tree(local_path, full_repo_path) + self._git('add', '.') + try: + self._git('commit', '-m', message) + except subprocess.CalledProcessError: + logging.debug('No changes, skipping git push.') + return + + self._git('push', 'origin', branch) + + def upload_crashes(self, name, directory): + """Uploads the crashes at |directory| to |name|.""" + return self._ci_filestore.upload_crashes(name, directory) + + def upload_corpus(self, name, directory): + """Uploads the corpus at |directory| to |name|.""" + self._upload_to_git('Corpus upload', self.config.git_store_branch, + os.path.join(_CORPUS_DIR, name), directory) + + def upload_build(self, name, directory): + """Uploads the build at |directory| to |name|.""" + return self._ci_filestore.upload_build(name, directory) + + def upload_coverage(self, name, directory): + """Uploads the coverage report at |directory| to |name|.""" + self._upload_to_git('Coverage upload', + self.config.git_store_branch_coverage, + os.path.join(_COVERAGE_DIR, name), + directory, + replace=True) + + def download_corpus(self, name, dst_directory): + """Downloads the corpus located at |name| to |dst_directory|.""" + self._reset_git(self.config.git_store_branch) + path = os.path.join(self.repo_path, _CORPUS_DIR, name) + if not os.path.exists(path): + logging.debug('Corpus does not exist at %s.', path) + return False + + dir_util.copy_tree(path, dst_directory) + return True + + def download_build(self, name, dst_directory): + """Downloads the build with |name| to |dst_directory|.""" + return self._ci_filestore.download_build(name, dst_directory) + + def download_coverage(self, name, dst_directory): + """Downloads the latest project coverage report.""" + self._reset_git(self.config.git_store_branch_coverage) + path = os.path.join(self.repo_path, _COVERAGE_DIR, name) + if not os.path.exists(path): + logging.debug('Coverage does not exist at %s.', path) + return False + + dir_util.copy_tree(path, dst_directory) + return True diff --git a/infra/cifuzz/filestore/git/git_test.py b/infra/cifuzz/filestore/git/git_test.py new file mode 100644 index 000000000..56be23bac --- /dev/null +++ b/infra/cifuzz/filestore/git/git_test.py @@ -0,0 +1,122 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for git.""" +import filecmp +import os +import tempfile +import subprocess +import sys +import unittest +from unittest import mock + +# pylint: disable=wrong-import-position +INFRA_DIR = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) +sys.path.append(INFRA_DIR) + +from filestore import git +import test_helpers + +# pylint: disable=protected-access,no-self-use + + +class GitFilestoreTest(unittest.TestCase): + """Tests for GitFilestore.""" + + def setUp(self): + self.git_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.git_dir.cleanup) + + self.local_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.local_dir.cleanup) + + self.download_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.download_dir.cleanup) + + with open(os.path.join(self.local_dir.name, 'a'), 'w') as handle: + handle.write('') + + os.makedirs(os.path.join(self.local_dir.name, 'b')) + + with open(os.path.join(self.local_dir.name, 'b', 'c'), 'w') as handle: + handle.write('') + + self.git_repo = git.git_runner(self.git_dir.name) + self.git_repo('init', '--bare') + + self.config = test_helpers.create_run_config( + git_store_repo='file://' + self.git_dir.name, + git_store_branch='main', + git_store_branch_coverage='cov-branch') + + self.mock_ci_filestore = mock.MagicMock() + self.git_store = git.GitFilestore(self.config, self.mock_ci_filestore) + + def assert_dirs_same(self, first, second): + """Asserts two dirs are the same.""" + dcmp = filecmp.dircmp(first, second) + if dcmp.diff_files or dcmp.left_only or dcmp.right_only: + return False + + return all( + self.assert_dirs_same(os.path.join(first, subdir), + os.path.join(second, subdir)) + for subdir in dcmp.common_dirs) + + def get_repo_filelist(self, branch): + """Get files in repo.""" + return subprocess.check_output([ + 'git', '-C', self.git_dir.name, 'ls-tree', '-r', '--name-only', branch + ]).decode().splitlines() + + def test_upload_download_corpus(self): + """Tests uploading and downloading corpus.""" + self.git_store.upload_corpus('target', self.local_dir.name) + self.git_store.download_corpus('target', self.download_dir.name) + self.assert_dirs_same(self.local_dir.name, self.download_dir.name) + + self.assertCountEqual([ + 'corpus/target/a', + 'corpus/target/b/c', + ], self.get_repo_filelist('main')) + + def test_upload_download_coverage(self): + """Tests uploading and downloading corpus.""" + self.git_store.upload_coverage('latest', self.local_dir.name) + self.git_store.download_coverage('latest', self.download_dir.name) + self.assert_dirs_same(self.local_dir.name, self.download_dir.name) + + self.assertCountEqual([ + 'coverage/latest/a', + 'coverage/latest/b/c', + ], self.get_repo_filelist('cov-branch')) + + def test_upload_crashes(self): + """Tests uploading crashes.""" + self.git_store.upload_crashes('current', self.local_dir.name) + self.mock_ci_filestore.upload_crashes.assert_called_with( + 'current', self.local_dir.name) + + def test_upload_build(self): + """Tests uploading build.""" + self.git_store.upload_build('sanitizer', self.local_dir.name) + self.mock_ci_filestore.upload_build.assert_called_with( + 'sanitizer', self.local_dir.name) + + def test_download_build(self): + """Tests downloading build.""" + self.git_store.download_build('sanitizer', self.download_dir.name) + self.mock_ci_filestore.download_build.assert_called_with( + 'sanitizer', self.download_dir.name) diff --git a/infra/cifuzz/filestore/github_actions/__init__.py b/infra/cifuzz/filestore/github_actions/__init__.py index 2a9f64eb7..6d04ccfbd 100644 --- a/infra/cifuzz/filestore/github_actions/__init__.py +++ b/infra/cifuzz/filestore/github_actions/__init__.py @@ -47,6 +47,10 @@ class GithubActionsFilestore(filestore.BaseFilestore): this however.""" ARTIFACT_PREFIX = 'cifuzz-' + BUILD_PREFIX = 'build-' + CRASHES_PREFIX = 'crashes-' + CORPUS_PREFIX = 'corpus-' + COVERAGE_PREFIX = 'coverage-' def __init__(self, config): super().__init__(config) @@ -59,7 +63,7 @@ class GithubActionsFilestore(filestore.BaseFilestore): return name return f'{self.ARTIFACT_PREFIX}{name}' - def upload_directory(self, name, directory): # pylint: disable=no-self-use + def _upload_directory(self, name, directory): # pylint: disable=no-self-use """Uploads |directory| as artifact with |name|.""" name = self._get_artifact_name(name) with tempfile.TemporaryDirectory() as temp_dir: @@ -67,9 +71,25 @@ class GithubActionsFilestore(filestore.BaseFilestore): tar_directory(directory, archive_path) _raw_upload_directory(name, temp_dir) + def upload_crashes(self, name, directory): + """Uploads the crashes at |directory| to |name|.""" + return _raw_upload_directory(self.CRASHES_PREFIX + name, directory) + + def upload_corpus(self, name, directory): + """Uploads the corpus at |directory| to |name|.""" + return self._upload_directory(self.CORPUS_PREFIX + name, directory) + + def upload_build(self, name, directory): + """Uploads the build at |directory| to |name|.""" + return self._upload_directory(self.BUILD_PREFIX + name, directory) + + def upload_coverage(self, name, directory): + """Uploads the coverage report at |directory| to |name|.""" + return self._upload_directory(self.COVERAGE_PREFIX + name, directory) + def download_corpus(self, name, dst_directory): # pylint: disable=unused-argument,no-self-use """Downloads the corpus located at |name| to |dst_directory|.""" - return self._download_artifact(name, dst_directory) + return self._download_artifact(self.CORPUS_PREFIX + name, dst_directory) def _find_artifact(self, name): """Finds an artifact using the GitHub API and returns it.""" @@ -117,13 +137,13 @@ class GithubActionsFilestore(filestore.BaseFilestore): self.config.project_repo_name, self.github_api_http_headers) - def download_latest_build(self, name, dst_directory): - """Downloads latest build with name |name| to |dst_directory|.""" - return self._download_artifact(name, dst_directory) + def download_build(self, name, dst_directory): + """Downloads the build with name |name| to |dst_directory|.""" + return self._download_artifact(self.BUILD_PREFIX + name, dst_directory) def download_coverage(self, name, dst_directory): """Downloads the latest project coverage report.""" - return self._download_artifact(name, dst_directory) + return self._download_artifact(self.COVERAGE_PREFIX + name, dst_directory) def _raw_upload_directory(name, directory): diff --git a/infra/cifuzz/filestore/github_actions/github_actions_test.py b/infra/cifuzz/filestore/github_actions/github_actions_test.py index d3a69ac91..1c5610f29 100644 --- a/infra/cifuzz/filestore/github_actions/github_actions_test.py +++ b/infra/cifuzz/filestore/github_actions/github_actions_test.py @@ -44,9 +44,8 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): self.repo = 'examplerepo' os.environ['GITHUB_REPOSITORY'] = f'{self.owner}/{self.repo}' self.config = test_helpers.create_run_config(github_token=self.github_token) - self.artifact_name = 'corpus' - self.corpus_dir = '/corpus-dir' - self.testcase = os.path.join(self.corpus_dir, 'testcase') + self.local_dir = '/local-dir' + self.testcase = os.path.join(self.local_dir, 'testcase') def _get_expected_http_headers(self): return { @@ -67,15 +66,15 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): return_value=None) @mock.patch('filestore.github_actions.github_api.find_artifact', return_value=None) - def test_download_latest_build_no_artifact(self, _, __, mocked_warning): - """Tests that download_latest_build returns None and doesn't exception when + def test_download_build_no_artifact(self, _, __, mocked_warning): + """Tests that download_build returns None and doesn't exception when find_artifact can't find an artifact.""" filestore = github_actions.GithubActionsFilestore(self.config) - name = 'build-name' + name = 'name' build_dir = 'build-dir' - self.assertFalse(filestore.download_latest_build(name, build_dir)) + self.assertFalse(filestore.download_build(name, build_dir)) mocked_warning.assert_called_with('Could not download artifact: %s.', - 'cifuzz-' + name) + 'cifuzz-build-' + name) @mock.patch('logging.warning') @mock.patch('filestore.github_actions.GithubActionsFilestore._list_artifacts', @@ -86,35 +85,86 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): """Tests that download_corpus_build returns None and doesn't exception when find_artifact can't find an artifact.""" filestore = github_actions.GithubActionsFilestore(self.config) - name = 'corpus-name' - dst_dir = 'corpus-dir' + name = 'name' + dst_dir = 'local-dir' self.assertFalse(filestore.download_corpus(name, dst_dir)) mocked_warning.assert_called_with('Could not download artifact: %s.', - 'cifuzz-' + name) + 'cifuzz-corpus-' + name) @mock.patch('filestore.github_actions.tar_directory') @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client' '.upload_artifact') - def test_upload_directory(self, mocked_upload_artifact, mocked_tar_directory): - """Tests that upload_directory invokes tar_directory and - artifact_client.upload_artifact properly.""" - self._create_corpus_dir() + def test_upload_corpus(self, mocked_upload_artifact, mocked_tar_directory): + """Test uploading corpus.""" + self._create_local_dir() + + def mock_tar_directory(_, archive_path): + self.fs.create_file(archive_path) + + mocked_tar_directory.side_effect = mock_tar_directory + + filestore = github_actions.GithubActionsFilestore(self.config) + filestore.upload_corpus('target', self.local_dir) + self.assert_upload(mocked_upload_artifact, mocked_tar_directory, + 'corpus-target') + + @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client' + '.upload_artifact') + def test_upload_crashes(self, mocked_upload_artifact): + """Test uploading crashes.""" + self._create_local_dir() + + filestore = github_actions.GithubActionsFilestore(self.config) + filestore.upload_crashes('current', self.local_dir) + mocked_upload_artifact.assert_has_calls( + [mock.call('crashes-current', ['/local-dir/testcase'], '/local-dir')]) + + @mock.patch('filestore.github_actions.tar_directory') + @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client' + '.upload_artifact') + def test_upload_build(self, mocked_upload_artifact, mocked_tar_directory): + """Test uploading build.""" + self._create_local_dir() + + def mock_tar_directory(_, archive_path): + self.fs.create_file(archive_path) + + mocked_tar_directory.side_effect = mock_tar_directory + + filestore = github_actions.GithubActionsFilestore(self.config) + filestore.upload_build('sanitizer', self.local_dir) + self.assert_upload(mocked_upload_artifact, mocked_tar_directory, + 'build-sanitizer') + + @mock.patch('filestore.github_actions.tar_directory') + @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client' + '.upload_artifact') + def test_upload_coverage(self, mocked_upload_artifact, mocked_tar_directory): + """Test uploading coverage.""" + self._create_local_dir() def mock_tar_directory(_, archive_path): self.fs.create_file(archive_path) mocked_tar_directory.side_effect = mock_tar_directory + filestore = github_actions.GithubActionsFilestore(self.config) - filestore.upload_directory(self.artifact_name, self.corpus_dir) + filestore.upload_coverage('latest', self.local_dir) + self.assert_upload(mocked_upload_artifact, mocked_tar_directory, + 'coverage-latest') + def assert_upload(self, mocked_upload_artifact, mocked_tar_directory, + expected_artifact_name): + """Tests that upload_directory invokes tar_directory and + artifact_client.upload_artifact properly.""" # Don't assert what second argument will be since it's a temporary # directory. self.assertEqual(mocked_tar_directory.call_args_list[0][0][0], - self.corpus_dir) + self.local_dir) # Don't assert what second and third arguments will be since they are # temporary directories. - expected_artifact_name = 'cifuzz-' + self.artifact_name + expected_artifact_name = 'cifuzz-' + expected_artifact_name self.assertEqual(mocked_upload_artifact.call_args_list[0][0][0], expected_artifact_name) @@ -124,7 +174,7 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): self.assertEqual(os.path.basename(artifacts_list[0]), expected_artifact_name + '.tar') - def _create_corpus_dir(self): + def _create_local_dir(self): """Sets up pyfakefs and creates a corpus directory containing self.testcase.""" self.setUpPyfakefs() @@ -138,16 +188,16 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): artifact_download_url = 'http://example.com/download' artifact_listing = { 'expired': False, - 'name': self.artifact_name, + 'name': 'corpus', 'archive_download_url': artifact_download_url } mocked_find_artifact.return_value = artifact_listing - self._create_corpus_dir() + self._create_local_dir() with tempfile.TemporaryDirectory() as temp_dir: # Create a tarball. - archive_path = os.path.join(temp_dir, f'cifuzz-{self.artifact_name}.tar') - github_actions.tar_directory(self.corpus_dir, archive_path) + archive_path = os.path.join(temp_dir, 'cifuzz-corpus.tar') + github_actions.tar_directory(self.local_dir, archive_path) artifact_download_dst_dir = os.path.join(temp_dir, 'dst') os.mkdir(artifact_download_dst_dir) @@ -165,9 +215,8 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): mocked_download_and_unpack_zip.side_effect = mock_download_and_unpack_zip filestore = github_actions.GithubActionsFilestore(self.config) self.assertTrue( - filestore._download_artifact(self.artifact_name, - artifact_download_dst_dir)) - mocked_find_artifact.assert_called_with(f'cifuzz-{self.artifact_name}') + filestore._download_artifact('corpus', artifact_download_dst_dir)) + mocked_find_artifact.assert_called_with('cifuzz-corpus') self.assertTrue( os.path.exists( os.path.join(artifact_download_dst_dir, @@ -183,17 +232,17 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): } artifact_listing_2 = { 'expired': False, - 'name': self.artifact_name, + 'name': 'artifact', 'archive_download_url': 'http://download2' } artifact_listing_3 = { 'expired': True, - 'name': self.artifact_name, + 'name': 'artifact', 'archive_download_url': 'http://download3' } artifact_listing_4 = { 'expired': False, - 'name': self.artifact_name, + 'name': 'artifact', 'archive_download_url': 'http://download4' } artifacts = [ @@ -204,8 +253,7 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): filestore = github_actions.GithubActionsFilestore(self.config) # Test that find_artifact will return the most recent unexpired artifact # with the correct name. - self.assertEqual(filestore._find_artifact(self.artifact_name), - artifact_listing_2) + self.assertEqual(filestore._find_artifact('artifact'), artifact_listing_2) mocked_list_artifacts.assert_called_with(self.owner, self.repo, self._get_expected_http_headers()) diff --git a/infra/cifuzz/filestore_utils.py b/infra/cifuzz/filestore_utils.py index d100b851d..d3aaecd82 100644 --- a/infra/cifuzz/filestore_utils.py +++ b/infra/cifuzz/filestore_utils.py @@ -13,6 +13,7 @@ # limitations under the License. """External filestore interface. Cannot be depended on by filestore code.""" import filestore +import filestore.git import filestore.github_actions @@ -21,5 +22,10 @@ def get_filestore(config): Raises an exception if there is no correct filestore for the platform.""" # TODO(metzman): Force specifying of filestore. if config.platform == config.Platform.EXTERNAL_GITHUB: - return filestore.github_actions.GithubActionsFilestore(config) + ci_filestore = filestore.github_actions.GithubActionsFilestore(config) + if not config.git_store_repo: + return ci_filestore + + return filestore.git.GitFilestore(config, ci_filestore) + raise filestore.FilestoreError('Filestore doesn\'t support platform.') |