aboutsummaryrefslogtreecommitdiff
path: root/infra/cifuzz
diff options
context:
space:
mode:
authorOliver Chang <oliverchang@users.noreply.github.com>2021-07-27 10:46:20 +1000
committerGitHub <noreply@github.com>2021-07-27 10:46:20 +1000
commit69400fb24bf11cc5c4479962a3e12103e880a848 (patch)
treead7f617ba1b72b2de950024212ec42a6fb79ba8c /infra/cifuzz
parent7c3f3ddc2b8c5a31fe4d3e4f8254a6b398660955 (diff)
downloadoss-fuzz-69400fb24bf11cc5c4479962a3e12103e880a848.tar.gz
Add a Git backed filestore. (#6088)
- Add storage-repo, storage-repo-branch, and storage-repo-branch-coverage fields to the actions, to indicate that the Git filestore should be used. - The CI provided filestore is still used for crashes and builds. - Replace generic Filestore.upload_directory with typed upload methods which matches the download methods. - Rename upload_latest_build to upload_build to make it more generic. - Make artifact name prefixes an implementation detail of the store. For #6052.
Diffstat (limited to 'infra/cifuzz')
-rw-r--r--infra/cifuzz/clusterfuzz_deployment.py25
-rw-r--r--infra/cifuzz/clusterfuzz_deployment_test.py29
-rw-r--r--infra/cifuzz/config_utils.py4
-rw-r--r--infra/cifuzz/external-actions/build_fuzzers/action.yml13
-rw-r--r--infra/cifuzz/external-actions/run_fuzzers/action.yml16
-rw-r--r--infra/cifuzz/filestore/__init__.py20
-rw-r--r--infra/cifuzz/filestore/git/__init__.py149
-rw-r--r--infra/cifuzz/filestore/git/git_test.py122
-rw-r--r--infra/cifuzz/filestore/github_actions/__init__.py32
-rw-r--r--infra/cifuzz/filestore/github_actions/github_actions_test.py110
-rw-r--r--infra/cifuzz/filestore_utils.py8
11 files changed, 457 insertions, 71 deletions
diff --git a/infra/cifuzz/clusterfuzz_deployment.py b/infra/cifuzz/clusterfuzz_deployment.py
index ef09723de..694905bd0 100644
--- a/infra/cifuzz/clusterfuzz_deployment.py
+++ b/infra/cifuzz/clusterfuzz_deployment.py
@@ -89,8 +89,7 @@ class BaseClusterFuzzDeployment:
class ClusterFuzzLite(BaseClusterFuzzDeployment):
"""Class representing a deployment of ClusterFuzzLite."""
- BASE_BUILD_NAME = 'build-'
- COVERAGE_NAME = 'coverage'
+ COVERAGE_NAME = 'latest'
def __init__(self, config, workspace):
super().__init__(config, workspace)
@@ -110,8 +109,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
try:
logging.info('Downloading latest build.')
- if self.filestore.download_latest_build(build_name,
- self.workspace.clusterfuzz_build):
+ if self.filestore.download_build(build_name,
+ self.workspace.clusterfuzz_build):
logging.info('Done downloading latest build.')
return self.workspace.clusterfuzz_build
except Exception as err: # pylint: disable=broad-except
@@ -133,15 +132,15 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
return corpus_dir
def _get_build_name(self):
- return self.BASE_BUILD_NAME + self.config.sanitizer
+ return self.config.sanitizer + '-latest'
def _get_corpus_name(self, target_name): # pylint: disable=no-self-use
"""Returns the name of the corpus artifact."""
- return 'corpus-{target_name}'.format(target_name=target_name)
+ return target_name
def _get_crashes_artifact_name(self): # pylint: disable=no-self-use
"""Returns the name of the crashes artifact."""
- return 'crashes'
+ return 'current'
def upload_corpus(self, target_name):
"""Upload the corpus produced by |target_name|."""
@@ -149,7 +148,7 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
logging.info('Uploading corpus in %s for %s.', corpus_dir, target_name)
name = self._get_corpus_name(target_name)
try:
- self.filestore.upload_directory(name, corpus_dir)
+ self.filestore.upload_corpus(name, corpus_dir)
logging.info('Done uploading corpus.')
except Exception as error: # pylint: disable=broad-except
logging.error('Failed to upload corpus for target: %s. Error: %s.',
@@ -160,7 +159,7 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
logging.info('Uploading latest build in %s.', self.workspace.out)
build_name = self._get_build_name()
try:
- result = self.filestore.upload_directory(build_name, self.workspace.out)
+ result = self.filestore.upload_build(build_name, self.workspace.out)
logging.info('Done uploading latest build.')
return result
except Exception as error: # pylint: disable=broad-except
@@ -177,16 +176,16 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
logging.info('Uploading crashes in %s.', self.workspace.artifacts)
try:
- self.filestore.upload_directory(crashes_artifact_name,
- self.workspace.artifacts)
+ self.filestore.upload_crashes(crashes_artifact_name,
+ self.workspace.artifacts)
logging.info('Done uploading crashes.')
except Exception as error: # pylint: disable=broad-except
logging.error('Failed to upload crashes. Error: %s', error)
def upload_coverage(self):
"""Uploads the coverage report to the filestore."""
- self.filestore.upload_directory(self.COVERAGE_NAME,
- self.workspace.coverage_report)
+ self.filestore.upload_coverage(self.COVERAGE_NAME,
+ self.workspace.coverage_report)
def get_coverage(self, repo_path):
"""Returns the project coverage object for the project."""
diff --git a/infra/cifuzz/clusterfuzz_deployment_test.py b/infra/cifuzz/clusterfuzz_deployment_test.py
index bf10e0541..c896d86eb 100644
--- a/infra/cifuzz/clusterfuzz_deployment_test.py
+++ b/infra/cifuzz/clusterfuzz_deployment_test.py
@@ -143,7 +143,7 @@ class ClusterFuzzLiteTest(fake_filesystem_unittest.TestCase):
expected_corpus_dir = os.path.join(WORKSPACE, 'cifuzz-corpus',
EXAMPLE_FUZZER)
self.assertEqual(result, expected_corpus_dir)
- mocked_download_corpus.assert_called_with('corpus-example_crash_fuzzer',
+ mocked_download_corpus.assert_called_with('example_crash_fuzzer',
expected_corpus_dir)
@mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus',
@@ -156,33 +156,30 @@ class ClusterFuzzLiteTest(fake_filesystem_unittest.TestCase):
'/workspace/cifuzz-corpus/example_crash_fuzzer')
self.assertEqual(os.listdir(corpus_path), [])
- @mock.patch(
- 'filestore.github_actions.GithubActionsFilestore.download_latest_build',
- return_value=True)
- def test_download_latest_build(self, mocked_download_latest_build):
+ @mock.patch('filestore.github_actions.GithubActionsFilestore.download_build',
+ return_value=True)
+ def test_download_latest_build(self, mocked_download_build):
"""Tests that downloading the latest build works as intended under normal
circumstances."""
self.assertEqual(self.deployment.download_latest_build(),
EXPECTED_LATEST_BUILD_PATH)
- expected_artifact_name = 'build-address'
- mocked_download_latest_build.assert_called_with(expected_artifact_name,
- EXPECTED_LATEST_BUILD_PATH)
+ expected_artifact_name = 'address-latest'
+ mocked_download_build.assert_called_with(expected_artifact_name,
+ EXPECTED_LATEST_BUILD_PATH)
- @mock.patch(
- 'filestore.github_actions.GithubActionsFilestore.download_latest_build',
- side_effect=Exception)
+ @mock.patch('filestore.github_actions.GithubActionsFilestore.download_build',
+ side_effect=Exception)
def test_download_latest_build_fail(self, _):
"""Tests that download_latest_build returns None when it fails to download a
build."""
self.assertIsNone(self.deployment.download_latest_build())
- @mock.patch('filestore.github_actions.GithubActionsFilestore.'
- 'upload_directory')
- def test_upload_latest_build(self, mocked_upload_directory):
+ @mock.patch('filestore.github_actions.GithubActionsFilestore.' 'upload_build')
+ def test_upload_latest_build(self, mocked_upload_build):
"""Tests that upload_latest_build works as intended."""
self.deployment.upload_latest_build()
- mocked_upload_directory.assert_called_with('build-address',
- '/workspace/build-out')
+ mocked_upload_build.assert_called_with('address-latest',
+ '/workspace/build-out')
class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase):
diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py
index 68dcde6a0..c60c3fc11 100644
--- a/infra/cifuzz/config_utils.py
+++ b/infra/cifuzz/config_utils.py
@@ -107,6 +107,10 @@ class BaseConfig:
self.low_disk_space = environment.get('LOW_DISK_SPACE', False)
self.github_token = os.environ.get('GITHUB_TOKEN')
+ self.git_store_repo = os.environ.get('GIT_STORE_REPO')
+ self.git_store_branch = os.environ.get('GIT_STORE_BRANCH')
+ self.git_store_branch_coverage = os.environ.get('GIT_STORE_BRANCH_COVERAGE',
+ self.git_store_branch)
@property
def is_internal(self):
diff --git a/infra/cifuzz/external-actions/build_fuzzers/action.yml b/infra/cifuzz/external-actions/build_fuzzers/action.yml
index 34adc08a7..c20442491 100644
--- a/infra/cifuzz/external-actions/build_fuzzers/action.yml
+++ b/infra/cifuzz/external-actions/build_fuzzers/action.yml
@@ -25,6 +25,19 @@ inputs:
description: "Whether or not OSS-Fuzz's check for bad builds should be done."
required: false
default: true
+ storage-repo:
+ description: |
+ The git repo to use for storing certain artifacts from fuzzing.
+ required: false
+ storage-repo-branch:
+ description: |
+ The branch of the git repo to use for storing certain artifacts from
+ fuzzing.
+ required: false
+ storage-repo-branch-coverage:
+ description: |
+ The branch of the git repo to use for storing coverage reports.
+ required: false
runs:
using: 'docker'
image: '../../../build_fuzzers.Dockerfile'
diff --git a/infra/cifuzz/external-actions/run_fuzzers/action.yml b/infra/cifuzz/external-actions/run_fuzzers/action.yml
index 24b93325c..f1ef5da3a 100644
--- a/infra/cifuzz/external-actions/run_fuzzers/action.yml
+++ b/infra/cifuzz/external-actions/run_fuzzers/action.yml
@@ -35,6 +35,19 @@ inputs:
TODO(https://github.com/google/oss-fuzz/pull/5841#discussion_r639393361):
Document locking this down.
required: true
+ storage-repo:
+ description: |
+ The git repo to use for storing certain artifacts from fuzzing.
+ required: false
+ storage-repo-branch:
+ description: |
+ The branch of the git repo to use for storing certain artifacts from
+ fuzzing.
+ required: false
+ storage-repo-branch-coverage:
+ description: |
+ The branch of the git repo to use for storing coverage reports.
+ required: false
runs:
using: 'docker'
image: '../../../run_fuzzers.Dockerfile'
@@ -48,3 +61,6 @@ runs:
BUILD_INTEGRATION_PATH: ${{ inputs.build-integration-path }}
GITHUB_TOKEN: ${{ inputs.github-token }}
LOW_DISK_SPACE: 'True'
+ GIT_STORE_REPO: ${{ inputs.storage-repo }}
+ GIT_STORE_BRANCH: ${{ inputs.storage-repo-branch }}
+ GIT_STORE_BRANCH_COVERAGE: ${{ inputs.storage-repo-branch-coverage }}
diff --git a/infra/cifuzz/filestore/__init__.py b/infra/cifuzz/filestore/__init__.py
index ab240c832..49999d081 100644
--- a/infra/cifuzz/filestore/__init__.py
+++ b/infra/cifuzz/filestore/__init__.py
@@ -25,16 +25,28 @@ class BaseFilestore:
def __init__(self, config):
self.config = config
- def upload_directory(self, name, directory):
- """Uploads the |directory| to |name|."""
+ def upload_crashes(self, name, directory):
+ """Uploads the crashes at |directory| to |name|."""
+ raise NotImplementedError('Child class must implement method.')
+
+ def upload_corpus(self, name, directory):
+ """Uploads the corpus at |directory| to |name|."""
+ raise NotImplementedError('Child class must implement method.')
+
+ def upload_build(self, name, directory):
+ """Uploads the build at |directory| to |name|."""
+ raise NotImplementedError('Child class must implement method.')
+
+ def upload_coverage(self, name, directory):
+ """Uploads the coverage report at |directory| to |name|."""
raise NotImplementedError('Child class must implement method.')
def download_corpus(self, name, dst_directory):
"""Downloads the corpus located at |name| to |dst_directory|."""
raise NotImplementedError('Child class must implement method.')
- def download_latest_build(self, name, dst_directory):
- """Downloads the latest build with |name| to |dst_directory|."""
+ def download_build(self, name, dst_directory):
+ """Downloads the build with |name| to |dst_directory|."""
raise NotImplementedError('Child class must implement method.')
def download_coverage(self, dst_directory):
diff --git a/infra/cifuzz/filestore/git/__init__.py b/infra/cifuzz/filestore/git/__init__.py
new file mode 100644
index 000000000..5281c10a1
--- /dev/null
+++ b/infra/cifuzz/filestore/git/__init__.py
@@ -0,0 +1,149 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for a git based filestore."""
+
+from distutils import dir_util
+import logging
+import os
+import shutil
+import subprocess
+import tempfile
+
+import filestore
+
+import retry
+
+_PUSH_RETRIES = 3
+_PUSH_BACKOFF = 1
+_GIT_EMAIL = 'cifuzz@clusterfuzz.com'
+_GIT_NAME = 'CIFuzz'
+_CORPUS_DIR = 'corpus'
+_COVERAGE_DIR = 'coverage'
+
+
+def git_runner(repo_path):
+ """Returns a gits runner for the repo_path."""
+
+ def func(*args):
+ return subprocess.check_call(('git', '-C', repo_path) + args)
+
+ return func
+
+
+# pylint: disable=unused-argument,no-self-use
+class GitFilestore(filestore.BaseFilestore):
+ """Generic git filestore. This still relies on another filestore provided by
+ the CI for larger artifacts or artifacts which make sense to be included as
+ the result of a workflow run."""
+
+ def __init__(self, config, ci_filestore):
+ super().__init__(config)
+ self.repo_path = tempfile.mkdtemp()
+ self._git = git_runner(self.repo_path)
+ self._clone(self.config.git_store_repo)
+
+ self._ci_filestore = ci_filestore
+
+ def __del__(self):
+ shutil.rmtree(self.repo_path)
+
+ def _clone(self, repo_url):
+ """Clones repo URL."""
+ self._git('clone', repo_url, '.')
+ self._git('config', '--local', 'user.email', _GIT_EMAIL)
+ self._git('config', '--local', 'user.name', _GIT_NAME)
+
+ def _reset_git(self, branch):
+ """Resets the git repo."""
+ self._git('fetch', 'origin')
+ try:
+ self._git('checkout', '-B', branch, 'origin/' + branch)
+ self._git('reset', '--hard', 'HEAD')
+ except subprocess.CalledProcessError:
+ self._git('checkout', '--orphan', branch)
+
+ self._git('clean', '-fxd')
+
+ # pylint: disable=too-many-arguments
+ @retry.wrap(_PUSH_RETRIES, _PUSH_BACKOFF)
+ def _upload_to_git(self,
+ message,
+ branch,
+ upload_path,
+ local_path,
+ replace=False):
+ """Uploads a directory to git. If `replace` is True, then existing contents in
+ the upload_path is deleted."""
+ self._reset_git(branch)
+
+ full_repo_path = os.path.join(self.repo_path, upload_path)
+ if replace and os.path.exists(full_repo_path):
+ shutil.rmtree(full_repo_path)
+
+ dir_util.copy_tree(local_path, full_repo_path)
+ self._git('add', '.')
+ try:
+ self._git('commit', '-m', message)
+ except subprocess.CalledProcessError:
+ logging.debug('No changes, skipping git push.')
+ return
+
+ self._git('push', 'origin', branch)
+
+ def upload_crashes(self, name, directory):
+ """Uploads the crashes at |directory| to |name|."""
+ return self._ci_filestore.upload_crashes(name, directory)
+
+ def upload_corpus(self, name, directory):
+ """Uploads the corpus at |directory| to |name|."""
+ self._upload_to_git('Corpus upload', self.config.git_store_branch,
+ os.path.join(_CORPUS_DIR, name), directory)
+
+ def upload_build(self, name, directory):
+ """Uploads the build at |directory| to |name|."""
+ return self._ci_filestore.upload_build(name, directory)
+
+ def upload_coverage(self, name, directory):
+ """Uploads the coverage report at |directory| to |name|."""
+ self._upload_to_git('Coverage upload',
+ self.config.git_store_branch_coverage,
+ os.path.join(_COVERAGE_DIR, name),
+ directory,
+ replace=True)
+
+ def download_corpus(self, name, dst_directory):
+ """Downloads the corpus located at |name| to |dst_directory|."""
+ self._reset_git(self.config.git_store_branch)
+ path = os.path.join(self.repo_path, _CORPUS_DIR, name)
+ if not os.path.exists(path):
+ logging.debug('Corpus does not exist at %s.', path)
+ return False
+
+ dir_util.copy_tree(path, dst_directory)
+ return True
+
+ def download_build(self, name, dst_directory):
+ """Downloads the build with |name| to |dst_directory|."""
+ return self._ci_filestore.download_build(name, dst_directory)
+
+ def download_coverage(self, name, dst_directory):
+ """Downloads the latest project coverage report."""
+ self._reset_git(self.config.git_store_branch_coverage)
+ path = os.path.join(self.repo_path, _COVERAGE_DIR, name)
+ if not os.path.exists(path):
+ logging.debug('Coverage does not exist at %s.', path)
+ return False
+
+ dir_util.copy_tree(path, dst_directory)
+ return True
diff --git a/infra/cifuzz/filestore/git/git_test.py b/infra/cifuzz/filestore/git/git_test.py
new file mode 100644
index 000000000..56be23bac
--- /dev/null
+++ b/infra/cifuzz/filestore/git/git_test.py
@@ -0,0 +1,122 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for git."""
+import filecmp
+import os
+import tempfile
+import subprocess
+import sys
+import unittest
+from unittest import mock
+
+# pylint: disable=wrong-import-position
+INFRA_DIR = os.path.dirname(
+ os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.abspath(__file__)))))
+sys.path.append(INFRA_DIR)
+
+from filestore import git
+import test_helpers
+
+# pylint: disable=protected-access,no-self-use
+
+
+class GitFilestoreTest(unittest.TestCase):
+ """Tests for GitFilestore."""
+
+ def setUp(self):
+ self.git_dir = tempfile.TemporaryDirectory()
+ self.addCleanup(self.git_dir.cleanup)
+
+ self.local_dir = tempfile.TemporaryDirectory()
+ self.addCleanup(self.local_dir.cleanup)
+
+ self.download_dir = tempfile.TemporaryDirectory()
+ self.addCleanup(self.download_dir.cleanup)
+
+ with open(os.path.join(self.local_dir.name, 'a'), 'w') as handle:
+ handle.write('')
+
+ os.makedirs(os.path.join(self.local_dir.name, 'b'))
+
+ with open(os.path.join(self.local_dir.name, 'b', 'c'), 'w') as handle:
+ handle.write('')
+
+ self.git_repo = git.git_runner(self.git_dir.name)
+ self.git_repo('init', '--bare')
+
+ self.config = test_helpers.create_run_config(
+ git_store_repo='file://' + self.git_dir.name,
+ git_store_branch='main',
+ git_store_branch_coverage='cov-branch')
+
+ self.mock_ci_filestore = mock.MagicMock()
+ self.git_store = git.GitFilestore(self.config, self.mock_ci_filestore)
+
+ def assert_dirs_same(self, first, second):
+ """Asserts two dirs are the same."""
+ dcmp = filecmp.dircmp(first, second)
+ if dcmp.diff_files or dcmp.left_only or dcmp.right_only:
+ return False
+
+ return all(
+ self.assert_dirs_same(os.path.join(first, subdir),
+ os.path.join(second, subdir))
+ for subdir in dcmp.common_dirs)
+
+ def get_repo_filelist(self, branch):
+ """Get files in repo."""
+ return subprocess.check_output([
+ 'git', '-C', self.git_dir.name, 'ls-tree', '-r', '--name-only', branch
+ ]).decode().splitlines()
+
+ def test_upload_download_corpus(self):
+ """Tests uploading and downloading corpus."""
+ self.git_store.upload_corpus('target', self.local_dir.name)
+ self.git_store.download_corpus('target', self.download_dir.name)
+ self.assert_dirs_same(self.local_dir.name, self.download_dir.name)
+
+ self.assertCountEqual([
+ 'corpus/target/a',
+ 'corpus/target/b/c',
+ ], self.get_repo_filelist('main'))
+
+ def test_upload_download_coverage(self):
+ """Tests uploading and downloading corpus."""
+ self.git_store.upload_coverage('latest', self.local_dir.name)
+ self.git_store.download_coverage('latest', self.download_dir.name)
+ self.assert_dirs_same(self.local_dir.name, self.download_dir.name)
+
+ self.assertCountEqual([
+ 'coverage/latest/a',
+ 'coverage/latest/b/c',
+ ], self.get_repo_filelist('cov-branch'))
+
+ def test_upload_crashes(self):
+ """Tests uploading crashes."""
+ self.git_store.upload_crashes('current', self.local_dir.name)
+ self.mock_ci_filestore.upload_crashes.assert_called_with(
+ 'current', self.local_dir.name)
+
+ def test_upload_build(self):
+ """Tests uploading build."""
+ self.git_store.upload_build('sanitizer', self.local_dir.name)
+ self.mock_ci_filestore.upload_build.assert_called_with(
+ 'sanitizer', self.local_dir.name)
+
+ def test_download_build(self):
+ """Tests downloading build."""
+ self.git_store.download_build('sanitizer', self.download_dir.name)
+ self.mock_ci_filestore.download_build.assert_called_with(
+ 'sanitizer', self.download_dir.name)
diff --git a/infra/cifuzz/filestore/github_actions/__init__.py b/infra/cifuzz/filestore/github_actions/__init__.py
index 2a9f64eb7..6d04ccfbd 100644
--- a/infra/cifuzz/filestore/github_actions/__init__.py
+++ b/infra/cifuzz/filestore/github_actions/__init__.py
@@ -47,6 +47,10 @@ class GithubActionsFilestore(filestore.BaseFilestore):
this however."""
ARTIFACT_PREFIX = 'cifuzz-'
+ BUILD_PREFIX = 'build-'
+ CRASHES_PREFIX = 'crashes-'
+ CORPUS_PREFIX = 'corpus-'
+ COVERAGE_PREFIX = 'coverage-'
def __init__(self, config):
super().__init__(config)
@@ -59,7 +63,7 @@ class GithubActionsFilestore(filestore.BaseFilestore):
return name
return f'{self.ARTIFACT_PREFIX}{name}'
- def upload_directory(self, name, directory): # pylint: disable=no-self-use
+ def _upload_directory(self, name, directory): # pylint: disable=no-self-use
"""Uploads |directory| as artifact with |name|."""
name = self._get_artifact_name(name)
with tempfile.TemporaryDirectory() as temp_dir:
@@ -67,9 +71,25 @@ class GithubActionsFilestore(filestore.BaseFilestore):
tar_directory(directory, archive_path)
_raw_upload_directory(name, temp_dir)
+ def upload_crashes(self, name, directory):
+ """Uploads the crashes at |directory| to |name|."""
+ return _raw_upload_directory(self.CRASHES_PREFIX + name, directory)
+
+ def upload_corpus(self, name, directory):
+ """Uploads the corpus at |directory| to |name|."""
+ return self._upload_directory(self.CORPUS_PREFIX + name, directory)
+
+ def upload_build(self, name, directory):
+ """Uploads the build at |directory| to |name|."""
+ return self._upload_directory(self.BUILD_PREFIX + name, directory)
+
+ def upload_coverage(self, name, directory):
+ """Uploads the coverage report at |directory| to |name|."""
+ return self._upload_directory(self.COVERAGE_PREFIX + name, directory)
+
def download_corpus(self, name, dst_directory): # pylint: disable=unused-argument,no-self-use
"""Downloads the corpus located at |name| to |dst_directory|."""
- return self._download_artifact(name, dst_directory)
+ return self._download_artifact(self.CORPUS_PREFIX + name, dst_directory)
def _find_artifact(self, name):
"""Finds an artifact using the GitHub API and returns it."""
@@ -117,13 +137,13 @@ class GithubActionsFilestore(filestore.BaseFilestore):
self.config.project_repo_name,
self.github_api_http_headers)
- def download_latest_build(self, name, dst_directory):
- """Downloads latest build with name |name| to |dst_directory|."""
- return self._download_artifact(name, dst_directory)
+ def download_build(self, name, dst_directory):
+ """Downloads the build with name |name| to |dst_directory|."""
+ return self._download_artifact(self.BUILD_PREFIX + name, dst_directory)
def download_coverage(self, name, dst_directory):
"""Downloads the latest project coverage report."""
- return self._download_artifact(name, dst_directory)
+ return self._download_artifact(self.COVERAGE_PREFIX + name, dst_directory)
def _raw_upload_directory(name, directory):
diff --git a/infra/cifuzz/filestore/github_actions/github_actions_test.py b/infra/cifuzz/filestore/github_actions/github_actions_test.py
index d3a69ac91..1c5610f29 100644
--- a/infra/cifuzz/filestore/github_actions/github_actions_test.py
+++ b/infra/cifuzz/filestore/github_actions/github_actions_test.py
@@ -44,9 +44,8 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
self.repo = 'examplerepo'
os.environ['GITHUB_REPOSITORY'] = f'{self.owner}/{self.repo}'
self.config = test_helpers.create_run_config(github_token=self.github_token)
- self.artifact_name = 'corpus'
- self.corpus_dir = '/corpus-dir'
- self.testcase = os.path.join(self.corpus_dir, 'testcase')
+ self.local_dir = '/local-dir'
+ self.testcase = os.path.join(self.local_dir, 'testcase')
def _get_expected_http_headers(self):
return {
@@ -67,15 +66,15 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
return_value=None)
@mock.patch('filestore.github_actions.github_api.find_artifact',
return_value=None)
- def test_download_latest_build_no_artifact(self, _, __, mocked_warning):
- """Tests that download_latest_build returns None and doesn't exception when
+ def test_download_build_no_artifact(self, _, __, mocked_warning):
+ """Tests that download_build returns None and doesn't exception when
find_artifact can't find an artifact."""
filestore = github_actions.GithubActionsFilestore(self.config)
- name = 'build-name'
+ name = 'name'
build_dir = 'build-dir'
- self.assertFalse(filestore.download_latest_build(name, build_dir))
+ self.assertFalse(filestore.download_build(name, build_dir))
mocked_warning.assert_called_with('Could not download artifact: %s.',
- 'cifuzz-' + name)
+ 'cifuzz-build-' + name)
@mock.patch('logging.warning')
@mock.patch('filestore.github_actions.GithubActionsFilestore._list_artifacts',
@@ -86,35 +85,86 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
"""Tests that download_corpus_build returns None and doesn't exception when
find_artifact can't find an artifact."""
filestore = github_actions.GithubActionsFilestore(self.config)
- name = 'corpus-name'
- dst_dir = 'corpus-dir'
+ name = 'name'
+ dst_dir = 'local-dir'
self.assertFalse(filestore.download_corpus(name, dst_dir))
mocked_warning.assert_called_with('Could not download artifact: %s.',
- 'cifuzz-' + name)
+ 'cifuzz-corpus-' + name)
@mock.patch('filestore.github_actions.tar_directory')
@mock.patch('third_party.github_actions_toolkit.artifact.artifact_client'
'.upload_artifact')
- def test_upload_directory(self, mocked_upload_artifact, mocked_tar_directory):
- """Tests that upload_directory invokes tar_directory and
- artifact_client.upload_artifact properly."""
- self._create_corpus_dir()
+ def test_upload_corpus(self, mocked_upload_artifact, mocked_tar_directory):
+ """Test uploading corpus."""
+ self._create_local_dir()
+
+ def mock_tar_directory(_, archive_path):
+ self.fs.create_file(archive_path)
+
+ mocked_tar_directory.side_effect = mock_tar_directory
+
+ filestore = github_actions.GithubActionsFilestore(self.config)
+ filestore.upload_corpus('target', self.local_dir)
+ self.assert_upload(mocked_upload_artifact, mocked_tar_directory,
+ 'corpus-target')
+
+ @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client'
+ '.upload_artifact')
+ def test_upload_crashes(self, mocked_upload_artifact):
+ """Test uploading crashes."""
+ self._create_local_dir()
+
+ filestore = github_actions.GithubActionsFilestore(self.config)
+ filestore.upload_crashes('current', self.local_dir)
+ mocked_upload_artifact.assert_has_calls(
+ [mock.call('crashes-current', ['/local-dir/testcase'], '/local-dir')])
+
+ @mock.patch('filestore.github_actions.tar_directory')
+ @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client'
+ '.upload_artifact')
+ def test_upload_build(self, mocked_upload_artifact, mocked_tar_directory):
+ """Test uploading build."""
+ self._create_local_dir()
+
+ def mock_tar_directory(_, archive_path):
+ self.fs.create_file(archive_path)
+
+ mocked_tar_directory.side_effect = mock_tar_directory
+
+ filestore = github_actions.GithubActionsFilestore(self.config)
+ filestore.upload_build('sanitizer', self.local_dir)
+ self.assert_upload(mocked_upload_artifact, mocked_tar_directory,
+ 'build-sanitizer')
+
+ @mock.patch('filestore.github_actions.tar_directory')
+ @mock.patch('third_party.github_actions_toolkit.artifact.artifact_client'
+ '.upload_artifact')
+ def test_upload_coverage(self, mocked_upload_artifact, mocked_tar_directory):
+ """Test uploading coverage."""
+ self._create_local_dir()
def mock_tar_directory(_, archive_path):
self.fs.create_file(archive_path)
mocked_tar_directory.side_effect = mock_tar_directory
+
filestore = github_actions.GithubActionsFilestore(self.config)
- filestore.upload_directory(self.artifact_name, self.corpus_dir)
+ filestore.upload_coverage('latest', self.local_dir)
+ self.assert_upload(mocked_upload_artifact, mocked_tar_directory,
+ 'coverage-latest')
+ def assert_upload(self, mocked_upload_artifact, mocked_tar_directory,
+ expected_artifact_name):
+ """Tests that upload_directory invokes tar_directory and
+ artifact_client.upload_artifact properly."""
# Don't assert what second argument will be since it's a temporary
# directory.
self.assertEqual(mocked_tar_directory.call_args_list[0][0][0],
- self.corpus_dir)
+ self.local_dir)
# Don't assert what second and third arguments will be since they are
# temporary directories.
- expected_artifact_name = 'cifuzz-' + self.artifact_name
+ expected_artifact_name = 'cifuzz-' + expected_artifact_name
self.assertEqual(mocked_upload_artifact.call_args_list[0][0][0],
expected_artifact_name)
@@ -124,7 +174,7 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
self.assertEqual(os.path.basename(artifacts_list[0]),
expected_artifact_name + '.tar')
- def _create_corpus_dir(self):
+ def _create_local_dir(self):
"""Sets up pyfakefs and creates a corpus directory containing
self.testcase."""
self.setUpPyfakefs()
@@ -138,16 +188,16 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
artifact_download_url = 'http://example.com/download'
artifact_listing = {
'expired': False,
- 'name': self.artifact_name,
+ 'name': 'corpus',
'archive_download_url': artifact_download_url
}
mocked_find_artifact.return_value = artifact_listing
- self._create_corpus_dir()
+ self._create_local_dir()
with tempfile.TemporaryDirectory() as temp_dir:
# Create a tarball.
- archive_path = os.path.join(temp_dir, f'cifuzz-{self.artifact_name}.tar')
- github_actions.tar_directory(self.corpus_dir, archive_path)
+ archive_path = os.path.join(temp_dir, 'cifuzz-corpus.tar')
+ github_actions.tar_directory(self.local_dir, archive_path)
artifact_download_dst_dir = os.path.join(temp_dir, 'dst')
os.mkdir(artifact_download_dst_dir)
@@ -165,9 +215,8 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
mocked_download_and_unpack_zip.side_effect = mock_download_and_unpack_zip
filestore = github_actions.GithubActionsFilestore(self.config)
self.assertTrue(
- filestore._download_artifact(self.artifact_name,
- artifact_download_dst_dir))
- mocked_find_artifact.assert_called_with(f'cifuzz-{self.artifact_name}')
+ filestore._download_artifact('corpus', artifact_download_dst_dir))
+ mocked_find_artifact.assert_called_with('cifuzz-corpus')
self.assertTrue(
os.path.exists(
os.path.join(artifact_download_dst_dir,
@@ -183,17 +232,17 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
}
artifact_listing_2 = {
'expired': False,
- 'name': self.artifact_name,
+ 'name': 'artifact',
'archive_download_url': 'http://download2'
}
artifact_listing_3 = {
'expired': True,
- 'name': self.artifact_name,
+ 'name': 'artifact',
'archive_download_url': 'http://download3'
}
artifact_listing_4 = {
'expired': False,
- 'name': self.artifact_name,
+ 'name': 'artifact',
'archive_download_url': 'http://download4'
}
artifacts = [
@@ -204,8 +253,7 @@ class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase):
filestore = github_actions.GithubActionsFilestore(self.config)
# Test that find_artifact will return the most recent unexpired artifact
# with the correct name.
- self.assertEqual(filestore._find_artifact(self.artifact_name),
- artifact_listing_2)
+ self.assertEqual(filestore._find_artifact('artifact'), artifact_listing_2)
mocked_list_artifacts.assert_called_with(self.owner, self.repo,
self._get_expected_http_headers())
diff --git a/infra/cifuzz/filestore_utils.py b/infra/cifuzz/filestore_utils.py
index d100b851d..d3aaecd82 100644
--- a/infra/cifuzz/filestore_utils.py
+++ b/infra/cifuzz/filestore_utils.py
@@ -13,6 +13,7 @@
# limitations under the License.
"""External filestore interface. Cannot be depended on by filestore code."""
import filestore
+import filestore.git
import filestore.github_actions
@@ -21,5 +22,10 @@ def get_filestore(config):
Raises an exception if there is no correct filestore for the platform."""
# TODO(metzman): Force specifying of filestore.
if config.platform == config.Platform.EXTERNAL_GITHUB:
- return filestore.github_actions.GithubActionsFilestore(config)
+ ci_filestore = filestore.github_actions.GithubActionsFilestore(config)
+ if not config.git_store_repo:
+ return ci_filestore
+
+ return filestore.git.GitFilestore(config, ci_filestore)
+
raise filestore.FilestoreError('Filestore doesn\'t support platform.')