diff options
Diffstat (limited to 'infra/cifuzz/filestore')
-rw-r--r-- | infra/cifuzz/filestore/__init__.py | 54 | ||||
-rw-r--r-- | infra/cifuzz/filestore/git/__init__.py | 159 | ||||
-rw-r--r-- | infra/cifuzz/filestore/git/git_test.py | 122 | ||||
-rw-r--r-- | infra/cifuzz/filestore/github_actions/__init__.py | 177 | ||||
-rw-r--r-- | infra/cifuzz/filestore/github_actions/github_actions_test.py | 281 | ||||
-rw-r--r-- | infra/cifuzz/filestore/github_actions/github_api.py | 108 | ||||
-rw-r--r-- | infra/cifuzz/filestore/github_actions/github_api_test.py | 33 | ||||
-rwxr-xr-x | infra/cifuzz/filestore/github_actions/upload.js | 33 |
8 files changed, 967 insertions, 0 deletions
diff --git a/infra/cifuzz/filestore/__init__.py b/infra/cifuzz/filestore/__init__.py new file mode 100644 index 000000000..d112f7b8c --- /dev/null +++ b/infra/cifuzz/filestore/__init__.py @@ -0,0 +1,54 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for a generic filestore.""" + + +class FilestoreError(Exception): + """Error using the filestore.""" + + +# pylint: disable=unused-argument,no-self-use +class BaseFilestore: + """Base class for a filestore.""" + + def __init__(self, config): + self.config = config + + def upload_crashes(self, name, directory): + """Uploads the crashes at |directory| to |name|.""" + raise NotImplementedError('Child class must implement method.') + + def upload_corpus(self, name, directory, replace=False): + """Uploads the corpus at |directory| to |name|.""" + raise NotImplementedError('Child class must implement method.') + + def upload_build(self, name, directory): + """Uploads the build at |directory| to |name|.""" + raise NotImplementedError('Child class must implement method.') + + def upload_coverage(self, name, directory): + """Uploads the coverage report at |directory| to |name|.""" + raise NotImplementedError('Child class must implement method.') + + def download_corpus(self, name, dst_directory): + """Downloads the corpus located at |name| to |dst_directory|.""" + raise NotImplementedError('Child class must implement method.') + + def download_build(self, name, dst_directory): + """Downloads the build with |name| to |dst_directory|.""" + raise NotImplementedError('Child class must implement method.') + + def download_coverage(self, dst_directory): + """Downloads the latest project coverage report.""" + raise NotImplementedError('Child class must implement method.') diff --git a/infra/cifuzz/filestore/git/__init__.py b/infra/cifuzz/filestore/git/__init__.py new file mode 100644 index 000000000..5414003da --- /dev/null +++ b/infra/cifuzz/filestore/git/__init__.py @@ -0,0 +1,159 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for a git based filestore.""" + +from distutils import dir_util +import logging +import os +import shutil +import subprocess +import sys +import tempfile + +import filestore + +# pylint: disable=wrong-import-position +INFRA_DIR = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) +sys.path.append(INFRA_DIR) + +import retry + +_PUSH_RETRIES = 3 +_PUSH_BACKOFF = 1 +_GIT_EMAIL = 'cifuzz@clusterfuzz.com' +_GIT_NAME = 'CIFuzz' +_CORPUS_DIR = 'corpus' +_COVERAGE_DIR = 'coverage' + + +def git_runner(repo_path): + """Returns a gits runner for the repo_path.""" + + def func(*args): + return subprocess.check_call(('git', '-C', repo_path) + args) + + return func + + +# pylint: disable=unused-argument,no-self-use +class GitFilestore(filestore.BaseFilestore): + """Generic git filestore. This still relies on another filestore provided by + the CI for larger artifacts or artifacts which make sense to be included as + the result of a workflow run.""" + + def __init__(self, config, ci_filestore): + super().__init__(config) + self.repo_path = tempfile.mkdtemp() + self._git = git_runner(self.repo_path) + self._clone(self.config.git_store_repo) + + self._ci_filestore = ci_filestore + + def __del__(self): + shutil.rmtree(self.repo_path) + + def _clone(self, repo_url): + """Clones repo URL.""" + self._git('clone', repo_url, '.') + self._git('config', '--local', 'user.email', _GIT_EMAIL) + self._git('config', '--local', 'user.name', _GIT_NAME) + + def _reset_git(self, branch): + """Resets the git repo.""" + self._git('fetch', 'origin') + try: + self._git('checkout', '-B', branch, 'origin/' + branch) + self._git('reset', '--hard', 'HEAD') + except subprocess.CalledProcessError: + self._git('checkout', '--orphan', branch) + + self._git('clean', '-fxd') + + # pylint: disable=too-many-arguments + @retry.wrap(_PUSH_RETRIES, _PUSH_BACKOFF) + def _upload_to_git(self, + message, + branch, + upload_path, + local_path, + replace=False): + """Uploads a directory to git. If `replace` is True, then existing contents + in the upload_path is deleted.""" + self._reset_git(branch) + + full_repo_path = os.path.join(self.repo_path, upload_path) + if replace and os.path.exists(full_repo_path): + shutil.rmtree(full_repo_path) + + dir_util.copy_tree(local_path, full_repo_path) + self._git('add', '.') + try: + self._git('commit', '-m', message) + except subprocess.CalledProcessError: + logging.debug('No changes, skipping git push.') + return + + self._git('push', 'origin', branch) + + def upload_crashes(self, name, directory): + """Uploads the crashes at |directory| to |name|.""" + return self._ci_filestore.upload_crashes(name, directory) + + def upload_corpus(self, name, directory, replace=False): + """Uploads the corpus at |directory| to |name|.""" + self._upload_to_git('Corpus upload', + self.config.git_store_branch, + os.path.join(_CORPUS_DIR, name), + directory, + replace=replace) + + def upload_build(self, name, directory): + """Uploads the build at |directory| to |name|.""" + return self._ci_filestore.upload_build(name, directory) + + def upload_coverage(self, name, directory): + """Uploads the coverage report at |directory| to |name|.""" + self._upload_to_git('Coverage upload', + self.config.git_store_branch_coverage, + os.path.join(_COVERAGE_DIR, name), + directory, + replace=True) + + def download_corpus(self, name, dst_directory): + """Downloads the corpus located at |name| to |dst_directory|.""" + self._reset_git(self.config.git_store_branch) + path = os.path.join(self.repo_path, _CORPUS_DIR, name) + if not os.path.exists(path): + logging.debug('Corpus does not exist at %s.', path) + return False + + dir_util.copy_tree(path, dst_directory) + return True + + def download_build(self, name, dst_directory): + """Downloads the build with |name| to |dst_directory|.""" + return self._ci_filestore.download_build(name, dst_directory) + + def download_coverage(self, name, dst_directory): + """Downloads the latest project coverage report.""" + self._reset_git(self.config.git_store_branch_coverage) + path = os.path.join(self.repo_path, _COVERAGE_DIR, name) + if not os.path.exists(path): + logging.debug('Coverage does not exist at %s.', path) + return False + + dir_util.copy_tree(path, dst_directory) + return True diff --git a/infra/cifuzz/filestore/git/git_test.py b/infra/cifuzz/filestore/git/git_test.py new file mode 100644 index 000000000..56be23bac --- /dev/null +++ b/infra/cifuzz/filestore/git/git_test.py @@ -0,0 +1,122 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for git.""" +import filecmp +import os +import tempfile +import subprocess +import sys +import unittest +from unittest import mock + +# pylint: disable=wrong-import-position +INFRA_DIR = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) +sys.path.append(INFRA_DIR) + +from filestore import git +import test_helpers + +# pylint: disable=protected-access,no-self-use + + +class GitFilestoreTest(unittest.TestCase): + """Tests for GitFilestore.""" + + def setUp(self): + self.git_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.git_dir.cleanup) + + self.local_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.local_dir.cleanup) + + self.download_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.download_dir.cleanup) + + with open(os.path.join(self.local_dir.name, 'a'), 'w') as handle: + handle.write('') + + os.makedirs(os.path.join(self.local_dir.name, 'b')) + + with open(os.path.join(self.local_dir.name, 'b', 'c'), 'w') as handle: + handle.write('') + + self.git_repo = git.git_runner(self.git_dir.name) + self.git_repo('init', '--bare') + + self.config = test_helpers.create_run_config( + git_store_repo='file://' + self.git_dir.name, + git_store_branch='main', + git_store_branch_coverage='cov-branch') + + self.mock_ci_filestore = mock.MagicMock() + self.git_store = git.GitFilestore(self.config, self.mock_ci_filestore) + + def assert_dirs_same(self, first, second): + """Asserts two dirs are the same.""" + dcmp = filecmp.dircmp(first, second) + if dcmp.diff_files or dcmp.left_only or dcmp.right_only: + return False + + return all( + self.assert_dirs_same(os.path.join(first, subdir), + os.path.join(second, subdir)) + for subdir in dcmp.common_dirs) + + def get_repo_filelist(self, branch): + """Get files in repo.""" + return subprocess.check_output([ + 'git', '-C', self.git_dir.name, 'ls-tree', '-r', '--name-only', branch + ]).decode().splitlines() + + def test_upload_download_corpus(self): + """Tests uploading and downloading corpus.""" + self.git_store.upload_corpus('target', self.local_dir.name) + self.git_store.download_corpus('target', self.download_dir.name) + self.assert_dirs_same(self.local_dir.name, self.download_dir.name) + + self.assertCountEqual([ + 'corpus/target/a', + 'corpus/target/b/c', + ], self.get_repo_filelist('main')) + + def test_upload_download_coverage(self): + """Tests uploading and downloading corpus.""" + self.git_store.upload_coverage('latest', self.local_dir.name) + self.git_store.download_coverage('latest', self.download_dir.name) + self.assert_dirs_same(self.local_dir.name, self.download_dir.name) + + self.assertCountEqual([ + 'coverage/latest/a', + 'coverage/latest/b/c', + ], self.get_repo_filelist('cov-branch')) + + def test_upload_crashes(self): + """Tests uploading crashes.""" + self.git_store.upload_crashes('current', self.local_dir.name) + self.mock_ci_filestore.upload_crashes.assert_called_with( + 'current', self.local_dir.name) + + def test_upload_build(self): + """Tests uploading build.""" + self.git_store.upload_build('sanitizer', self.local_dir.name) + self.mock_ci_filestore.upload_build.assert_called_with( + 'sanitizer', self.local_dir.name) + + def test_download_build(self): + """Tests downloading build.""" + self.git_store.download_build('sanitizer', self.download_dir.name) + self.mock_ci_filestore.download_build.assert_called_with( + 'sanitizer', self.download_dir.name) diff --git a/infra/cifuzz/filestore/github_actions/__init__.py b/infra/cifuzz/filestore/github_actions/__init__.py new file mode 100644 index 000000000..3b03f9c0b --- /dev/null +++ b/infra/cifuzz/filestore/github_actions/__init__.py @@ -0,0 +1,177 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of a filestore using Github actions artifacts.""" +import logging +import os +import shutil +import sys +import tarfile +import tempfile + +# pylint: disable=wrong-import-position,import-error +sys.path.append( + os.path.join(os.path.pardir, os.path.pardir, os.path.pardir, + os.path.dirname(os.path.abspath(__file__)))) + +import utils +import http_utils +import filestore +from filestore.github_actions import github_api + +UPLOAD_JS = os.path.join(os.path.dirname(__file__), 'upload.js') + + +def tar_directory(directory, archive_path): + """Tars a |directory| and stores archive at |archive_path|. |archive_path| + must end in .tar""" + assert archive_path.endswith('.tar') + # Do this because make_archive will append the extension to archive_path. + archive_path = os.path.splitext(archive_path)[0] + + root_directory = os.path.abspath(directory) + shutil.make_archive(archive_path, + 'tar', + root_dir=root_directory, + base_dir='./') + + +class GithubActionsFilestore(filestore.BaseFilestore): + """Implementation of BaseFilestore using Github actions artifacts. Relies on + github_actions_toolkit for using the GitHub actions API and the github_api + module for using GitHub's standard API. We need to use both because the GitHub + actions API is the only way to upload an artifact but it does not support + downloading artifacts from other runs. The standard GitHub API does support + this however.""" + + ARTIFACT_PREFIX = 'cifuzz-' + BUILD_PREFIX = 'build-' + CRASHES_PREFIX = 'crashes-' + CORPUS_PREFIX = 'corpus-' + COVERAGE_PREFIX = 'coverage-' + + def __init__(self, config): + super().__init__(config) + self.github_api_http_headers = github_api.get_http_auth_headers(config) + + def _get_artifact_name(self, name): + """Returns |name| prefixed with |self.ARITFACT_PREFIX| if it isn't already + prefixed. Otherwise returns |name|.""" + if name.startswith(self.ARTIFACT_PREFIX): + return name + return f'{self.ARTIFACT_PREFIX}{name}' + + def _upload_directory(self, name, directory): # pylint: disable=no-self-use + """Uploads |directory| as artifact with |name|.""" + name = self._get_artifact_name(name) + with tempfile.TemporaryDirectory() as temp_dir: + archive_path = os.path.join(temp_dir, name + '.tar') + tar_directory(directory, archive_path) + _raw_upload_directory(name, temp_dir) + + def upload_crashes(self, name, directory): + """Uploads the crashes at |directory| to |name|.""" + return _raw_upload_directory(self.CRASHES_PREFIX + name, directory) + + def upload_corpus(self, name, directory, replace=False): + """Uploads the corpus at |directory| to |name|.""" + # Not applicable as the the entire corpus is uploaded under a single + # artifact name. + del replace + return self._upload_directory(self.CORPUS_PREFIX + name, directory) + + def upload_build(self, name, directory): + """Uploads the build at |directory| to |name|.""" + return self._upload_directory(self.BUILD_PREFIX + name, directory) + + def upload_coverage(self, name, directory): + """Uploads the coverage report at |directory| to |name|.""" + return self._upload_directory(self.COVERAGE_PREFIX + name, directory) + + def download_corpus(self, name, dst_directory): # pylint: disable=unused-argument,no-self-use + """Downloads the corpus located at |name| to |dst_directory|.""" + return self._download_artifact(self.CORPUS_PREFIX + name, dst_directory) + + def _find_artifact(self, name): + """Finds an artifact using the GitHub API and returns it.""" + logging.debug('Listing artifacts.') + artifacts = self._list_artifacts() + artifact = github_api.find_artifact(name, artifacts) + logging.debug('Artifact: %s.', artifact) + return artifact + + def _download_artifact(self, name, dst_directory): + """Downloads artifact with |name| to |dst_directory|. Returns True on + success.""" + name = self._get_artifact_name(name) + + with tempfile.TemporaryDirectory() as temp_dir: + if not self._raw_download_artifact(name, temp_dir): + logging.warning('Could not download artifact: %s.', name) + return False + + artifact_tarfile_path = os.path.join(temp_dir, name + '.tar') + if not os.path.exists(artifact_tarfile_path): + logging.error('Artifact zip did not contain a tarfile.') + return False + + # TODO(jonathanmetzman): Replace this with archive.unpack from + # libClusterFuzz so we can avoid path traversal issues. + with tarfile.TarFile(artifact_tarfile_path) as artifact_tarfile: + artifact_tarfile.extractall(dst_directory) + return True + + def _raw_download_artifact(self, name, dst_directory): + """Downloads the artifact with |name| to |dst_directory|. Returns True on + success. Does not do any untarring or adding prefix to |name|.""" + artifact = self._find_artifact(name) + if not artifact: + logging.warning('Could not find artifact: %s.', name) + return False + download_url = artifact['archive_download_url'] + return http_utils.download_and_unpack_zip( + download_url, dst_directory, headers=self.github_api_http_headers) + + def _list_artifacts(self): + """Returns a list of artifacts.""" + return github_api.list_artifacts(self.config.project_repo_owner, + self.config.project_repo_name, + self.github_api_http_headers) + + def download_build(self, name, dst_directory): + """Downloads the build with name |name| to |dst_directory|.""" + return self._download_artifact(self.BUILD_PREFIX + name, dst_directory) + + def download_coverage(self, name, dst_directory): + """Downloads the latest project coverage report.""" + return self._download_artifact(self.COVERAGE_PREFIX + name, dst_directory) + + +def _upload_artifact_with_upload_js(name, artifact_paths, directory): + """Uploads the artifacts in |artifact_paths| that are located in |directory| + to |name|, using the upload.js script.""" + command = [UPLOAD_JS, name, directory] + artifact_paths + _, _, retcode = utils.execute(command) + return retcode == 0 + + +def _raw_upload_directory(name, directory): + """Uploads the artifacts located in |directory| to |name|. Does not do any + tarring or adding prefixes to |name|.""" + # Get file paths. + artifact_paths = [] + for root, _, curr_file_paths in os.walk(directory): + for file_path in curr_file_paths: + artifact_paths.append(os.path.join(root, file_path)) + logging.debug('Artifact paths: %s.', artifact_paths) + return _upload_artifact_with_upload_js(name, artifact_paths, directory) diff --git a/infra/cifuzz/filestore/github_actions/github_actions_test.py b/infra/cifuzz/filestore/github_actions/github_actions_test.py new file mode 100644 index 000000000..7745065a9 --- /dev/null +++ b/infra/cifuzz/filestore/github_actions/github_actions_test.py @@ -0,0 +1,281 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for github_actions.""" +import os +import shutil +import sys +import tarfile +import tempfile +import unittest +from unittest import mock + +from pyfakefs import fake_filesystem_unittest + +# pylint: disable=wrong-import-position +INFRA_DIR = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) +sys.path.append(INFRA_DIR) + +from filestore import github_actions +import test_helpers + +# pylint: disable=protected-access,no-self-use + + +class GithubActionsFilestoreTest(fake_filesystem_unittest.TestCase): + """Tests for GithubActionsFilestore.""" + + def setUp(self): + test_helpers.patch_environ(self) + self.token = 'example githubtoken' + self.owner = 'exampleowner' + self.repo = 'examplerepo' + os.environ['GITHUB_REPOSITORY'] = f'{self.owner}/{self.repo}' + os.environ['GITHUB_EVENT_PATH'] = '/fake' + self.config = test_helpers.create_run_config(token=self.token) + self.local_dir = '/local-dir' + self.testcase = os.path.join(self.local_dir, 'testcase') + + def _get_expected_http_headers(self): + return { + 'Authorization': f'token {self.token}', + 'Accept': 'application/vnd.github.v3+json', + } + + @mock.patch('filestore.github_actions.github_api.list_artifacts') + def test_list_artifacts(self, mock_list_artifacts): + """Tests that _list_artifacts works as intended.""" + filestore = github_actions.GithubActionsFilestore(self.config) + filestore._list_artifacts() + mock_list_artifacts.assert_called_with(self.owner, self.repo, + self._get_expected_http_headers()) + + @mock.patch('logging.warning') + @mock.patch('filestore.github_actions.GithubActionsFilestore._list_artifacts', + return_value=None) + @mock.patch('filestore.github_actions.github_api.find_artifact', + return_value=None) + def test_download_build_no_artifact(self, _, __, mock_warning): + """Tests that download_build returns None and doesn't exception when + find_artifact can't find an artifact.""" + filestore = github_actions.GithubActionsFilestore(self.config) + name = 'name' + build_dir = 'build-dir' + self.assertFalse(filestore.download_build(name, build_dir)) + mock_warning.assert_called_with('Could not download artifact: %s.', + 'cifuzz-build-' + name) + + @mock.patch('logging.warning') + @mock.patch('filestore.github_actions.GithubActionsFilestore._list_artifacts', + return_value=None) + @mock.patch('filestore.github_actions.github_api.find_artifact', + return_value=None) + def test_download_corpus_no_artifact(self, _, __, mock_warning): + """Tests that download_corpus_build returns None and doesn't exception when + find_artifact can't find an artifact.""" + filestore = github_actions.GithubActionsFilestore(self.config) + name = 'name' + dst_dir = 'local-dir' + self.assertFalse(filestore.download_corpus(name, dst_dir)) + mock_warning.assert_called_with('Could not download artifact: %s.', + 'cifuzz-corpus-' + name) + + @mock.patch('filestore.github_actions.tar_directory') + @mock.patch('filestore.github_actions._upload_artifact_with_upload_js') + def test_upload_corpus(self, mock_upload_artifact, mock_tar_directory): + """Test uploading corpus.""" + self._create_local_dir() + + def mock_tar_directory_impl(_, archive_path): + self.fs.create_file(archive_path) + + mock_tar_directory.side_effect = mock_tar_directory_impl + + filestore = github_actions.GithubActionsFilestore(self.config) + filestore.upload_corpus('target', self.local_dir) + self.assert_upload(mock_upload_artifact, mock_tar_directory, + 'corpus-target') + + @mock.patch('filestore.github_actions._upload_artifact_with_upload_js') + def test_upload_crashes(self, mock_upload_artifact): + """Test uploading crashes.""" + self._create_local_dir() + + filestore = github_actions.GithubActionsFilestore(self.config) + filestore.upload_crashes('current', self.local_dir) + mock_upload_artifact.assert_has_calls( + [mock.call('crashes-current', ['/local-dir/testcase'], '/local-dir')]) + + @mock.patch('filestore.github_actions.tar_directory') + @mock.patch('filestore.github_actions._upload_artifact_with_upload_js') + def test_upload_build(self, mock_upload_artifact, mock_tar_directory): + """Test uploading build.""" + self._create_local_dir() + + def mock_tar_directory_impl(_, archive_path): + self.fs.create_file(archive_path) + + mock_tar_directory.side_effect = mock_tar_directory_impl + + filestore = github_actions.GithubActionsFilestore(self.config) + filestore.upload_build('sanitizer', self.local_dir) + self.assert_upload(mock_upload_artifact, mock_tar_directory, + 'build-sanitizer') + + @mock.patch('filestore.github_actions.tar_directory') + @mock.patch('filestore.github_actions._upload_artifact_with_upload_js') + def test_upload_coverage(self, mock_upload_artifact, mock_tar_directory): + """Test uploading coverage.""" + self._create_local_dir() + + def mock_tar_directory_impl(_, archive_path): + self.fs.create_file(archive_path) + + mock_tar_directory.side_effect = mock_tar_directory_impl + + filestore = github_actions.GithubActionsFilestore(self.config) + filestore.upload_coverage('latest', self.local_dir) + self.assert_upload(mock_upload_artifact, mock_tar_directory, + 'coverage-latest') + + def assert_upload(self, mock_upload_artifact, mock_tar_directory, + expected_artifact_name): + """Tests that upload_directory invokes tar_directory and + artifact_client.upload_artifact properly.""" + # Don't assert what second argument will be since it's a temporary + # directory. + self.assertEqual(mock_tar_directory.call_args_list[0][0][0], self.local_dir) + + # Don't assert what second and third arguments will be since they are + # temporary directories. + expected_artifact_name = 'cifuzz-' + expected_artifact_name + self.assertEqual(mock_upload_artifact.call_args_list[0][0][0], + expected_artifact_name) + + # Assert artifacts list contains one tarfile. + artifacts_list = mock_upload_artifact.call_args_list[0][0][1] + self.assertEqual(len(artifacts_list), 1) + self.assertEqual(os.path.basename(artifacts_list[0]), + expected_artifact_name + '.tar') + + def _create_local_dir(self): + """Sets up pyfakefs and creates a corpus directory containing + self.testcase.""" + self.setUpPyfakefs() + self.fs.create_file(self.testcase, contents='hi') + + @mock.patch('filestore.github_actions.GithubActionsFilestore._find_artifact') + @mock.patch('http_utils.download_and_unpack_zip') + def test_download_artifact(self, mock_download_and_unpack_zip, + mock_find_artifact): + """Tests that _download_artifact works as intended.""" + artifact_download_url = 'http://example.com/download' + artifact_listing = { + 'expired': False, + 'name': 'corpus', + 'archive_download_url': artifact_download_url + } + mock_find_artifact.return_value = artifact_listing + + self._create_local_dir() + with tempfile.TemporaryDirectory() as temp_dir: + # Create a tarball. + archive_path = os.path.join(temp_dir, 'cifuzz-corpus.tar') + github_actions.tar_directory(self.local_dir, archive_path) + + artifact_download_dst_dir = os.path.join(temp_dir, 'dst') + os.mkdir(artifact_download_dst_dir) + + def mock_download_and_unpack_zip_impl(url, download_artifact_temp_dir, + headers): + self.assertEqual(url, artifact_download_url) + self.assertEqual(headers, self._get_expected_http_headers()) + shutil.copy( + archive_path, + os.path.join(download_artifact_temp_dir, + os.path.basename(archive_path))) + return True + + mock_download_and_unpack_zip.side_effect = ( + mock_download_and_unpack_zip_impl) + filestore = github_actions.GithubActionsFilestore(self.config) + self.assertTrue( + filestore._download_artifact('corpus', artifact_download_dst_dir)) + mock_find_artifact.assert_called_with('cifuzz-corpus') + self.assertTrue( + os.path.exists( + os.path.join(artifact_download_dst_dir, + os.path.basename(self.testcase)))) + + @mock.patch('filestore.github_actions.github_api.list_artifacts') + def test_find_artifact(self, mock_list_artifacts): + """Tests that _find_artifact works as intended.""" + artifact_listing_1 = { + 'expired': False, + 'name': 'other', + 'archive_download_url': 'http://download1' + } + artifact_listing_2 = { + 'expired': False, + 'name': 'artifact', + 'archive_download_url': 'http://download2' + } + artifact_listing_3 = { + 'expired': True, + 'name': 'artifact', + 'archive_download_url': 'http://download3' + } + artifact_listing_4 = { + 'expired': False, + 'name': 'artifact', + 'archive_download_url': 'http://download4' + } + artifacts = [ + artifact_listing_1, artifact_listing_2, artifact_listing_3, + artifact_listing_4 + ] + mock_list_artifacts.return_value = artifacts + filestore = github_actions.GithubActionsFilestore(self.config) + # Test that find_artifact will return the most recent unexpired artifact + # with the correct name. + self.assertEqual(filestore._find_artifact('artifact'), artifact_listing_2) + mock_list_artifacts.assert_called_with(self.owner, self.repo, + self._get_expected_http_headers()) + + +class TarDirectoryTest(unittest.TestCase): + """Tests for tar_directory.""" + + def test_tar_directory(self): + """Tests that tar_directory writes the archive to the correct location and + archives properly.""" + with tempfile.TemporaryDirectory() as temp_dir: + archive_path = os.path.join(temp_dir, 'myarchive.tar') + archived_dir = os.path.join(temp_dir, 'toarchive') + os.mkdir(archived_dir) + archived_filename = 'file1' + archived_file_path = os.path.join(archived_dir, archived_filename) + with open(archived_file_path, 'w') as file_handle: + file_handle.write('hi') + github_actions.tar_directory(archived_dir, archive_path) + self.assertTrue(os.path.exists(archive_path)) + + # Now check it archives correctly. + unpacked_directory = os.path.join(temp_dir, 'unpacked') + with tarfile.TarFile(archive_path) as artifact_tarfile: + artifact_tarfile.extractall(unpacked_directory) + unpacked_archived_file_path = os.path.join(unpacked_directory, + archived_filename) + self.assertTrue(os.path.exists(unpacked_archived_file_path)) diff --git a/infra/cifuzz/filestore/github_actions/github_api.py b/infra/cifuzz/filestore/github_actions/github_api.py new file mode 100644 index 000000000..191b75058 --- /dev/null +++ b/infra/cifuzz/filestore/github_actions/github_api.py @@ -0,0 +1,108 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for dealing with the GitHub API. This is different from +github_actions_toolkit which only deals with the actions API. We need to use +both.""" +import logging +import os +import sys + +import requests + +import filestore + +# pylint: disable=wrong-import-position,import-error + +sys.path.append( + os.path.join(__file__, os.path.pardir, os.path.pardir, os.path.pardir, + os.path.pardir)) +import retry + +_MAX_ITEMS_PER_PAGE = 100 + +_GET_ATTEMPTS = 3 +_GET_BACKOFF = 1 + + +def get_http_auth_headers(config): + """Returns HTTP headers for authentication to the API.""" + authorization = f'token {config.token}' + return { + 'Authorization': authorization, + 'Accept': 'application/vnd.github.v3+json' + } + + +def _get_artifacts_list_api_url(repo_owner, repo_name): + """Returns the artifacts_api_url for |repo_name| owned by |repo_owner|.""" + return (f'https://api.github.com/repos/{repo_owner}/' + f'{repo_name}/actions/artifacts') + + +@retry.wrap(_GET_ATTEMPTS, _GET_BACKOFF) +def _do_get_request(*args, **kwargs): + """Wrapped version of requests.get that does retries.""" + return requests.get(*args, **kwargs) + + +def _get_items(url, headers): + """Generator that gets and yields items from a GitHub API endpoint (specified + by |URL|) sending |headers| with the get request.""" + # Github API response pages are 1-indexed. + page_counter = 1 + + # Set to infinity so we run loop at least once. + total_num_items = float('inf') + + item_num = 0 + while item_num < total_num_items: + params = {'per_page': _MAX_ITEMS_PER_PAGE, 'page': str(page_counter)} + response = _do_get_request(url, params=params, headers=headers) + response_json = response.json() + if not response.status_code == 200: + # Check that request was successful. + logging.error('Request to %s failed. Code: %d. Response: %s', + response.request.url, response.status_code, response_json) + raise filestore.FilestoreError('Github API request failed.') + + if total_num_items == float('inf'): + # Set proper total_num_items + total_num_items = response_json['total_count'] + + # Get the key for the items we are after. + keys = [key for key in response_json.keys() if key != 'total_count'] + assert len(keys) == 1, keys + items_key = keys[0] + + for item in response_json[items_key]: + yield item + item_num += 1 + + page_counter += 1 + + +def find_artifact(artifact_name, artifacts): + """Find the artifact with the name |artifact_name| in |artifacts|.""" + for artifact in artifacts: + # TODO(metzman): Handle multiple by making sure we download the latest. + if artifact['name'] == artifact_name and not artifact['expired']: + return artifact + return None + + +def list_artifacts(owner, repo, headers): + """Returns a generator of all the artifacts for |owner|/|repo|.""" + url = _get_artifacts_list_api_url(owner, repo) + logging.debug('Getting artifacts from: %s', url) + return _get_items(url, headers) diff --git a/infra/cifuzz/filestore/github_actions/github_api_test.py b/infra/cifuzz/filestore/github_actions/github_api_test.py new file mode 100644 index 000000000..c7cad6db0 --- /dev/null +++ b/infra/cifuzz/filestore/github_actions/github_api_test.py @@ -0,0 +1,33 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for github_api.""" +import unittest + +from filestore.github_actions import github_api +import test_helpers + + +class GetHttpAuthHeaders(unittest.TestCase): + """Tests for get_http_auth_headers.""" + + def test_get_http_auth_headers(self): + """Tests that get_http_auth_headers returns the correct result.""" + token = 'example githubtoken' + run_config = test_helpers.create_run_config(token=token) + expected_headers = { + 'Authorization': f'token {token}', + 'Accept': 'application/vnd.github.v3+json', + } + self.assertEqual(expected_headers, + github_api.get_http_auth_headers(run_config)) diff --git a/infra/cifuzz/filestore/github_actions/upload.js b/infra/cifuzz/filestore/github_actions/upload.js new file mode 100755 index 000000000..cd025e560 --- /dev/null +++ b/infra/cifuzz/filestore/github_actions/upload.js @@ -0,0 +1,33 @@ +#!/usr/bin/env node +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Script for uploading an artifact. Returns 0 on success. +// Usage: upload.js <aritfactName> <rootDirectory> <file 1>...<file N> + +const fs = require('fs'); +const artifact = require('@actions/artifact'); +const artifactClient = artifact.create() +const artifactName = process.argv[2]; +const rootDirectory = process.argv[3] +const files = process.argv.slice(4); +const options = { + continueOnError: true +} + +const uploadResult = artifactClient.uploadArtifact(artifactName, files, rootDirectory, options) +console.log(uploadResult); +if (uploadResult['failedItems']) { + return 1; +} +return 0; |