diff options
| -rwxr-xr-x | infra/base-images/base-runner/run_fuzzer | 1 | ||||
| -rw-r--r-- | infra/cifuzz/clusterfuzz_deployment.py | 116 | ||||
| -rw-r--r-- | infra/cifuzz/clusterfuzz_deployment_test.py | 166 | ||||
| -rw-r--r-- | infra/cifuzz/fuzz_target.py | 20 | ||||
| -rw-r--r-- | infra/cifuzz/run_fuzzers.py | 84 | ||||
| -rw-r--r-- | infra/cifuzz/run_fuzzers_test.py | 93 |
6 files changed, 402 insertions, 78 deletions
diff --git a/infra/base-images/base-runner/run_fuzzer b/infra/base-images/base-runner/run_fuzzer index 77e1350af..f53aed60a 100755 --- a/infra/base-images/base-runner/run_fuzzer +++ b/infra/base-images/base-runner/run_fuzzer @@ -35,7 +35,6 @@ then rm -rf $CORPUS_DIR && mkdir -p $CORPUS_DIR fi - SANITIZER=${SANITIZER:-} if [ -z $SANITIZER ]; then # If $SANITIZER is not specified (e.g. calling from `reproduce` command), it diff --git a/infra/cifuzz/clusterfuzz_deployment.py b/infra/cifuzz/clusterfuzz_deployment.py index a8cff1fd1..eb9f835af 100644 --- a/infra/cifuzz/clusterfuzz_deployment.py +++ b/infra/cifuzz/clusterfuzz_deployment.py @@ -18,6 +18,8 @@ import sys import urllib.error import urllib.request +import filestore_utils + import http_utils # pylint: disable=wrong-import-position,import-error @@ -50,7 +52,8 @@ class BaseClusterFuzzDeployment: raise NotImplementedError('Child class must implement method.') def download_corpus(self, target_name, parent_dir): - """Downloads the corpus for |target_name| from ClusterFuzz to |parent_dir|. + """Downloads the corpus for |target_name| from ClusterFuzz to a subdirectory + of |parent_dir|. Returns: A path to where the OSS-Fuzz build was stored, or None if it wasn't. @@ -78,28 +81,98 @@ class BaseClusterFuzzDeployment: """Uploads the corpus for |target_name| in |corpus_dir| to filestore.""" raise NotImplementedError('Child class must implement method.') + def make_empty_corpus_dir(self, target_name, parent_dir): + """Makes an empty corpus directory for |target_name| in |parent_dir| and + returns the path to the directory.""" + corpus_dir = self.get_target_corpus_dir(target_name, parent_dir) + os.makedirs(corpus_dir, exist_ok=True) + return corpus_dir + class ClusterFuzzLite(BaseClusterFuzzDeployment): """Class representing a deployment of ClusterFuzzLite.""" + BASE_BUILD_NAME = 'cifuzz-build-' + + def __init__(self, config): + super().__init__(config) + self.filestore = filestore_utils.get_filestore(self.config) + def download_latest_build(self, parent_dir): - logging.info('download_latest_build not implemented for ClusterFuzzLite.') + build_dir = self.get_build_dir(parent_dir) + if os.path.exists(build_dir): + # This path is necessary because download_latest_build can be called + # multiple times.That is the case because it is called only when we need + # to see if a bug is novel, i.e. until we want to check a bug is novel we + # don't want to waste time calling this, but therefore this method can be + # called if multiple bugs are found. + return build_dir - def download_corpus(self, target_name, parent_dir): - logging.info('download_corpus not implemented for ClusterFuzzLite.') + os.makedirs(build_dir, exist_ok=True) + build_name = self._get_build_name() - def upload_corpus(self, target_name, corpus_dir): # pylint: disable=no-self-use,unused-argument - logging.info('upload_corpus not implemented for ClusterFuzzLite.') + try: + if self.filestore.download_latest_build(build_name, build_dir): + return build_dir + except Exception as err: # pylint: disable=broad-except + logging.error('Could not download latest build because of: %s.', err) + + return None + + def download_corpus(self, target_name, parent_dir): + corpus_dir = self.make_empty_corpus_dir(target_name, parent_dir) + logging.debug('ClusterFuzzLite: downloading corpus for %s to %s.', + target_name, parent_dir) + corpus_name = self._get_corpus_name(target_name) + try: + self.filestore.download_corpus(corpus_name, corpus_dir) + except Exception as err: # pylint: disable=broad-except + logging.error('Failed to download corpus for target: %s. Error: %s.', + target_name, str(err)) + return corpus_dir + + def _get_build_name(self): + return self.BASE_BUILD_NAME + self.config.sanitizer + + def _get_corpus_name(self, target_name): # pylint: disable=no-self-use + """Returns the name of the corpus artifact.""" + return 'corpus-{target_name}'.format(target_name=target_name) + + def _get_crashes_artifact_name(self): # pylint: disable=no-self-use + """Returns the name of the crashes artifact.""" + return 'crashes' + + def upload_corpus(self, target_name, corpus_dir): + """Upload the corpus produced by |target_name| in |corpus_dir|.""" + logging.info('Uploading corpus in %s for %s.', corpus_dir, target_name) + name = self._get_corpus_name(target_name) + try: + self.filestore.upload_directory(name, corpus_dir) + except Exception as error: # pylint: disable=broad-except + logging.error('Failed to upload corpus for target: %s. Error: %s.', + target_name, error) def upload_latest_build(self, build_dir): - """Uploads the latest build to the filestore. - Returns: - True on success. - """ - logging.info('upload_latest_build not implemented for ClusterFuzzLite.') + logging.info('Uploading latest build in %s.', build_dir) + build_name = self._get_build_name() + try: + return self.filestore.upload_directory(build_name, build_dir) + except Exception as error: # pylint: disable=broad-except + logging.error('Failed to upload latest build: %s. Error: %s.', build_dir, + error) def upload_crashes(self, crashes_dir): - logging.info('upload_crashes not implemented for ClusterFuzzLite.') + if not os.listdir(crashes_dir): + logging.info('No crashes in %s. Not uploading.', crashes_dir) + return + + crashes_artifact_name = self._get_crashes_artifact_name() + + logging.info('Uploading crashes in %s', crashes_dir) + try: + self.filestore.upload_directory(crashes_artifact_name, crashes_dir) + except Exception as error: # pylint: disable=broad-except + logging.error('Failed to upload crashes. Error: %s.', error) class OSSFuzz(BaseClusterFuzzDeployment): @@ -166,7 +239,7 @@ class OSSFuzz(BaseClusterFuzzDeployment): def upload_crashes(self, crashes_dir): # pylint: disable=no-self-use,unused-argument """Noop Impelementation of upload_crashes.""" - logging.info('Not uploading crashes on OSS-Fuzz.') + logging.info('Not uploading crashes because on OSS-Fuzz.') def download_corpus(self, target_name, parent_dir): """Downloads the latest OSS-Fuzz corpus for the target. @@ -174,13 +247,9 @@ class OSSFuzz(BaseClusterFuzzDeployment): Returns: The local path to to corpus or None if download failed. """ - corpus_dir = self.get_target_corpus_dir(target_name, parent_dir) - - os.makedirs(corpus_dir, exist_ok=True) - # TODO(metzman): Clean up this code. + corpus_dir = self.make_empty_corpus_dir(target_name, parent_dir) project_qualified_fuzz_target_name = target_name qualified_name_prefix = self.config.project_name + '_' - if not target_name.startswith(qualified_name_prefix): project_qualified_fuzz_target_name = qualified_name_prefix + target_name @@ -189,10 +258,9 @@ class OSSFuzz(BaseClusterFuzzDeployment): f'libFuzzer/{project_qualified_fuzz_target_name}/' f'{self.CORPUS_ZIP_NAME}') - if http_utils.download_and_unpack_zip(corpus_url, corpus_dir): - return corpus_dir - - return None + if not http_utils.download_and_unpack_zip(corpus_url, corpus_dir): + logging.warning('Failed to download corpus for %s.', target_name) + return corpus_dir class NoClusterFuzzDeployment(BaseClusterFuzzDeployment): @@ -215,10 +283,12 @@ class NoClusterFuzzDeployment(BaseClusterFuzzDeployment): def download_corpus(self, target_name, parent_dir): # pylint: disable=no-self-use,unused-argument """Noop Impelementation of download_corpus.""" logging.info('Not downloading corpus because no ClusterFuzz deployment.') + return self.make_empty_corpus_dir(target_name, parent_dir) def download_latest_build(self, parent_dir): # pylint: disable=no-self-use,unused-argument """Noop Impelementation of download_latest_build.""" - logging.info('Not downloading build because no ClusterFuzz deployment.') + logging.info( + 'Not downloading latest build because no ClusterFuzz deployment.') def get_clusterfuzz_deployment(config): diff --git a/infra/cifuzz/clusterfuzz_deployment_test.py b/infra/cifuzz/clusterfuzz_deployment_test.py index 5adb1f897..3c7e27598 100644 --- a/infra/cifuzz/clusterfuzz_deployment_test.py +++ b/infra/cifuzz/clusterfuzz_deployment_test.py @@ -17,9 +17,11 @@ import os import unittest from unittest import mock +import parameterized from pyfakefs import fake_filesystem_unittest import clusterfuzz_deployment +import config_utils import test_helpers # NOTE: This integration test relies on @@ -29,6 +31,9 @@ EXAMPLE_PROJECT = 'example' # An example fuzzer that triggers an error. EXAMPLE_FUZZER = 'example_crash_fuzzer' +OUT_DIR = '/out' +EXPECTED_LATEST_BUILD_PATH = os.path.join(OUT_DIR, 'cifuzz-latest-build') + def _create_config(**kwargs): """Creates a config object and then sets every attribute that is a key in @@ -50,19 +55,16 @@ def _create_deployment(**kwargs): class OSSFuzzTest(fake_filesystem_unittest.TestCase): """Tests OSSFuzz.""" - OUT_DIR = '/out' - def setUp(self): self.setUpPyfakefs() self.deployment = _create_deployment() @mock.patch('http_utils.download_and_unpack_zip', return_value=True) def test_download_corpus(self, mocked_download_and_unpack_zip): - """Tests that we can download a corpus for a valid project.""" - result = self.deployment.download_corpus(EXAMPLE_FUZZER, self.OUT_DIR) + """Tests that download_corpus works for a valid project.""" + result = self.deployment.download_corpus(EXAMPLE_FUZZER, OUT_DIR) self.assertIsNotNone(result) - expected_corpus_dir = os.path.join(self.OUT_DIR, 'cifuzz-corpus', - EXAMPLE_FUZZER) + expected_corpus_dir = os.path.join(OUT_DIR, 'cifuzz-corpus', EXAMPLE_FUZZER) expected_url = ('https://storage.googleapis.com/example-backup.' 'clusterfuzz-external.appspot.com/corpus/libFuzzer/' 'example_crash_fuzzer/public.zip') @@ -70,10 +72,12 @@ class OSSFuzzTest(fake_filesystem_unittest.TestCase): self.assertEqual(call_args, (expected_url, expected_corpus_dir)) @mock.patch('http_utils.download_and_unpack_zip', return_value=False) - def test_download_fail(self, _): - """Tests that when downloading fails, None is returned.""" - corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER, self.OUT_DIR) - self.assertIsNone(corpus_path) + def test_download_corpus_fail(self, _): + """Tests that when downloading fails, an empty corpus directory is still + returned.""" + corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER, OUT_DIR) + self.assertEqual(corpus_path, '/out/cifuzz-corpus/example_crash_fuzzer') + self.assertEqual(os.listdir(corpus_path), []) def test_get_latest_build_name(self): """Tests that the latest build name can be retrieved from GCS.""" @@ -81,6 +85,148 @@ class OSSFuzzTest(fake_filesystem_unittest.TestCase): self.assertTrue(latest_build_name.endswith('.zip')) self.assertTrue('address' in latest_build_name) + @parameterized.parameterized.expand([ + ('upload_latest_build', ('build',), + 'Not uploading latest build because on OSS-Fuzz.'), + ('upload_corpus', ('target', 'corpus_dir'), + 'Not uploading corpus because on OSS-Fuzz.'), + ('upload_crashes', ('crashes_dir',), + 'Not uploading crashes because on OSS-Fuzz.'), + ]) + def test_noop_methods(self, method, method_args, expected_message): + """Tests that certain methods are noops for OSS-Fuzz.""" + with mock.patch('logging.info') as mocked_info: + method = getattr(self.deployment, method) + self.assertIsNone(method(*method_args)) + mocked_info.assert_called_with(expected_message) + + @mock.patch('http_utils.download_and_unpack_zip', return_value=True) + def test_download_latest_build(self, mocked_download_and_unpack_zip): + """Tests that downloading the latest build works as intended under normal + circumstances.""" + self.assertEqual(self.deployment.download_latest_build(OUT_DIR), + EXPECTED_LATEST_BUILD_PATH) + expected_url = ('https://storage.googleapis.com/clusterfuzz-builds/example/' + 'example-address-202008030600.zip') + mocked_download_and_unpack_zip.assert_called_with( + expected_url, EXPECTED_LATEST_BUILD_PATH) + + @mock.patch('http_utils.download_and_unpack_zip', return_value=False) + def test_download_latest_build_fail(self, _): + """Tests that download_latest_build returns None when it fails to download a + build.""" + self.assertIsNone(self.deployment.download_latest_build(OUT_DIR)) + + +class ClusterFuzzLiteTest(fake_filesystem_unittest.TestCase): + """Tests for ClusterFuzzLite.""" + + def setUp(self): + self.setUpPyfakefs() + self.deployment = _create_deployment(run_fuzzers_mode='batch', + build_integration_path='/') + + @mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus', + return_value=True) + def test_download_corpus(self, mocked_download_corpus): + """Tests that download_corpus works for a valid project.""" + result = self.deployment.download_corpus(EXAMPLE_FUZZER, OUT_DIR) + expected_corpus_dir = os.path.join(OUT_DIR, 'cifuzz-corpus', EXAMPLE_FUZZER) + self.assertEqual(result, expected_corpus_dir) + mocked_download_corpus.assert_called_with('corpus-example_crash_fuzzer', + expected_corpus_dir) + + @mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus', + side_effect=Exception) + def test_download_corpus_fail(self, _): + """Tests that when downloading fails, an empty corpus directory is still + returned.""" + corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER, OUT_DIR) + self.assertEqual(corpus_path, '/out/cifuzz-corpus/example_crash_fuzzer') + self.assertEqual(os.listdir(corpus_path), []) + + @mock.patch( + 'filestore.github_actions.GithubActionsFilestore.download_latest_build', + return_value=True) + def test_download_latest_build(self, mocked_download_latest_build): + """Tests that downloading the latest build works as intended under normal + circumstances.""" + self.assertEqual(self.deployment.download_latest_build(OUT_DIR), + EXPECTED_LATEST_BUILD_PATH) + expected_artifact_name = 'cifuzz-build-address' + mocked_download_latest_build.assert_called_with(expected_artifact_name, + EXPECTED_LATEST_BUILD_PATH) + + @mock.patch( + 'filestore.github_actions.GithubActionsFilestore.download_latest_build', + side_effect=Exception) + def test_download_latest_build_fail(self, _): + """Tests that download_latest_build returns None when it fails to download a + build.""" + self.assertIsNone(self.deployment.download_latest_build(OUT_DIR)) + + +class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase): + """Tests for NoClusterFuzzDeployment.""" + + def setUp(self): + self.setUpPyfakefs() + config = test_helpers.create_run_config(project_name=EXAMPLE_PROJECT, + build_integration_path='/', + is_github=False) + self.deployment = clusterfuzz_deployment.get_clusterfuzz_deployment(config) + + @mock.patch('logging.info') + def test_download_corpus(self, mocked_info): + """Tests that download corpus returns the path to the empty corpus + directory.""" + corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER, OUT_DIR) + self.assertEqual(corpus_path, '/out/cifuzz-corpus/example_crash_fuzzer') + mocked_info.assert_called_with( + 'Not downloading corpus because no ClusterFuzz deployment.') + + @parameterized.parameterized.expand([ + ('upload_latest_build', ('build',), + 'Not uploading latest build because no ClusterFuzz deployment.'), + ('upload_corpus', ('target', 'corpus_dir'), + 'Not uploading corpus because no ClusterFuzz deployment.'), + ('upload_crashes', ('crashes_dir',), + 'Not uploading crashes because no ClusterFuzz deployment.'), + ('download_latest_build', ('parent_dir',), + 'Not downloading latest build because no ClusterFuzz deployment.') + ]) + def test_noop_methods(self, method, method_args, expected_message): + """Tests that certain methods are noops for NoClusterFuzzDeployment.""" + with mock.patch('logging.info') as mocked_info: + method = getattr(self.deployment, method) + self.assertIsNone(method(*method_args)) + mocked_info.assert_called_with(expected_message) + + +class GetClusterFuzzDeploymentTest(unittest.TestCase): + """Tests for get_clusterfuzz_deployment.""" + + @parameterized.parameterized.expand([ + (config_utils.BaseConfig.Platform.INTERNAL_GENERIC_CI, + clusterfuzz_deployment.OSSFuzz), + (config_utils.BaseConfig.Platform.INTERNAL_GITHUB, + clusterfuzz_deployment.OSSFuzz), + (config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI, + clusterfuzz_deployment.NoClusterFuzzDeployment), + (config_utils.BaseConfig.Platform.EXTERNAL_GITHUB, + clusterfuzz_deployment.ClusterFuzzLite), + ]) + def test_get_clusterfuzz_deployment(self, platform, expected_deployment_cls): + """Tests that get_clusterfuzz_deployment returns the correct value.""" + with mock.patch('config_utils.BaseConfig.platform', + return_value=platform, + new_callable=mock.PropertyMock): + with mock.patch('filestore_utils.get_filestore', return_value=None): + config = test_helpers.create_run_config() + self.assertIsInstance( + clusterfuzz_deployment.get_clusterfuzz_deployment(config), + expected_deployment_cls) + if __name__ == '__main__': unittest.main() diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index 0cc963557..c8d1c6861 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -101,10 +101,9 @@ class FuzzTarget: # If corpus can be downloaded use it for fuzzing. self.latest_corpus_path = self.clusterfuzz_deployment.download_corpus( self.target_name, self.out_dir) - if self.latest_corpus_path: - command += docker.get_args_mapping_host_path_to_container( - self.latest_corpus_path) - command += ['-e', 'CORPUS_DIR=' + self.latest_corpus_path] + command += docker.get_args_mapping_host_path_to_container( + self.latest_corpus_path) + command += ['-e', 'CORPUS_DIR=' + self.latest_corpus_path] command += [ '-e', 'RUN_FUZZER_MODE=interactive', docker.BASE_RUNNER_TAG, 'bash', @@ -149,14 +148,14 @@ class FuzzTarget: # We found a bug but we won't report it. return FuzzResult(None, None, self.latest_corpus_path) - def free_disk_if_needed(self): + def free_disk_if_needed(self, delete_fuzz_target=True): """Deletes things that are no longer needed from fuzzing this fuzz target to save disk space if needed.""" if not self.config.low_disk_space: + logging.info('Not freeing disk space after running fuzz target.') return - logging.info( - 'Deleting corpus, seed corpus and fuzz target of %s to save disk.', - self.target_name) + logging.info('Deleting corpus and seed corpus of %s to save disk.', + self.target_name) # Delete the seed corpus, corpus, and fuzz target. if self.latest_corpus_path and os.path.exists(self.latest_corpus_path): @@ -164,10 +163,13 @@ class FuzzTarget: # https://github.com/google/oss-fuzz/issues/5383. shutil.rmtree(self.latest_corpus_path, ignore_errors=True) - os.remove(self.target_path) target_seed_corpus_path = self.target_path + '_seed_corpus.zip' if os.path.exists(target_seed_corpus_path): os.remove(target_seed_corpus_path) + + if delete_fuzz_target: + logging.info('Deleting fuzz target: %s.', self.target_name) + os.remove(self.target_path) logging.info('Done deleting.') def is_reproducible(self, testcase, target_path): diff --git a/infra/cifuzz/run_fuzzers.py b/infra/cifuzz/run_fuzzers.py index 99cdf5ca6..6b787fcc7 100644 --- a/infra/cifuzz/run_fuzzers.py +++ b/infra/cifuzz/run_fuzzers.py @@ -91,19 +91,19 @@ class BaseFuzzTargetRunner: return True + def cleanup_after_fuzz_target_run(self, fuzz_target_obj): # pylint: disable=no-self-use + """Cleans up after running |fuzz_target_obj|.""" + raise NotImplementedError('Child class must implement method.') + def run_fuzz_target(self, fuzz_target_obj): # pylint: disable=no-self-use """Fuzzes with |fuzz_target_obj| and returns the result.""" - # TODO(metzman): Make children implement this so that the batch runner can - # do things differently. - result = fuzz_target_obj.fuzz() - fuzz_target_obj.free_disk_if_needed() - return result + raise NotImplementedError('Child class must implement method.') @property def quit_on_bug_found(self): """Property that is checked to determine if fuzzing should quit after first bug is found.""" - raise NotImplementedError('Child class must implement method') + raise NotImplementedError('Child class must implement method.') def get_fuzz_target_artifact(self, target, artifact_name): """Returns the path of a fuzzing artifact named |artifact_name| for @@ -136,6 +136,7 @@ class BaseFuzzTargetRunner: target = self.create_fuzz_target_obj(target_path, run_seconds) start_time = time.time() result = self.run_fuzz_target(target) + self.cleanup_after_fuzz_target_run(target) # It's OK if this goes negative since we take max when determining # run_seconds. @@ -169,7 +170,7 @@ class CoverageTargetRunner(BaseFuzzTargetRunner): @property def quit_on_bug_found(self): - return False + raise NotImplementedError('Not implemented for CoverageTargetRunner.') def get_fuzz_targets(self): """Returns fuzz targets in out_dir.""" @@ -181,11 +182,21 @@ class CoverageTargetRunner(BaseFuzzTargetRunner): return utils.get_fuzz_targets(self.out_dir, top_level_only=True) def run_fuzz_targets(self): + """Generates a coverage report. Always returns False since it never finds + any bugs.""" generate_coverage_report.generate_coverage_report( self.fuzz_target_paths, self.out_dir, self.clusterfuzz_deployment, self.config) return False + def run_fuzz_target(self, fuzz_target_obj): # pylint: disable=no-self-use + """Fuzzes with |fuzz_target_obj| and returns the result.""" + raise NotImplementedError('Not implemented for CoverageTargetRunner.') + + def cleanup_after_fuzz_target_run(self, fuzz_target_obj): # pylint: disable=no-self-use + """Cleans up after running |fuzz_target_obj|.""" + raise NotImplementedError('Not implemented for CoverageTargetRunner.') + class CiFuzzTargetRunner(BaseFuzzTargetRunner): """Runner for fuzz targets used in CI (patch-fuzzing) context.""" @@ -194,6 +205,13 @@ class CiFuzzTargetRunner(BaseFuzzTargetRunner): def quit_on_bug_found(self): return True + def cleanup_after_fuzz_target_run(self, fuzz_target_obj): # pylint: disable=no-self-use + """Cleans up after running |fuzz_target_obj|.""" + fuzz_target_obj.free_disk_if_needed() + + def run_fuzz_target(self, fuzz_target_obj): # pylint: disable=no-self-use + return fuzz_target_obj.fuzz() + class BatchFuzzTargetRunner(BaseFuzzTargetRunner): """Runner for fuzz targets used in batch fuzzing context.""" @@ -202,6 +220,58 @@ class BatchFuzzTargetRunner(BaseFuzzTargetRunner): def quit_on_bug_found(self): return False + def run_fuzz_target(self, fuzz_target_obj): + """Fuzzes with |fuzz_target_obj| and returns the result.""" + result = fuzz_target_obj.fuzz() + logging.debug('corpus_path: %s', os.listdir(result.corpus_path)) + self.clusterfuzz_deployment.upload_corpus(fuzz_target_obj.target_name, + result.corpus_path) + return result + + def cleanup_after_fuzz_target_run(self, fuzz_target_obj): + """Cleans up after running |fuzz_target_obj|.""" + # This must be done after we upload the corpus, otherwise it will be deleted + # before we get a chance to upload it. We can't delete the fuzz target + # because it is needed when we upload the build. + fuzz_target_obj.free_disk_if_needed(delete_fuzz_target=False) + + def run_fuzz_targets(self): + result = super().run_fuzz_targets() + + self.clusterfuzz_deployment.upload_crashes(self.crashes_dir) + + # We want to upload the build to the filestore after we do batch fuzzing. + # There are some problems with this. First, we don't want to upload the + # build before fuzzing, because if we download the latest build, we will + # consider the build we just uploaded to be the latest even though it + # shouldn't be (we really intend to download the build before the curent + # one. + # Second, the out directory is mounted into the runnner container and is + # used to pass the runner corpora, old builds and for the runner to pass the + # host testcases. Thus, we will upload the build after fuzzing. But before + # we zip the out directory we will remove these extra things that are now in + # out. + # TODO(metzman): We should really be uploading latest build in build_fuzzers + # before we remove unaffected fuzzers. Otherwise, we can lose fuzzers. This + # is probably more of a theoretical concern since in batch fuzzing, there is + # no code change and thus no fuzzers that are removed, but it's inelegant to + # put this here. + # TODO(metzman): Don't pollute self.out_dir like this. + + for directory in [ + self.clusterfuzz_deployment.get_corpus_dir(self.out_dir), + # This is the directory of the ClusterFuzz build, not the build we just + # did. + self.clusterfuzz_deployment.get_build_dir(self.out_dir), + self.crashes_dir, + ]: + if os.path.exists(directory): + shutil.rmtree(directory) + + self.clusterfuzz_deployment.upload_latest_build(self.out_dir) + + return result + def get_fuzz_target_runner(config): """Returns a fuzz target runner object based on the run_fuzzers_mode of diff --git a/infra/cifuzz/run_fuzzers_test.py b/infra/cifuzz/run_fuzzers_test.py index eacf8fc91..83f18d517 100644 --- a/infra/cifuzz/run_fuzzers_test.py +++ b/infra/cifuzz/run_fuzzers_test.py @@ -258,55 +258,95 @@ class CiFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): - """Tests that CiFuzzTargetRunner works as intended.""" + """Tests that BatchFuzzTargetRunnerTest works as intended.""" + WORKSPACE = 'workspace' + STACKTRACE = b'stacktrace' + CORPUS_DIR = 'corpus' def setUp(self): self.setUpPyfakefs() - - @mock.patch('utils.get_fuzz_targets') + self.out_dir = os.path.join(self.WORKSPACE, 'out') + self.fs.create_dir(self.out_dir) + self.testcase1 = os.path.join(self.WORKSPACE, 'testcase-aaa') + self.fs.create_file(self.testcase1) + self.testcase2 = os.path.join(self.WORKSPACE, 'testcase-bbb') + self.fs.create_file(self.testcase2) + self.config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS, + workspace=self.WORKSPACE, + project_name=EXAMPLE_PROJECT, + build_integration_path='/', + is_github=True) + + @mock.patch('utils.get_fuzz_targets', return_value=['target1', 'target2']) + @mock.patch('clusterfuzz_deployment.ClusterFuzzLite.upload_latest_build', + return_value=True) @mock.patch('run_fuzzers.BatchFuzzTargetRunner.run_fuzz_target') @mock.patch('run_fuzzers.BatchFuzzTargetRunner.create_fuzz_target_obj') def test_run_fuzz_targets_quits(self, mocked_create_fuzz_target_obj, - mocked_run_fuzz_target, - mocked_get_fuzz_targets): + mocked_run_fuzz_target, _, __): """Tests that run_fuzz_targets doesn't quit on the first crash it finds.""" - workspace = 'workspace' - out_path = os.path.join(workspace, 'out') - self.fs.create_dir(out_path) - config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS, - workspace=workspace, - project_name=EXAMPLE_PROJECT) - runner = run_fuzzers.BatchFuzzTargetRunner(config) - - mocked_get_fuzz_targets.return_value = ['target1', 'target2'] + runner = run_fuzzers.BatchFuzzTargetRunner(self.config) runner.initialize() - testcase1 = os.path.join(workspace, 'testcase-aaa') - testcase2 = os.path.join(workspace, 'testcase-bbb') - self.fs.create_file(testcase1) - self.fs.create_file(testcase2) - stacktrace = b'stacktrace' + call_count = 0 - corpus_dir = 'corpus' def mock_run_fuzz_target(_): nonlocal call_count if call_count == 0: - testcase = testcase1 + testcase = self.testcase1 elif call_count == 1: - testcase = testcase2 + testcase = self.testcase2 assert call_count != 2 call_count += 1 - return fuzz_target.FuzzResult(testcase, stacktrace, corpus_dir) + if not os.path.exists(self.CORPUS_DIR): + self.fs.create_dir(self.CORPUS_DIR) + return fuzz_target.FuzzResult(testcase, self.STACKTRACE, self.CORPUS_DIR) mocked_run_fuzz_target.side_effect = mock_run_fuzz_target magic_mock = mock.MagicMock() magic_mock.target_name = 'target1' mocked_create_fuzz_target_obj.return_value = magic_mock self.assertTrue(runner.run_fuzz_targets()) - self.assertIn('target1-address-testcase-aaa', - os.listdir(runner.crashes_dir)) self.assertEqual(mocked_run_fuzz_target.call_count, 2) + @mock.patch('run_fuzzers.BaseFuzzTargetRunner.run_fuzz_targets', + return_value=False) + @mock.patch('clusterfuzz_deployment.ClusterFuzzLite.upload_latest_build') + @mock.patch('clusterfuzz_deployment.ClusterFuzzLite.upload_crashes') + def test_run_fuzz_targets_upload_crashes_and_builds( + self, mocked_upload_crashes, mocked_upload_latest_build, _): + """Tests that run_fuzz_targets uploads crashes and builds correctly.""" + runner = run_fuzzers.BatchFuzzTargetRunner(self.config) + runner.initialize() + + expected_crashes_dir = 'workspace/out/artifacts' + + def mock_upload_crashes(crashes_dir): + self.assertEqual(crashes_dir, expected_crashes_dir) + # Ensure it wasn't deleted first. + self.assertTrue(os.path.exists(crashes_dir)) + + mocked_upload_crashes.side_effect = mock_upload_crashes + + expected_out_dir = 'workspace/out' + expected_build_dir = 'workspace/out/cifuzz-latest-build' + expected_corpus_dir = 'workspace/out/cifuzz-corpus' + self.fs.create_dir(expected_build_dir) + self.fs.create_dir(expected_corpus_dir) + + def mock_upload_latest_build(out_dir): + self.assertEqual(out_dir, expected_out_dir) + # Ensure these were deleted before this function call. + self.assertFalse(os.path.exists(expected_crashes_dir)) + self.assertFalse(os.path.exists(expected_build_dir)) + self.assertFalse(os.path.exists(expected_corpus_dir)) + + mocked_upload_latest_build.side_effect = mock_upload_latest_build + + self.assertFalse(runner.run_fuzz_targets()) + self.assertEqual(mocked_upload_crashes.call_count, 1) + self.assertEqual(mocked_upload_latest_build.call_count, 1) + @unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 'INTEGRATION_TESTS=1 not set') @@ -394,9 +434,6 @@ class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin, side_effect=[True, True]) def test_old_bug_found(self, _): """Tests run_fuzzers with a bug found in OSS-Fuzz before.""" - config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS, - workspace=TEST_DATA_PATH, - project_name=EXAMPLE_PROJECT) with tempfile.TemporaryDirectory() as tmp_dir: workspace = os.path.join(tmp_dir, 'workspace') shutil.copytree(TEST_DATA_PATH, workspace) |