diff options
| -rw-r--r-- | infra/cifuzz/actions/run_fuzzers/action.yml | 9 | ||||
| -rw-r--r-- | infra/cifuzz/config_utils.py | 7 | ||||
| -rw-r--r-- | infra/cifuzz/continuous_integration.py | 8 | ||||
| -rw-r--r-- | infra/cifuzz/fuzz_target.py | 10 | ||||
| -rw-r--r-- | infra/cifuzz/run_fuzzers.py | 210 | ||||
| -rw-r--r-- | infra/cifuzz/run_fuzzers_test.py | 304 | ||||
| -rw-r--r-- | infra/cifuzz/stack_parser.py | 9 | ||||
| -rw-r--r-- | infra/cifuzz/stack_parser_test.py | 35 |
8 files changed, 460 insertions, 132 deletions
diff --git a/infra/cifuzz/actions/run_fuzzers/action.yml b/infra/cifuzz/actions/run_fuzzers/action.yml index 8434753f5..42cb2ddaf 100644 --- a/infra/cifuzz/actions/run_fuzzers/action.yml +++ b/infra/cifuzz/actions/run_fuzzers/action.yml @@ -15,6 +15,14 @@ inputs: sanitizer: description: 'The sanitizer to run the fuzzers with.' default: 'address' + run-fuzzers-mode: + description: | + The mode to run the fuzzers with ("ci" or "batch"). + "ci" is for fuzzing a pull request or commit. + "batch" is for non-interactive fuzzing of an entire project. + "batch" is in alpha and should not be used in production. + required: false + default: 'ci' runs: using: 'docker' image: '../../../run_fuzzers.Dockerfile' @@ -23,3 +31,4 @@ runs: FUZZ_SECONDS: ${{ inputs.fuzz-seconds }} DRY_RUN: ${{ inputs.dry-run}} SANITIZER: ${{ inputs.sanitizer }} + RUN_FUZZERS_MODE: ${{ inputs.run-fuzzers-mode }} diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index 15da2058f..e2093b5b1 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -98,9 +98,16 @@ class BaseConfig: class RunFuzzersConfig(BaseConfig): """Class containing constant configuration for running fuzzers in CIFuzz.""" + RUN_FUZZERS_MODES = {'batch', 'ci'} + def __init__(self): super().__init__() self.fuzz_seconds = int(os.environ.get('FUZZ_SECONDS', 600)) + self.run_fuzzers_mode = os.environ.get('RUN_FUZZERS_MODE', 'ci').lower() + if self.run_fuzzers_mode not in self.RUN_FUZZERS_MODES: + raise Exception( + ('Invalid RUN_FUZZERS_MODE %s not one of allowed choices: %s.' % + self.run_fuzzers_mode, self.RUN_FUZZERS_MODES)) class BuildFuzzersConfig(BaseConfig): diff --git a/infra/cifuzz/continuous_integration.py b/infra/cifuzz/continuous_integration.py index ec8c87b68..75a9e2245 100644 --- a/infra/cifuzz/continuous_integration.py +++ b/infra/cifuzz/continuous_integration.py @@ -197,10 +197,15 @@ class ExternalGithub(GithubCiMixin, BaseCi): logging.info('Building external project.') git_workspace = os.path.join(self.config.workspace, 'storage') os.makedirs(git_workspace, exist_ok=True) + # Checkout before building, so we don't need to rely on copying the source + # into the image. + # TODO(metzman): Figure out if we want second copy at all. manager = repo_manager.clone_repo_and_get_manager( self.config.git_url, git_workspace, repo_name=self.config.project_repo_name) + checkout_specified_commit(manager, self.config.pr_ref, + self.config.commit_sha) build_integration_path = os.path.join(manager.repo_dir, self.config.build_integration_path) @@ -209,8 +214,5 @@ class ExternalGithub(GithubCiMixin, BaseCi): logging.error('Failed to build external project.') return BuildPreparationResult(False, None, None) - checkout_specified_commit(manager, self.config.pr_ref, - self.config.commit_sha) - image_repo_path = os.path.join('/src', self.config.project_repo_name) return BuildPreparationResult(True, image_repo_path, manager) diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index e061b4336..6d42563e1 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -76,7 +76,7 @@ class FuzzTarget: project_name: The name of the relevant OSS-Fuzz project. """ - #pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments def __init__(self, target_path, duration, @@ -94,9 +94,12 @@ class FuzzTarget: out_dir: The location of where the output from crashes should be stored. project_name: The name of the relevant OSS-Fuzz project. """ - self.target_name = os.path.basename(target_path) - self.duration = int(duration) + # TODO(metzman): Get rid of sanitizer defaulting to address. config_utils + # implements this functionality. Also look into why project_name defaults to + # None. Maybe accept config and get those values from there. self.target_path = target_path + self.target_name = os.path.basename(self.target_path) + self.duration = int(duration) self.out_dir = out_dir self.project_name = project_name self.sanitizer = sanitizer @@ -108,6 +111,7 @@ class FuzzTarget: (testcase, stacktrace, time in seconds) on crash or (None, None, time in seconds) on timeout or error. """ + # TODO(metzman): Change return value to a FuzzResult object. logging.info('Fuzzer %s, started.', self.target_name) docker_container = utils.get_container_name() command = ['docker', 'run', '--rm', '--privileged'] diff --git a/infra/cifuzz/run_fuzzers.py b/infra/cifuzz/run_fuzzers.py index 1bb08fe6c..1ba6865c4 100644 --- a/infra/cifuzz/run_fuzzers.py +++ b/infra/cifuzz/run_fuzzers.py @@ -27,61 +27,175 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import utils +class BaseFuzzTargetRunner: + """Base class for fuzzer runners.""" + + def __init__(self, config): + self.config = config + # Set by the initialize method. + self.out_dir = None + self.fuzz_target_paths = None + self.artifacts_dir = None + + def initialize(self): + """Initialization method. Must be called before calling run_fuzz_targets. + Returns True on success.""" + # Use a seperate initialization function so we can return False on failure + # instead of exceptioning like we need to do if this were done in the + # __init__ method. + + logging.info('Using %s sanitizer.', self.config.sanitizer) + + # TODO(metzman) Add a check to ensure we aren't over time limit. + if not self.config.fuzz_seconds or self.config.fuzz_seconds < 1: + logging.error( + 'Fuzz_seconds argument must be greater than 1, but was: %s.', + self.config.fuzz_seconds) + return False + + self.out_dir = os.path.join(self.config.workspace, 'out') + if not os.path.exists(self.out_dir): + logging.error('Out directory: %s does not exist.', self.out_dir) + return False + + self.artifacts_dir = os.path.join(self.out_dir, 'artifacts') + if not os.path.exists(self.artifacts_dir): + os.mkdir(self.artifacts_dir) + elif (not os.path.isdir(self.artifacts_dir) or + os.listdir(self.artifacts_dir)): + logging.error('Artifacts path: %s exists and is not an empty directory.', + self.artifacts_dir) + return False + + self.fuzz_target_paths = utils.get_fuzz_targets(self.out_dir) + logging.info('Fuzz targets: %s', self.fuzz_target_paths) + if not self.fuzz_target_paths: + logging.error('No fuzz targets were found in out directory: %s.', + self.out_dir) + return False + + return True + + def run_fuzz_target(self, fuzz_target_obj): # pylint: disable=no-self-use + """Fuzzes with |fuzz_target_obj| and returns the result.""" + # TODO(metzman): Make children implement this so that the batch runner can + # do things differently. + return fuzz_target_obj.fuzz() + + @property + def quit_on_bug_found(self): + """Property that is checked to determine if fuzzing should quit after first + bug is found.""" + raise NotImplementedError('Child class must implement method') + + def get_fuzz_target_artifact(self, target, artifact_name): + """Returns the path of a fuzzing |artifact| named |artifact_name| for + |target|.""" + artifact_name = target.target_name + '-' + artifact_name + return os.path.join(self.artifacts_dir, artifact_name) + + def create_fuzz_target_obj(self, target_path, run_seconds): + """Returns a fuzz target object.""" + return fuzz_target.FuzzTarget(target_path, + run_seconds, + self.out_dir, + self.config.project_name, + sanitizer=self.config.sanitizer) + + def run_fuzz_targets(self): + """Runs fuzz targets. Returns True if a bug was found.""" + fuzzers_left_to_run = len(self.fuzz_target_paths) + + # Make a copy since we will mutate it. + fuzz_seconds = self.config.fuzz_seconds + + min_seconds_per_fuzzer = fuzz_seconds // fuzzers_left_to_run + bug_found = False + for target_path in self.fuzz_target_paths: + # By doing this, we can ensure that every fuzz target runs for at least + # min_seconds_per_fuzzer, but that other fuzzers will have longer to run + # if one ends early. + run_seconds = max(fuzz_seconds // fuzzers_left_to_run, + min_seconds_per_fuzzer) + + target = self.create_fuzz_target_obj(target_path, run_seconds) + start_time = time.time() + testcase, stacktrace = self.run_fuzz_target(target) + + # It's OK if this goes negative since we take max when determining + # run_seconds. + fuzz_seconds -= time.time() - start_time + + fuzzers_left_to_run -= 1 + if not testcase or not stacktrace: + logging.info('Fuzzer %s finished running without crashes.', + target.target_name) + continue + + # We found a bug in the fuzz target. + utils.binary_print(b'Fuzzer: %s. Detected bug:\n%s' % + (target.target_name.encode(), stacktrace)) + + # TODO(metzman): Do this with filestore. + testcase_artifact = self.get_fuzz_target_artifact(target, 'testcase') + shutil.move(testcase, testcase_artifact) + bug_summary_artifact = self.get_fuzz_target_artifact( + target, 'bug-summary.txt') + stack_parser.parse_fuzzer_output(stacktrace, bug_summary_artifact) + + bug_found = True + if self.quit_on_bug_found: + logging.info('Bug found. Stopping fuzzing.') + return bug_found + + return bug_found + + +class CiFuzzTargetRunner(BaseFuzzTargetRunner): + """Runner for fuzz targets used in CI (patch-fuzzing) context.""" + + @property + def quit_on_bug_found(self): + return True + + +class BatchFuzzTargetRunner(BaseFuzzTargetRunner): + """Runner for fuzz targets used in batch fuzzing context.""" + + @property + def quit_on_bug_found(self): + return False + + +def get_fuzz_target_runner(config): + """Returns a fuzz target runner object based on the run_fuzzers_mode of + |config|.""" + logging.info('RUN_FUZZERS_MODE is: %s', config.run_fuzzers_mode) + if config.run_fuzzers_mode == 'batch': + return BatchFuzzTargetRunner(config) + return CiFuzzTargetRunner(config) + + def run_fuzzers(config): # pylint: disable=too-many-locals """Runs fuzzers for a specific OSS-Fuzz project. Args: - fuzz_seconds: The total time allotted for fuzzing. - workspace: The location in a shared volume to store a git repo and build - artifacts. - project_name: The name of the relevant OSS-Fuzz project. - sanitizer: The sanitizer the fuzzers should be run with. + config: A RunFuzzTargetsConfig. Returns: - (True if run was successful, True if bug was found). + (True if no (internal) errors fuzzing, True if bug found fuzzing). """ - # Validate inputs. - logging.info('Using %s sanitizer.', config.sanitizer) - - out_dir = os.path.join(config.workspace, 'out') - artifacts_dir = os.path.join(out_dir, 'artifacts') - os.makedirs(artifacts_dir, exist_ok=True) - - if not config.fuzz_seconds or config.fuzz_seconds < 1: - logging.error('Fuzz_seconds argument must be greater than 1, but was: %s.', - config.fuzz_seconds) - return False, False - - # Get fuzzer information. - fuzzer_paths = utils.get_fuzz_targets(out_dir) - if not fuzzer_paths: - logging.error('No fuzzers were found in out directory: %s.', out_dir) + fuzz_target_runner = get_fuzz_target_runner(config) + # TODO(metzman): Multiple return bools is confusing. Change to one enum + # return value. + if not fuzz_target_runner.initialize(): + # We didn't fuzz at all because of internal (CIFuzz) errors. And we didn't + # find any bugs. return False, False - # Run fuzzers for allotted time. - total_num_fuzzers = len(fuzzer_paths) - fuzzers_left_to_run = total_num_fuzzers - min_seconds_per_fuzzer = config.fuzz_seconds // total_num_fuzzers - for fuzzer_path in fuzzer_paths: - run_seconds = max(config.fuzz_seconds // fuzzers_left_to_run, - min_seconds_per_fuzzer) - - target = fuzz_target.FuzzTarget(fuzzer_path, - run_seconds, - out_dir, - config.project_name, - sanitizer=config.sanitizer) - start_time = time.time() - testcase, stacktrace = target.fuzz() - config.fuzz_seconds -= (time.time() - start_time) - if not testcase or not stacktrace: - logging.info('Fuzzer %s, finished running.', target.target_name) - else: - utils.binary_print(b'Fuzzer %s, detected error:\n%s' % - (target.target_name.encode(), stacktrace)) - shutil.move(testcase, os.path.join(artifacts_dir, 'test_case')) - stack_parser.parse_fuzzer_output(stacktrace, artifacts_dir) - return True, True - fuzzers_left_to_run -= 1 + if not fuzz_target_runner.run_fuzz_targets(): + # We fuzzed successfully, but didn't find any bugs (in the fuzz target). + return True, False - return True, False + # We fuzzed successfully and found bug(s) in the fuzz targets. + return True, True diff --git a/infra/cifuzz/run_fuzzers_test.py b/infra/cifuzz/run_fuzzers_test.py index 75b27b104..a55ed8495 100644 --- a/infra/cifuzz/run_fuzzers_test.py +++ b/infra/cifuzz/run_fuzzers_test.py @@ -14,12 +14,15 @@ """Tests for running fuzzers.""" import os import sys +import shutil import tempfile import unittest from unittest import mock +import parameterized +from pyfakefs import fake_filesystem_unittest + import config_utils -import fuzz_target import run_fuzzers # pylint: disable=wrong-import-position @@ -42,8 +45,10 @@ MEMORY_FUZZER = 'curl_fuzzer_memory' UNDEFINED_FUZZER_DIR = os.path.join(TEST_FILES_PATH, 'undefined') UNDEFINED_FUZZER = 'curl_fuzzer_undefined' +FUZZ_SECONDS = 10 + -def create_config(**kwargs): +def _create_config(**kwargs): """Creates a config object and then sets every attribute that is a key in |kwargs| to the corresponding value. Asserts that each key in |kwargs| is an attribute of Config.""" @@ -70,10 +75,10 @@ class RunFuzzerIntegrationTestMixin: # pylint: disable=too-few-public-methods,i """Calls run_fuzzers on fuzzer_dir and |sanitizer| and asserts the run succeeded and that no bug was found.""" with test_helpers.temp_dir_copy(fuzzer_dir) as fuzzer_dir_copy: - config = create_config(fuzz_seconds=10, - workspace=fuzzer_dir_copy, - project_name='curl', - sanitizer=sanitizer) + config = _create_config(fuzz_seconds=FUZZ_SECONDS, + workspace=fuzzer_dir_copy, + project_name='curl', + sanitizer=sanitizer) run_success, bug_found = run_fuzzers.run_fuzzers(config) self.assertTrue(run_success) self.assertFalse(bug_found) @@ -105,6 +110,209 @@ class RunUndefinedFuzzerIntegrationTest(RunFuzzerIntegrationTestMixin, self._test_run_with_sanitizer(self.FUZZER_DIR, 'undefined') +class BaseFuzzTargetRunnerTest(unittest.TestCase): + """Tests BaseFuzzTargetRunner.""" + + def _create_runner(self, **kwargs): # pylint: disable=no-self-use + defaults = {'fuzz_seconds': FUZZ_SECONDS, 'project_name': EXAMPLE_PROJECT} + for default_key, default_value in defaults.items(): + if default_key not in kwargs: + kwargs[default_key] = default_value + + config = _create_config(**kwargs) + return run_fuzzers.BaseFuzzTargetRunner(config) + + def _test_initialize_fail(self, expected_error_args, **create_runner_kwargs): + with mock.patch('logging.error') as mocked_error: + runner = self._create_runner(**create_runner_kwargs) + self.assertFalse(runner.initialize()) + mocked_error.assert_called_with(*expected_error_args) + + @parameterized.parameterized.expand([(0,), (None,), (-1,)]) + def test_initialize_invalid_fuzz_seconds(self, fuzz_seconds): + """Tests initialize fails with an invalid fuzz seconds.""" + expected_error_args = ('Fuzz_seconds argument must be greater than 1, ' + 'but was: %s.', fuzz_seconds) + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + os.mkdir(out_path) + with mock.patch('utils.get_fuzz_targets') as mocked_get_fuzz_targets: + mocked_get_fuzz_targets.return_value = [ + os.path.join(out_path, 'fuzz_target') + ] + self._test_initialize_fail(expected_error_args, + fuzz_seconds=fuzz_seconds, + workspace=tmp_dir) + + def test_initialize_no_out_dir(self): + """Tests initialize fails with no out dir.""" + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + expected_error_args = ('Out directory: %s does not exist.', out_path) + self._test_initialize_fail(expected_error_args, workspace=tmp_dir) + + def test_initialize_nonempty_artifacts(self): + """Tests initialize with a file artifacts path.""" + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + os.mkdir(out_path) + artifacts_path = os.path.join(out_path, 'artifacts') + with open(artifacts_path, 'w') as artifacts_handle: + artifacts_handle.write('fake') + expected_error_args = ( + 'Artifacts path: %s exists and is not an empty directory.', + artifacts_path) + self._test_initialize_fail(expected_error_args, workspace=tmp_dir) + + def test_initialize_bad_artifacts(self): + """Tests initialize with a non-empty artifacts path.""" + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + artifacts_path = os.path.join(out_path, 'artifacts') + os.makedirs(artifacts_path) + artifact_path = os.path.join(artifacts_path, 'artifact') + with open(artifact_path, 'w') as artifact_handle: + artifact_handle.write('fake') + expected_error_args = ( + 'Artifacts path: %s exists and is not an empty directory.', + artifacts_path) + self._test_initialize_fail(expected_error_args, workspace=tmp_dir) + + @mock.patch('utils.get_fuzz_targets') + @mock.patch('logging.error') + def test_initialize_empty_artifacts(self, mocked_log_error, + mocked_get_fuzz_targets): + """Tests initialize with an empty artifacts dir.""" + mocked_get_fuzz_targets.return_value = ['fuzz-target'] + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + artifacts_path = os.path.join(out_path, 'artifacts') + os.makedirs(artifacts_path) + runner = self._create_runner(workspace=tmp_dir) + self.assertTrue(runner.initialize()) + mocked_log_error.assert_not_called() + self.assertTrue(os.path.isdir(artifacts_path)) + + @mock.patch('utils.get_fuzz_targets') + @mock.patch('logging.error') + def test_initialize_no_artifacts(self, mocked_log_error, + mocked_get_fuzz_targets): + """Tests initialize with no artifacts dir (the expected setting).""" + mocked_get_fuzz_targets.return_value = ['fuzz-target'] + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + os.makedirs(out_path) + runner = self._create_runner(workspace=tmp_dir) + self.assertTrue(runner.initialize()) + mocked_log_error.assert_not_called() + self.assertTrue(os.path.isdir(os.path.join(out_path, 'artifacts'))) + + def test_initialize_no_fuzz_targets(self): + """Tests initialize with no fuzz targets.""" + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + os.makedirs(out_path) + expected_error_args = ('No fuzz targets were found in out directory: %s.', + out_path) + self._test_initialize_fail(expected_error_args, workspace=tmp_dir) + + def test_get_fuzz_target_artifact(self): + """Tests that get_fuzz_target_artifact works as intended.""" + runner = self._create_runner() + artifacts_dir = 'artifacts-dir' + runner.artifacts_dir = artifacts_dir + artifact_name = 'artifact-name' + target = mock.MagicMock() + target_name = 'target_name' + target.target_name = target_name + fuzz_target_artifact = runner.get_fuzz_target_artifact( + target, artifact_name) + expected_fuzz_target_artifact = 'artifacts-dir/target_name-artifact-name' + self.assertEqual(fuzz_target_artifact, expected_fuzz_target_artifact) + + +class CiFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): + """Tests that CiFuzzTargetRunner works as intended.""" + + def setUp(self): + self.setUpPyfakefs() + + @mock.patch('run_fuzzers.CiFuzzTargetRunner.run_fuzz_target') + @mock.patch('run_fuzzers.CiFuzzTargetRunner.create_fuzz_target_obj') + def test_run_fuzz_targets_quits(self, mocked_create_fuzz_target_obj, + mocked_run_fuzz_target): + """Tests that run_fuzz_targets quits on the first crash it finds.""" + workspace = 'workspace' + out_path = os.path.join(workspace, 'out') + self.fs.create_dir(out_path) + config = _create_config(fuzz_seconds=FUZZ_SECONDS, + workspace=workspace, + project_name=EXAMPLE_PROJECT) + runner = run_fuzzers.CiFuzzTargetRunner(config) + + with mock.patch('utils.get_fuzz_targets') as mocked_get_fuzz_targets: + mocked_get_fuzz_targets.return_value = ['target1', 'target2'] + runner.initialize() + testcase = os.path.join(workspace, 'testcase') + self.fs.create_file(testcase) + stacktrace = b'stacktrace' + mocked_run_fuzz_target.return_value = (testcase, stacktrace) + magic_mock = mock.MagicMock() + magic_mock.target_name = 'target1' + mocked_create_fuzz_target_obj.return_value = magic_mock + self.assertTrue(runner.run_fuzz_targets()) + self.assertIn('target1-testcase', os.listdir(runner.artifacts_dir)) + self.assertEqual(mocked_run_fuzz_target.call_count, 1) + + +class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): + """Tests that CiFuzzTargetRunner works as intended.""" + + def setUp(self): + self.setUpPyfakefs() + + @mock.patch('run_fuzzers.BatchFuzzTargetRunner.run_fuzz_target') + @mock.patch('run_fuzzers.BatchFuzzTargetRunner.create_fuzz_target_obj') + def test_run_fuzz_targets_quits(self, mocked_create_fuzz_target_obj, + mocked_run_fuzz_target): + """Tests that run_fuzz_targets quits on the first crash it finds.""" + workspace = 'workspace' + out_path = os.path.join(workspace, 'out') + self.fs.create_dir(out_path) + config = _create_config(fuzz_seconds=FUZZ_SECONDS, + workspace=workspace, + project_name=EXAMPLE_PROJECT) + runner = run_fuzzers.BatchFuzzTargetRunner(config) + + with mock.patch('utils.get_fuzz_targets') as mocked_get_fuzz_targets: + mocked_get_fuzz_targets.return_value = ['target1', 'target2'] + runner.initialize() + testcase1 = os.path.join(workspace, 'testcase1') + testcase2 = os.path.join(workspace, 'testcase2') + self.fs.create_file(testcase1) + self.fs.create_file(testcase2) + stacktrace = b'stacktrace' + call_count = 0 + + def mock_run_fuzz_target(_): + nonlocal call_count + if call_count == 0: + testcase = testcase1 + elif call_count == 1: + testcase = testcase2 + assert call_count != 2 + call_count += 1 + return testcase, stacktrace + + mocked_run_fuzz_target.side_effect = mock_run_fuzz_target + magic_mock = mock.MagicMock() + magic_mock.target_name = 'target1' + mocked_create_fuzz_target_obj.return_value = magic_mock + self.assertTrue(runner.run_fuzz_targets()) + self.assertIn('target1-testcase', os.listdir(runner.artifacts_dir)) + self.assertEqual(mocked_run_fuzz_target.call_count, 2) + + class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin, unittest.TestCase): """Integration tests for build_fuzzers with an ASAN build.""" @@ -116,72 +324,54 @@ class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin, # Set the first return value to True, then the second to False to # emulate a bug existing in the current PR but not on the downloaded # OSS-Fuzz build. - with mock.patch.object(fuzz_target.FuzzTarget, - 'is_reproducible', - side_effect=[True, False]): - config = create_config(fuzz_seconds=10, - workspace=TEST_FILES_PATH, - project_name=EXAMPLE_PROJECT) - run_success, bug_found = run_fuzzers.run_fuzzers(config) - build_dir = os.path.join(TEST_FILES_PATH, 'out', 'oss_fuzz_latest') - self.assertTrue(os.path.exists(build_dir)) - self.assertNotEqual(0, len(os.listdir(build_dir))) - self.assertTrue(run_success) - self.assertTrue(bug_found) + with mock.patch('fuzz_target.FuzzTarget.is_reproducible', + side_effect=[True, False]): + with tempfile.TemporaryDirectory() as tmp_dir: + workspace = os.path.join(tmp_dir, 'workspace') + shutil.copytree(TEST_FILES_PATH, workspace) + config = _create_config(fuzz_seconds=FUZZ_SECONDS, + workspace=workspace, + project_name=EXAMPLE_PROJECT) + run_success, bug_found = run_fuzzers.run_fuzzers(config) + self.assertTrue(run_success) + self.assertTrue(bug_found) + build_dir = os.path.join(workspace, 'out', 'oss_fuzz_latest') + self.assertNotEqual(0, len(os.listdir(build_dir))) @unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 'INTEGRATION_TESTS=1 not set') def test_old_bug_found(self): """Tests run_fuzzers with a bug found in OSS-Fuzz before.""" - config = create_config(fuzz_seconds=10, - workspace=TEST_FILES_PATH, - project_name=EXAMPLE_PROJECT) - with mock.patch.object(fuzz_target.FuzzTarget, - 'is_reproducible', - side_effect=[True, True]): - config = create_config(fuzz_seconds=10, - workspace=TEST_FILES_PATH, - project_name=EXAMPLE_PROJECT) - run_success, bug_found = run_fuzzers.run_fuzzers(config) - build_dir = os.path.join(TEST_FILES_PATH, 'out', 'oss_fuzz_latest') - self.assertTrue(os.path.exists(build_dir)) - self.assertNotEqual(0, len(os.listdir(build_dir))) - self.assertTrue(run_success) - self.assertFalse(bug_found) + config = _create_config(fuzz_seconds=FUZZ_SECONDS, + workspace=TEST_FILES_PATH, + project_name=EXAMPLE_PROJECT) + with mock.patch('fuzz_target.FuzzTarget.is_reproducible', + side_effect=[True, True]): + with tempfile.TemporaryDirectory() as tmp_dir: + workspace = os.path.join(tmp_dir, 'workspace') + shutil.copytree(TEST_FILES_PATH, workspace) + config = _create_config(fuzz_seconds=FUZZ_SECONDS, + workspace=TEST_FILES_PATH, + project_name=EXAMPLE_PROJECT) + run_success, bug_found = run_fuzzers.run_fuzzers(config) + build_dir = os.path.join(TEST_FILES_PATH, 'out', 'oss_fuzz_latest') + self.assertTrue(os.path.exists(build_dir)) + self.assertNotEqual(0, len(os.listdir(build_dir))) + self.assertTrue(run_success) + self.assertFalse(bug_found) def test_invalid_build(self): """Tests run_fuzzers with an invalid ASAN build.""" with tempfile.TemporaryDirectory() as tmp_dir: out_path = os.path.join(tmp_dir, 'out') os.mkdir(out_path) - config = create_config(fuzz_seconds=10, - workspace=tmp_dir, - project_name=EXAMPLE_PROJECT) + config = _create_config(fuzz_seconds=FUZZ_SECONDS, + workspace=tmp_dir, + project_name=EXAMPLE_PROJECT) run_success, bug_found = run_fuzzers.run_fuzzers(config) self.assertFalse(run_success) self.assertFalse(bug_found) - def test_invalid_fuzz_seconds(self): - """Tests run_fuzzers with an invalid fuzz seconds.""" - with tempfile.TemporaryDirectory() as tmp_dir: - out_path = os.path.join(tmp_dir, 'out') - os.mkdir(out_path) - config = create_config(fuzz_seconds=0, - workspace=tmp_dir, - project_name=EXAMPLE_PROJECT) - run_success, bug_found = run_fuzzers.run_fuzzers(config) - self.assertFalse(run_success) - self.assertFalse(bug_found) - - def test_invalid_out_dir(self): - """Tests run_fuzzers with an invalid out directory.""" - config = create_config(fuzz_seconds=10, - workspace='not/a/valid/path', - project_name=EXAMPLE_PROJECT) - run_success, bug_found = run_fuzzers.run_fuzzers(config) - self.assertFalse(run_success) - self.assertFalse(bug_found) - if __name__ == '__main__': unittest.main() diff --git a/infra/cifuzz/stack_parser.py b/infra/cifuzz/stack_parser.py index ae0a659c5..0077caae9 100644 --- a/infra/cifuzz/stack_parser.py +++ b/infra/cifuzz/stack_parser.py @@ -13,8 +13,6 @@ # limitations under the License. """Module for parsing stacks from fuzz targets.""" -import os - # From clusterfuzz: src/python/crash_analysis/crash_analyzer.py # Used to get the beginning of the stacktrace. STACKTRACE_TOOL_MARKERS = [ @@ -45,12 +43,12 @@ STACKTRACE_END_MARKERS = [ ] -def parse_fuzzer_output(fuzzer_output, out_dir): +def parse_fuzzer_output(fuzzer_output, parsed_output_file_path): """Parses the fuzzer output from a fuzz target binary. Args: fuzzer_output: A fuzz target binary output string to be parsed. - out_dir: The location to store the parsed output files. + parsed_output_file_path: The location to store the parsed output. """ # Get index of key file points. for marker in STACKTRACE_TOOL_MARKERS: @@ -74,6 +72,5 @@ def parse_fuzzer_output(fuzzer_output, out_dir): return # Write sections of fuzzer output to specific files. - summary_file_path = os.path.join(out_dir, 'bug_summary.txt') - with open(summary_file_path, 'ab') as summary_handle: + with open(parsed_output_file_path, 'ab') as summary_handle: summary_handle.write(summary_str) diff --git a/infra/cifuzz/stack_parser_test.py b/infra/cifuzz/stack_parser_test.py index 24fe8bb3b..0d3969bd3 100644 --- a/infra/cifuzz/stack_parser_test.py +++ b/infra/cifuzz/stack_parser_test.py @@ -32,26 +32,31 @@ class ParseOutputTest(unittest.TestCase): def test_parse_valid_output(self): """Checks that the parse fuzzer output can correctly parse output.""" - test_output_path = os.path.join(TEST_FILES_PATH, - 'example_crash_fuzzer_output.txt') - test_summary_path = os.path.join(TEST_FILES_PATH, 'bug_summary_example.txt') + # Read the fuzzer output from disk. + fuzzer_output_path = os.path.join(TEST_FILES_PATH, + 'example_crash_fuzzer_output.txt') + with open(fuzzer_output_path, 'rb') as fuzzer_output_handle: + fuzzer_output = fuzzer_output_handle.read() with tempfile.TemporaryDirectory() as tmp_dir: - with open(test_output_path, 'rb') as test_fuzz_output: - stack_parser.parse_fuzzer_output(test_fuzz_output.read(), tmp_dir) - result_files = ['bug_summary.txt'] - self.assertCountEqual(os.listdir(tmp_dir), result_files) - - # Compare the bug summaries. - with open(os.path.join(tmp_dir, 'bug_summary.txt')) as bug_summary: - detected_summary = bug_summary.read() - with open(test_summary_path) as bug_summary: - real_summary = bug_summary.read() - self.assertEqual(detected_summary, real_summary) + bug_summary_filename = 'bug-summary.txt' + bug_summary_path = os.path.join(tmp_dir, bug_summary_filename) + stack_parser.parse_fuzzer_output(fuzzer_output, bug_summary_path) + self.assertEqual(os.listdir(tmp_dir), [bug_summary_filename]) + with open(bug_summary_path) as bug_summary_handle: + bug_summary = bug_summary_handle.read() + + # Compare the bug to the expected one. + expected_bug_summary_path = os.path.join(TEST_FILES_PATH, + 'bug_summary_example.txt') + with open(expected_bug_summary_path) as expected_bug_summary_handle: + expected_bug_summary = expected_bug_summary_handle.read() + self.assertEqual(expected_bug_summary, bug_summary) def test_parse_invalid_output(self): """Checks that no files are created when an invalid input was given.""" with tempfile.TemporaryDirectory() as tmp_dir: - stack_parser.parse_fuzzer_output(b'not a valid output_string', tmp_dir) + artifact = os.path.join(tmp_dir, 'bug-summary.txt') + stack_parser.parse_fuzzer_output(b'not a valid output_string', artifact) self.assertEqual(len(os.listdir(tmp_dir)), 0) |