diff options
Diffstat (limited to 'infra/cifuzz/cifuzz_test.py')
| -rw-r--r-- | infra/cifuzz/cifuzz_test.py | 289 |
1 files changed, 133 insertions, 156 deletions
diff --git a/infra/cifuzz/cifuzz_test.py b/infra/cifuzz/cifuzz_test.py index 1de18e8a3..741c41e07 100644 --- a/infra/cifuzz/cifuzz_test.py +++ b/infra/cifuzz/cifuzz_test.py @@ -11,19 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Test the functionality of the cifuzz module's functions: +"""Tests the functionality of the cifuzz module's functions: 1. Building fuzzers. 2. Running fuzzers. """ import json import os -import pickle import shutil import sys import tempfile import unittest from unittest import mock +import parameterized + # pylint: disable=wrong-import-position INFRA_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(INFRA_DIR) @@ -64,10 +65,10 @@ UNDEFINED_FUZZER = 'curl_fuzzer_undefined' class BuildFuzzersIntegrationTest(unittest.TestCase): - """Test build_fuzzers function in the utils module.""" + """Integration tests for build_fuzzers.""" def test_valid_commit(self): - """Test building fuzzers with valid inputs.""" + """Tests building fuzzers with valid inputs.""" with tempfile.TemporaryDirectory() as tmp_dir: out_path = os.path.join(tmp_dir, 'out') os.mkdir(out_path) @@ -81,7 +82,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): os.path.exists(os.path.join(out_path, EXAMPLE_BUILD_FUZZER))) def test_valid_pull_request(self): - """Test building fuzzers with valid pull request.""" + """Tests building fuzzers with valid pull request.""" with tempfile.TemporaryDirectory() as tmp_dir: out_path = os.path.join(tmp_dir, 'out') os.mkdir(out_path) @@ -94,7 +95,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): os.path.exists(os.path.join(out_path, EXAMPLE_BUILD_FUZZER))) def test_invalid_pull_request(self): - """Test building fuzzers with invalid pull request.""" + """Tests building fuzzers with invalid pull request.""" with tempfile.TemporaryDirectory() as tmp_dir: out_path = os.path.join(tmp_dir, 'out') os.mkdir(out_path) @@ -105,7 +106,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): pr_ref='ref-1/merge')) def test_invalid_project_name(self): - """Test building fuzzers with invalid project name.""" + """Tests building fuzzers with invalid project name.""" with tempfile.TemporaryDirectory() as tmp_dir: self.assertFalse( cifuzz.build_fuzzers( @@ -115,7 +116,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523')) def test_invalid_repo_name(self): - """Test building fuzzers with invalid repo name.""" + """Tests building fuzzers with invalid repo name.""" with tempfile.TemporaryDirectory() as tmp_dir: self.assertFalse( cifuzz.build_fuzzers( @@ -125,7 +126,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523')) def test_invalid_commit_sha(self): - """Test building fuzzers with invalid commit SHA.""" + """Tests building fuzzers with invalid commit SHA.""" with tempfile.TemporaryDirectory() as tmp_dir: with self.assertRaises(AssertionError): cifuzz.build_fuzzers(EXAMPLE_PROJECT, @@ -134,7 +135,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): commit_sha='') def test_invalid_workspace(self): - """Test building fuzzers with invalid workspace.""" + """Tests building fuzzers with invalid workspace.""" self.assertFalse( cifuzz.build_fuzzers( EXAMPLE_PROJECT, @@ -144,78 +145,77 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): )) -class RunMemoryFuzzerIntegrationTest(unittest.TestCase): - """Test build_fuzzers function in the cifuzz module.""" +def remove_test_files(out_parent_dir, allowlist): + """Removes test files from |out_parent_dir| that are not in |allowlist|, a + list of files with paths relative to the out directory.""" + out_dir = os.path.join(out_parent_dir, 'out') + allowlist = set(allowlist) + for rel_out_path in os.listdir(out_dir): + if rel_out_path in allowlist: + continue + path_to_remove = os.path.join(out_dir, rel_out_path) + if os.path.isdir(path_to_remove): + shutil.rmtree(path_to_remove) + else: + os.remove(path_to_remove) + + +class RunFuzzerIntegrationTestMixin: # pylint: disable=too-few-public-methods,invalid-name + """Mixin for integration test classes that runbuild_fuzzers on builds of a + specific sanitizer.""" + # These must be defined by children. + FUZZER_DIR = None + FUZZER = None def tearDown(self): - """Remove any existing crashes and test files.""" - out_dir = os.path.join(MEMORY_FUZZER_DIR, 'out') - for out_file in os.listdir(out_dir): - out_path = os.path.join(out_dir, out_file) - #pylint: disable=consider-using-in - if out_file == MEMORY_FUZZER: - continue - if os.path.isdir(out_path): - shutil.rmtree(out_path) - else: - os.remove(out_path) + """Removes any existing crashes and test files.""" + remove_test_files(self.FUZZER_DIR, self.FUZZER) - def test_run_with_memory_sanitizer(self): - """Test run_fuzzers with a valid build.""" + def _test_run_with_sanitizer(self, fuzzer_dir, sanitizer): + """Calls run_fuzzers on fuzzer_dir and |sanitizer| and asserts + the run succeeded and that no bug was found.""" run_success, bug_found = cifuzz.run_fuzzers(10, - MEMORY_FUZZER_DIR, + fuzzer_dir, 'curl', - sanitizer='memory') + sanitizer=sanitizer) self.assertTrue(run_success) self.assertFalse(bug_found) -class RunUndefinedFuzzerIntegrationTest(unittest.TestCase): - """Test build_fuzzers function in the cifuzz module.""" +class RunMemoryFuzzerIntegrationTest(unittest.TestCase, + RunFuzzerIntegrationTestMixin): + """Integration test for build_fuzzers with an MSAN build.""" + FUZZER_DIR = MEMORY_FUZZER_DIR + FUZZER = MEMORY_FUZZER - def tearDown(self): - """Remove any existing crashes and test files.""" - out_dir = os.path.join(UNDEFINED_FUZZER_DIR, 'out') - for out_file in os.listdir(out_dir): - out_path = os.path.join(out_dir, out_file) - #pylint: disable=consider-using-in - if out_file == UNDEFINED_FUZZER: - continue - if os.path.isdir(out_path): - shutil.rmtree(out_path) - else: - os.remove(out_path) + def test_run_with_memory_sanitizer(self): + """Tests run_fuzzers with a valid MSAN build.""" + self._test_run_with_sanitizer(self.FUZZER_DIR, 'memory') + + +class RunUndefinedFuzzerIntegrationTest(unittest.TestCase, + RunFuzzerIntegrationTestMixin): + """Integration test for build_fuzzers with an UBSAN build.""" + FUZZER_DIR = UNDEFINED_FUZZER_DIR + FUZZER = UNDEFINED_FUZZER def test_run_with_undefined_sanitizer(self): - """Test run_fuzzers with a valid build.""" - run_success, bug_found = cifuzz.run_fuzzers(10, - UNDEFINED_FUZZER_DIR, - 'curl', - sanitizer='undefined') - self.assertTrue(run_success) - self.assertFalse(bug_found) + """Tests run_fuzzers with a valid MSAN build.""" + self._test_run_with_sanitizer(self.FUZZER_DIR, 'undefined') class RunAddressFuzzersIntegrationTest(unittest.TestCase): - """Test build_fuzzers function in the cifuzz module.""" + """Integration tests for build_fuzzers with an ASAN build.""" def tearDown(self): - """Remove any existing crashes and test files.""" - out_dir = os.path.join(TEST_FILES_PATH, 'out') + """Removes any existing crashes and test files.""" files_to_keep = [ 'undefined', 'memory', EXAMPLE_CRASH_FUZZER, EXAMPLE_NOCRASH_FUZZER ] - for out_file in os.listdir(out_dir): - out_path = os.path.join(out_dir, out_file) - if out_file in files_to_keep: - continue - if os.path.isdir(out_path): - shutil.rmtree(out_path) - else: - os.remove(out_path) + remove_test_files(TEST_FILES_PATH, files_to_keep) def test_new_bug_found(self): - """Test run_fuzzers with a valid build.""" + """Tests run_fuzzers with a valid ASAN build.""" # Set the first return value to True, then the second to False to # emulate a bug existing in the current PR but not on the downloaded # OSS-Fuzz build. @@ -231,7 +231,7 @@ class RunAddressFuzzersIntegrationTest(unittest.TestCase): self.assertTrue(bug_found) def test_old_bug_found(self): - """Test run_fuzzers with a bug found in OSS-Fuzz before.""" + """Tests run_fuzzers with a bug found in OSS-Fuzz before.""" with mock.patch.object(fuzz_target.FuzzTarget, 'is_reproducible', side_effect=[True, True]): @@ -244,7 +244,7 @@ class RunAddressFuzzersIntegrationTest(unittest.TestCase): self.assertFalse(bug_found) def test_invalid_build(self): - """Test run_fuzzers with an invalid build.""" + """Tests run_fuzzers with an invalid ASAN build.""" with tempfile.TemporaryDirectory() as tmp_dir: out_path = os.path.join(tmp_dir, 'out') os.mkdir(out_path) @@ -269,8 +269,8 @@ class RunAddressFuzzersIntegrationTest(unittest.TestCase): self.assertFalse(bug_found) -class ParseOutputUnitTest(unittest.TestCase): - """Test parse_fuzzer_output function in the cifuzz module.""" +class ParseOutputTest(unittest.TestCase): + """Tests parse_fuzzer_output.""" def test_parse_valid_output(self): """Checks that the parse fuzzer output can correctly parse output.""" @@ -284,9 +284,9 @@ class ParseOutputUnitTest(unittest.TestCase): self.assertCountEqual(os.listdir(tmp_dir), result_files) # Compare the bug summaries. - with open(os.path.join(tmp_dir, 'bug_summary.txt'), 'r') as bug_summary: + with open(os.path.join(tmp_dir, 'bug_summary.txt')) as bug_summary: detected_summary = bug_summary.read() - with open(os.path.join(test_summary_path), 'r') as bug_summary: + with open(test_summary_path) as bug_summary: real_summary = bug_summary.read() self.assertEqual(detected_summary, real_summary) @@ -297,7 +297,7 @@ class ParseOutputUnitTest(unittest.TestCase): self.assertEqual(len(os.listdir(tmp_dir)), 0) -class CheckFuzzerBuildUnitTest(unittest.TestCase): +class CheckFuzzerBuildTest(unittest.TestCase): """Tests the check_fuzzer_build function in the cifuzz module.""" def test_correct_fuzzer_build(self): @@ -318,28 +318,27 @@ class CheckFuzzerBuildUnitTest(unittest.TestCase): def test_allow_broken_fuzz_targets_percentage(self, mocked_docker_run): """Tests that ALLOWED_BROKEN_TARGETS_PERCENTAGE is set when running docker if it is set in the environment.""" - test_fuzzer_dir = os.path.join(TEST_FILES_PATH, 'out') mocked_docker_run.return_value = 0 + test_fuzzer_dir = os.path.join(TEST_FILES_PATH, 'out') cifuzz.check_fuzzer_build(test_fuzzer_dir) self.assertIn('-e ALLOWED_BROKEN_TARGETS_PERCENTAGE=0', ' '.join(mocked_docker_run.call_args[0][0])) -class GetFilesCoveredByTargetUnitTest(unittest.TestCase): - """Test to get the files covered by a fuzz target in the cifuzz module.""" +class GetFilesCoveredByTargetTest(unittest.TestCase): + """Tests get_files_covered_by_target.""" example_cov_json = 'example_curl_cov.json' example_fuzzer_cov_json = 'example_curl_fuzzer_cov.json' example_fuzzer = 'curl_fuzzer' - example_curl_file_list = 'example_curl_file_list' def setUp(self): - with open(os.path.join(TEST_FILES_PATH, self.example_cov_json), - 'r') as file: - self.proj_cov_report_example = json.loads(file.read()) - with open(os.path.join(TEST_FILES_PATH, self.example_fuzzer_cov_json), - 'r') as file: - self.fuzzer_cov_report_example = json.loads(file.read()) + with open(os.path.join(TEST_FILES_PATH, self.example_cov_json) + ) as file_handle: + self.proj_cov_report_example = json.loads(file_handle.read()) + with open(os.path.join(TEST_FILES_PATH, self.example_fuzzer_cov_json) + ) as file_handle: + self.fuzzer_cov_report_example = json.loads(file_handle.read()) def test_valid_target(self): """Tests that covered files can be retrieved from a coverage report.""" @@ -350,13 +349,14 @@ class GetFilesCoveredByTargetUnitTest(unittest.TestCase): file_list = cifuzz.get_files_covered_by_target( self.proj_cov_report_example, self.example_fuzzer, '/src/curl') - with open(os.path.join(TEST_FILES_PATH, 'example_curl_file_list'), - 'rb') as file_handle: - true_files_list = pickle.load(file_handle) + curl_files_list_path = os.path.join( + TEST_FILES_PATH, 'example_curl_file_list.json') + with open(curl_files_list_path) as file_handle: + true_files_list = json.load(file_handle) self.assertCountEqual(file_list, true_files_list) def test_invalid_target(self): - """Test asserts an invalid fuzzer returns None.""" + """Tests passing invalid fuzz target returns None.""" self.assertIsNone( cifuzz.get_files_covered_by_target(self.proj_cov_report_example, 'not-a-fuzzer', '/src/curl')) @@ -365,7 +365,7 @@ class GetFilesCoveredByTargetUnitTest(unittest.TestCase): '/src/curl')) def test_invalid_project_build_dir(self): - """Test asserts an invalid build dir returns None.""" + """Tests passing an invalid build directory returns None.""" self.assertIsNone( cifuzz.get_files_covered_by_target(self.proj_cov_report_example, self.example_fuzzer, '/no/pe')) @@ -374,43 +374,44 @@ class GetFilesCoveredByTargetUnitTest(unittest.TestCase): self.example_fuzzer, '')) -class GetTargetCoverageReporUnitTest(unittest.TestCase): - """Test get_target_coverage_report function in the cifuzz module.""" +class GetTargetCoverageReportTest(unittest.TestCase): + """Tests get_target_coverage_report.""" example_cov_json = 'example_curl_cov.json' example_fuzzer = 'curl_fuzzer' def setUp(self): with open(os.path.join(TEST_FILES_PATH, self.example_cov_json), - 'r') as file: - self.cov_exmp = json.loads(file.read()) + 'r') as file_handle: + self.example_cov = json.loads(file_handle.read()) def test_valid_target(self): - """Test a target's coverage report can be downloaded and parsed.""" + """Tests that a target's coverage report can be downloaded and parsed.""" with mock.patch.object(cifuzz, 'get_json_from_url', return_value='{}') as mock_get_json: - cifuzz.get_target_coverage_report(self.cov_exmp, self.example_fuzzer) + cifuzz.get_target_coverage_report(self.example_cov, self.example_fuzzer) (url,), _ = mock_get_json.call_args self.assertEqual( 'https://storage.googleapis.com/oss-fuzz-coverage/' 'curl/fuzzer_stats/20200226/curl_fuzzer.json', url) def test_invalid_target(self): - """Test an invalid target coverage report will be None.""" + """Tests that passing an invalid target coverage report returns None.""" self.assertIsNone( - cifuzz.get_target_coverage_report(self.cov_exmp, 'not-valid-target')) - self.assertIsNone(cifuzz.get_target_coverage_report(self.cov_exmp, '')) + cifuzz.get_target_coverage_report(self.example_cov, 'not-valid-target')) + self.assertIsNone(cifuzz.get_target_coverage_report(self.example_cov, '')) def test_invalid_project_json(self): - """Test a project json coverage report will be None.""" + """Tests that passing an invalid project json coverage report returns + None.""" self.assertIsNone( cifuzz.get_target_coverage_report('not-a-proj', self.example_fuzzer)) self.assertIsNone(cifuzz.get_target_coverage_report('', self.example_fuzzer)) -class GetLatestCoverageReportUnitTest(unittest.TestCase): - """Test get_latest_cov_report_info function in the cifuzz module.""" +class GetLatestCoverageReportTest(unittest.TestCase): + """Tests get_latest_cov_report_info.""" test_project = 'curl' @@ -418,7 +419,7 @@ class GetLatestCoverageReportUnitTest(unittest.TestCase): """Tests that a project's coverage report can be downloaded and parsed. NOTE: This test relies on the test_project repo's coverage report. - Example was not used because it has no coverage reports. + The "example" project was not used because it has no coverage reports. """ with mock.patch.object(cifuzz, 'get_json_from_url', return_value='{}') as mock_fun: @@ -430,80 +431,56 @@ class GetLatestCoverageReportUnitTest(unittest.TestCase): 'latest_report_info/curl.json', url) def test_get_invalid_project(self): - """Tests a project's coverage report will return None if bad project.""" + """Tests that passing a bad project returns None.""" self.assertIsNone(cifuzz.get_latest_cov_report_info('not-a-proj')) self.assertIsNone(cifuzz.get_latest_cov_report_info('')) -class KeepAffectedFuzzersUnitTest(unittest.TestCase): - """Test the keep_affected_fuzzer method in the CIFuzz module.""" +EXAMPLE_FILE_CHANGED = 'test.txt' - test_fuzzer_1 = os.path.join(TEST_FILES_PATH, 'out', 'example_crash_fuzzer') - test_fuzzer_2 = os.path.join(TEST_FILES_PATH, 'out', 'example_nocrash_fuzzer') - example_file_changed = 'test.txt' - def test_keeping_fuzzer_w_no_coverage(self): - """Tests that a specific fuzzer is kept if it is deemed affected.""" - with tempfile.TemporaryDirectory() as tmp_dir, mock.patch.object( - cifuzz, 'get_latest_cov_report_info', return_value=1): - shutil.copy(self.test_fuzzer_1, tmp_dir) - shutil.copy(self.test_fuzzer_2, tmp_dir) - with mock.patch.object(cifuzz, - 'get_files_covered_by_target', - side_effect=[[self.example_file_changed], None]): - cifuzz.remove_unaffected_fuzzers(EXAMPLE_PROJECT, tmp_dir, - [self.example_file_changed], '') - self.assertEqual(2, len(os.listdir(tmp_dir))) - - def test_keeping_specific_fuzzer(self): - """Tests that a specific fuzzer is kept if it is deemed affected.""" - with tempfile.TemporaryDirectory() as tmp_dir, mock.patch.object( - cifuzz, 'get_latest_cov_report_info', return_value=1): - shutil.copy(self.test_fuzzer_1, tmp_dir) - shutil.copy(self.test_fuzzer_2, tmp_dir) - with mock.patch.object(cifuzz, - 'get_files_covered_by_target', - side_effect=[[self.example_file_changed], - ['not/a/real/file']]): - cifuzz.remove_unaffected_fuzzers(EXAMPLE_PROJECT, tmp_dir, - [self.example_file_changed], '') - self.assertEqual(1, len(os.listdir(tmp_dir))) - - def test_no_fuzzers_kept_fuzzer(self): - """Tests that if there is no affected then all fuzzers are kept.""" - with tempfile.TemporaryDirectory() as tmp_dir, mock.patch.object( - cifuzz, 'get_latest_cov_report_info', return_value=1): - shutil.copy(self.test_fuzzer_1, tmp_dir) - shutil.copy(self.test_fuzzer_2, tmp_dir) +class RemoveUnaffectedFuzzersTest(unittest.TestCase): + """Tests remove_unaffected_fuzzers.""" + + TEST_FUZZER_1 = os.path.join(TEST_FILES_PATH, 'out', 'example_crash_fuzzer') + TEST_FUZZER_2 = os.path.join(TEST_FILES_PATH, 'out', 'example_nocrash_fuzzer') + + # yapf: disable + @parameterized.parameterized.expand([ + # Tests a specific affected fuzzers is kept. + ([[EXAMPLE_FILE_CHANGED], None], 2,), + + # Tests specific affected fuzzer is kept. + ([[EXAMPLE_FILE_CHANGED], ['not/a/real/file']], 1), + + # Tests all fuzzers are kept if none are deemed affected. + ([None, None], 2), + + # Tests that multiple fuzzers are kept if multiple fuzzers are affected. + ([[EXAMPLE_FILE_CHANGED], [EXAMPLE_FILE_CHANGED]], 2), + ]) + # yapf: enable + def test_remove_unaffected_fuzzers(self, side_effect, expected_dir_len): + """Tests that remove_unaffected_fuzzers has the intended effect.""" + with tempfile.TemporaryDirectory() as tmp_dir, mock.patch( + 'cifuzz.get_latest_cov_report_info', return_value=1): with mock.patch.object(cifuzz, - 'get_files_covered_by_target', - side_effect=[None, None]): - cifuzz.remove_unaffected_fuzzers(EXAMPLE_PROJECT, tmp_dir, - [self.example_file_changed], '') - self.assertEqual(2, len(os.listdir(tmp_dir))) - - def test_both_fuzzers_kept_fuzzer(self): - """Tests that if both fuzzers are affected then both fuzzers are kept.""" - with tempfile.TemporaryDirectory() as tmp_dir, mock.patch.object( - cifuzz, 'get_latest_cov_report_info', return_value=1): - shutil.copy(self.test_fuzzer_1, tmp_dir) - shutil.copy(self.test_fuzzer_2, tmp_dir) - with mock.patch.object( - cifuzz, - 'get_files_covered_by_target', - side_effect=[self.example_file_changed, self.example_file_changed]): + 'get_files_covered_by_target') as mocked_get_files: + mocked_get_files.side_effect = side_effect + shutil.copy(self.TEST_FUZZER_1, tmp_dir) + shutil.copy(self.TEST_FUZZER_2, tmp_dir) cifuzz.remove_unaffected_fuzzers(EXAMPLE_PROJECT, tmp_dir, - [self.example_file_changed], '') - self.assertEqual(2, len(os.listdir(tmp_dir))) + [EXAMPLE_FILE_CHANGED], '') + self.assertEqual(expected_dir_len, len(os.listdir(tmp_dir))) @unittest.skip('Test is too long to be run with presubmit.') class BuildSantizerIntegrationTest(unittest.TestCase): - """Integration tests for the build_fuzzers function in the cifuzz module. - Note: This test relies on the curl project being an OSS-Fuzz project.""" + """Integration tests for the build_fuzzers. + Note: This test relies on "curl" being an OSS-Fuzz project.""" def test_valid_project_curl_memory(self): - """Test if sanitizers can be detected from project.yaml""" + """Tests that MSAN can be detected from project.yaml""" with tempfile.TemporaryDirectory() as tmp_dir: self.assertTrue( cifuzz.build_fuzzers('curl', @@ -513,7 +490,7 @@ class BuildSantizerIntegrationTest(unittest.TestCase): sanitizer='memory')) def test_valid_project_curl_undefined(self): - """Test if sanitizers can be detected from project.yaml""" + """Test that UBSAN can be detected from project.yaml""" with tempfile.TemporaryDirectory() as tmp_dir: self.assertTrue( cifuzz.build_fuzzers('curl', |