diff options
26 files changed, 367 insertions, 342 deletions
diff --git a/llvm_tools/auto_llvm_bisection.py b/llvm_tools/auto_llvm_bisection.py index 7e8fb1dd..dbd8f37e 100755 --- a/llvm_tools/auto_llvm_bisection.py +++ b/llvm_tools/auto_llvm_bisection.py @@ -52,7 +52,8 @@ class BuilderStatus(enum.Enum): builder_status_mapping = { BuilderStatus.PASS.value: update_tryjob_status.TryjobStatus.GOOD.value, BuilderStatus.FAIL.value: update_tryjob_status.TryjobStatus.BAD.value, - BuilderStatus.RUNNING.value: update_tryjob_status.TryjobStatus.PENDING.value + BuilderStatus.RUNNING.value: + update_tryjob_status.TryjobStatus.PENDING.value } diff --git a/llvm_tools/auto_llvm_bisection_unittest.py b/llvm_tools/auto_llvm_bisection_unittest.py index 07c0e715..3f7e821b 100755 --- a/llvm_tools/auto_llvm_bisection_unittest.py +++ b/llvm_tools/auto_llvm_bisection_unittest.py @@ -27,10 +27,9 @@ class AutoLLVMBisectionTest(unittest.TestCase): """Unittests for auto bisection of LLVM.""" @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) - @mock.patch.object( - llvm_bisection, - 'GetCommandLineArgs', - return_value=test_helpers.ArgsOutputTest()) + @mock.patch.object(llvm_bisection, + 'GetCommandLineArgs', + return_value=test_helpers.ArgsOutputTest()) @mock.patch.object(time, 'sleep') @mock.patch.object(traceback, 'print_exc') @mock.patch.object(llvm_bisection, 'main') @@ -62,9 +61,9 @@ class AutoLLVMBisectionTest(unittest.TestCase): ] mock_json_load.return_value = { 'start': - 369410, + 369410, 'end': - 369420, + 369420, 'jobs': [{ 'buildbucket_id': 12345, 'rev': 369411, @@ -93,10 +92,9 @@ class AutoLLVMBisectionTest(unittest.TestCase): @mock.patch.object(traceback, 'print_exc') @mock.patch.object(llvm_bisection, 'main') @mock.patch.object(os.path, 'isfile') - @mock.patch.object( - llvm_bisection, - 'GetCommandLineArgs', - return_value=test_helpers.ArgsOutputTest()) + @mock.patch.object(llvm_bisection, + 'GetCommandLineArgs', + return_value=test_helpers.ArgsOutputTest()) def testFailedToStartBisection(self, mock_get_args, mock_isfile, mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot): @@ -121,10 +119,9 @@ class AutoLLVMBisectionTest(unittest.TestCase): self.assertEqual(mock_sleep.call_count, 2) @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) - @mock.patch.object( - llvm_bisection, - 'GetCommandLineArgs', - return_value=test_helpers.ArgsOutputTest()) + @mock.patch.object(llvm_bisection, + 'GetCommandLineArgs', + return_value=test_helpers.ArgsOutputTest()) @mock.patch.object(time, 'time') @mock.patch.object(time, 'sleep') @mock.patch.object(os.path, 'isfile') @@ -154,9 +151,9 @@ class AutoLLVMBisectionTest(unittest.TestCase): mock_isfile.return_value = True mock_json_load.return_value = { 'start': - 369410, + 369410, 'end': - 369420, + 369420, 'jobs': [{ 'buildbucket_id': 12345, 'rev': 369411, @@ -235,8 +232,8 @@ class AutoLLVMBisectionTest(unittest.TestCase): auto_llvm_bisection.GetBuildResult(chroot_path, buildbucket_id) self.assertEqual( - str(err.exception), - '"cros buildresult" return value is invalid: %s' % invalid_build_status) + str(err.exception), '"cros buildresult" return value is invalid: %s' % + invalid_build_status) mock_chroot_command.assert_called_once_with( [ diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py index c53db179..d9b3d141 100755 --- a/llvm_tools/bisect_clang_crashes.py +++ b/llvm_tools/bisect_clang_crashes.py @@ -64,12 +64,13 @@ def main(argv): ) cur_dir = os.path.dirname(os.path.abspath(__file__)) parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--4c', dest='forcey', required=True, help='Path to a 4c client binary') - parser.add_argument( - '--state_file', - default=os.path.join(cur_dir, 'chromeos-state.json'), - help='The path to the state file.') + parser.add_argument('--4c', + dest='forcey', + required=True, + help='Path to a 4c client binary') + parser.add_argument('--state_file', + default=os.path.join(cur_dir, 'chromeos-state.json'), + help='The path to the state file.') parser.add_argument( '--nocleanup', action='store_false', @@ -81,8 +82,9 @@ def main(argv): os.makedirs(os.path.dirname(state_file), exist_ok=True) temporary_directory = '/tmp/bisect_clang_crashes' os.makedirs(temporary_directory, exist_ok=True) - urls = get_artifacts('gs://chromeos-toolchain-artifacts/clang-crash-diagnoses' - '/**/*clang_crash_diagnoses.tar.xz') + urls = get_artifacts( + 'gs://chromeos-toolchain-artifacts/clang-crash-diagnoses' + '/**/*clang_crash_diagnoses.tar.xz') logging.info('%d crash URLs found', len(urls)) visited = {} diff --git a/llvm_tools/bisect_clang_crashes_unittest.py b/llvm_tools/bisect_clang_crashes_unittest.py index a3dc0c6d..238b674d 100755 --- a/llvm_tools/bisect_clang_crashes_unittest.py +++ b/llvm_tools/bisect_clang_crashes_unittest.py @@ -63,7 +63,6 @@ class Test(unittest.TestCase): @mock.patch.object(glob, 'glob') def test_get_crash_reproducers_no_matching_script(self, mock_file_search, mock_file_check): - def silence_logging(): root = logging.getLogger() filt = self._SilencingFilter() diff --git a/llvm_tools/chroot_unittest.py b/llvm_tools/chroot_unittest.py index 5eec5675..9fb1d0c0 100755 --- a/llvm_tools/chroot_unittest.py +++ b/llvm_tools/chroot_unittest.py @@ -32,9 +32,8 @@ class HelperFunctionsTest(unittest.TestCase): chroot_path = '/test/chroot/path' package_list = ['new-test/package'] - self.assertEqual( - chroot.GetChrootEbuildPaths(chroot_path, package_list), - [package_chroot_path]) + self.assertEqual(chroot.GetChrootEbuildPaths(chroot_path, package_list), + [package_chroot_path]) mock_chroot_command.assert_called_once() @@ -58,8 +57,9 @@ class HelperFunctionsTest(unittest.TestCase): expected_abs_path = '/path/to/chroot/src/package.ebuild' self.assertEqual( - chroot.ConvertChrootPathsToAbsolutePaths( - chroot_path, chroot_file_paths), [expected_abs_path]) + chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, + chroot_file_paths), + [expected_abs_path]) if __name__ == '__main__': diff --git a/llvm_tools/fetch_cros_sdk_rolls.py b/llvm_tools/fetch_cros_sdk_rolls.py index 83d7025a..b8fdf943 100755 --- a/llvm_tools/fetch_cros_sdk_rolls.py +++ b/llvm_tools/fetch_cros_sdk_rolls.py @@ -68,15 +68,19 @@ def load_manifest_versions(manifest: Path) -> Dict[str, str]: def main(): parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '-d', '--debug', action='store_true', help='Emit debugging output') + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-d', + '--debug', + action='store_true', + help='Emit debugging output') parser.add_argument( '-n', '--number', type=int, default=20, - help='Number of recent manifests to fetch info about. 0 means unlimited.') + help='Number of recent manifests to fetch info about. 0 means unlimited.' + ) args = parser.parse_args() is_debug = args.debug diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py index 83b5ae76..f566f6f3 100755 --- a/llvm_tools/get_llvm_hash.py +++ b/llvm_tools/get_llvm_hash.py @@ -91,7 +91,8 @@ def ParseLLVMMajorVersion(cmakelist): Raises: ValueError: The major version cannot be parsed from cmakelist """ - match = re.search(r'\n\s+set\(LLVM_VERSION_MAJOR (?P<major>\d+)\)', cmakelist) + match = re.search(r'\n\s+set\(LLVM_VERSION_MAJOR (?P<major>\d+)\)', + cmakelist) if not match: raise ValueError('Failed to parse CMakeList for llvm major version') return match.group('major') @@ -158,8 +159,8 @@ def CreateTempLLVMRepo(temp_dir): finally: if os.path.isdir(temp_dir): check_output([ - 'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'remove', '-f', - temp_dir + 'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'remove', + '-f', temp_dir ]) @@ -189,11 +190,10 @@ def GetAndUpdateLLVMProjectInLLVMTools(): 'llvm-project-copy') if not os.path.isdir(abs_path_to_llvm_project_dir): - print( - (f'Checking out LLVM to {abs_path_to_llvm_project_dir}\n' - 'so that we can map between commit hashes and revision numbers.\n' - 'This may take a while, but only has to be done once.'), - file=sys.stderr) + print((f'Checking out LLVM to {abs_path_to_llvm_project_dir}\n' + 'so that we can map between commit hashes and revision numbers.\n' + 'This may take a while, but only has to be done once.'), + file=sys.stderr) os.mkdir(abs_path_to_llvm_project_dir) LLVMHash().CloneLLVMRepo(abs_path_to_llvm_project_dir) @@ -240,7 +240,8 @@ def GetGoogle3LLVMVersion(stable): git_hash = check_output(cmd) # Change type to an integer - return GetVersionFrom(GetAndUpdateLLVMProjectInLLVMTools(), git_hash.rstrip()) + return GetVersionFrom(GetAndUpdateLLVMProjectInLLVMTools(), + git_hash.rstrip()) def IsSvnOption(svn_option): diff --git a/llvm_tools/get_llvm_hash_unittest.py b/llvm_tools/get_llvm_hash_unittest.py index 49740f33..b7c9e972 100755 --- a/llvm_tools/get_llvm_hash_unittest.py +++ b/llvm_tools/get_llvm_hash_unittest.py @@ -20,7 +20,6 @@ from get_llvm_hash import LLVMHash def MakeMockPopen(return_code): - def MockPopen(*_args, **_kwargs): result = mock.MagicMock() result.returncode = return_code @@ -61,8 +60,8 @@ class TestGetLLVMHash(unittest.TestCase): def testGetGitHashWorks(self, mock_get_git_hash): mock_get_git_hash.return_value = 'a13testhash2' - self.assertEqual( - get_llvm_hash.GetGitHashFrom('/tmp/tmpTest', 100), 'a13testhash2') + self.assertEqual(get_llvm_hash.GetGitHashFrom('/tmp/tmpTest', 100), + 'a13testhash2') mock_get_git_hash.assert_called_once() @@ -92,14 +91,14 @@ class TestGetLLVMHash(unittest.TestCase): @mock.patch.object(subprocess, 'Popen') def testCheckoutBranch(self, mock_popen): - mock_popen.return_value = mock.MagicMock( - communicate=lambda: (None, None), returncode=0) + mock_popen.return_value = mock.MagicMock(communicate=lambda: (None, None), + returncode=0) get_llvm_hash.CheckoutBranch('fake/src_dir', 'fake_branch') self.assertEqual( mock_popen.call_args_list[0][0], - (['git', '-C', 'fake/src_dir', 'checkout', 'fake_branch'],)) + (['git', '-C', 'fake/src_dir', 'checkout', 'fake_branch'], )) self.assertEqual(mock_popen.call_args_list[1][0], - (['git', '-C', 'fake/src_dir', 'pull'],)) + (['git', '-C', 'fake/src_dir', 'pull'], )) def testParseLLVMMajorVersion(self): cmakelist_42 = ('set(CMAKE_BUILD_WITH_INSTALL_NAME_DIR ON)\n' @@ -117,10 +116,9 @@ class TestGetLLVMHash(unittest.TestCase): @mock.patch.object(get_llvm_hash, 'ParseLLVMMajorVersion') @mock.patch.object(get_llvm_hash, 'CheckCommand') @mock.patch.object(get_llvm_hash, 'CheckoutBranch') - @mock.patch( - 'get_llvm_hash.open', - mock.mock_open(read_data='mock contents'), - create=True) + @mock.patch('get_llvm_hash.open', + mock.mock_open(read_data='mock contents'), + create=True) def testGetLLVMMajorVersion(self, mock_checkout_branch, mock_git_checkout, mock_major_version, mock_llvm_project_path): mock_llvm_project_path.return_value = 'path/to/llvm-project' @@ -132,7 +130,8 @@ class TestGetLLVMHash(unittest.TestCase): mock_major_version.assert_called_with('mock contents') mock_git_checkout.assert_called_once_with( ['git', '-C', 'path/to/llvm-project', 'checkout', '314159265']) - mock_checkout_branch.assert_called_once_with('path/to/llvm-project', 'main') + mock_checkout_branch.assert_called_once_with('path/to/llvm-project', + 'main') if __name__ == '__main__': diff --git a/llvm_tools/git.py b/llvm_tools/git.py index 22c7002a..2fa99de8 100755 --- a/llvm_tools/git.py +++ b/llvm_tools/git.py @@ -131,5 +131,5 @@ def UploadChanges(repo, branch, commit_messages, reviewers=None, cc=None): if not found_url: raise ValueError('Failed to find change list URL.') - return CommitContents( - url=found_url.group(0), cl_number=int(found_url.group(1))) + return CommitContents(url=found_url.group(0), + cl_number=int(found_url.group(1))) diff --git a/llvm_tools/git_llvm_rev.py b/llvm_tools/git_llvm_rev.py index b62b26e2..8ca60dca 100755 --- a/llvm_tools/git_llvm_rev.py +++ b/llvm_tools/git_llvm_rev.py @@ -246,8 +246,8 @@ def translate_prebase_rev_to_sha(llvm_config: LLVMConfig, rev: Rev) -> str: looking_for = f'llvm-svn: {rev.number}' git_command = [ - 'git', 'log', '--grep', f'^{looking_for}$', f'--format=%H%n%B{separator}', - base_llvm_sha + 'git', 'log', '--grep', f'^{looking_for}$', + f'--format=%H%n%B{separator}', base_llvm_sha ] subp = subprocess.Popen( @@ -353,8 +353,8 @@ def main(argv: t.List[str]) -> None: default='origin', help="LLVM upstream's remote name. Defaults to %(default)s.") sha_or_rev = parser.add_mutually_exclusive_group(required=True) - sha_or_rev.add_argument( - '--sha', help='A git SHA (or ref) to convert to a rev') + sha_or_rev.add_argument('--sha', + help='A git SHA (or ref) to convert to a rev') sha_or_rev.add_argument('--rev', help='A rev to convert into a sha') opts = parser.parse_args(argv) diff --git a/llvm_tools/git_llvm_rev_test.py b/llvm_tools/git_llvm_rev_test.py index d05093a8..0a6719c1 100755 --- a/llvm_tools/git_llvm_rev_test.py +++ b/llvm_tools/git_llvm_rev_test.py @@ -14,8 +14,8 @@ from git_llvm_rev import MAIN_BRANCH def get_llvm_config() -> git_llvm_rev.LLVMConfig: - return git_llvm_rev.LLVMConfig( - dir=llvm_project.get_location(), remote='origin') + return git_llvm_rev.LLVMConfig(dir=llvm_project.get_location(), + remote='origin') class Test(unittest.TestCase): @@ -30,8 +30,8 @@ class Test(unittest.TestCase): def test_sha_to_rev_on_base_sha_works(self) -> None: sha = self.rev_to_sha_with_round_trip( - git_llvm_rev.Rev( - branch=MAIN_BRANCH, number=git_llvm_rev.base_llvm_revision)) + git_llvm_rev.Rev(branch=MAIN_BRANCH, + number=git_llvm_rev.base_llvm_revision)) self.assertEqual(sha, git_llvm_rev.base_llvm_sha) def test_sha_to_rev_prior_to_base_rev_works(self) -> None: diff --git a/llvm_tools/git_unittest.py b/llvm_tools/git_unittest.py index 47927716..7c654475 100755 --- a/llvm_tools/git_unittest.py +++ b/llvm_tools/git_unittest.py @@ -32,9 +32,8 @@ class HelperFunctionsTest(unittest.TestCase): with self.assertRaises(ValueError) as err: git.CreateBranch(path_to_repo, branch) - self.assertEqual( - str(err.exception), - 'Invalid directory path provided: %s' % path_to_repo) + self.assertEqual(str(err.exception), + 'Invalid directory path provided: %s' % path_to_repo) mock_isdir.assert_called_once() @@ -59,9 +58,8 @@ class HelperFunctionsTest(unittest.TestCase): with self.assertRaises(ValueError) as err: git.DeleteBranch(path_to_repo, branch) - self.assertEqual( - str(err.exception), - 'Invalid directory path provided: %s' % path_to_repo) + self.assertEqual(str(err.exception), + 'Invalid directory path provided: %s' % path_to_repo) mock_isdir.assert_called_once() @@ -87,8 +85,8 @@ class HelperFunctionsTest(unittest.TestCase): with self.assertRaises(ValueError) as err: git.UploadChanges(path_to_repo, branch, commit_messages) - self.assertEqual( - str(err.exception), 'Invalid path provided: %s' % path_to_repo) + self.assertEqual(str(err.exception), + 'Invalid path provided: %s' % path_to_repo) mock_isdir.assert_called_once() @@ -129,11 +127,10 @@ class HelperFunctionsTest(unittest.TestCase): ] self.assertEqual( mock_commands.call_args_list[1], - mock.call( - expected_cmd, - stderr=subprocess.STDOUT, - cwd=path_to_repo, - encoding='utf-8')) + mock.call(expected_cmd, + stderr=subprocess.STDOUT, + cwd=path_to_repo, + encoding='utf-8')) self.assertEqual( change_list.url, diff --git a/llvm_tools/llvm_bisection_unittest.py b/llvm_tools/llvm_bisection_unittest.py index cc22dfa4..207f4c24 100755 --- a/llvm_tools/llvm_bisection_unittest.py +++ b/llvm_tools/llvm_bisection_unittest.py @@ -128,7 +128,8 @@ class LLVMBisectionTest(unittest.TestCase): mock_get_git_hash.side_effect = revs git_hashes = [ - git_llvm_rev.base_llvm_revision + 3, git_llvm_rev.base_llvm_revision + 5 + git_llvm_rev.base_llvm_revision + 3, + git_llvm_rev.base_llvm_revision + 5 ] self.assertEqual( @@ -160,9 +161,8 @@ class LLVMBisectionTest(unittest.TestCase): last_tested = '/abs/path/to/file_that_does_not_exist.json' - self.assertEqual( - llvm_bisection.LoadStatusFile(last_tested, start, end), - expected_bisect_state) + self.assertEqual(llvm_bisection.LoadStatusFile(last_tested, start, end), + expected_bisect_state) @mock.patch.object(modify_a_tryjob, 'AddTryjob') def testBisectPassed(self, mock_add_tryjob): @@ -173,9 +173,9 @@ class LLVMBisectionTest(unittest.TestCase): # Simulate behavior of `AddTryjob()` when successfully launched a tryjob for # the updated packages. @test_helpers.CallCountsToMockFunctions - def MockAddTryjob(call_count, _packages, _git_hash, _revision, _chroot_path, - _patch_file, _extra_cls, _options, _builder, _verbose, - _svn_revision): + def MockAddTryjob(call_count, _packages, _git_hash, _revision, + _chroot_path, _patch_file, _extra_cls, _options, + _builder, _verbose, _svn_revision): if call_count < 2: return {'rev': revisions_list[call_count], 'status': 'pending'} @@ -208,17 +208,18 @@ class LLVMBisectionTest(unittest.TestCase): # Verify that the status file is updated when an exception happened when # attempting to launch a revision (i.e. progress is not lost). with self.assertRaises(ValueError) as err: - llvm_bisection.Bisect(revisions_list, git_hash_list, bisection_contents, - temp_json_file, packages, args_output.chroot_path, - patch_file, args_output.extra_change_lists, + llvm_bisection.Bisect(revisions_list, git_hash_list, + bisection_contents, temp_json_file, packages, + args_output.chroot_path, patch_file, + args_output.extra_change_lists, args_output.options, args_output.builders, args_output.verbose) expected_bisection_contents = { 'start': - start, + start, 'end': - end, + end, 'jobs': [{ 'rev': revisions_list[0], 'status': 'pending' @@ -240,8 +241,9 @@ class LLVMBisectionTest(unittest.TestCase): self.assertEqual(mock_add_tryjob.call_count, 3) @mock.patch.object(subprocess, 'check_output', return_value=None) - @mock.patch.object( - get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') + @mock.patch.object(get_llvm_hash.LLVMHash, + 'GetLLVMHash', + return_value='a123testhash4') @mock.patch.object(llvm_bisection, 'GetCommitsBetween') @mock.patch.object(llvm_bisection, 'GetRemainingRange') @mock.patch.object(llvm_bisection, 'LoadStatusFile') @@ -331,9 +333,10 @@ class LLVMBisectionTest(unittest.TestCase): with self.assertRaises(ValueError) as err: llvm_bisection.main(args_output) - error_message = (f'The start {start} or the end {end} version provided is ' - f'different than "start" {bisect_state["start"]} or "end" ' - f'{bisect_state["end"]} in the .JSON file') + error_message = ( + f'The start {start} or the end {end} version provided is ' + f'different than "start" {bisect_state["start"]} or "end" ' + f'{bisect_state["end"]} in the .JSON file') self.assertEqual(str(err.exception), error_message) @@ -448,8 +451,9 @@ class LLVMBisectionTest(unittest.TestCase): mock_get_revision_and_hash_list.assert_called_once() @mock.patch.object(subprocess, 'check_output', return_value=None) - @mock.patch.object( - get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') + @mock.patch.object(get_llvm_hash.LLVMHash, + 'GetLLVMHash', + return_value='a123testhash4') @mock.patch.object(llvm_bisection, 'GetCommitsBetween') @mock.patch.object(llvm_bisection, 'GetRemainingRange') @mock.patch.object(llvm_bisection, 'LoadStatusFile') diff --git a/llvm_tools/llvm_patch_management.py b/llvm_tools/llvm_patch_management.py index 90f9a5c0..53ffc3c2 100755 --- a/llvm_tools/llvm_patch_management.py +++ b/llvm_tools/llvm_patch_management.py @@ -36,7 +36,8 @@ def GetCommandLineArgs(): cros_root = os.path.join(cros_root, 'chromiumos') # Create parser and add optional command-line arguments. - parser = argparse.ArgumentParser(description='Patch management for packages.') + parser = argparse.ArgumentParser( + description='Patch management for packages.') # Add argument for a specific chroot path. parser.add_argument( @@ -54,11 +55,10 @@ def GetCommandLineArgs(): help='the packages to manage their patches (default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for the LLVM version to use for patch management. parser.add_argument( @@ -95,8 +95,8 @@ def GetCommandLineArgs(): # Duplicate packages were passed into the command line if len(unique_packages) != len(args_output.packages): - raise ValueError('Duplicate packages were passed in: %s' % ' '.join( - args_output.packages)) + raise ValueError('Duplicate packages were passed in: %s' % + ' '.join(args_output.packages)) args_output.packages = unique_packages @@ -178,7 +178,8 @@ def _MoveSrcTreeHEADToGitHash(src_path, git_hash): move_head_cmd = ['git', '-C', src_path, 'checkout', git_hash] - subprocess_helpers.ExecCommandAndCaptureOutput(move_head_cmd, verbose=verbose) + subprocess_helpers.ExecCommandAndCaptureOutput(move_head_cmd, + verbose=verbose) def UpdatePackagesPatchMetadataFile(chroot_path, svn_version, @@ -230,8 +231,10 @@ def UpdatePackagesPatchMetadataFile(chroot_path, svn_version, patch_manager.CleanSrcTree(src_path) # Get the patch results for the current package. - patches_info = patch_manager.HandlePatches( - svn_version, patch_metadata_path, filesdir_path, src_path, mode) + patches_info = patch_manager.HandlePatches(svn_version, + patch_metadata_path, + filesdir_path, src_path, + mode) package_info[cur_package] = patches_info._asdict() @@ -262,8 +265,8 @@ def main(): # Only 'disable_patches' and 'remove_patches' can potentially modify the patch # metadata file. - if args_output.failure_mode == FailureModes.DISABLE_PATCHES.value or \ - args_output.failure_mode == FailureModes.REMOVE_PATCHES.value: + if (args_output.failure_mode == FailureModes.DISABLE_PATCHES.value + or args_output.failure_mode == FailureModes.REMOVE_PATCHES.value): print('The patch file %s has been modified for the packages:' % args_output.patch_metadata_file) print('\n'.join(args_output.packages)) diff --git a/llvm_tools/llvm_patch_management_unittest.py b/llvm_tools/llvm_patch_management_unittest.py index 968a816a..92dc64a9 100755 --- a/llvm_tools/llvm_patch_management_unittest.py +++ b/llvm_tools/llvm_patch_management_unittest.py @@ -36,8 +36,8 @@ class LlvmPatchManagementTest(unittest.TestCase): with self.assertRaises(ValueError) as err: llvm_patch_management.GetPathToFilesDirectory(chroot_path, package) - self.assertEqual( - str(err.exception), 'Invalid chroot provided: %s' % chroot_path) + self.assertEqual(str(err.exception), + 'Invalid chroot provided: %s' % chroot_path) mock_isdir.assert_called_once() @@ -46,8 +46,9 @@ class LlvmPatchManagementTest(unittest.TestCase): @mock.patch.object(os.path, 'isdir', return_value=True) @mock.patch.object(subprocess_helpers, 'ChrootRunCommand') @mock.patch.object(llvm_patch_management, '_GetRelativePathOfChrootPath') - def testSuccessfullyGetPathToFilesDir( - self, mock_get_relative_path_of_chroot_path, mock_chroot_cmd, mock_isdir): + def testSuccessfullyGetPathToFilesDir(self, + mock_get_relative_path_of_chroot_path, + mock_chroot_cmd, mock_isdir): package_chroot_path = '/mnt/host/source/path/to/llvm/llvm.ebuild' @@ -95,8 +96,8 @@ class LlvmPatchManagementTest(unittest.TestCase): package_rel_path = 'path/to/llvm' self.assertEqual( - llvm_patch_management._GetRelativePathOfChrootPath(package_chroot_path), - package_rel_path) + llvm_patch_management._GetRelativePathOfChrootPath( + package_chroot_path), package_rel_path) # Simulate behavior of 'os.path.isfile()' when the patch metadata file does # not exist. @@ -109,9 +110,8 @@ class LlvmPatchManagementTest(unittest.TestCase): with self.assertRaises(ValueError) as err: llvm_patch_management._CheckPatchMetadataPath(abs_path_to_patch_file) - self.assertEqual( - str(err.exception), - 'Invalid file provided: %s' % abs_path_to_patch_file) + self.assertEqual(str(err.exception), + 'Invalid file provided: %s' % abs_path_to_patch_file) mock_isfile.assert_called_once() @@ -146,8 +146,9 @@ class LlvmPatchManagementTest(unittest.TestCase): # Simulate `GetGitHashFrom()` when successfully retrieved the git hash # of the version passed in. - @mock.patch.object( - get_llvm_hash, 'GetGitHashFrom', return_value='a123testhash1') + @mock.patch.object(get_llvm_hash, + 'GetGitHashFrom', + return_value='a123testhash1') # Simulate `CreateTempLLVMRepo()` when successfully created a work tree from # the LLVM repo copy in `llvm_tools` directory. @mock.patch.object(get_llvm_hash, 'CreateTempLLVMRepo') @@ -160,16 +161,16 @@ class LlvmPatchManagementTest(unittest.TestCase): self, mock_check_patch_metadata_path, mock_get_filesdir_path, mock_move_head_pointer, mock_create_temp_llvm_repo, mock_get_git_hash): - abs_path_to_patch_file = \ - '/some/path/to/chroot/some/path/to/filesdir/PATCHES' + abs_path_to_patch_file = ( + '/some/path/to/chroot/some/path/to/filesdir/PATCHES') # Simulate the behavior of '_CheckPatchMetadataPath()' when the patch # metadata file in $FILESDIR does not exist or does not end in '.json'. def InvalidPatchMetadataFile(patch_metadata_path): self.assertEqual(patch_metadata_path, abs_path_to_patch_file) - raise ValueError( - 'File does not end in ".json": %s' % abs_path_to_patch_file) + raise ValueError('File does not end in ".json": %s' % + abs_path_to_patch_file) # Use the test function to simulate behavior of '_CheckPatchMetadataPath()'. mock_check_patch_metadata_path.side_effect = InvalidPatchMetadataFile @@ -184,8 +185,8 @@ class LlvmPatchManagementTest(unittest.TestCase): # Simulate the behavior of returning the absolute path to a worktree via # `git worktree add`. - mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = \ - temp_work_tree + mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = ( + temp_work_tree) chroot_path = '/some/path/to/chroot' revision = 1000 @@ -219,8 +220,9 @@ class LlvmPatchManagementTest(unittest.TestCase): @mock.patch.object(patch_manager, 'CleanSrcTree') # Simulate `GetGitHashFrom()` when successfully retrieved the git hash # of the version passed in. - @mock.patch.object( - get_llvm_hash, 'GetGitHashFrom', return_value='a123testhash1') + @mock.patch.object(get_llvm_hash, + 'GetGitHashFrom', + return_value='a123testhash1') # Simulate `CreateTempLLVMRepo()` when successfully created a work tree from # the LLVM repo copy in `llvm_tools` directory. @mock.patch.object(get_llvm_hash, 'CreateTempLLVMRepo') @@ -237,8 +239,8 @@ class LlvmPatchManagementTest(unittest.TestCase): abs_path_to_filesdir = '/some/path/to/chroot/some/path/to/filesdir' - abs_path_to_patch_file = \ - '/some/path/to/chroot/some/path/to/filesdir/PATCHES.json' + abs_path_to_patch_file = ( + '/some/path/to/chroot/some/path/to/filesdir/PATCHES.json') # Simulate the behavior of 'GetPathToFilesDirectory()' when successfully # constructed the absolute path to $FILESDIR of a package. @@ -264,8 +266,8 @@ class LlvmPatchManagementTest(unittest.TestCase): # Simulate the behavior of returning the absolute path to a worktree via # `git worktree add`. - mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = \ - temp_work_tree + mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = ( + temp_work_tree) expected_patch_results = { 'applied_patches': ['fixes_something.patch'], diff --git a/llvm_tools/llvm_project.py b/llvm_tools/llvm_project.py index 7937729f..e059ae29 100644 --- a/llvm_tools/llvm_project.py +++ b/llvm_tools/llvm_project.py @@ -27,11 +27,11 @@ def ensure_up_to_date(): checkout = get_location() if not os.path.isdir(checkout): - print( - 'No llvm-project exists locally; syncing it. This takes a while.', - file=sys.stderr) + print('No llvm-project exists locally; syncing it. This takes a while.', + file=sys.stderr) actual_checkout = get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools() - assert checkout == actual_checkout, '%s != %s' % (actual_checkout, checkout) + assert checkout == actual_checkout, '%s != %s' % (actual_checkout, + checkout) commit_timestamp = subprocess.check_output( [ @@ -52,13 +52,13 @@ def ensure_up_to_date(): if time_since_last_commit <= datetime.timedelta(days=2): return - print( - '%d days have elapsed since the last commit to %s; auto-syncing' % - (time_since_last_commit.days, checkout), - file=sys.stderr) + print('%d days have elapsed since the last commit to %s; auto-syncing' % + (time_since_last_commit.days, checkout), + file=sys.stderr) - result = subprocess.run(['git', 'fetch', 'origin'], check=False, cwd=checkout) + result = subprocess.run(['git', 'fetch', 'origin'], + check=False, + cwd=checkout) if result.returncode: - print( - 'Sync failed somehow; hoping that things are fresh enough, then...', - file=sys.stderr) + print('Sync failed somehow; hoping that things are fresh enough, then...', + file=sys.stderr) diff --git a/llvm_tools/modify_a_tryjob_unittest.py b/llvm_tools/modify_a_tryjob_unittest.py index e3c62972..c03a1e18 100755 --- a/llvm_tools/modify_a_tryjob_unittest.py +++ b/llvm_tools/modify_a_tryjob_unittest.py @@ -42,14 +42,17 @@ class ModifyATryjobTest(unittest.TestCase): with self.assertRaises(SystemExit) as err: modify_a_tryjob.PerformTryjobModification( revision_to_modify, modify_a_tryjob.ModifyTryjob.REMOVE, - temp_json_file, args_output.extra_change_lists, args_output.options, - args_output.builders, args_output.chroot_path, args_output.verbose) + temp_json_file, args_output.extra_change_lists, + args_output.options, args_output.builders, args_output.chroot_path, + args_output.verbose) self.assertEqual(str(err.exception), 'No tryjobs in %s' % temp_json_file) # Simulate the behavior of `FindTryjobIndex()` when the index of the tryjob # was not found. - @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) + @mock.patch.object(update_tryjob_status, + 'FindTryjobIndex', + return_value=None) def testNoTryjobIndexFound(self, mock_find_tryjob_index): bisect_test_contents = { 'start': 369410, @@ -78,8 +81,9 @@ class ModifyATryjobTest(unittest.TestCase): with self.assertRaises(ValueError) as err: modify_a_tryjob.PerformTryjobModification( revision_to_modify, modify_a_tryjob.ModifyTryjob.REMOVE, - temp_json_file, args_output.extra_change_lists, args_output.options, - args_output.builders, args_output.chroot_path, args_output.verbose) + temp_json_file, args_output.extra_change_lists, + args_output.options, args_output.builders, args_output.chroot_path, + args_output.verbose) self.assertEqual( str(err.exception), 'Unable to find tryjob for %d in %s' % @@ -139,9 +143,9 @@ class ModifyATryjobTest(unittest.TestCase): bisect_test_contents = { 'start': - 369410, + 369410, 'end': - 369420, + 369420, 'jobs': [{ 'rev': 369411, 'status': 'bad', @@ -185,9 +189,9 @@ class ModifyATryjobTest(unittest.TestCase): expected_file_contents = { 'start': - 369410, + 369410, 'end': - 369420, + 369420, 'jobs': [{ 'rev': 369411, 'status': 'pending', @@ -249,7 +253,9 @@ class ModifyATryjobTest(unittest.TestCase): mock_find_tryjob_index.assert_called_once() # Simulate the behavior of `FindTryjobIndex()` when the tryjob was not found. - @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) + @mock.patch.object(update_tryjob_status, + 'FindTryjobIndex', + return_value=None) def testSuccessfullyDidNotAddTryjobOutsideOfBisectionBounds( self, mock_find_tryjob_index): @@ -282,8 +288,8 @@ class ModifyATryjobTest(unittest.TestCase): args_output.extra_change_lists, args_output.options, args_output.builders, args_output.chroot_path, args_output.verbose) - self.assertEqual( - str(err.exception), 'Failed to add tryjob to %s' % temp_json_file) + self.assertEqual(str(err.exception), + 'Failed to add tryjob to %s' % temp_json_file) mock_find_tryjob_index.assert_called_once() @@ -292,12 +298,13 @@ class ModifyATryjobTest(unittest.TestCase): @mock.patch.object(modify_a_tryjob, 'AddTryjob') # Simulate the behavior of `GetLLVMHashAndVersionFromSVNOption()` when # successfully retrieved the git hash of the revision to launch a tryjob for. - @mock.patch.object( - get_llvm_hash, - 'GetLLVMHashAndVersionFromSVNOption', - return_value=('a123testhash1', 369418)) + @mock.patch.object(get_llvm_hash, + 'GetLLVMHashAndVersionFromSVNOption', + return_value=('a123testhash1', 369418)) # Simulate the behavior of `FindTryjobIndex()` when the tryjob was not found. - @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) + @mock.patch.object(update_tryjob_status, + 'FindTryjobIndex', + return_value=None) def testSuccessfullyAddedTryjob(self, mock_find_tryjob_index, mock_get_llvm_hash, mock_add_tryjob): @@ -391,9 +398,8 @@ class ModifyATryjobTest(unittest.TestCase): args_output.extra_change_lists, args_output.options, args_output.builders, args_output.chroot_path, args_output.verbose) - self.assertEqual( - str(err.exception), - 'Invalid "modify_tryjob" option provided: remove_link') + self.assertEqual(str(err.exception), + 'Invalid "modify_tryjob" option provided: remove_link') mock_find_tryjob_index.assert_called_once() diff --git a/llvm_tools/nightly_revert_checker_test.py b/llvm_tools/nightly_revert_checker_test.py index a8ab4195..f68513af 100755 --- a/llvm_tools/nightly_revert_checker_test.py +++ b/llvm_tools/nightly_revert_checker_test.py @@ -24,7 +24,6 @@ class Test(unittest.TestCase): """Tests for nightly_revert_checker.""" def test_email_rendering_works_for_singular_revert(self): - def prettify_sha(sha: str) -> tiny_render.Piece: return 'pretty_' + sha @@ -38,8 +37,8 @@ class Test(unittest.TestCase): prettify_sha=prettify_sha, get_sha_description=get_sha_description, new_reverts=[ - revert_checker.Revert( - sha='${revert_sha}', reverted_sha='${reverted_sha}') + revert_checker.Revert(sha='${revert_sha}', + reverted_sha='${reverted_sha}') ]) expected_email = nightly_revert_checker._Email( @@ -65,7 +64,6 @@ class Test(unittest.TestCase): self.assertEqual(email, expected_email) def test_email_rendering_works_for_multiple_reverts(self): - def prettify_sha(sha: str) -> tiny_render.Piece: return 'pretty_' + sha @@ -79,13 +77,13 @@ class Test(unittest.TestCase): prettify_sha=prettify_sha, get_sha_description=get_sha_description, new_reverts=[ - revert_checker.Revert( - sha='${revert_sha1}', reverted_sha='${reverted_sha1}'), - revert_checker.Revert( - sha='${revert_sha2}', reverted_sha='${reverted_sha2}'), + revert_checker.Revert(sha='${revert_sha1}', + reverted_sha='${reverted_sha1}'), + revert_checker.Revert(sha='${revert_sha2}', + reverted_sha='${reverted_sha2}'), # Keep this out-of-order to check that we sort based on SHAs - revert_checker.Revert( - sha='${revert_sha0}', reverted_sha='${reverted_sha0}'), + revert_checker.Revert(sha='${revert_sha0}', + reverted_sha='${reverted_sha0}'), ]) expected_email = nightly_revert_checker._Email( @@ -161,13 +159,13 @@ class Test(unittest.TestCase): find_reverts.return_value = [ revert_checker.Revert('12345abcdef', 'fedcba54321') ] - nightly_revert_checker.do_cherrypick( - chroot_path='/path/to/chroot', - llvm_dir='/path/to/llvm', - interesting_shas=[('12345abcdef', 'fedcba54321')], - state={}, - reviewers=['meow@chromium.org'], - cc=['purr@chromium.org']) + nightly_revert_checker.do_cherrypick(chroot_path='/path/to/chroot', + llvm_dir='/path/to/llvm', + interesting_shas=[('12345abcdef', + 'fedcba54321')], + state={}, + reviewers=['meow@chromium.org'], + cc=['purr@chromium.org']) do_cherrypick.assert_called_once() find_reverts.assert_called_once() @@ -181,13 +179,13 @@ class Test(unittest.TestCase): ] do_cherrypick.side_effect = get_upstream_patch.CherrypickError( 'Patch at 12345abcdef already exists in PATCHES.json') - nightly_revert_checker.do_cherrypick( - chroot_path='/path/to/chroot', - llvm_dir='/path/to/llvm', - interesting_shas=[('12345abcdef', 'fedcba54321')], - state={}, - reviewers=['meow@chromium.org'], - cc=['purr@chromium.org']) + nightly_revert_checker.do_cherrypick(chroot_path='/path/to/chroot', + llvm_dir='/path/to/llvm', + interesting_shas=[('12345abcdef', + 'fedcba54321')], + state={}, + reviewers=['meow@chromium.org'], + cc=['purr@chromium.org']) do_cherrypick.assert_called_once() find_reverts.assert_called_once() diff --git a/llvm_tools/patch_manager.py b/llvm_tools/patch_manager.py index eff1ba8d..303b0f39 100755 --- a/llvm_tools/patch_manager.py +++ b/llvm_tools/patch_manager.py @@ -34,12 +34,12 @@ def is_patch_metadata_file(patch_metadata_file): """Valides the argument into 'argparse' is a patch file.""" if not os.path.isfile(patch_metadata_file): - raise ValueError( - 'Invalid patch metadata file provided: %s' % patch_metadata_file) + raise ValueError('Invalid patch metadata file provided: %s' % + patch_metadata_file) if not patch_metadata_file.endswith('.json'): - raise ValueError( - 'Patch metadata file does not end in ".json": %s' % patch_metadata_file) + raise ValueError('Patch metadata file does not end in ".json": %s' % + patch_metadata_file) return patch_metadata_file @@ -61,8 +61,8 @@ def EnsureBisectModeAndSvnVersionAreSpecifiedTogether(failure_mode, if failure_mode != FailureModes.BISECT_PATCHES.value and good_svn_version: raise ValueError('"good_svn_version" is only available for bisection.') - elif failure_mode == FailureModes.BISECT_PATCHES.value and \ - not good_svn_version: + elif (failure_mode == FailureModes.BISECT_PATCHES.value + and not good_svn_version): raise ValueError('A good SVN version is required for bisection (used by' '"git bisect start".') @@ -75,15 +75,15 @@ def GetCommandLineArgs(): # Add argument for the last good SVN version which is required by # `git bisect start` (only valid for bisection mode). - parser.add_argument( - '--good_svn_version', - type=int, - help='INTERNAL USE ONLY... (used for bisection.)') + parser.add_argument('--good_svn_version', + type=int, + help='INTERNAL USE ONLY... (used for bisection.)') # Add argument for the number of patches it iterate. Only used when performing # `git bisect run`. - parser.add_argument( - '--num_patches_to_iterate', type=int, help=argparse.SUPPRESS) + parser.add_argument('--num_patches_to_iterate', + type=int, + help=argparse.SUPPRESS) # Add argument for whether bisection should continue. Only used for # 'bisect_patches.' @@ -127,11 +127,10 @@ def GetCommandLineArgs(): help='the absolute path to the ebuild "files/" directory') # Add argument for the absolute path to the unpacked sources. - parser.add_argument( - '--src_path', - required=True, - type=is_directory, - help='the absolute path to the unpacked LLVM sources') + parser.add_argument('--src_path', + required=True, + type=is_directory, + help='the absolute path to the unpacked LLVM sources') # Add argument for the mode of the patch manager when handling failing # applicable patches. @@ -139,8 +138,8 @@ def GetCommandLineArgs(): '--failure_mode', default=FailureModes.FAIL.value, type=is_valid_failure_mode, - help='the mode of the patch manager when handling failed patches ' \ - '(default: %(default)s)') + help='the mode of the patch manager when handling failed patches ' + '(default: %(default)s)') # Parse the command line. args_output = parser.parse_args() @@ -462,8 +461,8 @@ def HandlePatches(svn_version, for patch_dict_index, cur_patch_dict in enumerate(patch_file_contents): # Used by the internal bisection. All the patches in the interval [0, N] # have been iterated. - if num_patches_to_iterate and \ - (patch_dict_index + 1) > num_patches_to_iterate: + if (num_patches_to_iterate + and (patch_dict_index + 1) > num_patches_to_iterate): break # Get the absolute path to the patch in $FILESDIR. @@ -487,8 +486,8 @@ def HandlePatches(svn_version, else: # Patch is applicable if 'svn_version' >= 'from' && # "svn_version" < "until". - patch_applicable = (svn_version >= patch_metadata[0] and \ - svn_version < patch_metadata[1]) + patch_applicable = (svn_version >= patch_metadata[0] + and svn_version < patch_metadata[1]) if can_modify_patches: # Add to the list only if the mode can potentially modify a patch. @@ -498,8 +497,8 @@ def HandlePatches(svn_version, # file and all patches that are not applicable will be added to the # remove patches list which will not be included in the updated .json # file. - if patch_applicable or svn_version < patch_metadata[0] or \ - mode != FailureModes.REMOVE_PATCHES: + if (patch_applicable or svn_version < patch_metadata[0] + or mode != FailureModes.REMOVE_PATCHES): applicable_patches.append(cur_patch_dict) elif mode == FailureModes.REMOVE_PATCHES: removed_patches.append(path_to_patch) @@ -553,15 +552,17 @@ def HandlePatches(svn_version, CleanSrcTree(src_path) print('\nStarting to bisect patch %s for SVN version %d:\n' % - (os.path.basename(cur_patch_dict['rel_patch_path']), - svn_version)) + (os.path.basename( + cur_patch_dict['rel_patch_path']), svn_version)) # Performs the bisection: calls `git bisect start` and # `git bisect run`, where `git bisect run` is going to call this # script as many times as needed with specific arguments. - bad_svn_version = PerformBisection( - src_path, good_commit, bad_commit, svn_version, - patch_metadata_file, filesdir_path, patch_dict_index + 1) + bad_svn_version = PerformBisection(src_path, good_commit, + bad_commit, svn_version, + patch_metadata_file, + filesdir_path, + patch_dict_index + 1) print('\nSuccessfully bisected patch %s, starts to fail to apply ' 'at %d\n' % (os.path.basename( @@ -605,8 +606,8 @@ def HandlePatches(svn_version, print('\n'.join(applied_patches)) # Throw an exception on the first patch that failed to apply. - raise ValueError( - 'Failed to apply patch: %s' % os.path.basename(path_to_patch)) + raise ValueError('Failed to apply patch: %s' % + os.path.basename(path_to_patch)) elif mode == FailureModes.INTERNAL_BISECTION: # Determine the exit status for `git bisect run` because of the # failed patch in the interval [0, N]. @@ -662,13 +663,12 @@ def HandlePatches(svn_version, 'disabled_patches', 'removed_patches', 'modified_metadata' ]) - patch_info = PatchInfo( - applied_patches=applied_patches, - failed_patches=failed_patches, - non_applicable_patches=non_applicable_patches, - disabled_patches=disabled_patches, - removed_patches=removed_patches, - modified_metadata=modified_metadata) + patch_info = PatchInfo(applied_patches=applied_patches, + failed_patches=failed_patches, + non_applicable_patches=non_applicable_patches, + disabled_patches=disabled_patches, + removed_patches=removed_patches, + modified_metadata=modified_metadata) # Determine post actions after iterating through the patches. if mode == FailureModes.REMOVE_PATCHES: @@ -711,8 +711,8 @@ def PrintPatchResults(patch_info): print('\n'.join(patch_info.non_applicable_patches)) if patch_info.modified_metadata: - print('\nThe patch metadata file %s has been modified' % os.path.basename( - patch_info.modified_metadata)) + print('\nThe patch metadata file %s has been modified' % + os.path.basename(patch_info.modified_metadata)) if patch_info.disabled_patches: print('\nThe following patches were disabled:') @@ -746,11 +746,13 @@ def main(): args_output.svn_version = GetHEADSVNVersion(args_output.src_path) # Get the results of handling the patches of the package. - patch_info = HandlePatches( - args_output.svn_version, args_output.patch_metadata_file, - args_output.filesdir_path, args_output.src_path, - FailureModes(args_output.failure_mode), args_output.good_svn_version, - args_output.num_patches_to_iterate, args_output.continue_bisection) + patch_info = HandlePatches(args_output.svn_version, + args_output.patch_metadata_file, + args_output.filesdir_path, args_output.src_path, + FailureModes(args_output.failure_mode), + args_output.good_svn_version, + args_output.num_patches_to_iterate, + args_output.continue_bisection) PrintPatchResults(patch_info) diff --git a/llvm_tools/subprocess_helpers.py b/llvm_tools/subprocess_helpers.py index 8845112c..2e013780 100644 --- a/llvm_tools/subprocess_helpers.py +++ b/llvm_tools/subprocess_helpers.py @@ -13,8 +13,10 @@ import subprocess def CheckCommand(cmd): """Executes the command using Popen().""" - cmd_obj = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='UTF-8') + cmd_obj = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding='UTF-8') stdout, _ = cmd_obj.communicate() @@ -43,8 +45,9 @@ def ChrootRunCommand(chroot_path, cmd, verbose=False): exec_chroot_cmd = ['cros_sdk', '--'] exec_chroot_cmd.extend(cmd) - return ExecCommandAndCaptureOutput( - exec_chroot_cmd, cwd=chroot_path, verbose=verbose) + return ExecCommandAndCaptureOutput(exec_chroot_cmd, + cwd=chroot_path, + verbose=verbose) def ExecCommandAndCaptureOutput(cmd, cwd=None, verbose=False): diff --git a/llvm_tools/update_chromeos_llvm_hash_unittest.py b/llvm_tools/update_chromeos_llvm_hash_unittest.py index adb20598..2e93eae9 100755 --- a/llvm_tools/update_chromeos_llvm_hash_unittest.py +++ b/llvm_tools/update_chromeos_llvm_hash_unittest.py @@ -11,7 +11,6 @@ from __future__ import print_function import collections import datetime import os -import re import subprocess import unittest import unittest.mock as mock @@ -58,8 +57,8 @@ class UpdateLLVMHashTest(unittest.TestCase): update_chromeos_llvm_hash.UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version) - self.assertEqual( - str(err.exception), 'Invalid ebuild path provided: %s' % ebuild_path) + self.assertEqual(str(err.exception), + 'Invalid ebuild path provided: %s' % ebuild_path) mock_isfile.assert_called_once() @@ -195,8 +194,8 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion') @mock.patch.object(os.path, 'islink', return_value=False) - def testFailedToUprevEbuildToVersionForInvalidSymlink(self, mock_islink, - mock_llvm_version): + def testFailedToUprevEbuildToVersionForInvalidSymlink( + self, mock_islink, mock_llvm_version): symlink_path = '/path/to/chroot/package/package.ebuild' svn_version = 1000 git_hash = 'badf00d' @@ -207,8 +206,8 @@ class UpdateLLVMHashTest(unittest.TestCase): update_chromeos_llvm_hash.UprevEbuildToVersion(symlink_path, svn_version, git_hash) - self.assertEqual( - str(err.exception), 'Invalid symlink provided: %s' % symlink_path) + self.assertEqual(str(err.exception), + 'Invalid symlink provided: %s' % symlink_path) mock_islink.assert_called_once() mock_llvm_version.assert_not_called() @@ -221,8 +220,8 @@ class UpdateLLVMHashTest(unittest.TestCase): with self.assertRaises(ValueError) as err: update_chromeos_llvm_hash.UprevEbuildSymlink(symlink_path) - self.assertEqual( - str(err.exception), 'Invalid symlink provided: %s' % symlink_path) + self.assertEqual(str(err.exception), + 'Invalid symlink provided: %s' % symlink_path) mock_islink.assert_called_once() @@ -418,8 +417,8 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(update_chromeos_llvm_hash, 'GetEbuildPathsFromSymLinkPaths') def testSuccessfullyCreatedPathDictionaryFromPackages( - self, mock_ebuild_paths_from_symlink_paths, mock_chroot_paths_to_symlinks, - mock_get_chroot_paths): + self, mock_ebuild_paths_from_symlink_paths, + mock_chroot_paths_to_symlinks, mock_get_chroot_paths): package_chroot_path = '/mnt/host/source/src/path/to/package-r1.ebuild' @@ -662,12 +661,10 @@ class UpdateLLVMHashTest(unittest.TestCase): # Verify exception is raised when an exception is thrown within # the 'try' block by UprevEbuildSymlink function. with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash.UpdatePackages(packages_to_update, llvm_variant, - git_hash, svn_version, - chroot_path, patch_metadata_file, - failure_modes.FailureModes.FAIL, - git_hash_source, - extra_commit_msg) + update_chromeos_llvm_hash.UpdatePackages( + packages_to_update, llvm_variant, git_hash, svn_version, chroot_path, + patch_metadata_file, failure_modes.FailureModes.FAIL, + git_hash_source, extra_commit_msg) self.assertEqual(str(err.exception), 'Failed to uprev the ebuild.') @@ -698,12 +695,11 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(llvm_patch_management, 'UpdatePackagesPatchMetadataFile') @mock.patch.object(update_chromeos_llvm_hash, 'StagePatchMetadataFileForCommit') - def testSuccessfullyUpdatedPackages(self, mock_stage_patch_file, - mock_update_package_metadata_file, - mock_delete_repo, mock_upload_changes, - mock_uprev_symlink, mock_update_llvm_next, - mock_create_repo, mock_create_path_dict, - mock_llvm_version, mock_mask_contains): + def testSuccessfullyUpdatedPackages( + self, mock_stage_patch_file, mock_update_package_metadata_file, + mock_delete_repo, mock_upload_changes, mock_uprev_symlink, + mock_update_llvm_next, mock_create_repo, mock_create_path_dict, + mock_llvm_version, mock_mask_contains): path_to_package_dir = '/some/path/to/chroot/src/path/to' abs_path_to_package = os.path.join(path_to_package_dir, 'package.ebuild') @@ -848,7 +844,8 @@ class UpdateLLVMHashTest(unittest.TestCase): 'update_chromeos_llvm_hash.open', mock.mock_open(read_data='\n=sys-devel/llvm-1234.0_pre*\n'), create=True) as mock_file: - update_chromeos_llvm_hash.EnsurePackageMaskContains(chroot_path, git_hash) + update_chromeos_llvm_hash.EnsurePackageMaskContains( + chroot_path, git_hash) handle = mock_file() handle.write.assert_not_called() mock_llvm_version.assert_called_once_with(git_hash) @@ -865,11 +862,11 @@ class UpdateLLVMHashTest(unittest.TestCase): chroot_path = 'absolute/path/to/chroot' git_hash = 'badf00d' mock_llvm_version.return_value = '1234' - with mock.patch( - 'update_chromeos_llvm_hash.open', - mock.mock_open(read_data='nothing relevant'), - create=True) as mock_file: - update_chromeos_llvm_hash.EnsurePackageMaskContains(chroot_path, git_hash) + with mock.patch('update_chromeos_llvm_hash.open', + mock.mock_open(read_data='nothing relevant'), + create=True) as mock_file: + update_chromeos_llvm_hash.EnsurePackageMaskContains( + chroot_path, git_hash) handle = mock_file() handle.write.assert_called_once_with('=sys-devel/llvm-1234.0_pre*\n') mock_llvm_version.assert_called_once_with(git_hash) diff --git a/llvm_tools/update_packages_and_run_tests_unittest.py b/llvm_tools/update_packages_and_run_tests_unittest.py index 11f2b7f8..b48f6338 100755 --- a/llvm_tools/update_packages_and_run_tests_unittest.py +++ b/llvm_tools/update_packages_and_run_tests_unittest.py @@ -46,7 +46,8 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): def testMatchedLastTestedFile(self): with test_helpers.CreateTemporaryFile() as last_tested_file: arg_dict = { - 'svn_version': 1234, + 'svn_version': + 1234, 'ebuilds': [ '/path/to/package1-r2.ebuild', '/path/to/package2/package2-r3.ebuild' @@ -107,10 +108,9 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): options, builder), expected_cmd) - @mock.patch.object( - update_packages_and_run_tests, - 'GetCurrentTimeInUTC', - return_value='2019-09-09') + @mock.patch.object(update_packages_and_run_tests, + 'GetCurrentTimeInUTC', + return_value='2019-09-09') @mock.patch.object(update_packages_and_run_tests, 'AddLinksToCL') @mock.patch.object(subprocess, 'check_output') def testSuccessfullySubmittedTryJob(self, mock_cmd, mock_add_links_to_cl, @@ -147,8 +147,9 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): self.assertEqual(tests, expected_tests) - mock_cmd.assert_called_once_with( - expected_cmd, cwd=chroot_path, encoding='utf-8') + mock_cmd.assert_called_once_with(expected_cmd, + cwd=chroot_path, + encoding='utf-8') mock_add_links_to_cl.assert_called_once() @@ -166,7 +167,10 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): bb_id = '1234' create_time = '2020-04-18T00:03:53.978767Z' - mock_cmd.return_value = json.dumps({'id': bb_id, 'createTime': create_time}) + mock_cmd.return_value = json.dumps({ + 'id': bb_id, + 'createTime': create_time + }) chroot_path = '/some/path/to/chroot' cl = 900 @@ -188,8 +192,9 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): self.assertEqual(tests, expected_tests) - mock_cmd.assert_called_once_with( - expected_cmd, cwd=chroot_path, encoding='utf-8') + mock_cmd.assert_called_once_with(expected_cmd, + cwd=chroot_path, + encoding='utf-8') mock_add_links_to_cl.assert_called_once() @@ -304,8 +309,8 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase): update_packages_and_run_tests.GetCQDependString(test_no_changelists)) self.assertEqual( - update_packages_and_run_tests.GetCQDependString(test_single_changelist), - '\nCq-Depend: chromium:1234') + update_packages_and_run_tests.GetCQDependString( + test_single_changelist), '\nCq-Depend: chromium:1234') self.assertEqual( update_packages_and_run_tests.GetCQDependString( @@ -318,7 +323,8 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase): test_invalid_trybot = 'invalid-name' self.assertIsNone( - update_packages_and_run_tests.GetCQIncludeTrybotsString(test_no_trybot)) + update_packages_and_run_tests.GetCQIncludeTrybotsString( + test_no_trybot)) self.assertEqual( update_packages_and_run_tests.GetCQIncludeTrybotsString( diff --git a/llvm_tools/update_tryjob_status.py b/llvm_tools/update_tryjob_status.py index f25fadca..61aa9d1c 100755 --- a/llvm_tools/update_tryjob_status.py +++ b/llvm_tools/update_tryjob_status.py @@ -82,11 +82,10 @@ def GetCommandLineArgs(): # Add argument that determines which revision to search for in the list of # tryjobs. - parser.add_argument( - '--revision', - required=True, - type=int, - help='The revision to set its status.') + parser.add_argument('--revision', + required=True, + type=int, + help='The revision to set its status.') # Add argument for the custom script to execute for the 'custom_script' # option in '--set_status'. @@ -99,14 +98,13 @@ def GetCommandLineArgs(): args_output = parser.parse_args() - if not (os.path.isfile( - args_output.status_file and - not args_output.status_file.endswith('.json'))): + if not (os.path.isfile(args_output.status_file + and not args_output.status_file.endswith('.json'))): raise ValueError('File does not exist or does not ending in ".json" ' ': %s' % args_output.status_file) - if (args_output.set_status == TryjobStatus.CUSTOM_SCRIPT.value and - not args_output.custom_script): + if (args_output.set_status == TryjobStatus.CUSTOM_SCRIPT.value + and not args_output.custom_script): raise ValueError('Please provide the absolute path to the script to ' 'execute.') @@ -169,15 +167,16 @@ def GetCustomScriptResult(custom_script, status_file, tryjob_contents): exec_script_cmd = [custom_script, temp_json_file] # Execute the custom script to get the exit code. - exec_script_cmd_obj = subprocess.Popen( - exec_script_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + exec_script_cmd_obj = subprocess.Popen(exec_script_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) _, stderr = exec_script_cmd_obj.communicate() # Invalid exit code by the custom script. if exec_script_cmd_obj.returncode not in custom_script_exit_value_mapping: # Save the .JSON file to the directory of 'status_file'. - name_of_json_file = os.path.join( - os.path.dirname(status_file), os.path.basename(temp_json_file)) + name_of_json_file = os.path.join(os.path.dirname(status_file), + os.path.basename(temp_json_file)) os.rename(temp_json_file, name_of_json_file) @@ -236,7 +235,8 @@ def UpdateTryjobStatus(revision, set_status, status_file, custom_script): elif set_status == TryjobStatus.BAD: bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.BAD.value elif set_status == TryjobStatus.PENDING: - bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.PENDING.value + bisect_contents['jobs'][tryjob_index][ + 'status'] = TryjobStatus.PENDING.value elif set_status == TryjobStatus.SKIP: bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.SKIP.value elif set_status == TryjobStatus.CUSTOM_SCRIPT: @@ -246,7 +246,10 @@ def UpdateTryjobStatus(revision, set_status, status_file, custom_script): raise ValueError('Invalid "set_status" option provided: %s' % set_status) with open(status_file, 'w') as update_tryjobs: - json.dump(bisect_contents, update_tryjobs, indent=4, separators=(',', ': ')) + json.dump(bisect_contents, + update_tryjobs, + indent=4, + separators=(',', ': ')) def main(): @@ -256,7 +259,8 @@ def main(): args_output = GetCommandLineArgs() - UpdateTryjobStatus(args_output.revision, TryjobStatus(args_output.set_status), + UpdateTryjobStatus(args_output.revision, + TryjobStatus(args_output.set_status), args_output.status_file, args_output.custom_script) diff --git a/llvm_tools/update_tryjob_status_unittest.py b/llvm_tools/update_tryjob_status_unittest.py index c42c6718..bf078f3b 100755 --- a/llvm_tools/update_tryjob_status_unittest.py +++ b/llvm_tools/update_tryjob_status_unittest.py @@ -100,8 +100,8 @@ class UpdateTryjobStatusTest(unittest.TestCase): custom_script_path = '/abs/path/to/script.py' status_file_path = '/abs/path/to/status_file.json' - name_json_file = os.path.join( - os.path.dirname(status_file_path), 'tmpFile.json') + name_json_file = os.path.join(os.path.dirname(status_file_path), + 'tmpFile.json') expected_error_message = ( 'Custom script %s exit code %d did not match ' @@ -142,7 +142,8 @@ class UpdateTryjobStatusTest(unittest.TestCase): # script. # # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. - mock_exec_custom_script.return_value.communicate.return_value = (None, None) + mock_exec_custom_script.return_value.communicate.return_value = (None, + None) mock_exec_custom_script.return_value.returncode = ( CustomScriptStatus.GOOD.value) @@ -193,7 +194,9 @@ class UpdateTryjobStatusTest(unittest.TestCase): # Simulate the behavior of `FindTryjobIndex()` when the tryjob does not exist # in the status file. - @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) + @mock.patch.object(update_tryjob_status, + 'FindTryjobIndex', + return_value=None) def testNotFindTryjobIndexWhenUpdatingTryjobStatus(self, mock_find_tryjob_index): @@ -256,8 +259,8 @@ class UpdateTryjobStatusTest(unittest.TestCase): custom_script = None update_tryjob_status.UpdateTryjobStatus(revision_to_update, - TryjobStatus.GOOD, temp_json_file, - custom_script) + TryjobStatus.GOOD, + temp_json_file, custom_script) # Verify that the tryjob's 'status' has been updated in the status file. with open(temp_json_file) as status_file: @@ -387,10 +390,9 @@ class UpdateTryjobStatusTest(unittest.TestCase): mock_find_tryjob_index.assert_called_once() @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) - @mock.patch.object( - update_tryjob_status, - 'GetCustomScriptResult', - return_value=TryjobStatus.SKIP.value) + @mock.patch.object(update_tryjob_status, + 'GetCustomScriptResult', + return_value=TryjobStatus.SKIP.value) def testUpdatedTryjobStatusToAutoPassedWithCustomScript( self, mock_get_custom_script_result, mock_find_tryjob_index): bisect_test_contents = { @@ -434,8 +436,8 @@ class UpdateTryjobStatusTest(unittest.TestCase): # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the # status file. @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) - def testSetStatusDoesNotExistWhenUpdatingTryjobStatus(self, - mock_find_tryjob_index): + def testSetStatusDoesNotExistWhenUpdatingTryjobStatus( + self, mock_find_tryjob_index): bisect_test_contents = { 'start': 369410, @@ -466,9 +468,8 @@ class UpdateTryjobStatusTest(unittest.TestCase): nonexistent_update_status, temp_json_file, custom_script) - self.assertEqual( - str(err.exception), - 'Invalid "set_status" option provided: revert_status') + self.assertEqual(str(err.exception), + 'Invalid "set_status" option provided: revert_status') mock_find_tryjob_index.assert_called_once() diff --git a/llvm_tools/upload_lexan_crashes_to_forcey.py b/llvm_tools/upload_lexan_crashes_to_forcey.py index 61bf6b7d..5b038f53 100755 --- a/llvm_tools/upload_lexan_crashes_to_forcey.py +++ b/llvm_tools/upload_lexan_crashes_to_forcey.py @@ -50,7 +50,7 @@ def get_available_test_case_urls(year: int, month: int, day: int) -> List[str]: def test_cases_on_or_after(date: datetime.datetime - ) -> Generator[str, None, None]: + ) -> Generator[str, None, None]: """Yields all test-cases submitted on or after the given date.""" for year in get_available_year_numbers(): if year < date.year: @@ -118,8 +118,7 @@ def submit_test_case(gs_url: str, cr_tool: str) -> None: # chromium.clang-ToTiOS-12754-GTXToolKit-2bfcde.tgz) # we'll get `.crash` files. Unclear why, but let's filter them out anyway. repro_files = [ - os.path.join(tempdir, x) - for x in os.listdir(tempdir) + os.path.join(tempdir, x) for x in os.listdir(tempdir) if not x.endswith('.crash') ] assert len(repro_files) == 2, repro_files @@ -133,8 +132,8 @@ def submit_test_case(gs_url: str, cr_tool: str) -> None: # Peephole: lexan got a crash upload with a way old clang. Ignore it. with open(sh_file, encoding='utf-8') as f: if 'Crash reproducer for clang version 9.0.0' in f.read(): - logging.warning('Skipping upload for %s; seems to be with an old clang', - gs_url) + logging.warning( + 'Skipping upload for %s; seems to be with an old clang', gs_url) return subprocess.run( @@ -226,14 +225,16 @@ def main(argv: List[str]): my_dir = os.path.dirname(os.path.abspath(__file__)) parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--state_file', default=os.path.join(my_dir, 'lexan-state.json')) + parser.add_argument('--state_file', + default=os.path.join(my_dir, 'lexan-state.json')) parser.add_argument( '--last_date', help='The earliest date that we care about. All test cases from here ' 'on will be picked up. Format is YYYY-MM-DD.') - parser.add_argument( - '--4c', dest='forcey', required=True, help='Path to a 4c client binary') + parser.add_argument('--4c', + dest='forcey', + required=True, + help='Path to a 4c client binary') opts = parser.parse_args(argv) forcey = opts.forcey diff --git a/llvm_tools/upload_lexan_crashes_to_forcey_test.py b/llvm_tools/upload_lexan_crashes_to_forcey_test.py index 937cbf8e..36a35048 100755 --- a/llvm_tools/upload_lexan_crashes_to_forcey_test.py +++ b/llvm_tools/upload_lexan_crashes_to_forcey_test.py @@ -18,21 +18,19 @@ class Test(unittest.TestCase): """Tests for upload_lexan_crashes_to_forcey.""" def test_date_parsing_functions(self): - self.assertEqual( - datetime.date(2020, 2, 1), - upload_lexan_crashes_to_forcey.from_ymd('2020-02-01')) - - @unittest.mock.patch( - 'upload_lexan_crashes_to_forcey.test_cases_on_or_after', - return_value=( - ( - datetime.date(2020, 1, 1), - ('gs://test-case-1', 'gs://test-case-1.1'), - ), - (datetime.date(2020, 1, 2), ('gs://test-case-2',)), - (datetime.date(2020, 1, 1), ('gs://test-case-3',)), - (datetime.date(2020, 1, 4), ('gs://test-case-4',)), - )) + self.assertEqual(datetime.date(2020, 2, 1), + upload_lexan_crashes_to_forcey.from_ymd('2020-02-01')) + + @unittest.mock.patch('upload_lexan_crashes_to_forcey.test_cases_on_or_after', + return_value=( + ( + datetime.date(2020, 1, 1), + ('gs://test-case-1', 'gs://test-case-1.1'), + ), + (datetime.date(2020, 1, 2), ('gs://test-case-2', )), + (datetime.date(2020, 1, 1), ('gs://test-case-3', )), + (datetime.date(2020, 1, 4), ('gs://test-case-4', )), + )) @unittest.mock.patch('upload_lexan_crashes_to_forcey.submit_test_case') @unittest.mock.patch('upload_lexan_crashes_to_forcey.persist_state') def test_new_test_case_submission_functions(self, persist_state_mock, @@ -132,8 +130,8 @@ class Test(unittest.TestCase): # All we need is an empty file here. pass - with open( - os.path.join(tempdir, 'test_case.sh'), 'w', encoding='utf-8') as f: + with open(os.path.join(tempdir, 'test_case.sh'), 'w', + encoding='utf-8') as f: f.write('# Crash reproducer for clang version 9.0.0 (...)\n') f.write('clang something or other\n') |