diff options
-rwxr-xr-x | llvm_tools/auto_llvm_bisection.py | 9 | ||||
-rwxr-xr-x | llvm_tools/get_llvm_hash.py | 30 | ||||
-rwxr-xr-x | llvm_tools/llvm_bisection.py | 30 | ||||
-rwxr-xr-x | llvm_tools/llvm_patch_management.py | 50 | ||||
-rwxr-xr-x | llvm_tools/modify_a_tryjob.py | 56 | ||||
-rwxr-xr-x | llvm_tools/patch_manager.py | 66 | ||||
-rw-r--r-- | llvm_tools/subprocess_helpers.py | 58 | ||||
-rwxr-xr-x | llvm_tools/update_all_tryjobs_with_auto.py | 7 | ||||
-rwxr-xr-x | llvm_tools/update_chromeos_llvm_next_hash.py | 197 | ||||
-rwxr-xr-x | llvm_tools/update_packages_and_run_tryjobs.py | 76 | ||||
-rwxr-xr-x | llvm_tools/update_tryjob_status.py | 38 |
11 files changed, 318 insertions, 299 deletions
diff --git a/llvm_tools/auto_llvm_bisection.py b/llvm_tools/auto_llvm_bisection.py index 125b504b..ede99e51 100755 --- a/llvm_tools/auto_llvm_bisection.py +++ b/llvm_tools/auto_llvm_bisection.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -6,7 +6,6 @@ """Performs bisection on LLVM based off a .JSON file.""" -from __future__ import division from __future__ import print_function import os @@ -68,8 +67,8 @@ def main(): # Update all tryjobs whose status is 'pending' to the result of `cros # buildresult`. while True: - print('\nAttempting to update all tryjobs whose \'status\' is ' - '\'pending\':') + print('\nAttempting to update all tryjobs whose "status" is ' + '"pending":') print('-' * 40) update_ret = subprocess.call(exec_update_tryjobs) @@ -84,7 +83,7 @@ def main(): delta_time = time.time() - update_start_time if delta_time > POLLING_LIMIT_SECS: - print('Unable to update tryjobs whose status is \'pending\' to ' + print('Unable to update tryjobs whose status is "pending" to ' 'the result of `cros buildresult`.') # Something is wrong with updating the tryjobs's 'status' via diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py index c4a72bba..e08c30f8 100755 --- a/llvm_tools/get_llvm_hash.py +++ b/llvm_tools/get_llvm_hash.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -14,7 +14,10 @@ import re import shutil import subprocess import tempfile + from contextlib import contextmanager +from subprocess_helpers import CheckCommand +from subprocess_helpers import check_output import requests @@ -22,19 +25,6 @@ _LLVM_GIT_URL = ('https://chromium.googlesource.com/external/github.com/llvm' '/llvm-project') -def CheckCommand(cmd): - """Executes the command using Popen().""" - - cmd_obj = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - stdout, _ = cmd_obj.communicate() - - if cmd_obj.returncode: - print(stdout) - raise subprocess.CalledProcessError(cmd_obj.returncode, cmd) - - @contextmanager def CreateTempLLVMRepo(temp_dir): """Adds a LLVM worktree to 'temp_dir'. @@ -70,7 +60,7 @@ def CreateTempLLVMRepo(temp_dir): yield temp_dir finally: if os.path.isdir(temp_dir): - subprocess.check_output([ + check_output([ 'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'remove', '-f', temp_dir ]) @@ -106,7 +96,7 @@ def GetAndUpdateLLVMProjectInLLVMTools(): # `git status` has a '-s'/'--short' option that shortens the output. # With the '-s' option, if no changes were made to the LLVM repo, then the # output (assigned to 'repo_status') would be empty. - repo_status = subprocess.check_output( + repo_status = check_output( ['git', '-C', abs_path_to_llvm_project_dir, 'status', '-s']) if repo_status.rstrip(): @@ -145,7 +135,7 @@ def GetGoogle3LLVMVersion(): cat_cmd = ['cat', path_to_google3_llvm_version] # Get latest version. - g3_version = subprocess.check_output(cat_cmd) + g3_version = check_output(cat_cmd) # Change type to an integer return int(g3_version.rstrip()) @@ -323,7 +313,7 @@ class LLVMHash(object): 'git', '-C', subdir, 'log', '--format=%B', '-n', '1', cur_hash ] - out = subprocess.check_output(find_llvm_cmd) + out = check_output(find_llvm_cmd) commit_svn_version = self.GetSVNVersionFromCommitMessage(out.rstrip()) @@ -356,7 +346,7 @@ class LLVMHash(object): 'llvm-svn: %d' % llvm_version ] - hash_vals = subprocess.check_output(hash_cmd) + hash_vals = check_output(hash_cmd) return self._ParseCommitMessages(llvm_git_dir, hash_vals.rstrip(), llvm_version) @@ -390,7 +380,7 @@ class LLVMHash(object): 'git', 'ls-remote', _LLVM_GIT_URL, path_to_master_branch ] - llvm_tot_git_hash = subprocess.check_output(llvm_tot_git_hash_cmd) + llvm_tot_git_hash = check_output(llvm_tot_git_hash_cmd) return llvm_tot_git_hash.rstrip().split()[0] diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index d9eecce6..04fbdd55 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -6,7 +6,6 @@ """Performs bisection on LLVM based off a .JSON file.""" -from __future__ import division from __future__ import print_function import argparse @@ -20,7 +19,6 @@ from assert_not_in_chroot import VerifyOutsideChroot from get_llvm_hash import CreateTempLLVMRepo from get_llvm_hash import LLVMHash from modify_a_tryjob import AddTryjob -from patch_manager import _ConvertToASCII from update_tryjob_status import FindTryjobIndex from update_tryjob_status import TryjobStatus @@ -118,12 +116,12 @@ def GetCommandLineArgs(): default=cros_root, help='the path to the chroot (default: %(default)s)') - # Add argument for the log level. + # Add argument for whether to display command contents to `stdout`. parser.add_argument( - '--log_level', - default='none', - choices=['none', 'quiet', 'average', 'verbose'], - help='the level for the logs (default: %(default)s)') + '--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') args_output = parser.parse_args() @@ -133,7 +131,7 @@ def GetCommandLineArgs(): if args_output.last_tested and not args_output.last_tested.endswith('.json'): raise ValueError( - 'Filed provided %s does not end in \'.json\'' % args_output.last_tested) + 'Filed provided %s does not end in ".json"' % args_output.last_tested) return args_output @@ -144,7 +142,7 @@ def _ValidateStartAndEndAgainstJSONStartAndEnd(start, end, json_start, if start != json_start or end != json_end: raise ValueError('The start %d or the end %d version provided is ' - 'different than \'start\' %d or \'end\' %d in the .JSON ' + 'different than "start" %d or "end" %d in the .JSON ' 'file' % (start, end, json_start, json_end)) @@ -179,7 +177,7 @@ def GetStartAndEndRevision(start, end, tryjobs): # Verify that each tryjob has a value for the 'status' key. for cur_tryjob_dict in tryjobs: if not cur_tryjob_dict.get('status', None): - raise ValueError('\'status\' is missing or has no value, please ' + raise ValueError('"status" is missing or has no value, please ' 'go to %s and update it' % cur_tryjob_dict['link']) all_bad_revisions = [end] @@ -333,19 +331,19 @@ def CheckForExistingTryjobsInRevisionsToLaunch(revisions, jobs): for rev in revisions: if FindTryjobIndex(rev, jobs) is not None: - raise ValueError('Revision %d exists already in \'jobs\'' % rev) + raise ValueError('Revision %d exists already in "jobs"' % rev) def UpdateBisection(revisions, git_hashes, bisect_contents, last_tested, update_packages, chroot_path, patch_metadata_file, - extra_change_lists, options, builder, log_level): + extra_change_lists, options, builder, verbose): """Adds tryjobs and updates the status file with the new tryjobs.""" try: for svn_revision, git_hash in zip(revisions, git_hashes): tryjob_dict = AddTryjob(update_packages, git_hash, svn_revision, chroot_path, patch_metadata_file, - extra_change_lists, options, builder, log_level, + extra_change_lists, options, builder, verbose, svn_revision) bisect_contents['jobs'].append(tryjob_dict) @@ -394,7 +392,7 @@ def main(args_output): try: with open(args_output.last_tested) as f: - bisect_contents = _ConvertToASCII(json.load(f)) + bisect_contents = json.load(f) except IOError as err: if err.errno != errno.ENOENT: raise @@ -450,7 +448,7 @@ def main(args_output): args_output.last_tested, update_packages, args_output.chroot_path, patch_metadata_file, args_output.extra_change_lists, args_output.options, - args_output.builder, args_output.log_level) + args_output.builder, args_output.verbose) if __name__ == '__main__': diff --git a/llvm_tools/llvm_patch_management.py b/llvm_tools/llvm_patch_management.py index 90f82d30..522d4e34 100755 --- a/llvm_tools/llvm_patch_management.py +++ b/llvm_tools/llvm_patch_management.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,19 +8,21 @@ from __future__ import print_function -from pipes import quote import argparse import os import patch_manager from assert_not_in_chroot import VerifyOutsideChroot -from cros_utils import command_executer from failure_modes import FailureModes from get_llvm_hash import CreateTempLLVMRepo from get_llvm_hash import GetGoogle3LLVMVersion from get_llvm_hash import LLVMHash +from subprocess_helpers import ChrootRunCommand +from subprocess_helpers import ExecCommandAndCaptureOutput -ce = command_executer.GetCommandExecuter() +# If set to `True`, then the contents of `stdout` after executing a command will +# be displayed to the terminal. +verbose = False def GetCommandLineArgs(): @@ -52,12 +54,12 @@ def GetCommandLineArgs(): default=['sys-devel/llvm'], help='the packages to manage their patches (default: %(default)s)') - # Add argument for the log level. + # Add argument for whether to display command contents to `stdout`. parser.add_argument( - '--log_level', - default='none', - choices=['none', 'quiet', 'average', 'verbose'], - help='the level for the logs (default: %(default)s)') + '--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for the LLVM version to use for patch management. parser.add_argument( @@ -86,8 +88,9 @@ def GetCommandLineArgs(): # Parse the command line. args_output = parser.parse_args() - # Set the log level for the command executer. - ce.SetLogLevel(log_level=args_output.log_level) + global verbose + + verbose = args_output.verbose unique_packages = list(set(args_output.packages)) @@ -119,15 +122,8 @@ def GetPathToFilesDirectory(chroot_path, package): raise ValueError('Invalid chroot provided: %s' % chroot_path) # Get the absolute chroot path to the ebuild. - ret, chroot_ebuild_path, err = ce.ChrootRunCommandWOutput( - chromeos_root=chroot_path, - command='equery w %s' % package, - print_to_console=ce.GetLogLevel() == 'verbose') - - if ret: # Failed to get the absolute chroot path to package's ebuild. - raise ValueError( - 'Failed to get the absolute chroot path of the package %s: %s' % - (package, err)) + chroot_ebuild_path = ChrootRunCommand( + chroot_path, ['equery', 'w', package], verbose=verbose) # Get the absolute chroot path to $FILESDIR's parent directory. filesdir_parent_path = os.path.dirname(chroot_ebuild_path.strip()) @@ -175,20 +171,15 @@ def _CheckPatchMetadataPath(patch_metadata_path): raise ValueError('Invalid file provided: %s' % patch_metadata_path) if not patch_metadata_path.endswith('.json'): - raise ValueError('File does not end in \'.json\': %s' % patch_metadata_path) + raise ValueError('File does not end in ".json": %s' % patch_metadata_path) def _MoveSrcTreeHEADToGitHash(src_path, git_hash): """Moves HEAD to 'git_hash'.""" - move_head_cmd = 'git -C %s checkout %s' % (quote(src_path), git_hash) + move_head_cmd = ['git', '-C', src_path, 'checkout', git_hash] - ret, _, err = ce.RunCommandWOutput( - move_head_cmd, print_to_console=ce.GetLogLevel() == 'verbose') - - if ret: # Failed to checkout to 'git_hash'. - raise ValueError('Failed to moved HEAD in %s to %s: %s' % (quote(src_path), - git_hash, err)) + ExecCommandAndCaptureOutput(move_head_cmd, verbose=verbose) def UpdatePackagesPatchMetadataFile(chroot_path, svn_version, @@ -275,6 +266,9 @@ def main(): print('The patch file %s has been modified for the packages:' % args_output.patch_metadata_file) print('\n'.join(args_output.packages)) + else: + print('Applicable patches in %s applied successfully.' % + args_output.patch_metadata_file) if __name__ == '__main__': diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py index f50f2bdc..20ba3541 100755 --- a/llvm_tools/modify_a_tryjob.py +++ b/llvm_tools/modify_a_tryjob.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -17,7 +17,6 @@ import sys from assert_not_in_chroot import VerifyOutsideChroot from failure_modes import FailureModes from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption -from patch_manager import _ConvertToASCII from update_packages_and_run_tryjobs import RunTryJobs from update_tryjob_status import FindTryjobIndex from update_tryjob_status import TryjobStatus @@ -89,12 +88,12 @@ def GetCommandLineArgs(): default=cros_root, help='the path to the chroot (default: %(default)s)') - # Add argument for the log level. + # Add argument for whether to display command contents to `stdout`. parser.add_argument( - '--log_level', - default='none', - choices=['none', 'quiet', 'average', 'verbose'], - help='the level for the logs (default: %(default)s)') + '--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') args_output = parser.parse_args() @@ -130,7 +129,7 @@ def GetCLAfterUpdatingPackages(packages, git_hash, svn_version, chroot_path, def CreateNewTryjobEntryForBisection(cl, extra_cls, options, builder, - chroot_path, log_level, cl_url, revision): + chroot_path, verbose, cl_url, revision): """Submits a tryjob and adds additional information.""" # Get the tryjob results after submitting the tryjob. @@ -145,7 +144,7 @@ def CreateNewTryjobEntryForBisection(cl, extra_cls, options, builder, # } # ] tryjob_results = RunTryJobs(cl, extra_cls, options, [builder], chroot_path, - log_level) + verbose) print('\nTryjob:') print(tryjob_results[0]) @@ -159,24 +158,24 @@ def CreateNewTryjobEntryForBisection(cl, extra_cls, options, builder, def AddTryjob(packages, git_hash, revision, chroot_path, patch_metadata_file, - extra_cls, options, builder, log_level, svn_option): + extra_cls, options, builder, verbose, svn_option): """Submits a tryjob.""" - update_chromeos_llvm_next_hash.ce.SetLogLevel(log_level=log_level) + update_chromeos_llvm_next_hash.verbose = verbose change_list = GetCLAfterUpdatingPackages(packages, git_hash, revision, chroot_path, patch_metadata_file, svn_option) tryjob_dict = CreateNewTryjobEntryForBisection( - change_list.cl_number, extra_cls, options, builder, chroot_path, - log_level, change_list.url, revision) + change_list.cl_number, extra_cls, options, builder, chroot_path, verbose, + change_list.url, revision) return tryjob_dict def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, - options, builder, chroot_path, log_level): + options, builder, chroot_path, verbose): """Removes, relaunches, or adds a tryjob. Args: @@ -189,7 +188,7 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, builder: The builder to use for 'cros tryjob'. chroot_path: The absolute path to the chroot (used by 'cros tryjob' when relaunching a tryjob). - log_level: The level to use for the logs. + verbose: Determines whether to print the contents of a command to `stdout`. """ # Format of 'bisect_contents': @@ -204,7 +203,7 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, # ] # } with open(status_file) as tryjobs: - bisect_contents = _ConvertToASCII(json.load(tryjobs)) + bisect_contents = json.load(tryjobs) if not bisect_contents['jobs'] and modify_tryjob != ModifyTryjob.ADD: sys.exit('No tryjobs in %s' % status_file) @@ -219,14 +218,23 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, # Determine the action to take based off of 'modify_tryjob'. if modify_tryjob == ModifyTryjob.REMOVE: del bisect_contents['jobs'][tryjob_index] + + print('Successfully deleted the tryjob of revision %d' % revision) elif modify_tryjob == ModifyTryjob.RELAUNCH: - RunTryJobs(bisect_contents['jobs'][tryjob_index]['cl'], - bisect_contents['jobs'][tryjob_index]['extra_cls'], - bisect_contents['jobs'][tryjob_index]['options'], - bisect_contents['jobs'][tryjob_index]['builder'], chroot_path, - log_level) + # Need to update the tryjob link and buildbucket ID. + tryjob_results = RunTryJobs( + bisect_contents['jobs'][tryjob_index]['cl'], + bisect_contents['jobs'][tryjob_index]['extra_cls'], + bisect_contents['jobs'][tryjob_index]['options'], + bisect_contents['jobs'][tryjob_index]['builder'], chroot_path, verbose) bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.PENDING.value + bisect_contents['jobs'][tryjob_index]['link'] = tryjob_results[0]['link'] + bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[0][ + 'buildbucket_id'] + + print('Successfully relaunched the tryjob for revision %d and updated ' + 'the tryjob link to %s' % (revision, tryjob_results[0]['link'])) elif modify_tryjob == ModifyTryjob.ADD: # Tryjob exists already. if tryjob_index is not None: @@ -247,9 +255,11 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, tryjob_dict = AddTryjob(update_packages, git_hash, revision, chroot_path, patch_metadata_file, extra_cls, options, builder, - log_level, revision) + verbose, revision) bisect_contents['jobs'].append(tryjob_dict) + + print('Successfully added tryjob of revision %d' % revision) else: raise ValueError('Failed to add tryjob to %s' % status_file) else: @@ -271,7 +281,7 @@ def main(): args_output.revision, ModifyTryjob( args_output.modify_tryjob), args_output.status_file, args_output.extra_change_lists, args_output.options, args_output.builder, - args_output.chroot_path, args_output.log_level) + args_output.chroot_path, args_output.verbose) if __name__ == '__main__': diff --git a/llvm_tools/patch_manager.py b/llvm_tools/patch_manager.py index 07c88e6a..806b944f 100755 --- a/llvm_tools/patch_manager.py +++ b/llvm_tools/patch_manager.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -17,6 +17,8 @@ import sys from collections import namedtuple from failure_modes import FailureModes from get_llvm_hash import LLVMHash +from subprocess_helpers import check_call +from subprocess_helpers import check_output def is_directory(dir_path): @@ -36,8 +38,8 @@ def is_patch_metadata_file(patch_metadata_file): 'Invalid patch metadata file provided: %s' % patch_metadata_file) if not patch_metadata_file.endswith('.json'): - raise ValueError('Patch metadata file does not end in \'.json\': %s' % - patch_metadata_file) + raise ValueError( + 'Patch metadata file does not end in ".json": %s' % patch_metadata_file) return patch_metadata_file @@ -58,11 +60,11 @@ def EnsureBisectModeAndSvnVersionAreSpecifiedTogether(failure_mode, """Validates that 'good_svn_version' is passed in only for bisection.""" if failure_mode != FailureModes.BISECT_PATCHES.value and good_svn_version: - raise ValueError('\'good_svn_version\' is only available for bisection.') + raise ValueError('"good_svn_version" is only available for bisection.') elif failure_mode == FailureModes.BISECT_PATCHES.value and \ not good_svn_version: raise ValueError('A good SVN version is required for bisection (used by' - '\'git bisect start\'.') + '"git bisect start".') def GetCommandLineArgs(): @@ -91,7 +93,7 @@ def GetCommandLineArgs(): default=False, help='Determines whether bisection should continue after successfully ' 'bisecting a patch (default: %(default)s) - only used for ' - '\'bisect_patches\'') + '"bisect_patches"') # Add argument for the LLVM version to use for patch management. parser.add_argument( @@ -106,7 +108,7 @@ def GetCommandLineArgs(): '--patch_metadata_file', required=True, type=is_patch_metadata_file, - help='the absolute path to the .json file in \'$FILESDIR/\' of the ' + help='the absolute path to the .json file in "$FILESDIR/" of the ' 'package which has all the patches and their metadata if applicable') # Add argument for the absolute path to the ebuild's $FILESDIR path. @@ -115,7 +117,7 @@ def GetCommandLineArgs(): '--filesdir_path', required=True, type=is_directory, - help='the absolute path to the ebuild \'files/\' directory') + help='the absolute path to the ebuild "files/" directory') # Add argument for the absolute path to the unpacked sources. parser.add_argument( @@ -147,7 +149,7 @@ def GetHEADSVNVersion(src_path): get_head_cmd = ['git', '-C', src_path, 'log', '-1', '--pretty=%B'] - head_commit_message = subprocess.check_output(get_head_cmd) + head_commit_message = check_output(get_head_cmd) head_svn_version = LLVMHash().GetSVNVersionFromCommitMessage( head_commit_message) @@ -161,8 +163,8 @@ def VerifyHEADIsTheSameAsSVNVersion(src_path, svn_version): head_svn_version = GetHEADSVNVersion(src_path) if head_svn_version != svn_version: - raise ValueError('HEAD\'s SVN version %d does not match \'svn_version\'' - ' %d, please move HEAD to \'svn_version\'s\' git hash.' % + raise ValueError('HEAD\'s SVN version %d does not match "svn_version"' + ' %d, please move HEAD to "svn_version"s\' git hash.' % (head_svn_version, svn_version)) @@ -242,7 +244,7 @@ def ApplyPatch(src_path, patch_path): ] try: - subprocess.check_output(test_patch_cmd) + check_output(test_patch_cmd) # If the mode is 'continue', then catching the exception makes sure that # the program does not exit on the first failed applicable patch. @@ -251,7 +253,7 @@ def ApplyPatch(src_path, patch_path): return False # Test run succeeded on the patch. - subprocess.check_output(apply_patch_cmd) + check_output(apply_patch_cmd) return True @@ -269,27 +271,12 @@ def UpdatePatchMetadataFile(patch_metadata_file, patches): """ if not patch_metadata_file.endswith('.json'): - raise ValueError('File does not end in \'.json\': %s' % patch_metadata_file) + raise ValueError('File does not end in ".json": %s' % patch_metadata_file) with open(patch_metadata_file, 'w') as patch_file: json.dump(patches, patch_file, indent=4, separators=(',', ': ')) -def _ConvertToASCII(obj): - """Convert an object loaded from JSON to ASCII; JSON gives us unicode.""" - - # Using something like `object_hook` is insufficient, since it only fires on - # actual JSON objects. `encoding` fails, too, since the default decoder always - # uses unicode() to decode strings. - if isinstance(obj, unicode): - return str(obj) - if isinstance(obj, dict): - return {_ConvertToASCII(k): _ConvertToASCII(v) for k, v in obj.iteritems()} - if isinstance(obj, list): - return [_ConvertToASCII(v) for v in obj] - return obj - - def GetCommitHashesForBisection(src_path, good_svn_version, bad_svn_version): """Gets the good and bad commit hashes required by `git bisect start`.""" @@ -312,7 +299,7 @@ def PerformBisection(src_path, good_commit, bad_commit, svn_version, 'git', '-C', src_path, 'bisect', 'start', bad_commit, good_commit ] - subprocess.check_output(bisect_start_cmd) + check_output(bisect_start_cmd) bisect_run_cmd = [ 'git', '-C', src_path, 'bisect', 'run', @@ -323,7 +310,7 @@ def PerformBisection(src_path, good_commit, bad_commit, svn_version, '%d' % num_patches ] - subprocess.check_call(bisect_run_cmd) + check_call(bisect_run_cmd) # Successfully bisected the patch, so retrieve the SVN version from the # commit message. @@ -331,12 +318,11 @@ def PerformBisection(src_path, good_commit, bad_commit, svn_version, 'git', '-C', src_path, 'show', 'refs/bisect/bad' ] - bad_commit_message = subprocess.check_output( - get_bad_commit_from_bisect_run_cmd) + bad_commit_message = check_output(get_bad_commit_from_bisect_run_cmd) end_bisection_cmd = ['git', '-C', src_path, 'bisect', 'reset'] - subprocess.check_output(end_bisection_cmd) + check_output(end_bisection_cmd) # `git bisect run` returns the bad commit hash and the commit message. bad_version = LLVMHash().GetSVNVersionFromCommitMessage( @@ -350,11 +336,11 @@ def CleanSrcTree(src_path): reset_src_tree_cmd = ['git', '-C', src_path, 'reset', 'HEAD', '--hard'] - subprocess.check_output(reset_src_tree_cmd) + check_output(reset_src_tree_cmd) clean_src_tree_cmd = ['git', '-C', src_path, 'clean', '-fd'] - subprocess.check_output(clean_src_tree_cmd) + check_output(clean_src_tree_cmd) def SaveSrcTreeState(src_path): @@ -362,7 +348,7 @@ def SaveSrcTreeState(src_path): save_src_tree_cmd = ['git', '-C', src_path, 'stash', '-a'] - subprocess.check_output(save_src_tree_cmd) + check_output(save_src_tree_cmd) def RestoreSrcTreeState(src_path, bad_commit_hash): @@ -370,11 +356,11 @@ def RestoreSrcTreeState(src_path, bad_commit_hash): checkout_cmd = ['git', '-C', src_path, 'checkout', bad_commit_hash] - subprocess.check_output(checkout_cmd) + check_output(checkout_cmd) get_changes_cmd = ['git', '-C', src_path, 'stash', 'pop'] - subprocess.check_output(get_changes_cmd) + check_output(get_changes_cmd) def HandlePatches(svn_version, @@ -454,7 +440,7 @@ def HandlePatches(svn_version, failed_patches = [] with open(patch_metadata_file) as patch_file: - patch_file_contents = _ConvertToASCII(json.load(patch_file)) + patch_file_contents = json.load(patch_file) if mode == FailureModes.BISECT_PATCHES: # A good and bad commit are required by `git bisect start`. diff --git a/llvm_tools/subprocess_helpers.py b/llvm_tools/subprocess_helpers.py new file mode 100644 index 00000000..8845112c --- /dev/null +++ b/llvm_tools/subprocess_helpers.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Helpers/wrappers for the subprocess module for migration to python3.""" + +from __future__ import print_function + +import subprocess + + +def CheckCommand(cmd): + """Executes the command using Popen().""" + + cmd_obj = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='UTF-8') + + stdout, _ = cmd_obj.communicate() + + if cmd_obj.returncode: + print(stdout) + raise subprocess.CalledProcessError(cmd_obj.returncode, cmd) + + +def check_output(cmd, cwd=None): + """Wrapper for pre-python3 subprocess.check_output().""" + + return subprocess.check_output(cmd, encoding='UTF-8', cwd=cwd) + + +def check_call(cmd, cwd=None): + """Wrapper for pre-python3 subprocess.check_call().""" + + subprocess.check_call(cmd, encoding='UTF-8', cwd=cwd) + + +# FIXME: CTRL+C does not work when executing a command inside the chroot via +# `cros_sdk`. +def ChrootRunCommand(chroot_path, cmd, verbose=False): + """Runs the command inside the chroot.""" + + exec_chroot_cmd = ['cros_sdk', '--'] + exec_chroot_cmd.extend(cmd) + + return ExecCommandAndCaptureOutput( + exec_chroot_cmd, cwd=chroot_path, verbose=verbose) + + +def ExecCommandAndCaptureOutput(cmd, cwd=None, verbose=False): + """Executes the command and prints to stdout if possible.""" + + out = check_output(cmd, cwd=cwd).rstrip() + + if verbose and out: + print(out) + + return out diff --git a/llvm_tools/update_all_tryjobs_with_auto.py b/llvm_tools/update_all_tryjobs_with_auto.py index 4a670c34..511bfffa 100755 --- a/llvm_tools/update_all_tryjobs_with_auto.py +++ b/llvm_tools/update_all_tryjobs_with_auto.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -13,7 +13,6 @@ import json import os from assert_not_in_chroot import VerifyOutsideChroot -from patch_manager import _ConvertToASCII from update_tryjob_status import GetAutoResult from update_tryjob_status import TryjobStatus @@ -51,7 +50,7 @@ def GetCommandLineArgs(): if not os.path.isfile(args_output.last_tested) or \ not args_output.last_tested.endswith('.json'): - raise ValueError('File does not exist or does not ending in \'.json\' ' + raise ValueError('File does not exist or does not ending in ".json" ' ': %s' % args_output.last_tested) return args_output @@ -65,7 +64,7 @@ def main(): args_output = GetCommandLineArgs() with open(args_output.last_tested) as tryjobs: - bisect_contents = _ConvertToASCII(json.load(tryjobs)) + bisect_contents = json.load(tryjobs) for tryjob in bisect_contents['jobs']: if tryjob['status'] == TryjobStatus.PENDING.value: diff --git a/llvm_tools/update_chromeos_llvm_next_hash.py b/llvm_tools/update_chromeos_llvm_next_hash.py index aa63b76c..cf92b489 100755 --- a/llvm_tools/update_chromeos_llvm_next_hash.py +++ b/llvm_tools/update_chromeos_llvm_next_hash.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -13,19 +13,22 @@ for review. from __future__ import print_function from collections import namedtuple -from pipes import quote import argparse -import llvm_patch_management import os import re +import subprocess from assert_not_in_chroot import VerifyOutsideChroot -from cros_utils import command_executer from failure_modes import FailureModes from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption from get_llvm_hash import is_svn_option +from subprocess_helpers import ChrootRunCommand +from subprocess_helpers import ExecCommandAndCaptureOutput +import llvm_patch_management -ce = command_executer.GetCommandExecuter() +# If set to `True`, then the contents of `stdout` after executing a command will +# be displayed to the terminal. +verbose = False CommitContents = namedtuple('CommitContents', ['url', 'cl_number']) @@ -63,12 +66,12 @@ def GetCommandLineArgs(): help='the ebuilds to update their hash for llvm-next ' \ '(default: %(default)s)') - # Add argument for the log level. + # Add argument for whether to display command contents to `stdout`. parser.add_argument( - '--log_level', - default='none', - choices=['none', 'quiet', 'average', 'verbose'], - help='the level for the logs (default: %(default)s)') + '--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for the LLVM version to use. parser.add_argument( @@ -100,8 +103,9 @@ def GetCommandLineArgs(): # Parse the command line. args_output = parser.parse_args() - # Set the log level for the command executer. - ce.SetLogLevel(log_level=args_output.log_level) + global verbose + + verbose = args_output.verbose return args_output @@ -127,17 +131,9 @@ def GetChrootBuildPaths(chromeos_root, package_list): # Find the chroot path for each package's ebuild. for cur_package in sorted(set(package_list)): # Cmd to find the chroot path for the package. - equery_cmd = 'equery w %s' % cur_package - - # Find the chroot path for the package. - ret, chroot_path, err = ce.ChrootRunCommandWOutput( - chromeos_root=chromeos_root, - command=equery_cmd, - print_to_console=ce.GetLogLevel() == 'verbose') + equery_cmd = ['equery', 'w', cur_package] - if ret: # failed to get the chroot path - raise ValueError('Failed to get chroot path for the package (%s): %s' % - (cur_package, err)) + chroot_path = ChrootRunCommand(chromeos_root, equery_cmd, verbose=verbose) chroot_paths.append(chroot_path.strip()) @@ -269,12 +265,9 @@ def UpdateBuildLLVMNextHash(ebuild_path, llvm_hash, llvm_version): parent_dir = os.path.dirname(ebuild_path) # Stage the changes. - ret, _, err = ce.RunCommandWOutput( - 'git -C %s add %s' % (parent_dir, ebuild_path), - print_to_console=ce.GetLogLevel() == 'verbose') + stage_changes_cmd = ['git', '-C', parent_dir, 'add', ebuild_path] - if ret: # failed to stage the changes - raise ValueError('Failed to stage the ebuild for commit: %s' % err) + ExecCommandAndCaptureOutput(stage_changes_cmd, verbose=verbose) def ReplaceLLVMNextHash(ebuild_lines, is_updated, llvm_regex, llvm_hash, @@ -333,12 +326,11 @@ def UprevEbuild(symlink): path_to_symlink_dir = os.path.dirname(symlink) # Stage the new symlink for commit. - ret, _, err = ce.RunCommandWOutput( - 'git -C %s mv %s %s' % (path_to_symlink_dir, symlink, new_symlink), - print_to_console=ce.GetLogLevel() == 'verbose') + stage_symlink_cmd = [ + 'git', '-C', path_to_symlink_dir, 'mv', symlink, new_symlink + ] - if ret: # failed to stage the symlink for commit - raise ValueError('Failed to stage the symlink for commit: %s' % err) + ExecCommandAndCaptureOutput(stage_symlink_cmd, verbose=verbose) def _CreateRepo(path_to_repo_dir, llvm_hash): @@ -355,18 +347,21 @@ def _CreateRepo(path_to_repo_dir, llvm_hash): if not os.path.isdir(path_to_repo_dir): raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir) - create_repo_cmd = ' && '.join([ - 'cd %s' % path_to_repo_dir, - 'git reset HEAD --hard', - 'repo start llvm-next-update-%s' % llvm_hash, - ]) + reset_changes_cmd = [ + 'git', + '-C', + path_to_repo_dir, + 'reset', + 'HEAD', + '--hard', + ] - ret, _, err = ce.RunCommandWOutput( - create_repo_cmd, print_to_console=ce.GetLogLevel() == 'verbose') + ExecCommandAndCaptureOutput(reset_changes_cmd, verbose=verbose) - if ret: # failed to create a repo for the changes - raise ValueError('Failed to create the repo (llvm-next-update-%s): %s' % - (llvm_hash, err)) + create_repo_cmd = ['repo', 'start', 'llvm-next-update-%s' % llvm_hash] + + ExecCommandAndCaptureOutput( + create_repo_cmd, cwd=path_to_repo_dir, verbose=verbose) def _DeleteRepo(path_to_repo_dir, llvm_hash): @@ -383,18 +378,22 @@ def _DeleteRepo(path_to_repo_dir, llvm_hash): if not os.path.isdir(path_to_repo_dir): raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir) - delete_repo_cmd = ' && '.join([ - 'cd %s' % path_to_repo_dir, 'git checkout cros/master', - 'git reset HEAD --hard', - 'git branch -D llvm-next-update-%s' % llvm_hash - ]) + checkout_to_master_cmd = [ + 'git', '-C', path_to_repo_dir, 'checkout', 'cros/master' + ] + + ExecCommandAndCaptureOutput(checkout_to_master_cmd, verbose=verbose) + + reset_head_cmd = ['git', '-C', path_to_repo_dir, 'reset', 'HEAD', '--hard'] - ret, _, err = ce.RunCommandWOutput( - delete_repo_cmd, print_to_console=ce.GetLogLevel() == 'verbose') + ExecCommandAndCaptureOutput(reset_head_cmd, verbose=verbose) - if ret: # failed to delete the repo - raise ValueError('Failed to delete the repo (llvm-next-update-%s): %s' % - (llvm_hash, err)) + delete_repo_cmd = [ + 'git', '-C', path_to_repo_dir, 'branch', '-D', + 'llvm-next-update-%s' % llvm_hash + ] + + ExecCommandAndCaptureOutput(delete_repo_cmd, verbose=verbose) def GetGerritRepoUploadContents(repo_upload_contents): @@ -446,26 +445,36 @@ def UploadChanges(path_to_repo_dir, llvm_hash, commit_messages): if not os.path.isdir(path_to_repo_dir): raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir) - commit_cmd = 'cd %s && git commit %s' % (path_to_repo_dir, commit_messages) - - ret, _, err = ce.RunCommandWOutput( - commit_cmd, print_to_console=ce.GetLogLevel() == 'verbose') + commit_cmd = [ + 'git', + 'commit', + ] + commit_cmd.extend(commit_messages) - if ret: # failed to commit the changes - raise ValueError('Failed to create a commit for the changes: %s' % err) + ExecCommandAndCaptureOutput(commit_cmd, cwd=path_to_repo_dir, verbose=verbose) # Upload the changes for review. - upload_change_cmd = 'cd %s && ' \ - 'yes | repo upload --br=llvm-next-update-%s --no-verify' % ( - path_to_repo_dir, llvm_hash) + upload_change_cmd = ( + 'yes | repo upload --br=llvm-next-update-%s --no-verify' % llvm_hash) + + # NOTE: Need `shell=True` in order to pipe `yes` into `repo upload ...`. + # + # The CL URL is sent to 'stderr', so need to redirect 'stderr' to 'stdout'. + upload_changes_obj = subprocess.Popen( + upload_change_cmd, + cwd=path_to_repo_dir, + shell=True, + encoding='UTF-8', + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) - ret, _, err = ce.RunCommandWOutput( - upload_change_cmd, print_to_console=ce.GetLogLevel() == 'verbose') + out, _ = upload_changes_obj.communicate() - if ret: # failed to upload the changes for review - raise ValueError('Failed to upload changes for review: %s' % err) + if upload_changes_obj.returncode: # Failed to upload changes. + print(out) + raise ValueError('Failed to upload changes for review') - return GetGerritRepoUploadContents(err) + return GetGerritRepoUploadContents(out.rstrip()) def CreatePathDictionaryFromPackages(chroot_path, update_packages): @@ -504,13 +513,12 @@ def RemovePatchesFromFilesDir(patches_to_remove): """ for cur_patch in patches_to_remove: - ret, _, err = ce.RunCommandWOutput( - 'git -C %s rm -f %s' % (os.path.dirname(cur_patch), cur_patch), - print_to_console=ce.GetLogLevel() == 'verbose') + remove_patch_cmd = [ + 'git', '-C', + os.path.dirname(cur_patch), 'rm', '-f', cur_patch + ] - if ret: # Failed to remove the patch in $FILESDIR. - raise ValueError( - 'Failed to remove patch %s: %s' % (os.path.basename(cur_patch), err)) + ExecCommandAndCaptureOutput(remove_patch_cmd, verbose=verbose) def StagePatchMetadataFileForCommit(patch_metadata_file_path): @@ -529,15 +537,12 @@ def StagePatchMetadataFileForCommit(patch_metadata_file_path): 'Invalid patch metadata file provided: %s' % patch_metadata_file_path) # Cmd to stage the patch metadata file for commit. - stage_patch_file = 'git -C %s add %s' % ( - os.path.dirname(patch_metadata_file_path), patch_metadata_file_path) - - ret, _, err = ce.RunCommandWOutput( - stage_patch_file, print_to_console=ce.GetLogLevel() == 'verbose') + stage_patch_file = [ + 'git', '-C', + os.path.dirname(patch_metadata_file_path), 'add', patch_metadata_file_path + ] - if ret: # Failed to stage the patch metadata file for commit. - raise ValueError('Failed to stage patch metadata file %s for commit: %s' % - (os.path.basename(patch_metadata_file_path), err)) + ExecCommandAndCaptureOutput(stage_patch_file, verbose=verbose) def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): @@ -559,32 +564,29 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): patch_info_dict['removed_patches'] or \ patch_info_dict['modified_metadata']: cur_package_header = 'For the package %s:' % package_name - commit_messages.append('-m %s' % quote(cur_package_header)) + commit_messages.append('-m %s' % cur_package_header) # Add to the commit message that the patch metadata file was modified. if patch_info_dict['modified_metadata']: patch_metadata_path = patch_info_dict['modified_metadata'] - commit_messages.append( - '-m %s' % quote('The patch metadata file %s was ' - 'modified' % os.path.basename(patch_metadata_path))) + commit_messages.append('-m %s' % 'The patch metadata file %s was ' + 'modified' % os.path.basename(patch_metadata_path)) StagePatchMetadataFileForCommit(patch_metadata_path) # Add each disabled patch to the commit message. if patch_info_dict['disabled_patches']: - commit_messages.append( - '-m %s' % quote('The following patches were disabled:')) + commit_messages.append('-m %s' % 'The following patches were disabled:') for patch_path in patch_info_dict['disabled_patches']: - commit_messages.append('-m %s' % quote(os.path.basename(patch_path))) + commit_messages.append('-m %s' % os.path.basename(patch_path)) # Add each removed patch to the commit message. if patch_info_dict['removed_patches']: - commit_messages.append( - '-m %s' % quote('The following patches were removed:')) + commit_messages.append('-m %s' % 'The following patches were removed:') for patch_path in patch_info_dict['removed_patches']: - commit_messages.append('-m %s' % quote(os.path.basename(patch_path))) + commit_messages.append('-m %s' % os.path.basename(patch_path)) RemovePatchesFromFilesDir(patch_info_dict['removed_patches']) @@ -616,11 +618,14 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path, Gerrit commit URL and the second pair is the change list number. """ + # Determines whether to print the result of each executed command. + llvm_patch_management.verbose = verbose + # Construct a dictionary where the key is the absolute path of the symlink to # the package and the value is the absolute path to the ebuild of the package. paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages) - repo_path = os.path.dirname(paths_dict.itervalues().next()) + repo_path = os.path.dirname(next(iter(paths_dict.values()))) _CreateRepo(repo_path, llvm_hash) @@ -634,10 +639,9 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path, else: commit_message_header = 'llvm-next: Update packages to r%d' % llvm_version - commit_messages = ['-m %s' % quote(commit_message_header)] + commit_messages = ['-m %s' % commit_message_header] - commit_messages.append( - '-m %s' % quote('Following packages have been updated:')) + commit_messages.append('-m %s' % 'Following packages have been updated:') # Holds the list of packages that are updating. packages = [] @@ -662,10 +666,9 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path, new_commit_message = '%s/%s' % (parent_dir_name, cur_dir_name) - commit_messages.append('-m %s' % quote(new_commit_message)) + commit_messages.append('-m %s' % new_commit_message) # Handle the patches for each package. - llvm_patch_management.ce.SetLogLevel(log_level=ce.GetLogLevel()) package_info_dict = llvm_patch_management.UpdatePackagesPatchMetadataFile( chroot_path, llvm_version, patch_metadata_file, packages, mode) @@ -673,7 +676,7 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path, commit_messages = StagePackagesPatchResultsForCommit( package_info_dict, commit_messages) - change_list = UploadChanges(repo_path, llvm_hash, ' '.join(commit_messages)) + change_list = UploadChanges(repo_path, llvm_hash, commit_messages) finally: _DeleteRepo(repo_path, llvm_hash) diff --git a/llvm_tools/update_packages_and_run_tryjobs.py b/llvm_tools/update_packages_and_run_tryjobs.py index 487b7f6a..f4fc234f 100755 --- a/llvm_tools/update_packages_and_run_tryjobs.py +++ b/llvm_tools/update_packages_and_run_tryjobs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,19 +8,17 @@ from __future__ import print_function -from pipes import quote import argparse import datetime import json import os -import sys from assert_not_in_chroot import VerifyOutsideChroot -from cros_utils import command_executer -from cros_utils.buildbot_utils import ParseTryjobBuildbucketId from failure_modes import FailureModes from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption from get_llvm_hash import is_svn_option +from subprocess_helpers import ChrootRunCommand +from subprocess_helpers import ExecCommandAndCaptureOutput import update_chromeos_llvm_next_hash @@ -41,7 +39,7 @@ def GetCommandLineArgs(): # Create parser and add optional command-line arguments. parser = argparse.ArgumentParser( description='Runs a tryjob if successfully updated packages\'' - '\'LLVM_NEXT_HASH\'.') + '"LLVM_NEXT_HASH".') # Add argument for the absolute path to the file that contains information on # the previous tested svn version. @@ -86,12 +84,12 @@ def GetCommandLineArgs(): default=cros_root, help='the path to the chroot (default: %(default)s)') - # Add argument for the log level. + # Add argument for whether to display command contents to `stdout`. parser.add_argument( - '--log_level', - default='none', - choices=['none', 'quiet', 'average', 'verbose'], - help='the level for the logs (default: %(default)s)') + '--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for the LLVM version to use. parser.add_argument( @@ -166,11 +164,11 @@ def GetTryJobCommand(change_list, extra_change_lists, options, builder): if options: tryjob_cmd.extend('--%s' % option for option in options) - return ' '.join(tryjob_cmd) + return tryjob_cmd def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, - log_level): + verbose): """Runs a tryjob/tryjobs. Args: @@ -180,7 +178,7 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, options: Any options to be passed into the 'tryjob' command. builders: All the builders to run the 'tryjob' with. chroot_path: The absolute path to the chroot. - log_level: The log level to be used for the command_executer. + verbose: Print command contents to `stdout`. Returns: A list that contains stdout contents of each tryjob, where stdout is @@ -191,8 +189,6 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, ValueError: Failed to submit a tryjob. """ - ce = command_executer.GetCommandExecuter(log_level=log_level) - # Contains the results of each tryjob. The results are retrieved from 'out' # which is stdout of the command executer. tryjob_results = [] @@ -206,23 +202,13 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, tryjob_cmd = GetTryJobCommand(cl_number, extra_change_lists, options, cur_builder) - ret, out, err = ce.ChrootRunCommandWOutput( - chromeos_root=chroot_path, - command=tryjob_cmd, - print_to_console=log_level == 'verbose') - - if ret: # Failed to submit a tryjob. - print(err, file=sys.stderr) - raise ValueError('Failed to submit tryjob.') - - # stderr can be noisy e.g. warnings when entering chroot, so ignore it. - # e.g. cros_sdk:enter_chroot: Gclient cache dir "/tmp/git-cache" is not... + out = ChrootRunCommand(chroot_path, tryjob_cmd, verbose=verbose) tryjob_launch_time = datetime.datetime.utcnow() - buildbucket_id = int(ParseTryjobBuildbucketId(out.rstrip())) + tryjob_contents = json.loads(out) - tryjob_contents = json.loads(out.rstrip()) + buildbucket_id = int(tryjob_contents[0]['buildbucket_id']) new_tryjob = { 'launch_time': str(tryjob_launch_time), @@ -235,27 +221,31 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, tryjob_results.append(new_tryjob) - AddTryjobLinkToCL(tryjob_results, cl_number, chroot_path, log_level) + AddTryjobLinkToCL(tryjob_results, cl_number, chroot_path) return tryjob_results -def AddTryjobLinkToCL(tryjobs, cl, chroot_path, log_level): +def AddTryjobLinkToCL(tryjobs, cl, chroot_path): """Adds the tryjob link(s) to the CL via `gerrit message <CL> <message>`.""" - tryjob_links = ['Started the following tryjobs:'] - tryjob_links.extend(quote(tryjob['link']) for tryjob in tryjobs) + # NOTE: Invoking `cros_sdk` does not make each tryjob link appear on its own + # line, so invoking the `gerrit` command directly instead of using `cros_sdk` + # to do it for us. + # + # FIXME: Need to figure out why `cros_sdk` does not add each tryjob link as a + # newline. + gerrit_abs_path = os.path.join(chroot_path, 'chromite/bin/gerrit') - add_message_cmd = 'gerrit message %d \"%s\"' % (cl, '\n'.join(tryjob_links)) + tryjob_links = ['Started the following tryjobs:'] + tryjob_links.extend(tryjob['link'] for tryjob in tryjobs) - ce = command_executer.GetCommandExecuter(log_level=log_level) - ret, _, err = ce.ChrootRunCommandWOutput( - chromeos_root=chroot_path, - command=add_message_cmd, - print_to_console=log_level == 'verbose') + add_message_cmd = [ + gerrit_abs_path, 'message', + str(cl), '\n'.join(tryjob_links) + ] - if ret: # Failed to add tryjob link(s) to CL. - raise ValueError('Failed to add tryjob links to CL %d: %s' % (cl, err)) + ExecCommandAndCaptureOutput(add_message_cmd) def main(): @@ -289,7 +279,7 @@ def main(): (svn_version, last_svn_version, args_output.last_tested)) return - update_chromeos_llvm_next_hash.ce.SetLogLevel(log_level=args_output.log_level) + update_chromeos_llvm_next_hash.verbose = args_output.verbose change_list = update_chromeos_llvm_next_hash.UpdatePackages( update_packages, git_hash, svn_version, args_output.chroot_path, @@ -302,7 +292,7 @@ def main(): tryjob_results = RunTryJobs(change_list.cl_number, args_output.extra_change_lists, args_output.options, args_output.builders, - args_output.chroot_path, args_output.log_level) + args_output.chroot_path, args_output.verbose) print('Tryjobs:') for tryjob in tryjob_results: diff --git a/llvm_tools/update_tryjob_status.py b/llvm_tools/update_tryjob_status.py index 4648db12..38eab8e5 100755 --- a/llvm_tools/update_tryjob_status.py +++ b/llvm_tools/update_tryjob_status.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -16,8 +16,7 @@ import subprocess import sys from assert_not_in_chroot import VerifyOutsideChroot -from cros_utils import command_executer -from patch_manager import _ConvertToASCII +from subprocess_helpers import ChrootRunCommand from test_helpers import CreateTemporaryJsonFile @@ -97,7 +96,7 @@ def GetCommandLineArgs(): '--set_status', required=True, choices=[tryjob_status.value for tryjob_status in TryjobStatus], - help='Sets the \'status\' field of the tryjob.') + help='Sets the "status" field of the tryjob.') # Add argument that determines which revision to search for in the list of # tryjobs. @@ -118,7 +117,7 @@ def GetCommandLineArgs(): parser.add_argument( '--custom_script', help='The absolute path to the custom script to execute (its exit code ' - 'should be %d for \'good\', %d for \'bad\', or %d for \'skip\')' % + 'should be %d for "good", %d for "bad", or %d for "skip")' % (CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value, CustomScriptStatus.SKIP.value)) @@ -126,7 +125,7 @@ def GetCommandLineArgs(): if not os.path.isfile(args_output.status_file) or \ not args_output.status_file.endswith('.json'): - raise ValueError('File does not exist or does not ending in \'.json\' ' + raise ValueError('File does not exist or does not ending in ".json" ' ': %s' % args_output.status_file) if args_output.set_status == TryjobStatus.CUSTOM_SCRIPT.value and \ @@ -169,19 +168,12 @@ def FindTryjobIndex(revision, tryjobs_list): def GetStatusFromCrosBuildResult(chroot_path, buildbucket_id): """Retrieves the 'status' using 'cros buildresult'.""" - ce = command_executer.GetCommandExecuter(log_level=None) + get_buildbucket_id_cmd = [ + 'cros', 'buildresult', '--buildbucket-id', + str(buildbucket_id), '--report', 'json' + ] - get_buildbucket_id_cmd = ('cros buildresult --buildbucket-id %d ' - '--report json' % buildbucket_id) - - ret, tryjob_json, err = ce.ChrootRunCommandWOutput( - chromeos_root=chroot_path, - command=get_buildbucket_id_cmd, - print_to_console=False) - - if ret: # Failed to get the contents of the tryjob. - raise ValueError('Failed to get JSON contents of the tryjobs buildbucket ' - 'ID %d: %s' % (buildbucket_id, err)) + tryjob_json = ChrootRunCommand(chroot_path, get_buildbucket_id_cmd) tryjob_contents = json.loads(tryjob_json) @@ -197,7 +189,7 @@ def GetAutoResult(chroot_path, buildbucket_id): # The string returned by 'cros buildresult' might not be in the mapping. if build_result not in builder_status_mapping: raise ValueError( - '\'cros buildresult\' return value is invalid: %s' % build_result) + '"cros buildresult" return value is invalid: %s' % build_result) return builder_status_mapping[build_result] @@ -243,8 +235,8 @@ def GetCustomScriptResult(custom_script, status_file, tryjob_contents): raise ValueError( 'Custom script %s exit code %d did not match ' - 'any of the expected exit codes: %d for \'good\', %d ' - 'for \'bad\', or %d for \'skip\'.\nPlease check %s for information ' + 'any of the expected exit codes: %d for "good", %d ' + 'for "bad", or %d for "skip".\nPlease check %s for information ' 'about the tryjob: %s' % (custom_script, exec_script_cmd_obj.returncode, CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value, @@ -281,7 +273,7 @@ def UpdateTryjobStatus(revision, set_status, status_file, chroot_path, # ] # } with open(status_file) as tryjobs: - bisect_contents = _ConvertToASCII(json.load(tryjobs)) + bisect_contents = json.load(tryjobs) if not bisect_contents['jobs']: sys.exit('No tryjobs in %s' % status_file) @@ -309,7 +301,7 @@ def UpdateTryjobStatus(revision, set_status, status_file, chroot_path, bisect_contents['jobs'][tryjob_index]['status'] = GetCustomScriptResult( custom_script, status_file, bisect_contents['jobs'][tryjob_index]) else: - raise ValueError('Invalid \'set_status\' option provided: %s' % set_status) + raise ValueError('Invalid "set_status" option provided: %s' % set_status) with open(status_file, 'w') as update_tryjobs: json.dump(bisect_contents, update_tryjobs, indent=4, separators=(',', ': ')) |