aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSalud Lemus <saludlemus@google.com>2019-09-04 18:15:35 -0700
committerSalud Lemus <saludlemus@google.com>2019-09-09 20:23:50 +0000
commit6270cccfcd886478a9c1b20acf9e8cd50a673a19 (patch)
treef9b5938b388f0be67333882113a2e66e1a969860
parent2ad170bf822e26b07b4ed69f500be2ca9a5c3df1 (diff)
downloadtoolchain-utils-6270cccfcd886478a9c1b20acf9e8cd50a673a19.tar.gz
LLVM tools: Migrated all scripts to python3
This CL includes changes such as replacements of `\'` with `"` and adding extra debugging output to some scripts. Currently, the scripts are in python2, so migrating them to python3 so they are more maintainable. BUG=None TEST=Ran each script by itself with various input (e.g. different google3, tot, etc.). Change-Id: Ib72b7744c6f7c13711c2db427f6524ff3cbc6205 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1787738 Tested-by: Salud Lemus <saludlemus@google.com> Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
-rwxr-xr-xllvm_tools/auto_llvm_bisection.py9
-rwxr-xr-xllvm_tools/get_llvm_hash.py30
-rwxr-xr-xllvm_tools/llvm_bisection.py30
-rwxr-xr-xllvm_tools/llvm_patch_management.py50
-rwxr-xr-xllvm_tools/modify_a_tryjob.py56
-rwxr-xr-xllvm_tools/patch_manager.py66
-rw-r--r--llvm_tools/subprocess_helpers.py58
-rwxr-xr-xllvm_tools/update_all_tryjobs_with_auto.py7
-rwxr-xr-xllvm_tools/update_chromeos_llvm_next_hash.py197
-rwxr-xr-xllvm_tools/update_packages_and_run_tryjobs.py76
-rwxr-xr-xllvm_tools/update_tryjob_status.py38
11 files changed, 318 insertions, 299 deletions
diff --git a/llvm_tools/auto_llvm_bisection.py b/llvm_tools/auto_llvm_bisection.py
index 125b504b..ede99e51 100755
--- a/llvm_tools/auto_llvm_bisection.py
+++ b/llvm_tools/auto_llvm_bisection.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -6,7 +6,6 @@
"""Performs bisection on LLVM based off a .JSON file."""
-from __future__ import division
from __future__ import print_function
import os
@@ -68,8 +67,8 @@ def main():
# Update all tryjobs whose status is 'pending' to the result of `cros
# buildresult`.
while True:
- print('\nAttempting to update all tryjobs whose \'status\' is '
- '\'pending\':')
+ print('\nAttempting to update all tryjobs whose "status" is '
+ '"pending":')
print('-' * 40)
update_ret = subprocess.call(exec_update_tryjobs)
@@ -84,7 +83,7 @@ def main():
delta_time = time.time() - update_start_time
if delta_time > POLLING_LIMIT_SECS:
- print('Unable to update tryjobs whose status is \'pending\' to '
+ print('Unable to update tryjobs whose status is "pending" to '
'the result of `cros buildresult`.')
# Something is wrong with updating the tryjobs's 'status' via
diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py
index c4a72bba..e08c30f8 100755
--- a/llvm_tools/get_llvm_hash.py
+++ b/llvm_tools/get_llvm_hash.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -14,7 +14,10 @@ import re
import shutil
import subprocess
import tempfile
+
from contextlib import contextmanager
+from subprocess_helpers import CheckCommand
+from subprocess_helpers import check_output
import requests
@@ -22,19 +25,6 @@ _LLVM_GIT_URL = ('https://chromium.googlesource.com/external/github.com/llvm'
'/llvm-project')
-def CheckCommand(cmd):
- """Executes the command using Popen()."""
-
- cmd_obj = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
- stdout, _ = cmd_obj.communicate()
-
- if cmd_obj.returncode:
- print(stdout)
- raise subprocess.CalledProcessError(cmd_obj.returncode, cmd)
-
-
@contextmanager
def CreateTempLLVMRepo(temp_dir):
"""Adds a LLVM worktree to 'temp_dir'.
@@ -70,7 +60,7 @@ def CreateTempLLVMRepo(temp_dir):
yield temp_dir
finally:
if os.path.isdir(temp_dir):
- subprocess.check_output([
+ check_output([
'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'remove', '-f',
temp_dir
])
@@ -106,7 +96,7 @@ def GetAndUpdateLLVMProjectInLLVMTools():
# `git status` has a '-s'/'--short' option that shortens the output.
# With the '-s' option, if no changes were made to the LLVM repo, then the
# output (assigned to 'repo_status') would be empty.
- repo_status = subprocess.check_output(
+ repo_status = check_output(
['git', '-C', abs_path_to_llvm_project_dir, 'status', '-s'])
if repo_status.rstrip():
@@ -145,7 +135,7 @@ def GetGoogle3LLVMVersion():
cat_cmd = ['cat', path_to_google3_llvm_version]
# Get latest version.
- g3_version = subprocess.check_output(cat_cmd)
+ g3_version = check_output(cat_cmd)
# Change type to an integer
return int(g3_version.rstrip())
@@ -323,7 +313,7 @@ class LLVMHash(object):
'git', '-C', subdir, 'log', '--format=%B', '-n', '1', cur_hash
]
- out = subprocess.check_output(find_llvm_cmd)
+ out = check_output(find_llvm_cmd)
commit_svn_version = self.GetSVNVersionFromCommitMessage(out.rstrip())
@@ -356,7 +346,7 @@ class LLVMHash(object):
'llvm-svn: %d' % llvm_version
]
- hash_vals = subprocess.check_output(hash_cmd)
+ hash_vals = check_output(hash_cmd)
return self._ParseCommitMessages(llvm_git_dir, hash_vals.rstrip(),
llvm_version)
@@ -390,7 +380,7 @@ class LLVMHash(object):
'git', 'ls-remote', _LLVM_GIT_URL, path_to_master_branch
]
- llvm_tot_git_hash = subprocess.check_output(llvm_tot_git_hash_cmd)
+ llvm_tot_git_hash = check_output(llvm_tot_git_hash_cmd)
return llvm_tot_git_hash.rstrip().split()[0]
diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py
index d9eecce6..04fbdd55 100755
--- a/llvm_tools/llvm_bisection.py
+++ b/llvm_tools/llvm_bisection.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -6,7 +6,6 @@
"""Performs bisection on LLVM based off a .JSON file."""
-from __future__ import division
from __future__ import print_function
import argparse
@@ -20,7 +19,6 @@ from assert_not_in_chroot import VerifyOutsideChroot
from get_llvm_hash import CreateTempLLVMRepo
from get_llvm_hash import LLVMHash
from modify_a_tryjob import AddTryjob
-from patch_manager import _ConvertToASCII
from update_tryjob_status import FindTryjobIndex
from update_tryjob_status import TryjobStatus
@@ -118,12 +116,12 @@ def GetCommandLineArgs():
default=cros_root,
help='the path to the chroot (default: %(default)s)')
- # Add argument for the log level.
+ # Add argument for whether to display command contents to `stdout`.
parser.add_argument(
- '--log_level',
- default='none',
- choices=['none', 'quiet', 'average', 'verbose'],
- help='the level for the logs (default: %(default)s)')
+ '--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
args_output = parser.parse_args()
@@ -133,7 +131,7 @@ def GetCommandLineArgs():
if args_output.last_tested and not args_output.last_tested.endswith('.json'):
raise ValueError(
- 'Filed provided %s does not end in \'.json\'' % args_output.last_tested)
+ 'Filed provided %s does not end in ".json"' % args_output.last_tested)
return args_output
@@ -144,7 +142,7 @@ def _ValidateStartAndEndAgainstJSONStartAndEnd(start, end, json_start,
if start != json_start or end != json_end:
raise ValueError('The start %d or the end %d version provided is '
- 'different than \'start\' %d or \'end\' %d in the .JSON '
+ 'different than "start" %d or "end" %d in the .JSON '
'file' % (start, end, json_start, json_end))
@@ -179,7 +177,7 @@ def GetStartAndEndRevision(start, end, tryjobs):
# Verify that each tryjob has a value for the 'status' key.
for cur_tryjob_dict in tryjobs:
if not cur_tryjob_dict.get('status', None):
- raise ValueError('\'status\' is missing or has no value, please '
+ raise ValueError('"status" is missing or has no value, please '
'go to %s and update it' % cur_tryjob_dict['link'])
all_bad_revisions = [end]
@@ -333,19 +331,19 @@ def CheckForExistingTryjobsInRevisionsToLaunch(revisions, jobs):
for rev in revisions:
if FindTryjobIndex(rev, jobs) is not None:
- raise ValueError('Revision %d exists already in \'jobs\'' % rev)
+ raise ValueError('Revision %d exists already in "jobs"' % rev)
def UpdateBisection(revisions, git_hashes, bisect_contents, last_tested,
update_packages, chroot_path, patch_metadata_file,
- extra_change_lists, options, builder, log_level):
+ extra_change_lists, options, builder, verbose):
"""Adds tryjobs and updates the status file with the new tryjobs."""
try:
for svn_revision, git_hash in zip(revisions, git_hashes):
tryjob_dict = AddTryjob(update_packages, git_hash, svn_revision,
chroot_path, patch_metadata_file,
- extra_change_lists, options, builder, log_level,
+ extra_change_lists, options, builder, verbose,
svn_revision)
bisect_contents['jobs'].append(tryjob_dict)
@@ -394,7 +392,7 @@ def main(args_output):
try:
with open(args_output.last_tested) as f:
- bisect_contents = _ConvertToASCII(json.load(f))
+ bisect_contents = json.load(f)
except IOError as err:
if err.errno != errno.ENOENT:
raise
@@ -450,7 +448,7 @@ def main(args_output):
args_output.last_tested, update_packages,
args_output.chroot_path, patch_metadata_file,
args_output.extra_change_lists, args_output.options,
- args_output.builder, args_output.log_level)
+ args_output.builder, args_output.verbose)
if __name__ == '__main__':
diff --git a/llvm_tools/llvm_patch_management.py b/llvm_tools/llvm_patch_management.py
index 90f82d30..522d4e34 100755
--- a/llvm_tools/llvm_patch_management.py
+++ b/llvm_tools/llvm_patch_management.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,19 +8,21 @@
from __future__ import print_function
-from pipes import quote
import argparse
import os
import patch_manager
from assert_not_in_chroot import VerifyOutsideChroot
-from cros_utils import command_executer
from failure_modes import FailureModes
from get_llvm_hash import CreateTempLLVMRepo
from get_llvm_hash import GetGoogle3LLVMVersion
from get_llvm_hash import LLVMHash
+from subprocess_helpers import ChrootRunCommand
+from subprocess_helpers import ExecCommandAndCaptureOutput
-ce = command_executer.GetCommandExecuter()
+# If set to `True`, then the contents of `stdout` after executing a command will
+# be displayed to the terminal.
+verbose = False
def GetCommandLineArgs():
@@ -52,12 +54,12 @@ def GetCommandLineArgs():
default=['sys-devel/llvm'],
help='the packages to manage their patches (default: %(default)s)')
- # Add argument for the log level.
+ # Add argument for whether to display command contents to `stdout`.
parser.add_argument(
- '--log_level',
- default='none',
- choices=['none', 'quiet', 'average', 'verbose'],
- help='the level for the logs (default: %(default)s)')
+ '--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
# Add argument for the LLVM version to use for patch management.
parser.add_argument(
@@ -86,8 +88,9 @@ def GetCommandLineArgs():
# Parse the command line.
args_output = parser.parse_args()
- # Set the log level for the command executer.
- ce.SetLogLevel(log_level=args_output.log_level)
+ global verbose
+
+ verbose = args_output.verbose
unique_packages = list(set(args_output.packages))
@@ -119,15 +122,8 @@ def GetPathToFilesDirectory(chroot_path, package):
raise ValueError('Invalid chroot provided: %s' % chroot_path)
# Get the absolute chroot path to the ebuild.
- ret, chroot_ebuild_path, err = ce.ChrootRunCommandWOutput(
- chromeos_root=chroot_path,
- command='equery w %s' % package,
- print_to_console=ce.GetLogLevel() == 'verbose')
-
- if ret: # Failed to get the absolute chroot path to package's ebuild.
- raise ValueError(
- 'Failed to get the absolute chroot path of the package %s: %s' %
- (package, err))
+ chroot_ebuild_path = ChrootRunCommand(
+ chroot_path, ['equery', 'w', package], verbose=verbose)
# Get the absolute chroot path to $FILESDIR's parent directory.
filesdir_parent_path = os.path.dirname(chroot_ebuild_path.strip())
@@ -175,20 +171,15 @@ def _CheckPatchMetadataPath(patch_metadata_path):
raise ValueError('Invalid file provided: %s' % patch_metadata_path)
if not patch_metadata_path.endswith('.json'):
- raise ValueError('File does not end in \'.json\': %s' % patch_metadata_path)
+ raise ValueError('File does not end in ".json": %s' % patch_metadata_path)
def _MoveSrcTreeHEADToGitHash(src_path, git_hash):
"""Moves HEAD to 'git_hash'."""
- move_head_cmd = 'git -C %s checkout %s' % (quote(src_path), git_hash)
+ move_head_cmd = ['git', '-C', src_path, 'checkout', git_hash]
- ret, _, err = ce.RunCommandWOutput(
- move_head_cmd, print_to_console=ce.GetLogLevel() == 'verbose')
-
- if ret: # Failed to checkout to 'git_hash'.
- raise ValueError('Failed to moved HEAD in %s to %s: %s' % (quote(src_path),
- git_hash, err))
+ ExecCommandAndCaptureOutput(move_head_cmd, verbose=verbose)
def UpdatePackagesPatchMetadataFile(chroot_path, svn_version,
@@ -275,6 +266,9 @@ def main():
print('The patch file %s has been modified for the packages:' %
args_output.patch_metadata_file)
print('\n'.join(args_output.packages))
+ else:
+ print('Applicable patches in %s applied successfully.' %
+ args_output.patch_metadata_file)
if __name__ == '__main__':
diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py
index f50f2bdc..20ba3541 100755
--- a/llvm_tools/modify_a_tryjob.py
+++ b/llvm_tools/modify_a_tryjob.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -17,7 +17,6 @@ import sys
from assert_not_in_chroot import VerifyOutsideChroot
from failure_modes import FailureModes
from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption
-from patch_manager import _ConvertToASCII
from update_packages_and_run_tryjobs import RunTryJobs
from update_tryjob_status import FindTryjobIndex
from update_tryjob_status import TryjobStatus
@@ -89,12 +88,12 @@ def GetCommandLineArgs():
default=cros_root,
help='the path to the chroot (default: %(default)s)')
- # Add argument for the log level.
+ # Add argument for whether to display command contents to `stdout`.
parser.add_argument(
- '--log_level',
- default='none',
- choices=['none', 'quiet', 'average', 'verbose'],
- help='the level for the logs (default: %(default)s)')
+ '--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
args_output = parser.parse_args()
@@ -130,7 +129,7 @@ def GetCLAfterUpdatingPackages(packages, git_hash, svn_version, chroot_path,
def CreateNewTryjobEntryForBisection(cl, extra_cls, options, builder,
- chroot_path, log_level, cl_url, revision):
+ chroot_path, verbose, cl_url, revision):
"""Submits a tryjob and adds additional information."""
# Get the tryjob results after submitting the tryjob.
@@ -145,7 +144,7 @@ def CreateNewTryjobEntryForBisection(cl, extra_cls, options, builder,
# }
# ]
tryjob_results = RunTryJobs(cl, extra_cls, options, [builder], chroot_path,
- log_level)
+ verbose)
print('\nTryjob:')
print(tryjob_results[0])
@@ -159,24 +158,24 @@ def CreateNewTryjobEntryForBisection(cl, extra_cls, options, builder,
def AddTryjob(packages, git_hash, revision, chroot_path, patch_metadata_file,
- extra_cls, options, builder, log_level, svn_option):
+ extra_cls, options, builder, verbose, svn_option):
"""Submits a tryjob."""
- update_chromeos_llvm_next_hash.ce.SetLogLevel(log_level=log_level)
+ update_chromeos_llvm_next_hash.verbose = verbose
change_list = GetCLAfterUpdatingPackages(packages, git_hash, revision,
chroot_path, patch_metadata_file,
svn_option)
tryjob_dict = CreateNewTryjobEntryForBisection(
- change_list.cl_number, extra_cls, options, builder, chroot_path,
- log_level, change_list.url, revision)
+ change_list.cl_number, extra_cls, options, builder, chroot_path, verbose,
+ change_list.url, revision)
return tryjob_dict
def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
- options, builder, chroot_path, log_level):
+ options, builder, chroot_path, verbose):
"""Removes, relaunches, or adds a tryjob.
Args:
@@ -189,7 +188,7 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
builder: The builder to use for 'cros tryjob'.
chroot_path: The absolute path to the chroot (used by 'cros tryjob' when
relaunching a tryjob).
- log_level: The level to use for the logs.
+ verbose: Determines whether to print the contents of a command to `stdout`.
"""
# Format of 'bisect_contents':
@@ -204,7 +203,7 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
# ]
# }
with open(status_file) as tryjobs:
- bisect_contents = _ConvertToASCII(json.load(tryjobs))
+ bisect_contents = json.load(tryjobs)
if not bisect_contents['jobs'] and modify_tryjob != ModifyTryjob.ADD:
sys.exit('No tryjobs in %s' % status_file)
@@ -219,14 +218,23 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
# Determine the action to take based off of 'modify_tryjob'.
if modify_tryjob == ModifyTryjob.REMOVE:
del bisect_contents['jobs'][tryjob_index]
+
+ print('Successfully deleted the tryjob of revision %d' % revision)
elif modify_tryjob == ModifyTryjob.RELAUNCH:
- RunTryJobs(bisect_contents['jobs'][tryjob_index]['cl'],
- bisect_contents['jobs'][tryjob_index]['extra_cls'],
- bisect_contents['jobs'][tryjob_index]['options'],
- bisect_contents['jobs'][tryjob_index]['builder'], chroot_path,
- log_level)
+ # Need to update the tryjob link and buildbucket ID.
+ tryjob_results = RunTryJobs(
+ bisect_contents['jobs'][tryjob_index]['cl'],
+ bisect_contents['jobs'][tryjob_index]['extra_cls'],
+ bisect_contents['jobs'][tryjob_index]['options'],
+ bisect_contents['jobs'][tryjob_index]['builder'], chroot_path, verbose)
bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.PENDING.value
+ bisect_contents['jobs'][tryjob_index]['link'] = tryjob_results[0]['link']
+ bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[0][
+ 'buildbucket_id']
+
+ print('Successfully relaunched the tryjob for revision %d and updated '
+ 'the tryjob link to %s' % (revision, tryjob_results[0]['link']))
elif modify_tryjob == ModifyTryjob.ADD:
# Tryjob exists already.
if tryjob_index is not None:
@@ -247,9 +255,11 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
tryjob_dict = AddTryjob(update_packages, git_hash, revision, chroot_path,
patch_metadata_file, extra_cls, options, builder,
- log_level, revision)
+ verbose, revision)
bisect_contents['jobs'].append(tryjob_dict)
+
+ print('Successfully added tryjob of revision %d' % revision)
else:
raise ValueError('Failed to add tryjob to %s' % status_file)
else:
@@ -271,7 +281,7 @@ def main():
args_output.revision, ModifyTryjob(
args_output.modify_tryjob), args_output.status_file,
args_output.extra_change_lists, args_output.options, args_output.builder,
- args_output.chroot_path, args_output.log_level)
+ args_output.chroot_path, args_output.verbose)
if __name__ == '__main__':
diff --git a/llvm_tools/patch_manager.py b/llvm_tools/patch_manager.py
index 07c88e6a..806b944f 100755
--- a/llvm_tools/patch_manager.py
+++ b/llvm_tools/patch_manager.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -17,6 +17,8 @@ import sys
from collections import namedtuple
from failure_modes import FailureModes
from get_llvm_hash import LLVMHash
+from subprocess_helpers import check_call
+from subprocess_helpers import check_output
def is_directory(dir_path):
@@ -36,8 +38,8 @@ def is_patch_metadata_file(patch_metadata_file):
'Invalid patch metadata file provided: %s' % patch_metadata_file)
if not patch_metadata_file.endswith('.json'):
- raise ValueError('Patch metadata file does not end in \'.json\': %s' %
- patch_metadata_file)
+ raise ValueError(
+ 'Patch metadata file does not end in ".json": %s' % patch_metadata_file)
return patch_metadata_file
@@ -58,11 +60,11 @@ def EnsureBisectModeAndSvnVersionAreSpecifiedTogether(failure_mode,
"""Validates that 'good_svn_version' is passed in only for bisection."""
if failure_mode != FailureModes.BISECT_PATCHES.value and good_svn_version:
- raise ValueError('\'good_svn_version\' is only available for bisection.')
+ raise ValueError('"good_svn_version" is only available for bisection.')
elif failure_mode == FailureModes.BISECT_PATCHES.value and \
not good_svn_version:
raise ValueError('A good SVN version is required for bisection (used by'
- '\'git bisect start\'.')
+ '"git bisect start".')
def GetCommandLineArgs():
@@ -91,7 +93,7 @@ def GetCommandLineArgs():
default=False,
help='Determines whether bisection should continue after successfully '
'bisecting a patch (default: %(default)s) - only used for '
- '\'bisect_patches\'')
+ '"bisect_patches"')
# Add argument for the LLVM version to use for patch management.
parser.add_argument(
@@ -106,7 +108,7 @@ def GetCommandLineArgs():
'--patch_metadata_file',
required=True,
type=is_patch_metadata_file,
- help='the absolute path to the .json file in \'$FILESDIR/\' of the '
+ help='the absolute path to the .json file in "$FILESDIR/" of the '
'package which has all the patches and their metadata if applicable')
# Add argument for the absolute path to the ebuild's $FILESDIR path.
@@ -115,7 +117,7 @@ def GetCommandLineArgs():
'--filesdir_path',
required=True,
type=is_directory,
- help='the absolute path to the ebuild \'files/\' directory')
+ help='the absolute path to the ebuild "files/" directory')
# Add argument for the absolute path to the unpacked sources.
parser.add_argument(
@@ -147,7 +149,7 @@ def GetHEADSVNVersion(src_path):
get_head_cmd = ['git', '-C', src_path, 'log', '-1', '--pretty=%B']
- head_commit_message = subprocess.check_output(get_head_cmd)
+ head_commit_message = check_output(get_head_cmd)
head_svn_version = LLVMHash().GetSVNVersionFromCommitMessage(
head_commit_message)
@@ -161,8 +163,8 @@ def VerifyHEADIsTheSameAsSVNVersion(src_path, svn_version):
head_svn_version = GetHEADSVNVersion(src_path)
if head_svn_version != svn_version:
- raise ValueError('HEAD\'s SVN version %d does not match \'svn_version\''
- ' %d, please move HEAD to \'svn_version\'s\' git hash.' %
+ raise ValueError('HEAD\'s SVN version %d does not match "svn_version"'
+ ' %d, please move HEAD to "svn_version"s\' git hash.' %
(head_svn_version, svn_version))
@@ -242,7 +244,7 @@ def ApplyPatch(src_path, patch_path):
]
try:
- subprocess.check_output(test_patch_cmd)
+ check_output(test_patch_cmd)
# If the mode is 'continue', then catching the exception makes sure that
# the program does not exit on the first failed applicable patch.
@@ -251,7 +253,7 @@ def ApplyPatch(src_path, patch_path):
return False
# Test run succeeded on the patch.
- subprocess.check_output(apply_patch_cmd)
+ check_output(apply_patch_cmd)
return True
@@ -269,27 +271,12 @@ def UpdatePatchMetadataFile(patch_metadata_file, patches):
"""
if not patch_metadata_file.endswith('.json'):
- raise ValueError('File does not end in \'.json\': %s' % patch_metadata_file)
+ raise ValueError('File does not end in ".json": %s' % patch_metadata_file)
with open(patch_metadata_file, 'w') as patch_file:
json.dump(patches, patch_file, indent=4, separators=(',', ': '))
-def _ConvertToASCII(obj):
- """Convert an object loaded from JSON to ASCII; JSON gives us unicode."""
-
- # Using something like `object_hook` is insufficient, since it only fires on
- # actual JSON objects. `encoding` fails, too, since the default decoder always
- # uses unicode() to decode strings.
- if isinstance(obj, unicode):
- return str(obj)
- if isinstance(obj, dict):
- return {_ConvertToASCII(k): _ConvertToASCII(v) for k, v in obj.iteritems()}
- if isinstance(obj, list):
- return [_ConvertToASCII(v) for v in obj]
- return obj
-
-
def GetCommitHashesForBisection(src_path, good_svn_version, bad_svn_version):
"""Gets the good and bad commit hashes required by `git bisect start`."""
@@ -312,7 +299,7 @@ def PerformBisection(src_path, good_commit, bad_commit, svn_version,
'git', '-C', src_path, 'bisect', 'start', bad_commit, good_commit
]
- subprocess.check_output(bisect_start_cmd)
+ check_output(bisect_start_cmd)
bisect_run_cmd = [
'git', '-C', src_path, 'bisect', 'run',
@@ -323,7 +310,7 @@ def PerformBisection(src_path, good_commit, bad_commit, svn_version,
'%d' % num_patches
]
- subprocess.check_call(bisect_run_cmd)
+ check_call(bisect_run_cmd)
# Successfully bisected the patch, so retrieve the SVN version from the
# commit message.
@@ -331,12 +318,11 @@ def PerformBisection(src_path, good_commit, bad_commit, svn_version,
'git', '-C', src_path, 'show', 'refs/bisect/bad'
]
- bad_commit_message = subprocess.check_output(
- get_bad_commit_from_bisect_run_cmd)
+ bad_commit_message = check_output(get_bad_commit_from_bisect_run_cmd)
end_bisection_cmd = ['git', '-C', src_path, 'bisect', 'reset']
- subprocess.check_output(end_bisection_cmd)
+ check_output(end_bisection_cmd)
# `git bisect run` returns the bad commit hash and the commit message.
bad_version = LLVMHash().GetSVNVersionFromCommitMessage(
@@ -350,11 +336,11 @@ def CleanSrcTree(src_path):
reset_src_tree_cmd = ['git', '-C', src_path, 'reset', 'HEAD', '--hard']
- subprocess.check_output(reset_src_tree_cmd)
+ check_output(reset_src_tree_cmd)
clean_src_tree_cmd = ['git', '-C', src_path, 'clean', '-fd']
- subprocess.check_output(clean_src_tree_cmd)
+ check_output(clean_src_tree_cmd)
def SaveSrcTreeState(src_path):
@@ -362,7 +348,7 @@ def SaveSrcTreeState(src_path):
save_src_tree_cmd = ['git', '-C', src_path, 'stash', '-a']
- subprocess.check_output(save_src_tree_cmd)
+ check_output(save_src_tree_cmd)
def RestoreSrcTreeState(src_path, bad_commit_hash):
@@ -370,11 +356,11 @@ def RestoreSrcTreeState(src_path, bad_commit_hash):
checkout_cmd = ['git', '-C', src_path, 'checkout', bad_commit_hash]
- subprocess.check_output(checkout_cmd)
+ check_output(checkout_cmd)
get_changes_cmd = ['git', '-C', src_path, 'stash', 'pop']
- subprocess.check_output(get_changes_cmd)
+ check_output(get_changes_cmd)
def HandlePatches(svn_version,
@@ -454,7 +440,7 @@ def HandlePatches(svn_version,
failed_patches = []
with open(patch_metadata_file) as patch_file:
- patch_file_contents = _ConvertToASCII(json.load(patch_file))
+ patch_file_contents = json.load(patch_file)
if mode == FailureModes.BISECT_PATCHES:
# A good and bad commit are required by `git bisect start`.
diff --git a/llvm_tools/subprocess_helpers.py b/llvm_tools/subprocess_helpers.py
new file mode 100644
index 00000000..8845112c
--- /dev/null
+++ b/llvm_tools/subprocess_helpers.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helpers/wrappers for the subprocess module for migration to python3."""
+
+from __future__ import print_function
+
+import subprocess
+
+
+def CheckCommand(cmd):
+ """Executes the command using Popen()."""
+
+ cmd_obj = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='UTF-8')
+
+ stdout, _ = cmd_obj.communicate()
+
+ if cmd_obj.returncode:
+ print(stdout)
+ raise subprocess.CalledProcessError(cmd_obj.returncode, cmd)
+
+
+def check_output(cmd, cwd=None):
+ """Wrapper for pre-python3 subprocess.check_output()."""
+
+ return subprocess.check_output(cmd, encoding='UTF-8', cwd=cwd)
+
+
+def check_call(cmd, cwd=None):
+ """Wrapper for pre-python3 subprocess.check_call()."""
+
+ subprocess.check_call(cmd, encoding='UTF-8', cwd=cwd)
+
+
+# FIXME: CTRL+C does not work when executing a command inside the chroot via
+# `cros_sdk`.
+def ChrootRunCommand(chroot_path, cmd, verbose=False):
+ """Runs the command inside the chroot."""
+
+ exec_chroot_cmd = ['cros_sdk', '--']
+ exec_chroot_cmd.extend(cmd)
+
+ return ExecCommandAndCaptureOutput(
+ exec_chroot_cmd, cwd=chroot_path, verbose=verbose)
+
+
+def ExecCommandAndCaptureOutput(cmd, cwd=None, verbose=False):
+ """Executes the command and prints to stdout if possible."""
+
+ out = check_output(cmd, cwd=cwd).rstrip()
+
+ if verbose and out:
+ print(out)
+
+ return out
diff --git a/llvm_tools/update_all_tryjobs_with_auto.py b/llvm_tools/update_all_tryjobs_with_auto.py
index 4a670c34..511bfffa 100755
--- a/llvm_tools/update_all_tryjobs_with_auto.py
+++ b/llvm_tools/update_all_tryjobs_with_auto.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -13,7 +13,6 @@ import json
import os
from assert_not_in_chroot import VerifyOutsideChroot
-from patch_manager import _ConvertToASCII
from update_tryjob_status import GetAutoResult
from update_tryjob_status import TryjobStatus
@@ -51,7 +50,7 @@ def GetCommandLineArgs():
if not os.path.isfile(args_output.last_tested) or \
not args_output.last_tested.endswith('.json'):
- raise ValueError('File does not exist or does not ending in \'.json\' '
+ raise ValueError('File does not exist or does not ending in ".json" '
': %s' % args_output.last_tested)
return args_output
@@ -65,7 +64,7 @@ def main():
args_output = GetCommandLineArgs()
with open(args_output.last_tested) as tryjobs:
- bisect_contents = _ConvertToASCII(json.load(tryjobs))
+ bisect_contents = json.load(tryjobs)
for tryjob in bisect_contents['jobs']:
if tryjob['status'] == TryjobStatus.PENDING.value:
diff --git a/llvm_tools/update_chromeos_llvm_next_hash.py b/llvm_tools/update_chromeos_llvm_next_hash.py
index aa63b76c..cf92b489 100755
--- a/llvm_tools/update_chromeos_llvm_next_hash.py
+++ b/llvm_tools/update_chromeos_llvm_next_hash.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -13,19 +13,22 @@ for review.
from __future__ import print_function
from collections import namedtuple
-from pipes import quote
import argparse
-import llvm_patch_management
import os
import re
+import subprocess
from assert_not_in_chroot import VerifyOutsideChroot
-from cros_utils import command_executer
from failure_modes import FailureModes
from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption
from get_llvm_hash import is_svn_option
+from subprocess_helpers import ChrootRunCommand
+from subprocess_helpers import ExecCommandAndCaptureOutput
+import llvm_patch_management
-ce = command_executer.GetCommandExecuter()
+# If set to `True`, then the contents of `stdout` after executing a command will
+# be displayed to the terminal.
+verbose = False
CommitContents = namedtuple('CommitContents', ['url', 'cl_number'])
@@ -63,12 +66,12 @@ def GetCommandLineArgs():
help='the ebuilds to update their hash for llvm-next ' \
'(default: %(default)s)')
- # Add argument for the log level.
+ # Add argument for whether to display command contents to `stdout`.
parser.add_argument(
- '--log_level',
- default='none',
- choices=['none', 'quiet', 'average', 'verbose'],
- help='the level for the logs (default: %(default)s)')
+ '--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
# Add argument for the LLVM version to use.
parser.add_argument(
@@ -100,8 +103,9 @@ def GetCommandLineArgs():
# Parse the command line.
args_output = parser.parse_args()
- # Set the log level for the command executer.
- ce.SetLogLevel(log_level=args_output.log_level)
+ global verbose
+
+ verbose = args_output.verbose
return args_output
@@ -127,17 +131,9 @@ def GetChrootBuildPaths(chromeos_root, package_list):
# Find the chroot path for each package's ebuild.
for cur_package in sorted(set(package_list)):
# Cmd to find the chroot path for the package.
- equery_cmd = 'equery w %s' % cur_package
-
- # Find the chroot path for the package.
- ret, chroot_path, err = ce.ChrootRunCommandWOutput(
- chromeos_root=chromeos_root,
- command=equery_cmd,
- print_to_console=ce.GetLogLevel() == 'verbose')
+ equery_cmd = ['equery', 'w', cur_package]
- if ret: # failed to get the chroot path
- raise ValueError('Failed to get chroot path for the package (%s): %s' %
- (cur_package, err))
+ chroot_path = ChrootRunCommand(chromeos_root, equery_cmd, verbose=verbose)
chroot_paths.append(chroot_path.strip())
@@ -269,12 +265,9 @@ def UpdateBuildLLVMNextHash(ebuild_path, llvm_hash, llvm_version):
parent_dir = os.path.dirname(ebuild_path)
# Stage the changes.
- ret, _, err = ce.RunCommandWOutput(
- 'git -C %s add %s' % (parent_dir, ebuild_path),
- print_to_console=ce.GetLogLevel() == 'verbose')
+ stage_changes_cmd = ['git', '-C', parent_dir, 'add', ebuild_path]
- if ret: # failed to stage the changes
- raise ValueError('Failed to stage the ebuild for commit: %s' % err)
+ ExecCommandAndCaptureOutput(stage_changes_cmd, verbose=verbose)
def ReplaceLLVMNextHash(ebuild_lines, is_updated, llvm_regex, llvm_hash,
@@ -333,12 +326,11 @@ def UprevEbuild(symlink):
path_to_symlink_dir = os.path.dirname(symlink)
# Stage the new symlink for commit.
- ret, _, err = ce.RunCommandWOutput(
- 'git -C %s mv %s %s' % (path_to_symlink_dir, symlink, new_symlink),
- print_to_console=ce.GetLogLevel() == 'verbose')
+ stage_symlink_cmd = [
+ 'git', '-C', path_to_symlink_dir, 'mv', symlink, new_symlink
+ ]
- if ret: # failed to stage the symlink for commit
- raise ValueError('Failed to stage the symlink for commit: %s' % err)
+ ExecCommandAndCaptureOutput(stage_symlink_cmd, verbose=verbose)
def _CreateRepo(path_to_repo_dir, llvm_hash):
@@ -355,18 +347,21 @@ def _CreateRepo(path_to_repo_dir, llvm_hash):
if not os.path.isdir(path_to_repo_dir):
raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir)
- create_repo_cmd = ' && '.join([
- 'cd %s' % path_to_repo_dir,
- 'git reset HEAD --hard',
- 'repo start llvm-next-update-%s' % llvm_hash,
- ])
+ reset_changes_cmd = [
+ 'git',
+ '-C',
+ path_to_repo_dir,
+ 'reset',
+ 'HEAD',
+ '--hard',
+ ]
- ret, _, err = ce.RunCommandWOutput(
- create_repo_cmd, print_to_console=ce.GetLogLevel() == 'verbose')
+ ExecCommandAndCaptureOutput(reset_changes_cmd, verbose=verbose)
- if ret: # failed to create a repo for the changes
- raise ValueError('Failed to create the repo (llvm-next-update-%s): %s' %
- (llvm_hash, err))
+ create_repo_cmd = ['repo', 'start', 'llvm-next-update-%s' % llvm_hash]
+
+ ExecCommandAndCaptureOutput(
+ create_repo_cmd, cwd=path_to_repo_dir, verbose=verbose)
def _DeleteRepo(path_to_repo_dir, llvm_hash):
@@ -383,18 +378,22 @@ def _DeleteRepo(path_to_repo_dir, llvm_hash):
if not os.path.isdir(path_to_repo_dir):
raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir)
- delete_repo_cmd = ' && '.join([
- 'cd %s' % path_to_repo_dir, 'git checkout cros/master',
- 'git reset HEAD --hard',
- 'git branch -D llvm-next-update-%s' % llvm_hash
- ])
+ checkout_to_master_cmd = [
+ 'git', '-C', path_to_repo_dir, 'checkout', 'cros/master'
+ ]
+
+ ExecCommandAndCaptureOutput(checkout_to_master_cmd, verbose=verbose)
+
+ reset_head_cmd = ['git', '-C', path_to_repo_dir, 'reset', 'HEAD', '--hard']
- ret, _, err = ce.RunCommandWOutput(
- delete_repo_cmd, print_to_console=ce.GetLogLevel() == 'verbose')
+ ExecCommandAndCaptureOutput(reset_head_cmd, verbose=verbose)
- if ret: # failed to delete the repo
- raise ValueError('Failed to delete the repo (llvm-next-update-%s): %s' %
- (llvm_hash, err))
+ delete_repo_cmd = [
+ 'git', '-C', path_to_repo_dir, 'branch', '-D',
+ 'llvm-next-update-%s' % llvm_hash
+ ]
+
+ ExecCommandAndCaptureOutput(delete_repo_cmd, verbose=verbose)
def GetGerritRepoUploadContents(repo_upload_contents):
@@ -446,26 +445,36 @@ def UploadChanges(path_to_repo_dir, llvm_hash, commit_messages):
if not os.path.isdir(path_to_repo_dir):
raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir)
- commit_cmd = 'cd %s && git commit %s' % (path_to_repo_dir, commit_messages)
-
- ret, _, err = ce.RunCommandWOutput(
- commit_cmd, print_to_console=ce.GetLogLevel() == 'verbose')
+ commit_cmd = [
+ 'git',
+ 'commit',
+ ]
+ commit_cmd.extend(commit_messages)
- if ret: # failed to commit the changes
- raise ValueError('Failed to create a commit for the changes: %s' % err)
+ ExecCommandAndCaptureOutput(commit_cmd, cwd=path_to_repo_dir, verbose=verbose)
# Upload the changes for review.
- upload_change_cmd = 'cd %s && ' \
- 'yes | repo upload --br=llvm-next-update-%s --no-verify' % (
- path_to_repo_dir, llvm_hash)
+ upload_change_cmd = (
+ 'yes | repo upload --br=llvm-next-update-%s --no-verify' % llvm_hash)
+
+ # NOTE: Need `shell=True` in order to pipe `yes` into `repo upload ...`.
+ #
+ # The CL URL is sent to 'stderr', so need to redirect 'stderr' to 'stdout'.
+ upload_changes_obj = subprocess.Popen(
+ upload_change_cmd,
+ cwd=path_to_repo_dir,
+ shell=True,
+ encoding='UTF-8',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
- ret, _, err = ce.RunCommandWOutput(
- upload_change_cmd, print_to_console=ce.GetLogLevel() == 'verbose')
+ out, _ = upload_changes_obj.communicate()
- if ret: # failed to upload the changes for review
- raise ValueError('Failed to upload changes for review: %s' % err)
+ if upload_changes_obj.returncode: # Failed to upload changes.
+ print(out)
+ raise ValueError('Failed to upload changes for review')
- return GetGerritRepoUploadContents(err)
+ return GetGerritRepoUploadContents(out.rstrip())
def CreatePathDictionaryFromPackages(chroot_path, update_packages):
@@ -504,13 +513,12 @@ def RemovePatchesFromFilesDir(patches_to_remove):
"""
for cur_patch in patches_to_remove:
- ret, _, err = ce.RunCommandWOutput(
- 'git -C %s rm -f %s' % (os.path.dirname(cur_patch), cur_patch),
- print_to_console=ce.GetLogLevel() == 'verbose')
+ remove_patch_cmd = [
+ 'git', '-C',
+ os.path.dirname(cur_patch), 'rm', '-f', cur_patch
+ ]
- if ret: # Failed to remove the patch in $FILESDIR.
- raise ValueError(
- 'Failed to remove patch %s: %s' % (os.path.basename(cur_patch), err))
+ ExecCommandAndCaptureOutput(remove_patch_cmd, verbose=verbose)
def StagePatchMetadataFileForCommit(patch_metadata_file_path):
@@ -529,15 +537,12 @@ def StagePatchMetadataFileForCommit(patch_metadata_file_path):
'Invalid patch metadata file provided: %s' % patch_metadata_file_path)
# Cmd to stage the patch metadata file for commit.
- stage_patch_file = 'git -C %s add %s' % (
- os.path.dirname(patch_metadata_file_path), patch_metadata_file_path)
-
- ret, _, err = ce.RunCommandWOutput(
- stage_patch_file, print_to_console=ce.GetLogLevel() == 'verbose')
+ stage_patch_file = [
+ 'git', '-C',
+ os.path.dirname(patch_metadata_file_path), 'add', patch_metadata_file_path
+ ]
- if ret: # Failed to stage the patch metadata file for commit.
- raise ValueError('Failed to stage patch metadata file %s for commit: %s' %
- (os.path.basename(patch_metadata_file_path), err))
+ ExecCommandAndCaptureOutput(stage_patch_file, verbose=verbose)
def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages):
@@ -559,32 +564,29 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages):
patch_info_dict['removed_patches'] or \
patch_info_dict['modified_metadata']:
cur_package_header = 'For the package %s:' % package_name
- commit_messages.append('-m %s' % quote(cur_package_header))
+ commit_messages.append('-m %s' % cur_package_header)
# Add to the commit message that the patch metadata file was modified.
if patch_info_dict['modified_metadata']:
patch_metadata_path = patch_info_dict['modified_metadata']
- commit_messages.append(
- '-m %s' % quote('The patch metadata file %s was '
- 'modified' % os.path.basename(patch_metadata_path)))
+ commit_messages.append('-m %s' % 'The patch metadata file %s was '
+ 'modified' % os.path.basename(patch_metadata_path))
StagePatchMetadataFileForCommit(patch_metadata_path)
# Add each disabled patch to the commit message.
if patch_info_dict['disabled_patches']:
- commit_messages.append(
- '-m %s' % quote('The following patches were disabled:'))
+ commit_messages.append('-m %s' % 'The following patches were disabled:')
for patch_path in patch_info_dict['disabled_patches']:
- commit_messages.append('-m %s' % quote(os.path.basename(patch_path)))
+ commit_messages.append('-m %s' % os.path.basename(patch_path))
# Add each removed patch to the commit message.
if patch_info_dict['removed_patches']:
- commit_messages.append(
- '-m %s' % quote('The following patches were removed:'))
+ commit_messages.append('-m %s' % 'The following patches were removed:')
for patch_path in patch_info_dict['removed_patches']:
- commit_messages.append('-m %s' % quote(os.path.basename(patch_path)))
+ commit_messages.append('-m %s' % os.path.basename(patch_path))
RemovePatchesFromFilesDir(patch_info_dict['removed_patches'])
@@ -616,11 +618,14 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path,
Gerrit commit URL and the second pair is the change list number.
"""
+ # Determines whether to print the result of each executed command.
+ llvm_patch_management.verbose = verbose
+
# Construct a dictionary where the key is the absolute path of the symlink to
# the package and the value is the absolute path to the ebuild of the package.
paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages)
- repo_path = os.path.dirname(paths_dict.itervalues().next())
+ repo_path = os.path.dirname(next(iter(paths_dict.values())))
_CreateRepo(repo_path, llvm_hash)
@@ -634,10 +639,9 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path,
else:
commit_message_header = 'llvm-next: Update packages to r%d' % llvm_version
- commit_messages = ['-m %s' % quote(commit_message_header)]
+ commit_messages = ['-m %s' % commit_message_header]
- commit_messages.append(
- '-m %s' % quote('Following packages have been updated:'))
+ commit_messages.append('-m %s' % 'Following packages have been updated:')
# Holds the list of packages that are updating.
packages = []
@@ -662,10 +666,9 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path,
new_commit_message = '%s/%s' % (parent_dir_name, cur_dir_name)
- commit_messages.append('-m %s' % quote(new_commit_message))
+ commit_messages.append('-m %s' % new_commit_message)
# Handle the patches for each package.
- llvm_patch_management.ce.SetLogLevel(log_level=ce.GetLogLevel())
package_info_dict = llvm_patch_management.UpdatePackagesPatchMetadataFile(
chroot_path, llvm_version, patch_metadata_file, packages, mode)
@@ -673,7 +676,7 @@ def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path,
commit_messages = StagePackagesPatchResultsForCommit(
package_info_dict, commit_messages)
- change_list = UploadChanges(repo_path, llvm_hash, ' '.join(commit_messages))
+ change_list = UploadChanges(repo_path, llvm_hash, commit_messages)
finally:
_DeleteRepo(repo_path, llvm_hash)
diff --git a/llvm_tools/update_packages_and_run_tryjobs.py b/llvm_tools/update_packages_and_run_tryjobs.py
index 487b7f6a..f4fc234f 100755
--- a/llvm_tools/update_packages_and_run_tryjobs.py
+++ b/llvm_tools/update_packages_and_run_tryjobs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,19 +8,17 @@
from __future__ import print_function
-from pipes import quote
import argparse
import datetime
import json
import os
-import sys
from assert_not_in_chroot import VerifyOutsideChroot
-from cros_utils import command_executer
-from cros_utils.buildbot_utils import ParseTryjobBuildbucketId
from failure_modes import FailureModes
from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption
from get_llvm_hash import is_svn_option
+from subprocess_helpers import ChrootRunCommand
+from subprocess_helpers import ExecCommandAndCaptureOutput
import update_chromeos_llvm_next_hash
@@ -41,7 +39,7 @@ def GetCommandLineArgs():
# Create parser and add optional command-line arguments.
parser = argparse.ArgumentParser(
description='Runs a tryjob if successfully updated packages\''
- '\'LLVM_NEXT_HASH\'.')
+ '"LLVM_NEXT_HASH".')
# Add argument for the absolute path to the file that contains information on
# the previous tested svn version.
@@ -86,12 +84,12 @@ def GetCommandLineArgs():
default=cros_root,
help='the path to the chroot (default: %(default)s)')
- # Add argument for the log level.
+ # Add argument for whether to display command contents to `stdout`.
parser.add_argument(
- '--log_level',
- default='none',
- choices=['none', 'quiet', 'average', 'verbose'],
- help='the level for the logs (default: %(default)s)')
+ '--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
# Add argument for the LLVM version to use.
parser.add_argument(
@@ -166,11 +164,11 @@ def GetTryJobCommand(change_list, extra_change_lists, options, builder):
if options:
tryjob_cmd.extend('--%s' % option for option in options)
- return ' '.join(tryjob_cmd)
+ return tryjob_cmd
def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
- log_level):
+ verbose):
"""Runs a tryjob/tryjobs.
Args:
@@ -180,7 +178,7 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
options: Any options to be passed into the 'tryjob' command.
builders: All the builders to run the 'tryjob' with.
chroot_path: The absolute path to the chroot.
- log_level: The log level to be used for the command_executer.
+ verbose: Print command contents to `stdout`.
Returns:
A list that contains stdout contents of each tryjob, where stdout is
@@ -191,8 +189,6 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
ValueError: Failed to submit a tryjob.
"""
- ce = command_executer.GetCommandExecuter(log_level=log_level)
-
# Contains the results of each tryjob. The results are retrieved from 'out'
# which is stdout of the command executer.
tryjob_results = []
@@ -206,23 +202,13 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
tryjob_cmd = GetTryJobCommand(cl_number, extra_change_lists, options,
cur_builder)
- ret, out, err = ce.ChrootRunCommandWOutput(
- chromeos_root=chroot_path,
- command=tryjob_cmd,
- print_to_console=log_level == 'verbose')
-
- if ret: # Failed to submit a tryjob.
- print(err, file=sys.stderr)
- raise ValueError('Failed to submit tryjob.')
-
- # stderr can be noisy e.g. warnings when entering chroot, so ignore it.
- # e.g. cros_sdk:enter_chroot: Gclient cache dir "/tmp/git-cache" is not...
+ out = ChrootRunCommand(chroot_path, tryjob_cmd, verbose=verbose)
tryjob_launch_time = datetime.datetime.utcnow()
- buildbucket_id = int(ParseTryjobBuildbucketId(out.rstrip()))
+ tryjob_contents = json.loads(out)
- tryjob_contents = json.loads(out.rstrip())
+ buildbucket_id = int(tryjob_contents[0]['buildbucket_id'])
new_tryjob = {
'launch_time': str(tryjob_launch_time),
@@ -235,27 +221,31 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
tryjob_results.append(new_tryjob)
- AddTryjobLinkToCL(tryjob_results, cl_number, chroot_path, log_level)
+ AddTryjobLinkToCL(tryjob_results, cl_number, chroot_path)
return tryjob_results
-def AddTryjobLinkToCL(tryjobs, cl, chroot_path, log_level):
+def AddTryjobLinkToCL(tryjobs, cl, chroot_path):
"""Adds the tryjob link(s) to the CL via `gerrit message <CL> <message>`."""
- tryjob_links = ['Started the following tryjobs:']
- tryjob_links.extend(quote(tryjob['link']) for tryjob in tryjobs)
+ # NOTE: Invoking `cros_sdk` does not make each tryjob link appear on its own
+ # line, so invoking the `gerrit` command directly instead of using `cros_sdk`
+ # to do it for us.
+ #
+ # FIXME: Need to figure out why `cros_sdk` does not add each tryjob link as a
+ # newline.
+ gerrit_abs_path = os.path.join(chroot_path, 'chromite/bin/gerrit')
- add_message_cmd = 'gerrit message %d \"%s\"' % (cl, '\n'.join(tryjob_links))
+ tryjob_links = ['Started the following tryjobs:']
+ tryjob_links.extend(tryjob['link'] for tryjob in tryjobs)
- ce = command_executer.GetCommandExecuter(log_level=log_level)
- ret, _, err = ce.ChrootRunCommandWOutput(
- chromeos_root=chroot_path,
- command=add_message_cmd,
- print_to_console=log_level == 'verbose')
+ add_message_cmd = [
+ gerrit_abs_path, 'message',
+ str(cl), '\n'.join(tryjob_links)
+ ]
- if ret: # Failed to add tryjob link(s) to CL.
- raise ValueError('Failed to add tryjob links to CL %d: %s' % (cl, err))
+ ExecCommandAndCaptureOutput(add_message_cmd)
def main():
@@ -289,7 +279,7 @@ def main():
(svn_version, last_svn_version, args_output.last_tested))
return
- update_chromeos_llvm_next_hash.ce.SetLogLevel(log_level=args_output.log_level)
+ update_chromeos_llvm_next_hash.verbose = args_output.verbose
change_list = update_chromeos_llvm_next_hash.UpdatePackages(
update_packages, git_hash, svn_version, args_output.chroot_path,
@@ -302,7 +292,7 @@ def main():
tryjob_results = RunTryJobs(change_list.cl_number,
args_output.extra_change_lists,
args_output.options, args_output.builders,
- args_output.chroot_path, args_output.log_level)
+ args_output.chroot_path, args_output.verbose)
print('Tryjobs:')
for tryjob in tryjob_results:
diff --git a/llvm_tools/update_tryjob_status.py b/llvm_tools/update_tryjob_status.py
index 4648db12..38eab8e5 100755
--- a/llvm_tools/update_tryjob_status.py
+++ b/llvm_tools/update_tryjob_status.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -16,8 +16,7 @@ import subprocess
import sys
from assert_not_in_chroot import VerifyOutsideChroot
-from cros_utils import command_executer
-from patch_manager import _ConvertToASCII
+from subprocess_helpers import ChrootRunCommand
from test_helpers import CreateTemporaryJsonFile
@@ -97,7 +96,7 @@ def GetCommandLineArgs():
'--set_status',
required=True,
choices=[tryjob_status.value for tryjob_status in TryjobStatus],
- help='Sets the \'status\' field of the tryjob.')
+ help='Sets the "status" field of the tryjob.')
# Add argument that determines which revision to search for in the list of
# tryjobs.
@@ -118,7 +117,7 @@ def GetCommandLineArgs():
parser.add_argument(
'--custom_script',
help='The absolute path to the custom script to execute (its exit code '
- 'should be %d for \'good\', %d for \'bad\', or %d for \'skip\')' %
+ 'should be %d for "good", %d for "bad", or %d for "skip")' %
(CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value,
CustomScriptStatus.SKIP.value))
@@ -126,7 +125,7 @@ def GetCommandLineArgs():
if not os.path.isfile(args_output.status_file) or \
not args_output.status_file.endswith('.json'):
- raise ValueError('File does not exist or does not ending in \'.json\' '
+ raise ValueError('File does not exist or does not ending in ".json" '
': %s' % args_output.status_file)
if args_output.set_status == TryjobStatus.CUSTOM_SCRIPT.value and \
@@ -169,19 +168,12 @@ def FindTryjobIndex(revision, tryjobs_list):
def GetStatusFromCrosBuildResult(chroot_path, buildbucket_id):
"""Retrieves the 'status' using 'cros buildresult'."""
- ce = command_executer.GetCommandExecuter(log_level=None)
+ get_buildbucket_id_cmd = [
+ 'cros', 'buildresult', '--buildbucket-id',
+ str(buildbucket_id), '--report', 'json'
+ ]
- get_buildbucket_id_cmd = ('cros buildresult --buildbucket-id %d '
- '--report json' % buildbucket_id)
-
- ret, tryjob_json, err = ce.ChrootRunCommandWOutput(
- chromeos_root=chroot_path,
- command=get_buildbucket_id_cmd,
- print_to_console=False)
-
- if ret: # Failed to get the contents of the tryjob.
- raise ValueError('Failed to get JSON contents of the tryjobs buildbucket '
- 'ID %d: %s' % (buildbucket_id, err))
+ tryjob_json = ChrootRunCommand(chroot_path, get_buildbucket_id_cmd)
tryjob_contents = json.loads(tryjob_json)
@@ -197,7 +189,7 @@ def GetAutoResult(chroot_path, buildbucket_id):
# The string returned by 'cros buildresult' might not be in the mapping.
if build_result not in builder_status_mapping:
raise ValueError(
- '\'cros buildresult\' return value is invalid: %s' % build_result)
+ '"cros buildresult" return value is invalid: %s' % build_result)
return builder_status_mapping[build_result]
@@ -243,8 +235,8 @@ def GetCustomScriptResult(custom_script, status_file, tryjob_contents):
raise ValueError(
'Custom script %s exit code %d did not match '
- 'any of the expected exit codes: %d for \'good\', %d '
- 'for \'bad\', or %d for \'skip\'.\nPlease check %s for information '
+ 'any of the expected exit codes: %d for "good", %d '
+ 'for "bad", or %d for "skip".\nPlease check %s for information '
'about the tryjob: %s' %
(custom_script, exec_script_cmd_obj.returncode,
CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value,
@@ -281,7 +273,7 @@ def UpdateTryjobStatus(revision, set_status, status_file, chroot_path,
# ]
# }
with open(status_file) as tryjobs:
- bisect_contents = _ConvertToASCII(json.load(tryjobs))
+ bisect_contents = json.load(tryjobs)
if not bisect_contents['jobs']:
sys.exit('No tryjobs in %s' % status_file)
@@ -309,7 +301,7 @@ def UpdateTryjobStatus(revision, set_status, status_file, chroot_path,
bisect_contents['jobs'][tryjob_index]['status'] = GetCustomScriptResult(
custom_script, status_file, bisect_contents['jobs'][tryjob_index])
else:
- raise ValueError('Invalid \'set_status\' option provided: %s' % set_status)
+ raise ValueError('Invalid "set_status" option provided: %s' % set_status)
with open(status_file, 'w') as update_tryjobs:
json.dump(bisect_contents, update_tryjobs, indent=4, separators=(',', ': '))