diff options
author | Salud Lemus <saludlemus@google.com> | 2019-08-02 17:20:58 -0700 |
---|---|---|
committer | Manoj Gupta <manojgupta@chromium.org> | 2019-08-06 20:44:01 +0000 |
commit | c369e299c1421cc5687bead8fb3b74888d817d55 (patch) | |
tree | 061c79d7bc9ef1e330c4b82097f1d20902f44d98 /llvm_tools | |
parent | ad16df98e04ca5afeb51793a5a0eef71610df71a (diff) | |
download | toolchain-utils-c369e299c1421cc5687bead8fb3b74888d817d55.tar.gz |
LLVM tools: Added script that runs tryjobs after updating the packages
BUG=None
TEST=Ran the script and a CL was generated for ToT correctly and two
trybots were created, one for kevin and one for nocturne
Change-Id: I856c23d4368f5e45d9837e6f4341ae1f97f8f136
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1733650
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Manoj Gupta <manojgupta@chromium.org>
Diffstat (limited to 'llvm_tools')
-rwxr-xr-x | llvm_tools/get_llvm_hash.py | 61 | ||||
-rwxr-xr-x | llvm_tools/llvm_patch_management.py | 5 | ||||
-rwxr-xr-x | llvm_tools/update_chromeos_llvm_next_hash.py | 105 | ||||
-rwxr-xr-x | llvm_tools/update_packages_and_run_tryjobs.py | 267 |
4 files changed, 411 insertions, 27 deletions
diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py index 0a4f19c1..78affb23 100755 --- a/llvm_tools/get_llvm_hash.py +++ b/llvm_tools/get_llvm_hash.py @@ -12,6 +12,7 @@ from pipes import quote import argparse import os import re +import requests import shutil import tempfile @@ -48,6 +49,38 @@ def is_svn_option(svn_option): raise ValueError('Invalid LLVM git hash option provided: %s' % svn_option) +def GetLLVMHashAndVersionFromSVNOption(svn_option): + """Gets the LLVM hash and LLVM version based off of the svn option. + + Args: + svn_option: A valid svn option obtained from the command line. + Ex: 'google3', 'tot', or <svn_version> such as 365123. + + Returns: + A tuple that is the LLVM git hash and LLVM version. + """ + + new_llvm_hash = LLVMHash() + + # Determine which LLVM git hash to retrieve. + if svn_option == 'tot': + llvm_hash = new_llvm_hash.GetTopOfTrunkGitHash() + + tot_commit_message = new_llvm_hash.GetCommitMessageForHash(llvm_hash) + + llvm_version = new_llvm_hash.GetSVNVersionFromCommitMessage( + tot_commit_message) + else: + if isinstance(svn_option, int): + llvm_version = svn_option + else: + llvm_version = LLVMVersion().GetGoogle3LLVMVersion() + + llvm_hash = new_llvm_hash.GetLLVMHash(llvm_version) + + return llvm_hash, llvm_version + + class LLVMHash(object): """Provides three methods to retrieve a LLVM hash.""" @@ -76,6 +109,34 @@ class LLVMHash(object): if ret: # Failed to create repo. raise ValueError('Failed to clone the llvm repo: %s' % err) + def GetCommitMessageForHash(self, git_hash): + """Gets the commit message from the git hash. + + Args: + git_hash: A git hash of LLVM. + + Returns: + The commit message of the git hash. + + Raises: + ValueError: Unable to retrieve json contents from the LLVM commit URL. + """ + + llvm_commit_url = ('https://api.github.com/repos/llvm/llvm-project/git/' + 'commits/') + + commit_url = os.path.join(llvm_commit_url, git_hash) + + url_response = requests.get(commit_url) + + if not url_response: + raise ValueError('Failed to get response from url %s: Status Code %d' % + (commit_url, url_response.status_code)) + + unicode_json_contents = url_response.json() + + return str(unicode_json_contents['message']) + def GetSVNVersionFromCommitMessage(self, commit_message): """Gets the 'llvm-svn' from the commit message. diff --git a/llvm_tools/llvm_patch_management.py b/llvm_tools/llvm_patch_management.py index c8d21d45..e50b99dd 100755 --- a/llvm_tools/llvm_patch_management.py +++ b/llvm_tools/llvm_patch_management.py @@ -205,7 +205,10 @@ def UnpackLLVMPackage(chroot_path, package): unpack_cmd = 'sudo ebuild %s clean unpack' % quote(ebuild_path) ret, _, err = ce.ChrootRunCommandWOutput( - chromeos_root=chroot_path, command=unpack_cmd, print_to_console=False) + chromeos_root=chroot_path, + command=unpack_cmd, + print_to_console=False, + env=dict(os.environ, USE='llvm-next')) if ret: # Failed to unpack the package. raise ValueError('Failed to unpack the package %s: %s' % (package, err)) diff --git a/llvm_tools/update_chromeos_llvm_next_hash.py b/llvm_tools/update_chromeos_llvm_next_hash.py index 0827d215..418c1a21 100755 --- a/llvm_tools/update_chromeos_llvm_next_hash.py +++ b/llvm_tools/update_chromeos_llvm_next_hash.py @@ -12,6 +12,7 @@ for review. from __future__ import print_function +from collections import namedtuple from pipes import quote import argparse import llvm_patch_management @@ -20,8 +21,8 @@ import re from cros_utils import command_executer from failure_modes import FailureModes -from get_google3_llvm_version import LLVMVersion -from get_llvm_hash import LLVMHash +from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption +from get_llvm_hash import is_svn_option ce = command_executer.GetCommandExecuter() @@ -69,9 +70,12 @@ def GetCommandLineArgs(): # Add argument for the LLVM version to use. parser.add_argument( '--llvm_version', - type=int, - help='the LLVM version to use for retrieving the LLVM hash ' \ - '(default: uses the google3 llvm version)') + type=is_svn_option, + required=True, + help='which git hash of LLVM to find ' + '{google3, ToT, <svn_version>} ' + '(default: finds the git hash of the google3 LLVM ' + 'version)') # Add argument for the mode of the patch management when handling patches. parser.add_argument( @@ -383,6 +387,36 @@ def _DeleteRepo(path_to_repo_dir, llvm_hash): (llvm_hash, err)) +def GetGerritRepoUploadContents(repo_upload_contents): + """Parses 'repo upload' to get the Gerrit commit URL and CL number. + + Args: + repo_upload_contents: The contents of the 'repo upload' command. + + Returns: + A nametuple that has two (key, value) pairs, where the first pair is the + Gerrit commit URL and the second pair is the change list number. + + Raises: + ValueError: The contents of the 'repo upload' command did not contain a + Gerrit commit URL. + """ + + found_url = re.search( + r'https://chromium-review.googlesource.com/c/' + 'chromiumos/overlays/chromiumos-overlay/\+/([0-9]+)', + repo_upload_contents) + + if not found_url: + raise ValueError('Failed to find change list URL.') + + CommitContents = namedtuple('CommitContents', ['url', 'cl_number']) + + cl_number = int(found_url.group(1)) + + return CommitContents(url=found_url.group(0), cl_number=cl_number) + + def UploadChanges(path_to_repo_dir, llvm_hash, commit_messages): """Uploads the changes (updating LLVM next hash and uprev symlink) for review. @@ -392,6 +426,10 @@ def UploadChanges(path_to_repo_dir, llvm_hash, commit_messages): commit_messages: A string of commit message(s) (i.e. '-m [message]' of the changes made. + Returns: + A nametuple that has two (key, value) pairs, where the first pair is the + Gerrit commit URL and the second pair is the change list number. + Raises: ValueError: Failed to create a commit or failed to upload the changes for review. @@ -417,6 +455,8 @@ def UploadChanges(path_to_repo_dir, llvm_hash, commit_messages): if ret: # failed to upload the changes for review raise ValueError('Failed to upload changes for review: %s' % err) + return GetGerritRepoUploadContents(err) + def CreatePathDictionaryFromPackages(chroot_path, update_packages): """Creates a symlink and ebuild path pair dictionary from the packages. @@ -540,17 +580,15 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): return commit_messages -def UpdatePackages(paths_dict, llvm_hash, llvm_version, chroot_path, - patch_metadata_file, mode): +def UpdatePackages(packages, llvm_hash, llvm_version, chroot_path, + patch_metadata_file, mode, svn_option): """Updates the package's LLVM_NEXT_HASH and uprevs the ebuild. A temporary repo is created for the changes. The changes are then uploaded for review. Args: - paths_dict: A dictionary that has absolute paths where the - key is the absolute path to the symlink of the package and the - value is the absolute path to the ebuild of the package. + packages: A list of all the packages that are going to be updated. llvm_hash: The LLVM hash to use for 'LLVM_NEXT_HASH'. llvm_version: The LLVM version of the 'llvm_hash'. chroot_path: The absolute path to the chroot. @@ -559,14 +597,32 @@ def UpdatePackages(paths_dict, llvm_hash, llvm_version, chroot_path, mode: The mode of the patch manager when handling an applicable patch that failed to apply. Ex: 'FailureModes.FAIL' + svn_option: The git hash to use based off of the svn option. + Ex: 'google3', 'tot', or <svn_version> such as 365123 + + Returns: + A nametuple that has two (key, value) pairs, where the first pair is the + Gerrit commit URL and the second pair is the change list number. """ + # Construct a dictionary where the key is the absolute path of the symlink to + # the package and the value is the absolute path to the ebuild of the package. + paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages) + repo_path = os.path.dirname(paths_dict.itervalues().next()) _CreateRepo(repo_path, llvm_hash) try: - commit_message_header = 'llvm-next: Update packages to r%d' % llvm_version + if svn_option == 'google3': + commit_message_header = ( + 'llvm-next/google3: Update packages to r%d' % llvm_version) + elif svn_option == 'tot': + commit_message_header = ( + 'llvm-next/tot: Update packages to r%d' % llvm_version) + else: + commit_message_header = 'llvm-next: Update packages to r%d' % llvm_version + commit_messages = ['-m %s' % quote(commit_message_header)] commit_messages.append( @@ -606,34 +662,31 @@ def UpdatePackages(paths_dict, llvm_hash, llvm_version, chroot_path, commit_messages = StagePackagesPatchResultsForCommit( package_info_dict, commit_messages) - UploadChanges(repo_path, llvm_hash, ' '.join(commit_messages)) + change_list = UploadChanges(repo_path, llvm_hash, ' '.join(commit_messages)) finally: _DeleteRepo(repo_path, llvm_hash) + return change_list + def main(): """Updates the LLVM next hash for each package.""" args_output = GetCommandLineArgs() - # Construct a dictionary where the key is the absolute path of the symlink to - # the package and the value is the absolute path to the ebuild of the package. - paths_dict = CreatePathDictionaryFromPackages(args_output.chroot_path, - args_output.update_packages) + svn_option = args_output.llvm_version - # Get the google3 LLVM version if a LLVM version was not provided. - if not args_output.llvm_version: - args_output.llvm_version = LLVMVersion( - log_level=args_output.log_level).GetGoogle3LLVMVersion() + llvm_hash, llvm_version = GetLLVMHashAndVersionFromSVNOption(svn_option) - # Get the LLVM hash. - llvm_hash = LLVMHash(log_level=args_output.log_level).GetLLVMHash( - args_output.llvm_version) + change_list = UpdatePackages( + args_output.update_packages, llvm_hash, llvm_version, + args_output.chroot_path, args_output.patch_metadata_file, + FailureModes(args_output.failure_mode), svn_option) - UpdatePackages(paths_dict, llvm_hash, args_output.llvm_version, - args_output.chroot_path, args_output.patch_metadata_file, - FailureModes(args_output.failure_mode)) + print('Successfully updated packages to %d' % llvm_version) + print('Gerrit URL: %s' % change_list.url) + print('Change list number: %d' % change_list.cl_number) if __name__ == '__main__': diff --git a/llvm_tools/update_packages_and_run_tryjobs.py b/llvm_tools/update_packages_and_run_tryjobs.py new file mode 100755 index 00000000..672aee04 --- /dev/null +++ b/llvm_tools/update_packages_and_run_tryjobs.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs a tryjob/tryjobs after updating the packages.""" + +from __future__ import print_function + +import argparse +import os +import sys + +from cros_utils import command_executer +from failure_modes import FailureModes +from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption +from get_llvm_hash import is_svn_option +import update_chromeos_llvm_next_hash + + +def GetCommandLineArgs(): + """Parses the command line for the command line arguments. + + Returns: + The log level to use when retrieving the LLVM hash or google3 LLVM version, + the chroot path to use for executing chroot commands, + a list of a package or packages to update their LLVM next hash, + and the LLVM version to use when retrieving the LLVM hash. + """ + + # Default path to the chroot if a path is not specified. + cros_root = os.path.expanduser('~') + cros_root = os.path.join(cros_root, 'chromiumos') + + # Create parser and add optional command-line arguments. + parser = argparse.ArgumentParser( + description='Runs a tryjob if successfully updated packages\'' + '\'LLVM_NEXT_HASH\'.') + + # Add argument for the absolute path to the file that contains information on + # the previous tested svn version. + parser.add_argument( + '--last_tested', + help='the absolute path to the file that contains the last tested ' + 'svn version') + + # Add argument for other change lists that want to run alongside the tryjob + # which has a change list of updating a package's git hash. + parser.add_argument( + '--extra_change_lists', + type=int, + nargs='+', + help='change lists that would like to be run alongside the change list ' + 'of updating the packages') + + # Add argument for custom options for the tryjob. + parser.add_argument( + '--options', + required=False, + nargs='+', + help='options to use for the tryjob testing') + + # Add argument for builders for the tryjob. + parser.add_argument( + '--builders', + required=True, + nargs='+', + help='builders to use for the tryjob testing') + + # Add argument for the description of the tryjob. + parser.add_argument( + '--description', + required=False, + nargs='+', + help='the description of the tryjob') + + # Add argument for a specific chroot path. + parser.add_argument( + '--chroot_path', + default=cros_root, + help='the path to the chroot (default: %(default)s)') + + # Add argument for the log level. + parser.add_argument( + '--log_level', + default='none', + choices=['none', 'quiet', 'average', 'verbose'], + help='the level for the logs (default: %(default)s)') + + # Add argument for the LLVM version to use. + parser.add_argument( + '--llvm_version', + type=is_svn_option, + required=True, + help='which git hash of LLVM to find ' + '{google3, ToT, <svn_version>} ' + '(default: finds the git hash of the google3 LLVM ' + 'version)') + + args_output = parser.parse_args() + + return args_output + + +def GetLastTestedSVNVersion(last_tested_file): + """Gets the lasted tested svn version from the file. + + Args: + last_tested_file: The absolute path to the file that contains the last + tested svn version. + + Returns: + The last tested svn version or 'None' if the file did not have a last tested + svn version (the file exists, but failed to convert the contents to an + integer) or the file does not exist. + """ + + if not last_tested_file: + return None + + last_svn_version = None + + # Get the last tested svn version if the file exists. + try: + with open(last_tested_file) as file_obj: + # For now, the first line contains the last tested svn version. + return int(file_obj.read().rstrip()) + + except IOError: + pass + except ValueError: + pass + + return last_svn_version + + +def GetTryJobCommand(change_list, extra_change_lists, options, builder): + """Constructs the 'tryjob' command. + + Args: + change_list: The CL obtained from updating the packages. + extra_change_lists: Extra change lists that would like to be run alongside + the change list of updating the packages. + options: Options to be passed into the tryjob command. + builder: The builder to be passed into the tryjob command. + + Returns: + The 'tryjob' command with the change list of updating the packages and + any extra information that was passed into the command line. + """ + + tryjob_cmd = ['cros', 'tryjob', '--yes', '--json', '-g', '%d' % change_list] + + if extra_change_lists: + for extra_cl in extra_change_lists: + tryjob_cmd.extend(['-g', '%d' % extra_cl]) + + tryjob_cmd.append(builder) + + if options: + tryjob_cmd.extend('--%s' % option for option in options) + + return ' '.join(tryjob_cmd) + + +def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, + log_level): + """Runs a tryjob/tryjobs. + + Args: + cl_number: The CL created by updating the packages. + extra_change_lists: Any extra change lists that would run alongside the CL + that was created by updating the packages ('cl_number'). + options: Any options to be passed into the 'tryjob' command. + builders: All the builders to run the 'tryjob' with. + chroot_path: The absolute path to the chroot. + log_level: The log level to be used for the command_executer. + + Returns: + A list that contains stdout contents of each tryjob, where stdout is + information (a hashmap) about the tryjob. The hashmap also contains stderr + if there was an error when running a tryjob. + + Raises: + ValueError: Failed to submit a tryjob. + """ + + ce = command_executer.GetCommandExecuter(log_level=log_level) + + # Contains the results of each tryjob. The results are retrieved from 'out' + # which is stdout of the command executer. + tryjob_results = [] + + # For each builder passed into the command line: + # + # Run a tryjob with the change list number obtained from updating the + # packages and append additional changes lists and options obtained from the + # command line. + for cur_builder in builders: + tryjob_cmd = GetTryJobCommand(cl_number, extra_change_lists, options, + cur_builder) + + ret, out, err = ce.ChrootRunCommandWOutput( + chromeos_root=chroot_path, command=tryjob_cmd, print_to_console=False) + + if ret: # Failed to submit a tryjob. + print(err, file=sys.stderr) + raise ValueError('Failed to submit tryjob.') + + assert not err, 'Unexpected contents in stderr: %s' % err + + tryjob_results.append(out.rstrip()) + + return tryjob_results + + +def main(): + args_output = GetCommandLineArgs() + + last_svn_version = GetLastTestedSVNVersion(args_output.last_tested) + + update_packages = [ + 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', + 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' + ] + + patch_metadata_file = 'PATCHES.json' + + svn_option = args_output.llvm_version + + git_hash, svn_version = GetLLVMHashAndVersionFromSVNOption(svn_option) + + # There is no need to run tryjobs when the SVN version matches the last tested + # SVN version. + if last_svn_version == svn_version: + print('svn version (%d) matches the last tested svn version (%d) in %s' % + (svn_version, last_svn_version, args_output.last_tested)) + return + + update_chromeos_llvm_next_hash.ce.SetLogLevel(log_level=args_output.log_level) + + change_list = update_chromeos_llvm_next_hash.UpdatePackages( + update_packages, git_hash, svn_version, args_output.chroot_path, + patch_metadata_file, FailureModes.DISABLE_PATCHES, svn_option) + + print('Successfully updated packages to %d' % svn_version) + print('Gerrit URL: %s' % change_list.url) + print('Change list number: %d' % change_list.cl_number) + + tryjob_results = RunTryJobs(change_list.cl_number, + args_output.extra_change_lists, + args_output.options, args_output.builders, + args_output.chroot_path, args_output.log_level) + + print('Tryjobs:') + print('\n'.join(tryjob_results)) + + # Updated the packages and submitted tryjobs successfully, so the file will + # contain 'svn_version' which will now become the last tested svn version. + if args_output.last_tested: + with open(args_output.last_tested, 'w') as file_obj: + file_obj.write(str(svn_version)) + + +if __name__ == '__main__': + main() |