diff options
Diffstat (limited to 'llvm_tools/cherrypick_cl.py')
| -rwxr-xr-x | llvm_tools/cherrypick_cl.py | 250 |
1 files changed, 250 insertions, 0 deletions
diff --git a/llvm_tools/cherrypick_cl.py b/llvm_tools/cherrypick_cl.py new file mode 100755 index 00000000..9e306725 --- /dev/null +++ b/llvm_tools/cherrypick_cl.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# pylint: disable=cros-logging-import + +"""Adds a cherrypick to LLVM's PATCHES.json.""" + +from __future__ import print_function + +import argparse +import json +import logging +import os +import shlex +import subprocess +import sys + +import chroot +import get_llvm_hash +import git +import git_llvm_rev +import update_chromeos_llvm_hash + + +def add_cherrypick(patches_json_path: str, patches_dir: str, + relative_patches_dir: str, start_version: git_llvm_rev.Rev, + llvm_dir: str, rev: git_llvm_rev.Rev, sha: str, + package: str): + with open(patches_json_path, encoding='utf-8') as f: + patches_json = json.load(f) + + file_name = sha + '.patch' + rel_patch_path = os.path.join(relative_patches_dir, file_name) + + for p in patches_json: + rel_path = p['rel_patch_path'] + if rel_path == rel_patch_path: + raise ValueError('Patch at %r already exists in PATCHES.json' % rel_path) + if sha in rel_path: + logging.warning( + 'Similarly-named patch already exists in PATCHES.json: %r', rel_path) + + with open(os.path.join(patches_dir, file_name), 'wb') as f: + cmd = ['git', 'show', sha] + # Only apply the part of the patch that belongs to this package, expect + # LLVM. This is because some packages are built with LLVM ebuild on X86 but + # not on the other architectures. e.g. compiler-rt. Therefore always apply + # the entire patch to LLVM ebuild as a workaround. + if package != 'llvm': + cmd.append(package_to_project(package)) + subprocess.check_call(cmd, stdout=f, cwd=llvm_dir) + + commit_subject = subprocess.check_output( + ['git', 'log', '-n1', '--format=%s', sha], cwd=llvm_dir, encoding='utf-8') + + patches_json.append({ + 'comment': commit_subject.strip(), + 'rel_patch_path': rel_patch_path, + 'start_version': start_version.number, + 'end_version': rev.number, + }) + + temp_file = patches_json_path + '.tmp' + with open(temp_file, 'w', encoding='utf-8') as f: + json.dump(patches_json, f, indent=4, separators=(',', ': ')) + os.rename(temp_file, patches_json_path) + + +def parse_ebuild_for_assignment(ebuild_path: str, var_name: str) -> str: + # '_pre' filters the LLVM 9.0 ebuild, which we never want to target, from + # this list. + candidates = [ + x for x in os.listdir(ebuild_path) + if x.endswith('.ebuild') and '_pre' in x + ] + + if not candidates: + raise ValueError('No ebuilds found under %r' % ebuild_path) + + ebuild = os.path.join(ebuild_path, max(candidates)) + with open(ebuild, encoding='utf-8') as f: + var_name_eq = var_name + '=' + for orig_line in f: + if not orig_line.startswith(var_name_eq): + continue + + # We shouldn't see much variety here, so do the simplest thing possible. + line = orig_line[len(var_name_eq):] + # Remove comments + line = line.split('#')[0] + # Remove quotes + line = shlex.split(line) + if len(line) != 1: + raise ValueError('Expected exactly one quoted value in %r' % orig_line) + return line[0].strip() + + raise ValueError('No %s= line found in %r' % (var_name, ebuild)) + + +# Resolves a git ref (or similar) to a LLVM SHA. +def resolve_llvm_ref(llvm_dir: str, sha: str) -> str: + return subprocess.check_output( + ['git', 'rev-parse', sha], + encoding='utf-8', + cwd=llvm_dir, + ).strip() + + +# Get the package name of an LLVM project +def project_to_package(project: str) -> str: + if project == 'libunwind': + return 'llvm-libunwind' + return project + + +# Get the LLVM project name of a package +def package_to_project(package: str) -> str: + if package == 'llvm-libunwind': + return 'libunwind' + return package + + +# Get the LLVM projects change in the specifed sha +def get_package_names(sha: str, llvm_dir: str) -> list: + paths = subprocess.check_output( + ['git', 'show', '--name-only', '--format=', sha], + cwd=llvm_dir, + encoding='utf-8').splitlines() + # Some LLVM projects are built by LLVM ebuild on X86, so always apply the + # patch to LLVM ebuild + packages = {'llvm'} + # Detect if there are more packages to apply the patch to + for path in paths: + package = project_to_package(path.split('/')[0]) + if package in ('compiler-rt', 'libcxx', 'libcxxabi', 'llvm-libunwind'): + packages.add(package) + packages = list(sorted(packages)) + return packages + + +def main(): + chroot.VerifyOutsideChroot() + logging.basicConfig( + format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s', + level=logging.INFO, + ) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--chroot_path', + default=os.path.join(os.path.expanduser('~'), 'chromiumos'), + help='the path to the chroot (default: %(default)s)') + parser.add_argument( + '--start_sha', + default='llvm-next', + help='LLVM SHA that the patch should start applying at. You can specify ' + '"llvm" or "llvm-next", as well. Defaults to %(default)s.') + parser.add_argument( + '--sha', + required=True, + action='append', + help='The LLVM git SHA to cherry-pick.') + parser.add_argument( + '--create_cl', + default=False, + action='store_true', + help='Automatically create a CL if specified') + args = parser.parse_args() + + llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( + args.chroot_path, + chroot.GetChrootEbuildPaths(args.chroot_path, ['sys-devel/llvm']))[0] + llvm_symlink_dir = os.path.dirname(llvm_symlink) + + git_status = subprocess.check_output(['git', 'status', '-s'], + cwd=llvm_symlink_dir, + encoding='utf-8') + if git_status: + raise ValueError('Uncommited changes detected in %s' % + os.path.dirname(os.path.dirname(llvm_symlink_dir))) + + start_sha = args.start_sha + if start_sha == 'llvm': + start_sha = parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_HASH') + elif start_sha == 'llvm-next': + start_sha = parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_NEXT_HASH') + logging.info('Base llvm hash == %s', start_sha) + + llvm_config = git_llvm_rev.LLVMConfig( + remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) + + start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) + start_rev = git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha) + + if args.create_cl: + branch = 'cherry-pick' + git.CreateBranch(llvm_symlink_dir, branch) + symlinks_to_uprev = [] + commit_messages = [ + 'llvm: cherry-pick CLs from upstream\n', + ] + + for sha in args.sha: + sha = resolve_llvm_ref(llvm_config.dir, sha) + rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + # Find out the llvm projects changed in this commit + packages = get_package_names(sha, llvm_config.dir) + # Find out the ebuild symlinks of the corresponding ChromeOS packages + symlinks = chroot.GetChrootEbuildPaths(args.chroot_path, [ + 'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package + for package in packages + ]) + symlinks = chroot.ConvertChrootPathsToAbsolutePaths(args.chroot_path, + symlinks) + + # Create a patch and add its metadata for each package + for package, symlink in zip(packages, symlinks): + symlink_dir = os.path.dirname(symlink) + patches_json_path = os.path.join(symlink_dir, 'files/PATCHES.json') + relative_patches_dir = 'cherry' if package == 'llvm' else '' + patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir) + logging.info('Cherrypicking %s (%s) into %s', rev, sha, package) + + add_cherrypick(patches_json_path, patches_dir, relative_patches_dir, + start_rev, llvm_config.dir, rev, sha, package) + if args.create_cl: + symlinks_to_uprev.extend(symlinks) + commit_messages.extend([ + '\n\nreviews.llvm.org/rG%s\n' % sha, + subprocess.check_output(['git', 'log', '-n1', '--oneline', sha], + cwd=llvm_config.dir, + encoding='utf-8') + ]) + + logging.info('Complete.') + + if args.create_cl: + symlinks_to_uprev = list(sorted(set(symlinks_to_uprev))) + for symlink in symlinks_to_uprev: + update_chromeos_llvm_hash.UprevEbuildSymlink(symlink) + subprocess.check_output(['git', 'add', '--all'], cwd=symlink_dir) + git.UploadChanges(llvm_symlink_dir, branch, commit_messages) + git.DeleteBranch(llvm_symlink_dir, branch) + + +if __name__ == '__main__': + sys.exit(main()) |