diff options
Diffstat (limited to 'llvm_tools/get_upstream_patch.py')
| -rwxr-xr-x | llvm_tools/get_upstream_patch.py | 465 |
1 files changed, 465 insertions, 0 deletions
diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py new file mode 100755 index 00000000..5669b023 --- /dev/null +++ b/llvm_tools/get_upstream_patch.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Get an upstream patch to LLVM's PATCHES.json.""" + +import argparse +import json +import logging +import os +import shlex +import subprocess +import sys +import typing as t +from datetime import datetime + +import dataclasses + +import chroot +import get_llvm_hash +import git +import git_llvm_rev +import update_chromeos_llvm_hash + +__DOC_EPILOGUE = """ +Example Usage: + get_upstream_patch --chroot_path ~/chromiumos --platform chromiumos \ +--sha 1234567 --sha 890abdc +""" + + +class CherrypickError(ValueError): + """A ValueError that highlights the cherry-pick has been seen before""" + + +def add_patch(patches_json_path: str, patches_dir: str, + relative_patches_dir: str, start_version: git_llvm_rev.Rev, + llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, str], sha: str, + package: str, platforms: t.List[str]): + """Gets the start and end intervals in 'json_file'. + + Args: + patches_json_path: The absolute path to PATCHES.json. + patches_dir: The aboslute path to the directory patches are in. + relative_patches_dir: The relative path to PATCHES.json. + start_version: The base LLVM revision this patch applies to. + llvm_dir: The path to LLVM checkout. + rev: An LLVM revision (git_llvm_rev.Rev) for a cherrypicking, or a + differential revision (str) otherwise. + sha: The LLVM git sha that corresponds to the patch. For differential + revisions, the git sha from the local commit created by 'arc patch' + is used. + package: The LLVM project name this patch applies to. + platforms: List of platforms this patch applies to. + + Raises: + CherrypickError: A ValueError that highlights the cherry-pick has been + seen before. + """ + + with open(patches_json_path, encoding='utf-8') as f: + patches_json = json.load(f) + + is_cherrypick = isinstance(rev, git_llvm_rev.Rev) + if is_cherrypick: + file_name = f'{sha}.patch' + else: + file_name = f'{rev}.patch' + rel_patch_path = os.path.join(relative_patches_dir, file_name) + + for p in patches_json: + rel_path = p['rel_patch_path'] + if rel_path == rel_patch_path: + raise CherrypickError( + f'Patch at {rel_path} already exists in PATCHES.json') + if is_cherrypick: + if sha in rel_path: + logging.warning( + 'Similarly-named patch already exists in PATCHES.json: %r', + rel_path) + + with open(os.path.join(patches_dir, file_name), 'wb') as f: + cmd = ['git', 'show', sha] + # Only apply the part of the patch that belongs to this package, expect + # LLVM. This is because some packages are built with LLVM ebuild on X86 but + # not on the other architectures. e.g. compiler-rt. Therefore always apply + # the entire patch to LLVM ebuild as a workaround. + if package != 'llvm': + cmd.append(package_to_project(package)) + subprocess.check_call(cmd, stdout=f, cwd=llvm_dir) + + commit_subject = subprocess.check_output( + ['git', 'log', '-n1', '--format=%s', sha], + cwd=llvm_dir, + encoding='utf-8') + + end_vers = rev.number if isinstance(rev, git_llvm_rev.Rev) else None + patch_props = { + 'rel_patch_path': rel_patch_path, + 'metadata': { + 'title': commit_subject.strip(), + 'info': [], + }, + 'platforms': sorted(platforms), + 'version_range': { + 'from': start_version.number, + 'until': end_vers, + }, + } + patches_json.append(patch_props) + + temp_file = patches_json_path + '.tmp' + with open(temp_file, 'w', encoding='utf-8') as f: + json.dump(patches_json, + f, + indent=4, + separators=(',', ': '), + sort_keys=True) + f.write('\n') + os.rename(temp_file, patches_json_path) + + +def parse_ebuild_for_assignment(ebuild_path: str, var_name: str) -> str: + # '_pre' filters the LLVM 9.0 ebuild, which we never want to target, from + # this list. + candidates = [ + x for x in os.listdir(ebuild_path) + if x.endswith('.ebuild') and '_pre' in x + ] + + if not candidates: + raise ValueError('No ebuilds found under %r' % ebuild_path) + + ebuild = os.path.join(ebuild_path, max(candidates)) + with open(ebuild, encoding='utf-8') as f: + var_name_eq = var_name + '=' + for orig_line in f: + if not orig_line.startswith(var_name_eq): + continue + + # We shouldn't see much variety here, so do the simplest thing possible. + line = orig_line[len(var_name_eq):] + # Remove comments + line = line.split('#')[0] + # Remove quotes + line = shlex.split(line) + if len(line) != 1: + raise ValueError('Expected exactly one quoted value in %r' % orig_line) + return line[0].strip() + + raise ValueError('No %s= line found in %r' % (var_name, ebuild)) + + +# Resolves a git ref (or similar) to a LLVM SHA. +def resolve_llvm_ref(llvm_dir: str, sha: str) -> str: + return subprocess.check_output( + ['git', 'rev-parse', sha], + encoding='utf-8', + cwd=llvm_dir, + ).strip() + + +# Get the package name of an LLVM project +def project_to_package(project: str) -> str: + if project == 'libunwind': + return 'llvm-libunwind' + return project + + +# Get the LLVM project name of a package +def package_to_project(package: str) -> str: + if package == 'llvm-libunwind': + return 'libunwind' + return package + + +# Get the LLVM projects change in the specifed sha +def get_package_names(sha: str, llvm_dir: str) -> list: + paths = subprocess.check_output( + ['git', 'show', '--name-only', '--format=', sha], + cwd=llvm_dir, + encoding='utf-8').splitlines() + # Some LLVM projects are built by LLVM ebuild on X86, so always apply the + # patch to LLVM ebuild + packages = {'llvm'} + # Detect if there are more packages to apply the patch to + for path in paths: + package = project_to_package(path.split('/')[0]) + if package in ('compiler-rt', 'libcxx', 'libcxxabi', 'llvm-libunwind'): + packages.add(package) + packages = list(sorted(packages)) + return packages + + +def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], + start_rev: git_llvm_rev.Rev, + rev: t.Union[git_llvm_rev.Rev, str], sha: str, + llvm_dir: str, platforms: t.List[str]): + """Create a patch and add its metadata for each package""" + for package, symlink in zip(packages, symlinks): + symlink_dir = os.path.dirname(symlink) + patches_json_path = os.path.join(symlink_dir, 'files/PATCHES.json') + relative_patches_dir = 'cherry' if package == 'llvm' else '' + patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir) + logging.info('Getting %s (%s) into %s', rev, sha, package) + add_patch(patches_json_path, + patches_dir, + relative_patches_dir, + start_rev, + llvm_dir, + rev, + sha, + package, + platforms=platforms) + + +def make_cl(symlinks_to_uprev: t.List[str], llvm_symlink_dir: str, branch: str, + commit_messages: t.List[str], reviewers: t.Optional[t.List[str]], + cc: t.Optional[t.List[str]]): + symlinks_to_uprev = sorted(set(symlinks_to_uprev)) + for symlink in symlinks_to_uprev: + update_chromeos_llvm_hash.UprevEbuildSymlink(symlink) + subprocess.check_output(['git', 'add', '--all'], + cwd=os.path.dirname(symlink)) + git.UploadChanges(llvm_symlink_dir, branch, commit_messages, reviewers, cc) + git.DeleteBranch(llvm_symlink_dir, branch) + + +def resolve_symbolic_sha(start_sha: str, llvm_symlink_dir: str) -> str: + if start_sha == 'llvm': + return parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_HASH') + + if start_sha == 'llvm-next': + return parse_ebuild_for_assignment(llvm_symlink_dir, 'LLVM_NEXT_HASH') + + return start_sha + + +def find_patches_and_make_cl( + chroot_path: str, patches: t.List[str], start_rev: git_llvm_rev.Rev, + llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str, + create_cl: bool, skip_dependencies: bool, + reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]], + platforms: t.List[str]): + + converted_patches = [ + _convert_patch(llvm_config, skip_dependencies, p) for p in patches + ] + potential_duplicates = _get_duplicate_shas(converted_patches) + if potential_duplicates: + err_msg = '\n'.join(f'{a.patch} == {b.patch}' + for a, b in potential_duplicates) + raise RuntimeError(f'Found Duplicate SHAs:\n{err_msg}') + + # CL Related variables, only used if `create_cl` + symlinks_to_uprev = [] + commit_messages = [ + 'llvm: get patches from upstream\n', + ] + branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' + + if create_cl: + git.CreateBranch(llvm_symlink_dir, branch) + + for parsed_patch in converted_patches: + # Find out the llvm projects changed in this commit + packages = get_package_names(parsed_patch.sha, llvm_config.dir) + # Find out the ebuild symlinks of the corresponding ChromeOS packages + symlinks = chroot.GetChrootEbuildPaths(chroot_path, [ + 'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package + for package in packages + ]) + symlinks = chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, symlinks) + # Create a local patch for all the affected llvm projects + create_patch_for_packages(packages, + symlinks, + start_rev, + parsed_patch.rev, + parsed_patch.sha, + llvm_config.dir, + platforms=platforms) + if create_cl: + symlinks_to_uprev.extend(symlinks) + + commit_messages.extend([ + parsed_patch.git_msg(), + subprocess.check_output( + ['git', 'log', '-n1', '--oneline', parsed_patch.sha], + cwd=llvm_config.dir, + encoding='utf-8') + ]) + + if parsed_patch.is_differential: + subprocess.check_output(['git', 'reset', '--hard', 'HEAD^'], + cwd=llvm_config.dir) + + if create_cl: + make_cl(symlinks_to_uprev, llvm_symlink_dir, branch, commit_messages, + reviewers, cc) + + +@dataclasses.dataclass(frozen=True) +class ParsedPatch: + """Class to keep track of bundled patch info.""" + patch: str + sha: str + is_differential: bool + rev: t.Union[git_llvm_rev.Rev, str] + + def git_msg(self) -> str: + if self.is_differential: + return f'\n\nreviews.llvm.org/{self.patch}\n' + return f'\n\nreviews.llvm.org/rG{self.sha}\n' + + +def _convert_patch(llvm_config: git_llvm_rev.LLVMConfig, + skip_dependencies: bool, patch: str) -> ParsedPatch: + """Extract git revision info from a patch. + + Args: + llvm_config: LLVM configuration object. + skip_dependencies: Pass --skip-dependecies for to `arc` + patch: A single patch referent string. + + Returns: + A [ParsedPatch] object. + """ + + # git hash should only have lower-case letters + is_differential = patch.startswith('D') + if is_differential: + subprocess.check_output( + [ + 'arc', 'patch', '--nobranch', + '--skip-dependencies' if skip_dependencies else '--revision', patch + ], + cwd=llvm_config.dir, + ) + sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') + rev = patch + else: + sha = resolve_llvm_ref(llvm_config.dir, patch) + rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + return ParsedPatch(patch=patch, + sha=sha, + rev=rev, + is_differential=is_differential) + + +def _get_duplicate_shas(patches: t.List[ParsedPatch] + ) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]: + """Return a list of Patches which have duplicate SHA's""" + return [(left, right) for i, left in enumerate(patches) + for right in patches[i + 1:] if left.sha == right.sha] + + +def get_from_upstream(chroot_path: str, + create_cl: bool, + start_sha: str, + patches: t.List[str], + platforms: t.List[str], + skip_dependencies: bool = False, + reviewers: t.List[str] = None, + cc: t.List[str] = None): + llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( + chroot_path, chroot.GetChrootEbuildPaths(chroot_path, + ['sys-devel/llvm']))[0] + llvm_symlink_dir = os.path.dirname(llvm_symlink) + + git_status = subprocess.check_output(['git', 'status', '-s'], + cwd=llvm_symlink_dir, + encoding='utf-8') + + if git_status: + error_path = os.path.dirname(os.path.dirname(llvm_symlink_dir)) + raise ValueError(f'Uncommited changes detected in {error_path}') + + start_sha = resolve_symbolic_sha(start_sha, llvm_symlink_dir) + logging.info('Base llvm hash == %s', start_sha) + + llvm_config = git_llvm_rev.LLVMConfig( + remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) + start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) + + find_patches_and_make_cl(chroot_path=chroot_path, + patches=patches, + platforms=platforms, + start_rev=git_llvm_rev.translate_sha_to_rev( + llvm_config, start_sha), + llvm_config=llvm_config, + llvm_symlink_dir=llvm_symlink_dir, + create_cl=create_cl, + skip_dependencies=skip_dependencies, + reviewers=reviewers, + cc=cc) + logging.info('Complete.') + + +def main(): + chroot.VerifyOutsideChroot() + logging.basicConfig( + format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s', + level=logging.INFO, + ) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__DOC_EPILOGUE) + parser.add_argument('--chroot_path', + default=os.path.join(os.path.expanduser('~'), + 'chromiumos'), + help='the path to the chroot (default: %(default)s)') + parser.add_argument( + '--start_sha', + default='llvm-next', + help='LLVM SHA that the patch should start applying at. You can specify ' + '"llvm" or "llvm-next", as well. Defaults to %(default)s.') + parser.add_argument('--sha', + action='append', + default=[], + help='The LLVM git SHA to cherry-pick.') + parser.add_argument( + '--differential', + action='append', + default=[], + help='The LLVM differential revision to apply. Example: D1234') + parser.add_argument( + '--platform', + action='append', + required=True, + help='Apply this patch to the give platform. Common options include ' + '"chromiumos" and "android". Can be specified multiple times to ' + 'apply to multiple platforms') + parser.add_argument('--create_cl', + action='store_true', + help='Automatically create a CL if specified') + parser.add_argument( + '--skip_dependencies', + action='store_true', + help="Skips a LLVM differential revision's dependencies. Only valid " + 'when --differential appears exactly once.') + args = parser.parse_args() + + if not (args.sha or args.differential): + parser.error('--sha or --differential required') + + if args.skip_dependencies and len(args.differential) != 1: + parser.error("--skip_dependencies is only valid when there's exactly one " + 'supplied differential') + + get_from_upstream( + chroot_path=args.chroot_path, + create_cl=args.create_cl, + start_sha=args.start_sha, + patches=args.sha + args.differential, + skip_dependencies=args.skip_dependencies, + platforms=args.platform, + ) + + +if __name__ == '__main__': + sys.exit(main()) |