diff options
Diffstat (limited to 'llvm_tools/get_upstream_patch.py')
| -rwxr-xr-x | llvm_tools/get_upstream_patch.py | 249 |
1 files changed, 176 insertions, 73 deletions
diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py index 77783d41..7a4be3eb 100755 --- a/llvm_tools/get_upstream_patch.py +++ b/llvm_tools/get_upstream_patch.py @@ -6,8 +6,6 @@ """Get an upstream patch to LLVM's PATCHES.json.""" -from __future__ import print_function - import argparse import json import logging @@ -18,12 +16,20 @@ import sys import typing as t from datetime import datetime +import dataclasses + import chroot import get_llvm_hash import git import git_llvm_rev import update_chromeos_llvm_hash +__DOC_EPILOGUE = """ +Example Usage: + get_upstream_patch --chroot_path ~/chromiumos --platform chromiumos \ +--sha 1234567 --sha 890abdc +""" + class CherrypickError(ValueError): """A ValueError that highlights the cherry-pick has been seen before""" @@ -32,7 +38,7 @@ class CherrypickError(ValueError): def add_patch(patches_json_path: str, patches_dir: str, relative_patches_dir: str, start_version: git_llvm_rev.Rev, llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, str], sha: str, - package: str): + package: str, platforms: t.List[str]): """Gets the start and end intervals in 'json_file'. Args: @@ -47,6 +53,7 @@ def add_patch(patches_json_path: str, patches_dir: str, revisions, the git sha from the local commit created by 'arc patch' is used. package: The LLVM project name this patch applies to. + platforms: List of platforms this patch applies to. Raises: CherrypickError: A ValueError that highlights the cherry-pick has been @@ -85,20 +92,29 @@ def add_patch(patches_json_path: str, patches_dir: str, subprocess.check_call(cmd, stdout=f, cwd=llvm_dir) commit_subject = subprocess.check_output( - ['git', 'log', '-n1', '--format=%s', sha], cwd=llvm_dir, encoding='utf-8') + ['git', 'log', '-n1', '--format=%s', sha], + cwd=llvm_dir, + encoding='utf-8') - patch_metadata = { - 'comment': commit_subject.strip(), + patch_props = { 'rel_patch_path': rel_patch_path, 'start_version': start_version.number, + 'metadata': { + 'title': commit_subject.strip(), + 'info': [], + }, + 'platforms': sorted(platforms), + 'end_version': rev.number if isinstance(rev, git_llvm_rev.Rev) else None, } - if isinstance(rev, git_llvm_rev.Rev): - patch_metadata['end_version'] = rev.number - patches_json.append(patch_metadata) + patches_json.append(patch_props) temp_file = patches_json_path + '.tmp' with open(temp_file, 'w', encoding='utf-8') as f: - json.dump(patches_json, f, indent=4, separators=(',', ': ')) + json.dump(patches_json, + f, + indent=4, + separators=(',', ': '), + sort_keys=True) f.write('\n') os.rename(temp_file, patches_json_path) @@ -178,7 +194,7 @@ def get_package_names(sha: str, llvm_dir: str) -> list: def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], start_rev: git_llvm_rev.Rev, rev: t.Union[git_llvm_rev.Rev, str], sha: str, - llvm_dir: str): + llvm_dir: str, platforms: t.List[str]): """Create a patch and add its metadata for each package""" for package, symlink in zip(packages, symlinks): symlink_dir = os.path.dirname(symlink) @@ -186,8 +202,15 @@ def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], relative_patches_dir = 'cherry' if package == 'llvm' else '' patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir) logging.info('Getting %s (%s) into %s', rev, sha, package) - add_patch(patches_json_path, patches_dir, relative_patches_dir, start_rev, - llvm_dir, rev, sha, package) + add_patch(patches_json_path, + patches_dir, + relative_patches_dir, + start_rev, + llvm_dir, + rev, + sha, + package, + platforms=platforms) def make_cl(symlinks_to_uprev: t.List[str], llvm_symlink_dir: str, branch: str, @@ -212,35 +235,35 @@ def resolve_symbolic_sha(start_sha: str, llvm_symlink_dir: str) -> str: return start_sha -def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], - start_rev: git_llvm_rev.Rev, - llvm_config: git_llvm_rev.LLVMConfig, - llvm_symlink_dir: str, create_cl: bool, - reviewers: t.Optional[t.List[str]], - cc: t.Optional[t.List[str]]): +def find_patches_and_make_cl( + chroot_path: str, patches: t.List[str], start_rev: git_llvm_rev.Rev, + llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str, + create_cl: bool, skip_dependencies: bool, + reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]], + platforms: t.List[str]): + + converted_patches = [ + _convert_patch(llvm_config, skip_dependencies, p) for p in patches + ] + potential_duplicates = _get_duplicate_shas(converted_patches) + if potential_duplicates: + err_msg = '\n'.join(f'{a.patch} == {b.patch}' + for a, b in potential_duplicates) + raise RuntimeError(f'Found Duplicate SHAs:\n{err_msg}') + + # CL Related variables, only used if `create_cl` + symlinks_to_uprev = [] + commit_messages = [ + 'llvm: get patches from upstream\n', + ] + branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' + if create_cl: - branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' git.CreateBranch(llvm_symlink_dir, branch) - symlinks_to_uprev = [] - commit_messages = [ - 'llvm: get patches from upstream\n', - ] - - for patch in patches: - # git hash should only have lower-case letters - is_differential = patch.startswith('D') - if is_differential: - subprocess.check_output( - ['arc', 'patch', '--nobranch', '--revision', patch], - cwd=llvm_config.dir, - ) - sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') - rev = patch - else: - sha = resolve_llvm_ref(llvm_config.dir, patch) - rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + + for parsed_patch in converted_patches: # Find out the llvm projects changed in this commit - packages = get_package_names(sha, llvm_config.dir) + packages = get_package_names(parsed_patch.sha, llvm_config.dir) # Find out the ebuild symlinks of the corresponding ChromeOS packages symlinks = chroot.GetChrootEbuildPaths(chroot_path, [ 'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package @@ -248,23 +271,25 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], ]) symlinks = chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, symlinks) # Create a local patch for all the affected llvm projects - create_patch_for_packages(packages, symlinks, start_rev, rev, sha, - llvm_config.dir) + create_patch_for_packages(packages, + symlinks, + start_rev, + parsed_patch.rev, + parsed_patch.sha, + llvm_config.dir, + platforms=platforms) if create_cl: symlinks_to_uprev.extend(symlinks) - if is_differential: - msg = f'\n\nreviews.llvm.org/{patch}\n' - else: - msg = f'\n\nreviews.llvm.org/rG{sha}\n' commit_messages.extend([ - msg, - subprocess.check_output(['git', 'log', '-n1', '--oneline', sha], - cwd=llvm_config.dir, - encoding='utf-8') + parsed_patch.git_msg(), + subprocess.check_output( + ['git', 'log', '-n1', '--oneline', parsed_patch.sha], + cwd=llvm_config.dir, + encoding='utf-8') ]) - if is_differential: + if parsed_patch.is_differential: subprocess.check_output(['git', 'reset', '--hard', 'HEAD^'], cwd=llvm_config.dir) @@ -273,10 +298,67 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], reviewers, cc) +@dataclasses.dataclass(frozen=True) +class ParsedPatch: + """Class to keep track of bundled patch info.""" + patch: str + sha: str + is_differential: bool + rev: t.Union[git_llvm_rev.Rev, str] + + def git_msg(self) -> str: + if self.is_differential: + return f'\n\nreviews.llvm.org/{self.patch}\n' + return f'\n\nreviews.llvm.org/rG{self.sha}\n' + + +def _convert_patch(llvm_config: git_llvm_rev.LLVMConfig, + skip_dependencies: bool, patch: str) -> ParsedPatch: + """Extract git revision info from a patch. + + Args: + llvm_config: LLVM configuration object. + skip_dependencies: Pass --skip-dependecies for to `arc` + patch: A single patch referent string. + + Returns: + A [ParsedPatch] object. + """ + + # git hash should only have lower-case letters + is_differential = patch.startswith('D') + if is_differential: + subprocess.check_output( + [ + 'arc', 'patch', '--nobranch', + '--skip-dependencies' if skip_dependencies else '--revision', patch + ], + cwd=llvm_config.dir, + ) + sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') + rev = patch + else: + sha = resolve_llvm_ref(llvm_config.dir, patch) + rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + return ParsedPatch(patch=patch, + sha=sha, + rev=rev, + is_differential=is_differential) + + +def _get_duplicate_shas( + patches: t.List[ParsedPatch]) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]: + """Return a list of Patches which have duplicate SHA's""" + return [(left, right) for i, left in enumerate(patches) + for right in patches[i + 1:] if left.sha == right.sha] + + def get_from_upstream(chroot_path: str, create_cl: bool, start_sha: str, patches: t.List[str], + platforms: t.List[str], + skip_dependencies: bool = False, reviewers: t.List[str] = None, cc: t.List[str] = None): llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( @@ -299,15 +381,17 @@ def get_from_upstream(chroot_path: str, remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) - find_patches_and_make_cl( - chroot_path=chroot_path, - patches=patches, - start_rev=git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha), - llvm_config=llvm_config, - llvm_symlink_dir=llvm_symlink_dir, - create_cl=create_cl, - reviewers=reviewers, - cc=cc) + find_patches_and_make_cl(chroot_path=chroot_path, + patches=patches, + platforms=platforms, + start_rev=git_llvm_rev.translate_sha_to_rev( + llvm_config, start_sha), + llvm_config=llvm_config, + llvm_symlink_dir=llvm_symlink_dir, + create_cl=create_cl, + skip_dependencies=skip_dependencies, + reviewers=reviewers, + cc=cc) logging.info('Complete.') @@ -318,41 +402,60 @@ def main(): level=logging.INFO, ) - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--chroot_path', - default=os.path.join(os.path.expanduser('~'), 'chromiumos'), - help='the path to the chroot (default: %(default)s)') + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__DOC_EPILOGUE) + parser.add_argument('--chroot_path', + default=os.path.join(os.path.expanduser('~'), + 'chromiumos'), + help='the path to the chroot (default: %(default)s)') parser.add_argument( '--start_sha', default='llvm-next', help='LLVM SHA that the patch should start applying at. You can specify ' '"llvm" or "llvm-next", as well. Defaults to %(default)s.') - parser.add_argument( - '--sha', - action='append', - default=[], - help='The LLVM git SHA to cherry-pick.') + parser.add_argument('--sha', + action='append', + default=[], + help='The LLVM git SHA to cherry-pick.') parser.add_argument( '--differential', action='append', default=[], help='The LLVM differential revision to apply. Example: D1234') parser.add_argument( - '--create_cl', - default=False, + '--platform', + action='append', + required=True, + help='Apply this patch to the give platform. Common options include ' + '"chromiumos" and "android". Can be specified multiple times to ' + 'apply to multiple platforms') + parser.add_argument('--create_cl', + action='store_true', + help='Automatically create a CL if specified') + parser.add_argument( + '--skip_dependencies', action='store_true', - help='Automatically create a CL if specified') + help="Skips a LLVM differential revision's dependencies. Only valid " + 'when --differential appears exactly once.') args = parser.parse_args() if not (args.sha or args.differential): parser.error('--sha or --differential required') + if args.skip_dependencies and len(args.differential) != 1: + parser.error("--skip_dependencies is only valid when there's exactly one " + 'supplied differential') + get_from_upstream( chroot_path=args.chroot_path, create_cl=args.create_cl, start_sha=args.start_sha, - patches=args.sha + args.differential) + patches=args.sha + args.differential, + skip_dependencies=args.skip_dependencies, + platforms=args.platform, + ) if __name__ == '__main__': |