diff options
Diffstat (limited to 'llvm_tools/update_chromeos_llvm_hash.py')
| -rwxr-xr-x | llvm_tools/update_chromeos_llvm_hash.py | 401 |
1 files changed, 215 insertions, 186 deletions
diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py index 75c6ce6c..9a8754d4 100755 --- a/llvm_tools/update_chromeos_llvm_hash.py +++ b/llvm_tools/update_chromeos_llvm_hash.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -17,12 +16,15 @@ import os from pathlib import Path import re import subprocess -from typing import Dict, Iterable +import textwrap +from typing import Dict, Iterable, List, Optional +import atomic_write_file import chroot import failure_modes import get_llvm_hash import git +import manifest_utils import patch_utils import subprocess_helpers @@ -33,6 +35,7 @@ DEFAULT_PACKAGES = [ "sys-libs/compiler-rt", "sys-libs/libcxx", "sys-libs/llvm-libunwind", + "sys-libs/scudo", ] DEFAULT_MANIFEST_PACKAGES = ["sys-devel/llvm"] @@ -46,9 +49,87 @@ class LLVMVariant(enum.Enum): next = "LLVM_NEXT_HASH" -# If set to `True`, then the contents of `stdout` after executing a command will -# be displayed to the terminal. -verbose = False +class PortagePackage: + """Represents a portage package with location info.""" + + def __init__(self, chromeos_root: Path, package: str): + """Create a new PortagePackage. + + Args: + chromeos_root: Path to the root of the code checkout. + package: "category/package" string. + """ + self.package = package + potential_ebuild_path = PortagePackage.find_package_ebuild( + chromeos_root, package + ) + if potential_ebuild_path.is_symlink(): + self.uprev_target = potential_ebuild_path.absolute() + self.ebuild_path = potential_ebuild_path.resolve() + else: + # Should have a 9999 ebuild, no uprevs needed. + self.uprev_target = None + self.ebuild_path = potential_ebuild_path.absolute() + + @staticmethod + def find_package_ebuild(chromeos_root: Path, package: str) -> Path: + """Look up the package's ebuild location.""" + chromeos_root_str = str(chromeos_root) + ebuild_paths = chroot.GetChrootEbuildPaths( + chromeos_root_str, + [package], + ) + converted = chroot.ConvertChrootPathsToAbsolutePaths( + chromeos_root_str, ebuild_paths + )[0] + return Path(converted) + + def package_dir(self) -> Path: + """Return the package directory.""" + return self.ebuild_path.parent + + def update( + self, llvm_variant: LLVMVariant, git_hash: str, svn_version: int + ): + """Update the package with the new LLVM git sha and revision. + + Args: + llvm_variant: Which LLVM hash to update. + Either LLVM_HASH or LLVM_NEXT_HASH. + git_hash: Upstream LLVM git hash to update to. + svn_version: Matching LLVM revision string for the git_hash. + """ + live_ebuild = self.live_ebuild() + if live_ebuild: + # Working with a -9999 ebuild package here, no + # upreving. + UpdateEbuildLLVMHash( + live_ebuild, llvm_variant, git_hash, svn_version + ) + return + if not self.uprev_target: + # We can exit early if we're not working with a live ebuild, + # and we don't have something to uprev. + raise RuntimeError( + f"Cannot update: no live ebuild or symlink found for {self.package}" + ) + + UpdateEbuildLLVMHash( + self.ebuild_path, llvm_variant, git_hash, svn_version + ) + if llvm_variant == LLVMVariant.current: + UprevEbuildToVersion(str(self.uprev_target), svn_version, git_hash) + else: + UprevEbuildSymlink(str(self.uprev_target)) + + def live_ebuild(self) -> Optional[Path]: + """Path to the live ebuild if it exists. + + Returns: + The patch to the live ebuild if it exists. None otherwise. + """ + matches = self.package_dir().glob("*-9999.ebuild") + return next(matches, None) def defaultCrosRoot() -> Path: @@ -104,14 +185,6 @@ def GetCommandLineArgs(): "(default: %(default)s)", ) - # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - "--verbose", - action="store_true", - help="display contents of a command to the terminal " - "(default: %(default)s)", - ) - # Add argument for the LLVM hash to update parser.add_argument( "--is_llvm_next", @@ -150,59 +223,23 @@ def GetCommandLineArgs(): help="the .json file that has all the patches and their " "metadata if applicable (default: PATCHES.json inside $FILESDIR)", ) + parser.add_argument( + "--repo_manifest", + action="store_true", + help="Updates the llvm-project revision attribute" + " in the internal manifest.", + ) # Parse the command line. - args_output = parser.parse_args() - - # FIXME: We shouldn't be using globals here, but until we fix it, make pylint - # stop complaining about it. - # pylint: disable=global-statement - global verbose + return parser.parse_args() - verbose = args_output.verbose - - return args_output - - -def GetEbuildPathsFromSymLinkPaths(symlinks): - """Reads the symlink(s) to get the ebuild path(s) to the package(s). - - Args: - symlinks: A list of absolute path symlink/symlinks that point - to the package's ebuild. - - Returns: - A dictionary where the key is the absolute path of the symlink and the value - is the absolute path to the ebuild that was read from the symlink. - Raises: - ValueError: Invalid symlink(s) were provided. - """ - - # A dictionary that holds: - # key: absolute symlink path - # value: absolute ebuild path - resolved_paths = {} - - # Iterate through each symlink. - # - # For each symlink, check that it is a valid symlink, - # and then construct the ebuild path, and - # then add the ebuild path to the dict. - for cur_symlink in symlinks: - if not os.path.islink(cur_symlink): - raise ValueError(f"Invalid symlink provided: {cur_symlink}") - - # Construct the absolute path to the ebuild. - ebuild_path = os.path.realpath(cur_symlink) - - if cur_symlink not in resolved_paths: - resolved_paths[cur_symlink] = ebuild_path - - return resolved_paths - - -def UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version): +def UpdateEbuildLLVMHash( + ebuild_path: Path, + llvm_variant: LLVMVariant, + git_hash: str, + svn_version: int, +): """Updates the LLVM hash in the ebuild. The build changes are staged for commit in the temporary repo. @@ -218,33 +255,26 @@ def UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version): of the changes or failed to update the LLVM hash. """ - # Iterate through each ebuild. - # # For each ebuild, read the file in # advance and then create a temporary file # that gets updated with the new LLVM hash # and revision number and then the ebuild file # gets updated to the temporary file. - if not os.path.isfile(ebuild_path): raise ValueError(f"Invalid ebuild path provided: {ebuild_path}") - temp_ebuild_file = f"{ebuild_path}.temp" - - with open(ebuild_path) as ebuild_file: - # write updates to a temporary file in case of interrupts - with open(temp_ebuild_file, "w") as temp_file: - for cur_line in ReplaceLLVMHash( - ebuild_file, llvm_variant, git_hash, svn_version - ): - temp_file.write(cur_line) - os.rename(temp_ebuild_file, ebuild_path) - - # Get the path to the parent directory. - parent_dir = os.path.dirname(ebuild_path) - + with open(ebuild_path, encoding="utf-8") as ebuild_file: + new_lines = list( + ReplaceLLVMHash(ebuild_file, llvm_variant, git_hash, svn_version) + ) + with atomic_write_file.atomic_write( + ebuild_path, "w", encoding="utf-8" + ) as ebuild_file: + ebuild_file.writelines(new_lines) # Stage the changes. - subprocess.check_output(["git", "-C", parent_dir, "add", ebuild_path]) + subprocess.check_output( + ["git", "-C", ebuild_path.parent, "add", ebuild_path] + ) def ReplaceLLVMHash(ebuild_lines, llvm_variant, git_hash, svn_version): @@ -367,46 +397,11 @@ def UprevEbuildToVersion(symlink, svn_version, git_hash): # Create a symlink of the renamed ebuild new_symlink = new_ebuild[: -len(".ebuild")] + "-r1.ebuild" subprocess.check_output(["ln", "-s", "-r", new_ebuild, new_symlink]) - - if not os.path.islink(new_symlink): - raise ValueError( - f'Invalid symlink name: {new_ebuild[:-len(".ebuild")]}' - ) - subprocess.check_output(["git", "-C", symlink_dir, "add", new_symlink]) - # Remove the old symlink subprocess.check_output(["git", "-C", symlink_dir, "rm", symlink]) -def CreatePathDictionaryFromPackages(chroot_path, update_packages): - """Creates a symlink and ebuild path pair dictionary from the packages. - - Args: - chroot_path: The absolute path to the chroot. - update_packages: The filtered packages to be updated. - - Returns: - A dictionary where the key is the absolute path to the symlink - of the package and the value is the absolute path to the ebuild of - the package. - """ - - # Construct a list containing the chroot file paths of the package(s). - chroot_file_paths = chroot.GetChrootEbuildPaths( - chroot_path, update_packages - ) - - # Construct a list containing the symlink(s) of the package(s). - symlink_file_paths = chroot.ConvertChrootPathsToAbsolutePaths( - chroot_path, chroot_file_paths - ) - - # Create a dictionary where the key is the absolute path of the symlink to - # the package and the value is the absolute path to the ebuild of the package. - return GetEbuildPathsFromSymLinkPaths(symlink_file_paths) - - def RemovePatchesFromFilesDir(patches): """Removes the patches from $FILESDIR of a package. @@ -506,8 +501,8 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): return commit_messages -def UpdateManifests(packages: Iterable[str], chroot_path: Path): - """Updates manifest files for packages. +def UpdatePortageManifests(packages: Iterable[str], chroot_path: Path): + """Updates portage manifest files for packages. Args: packages: A list of packages to update manifests for. @@ -518,17 +513,21 @@ def UpdateManifests(packages: Iterable[str], chroot_path: Path): """ manifest_ebuilds = chroot.GetChrootEbuildPaths(chroot_path, packages) for ebuild_path in manifest_ebuilds: + ebuild_dir = os.path.dirname(ebuild_path) subprocess_helpers.ChrootRunCommand( chroot_path, ["ebuild", ebuild_path, "manifest"] ) + subprocess_helpers.ChrootRunCommand( + chroot_path, ["git", "-C", ebuild_dir, "add", "Manifest"] + ) def UpdatePackages( packages: Iterable[str], manifest_packages: Iterable[str], - llvm_variant, - git_hash, - svn_version, + llvm_variant: LLVMVariant, + git_hash: str, + svn_version: int, chroot_path: Path, mode, git_hash_source, @@ -558,86 +557,55 @@ def UpdatePackages( Gerrit commit URL and the second pair is the change list number. """ - # Construct a dictionary where the key is the absolute path of the symlink to - # the package and the value is the absolute path to the ebuild of the package. - paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages) - - repo_path = os.path.dirname(next(iter(paths_dict.values()))) - - branch = "update-" + llvm_variant.value + "-" + git_hash + portage_packages = (PortagePackage(chroot_path, pkg) for pkg in packages) + chromiumos_overlay_path = ( + chroot_path / "src" / "third_party" / "chromiumos-overlay" + ) + branch_name = "update-" + llvm_variant.value + "-" + git_hash + + commit_message_header = "llvm" + if llvm_variant == LLVMVariant.next: + commit_message_header = "llvm-next" + if git_hash_source in get_llvm_hash.KNOWN_HASH_SOURCES: + commit_message_header += ( + f"/{git_hash_source}: upgrade to {git_hash} (r{svn_version})" + ) + else: + commit_message_header += f": upgrade to {git_hash} (r{svn_version})" - git.CreateBranch(repo_path, branch) + commit_lines = [ + commit_message_header + "\n", + "The following packages have been updated:", + ] + # Holds the list of packages that are updating. + updated_packages: List[str] = [] + git.CreateBranch(chromiumos_overlay_path, branch_name) try: - commit_message_header = "llvm" - if llvm_variant == LLVMVariant.next: - commit_message_header = "llvm-next" - if git_hash_source in get_llvm_hash.KNOWN_HASH_SOURCES: - commit_message_header += ( - f"/{git_hash_source}: upgrade to {git_hash} (r{svn_version})" - ) - else: - commit_message_header += f": upgrade to {git_hash} (r{svn_version})" - - commit_lines = [ - commit_message_header + "\n", - "The following packages have been updated:", - ] - - # Holds the list of packages that are updating. - packages = [] - - # Iterate through the dictionary. - # - # For each iteration: - # 1) Update the ebuild's LLVM hash. - # 2) Uprev the ebuild (symlink). - # 3) Add the modified package to the commit message. - for symlink_path, ebuild_path in paths_dict.items(): - path_to_ebuild_dir = os.path.dirname(ebuild_path) - - UpdateEbuildLLVMHash( - ebuild_path, llvm_variant, git_hash, svn_version - ) - - if llvm_variant == LLVMVariant.current: - UprevEbuildToVersion(symlink_path, svn_version, git_hash) - else: - UprevEbuildSymlink(symlink_path) - - cur_dir_name = os.path.basename(path_to_ebuild_dir) - parent_dir_name = os.path.basename( - os.path.dirname(path_to_ebuild_dir) - ) - - packages.append(f"{parent_dir_name}/{cur_dir_name}") - commit_lines.append(f"{parent_dir_name}/{cur_dir_name}") - + for pkg in portage_packages: + pkg.update(llvm_variant, git_hash, svn_version) + updated_packages.append(pkg.package) + commit_lines.append(pkg.package) if manifest_packages: - UpdateManifests(manifest_packages, chroot_path) + UpdatePortageManifests(manifest_packages, chroot_path) commit_lines.append("Updated manifest for:") commit_lines.extend(manifest_packages) - EnsurePackageMaskContains(chroot_path, git_hash) - # Handle the patches for each package. package_info_dict = UpdatePackagesPatchMetadataFile( - chroot_path, svn_version, packages, mode + chroot_path, svn_version, updated_packages, mode ) - # Update the commit message if changes were made to a package's patches. commit_lines = StagePackagesPatchResultsForCommit( package_info_dict, commit_lines ) - if extra_commit_msg: commit_lines.append(extra_commit_msg) - - change_list = git.UploadChanges(repo_path, branch, commit_lines) - + change_list = git.UploadChanges( + chromiumos_overlay_path, branch_name, commit_lines + ) finally: - git.DeleteBranch(repo_path, branch) - + git.DeleteBranch(chromiumos_overlay_path, branch_name) return change_list @@ -660,7 +628,7 @@ def EnsurePackageMaskContains(chroot_path, git_hash): mask_path = os.path.join( overlay_dir, "profiles/targets/chromeos/package.mask" ) - with open(mask_path, "r+") as mask_file: + with open(mask_path, "r+", encoding="utf-8") as mask_file: mask_contents = mask_file.read() expected_line = f"=sys-devel/llvm-{llvm_major_version}.0_pre*\n" if expected_line not in mask_contents: @@ -715,7 +683,7 @@ def UpdatePackagesPatchMetadataFile( ) chroot_ebuild_path = Path( chroot.ConvertChrootPathsToAbsolutePaths( - chroot_path, [chroot_ebuild_str] + str(chroot_path), [chroot_ebuild_str] )[0] ) patches_json_fp = ( @@ -747,12 +715,61 @@ def UpdatePackagesPatchMetadataFile( patches_info = patch_utils.update_version_ranges( svn_version, src_path, patches_json_fp ) + else: + raise RuntimeError(f"unsupported failure mode: {mode}") package_info[cur_package] = patches_info._asdict() return package_info +def ChangeRepoManifest(git_hash: str, src_tree: Path): + """Change the repo internal manifest for llvm-project. + + Args: + git_hash: The LLVM git hash to change to. + src_tree: ChromiumOS source tree checkout. + + Returns: + The uploaded changelist CommitContents. + """ + manifest_dir = manifest_utils.get_chromeos_manifest_path(src_tree).parent + branch_name = "update-llvm-project-" + git_hash + commit_lines = ( + textwrap.dedent( + f""" + manifest: Update llvm-project to {git_hash} + + Upgrade the local LLVM reversion to match the new llvm ebuild + hash. This must be merged along with any chromiumos-overlay + changes to LLVM. Automatic uprevs rely on the manifest hash + to match what is specified by LLVM_HASH. + + This CL is generated by the update_chromeos_llvm_hash.py script. + + BUG=None + TEST=CQ + """ + ) + .lstrip() + .splitlines() + ) + + git.CreateBranch(manifest_dir, branch_name) + try: + manifest_path = manifest_utils.update_chromeos_manifest( + git_hash, + src_tree, + ) + subprocess.run( + ["git", "-C", manifest_dir, "add", manifest_path.name], check=True + ) + change_list = git.UploadChanges(manifest_dir, branch_name, commit_lines) + finally: + git.DeleteBranch(manifest_dir, branch_name) + return change_list + + def main(): """Updates the LLVM next hash for each package. @@ -764,6 +781,8 @@ def main(): args_output = GetCommandLineArgs() + chroot.VerifyChromeOSRoot(args_output.chroot_path) + llvm_variant = LLVMVariant.current if args_output.is_llvm_next: llvm_variant = LLVMVariant.next @@ -773,7 +792,6 @@ def main(): git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( git_hash_source ) - # Filter out empty strings. For example "".split{",") returns [""]. packages = set(p for p in args_output.update_packages.split(",") if p) manifest_packages = set( @@ -798,6 +816,17 @@ def main(): print(f"Gerrit URL: {change_list.url}") print(f"Change list number: {change_list.cl_number}") + if args_output.repo_manifest: + print( + f"Updating internal manifest to {git_hash} ({svn_version})...", + end="", + ) + change_list = ChangeRepoManifest(git_hash, args_output.chroot_path) + print(" Done!") + print("New repo manifest CL:") + print(f" URL: {change_list.url}") + print(f" CL Number: {change_list.cl_number}") + if __name__ == "__main__": main() |