diff options
Diffstat (limited to 'rust_tools/rust_uprev.py')
| -rwxr-xr-x | rust_tools/rust_uprev.py | 1522 |
1 files changed, 866 insertions, 656 deletions
diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py index 011639df..382d991a 100755 --- a/rust_tools/rust_uprev.py +++ b/rust_tools/rust_uprev.py @@ -1,416 +1,437 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tool to automatically generate a new Rust uprev CL. -This tool is intended to automatically generate a CL to uprev Rust to a -newer version in Chrome OS, including creating a new Rust version or -removing an old version. It's based on -src/third_party/chromiumos-overlay/dev-lang/rust/UPGRADE.md. When using -the tool, the progress can be saved to a JSON file, so the user can resume -the process after a failing step is fixed. Example usage to create a new -version: - -1. (inside chroot) $ ./rust_tools/rust_uprev.py - --state_file /tmp/state-file.json - create --rust_version 1.45.0 -2. Step "compile rust" failed due to the patches can't apply to new version -3. Manually fix the patches -4. Execute the command in step 1 again. +This tool is intended to automatically generate a CL to uprev Rust to +a newer version in Chrome OS, including creating a new Rust version or +removing an old version. When using the tool, the progress can be +saved to a JSON file, so the user can resume the process after a +failing step is fixed. Example usage to create a new version: + +1. (inside chroot) $ ./rust_tools/rust_uprev.py \\ + --state_file /tmp/rust-to-1.60.0.json \\ + roll --uprev 1.60.0 +2. Step "compile rust" failed due to the patches can't apply to new version. +3. Manually fix the patches. +4. Execute the command in step 1 again, but add "--continue" before "roll". 5. Iterate 1-4 for each failed step until the tool passes. -Replace `create --rust_version 1.45.0` with `remove --rust_version 1.43.0` -if you want to remove all 1.43.0 related stuff in the same CL. Remember to -use a different state file if you choose to run different subcommands. - -If you want a hammer that can do everything for you, use the subcommand -`roll`. It can create a Rust uprev CL with `create` and `remove` and upload -the CL to chromium code review. +Besides "roll", the tool also support subcommands that perform +various parts of an uprev. See `--help` for all available options. """ import argparse -import pathlib import json import logging import os +import pathlib +from pathlib import Path import re +import shlex import shutil import subprocess import sys -from pathlib import Path from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple -from llvm_tools import chroot, git +from llvm_tools import chroot +from llvm_tools import git + -EQUERY = 'equery' -GSUTIL = 'gsutil.py' -MIRROR_PATH = 'gs://chromeos-localmirror/distfiles' -RUST_PATH = Path( - '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust') +EQUERY = "equery" +GSUTIL = "gsutil.py" +MIRROR_PATH = "gs://chromeos-localmirror/distfiles" +EBUILD_PREFIX = Path("/mnt/host/source/src/third_party/chromiumos-overlay") +RUST_PATH = Path(EBUILD_PREFIX, "dev-lang", "rust") def get_command_output(command: List[str], *args, **kwargs) -> str: - return subprocess.check_output(command, encoding='utf-8', *args, - **kwargs).strip() + return subprocess.check_output( + command, encoding="utf-8", *args, **kwargs + ).strip() def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str: - return subprocess.run(command, - check=False, - stdout=subprocess.PIPE, - encoding='utf-8', - *args, - **kwargs).stdout.strip() + return subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + encoding="utf-8", + *args, + **kwargs, + ).stdout.strip() class RustVersion(NamedTuple): - """NamedTuple represents a Rust version""" - major: int - minor: int - patch: int - - def __str__(self): - return f'{self.major}.{self.minor}.{self.patch}' - - @staticmethod - def parse_from_ebuild(ebuild_name: str) -> 'RustVersion': - input_re = re.compile(r'^rust-' - r'(?P<major>\d+)\.' - r'(?P<minor>\d+)\.' - r'(?P<patch>\d+)' - r'(:?-r\d+)?' - r'\.ebuild$') - m = input_re.match(ebuild_name) - assert m, f'failed to parse {ebuild_name!r}' - return RustVersion(int(m.group('major')), int(m.group('minor')), - int(m.group('patch'))) - - @staticmethod - def parse(x: str) -> 'RustVersion': - input_re = re.compile(r'^(?:rust-)?' - r'(?P<major>\d+)\.' - r'(?P<minor>\d+)\.' - r'(?P<patch>\d+)' - r'(?:.ebuild)?$') - m = input_re.match(x) - assert m, f'failed to parse {x!r}' - return RustVersion(int(m.group('major')), int(m.group('minor')), - int(m.group('patch'))) + """NamedTuple represents a Rust version""" + + major: int + minor: int + patch: int + + def __str__(self): + return f"{self.major}.{self.minor}.{self.patch}" + + @staticmethod + def parse_from_ebuild(ebuild_name: str) -> "RustVersion": + input_re = re.compile( + r"^rust-" + r"(?P<major>\d+)\." + r"(?P<minor>\d+)\." + r"(?P<patch>\d+)" + r"(:?-r\d+)?" + r"\.ebuild$" + ) + m = input_re.match(ebuild_name) + assert m, f"failed to parse {ebuild_name!r}" + return RustVersion( + int(m.group("major")), int(m.group("minor")), int(m.group("patch")) + ) + + @staticmethod + def parse(x: str) -> "RustVersion": + input_re = re.compile( + r"^(?:rust-)?" + r"(?P<major>\d+)\." + r"(?P<minor>\d+)\." + r"(?P<patch>\d+)" + r"(?:.ebuild)?$" + ) + m = input_re.match(x) + assert m, f"failed to parse {x!r}" + return RustVersion( + int(m.group("major")), int(m.group("minor")), int(m.group("patch")) + ) def compute_rustc_src_name(version: RustVersion) -> str: - return f'rustc-{version}-src.tar.gz' + return f"rustc-{version}-src.tar.gz" def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str: - return f'rust-bootstrap-{version}.tbz2' + return f"rust-bootstrap-{version}.tbz2" def find_ebuild_for_package(name: str) -> os.PathLike: - """Returns the path to the ebuild for the named package.""" - return get_command_output([EQUERY, 'w', name]) - - -def find_ebuild_path(directory: Path, - name: str, - version: Optional[RustVersion] = None) -> Path: - """Finds an ebuild in a directory. - - Returns the path to the ebuild file. Asserts if there is not - exactly one match. The match is constrained by name and optionally - by version, but can match any patch level. E.g. "rust" version - 1.3.4 can match rust-1.3.4.ebuild but also rust-1.3.4-r6.ebuild. - """ - if version: - pattern = f'{name}-{version}*.ebuild' - else: - pattern = f'{name}-*.ebuild' - matches = list(Path(directory).glob(pattern)) - assert len(matches) == 1, matches - return matches[0] + """Returns the path to the ebuild for the named package.""" + return get_command_output([EQUERY, "w", name]) + + +def find_ebuild_path( + directory: Path, name: str, version: Optional[RustVersion] = None +) -> Path: + """Finds an ebuild in a directory. + + Returns the path to the ebuild file. The match is constrained by + name and optionally by version, but can match any patch level. + E.g. "rust" version 1.3.4 can match rust-1.3.4.ebuild but also + rust-1.3.4-r6.ebuild. + + The expectation is that there is only one matching ebuild, and + an assert is raised if this is not the case. However, symlinks to + ebuilds in the same directory are ignored, so having a + rust-x.y.z-rn.ebuild symlink to rust-x.y.z.ebuild is allowed. + """ + if version: + pattern = f"{name}-{version}*.ebuild" + else: + pattern = f"{name}-*.ebuild" + matches = set(directory.glob(pattern)) + result = [] + # Only count matches that are not links to other matches. + for m in matches: + try: + target = os.readlink(directory / m) + except OSError: + # Getting here means the match is not a symlink to one of + # the matching ebuilds, so add it to the result list. + result.append(m) + continue + if directory / target not in matches: + result.append(m) + assert len(result) == 1, result + return result[0] def get_rust_bootstrap_version(): - """Get the version of the current rust-bootstrap package.""" - bootstrap_ebuild = find_ebuild_path(rust_bootstrap_path(), 'rust-bootstrap') - m = re.match(r'^rust-bootstrap-(\d+).(\d+).(\d+)', bootstrap_ebuild.name) - assert m, bootstrap_ebuild.name - return RustVersion(int(m.group(1)), int(m.group(2)), int(m.group(3))) + """Get the version of the current rust-bootstrap package.""" + bootstrap_ebuild = find_ebuild_path(rust_bootstrap_path(), "rust-bootstrap") + m = re.match(r"^rust-bootstrap-(\d+).(\d+).(\d+)", bootstrap_ebuild.name) + assert m, bootstrap_ebuild.name + return RustVersion(int(m.group(1)), int(m.group(2)), int(m.group(3))) def parse_commandline_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--state_file', - required=True, - help='A state file to hold previous completed steps. If the file ' - 'exists, it needs to be used together with --continue or --restart. ' - 'If not exist (do not use --continue in this case), we will create a ' - 'file for you.', - ) - parser.add_argument( - '--restart', - action='store_true', - help='Restart from the first step. Ignore the completed steps in ' - 'the state file', - ) - parser.add_argument( - '--continue', - dest='cont', - action='store_true', - help='Continue the steps from the state file', - ) - - create_parser_template = argparse.ArgumentParser(add_help=False) - create_parser_template.add_argument( - '--template', - type=RustVersion.parse, - default=None, - help='A template to use for creating a Rust uprev from, in the form ' - 'a.b.c The ebuild has to exist in the chroot. If not specified, the ' - 'tool will use the current Rust version in the chroot as template.', - ) - create_parser_template.add_argument( - '--skip_compile', - action='store_true', - help='Skip compiling rust to test the tool. Only for testing', - ) - - subparsers = parser.add_subparsers(dest='subparser_name') - subparser_names = [] - subparser_names.append('create') - create_parser = subparsers.add_parser( - 'create', - parents=[create_parser_template], - help='Create changes uprevs Rust to a new version', - ) - create_parser.add_argument( - '--rust_version', - type=RustVersion.parse, - required=True, - help='Rust version to uprev to, in the form a.b.c', - ) - - subparser_names.append('remove') - remove_parser = subparsers.add_parser( - 'remove', - help='Clean up old Rust version from chroot', - ) - remove_parser.add_argument( - '--rust_version', - type=RustVersion.parse, - default=None, - help='Rust version to remove, in the form a.b.c If not ' - 'specified, the tool will remove the oldest version in the chroot', - ) - - subparser_names.append('remove-bootstrap') - remove_bootstrap_parser = subparsers.add_parser( - 'remove-bootstrap', - help='Remove an old rust-bootstrap version', - ) - remove_bootstrap_parser.add_argument( - '--version', - type=RustVersion.parse, - required=True, - help='rust-bootstrap version to remove', - ) - - subparser_names.append('roll') - roll_parser = subparsers.add_parser( - 'roll', - parents=[create_parser_template], - help='A command can create and upload a Rust uprev CL, including ' - 'preparing the repo, creating new Rust uprev, deleting old uprev, ' - 'and upload a CL to crrev.', - ) - roll_parser.add_argument( - '--uprev', - type=RustVersion.parse, - required=True, - help='Rust version to uprev to, in the form a.b.c', - ) - roll_parser.add_argument( - '--remove', - type=RustVersion.parse, - default=None, - help='Rust version to remove, in the form a.b.c If not ' - 'specified, the tool will remove the oldest version in the chroot', - ) - roll_parser.add_argument( - '--skip_cross_compiler', - action='store_true', - help='Skip updating cross-compiler in the chroot', - ) - roll_parser.add_argument( - '--no_upload', - action='store_true', - help='If specified, the tool will not upload the CL for review', - ) - - args = parser.parse_args() - if args.subparser_name not in subparser_names: - parser.error('one of %s must be specified' % subparser_names) - - if args.cont and args.restart: - parser.error('Please select either --continue or --restart') - - if os.path.exists(args.state_file): - if not args.cont and not args.restart: - parser.error('State file exists, so you should either --continue ' - 'or --restart') - if args.cont and not os.path.exists(args.state_file): - parser.error('Indicate --continue but the state file does not exist') - - if args.restart and os.path.exists(args.state_file): - os.remove(args.state_file) - - return args - - -def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion] - ) -> Optional[Tuple[RustVersion, str, RustVersion]]: - if template is None: - ebuild_path = find_ebuild_for_package('rust') - ebuild_name = os.path.basename(ebuild_path) - template_version = RustVersion.parse_from_ebuild(ebuild_name) - else: - ebuild_path = find_ebuild_for_rust_version(template) - template_version = template - - bootstrap_version = get_rust_bootstrap_version() - - if rust_version <= template_version: - logging.info( - 'Requested version %s is not newer than the template version %s.', - rust_version, template_version) - return None + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--state_file", + required=True, + help="A state file to hold previous completed steps. If the file " + "exists, it needs to be used together with --continue or --restart. " + "If not exist (do not use --continue in this case), we will create a " + "file for you.", + ) + parser.add_argument( + "--restart", + action="store_true", + help="Restart from the first step. Ignore the completed steps in " + "the state file", + ) + parser.add_argument( + "--continue", + dest="cont", + action="store_true", + help="Continue the steps from the state file", + ) + + create_parser_template = argparse.ArgumentParser(add_help=False) + create_parser_template.add_argument( + "--template", + type=RustVersion.parse, + default=None, + help="A template to use for creating a Rust uprev from, in the form " + "a.b.c The ebuild has to exist in the chroot. If not specified, the " + "tool will use the current Rust version in the chroot as template.", + ) + create_parser_template.add_argument( + "--skip_compile", + action="store_true", + help="Skip compiling rust to test the tool. Only for testing", + ) - logging.info('Template Rust version is %s (ebuild: %r)', template_version, - ebuild_path) - logging.info('rust-bootstrap version is %s', bootstrap_version) + subparsers = parser.add_subparsers(dest="subparser_name") + subparser_names = [] + subparser_names.append("create") + create_parser = subparsers.add_parser( + "create", + parents=[create_parser_template], + help="Create changes uprevs Rust to a new version", + ) + create_parser.add_argument( + "--rust_version", + type=RustVersion.parse, + required=True, + help="Rust version to uprev to, in the form a.b.c", + ) - return template_version, ebuild_path, bootstrap_version + subparser_names.append("remove") + remove_parser = subparsers.add_parser( + "remove", + help="Clean up old Rust version from chroot", + ) + remove_parser.add_argument( + "--rust_version", + type=RustVersion.parse, + default=None, + help="Rust version to remove, in the form a.b.c If not " + "specified, the tool will remove the oldest version in the chroot", + ) + subparser_names.append("remove-bootstrap") + remove_bootstrap_parser = subparsers.add_parser( + "remove-bootstrap", + help="Remove an old rust-bootstrap version", + ) + remove_bootstrap_parser.add_argument( + "--version", + type=RustVersion.parse, + required=True, + help="rust-bootstrap version to remove", + ) -def copy_patches(directory: Path, template_version: RustVersion, - new_version: RustVersion) -> None: - patch_path = directory.joinpath('files') - prefix = '%s-%s-' % (directory.name, template_version) - new_prefix = '%s-%s-' % (directory.name, new_version) - for f in os.listdir(patch_path): - if not f.startswith(prefix): - continue - logging.info('Copy patch %s to new version', f) - new_name = f.replace(str(template_version), str(new_version)) - shutil.copyfile( - os.path.join(patch_path, f), - os.path.join(patch_path, new_name), + subparser_names.append("roll") + roll_parser = subparsers.add_parser( + "roll", + parents=[create_parser_template], + help="A command can create and upload a Rust uprev CL, including " + "preparing the repo, creating new Rust uprev, deleting old uprev, " + "and upload a CL to crrev.", + ) + roll_parser.add_argument( + "--uprev", + type=RustVersion.parse, + required=True, + help="Rust version to uprev to, in the form a.b.c", + ) + roll_parser.add_argument( + "--remove", + type=RustVersion.parse, + default=None, + help="Rust version to remove, in the form a.b.c If not " + "specified, the tool will remove the oldest version in the chroot", + ) + roll_parser.add_argument( + "--skip_cross_compiler", + action="store_true", + help="Skip updating cross-compiler in the chroot", + ) + roll_parser.add_argument( + "--no_upload", + action="store_true", + help="If specified, the tool will not upload the CL for review", ) - subprocess.check_call(['git', 'add', f'{new_prefix}*.patch'], cwd=patch_path) + args = parser.parse_args() + if args.subparser_name not in subparser_names: + parser.error("one of %s must be specified" % subparser_names) + + if args.cont and args.restart: + parser.error("Please select either --continue or --restart") + + if os.path.exists(args.state_file): + if not args.cont and not args.restart: + parser.error( + "State file exists, so you should either --continue " + "or --restart" + ) + if args.cont and not os.path.exists(args.state_file): + parser.error("Indicate --continue but the state file does not exist") + + if args.restart and os.path.exists(args.state_file): + os.remove(args.state_file) + + return args + + +def prepare_uprev( + rust_version: RustVersion, template: Optional[RustVersion] +) -> Optional[Tuple[RustVersion, str, RustVersion]]: + if template is None: + ebuild_path = find_ebuild_for_package("rust") + ebuild_name = os.path.basename(ebuild_path) + template_version = RustVersion.parse_from_ebuild(ebuild_name) + else: + ebuild_path = find_ebuild_for_rust_version(template) + template_version = template + + bootstrap_version = get_rust_bootstrap_version() + + if rust_version <= template_version: + logging.info( + "Requested version %s is not newer than the template version %s.", + rust_version, + template_version, + ) + return None + + logging.info( + "Template Rust version is %s (ebuild: %r)", + template_version, + ebuild_path, + ) + logging.info("rust-bootstrap version is %s", bootstrap_version) + + return template_version, ebuild_path, bootstrap_version + + +def copy_patches( + directory: Path, template_version: RustVersion, new_version: RustVersion +) -> None: + patch_path = directory / "files" + prefix = "%s-%s-" % (directory.name, template_version) + new_prefix = "%s-%s-" % (directory.name, new_version) + for f in os.listdir(patch_path): + if not f.startswith(prefix): + continue + logging.info("Copy patch %s to new version", f) + new_name = f.replace(str(template_version), str(new_version)) + shutil.copyfile( + os.path.join(patch_path, f), + os.path.join(patch_path, new_name), + ) + + subprocess.check_call( + ["git", "add", f"{new_prefix}*.patch"], cwd=patch_path + ) -def create_ebuild(template_ebuild: str, new_version: RustVersion) -> str: - shutil.copyfile(template_ebuild, - RUST_PATH.joinpath(f'rust-{new_version}.ebuild')) - subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], - cwd=RUST_PATH) - return os.path.join(RUST_PATH, f'rust-{new_version}.ebuild') +def create_ebuild( + template_ebuild: str, pkgatom: str, new_version: RustVersion +) -> str: + filename = f"{Path(pkgatom).name}-{new_version}.ebuild" + ebuild = EBUILD_PREFIX.joinpath(f"{pkgatom}/{filename}") + shutil.copyfile(template_ebuild, ebuild) + subprocess.check_call(["git", "add", filename], cwd=ebuild.parent) + return str(ebuild) def update_bootstrap_ebuild(new_bootstrap_version: RustVersion) -> None: - old_ebuild = find_ebuild_path(rust_bootstrap_path(), 'rust-bootstrap') - m = re.match(r'^rust-bootstrap-(\d+).(\d+).(\d+)', old_ebuild.name) - assert m, old_ebuild.name - old_version = RustVersion(m.group(1), m.group(2), m.group(3)) - new_ebuild = old_ebuild.parent.joinpath( - f'rust-bootstrap-{new_bootstrap_version}.ebuild') - old_text = old_ebuild.read_text(encoding='utf-8') - new_text, changes = re.subn(r'(RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=\([^)]*)', - f'\\1\t{old_version}\n', - old_text, - flags=re.MULTILINE) - assert changes == 1, 'Failed to update RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE' - new_ebuild.write_text(new_text, encoding='utf-8') - - -def update_ebuild(ebuild_file: str, - new_bootstrap_version: RustVersion) -> None: - contents = open(ebuild_file, encoding='utf-8').read() - contents, subs = re.subn(r'^BOOTSTRAP_VERSION=.*$', - 'BOOTSTRAP_VERSION="%s"' % - (new_bootstrap_version, ), - contents, - flags=re.MULTILINE) - if not subs: - raise RuntimeError('BOOTSTRAP_VERSION not found in rust ebuild') - open(ebuild_file, 'w', encoding='utf-8').write(contents) - logging.info('Rust ebuild file has BOOTSTRAP_VERSION updated to %s', - new_bootstrap_version) - - -def flip_mirror_in_ebuild(ebuild_file: Path, add: bool) -> None: - restrict_re = re.compile( - r'(?P<before>RESTRICT=")(?P<values>"[^"]*"|.*)(?P<after>")') - with open(ebuild_file, encoding='utf-8') as f: - contents = f.read() - m = restrict_re.search(contents) - assert m, 'failed to find RESTRICT variable in Rust ebuild' - values = m.group('values') - if add: - if 'mirror' in values: - return - values += ' mirror' - else: - if 'mirror' not in values: - return - values = values.replace(' mirror', '') - new_contents = restrict_re.sub(r'\g<before>%s\g<after>' % values, contents) - with open(ebuild_file, 'w', encoding='utf-8') as f: - f.write(new_contents) - - -def ebuild_actions(package: str, actions: List[str], - sudo: bool = False) -> None: - ebuild_path_inchroot = find_ebuild_for_package(package) - cmd = ['ebuild', ebuild_path_inchroot] + actions - if sudo: - cmd = ['sudo'] + cmd - subprocess.check_call(cmd) + old_ebuild = find_ebuild_path(rust_bootstrap_path(), "rust-bootstrap") + m = re.match(r"^rust-bootstrap-(\d+).(\d+).(\d+)", old_ebuild.name) + assert m, old_ebuild.name + old_version = RustVersion(m.group(1), m.group(2), m.group(3)) + new_ebuild = old_ebuild.parent.joinpath( + f"rust-bootstrap-{new_bootstrap_version}.ebuild" + ) + old_text = old_ebuild.read_text(encoding="utf-8") + new_text, changes = re.subn( + r"(RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=\([^)]*)", + f"\\1\t{old_version}\n", + old_text, + flags=re.MULTILINE, + ) + assert changes == 1, "Failed to update RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE" + new_ebuild.write_text(new_text, encoding="utf-8") + + +def update_bootstrap_version( + path: str, new_bootstrap_version: RustVersion +) -> None: + contents = open(path, encoding="utf-8").read() + contents, subs = re.subn( + r"^BOOTSTRAP_VERSION=.*$", + 'BOOTSTRAP_VERSION="%s"' % (new_bootstrap_version,), + contents, + flags=re.MULTILINE, + ) + if not subs: + raise RuntimeError(f"BOOTSTRAP_VERSION not found in {path}") + open(path, "w", encoding="utf-8").write(contents) + logging.info("Rust BOOTSTRAP_VERSION updated to %s", new_bootstrap_version) + + +def ebuild_actions( + package: str, actions: List[str], sudo: bool = False +) -> None: + ebuild_path_inchroot = find_ebuild_for_package(package) + cmd = ["ebuild", ebuild_path_inchroot] + actions + if sudo: + cmd = ["sudo"] + cmd + subprocess.check_call(cmd) def fetch_distfile_from_mirror(name: str) -> None: - """Gets the named file from the local mirror. - - This ensures that the file exists on the mirror, and - that we can read it. We overwrite any existing distfile - to ensure the checksums that update_manifest() records - match the file as it exists on the mirror. - - This function also attempts to verify the ACL for - the file (which is expected to have READER permission - for allUsers). We can only see the ACL if the user - gsutil runs with is the owner of the file. If not, - we get an access denied error. We also count this - as a success, because it means we were able to fetch - the file even though we don't own it. - """ - mirror_file = MIRROR_PATH + '/' + name - local_file = Path(get_distdir(), name) - cmd = [GSUTIL, 'cp', mirror_file, local_file] - logging.info('Running %r', cmd) - rc = subprocess.call(cmd) - if rc != 0: - logging.error( - """Could not fetch %s + """Gets the named file from the local mirror. + + This ensures that the file exists on the mirror, and + that we can read it. We overwrite any existing distfile + to ensure the checksums that update_manifest() records + match the file as it exists on the mirror. + + This function also attempts to verify the ACL for + the file (which is expected to have READER permission + for allUsers). We can only see the ACL if the user + gsutil runs with is the owner of the file. If not, + we get an access denied error. We also count this + as a success, because it means we were able to fetch + the file even though we don't own it. + """ + mirror_file = MIRROR_PATH + "/" + name + local_file = Path(get_distdir(), name) + cmd = [GSUTIL, "cp", mirror_file, local_file] + logging.info("Running %r", cmd) + rc = subprocess.call(cmd) + if rc != 0: + logging.error( + """Could not fetch %s If the file does not yet exist at %s please download the file, verify its integrity @@ -425,352 +446,541 @@ gpg --recv-keys 85AB96E6FA1BE5FE Once you have verify the integrity of the file, upload it to the local mirror using gsutil cp. -""", mirror_file, MIRROR_PATH, name, name) - raise Exception(f'Could not fetch {mirror_file}') - # Check that the ACL allows allUsers READER access. - # If we get an AccessDeniedAcception here, that also - # counts as a success, because we were able to fetch - # the file as a non-owner. - cmd = [GSUTIL, 'acl', 'get', mirror_file] - logging.info('Running %r', cmd) - output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT) - acl_verified = False - if 'AccessDeniedException:' in output: - acl_verified = True - else: - acl = json.loads(output) - for x in acl: - if x['entity'] == 'allUsers' and x['role'] == 'READER': +""", + mirror_file, + MIRROR_PATH, + name, + name, + ) + raise Exception(f"Could not fetch {mirror_file}") + # Check that the ACL allows allUsers READER access. + # If we get an AccessDeniedAcception here, that also + # counts as a success, because we were able to fetch + # the file as a non-owner. + cmd = [GSUTIL, "acl", "get", mirror_file] + logging.info("Running %r", cmd) + output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT) + acl_verified = False + if "AccessDeniedException:" in output: acl_verified = True - break - if not acl_verified: - logging.error('Output from acl get:\n%s', output) - raise Exception('Could not verify that allUsers has READER permission') - - -def fetch_bootstrap_distfiles(old_version: RustVersion, - new_version: RustVersion) -> None: - """Fetches rust-bootstrap distfiles from the local mirror - - Fetches the distfiles for a rust-bootstrap ebuild to ensure they - are available on the mirror and the local copies are the same as - the ones on the mirror. - """ - fetch_distfile_from_mirror(compute_rust_bootstrap_prebuilt_name(old_version)) - fetch_distfile_from_mirror(compute_rustc_src_name(new_version)) + else: + acl = json.loads(output) + for x in acl: + if x["entity"] == "allUsers" and x["role"] == "READER": + acl_verified = True + break + if not acl_verified: + logging.error("Output from acl get:\n%s", output) + raise Exception("Could not verify that allUsers has READER permission") + + +def fetch_bootstrap_distfiles( + old_version: RustVersion, new_version: RustVersion +) -> None: + """Fetches rust-bootstrap distfiles from the local mirror + + Fetches the distfiles for a rust-bootstrap ebuild to ensure they + are available on the mirror and the local copies are the same as + the ones on the mirror. + """ + fetch_distfile_from_mirror( + compute_rust_bootstrap_prebuilt_name(old_version) + ) + fetch_distfile_from_mirror(compute_rustc_src_name(new_version)) def fetch_rust_distfiles(version: RustVersion) -> None: - """Fetches rust distfiles from the local mirror + """Fetches rust distfiles from the local mirror - Fetches the distfiles for a rust ebuild to ensure they - are available on the mirror and the local copies are - the same as the ones on the mirror. - """ - fetch_distfile_from_mirror(compute_rustc_src_name(version)) + Fetches the distfiles for a rust ebuild to ensure they + are available on the mirror and the local copies are + the same as the ones on the mirror. + """ + fetch_distfile_from_mirror(compute_rustc_src_name(version)) def get_distdir() -> os.PathLike: - """Returns portage's distdir.""" - return get_command_output(['portageq', 'distdir']) + """Returns portage's distdir.""" + return get_command_output(["portageq", "distdir"]) def update_manifest(ebuild_file: os.PathLike) -> None: - """Updates the MANIFEST for the ebuild at the given path.""" - ebuild = Path(ebuild_file) - logging.info('Added "mirror" to RESTRICT to %s', ebuild.name) - flip_mirror_in_ebuild(ebuild, add=True) - ebuild_actions(ebuild.parent.name, ['manifest']) - logging.info('Removed "mirror" to RESTRICT from %s', ebuild.name) - flip_mirror_in_ebuild(ebuild, add=False) - - -def update_rust_packages(rust_version: RustVersion, add: bool) -> None: - package_file = RUST_PATH.joinpath( - '../../profiles/targets/chromeos/package.provided') - with open(package_file, encoding='utf-8') as f: - contents = f.read() - if add: - rust_packages_re = re.compile(r'dev-lang/rust-(\d+\.\d+\.\d+)') - rust_packages = rust_packages_re.findall(contents) - # Assume all the rust packages are in alphabetical order, so insert the new - # version to the place after the last rust_packages - new_str = f'dev-lang/rust-{rust_version}' - new_contents = contents.replace(rust_packages[-1], - f'{rust_packages[-1]}\n{new_str}') - logging.info('%s has been inserted into package.provided', new_str) - else: - old_str = f'dev-lang/rust-{rust_version}\n' - assert old_str in contents, f'{old_str!r} not found in package.provided' - new_contents = contents.replace(old_str, '') - logging.info('%s has been removed from package.provided', old_str) - - with open(package_file, 'w', encoding='utf-8') as f: - f.write(new_contents) - - -def update_virtual_rust(template_version: RustVersion, - new_version: RustVersion) -> None: - template_ebuild = find_ebuild_path(RUST_PATH.joinpath('../../virtual/rust'), - 'rust', template_version) - virtual_rust_dir = template_ebuild.parent - new_name = f'rust-{new_version}.ebuild' - new_ebuild = virtual_rust_dir.joinpath(new_name) - shutil.copyfile(template_ebuild, new_ebuild) - subprocess.check_call(['git', 'add', new_name], cwd=virtual_rust_dir) - - -def perform_step(state_file: pathlib.Path, - tmp_state_file: pathlib.Path, - completed_steps: Dict[str, Any], - step_name: str, - step_fn: Callable[[], T], - result_from_json: Optional[Callable[[Any], T]] = None, - result_to_json: Optional[Callable[[T], Any]] = None) -> T: - if step_name in completed_steps: - logging.info('Skipping previously completed step %s', step_name) - if result_from_json: - return result_from_json(completed_steps[step_name]) - return completed_steps[step_name] - - logging.info('Running step %s', step_name) - val = step_fn() - logging.info('Step %s complete', step_name) - if result_to_json: - completed_steps[step_name] = result_to_json(val) - else: - completed_steps[step_name] = val - - with tmp_state_file.open('w', encoding='utf-8') as f: - json.dump(completed_steps, f, indent=4) - tmp_state_file.rename(state_file) - return val + """Updates the MANIFEST for the ebuild at the given path.""" + ebuild = Path(ebuild_file) + ebuild_actions(ebuild.parent.name, ["manifest"]) + + +def update_rust_packages( + pkgatom: str, rust_version: RustVersion, add: bool +) -> None: + package_file = EBUILD_PREFIX.joinpath( + "profiles/targets/chromeos/package.provided" + ) + with open(package_file, encoding="utf-8") as f: + contents = f.read() + if add: + rust_packages_re = re.compile( + "^" + re.escape(pkgatom) + r"-\d+\.\d+\.\d+$", re.MULTILINE + ) + rust_packages = rust_packages_re.findall(contents) + # Assume all the rust packages are in alphabetical order, so insert + # the new version to the place after the last rust_packages + new_str = f"{pkgatom}-{rust_version}" + new_contents = contents.replace( + rust_packages[-1], f"{rust_packages[-1]}\n{new_str}" + ) + logging.info("%s has been inserted into package.provided", new_str) + else: + old_str = f"{pkgatom}-{rust_version}\n" + assert old_str in contents, f"{old_str!r} not found in package.provided" + new_contents = contents.replace(old_str, "") + logging.info("%s has been removed from package.provided", old_str) + + with open(package_file, "w", encoding="utf-8") as f: + f.write(new_contents) + + +def update_virtual_rust( + template_version: RustVersion, new_version: RustVersion +) -> None: + template_ebuild = find_ebuild_path( + EBUILD_PREFIX.joinpath("virtual/rust"), "rust", template_version + ) + virtual_rust_dir = template_ebuild.parent + new_name = f"rust-{new_version}.ebuild" + new_ebuild = virtual_rust_dir.joinpath(new_name) + shutil.copyfile(template_ebuild, new_ebuild) + subprocess.check_call(["git", "add", new_name], cwd=virtual_rust_dir) + + +def unmerge_package_if_installed(pkgatom: str) -> None: + """Unmerges a package if it is installed.""" + shpkg = shlex.quote(pkgatom) + subprocess.check_call( + [ + "sudo", + "bash", + "-c", + f"! emerge --pretend --quiet --unmerge {shpkg}" + f" || emerge --rage-clean {shpkg}", + ] + ) + + +def perform_step( + state_file: pathlib.Path, + tmp_state_file: pathlib.Path, + completed_steps: Dict[str, Any], + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, +) -> T: + if step_name in completed_steps: + logging.info("Skipping previously completed step %s", step_name) + if result_from_json: + return result_from_json(completed_steps[step_name]) + return completed_steps[step_name] + + logging.info("Running step %s", step_name) + val = step_fn() + logging.info("Step %s complete", step_name) + if result_to_json: + completed_steps[step_name] = result_to_json(val) + else: + completed_steps[step_name] = val + + with tmp_state_file.open("w", encoding="utf-8") as f: + json.dump(completed_steps, f, indent=4) + tmp_state_file.rename(state_file) + return val def prepare_uprev_from_json( - obj: Any) -> Optional[Tuple[RustVersion, str, RustVersion]]: - if not obj: - return None - version, ebuild_path, bootstrap_version = obj - return RustVersion(*version), ebuild_path, RustVersion(*bootstrap_version) - - -def create_rust_uprev(rust_version: RustVersion, - maybe_template_version: Optional[RustVersion], - skip_compile: bool, run_step: Callable[[], T]) -> None: - template_version, template_ebuild, old_bootstrap_version = run_step( - 'prepare uprev', - lambda: prepare_uprev(rust_version, maybe_template_version), - result_from_json=prepare_uprev_from_json, - ) - if template_ebuild is None: - return - - # The fetch steps will fail (on purpose) if the files they check for - # are not available on the mirror. To make them pass, fetch the - # required files yourself, verify their checksums, then upload them - # to the mirror. - run_step( - 'fetch bootstrap distfiles', lambda: fetch_bootstrap_distfiles( - old_bootstrap_version, template_version)) - run_step('fetch rust distfiles', lambda: fetch_rust_distfiles(rust_version)) - run_step('update bootstrap ebuild', lambda: update_bootstrap_ebuild( - template_version)) - run_step( - 'update bootstrap manifest', lambda: update_manifest(rust_bootstrap_path( - ).joinpath(f'rust-bootstrap-{template_version}.ebuild'))) - run_step('copy patches', lambda: copy_patches(RUST_PATH, template_version, - rust_version)) - ebuild_file = run_step( - 'create ebuild', lambda: create_ebuild(template_ebuild, rust_version)) - run_step( - 'update ebuild', lambda: update_ebuild(ebuild_file, template_version)) - run_step('update manifest to add new version', lambda: update_manifest( - Path(ebuild_file))) - if not skip_compile: + obj: Any, +) -> Optional[Tuple[RustVersion, str, RustVersion]]: + if not obj: + return None + version, ebuild_path, bootstrap_version = obj + return RustVersion(*version), ebuild_path, RustVersion(*bootstrap_version) + + +def create_rust_uprev( + rust_version: RustVersion, + maybe_template_version: Optional[RustVersion], + skip_compile: bool, + run_step: Callable[[], T], +) -> None: + template_version, template_ebuild, old_bootstrap_version = run_step( + "prepare uprev", + lambda: prepare_uprev(rust_version, maybe_template_version), + result_from_json=prepare_uprev_from_json, + ) + if template_ebuild is None: + return + + # The fetch steps will fail (on purpose) if the files they check for + # are not available on the mirror. To make them pass, fetch the + # required files yourself, verify their checksums, then upload them + # to the mirror. run_step( - 'emerge rust', lambda: subprocess.check_call( - ['sudo', 'emerge', 'dev-lang/rust'])) - run_step('insert version into rust packages', lambda: update_rust_packages( - rust_version, add=True)) - run_step('upgrade virtual/rust', lambda: update_virtual_rust( - template_version, rust_version)) + "fetch bootstrap distfiles", + lambda: fetch_bootstrap_distfiles( + old_bootstrap_version, template_version + ), + ) + run_step("fetch rust distfiles", lambda: fetch_rust_distfiles(rust_version)) + run_step( + "update bootstrap ebuild", + lambda: update_bootstrap_ebuild(template_version), + ) + run_step( + "update bootstrap manifest", + lambda: update_manifest( + rust_bootstrap_path().joinpath( + f"rust-bootstrap-{template_version}.ebuild" + ) + ), + ) + run_step( + "update bootstrap version", + lambda: update_bootstrap_version( + EBUILD_PREFIX.joinpath("eclass/cros-rustc.eclass"), template_version + ), + ) + run_step( + "copy patches", + lambda: copy_patches(RUST_PATH, template_version, rust_version), + ) + template_host_ebuild = EBUILD_PREFIX.joinpath( + f"dev-lang/rust-host/rust-host-{template_version}.ebuild" + ) + host_file = run_step( + "create host ebuild", + lambda: create_ebuild( + template_host_ebuild, "dev-lang/rust-host", rust_version + ), + ) + run_step( + "update host manifest to add new version", + lambda: update_manifest(Path(host_file)), + ) + target_file = run_step( + "create target ebuild", + lambda: create_ebuild(template_ebuild, "dev-lang/rust", rust_version), + ) + run_step( + "update target manifest to add new version", + lambda: update_manifest(Path(target_file)), + ) + if not skip_compile: + run_step("build packages", lambda: rebuild_packages(rust_version)) + run_step( + "insert host version into rust packages", + lambda: update_rust_packages( + "dev-lang/rust-host", rust_version, add=True + ), + ) + run_step( + "insert target version into rust packages", + lambda: update_rust_packages("dev-lang/rust", rust_version, add=True), + ) + run_step( + "upgrade virtual/rust", + lambda: update_virtual_rust(template_version, rust_version), + ) def find_rust_versions_in_chroot() -> List[Tuple[RustVersion, str]]: - return [(RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) - for x in os.listdir(RUST_PATH) if x.endswith('.ebuild')] + return [ + (RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) + for x in os.listdir(RUST_PATH) + if x.endswith(".ebuild") + ] -def find_oldest_rust_version_in_chroot() -> Tuple[RustVersion, str]: - rust_versions = find_rust_versions_in_chroot() - if len(rust_versions) <= 1: - raise RuntimeError('Expect to find more than one Rust versions') - return min(rust_versions) +def find_oldest_rust_version_in_chroot() -> RustVersion: + rust_versions = find_rust_versions_in_chroot() + if len(rust_versions) <= 1: + raise RuntimeError("Expect to find more than one Rust versions") + return min(rust_versions)[0] def find_ebuild_for_rust_version(version: RustVersion) -> str: - rust_ebuilds = [ - ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version - ] - if not rust_ebuilds: - raise ValueError(f'No Rust ebuilds found matching {version}') - if len(rust_ebuilds) > 1: - raise ValueError(f'Multiple Rust ebuilds found matching {version}: ' - f'{rust_ebuilds}') - return rust_ebuilds[0] + rust_ebuilds = [ + ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version + ] + if not rust_ebuilds: + raise ValueError(f"No Rust ebuilds found matching {version}") + if len(rust_ebuilds) > 1: + raise ValueError( + f"Multiple Rust ebuilds found matching {version}: " + f"{rust_ebuilds}" + ) + return rust_ebuilds[0] + + +def rebuild_packages(version: RustVersion): + """Rebuild packages modified by this script.""" + # Remove all packages we modify to avoid depending on preinstalled + # versions. This ensures that the packages can really be built. + packages = [ + "dev-lang/rust", + "dev-lang/rust-host", + "dev-lang/rust-bootstrap", + ] + for pkg in packages: + unmerge_package_if_installed(pkg) + # Mention only dev-lang/rust explicitly, so that others are pulled + # in as dependencies (letting us detect dependency errors). + # Packages we modify are listed in --usepkg-exclude to ensure they + # are built from source. + try: + subprocess.check_call( + [ + "sudo", + "emerge", + "--quiet-build", + "--usepkg-exclude", + " ".join(packages), + f"=dev-lang/rust-{version}", + ] + ) + except: + logging.warning( + "Failed to build dev-lang/rust or one of its dependencies." + " If necessary, you can restore rust and rust-host from" + " binary packages:\n sudo emerge --getbinpkgonly dev-lang/rust" + ) + raise + + +def remove_ebuild_version(path: os.PathLike, name: str, version: RustVersion): + """Remove the specified version of an ebuild. + + Removes {path}/{name}-{version}.ebuild and {path}/{name}-{version}-*.ebuild + using git rm. + + Args: + path: The directory in which the ebuild files are. + name: The name of the package (e.g. 'rust'). + version: The version of the ebuild to remove. + """ + path = Path(path) + pattern = f"{name}-{version}-*.ebuild" + matches = list(path.glob(pattern)) + ebuild = path / f"{name}-{version}.ebuild" + if ebuild.exists(): + matches.append(ebuild) + if not matches: + logging.warning( + "No ebuilds matching %s version %s in %r", name, version, str(path) + ) + for m in matches: + remove_files(m.name, path) def remove_files(filename: str, path: str) -> None: - subprocess.check_call(['git', 'rm', filename], cwd=path) - - -def remove_rust_bootstrap_version(version: RustVersion, - run_step: Callable[[], T]) -> None: - prefix = f'rust-bootstrap-{version}' - run_step('remove old bootstrap ebuild', lambda: remove_files( - f'{prefix}*.ebuild', rust_bootstrap_path())) - ebuild_file = find_ebuild_for_package('rust-bootstrap') - run_step('update bootstrap manifest to delete old version', lambda: - update_manifest(ebuild_file)) - - -def remove_rust_uprev(rust_version: Optional[RustVersion], - run_step: Callable[[], T]) -> None: - def find_desired_rust_version(): - if rust_version: - return rust_version, find_ebuild_for_rust_version(rust_version) - return find_oldest_rust_version_in_chroot() - - def find_desired_rust_version_from_json(obj: Any) -> Tuple[RustVersion, str]: - version, ebuild_path = obj - return RustVersion(*version), ebuild_path - - delete_version, delete_ebuild = run_step( - 'find rust version to delete', - find_desired_rust_version, - result_from_json=find_desired_rust_version_from_json, - ) - run_step( - 'remove patches', lambda: remove_files( - f'files/rust-{delete_version}-*.patch', RUST_PATH)) - run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH)) - ebuild_file = find_ebuild_for_package('rust') - run_step('update manifest to delete old version', lambda: update_manifest( - ebuild_file)) - run_step('remove version from rust packages', lambda: update_rust_packages( - delete_version, add=False)) - run_step('remove virtual/rust', lambda: remove_virtual_rust(delete_version)) + subprocess.check_call(["git", "rm", filename], cwd=path) + + +def remove_rust_bootstrap_version( + version: RustVersion, run_step: Callable[[], T] +) -> None: + run_step( + "remove old bootstrap ebuild", + lambda: remove_ebuild_version( + rust_bootstrap_path(), "rust-bootstrap", version + ), + ) + ebuild_file = find_ebuild_for_package("rust-bootstrap") + run_step( + "update bootstrap manifest to delete old version", + lambda: update_manifest(ebuild_file), + ) + + +def remove_rust_uprev( + rust_version: Optional[RustVersion], run_step: Callable[[], T] +) -> None: + def find_desired_rust_version() -> RustVersion: + if rust_version: + return rust_version + return find_oldest_rust_version_in_chroot() + + def find_desired_rust_version_from_json(obj: Any) -> RustVersion: + return RustVersion(*obj) + + delete_version = run_step( + "find rust version to delete", + find_desired_rust_version, + result_from_json=find_desired_rust_version_from_json, + ) + run_step( + "remove patches", + lambda: remove_files(f"files/rust-{delete_version}-*.patch", RUST_PATH), + ) + run_step( + "remove target ebuild", + lambda: remove_ebuild_version(RUST_PATH, "rust", delete_version), + ) + run_step( + "remove host ebuild", + lambda: remove_ebuild_version( + EBUILD_PREFIX.joinpath("dev-lang/rust-host"), + "rust-host", + delete_version, + ), + ) + target_file = find_ebuild_for_package("rust") + run_step( + "update target manifest to delete old version", + lambda: update_manifest(target_file), + ) + run_step( + "remove target version from rust packages", + lambda: update_rust_packages( + "dev-lang/rust", delete_version, add=False + ), + ) + host_file = find_ebuild_for_package("rust-host") + run_step( + "update host manifest to delete old version", + lambda: update_manifest(host_file), + ) + run_step( + "remove host version from rust packages", + lambda: update_rust_packages( + "dev-lang/rust-host", delete_version, add=False + ), + ) + run_step("remove virtual/rust", lambda: remove_virtual_rust(delete_version)) def remove_virtual_rust(delete_version: RustVersion) -> None: - ebuild = find_ebuild_path(RUST_PATH.joinpath('../../virtual/rust'), 'rust', - delete_version) - subprocess.check_call(['git', 'rm', str(ebuild.name)], cwd=ebuild.parent) + remove_ebuild_version( + EBUILD_PREFIX.joinpath("virtual/rust"), "rust", delete_version + ) def rust_bootstrap_path() -> Path: - return RUST_PATH.joinpath('../rust-bootstrap') + return EBUILD_PREFIX.joinpath("dev-lang/rust-bootstrap") def create_new_repo(rust_version: RustVersion) -> None: - output = get_command_output(['git', 'status', '--porcelain'], cwd=RUST_PATH) - if output: - raise RuntimeError( - f'{RUST_PATH} has uncommitted changes, please either discard them ' - 'or commit them.') - git.CreateBranch(RUST_PATH, f'rust-to-{rust_version}') + output = get_command_output( + ["git", "status", "--porcelain"], cwd=EBUILD_PREFIX + ) + if output: + raise RuntimeError( + f"{EBUILD_PREFIX} has uncommitted changes, please either discard " + "them or commit them." + ) + git.CreateBranch(EBUILD_PREFIX, f"rust-to-{rust_version}") def build_cross_compiler() -> None: - # Get target triples in ebuild - rust_ebuild = find_ebuild_for_package('rust') - with open(rust_ebuild, encoding='utf-8') as f: - contents = f.read() - - target_triples_re = re.compile(r'RUSTC_TARGET_TRIPLES=\(([^)]+)\)') - m = target_triples_re.search(contents) - assert m, 'RUST_TARGET_TRIPLES not found in rust ebuild' - target_triples = m.group(1).strip().split('\n') - - compiler_targets_to_install = [ - target.strip() for target in target_triples if 'cros-' in target - ] - for target in target_triples: - if 'cros-' not in target: - continue - target = target.strip() - - # We also always need arm-none-eabi, though it's not mentioned in - # RUSTC_TARGET_TRIPLES. - compiler_targets_to_install.append('arm-none-eabi') - - logging.info('Emerging cross compilers %s', compiler_targets_to_install) - subprocess.check_call( - ['sudo', 'emerge', '-j', '-G'] + - [f'cross-{target}/gcc' for target in compiler_targets_to_install]) + # Get target triples in ebuild + rust_ebuild = find_ebuild_for_package("rust") + with open(rust_ebuild, encoding="utf-8") as f: + contents = f.read() + + target_triples_re = re.compile(r"RUSTC_TARGET_TRIPLES=\(([^)]+)\)") + m = target_triples_re.search(contents) + assert m, "RUST_TARGET_TRIPLES not found in rust ebuild" + target_triples = m.group(1).strip().split("\n") + + compiler_targets_to_install = [ + target.strip() for target in target_triples if "cros-" in target + ] + for target in target_triples: + if "cros-" not in target: + continue + target = target.strip() + + # We also always need arm-none-eabi, though it's not mentioned in + # RUSTC_TARGET_TRIPLES. + compiler_targets_to_install.append("arm-none-eabi") + + logging.info("Emerging cross compilers %s", compiler_targets_to_install) + subprocess.check_call( + ["sudo", "emerge", "-j", "-G"] + + [f"cross-{target}/gcc" for target in compiler_targets_to_install] + ) def create_new_commit(rust_version: RustVersion) -> None: - subprocess.check_call(['git', 'add', '-A'], cwd=RUST_PATH) - messages = [ - f'[DO NOT SUBMIT] dev-lang/rust: upgrade to Rust {rust_version}', - '', - 'This CL is created by rust_uprev tool automatically.' - '', - 'BUG=None', - 'TEST=Use CQ to test the new Rust version', - ] - git.UploadChanges(RUST_PATH, f'rust-to-{rust_version}', messages) + subprocess.check_call(["git", "add", "-A"], cwd=EBUILD_PREFIX) + messages = [ + f"[DO NOT SUBMIT] dev-lang/rust: upgrade to Rust {rust_version}", + "", + "This CL is created by rust_uprev tool automatically." "", + "BUG=None", + "TEST=Use CQ to test the new Rust version", + ] + git.UploadChanges(EBUILD_PREFIX, f"rust-to-{rust_version}", messages) def main() -> None: - if not chroot.InChroot(): - raise RuntimeError('This script must be executed inside chroot') - - logging.basicConfig(level=logging.INFO) - - args = parse_commandline_args() - - state_file = pathlib.Path(args.state_file) - tmp_state_file = state_file.with_suffix('.tmp') - - try: - with state_file.open(encoding='utf-8') as f: - completed_steps = json.load(f) - except FileNotFoundError: - completed_steps = {} - - def run_step( - step_name: str, - step_fn: Callable[[], T], - result_from_json: Optional[Callable[[Any], T]] = None, - result_to_json: Optional[Callable[[T], Any]] = None, - ) -> T: - return perform_step(state_file, tmp_state_file, completed_steps, step_name, - step_fn, result_from_json, result_to_json) - - if args.subparser_name == 'create': - create_rust_uprev(args.rust_version, args.template, args.skip_compile, - run_step) - elif args.subparser_name == 'remove': - remove_rust_uprev(args.rust_version, run_step) - elif args.subparser_name == 'remove-bootstrap': - remove_rust_bootstrap_version(args.version, run_step) - else: - # If you have added more subparser_name, please also add the handlers above - assert args.subparser_name == 'roll' - run_step('create new repo', lambda: create_new_repo(args.uprev)) - if not args.skip_cross_compiler: - run_step('build cross compiler', build_cross_compiler) - create_rust_uprev(args.uprev, args.template, args.skip_compile, run_step) - remove_rust_uprev(args.remove, run_step) - bootstrap_version = prepare_uprev_from_json( - completed_steps['prepare uprev'])[2] - remove_rust_bootstrap_version(bootstrap_version, run_step) - if not args.no_upload: - run_step('create rust uprev CL', lambda: create_new_commit(args.uprev)) - - -if __name__ == '__main__': - sys.exit(main()) + if not chroot.InChroot(): + raise RuntimeError("This script must be executed inside chroot") + + logging.basicConfig(level=logging.INFO) + + args = parse_commandline_args() + + state_file = pathlib.Path(args.state_file) + tmp_state_file = state_file.with_suffix(".tmp") + + try: + with state_file.open(encoding="utf-8") as f: + completed_steps = json.load(f) + except FileNotFoundError: + completed_steps = {} + + def run_step( + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, + ) -> T: + return perform_step( + state_file, + tmp_state_file, + completed_steps, + step_name, + step_fn, + result_from_json, + result_to_json, + ) + + if args.subparser_name == "create": + create_rust_uprev( + args.rust_version, args.template, args.skip_compile, run_step + ) + elif args.subparser_name == "remove": + remove_rust_uprev(args.rust_version, run_step) + elif args.subparser_name == "remove-bootstrap": + remove_rust_bootstrap_version(args.version, run_step) + else: + # If you have added more subparser_name, please also add the handlers above + assert args.subparser_name == "roll" + run_step("create new repo", lambda: create_new_repo(args.uprev)) + if not args.skip_cross_compiler: + run_step("build cross compiler", build_cross_compiler) + create_rust_uprev( + args.uprev, args.template, args.skip_compile, run_step + ) + remove_rust_uprev(args.remove, run_step) + bootstrap_version = prepare_uprev_from_json( + completed_steps["prepare uprev"] + )[2] + remove_rust_bootstrap_version(bootstrap_version, run_step) + if not args.no_upload: + run_step( + "create rust uprev CL", lambda: create_new_commit(args.uprev) + ) + + +if __name__ == "__main__": + sys.exit(main()) |