diff options
Diffstat (limited to 'rust_tools')
| -rwxr-xr-x | rust_tools/copy_rust_bootstrap.py | 192 | ||||
| -rwxr-xr-x | rust_tools/rust_uprev.py | 1522 | ||||
| -rwxr-xr-x | rust_tools/rust_uprev_test.py | 1064 | ||||
| -rwxr-xr-x | rust_tools/rust_watch.py | 617 | ||||
| -rwxr-xr-x | rust_tools/rust_watch_test.py | 304 |
5 files changed, 2173 insertions, 1526 deletions
diff --git a/rust_tools/copy_rust_bootstrap.py b/rust_tools/copy_rust_bootstrap.py new file mode 100755 index 00000000..5da8007f --- /dev/null +++ b/rust_tools/copy_rust_bootstrap.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +# Copyright 2022 The ChromiumOS Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Copies rust-bootstrap artifacts from an SDK build to localmirror. + +We use localmirror to host these artifacts, but they've changed a bit over +time, so simply `gsutil.py cp $FROM $TO` doesn't work. This script allows the +convenience of the old `cp` command. +""" + +import argparse +import logging +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from typing import List + + +_LOCALMIRROR_ROOT = 'gs://chromeos-localmirror/distfiles/' + + +def _is_in_chroot() -> bool: + return Path('/etc/cros_chroot_version').exists() + + +def _ensure_pbzip2_is_installed(): + if shutil.which('pbzip2'): + return + + logging.info('Auto-installing pbzip2...') + subprocess.run(['sudo', 'emerge', '-G', 'pbzip2'], check=True) + + +def _determine_target_path(sdk_path: str) -> str: + """Determine where `sdk_path` should sit in localmirror.""" + gs_prefix = 'gs://' + if not sdk_path.startswith(gs_prefix): + raise ValueError(f'Invalid GS path: {sdk_path!r}') + + file_name = Path(sdk_path[len(gs_prefix):]).name + return _LOCALMIRROR_ROOT + file_name + + +def _download(remote_path: str, local_file: Path): + """Downloads the given gs:// path to the given local file.""" + logging.info('Downloading %s -> %s', remote_path, local_file) + subprocess.run( + ['gsutil.py', 'cp', remote_path, + str(local_file)], + check=True, + ) + + +def _debinpkgify(binpkg_file: Path) -> Path: + """Converts a binpkg into the files it installs. + + Note that this function makes temporary files in the same directory as + `binpkg_file`. It makes no attempt to clean them up. + """ + logging.info('Converting %s from a binpkg...', binpkg_file) + + # The SDK builder produces binary packages: + # https://wiki.gentoo.org/wiki/Binary_package_guide + # + # Which means that `binpkg_file` is in the XPAK format. We want to split + # that out, and recompress it from zstd (which is the compression format + # that CrOS uses) to bzip2 (which is what we've historically used, and + # which is what our ebuild expects). + tmpdir = binpkg_file.parent + + def _mkstemp(suffix=None) -> str: + fd, file_path = tempfile.mkstemp(dir=tmpdir, suffix=suffix) + os.close(fd) + return Path(file_path) + + # First, split the actual artifacts that land in the chroot out to + # `temp_file`. + artifacts_file = _mkstemp() + logging.info('Extracting artifacts from %s into %s...', binpkg_file, + artifacts_file) + with artifacts_file.open('wb') as f: + subprocess.run( + [ + 'qtbz2', + '-s', + '-t', + '-O', + str(binpkg_file), + ], + check=True, + stdout=f, + ) + + decompressed_artifacts_file = _mkstemp() + decompressed_artifacts_file.unlink() + logging.info('Decompressing artifacts from %s to %s...', artifacts_file, + decompressed_artifacts_file) + subprocess.run( + [ + 'zstd', + '-d', + str(artifacts_file), + '-o', + str(decompressed_artifacts_file), + ], + check=True, + ) + + # Finally, recompress it as a tbz2. + tbz2_file = _mkstemp('.tbz2') + logging.info( + 'Recompressing artifacts from %s to %s (this may take a while)...', + decompressed_artifacts_file, tbz2_file) + with tbz2_file.open('wb') as f: + subprocess.run( + [ + 'pbzip2', + '-9', + '-c', + str(decompressed_artifacts_file), + ], + check=True, + stdout=f, + ) + return tbz2_file + + +def _upload(local_file: Path, remote_path: str, force: bool): + """Uploads the local file to the given gs:// path.""" + logging.info('Uploading %s -> %s', local_file, remote_path) + cmd_base = ['gsutil.py', 'cp', '-a', 'public-read'] + if not force: + cmd_base.append('-n') + subprocess.run( + cmd_base + [str(local_file), remote_path], + check=True, + stdin=subprocess.DEVNULL, + ) + + +def main(argv: List[str]): + logging.basicConfig( + format='>> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: ' + '%(message)s', + level=logging.INFO, + ) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + 'sdk_artifact', + help='Path to the SDK rust-bootstrap artifact to copy. e.g., ' + 'gs://chromeos-prebuilt/host/amd64/amd64-host/' + 'chroot-2022.07.12.134334/packages/dev-lang/' + 'rust-bootstrap-1.59.0.tbz2.') + parser.add_argument( + '-n', + '--dry-run', + action='store_true', + help='Do everything except actually uploading the artifact.') + parser.add_argument( + '--force', + action='store_true', + help='Upload the artifact even if one exists in localmirror already.') + opts = parser.parse_args(argv) + + if not _is_in_chroot(): + parser.error('Run me from within the chroot.') + _ensure_pbzip2_is_installed() + + target_path = _determine_target_path(opts.sdk_artifact) + with tempfile.TemporaryDirectory() as tempdir: + download_path = Path(tempdir) / 'sdk_artifact' + _download(opts.sdk_artifact, download_path) + file_to_upload = _debinpkgify(download_path) + if opts.dry_run: + logging.info('--dry-run specified; skipping upload of %s to %s', + file_to_upload, target_path) + else: + _upload(file_to_upload, target_path, opts.force) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py index 011639df..382d991a 100755 --- a/rust_tools/rust_uprev.py +++ b/rust_tools/rust_uprev.py @@ -1,416 +1,437 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tool to automatically generate a new Rust uprev CL. -This tool is intended to automatically generate a CL to uprev Rust to a -newer version in Chrome OS, including creating a new Rust version or -removing an old version. It's based on -src/third_party/chromiumos-overlay/dev-lang/rust/UPGRADE.md. When using -the tool, the progress can be saved to a JSON file, so the user can resume -the process after a failing step is fixed. Example usage to create a new -version: - -1. (inside chroot) $ ./rust_tools/rust_uprev.py - --state_file /tmp/state-file.json - create --rust_version 1.45.0 -2. Step "compile rust" failed due to the patches can't apply to new version -3. Manually fix the patches -4. Execute the command in step 1 again. +This tool is intended to automatically generate a CL to uprev Rust to +a newer version in Chrome OS, including creating a new Rust version or +removing an old version. When using the tool, the progress can be +saved to a JSON file, so the user can resume the process after a +failing step is fixed. Example usage to create a new version: + +1. (inside chroot) $ ./rust_tools/rust_uprev.py \\ + --state_file /tmp/rust-to-1.60.0.json \\ + roll --uprev 1.60.0 +2. Step "compile rust" failed due to the patches can't apply to new version. +3. Manually fix the patches. +4. Execute the command in step 1 again, but add "--continue" before "roll". 5. Iterate 1-4 for each failed step until the tool passes. -Replace `create --rust_version 1.45.0` with `remove --rust_version 1.43.0` -if you want to remove all 1.43.0 related stuff in the same CL. Remember to -use a different state file if you choose to run different subcommands. - -If you want a hammer that can do everything for you, use the subcommand -`roll`. It can create a Rust uprev CL with `create` and `remove` and upload -the CL to chromium code review. +Besides "roll", the tool also support subcommands that perform +various parts of an uprev. See `--help` for all available options. """ import argparse -import pathlib import json import logging import os +import pathlib +from pathlib import Path import re +import shlex import shutil import subprocess import sys -from pathlib import Path from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple -from llvm_tools import chroot, git +from llvm_tools import chroot +from llvm_tools import git + -EQUERY = 'equery' -GSUTIL = 'gsutil.py' -MIRROR_PATH = 'gs://chromeos-localmirror/distfiles' -RUST_PATH = Path( - '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust') +EQUERY = "equery" +GSUTIL = "gsutil.py" +MIRROR_PATH = "gs://chromeos-localmirror/distfiles" +EBUILD_PREFIX = Path("/mnt/host/source/src/third_party/chromiumos-overlay") +RUST_PATH = Path(EBUILD_PREFIX, "dev-lang", "rust") def get_command_output(command: List[str], *args, **kwargs) -> str: - return subprocess.check_output(command, encoding='utf-8', *args, - **kwargs).strip() + return subprocess.check_output( + command, encoding="utf-8", *args, **kwargs + ).strip() def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str: - return subprocess.run(command, - check=False, - stdout=subprocess.PIPE, - encoding='utf-8', - *args, - **kwargs).stdout.strip() + return subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + encoding="utf-8", + *args, + **kwargs, + ).stdout.strip() class RustVersion(NamedTuple): - """NamedTuple represents a Rust version""" - major: int - minor: int - patch: int - - def __str__(self): - return f'{self.major}.{self.minor}.{self.patch}' - - @staticmethod - def parse_from_ebuild(ebuild_name: str) -> 'RustVersion': - input_re = re.compile(r'^rust-' - r'(?P<major>\d+)\.' - r'(?P<minor>\d+)\.' - r'(?P<patch>\d+)' - r'(:?-r\d+)?' - r'\.ebuild$') - m = input_re.match(ebuild_name) - assert m, f'failed to parse {ebuild_name!r}' - return RustVersion(int(m.group('major')), int(m.group('minor')), - int(m.group('patch'))) - - @staticmethod - def parse(x: str) -> 'RustVersion': - input_re = re.compile(r'^(?:rust-)?' - r'(?P<major>\d+)\.' - r'(?P<minor>\d+)\.' - r'(?P<patch>\d+)' - r'(?:.ebuild)?$') - m = input_re.match(x) - assert m, f'failed to parse {x!r}' - return RustVersion(int(m.group('major')), int(m.group('minor')), - int(m.group('patch'))) + """NamedTuple represents a Rust version""" + + major: int + minor: int + patch: int + + def __str__(self): + return f"{self.major}.{self.minor}.{self.patch}" + + @staticmethod + def parse_from_ebuild(ebuild_name: str) -> "RustVersion": + input_re = re.compile( + r"^rust-" + r"(?P<major>\d+)\." + r"(?P<minor>\d+)\." + r"(?P<patch>\d+)" + r"(:?-r\d+)?" + r"\.ebuild$" + ) + m = input_re.match(ebuild_name) + assert m, f"failed to parse {ebuild_name!r}" + return RustVersion( + int(m.group("major")), int(m.group("minor")), int(m.group("patch")) + ) + + @staticmethod + def parse(x: str) -> "RustVersion": + input_re = re.compile( + r"^(?:rust-)?" + r"(?P<major>\d+)\." + r"(?P<minor>\d+)\." + r"(?P<patch>\d+)" + r"(?:.ebuild)?$" + ) + m = input_re.match(x) + assert m, f"failed to parse {x!r}" + return RustVersion( + int(m.group("major")), int(m.group("minor")), int(m.group("patch")) + ) def compute_rustc_src_name(version: RustVersion) -> str: - return f'rustc-{version}-src.tar.gz' + return f"rustc-{version}-src.tar.gz" def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str: - return f'rust-bootstrap-{version}.tbz2' + return f"rust-bootstrap-{version}.tbz2" def find_ebuild_for_package(name: str) -> os.PathLike: - """Returns the path to the ebuild for the named package.""" - return get_command_output([EQUERY, 'w', name]) - - -def find_ebuild_path(directory: Path, - name: str, - version: Optional[RustVersion] = None) -> Path: - """Finds an ebuild in a directory. - - Returns the path to the ebuild file. Asserts if there is not - exactly one match. The match is constrained by name and optionally - by version, but can match any patch level. E.g. "rust" version - 1.3.4 can match rust-1.3.4.ebuild but also rust-1.3.4-r6.ebuild. - """ - if version: - pattern = f'{name}-{version}*.ebuild' - else: - pattern = f'{name}-*.ebuild' - matches = list(Path(directory).glob(pattern)) - assert len(matches) == 1, matches - return matches[0] + """Returns the path to the ebuild for the named package.""" + return get_command_output([EQUERY, "w", name]) + + +def find_ebuild_path( + directory: Path, name: str, version: Optional[RustVersion] = None +) -> Path: + """Finds an ebuild in a directory. + + Returns the path to the ebuild file. The match is constrained by + name and optionally by version, but can match any patch level. + E.g. "rust" version 1.3.4 can match rust-1.3.4.ebuild but also + rust-1.3.4-r6.ebuild. + + The expectation is that there is only one matching ebuild, and + an assert is raised if this is not the case. However, symlinks to + ebuilds in the same directory are ignored, so having a + rust-x.y.z-rn.ebuild symlink to rust-x.y.z.ebuild is allowed. + """ + if version: + pattern = f"{name}-{version}*.ebuild" + else: + pattern = f"{name}-*.ebuild" + matches = set(directory.glob(pattern)) + result = [] + # Only count matches that are not links to other matches. + for m in matches: + try: + target = os.readlink(directory / m) + except OSError: + # Getting here means the match is not a symlink to one of + # the matching ebuilds, so add it to the result list. + result.append(m) + continue + if directory / target not in matches: + result.append(m) + assert len(result) == 1, result + return result[0] def get_rust_bootstrap_version(): - """Get the version of the current rust-bootstrap package.""" - bootstrap_ebuild = find_ebuild_path(rust_bootstrap_path(), 'rust-bootstrap') - m = re.match(r'^rust-bootstrap-(\d+).(\d+).(\d+)', bootstrap_ebuild.name) - assert m, bootstrap_ebuild.name - return RustVersion(int(m.group(1)), int(m.group(2)), int(m.group(3))) + """Get the version of the current rust-bootstrap package.""" + bootstrap_ebuild = find_ebuild_path(rust_bootstrap_path(), "rust-bootstrap") + m = re.match(r"^rust-bootstrap-(\d+).(\d+).(\d+)", bootstrap_ebuild.name) + assert m, bootstrap_ebuild.name + return RustVersion(int(m.group(1)), int(m.group(2)), int(m.group(3))) def parse_commandline_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--state_file', - required=True, - help='A state file to hold previous completed steps. If the file ' - 'exists, it needs to be used together with --continue or --restart. ' - 'If not exist (do not use --continue in this case), we will create a ' - 'file for you.', - ) - parser.add_argument( - '--restart', - action='store_true', - help='Restart from the first step. Ignore the completed steps in ' - 'the state file', - ) - parser.add_argument( - '--continue', - dest='cont', - action='store_true', - help='Continue the steps from the state file', - ) - - create_parser_template = argparse.ArgumentParser(add_help=False) - create_parser_template.add_argument( - '--template', - type=RustVersion.parse, - default=None, - help='A template to use for creating a Rust uprev from, in the form ' - 'a.b.c The ebuild has to exist in the chroot. If not specified, the ' - 'tool will use the current Rust version in the chroot as template.', - ) - create_parser_template.add_argument( - '--skip_compile', - action='store_true', - help='Skip compiling rust to test the tool. Only for testing', - ) - - subparsers = parser.add_subparsers(dest='subparser_name') - subparser_names = [] - subparser_names.append('create') - create_parser = subparsers.add_parser( - 'create', - parents=[create_parser_template], - help='Create changes uprevs Rust to a new version', - ) - create_parser.add_argument( - '--rust_version', - type=RustVersion.parse, - required=True, - help='Rust version to uprev to, in the form a.b.c', - ) - - subparser_names.append('remove') - remove_parser = subparsers.add_parser( - 'remove', - help='Clean up old Rust version from chroot', - ) - remove_parser.add_argument( - '--rust_version', - type=RustVersion.parse, - default=None, - help='Rust version to remove, in the form a.b.c If not ' - 'specified, the tool will remove the oldest version in the chroot', - ) - - subparser_names.append('remove-bootstrap') - remove_bootstrap_parser = subparsers.add_parser( - 'remove-bootstrap', - help='Remove an old rust-bootstrap version', - ) - remove_bootstrap_parser.add_argument( - '--version', - type=RustVersion.parse, - required=True, - help='rust-bootstrap version to remove', - ) - - subparser_names.append('roll') - roll_parser = subparsers.add_parser( - 'roll', - parents=[create_parser_template], - help='A command can create and upload a Rust uprev CL, including ' - 'preparing the repo, creating new Rust uprev, deleting old uprev, ' - 'and upload a CL to crrev.', - ) - roll_parser.add_argument( - '--uprev', - type=RustVersion.parse, - required=True, - help='Rust version to uprev to, in the form a.b.c', - ) - roll_parser.add_argument( - '--remove', - type=RustVersion.parse, - default=None, - help='Rust version to remove, in the form a.b.c If not ' - 'specified, the tool will remove the oldest version in the chroot', - ) - roll_parser.add_argument( - '--skip_cross_compiler', - action='store_true', - help='Skip updating cross-compiler in the chroot', - ) - roll_parser.add_argument( - '--no_upload', - action='store_true', - help='If specified, the tool will not upload the CL for review', - ) - - args = parser.parse_args() - if args.subparser_name not in subparser_names: - parser.error('one of %s must be specified' % subparser_names) - - if args.cont and args.restart: - parser.error('Please select either --continue or --restart') - - if os.path.exists(args.state_file): - if not args.cont and not args.restart: - parser.error('State file exists, so you should either --continue ' - 'or --restart') - if args.cont and not os.path.exists(args.state_file): - parser.error('Indicate --continue but the state file does not exist') - - if args.restart and os.path.exists(args.state_file): - os.remove(args.state_file) - - return args - - -def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion] - ) -> Optional[Tuple[RustVersion, str, RustVersion]]: - if template is None: - ebuild_path = find_ebuild_for_package('rust') - ebuild_name = os.path.basename(ebuild_path) - template_version = RustVersion.parse_from_ebuild(ebuild_name) - else: - ebuild_path = find_ebuild_for_rust_version(template) - template_version = template - - bootstrap_version = get_rust_bootstrap_version() - - if rust_version <= template_version: - logging.info( - 'Requested version %s is not newer than the template version %s.', - rust_version, template_version) - return None + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--state_file", + required=True, + help="A state file to hold previous completed steps. If the file " + "exists, it needs to be used together with --continue or --restart. " + "If not exist (do not use --continue in this case), we will create a " + "file for you.", + ) + parser.add_argument( + "--restart", + action="store_true", + help="Restart from the first step. Ignore the completed steps in " + "the state file", + ) + parser.add_argument( + "--continue", + dest="cont", + action="store_true", + help="Continue the steps from the state file", + ) + + create_parser_template = argparse.ArgumentParser(add_help=False) + create_parser_template.add_argument( + "--template", + type=RustVersion.parse, + default=None, + help="A template to use for creating a Rust uprev from, in the form " + "a.b.c The ebuild has to exist in the chroot. If not specified, the " + "tool will use the current Rust version in the chroot as template.", + ) + create_parser_template.add_argument( + "--skip_compile", + action="store_true", + help="Skip compiling rust to test the tool. Only for testing", + ) - logging.info('Template Rust version is %s (ebuild: %r)', template_version, - ebuild_path) - logging.info('rust-bootstrap version is %s', bootstrap_version) + subparsers = parser.add_subparsers(dest="subparser_name") + subparser_names = [] + subparser_names.append("create") + create_parser = subparsers.add_parser( + "create", + parents=[create_parser_template], + help="Create changes uprevs Rust to a new version", + ) + create_parser.add_argument( + "--rust_version", + type=RustVersion.parse, + required=True, + help="Rust version to uprev to, in the form a.b.c", + ) - return template_version, ebuild_path, bootstrap_version + subparser_names.append("remove") + remove_parser = subparsers.add_parser( + "remove", + help="Clean up old Rust version from chroot", + ) + remove_parser.add_argument( + "--rust_version", + type=RustVersion.parse, + default=None, + help="Rust version to remove, in the form a.b.c If not " + "specified, the tool will remove the oldest version in the chroot", + ) + subparser_names.append("remove-bootstrap") + remove_bootstrap_parser = subparsers.add_parser( + "remove-bootstrap", + help="Remove an old rust-bootstrap version", + ) + remove_bootstrap_parser.add_argument( + "--version", + type=RustVersion.parse, + required=True, + help="rust-bootstrap version to remove", + ) -def copy_patches(directory: Path, template_version: RustVersion, - new_version: RustVersion) -> None: - patch_path = directory.joinpath('files') - prefix = '%s-%s-' % (directory.name, template_version) - new_prefix = '%s-%s-' % (directory.name, new_version) - for f in os.listdir(patch_path): - if not f.startswith(prefix): - continue - logging.info('Copy patch %s to new version', f) - new_name = f.replace(str(template_version), str(new_version)) - shutil.copyfile( - os.path.join(patch_path, f), - os.path.join(patch_path, new_name), + subparser_names.append("roll") + roll_parser = subparsers.add_parser( + "roll", + parents=[create_parser_template], + help="A command can create and upload a Rust uprev CL, including " + "preparing the repo, creating new Rust uprev, deleting old uprev, " + "and upload a CL to crrev.", + ) + roll_parser.add_argument( + "--uprev", + type=RustVersion.parse, + required=True, + help="Rust version to uprev to, in the form a.b.c", + ) + roll_parser.add_argument( + "--remove", + type=RustVersion.parse, + default=None, + help="Rust version to remove, in the form a.b.c If not " + "specified, the tool will remove the oldest version in the chroot", + ) + roll_parser.add_argument( + "--skip_cross_compiler", + action="store_true", + help="Skip updating cross-compiler in the chroot", + ) + roll_parser.add_argument( + "--no_upload", + action="store_true", + help="If specified, the tool will not upload the CL for review", ) - subprocess.check_call(['git', 'add', f'{new_prefix}*.patch'], cwd=patch_path) + args = parser.parse_args() + if args.subparser_name not in subparser_names: + parser.error("one of %s must be specified" % subparser_names) + + if args.cont and args.restart: + parser.error("Please select either --continue or --restart") + + if os.path.exists(args.state_file): + if not args.cont and not args.restart: + parser.error( + "State file exists, so you should either --continue " + "or --restart" + ) + if args.cont and not os.path.exists(args.state_file): + parser.error("Indicate --continue but the state file does not exist") + + if args.restart and os.path.exists(args.state_file): + os.remove(args.state_file) + + return args + + +def prepare_uprev( + rust_version: RustVersion, template: Optional[RustVersion] +) -> Optional[Tuple[RustVersion, str, RustVersion]]: + if template is None: + ebuild_path = find_ebuild_for_package("rust") + ebuild_name = os.path.basename(ebuild_path) + template_version = RustVersion.parse_from_ebuild(ebuild_name) + else: + ebuild_path = find_ebuild_for_rust_version(template) + template_version = template + + bootstrap_version = get_rust_bootstrap_version() + + if rust_version <= template_version: + logging.info( + "Requested version %s is not newer than the template version %s.", + rust_version, + template_version, + ) + return None + + logging.info( + "Template Rust version is %s (ebuild: %r)", + template_version, + ebuild_path, + ) + logging.info("rust-bootstrap version is %s", bootstrap_version) + + return template_version, ebuild_path, bootstrap_version + + +def copy_patches( + directory: Path, template_version: RustVersion, new_version: RustVersion +) -> None: + patch_path = directory / "files" + prefix = "%s-%s-" % (directory.name, template_version) + new_prefix = "%s-%s-" % (directory.name, new_version) + for f in os.listdir(patch_path): + if not f.startswith(prefix): + continue + logging.info("Copy patch %s to new version", f) + new_name = f.replace(str(template_version), str(new_version)) + shutil.copyfile( + os.path.join(patch_path, f), + os.path.join(patch_path, new_name), + ) + + subprocess.check_call( + ["git", "add", f"{new_prefix}*.patch"], cwd=patch_path + ) -def create_ebuild(template_ebuild: str, new_version: RustVersion) -> str: - shutil.copyfile(template_ebuild, - RUST_PATH.joinpath(f'rust-{new_version}.ebuild')) - subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], - cwd=RUST_PATH) - return os.path.join(RUST_PATH, f'rust-{new_version}.ebuild') +def create_ebuild( + template_ebuild: str, pkgatom: str, new_version: RustVersion +) -> str: + filename = f"{Path(pkgatom).name}-{new_version}.ebuild" + ebuild = EBUILD_PREFIX.joinpath(f"{pkgatom}/{filename}") + shutil.copyfile(template_ebuild, ebuild) + subprocess.check_call(["git", "add", filename], cwd=ebuild.parent) + return str(ebuild) def update_bootstrap_ebuild(new_bootstrap_version: RustVersion) -> None: - old_ebuild = find_ebuild_path(rust_bootstrap_path(), 'rust-bootstrap') - m = re.match(r'^rust-bootstrap-(\d+).(\d+).(\d+)', old_ebuild.name) - assert m, old_ebuild.name - old_version = RustVersion(m.group(1), m.group(2), m.group(3)) - new_ebuild = old_ebuild.parent.joinpath( - f'rust-bootstrap-{new_bootstrap_version}.ebuild') - old_text = old_ebuild.read_text(encoding='utf-8') - new_text, changes = re.subn(r'(RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=\([^)]*)', - f'\\1\t{old_version}\n', - old_text, - flags=re.MULTILINE) - assert changes == 1, 'Failed to update RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE' - new_ebuild.write_text(new_text, encoding='utf-8') - - -def update_ebuild(ebuild_file: str, - new_bootstrap_version: RustVersion) -> None: - contents = open(ebuild_file, encoding='utf-8').read() - contents, subs = re.subn(r'^BOOTSTRAP_VERSION=.*$', - 'BOOTSTRAP_VERSION="%s"' % - (new_bootstrap_version, ), - contents, - flags=re.MULTILINE) - if not subs: - raise RuntimeError('BOOTSTRAP_VERSION not found in rust ebuild') - open(ebuild_file, 'w', encoding='utf-8').write(contents) - logging.info('Rust ebuild file has BOOTSTRAP_VERSION updated to %s', - new_bootstrap_version) - - -def flip_mirror_in_ebuild(ebuild_file: Path, add: bool) -> None: - restrict_re = re.compile( - r'(?P<before>RESTRICT=")(?P<values>"[^"]*"|.*)(?P<after>")') - with open(ebuild_file, encoding='utf-8') as f: - contents = f.read() - m = restrict_re.search(contents) - assert m, 'failed to find RESTRICT variable in Rust ebuild' - values = m.group('values') - if add: - if 'mirror' in values: - return - values += ' mirror' - else: - if 'mirror' not in values: - return - values = values.replace(' mirror', '') - new_contents = restrict_re.sub(r'\g<before>%s\g<after>' % values, contents) - with open(ebuild_file, 'w', encoding='utf-8') as f: - f.write(new_contents) - - -def ebuild_actions(package: str, actions: List[str], - sudo: bool = False) -> None: - ebuild_path_inchroot = find_ebuild_for_package(package) - cmd = ['ebuild', ebuild_path_inchroot] + actions - if sudo: - cmd = ['sudo'] + cmd - subprocess.check_call(cmd) + old_ebuild = find_ebuild_path(rust_bootstrap_path(), "rust-bootstrap") + m = re.match(r"^rust-bootstrap-(\d+).(\d+).(\d+)", old_ebuild.name) + assert m, old_ebuild.name + old_version = RustVersion(m.group(1), m.group(2), m.group(3)) + new_ebuild = old_ebuild.parent.joinpath( + f"rust-bootstrap-{new_bootstrap_version}.ebuild" + ) + old_text = old_ebuild.read_text(encoding="utf-8") + new_text, changes = re.subn( + r"(RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=\([^)]*)", + f"\\1\t{old_version}\n", + old_text, + flags=re.MULTILINE, + ) + assert changes == 1, "Failed to update RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE" + new_ebuild.write_text(new_text, encoding="utf-8") + + +def update_bootstrap_version( + path: str, new_bootstrap_version: RustVersion +) -> None: + contents = open(path, encoding="utf-8").read() + contents, subs = re.subn( + r"^BOOTSTRAP_VERSION=.*$", + 'BOOTSTRAP_VERSION="%s"' % (new_bootstrap_version,), + contents, + flags=re.MULTILINE, + ) + if not subs: + raise RuntimeError(f"BOOTSTRAP_VERSION not found in {path}") + open(path, "w", encoding="utf-8").write(contents) + logging.info("Rust BOOTSTRAP_VERSION updated to %s", new_bootstrap_version) + + +def ebuild_actions( + package: str, actions: List[str], sudo: bool = False +) -> None: + ebuild_path_inchroot = find_ebuild_for_package(package) + cmd = ["ebuild", ebuild_path_inchroot] + actions + if sudo: + cmd = ["sudo"] + cmd + subprocess.check_call(cmd) def fetch_distfile_from_mirror(name: str) -> None: - """Gets the named file from the local mirror. - - This ensures that the file exists on the mirror, and - that we can read it. We overwrite any existing distfile - to ensure the checksums that update_manifest() records - match the file as it exists on the mirror. - - This function also attempts to verify the ACL for - the file (which is expected to have READER permission - for allUsers). We can only see the ACL if the user - gsutil runs with is the owner of the file. If not, - we get an access denied error. We also count this - as a success, because it means we were able to fetch - the file even though we don't own it. - """ - mirror_file = MIRROR_PATH + '/' + name - local_file = Path(get_distdir(), name) - cmd = [GSUTIL, 'cp', mirror_file, local_file] - logging.info('Running %r', cmd) - rc = subprocess.call(cmd) - if rc != 0: - logging.error( - """Could not fetch %s + """Gets the named file from the local mirror. + + This ensures that the file exists on the mirror, and + that we can read it. We overwrite any existing distfile + to ensure the checksums that update_manifest() records + match the file as it exists on the mirror. + + This function also attempts to verify the ACL for + the file (which is expected to have READER permission + for allUsers). We can only see the ACL if the user + gsutil runs with is the owner of the file. If not, + we get an access denied error. We also count this + as a success, because it means we were able to fetch + the file even though we don't own it. + """ + mirror_file = MIRROR_PATH + "/" + name + local_file = Path(get_distdir(), name) + cmd = [GSUTIL, "cp", mirror_file, local_file] + logging.info("Running %r", cmd) + rc = subprocess.call(cmd) + if rc != 0: + logging.error( + """Could not fetch %s If the file does not yet exist at %s please download the file, verify its integrity @@ -425,352 +446,541 @@ gpg --recv-keys 85AB96E6FA1BE5FE Once you have verify the integrity of the file, upload it to the local mirror using gsutil cp. -""", mirror_file, MIRROR_PATH, name, name) - raise Exception(f'Could not fetch {mirror_file}') - # Check that the ACL allows allUsers READER access. - # If we get an AccessDeniedAcception here, that also - # counts as a success, because we were able to fetch - # the file as a non-owner. - cmd = [GSUTIL, 'acl', 'get', mirror_file] - logging.info('Running %r', cmd) - output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT) - acl_verified = False - if 'AccessDeniedException:' in output: - acl_verified = True - else: - acl = json.loads(output) - for x in acl: - if x['entity'] == 'allUsers' and x['role'] == 'READER': +""", + mirror_file, + MIRROR_PATH, + name, + name, + ) + raise Exception(f"Could not fetch {mirror_file}") + # Check that the ACL allows allUsers READER access. + # If we get an AccessDeniedAcception here, that also + # counts as a success, because we were able to fetch + # the file as a non-owner. + cmd = [GSUTIL, "acl", "get", mirror_file] + logging.info("Running %r", cmd) + output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT) + acl_verified = False + if "AccessDeniedException:" in output: acl_verified = True - break - if not acl_verified: - logging.error('Output from acl get:\n%s', output) - raise Exception('Could not verify that allUsers has READER permission') - - -def fetch_bootstrap_distfiles(old_version: RustVersion, - new_version: RustVersion) -> None: - """Fetches rust-bootstrap distfiles from the local mirror - - Fetches the distfiles for a rust-bootstrap ebuild to ensure they - are available on the mirror and the local copies are the same as - the ones on the mirror. - """ - fetch_distfile_from_mirror(compute_rust_bootstrap_prebuilt_name(old_version)) - fetch_distfile_from_mirror(compute_rustc_src_name(new_version)) + else: + acl = json.loads(output) + for x in acl: + if x["entity"] == "allUsers" and x["role"] == "READER": + acl_verified = True + break + if not acl_verified: + logging.error("Output from acl get:\n%s", output) + raise Exception("Could not verify that allUsers has READER permission") + + +def fetch_bootstrap_distfiles( + old_version: RustVersion, new_version: RustVersion +) -> None: + """Fetches rust-bootstrap distfiles from the local mirror + + Fetches the distfiles for a rust-bootstrap ebuild to ensure they + are available on the mirror and the local copies are the same as + the ones on the mirror. + """ + fetch_distfile_from_mirror( + compute_rust_bootstrap_prebuilt_name(old_version) + ) + fetch_distfile_from_mirror(compute_rustc_src_name(new_version)) def fetch_rust_distfiles(version: RustVersion) -> None: - """Fetches rust distfiles from the local mirror + """Fetches rust distfiles from the local mirror - Fetches the distfiles for a rust ebuild to ensure they - are available on the mirror and the local copies are - the same as the ones on the mirror. - """ - fetch_distfile_from_mirror(compute_rustc_src_name(version)) + Fetches the distfiles for a rust ebuild to ensure they + are available on the mirror and the local copies are + the same as the ones on the mirror. + """ + fetch_distfile_from_mirror(compute_rustc_src_name(version)) def get_distdir() -> os.PathLike: - """Returns portage's distdir.""" - return get_command_output(['portageq', 'distdir']) + """Returns portage's distdir.""" + return get_command_output(["portageq", "distdir"]) def update_manifest(ebuild_file: os.PathLike) -> None: - """Updates the MANIFEST for the ebuild at the given path.""" - ebuild = Path(ebuild_file) - logging.info('Added "mirror" to RESTRICT to %s', ebuild.name) - flip_mirror_in_ebuild(ebuild, add=True) - ebuild_actions(ebuild.parent.name, ['manifest']) - logging.info('Removed "mirror" to RESTRICT from %s', ebuild.name) - flip_mirror_in_ebuild(ebuild, add=False) - - -def update_rust_packages(rust_version: RustVersion, add: bool) -> None: - package_file = RUST_PATH.joinpath( - '../../profiles/targets/chromeos/package.provided') - with open(package_file, encoding='utf-8') as f: - contents = f.read() - if add: - rust_packages_re = re.compile(r'dev-lang/rust-(\d+\.\d+\.\d+)') - rust_packages = rust_packages_re.findall(contents) - # Assume all the rust packages are in alphabetical order, so insert the new - # version to the place after the last rust_packages - new_str = f'dev-lang/rust-{rust_version}' - new_contents = contents.replace(rust_packages[-1], - f'{rust_packages[-1]}\n{new_str}') - logging.info('%s has been inserted into package.provided', new_str) - else: - old_str = f'dev-lang/rust-{rust_version}\n' - assert old_str in contents, f'{old_str!r} not found in package.provided' - new_contents = contents.replace(old_str, '') - logging.info('%s has been removed from package.provided', old_str) - - with open(package_file, 'w', encoding='utf-8') as f: - f.write(new_contents) - - -def update_virtual_rust(template_version: RustVersion, - new_version: RustVersion) -> None: - template_ebuild = find_ebuild_path(RUST_PATH.joinpath('../../virtual/rust'), - 'rust', template_version) - virtual_rust_dir = template_ebuild.parent - new_name = f'rust-{new_version}.ebuild' - new_ebuild = virtual_rust_dir.joinpath(new_name) - shutil.copyfile(template_ebuild, new_ebuild) - subprocess.check_call(['git', 'add', new_name], cwd=virtual_rust_dir) - - -def perform_step(state_file: pathlib.Path, - tmp_state_file: pathlib.Path, - completed_steps: Dict[str, Any], - step_name: str, - step_fn: Callable[[], T], - result_from_json: Optional[Callable[[Any], T]] = None, - result_to_json: Optional[Callable[[T], Any]] = None) -> T: - if step_name in completed_steps: - logging.info('Skipping previously completed step %s', step_name) - if result_from_json: - return result_from_json(completed_steps[step_name]) - return completed_steps[step_name] - - logging.info('Running step %s', step_name) - val = step_fn() - logging.info('Step %s complete', step_name) - if result_to_json: - completed_steps[step_name] = result_to_json(val) - else: - completed_steps[step_name] = val - - with tmp_state_file.open('w', encoding='utf-8') as f: - json.dump(completed_steps, f, indent=4) - tmp_state_file.rename(state_file) - return val + """Updates the MANIFEST for the ebuild at the given path.""" + ebuild = Path(ebuild_file) + ebuild_actions(ebuild.parent.name, ["manifest"]) + + +def update_rust_packages( + pkgatom: str, rust_version: RustVersion, add: bool +) -> None: + package_file = EBUILD_PREFIX.joinpath( + "profiles/targets/chromeos/package.provided" + ) + with open(package_file, encoding="utf-8") as f: + contents = f.read() + if add: + rust_packages_re = re.compile( + "^" + re.escape(pkgatom) + r"-\d+\.\d+\.\d+$", re.MULTILINE + ) + rust_packages = rust_packages_re.findall(contents) + # Assume all the rust packages are in alphabetical order, so insert + # the new version to the place after the last rust_packages + new_str = f"{pkgatom}-{rust_version}" + new_contents = contents.replace( + rust_packages[-1], f"{rust_packages[-1]}\n{new_str}" + ) + logging.info("%s has been inserted into package.provided", new_str) + else: + old_str = f"{pkgatom}-{rust_version}\n" + assert old_str in contents, f"{old_str!r} not found in package.provided" + new_contents = contents.replace(old_str, "") + logging.info("%s has been removed from package.provided", old_str) + + with open(package_file, "w", encoding="utf-8") as f: + f.write(new_contents) + + +def update_virtual_rust( + template_version: RustVersion, new_version: RustVersion +) -> None: + template_ebuild = find_ebuild_path( + EBUILD_PREFIX.joinpath("virtual/rust"), "rust", template_version + ) + virtual_rust_dir = template_ebuild.parent + new_name = f"rust-{new_version}.ebuild" + new_ebuild = virtual_rust_dir.joinpath(new_name) + shutil.copyfile(template_ebuild, new_ebuild) + subprocess.check_call(["git", "add", new_name], cwd=virtual_rust_dir) + + +def unmerge_package_if_installed(pkgatom: str) -> None: + """Unmerges a package if it is installed.""" + shpkg = shlex.quote(pkgatom) + subprocess.check_call( + [ + "sudo", + "bash", + "-c", + f"! emerge --pretend --quiet --unmerge {shpkg}" + f" || emerge --rage-clean {shpkg}", + ] + ) + + +def perform_step( + state_file: pathlib.Path, + tmp_state_file: pathlib.Path, + completed_steps: Dict[str, Any], + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, +) -> T: + if step_name in completed_steps: + logging.info("Skipping previously completed step %s", step_name) + if result_from_json: + return result_from_json(completed_steps[step_name]) + return completed_steps[step_name] + + logging.info("Running step %s", step_name) + val = step_fn() + logging.info("Step %s complete", step_name) + if result_to_json: + completed_steps[step_name] = result_to_json(val) + else: + completed_steps[step_name] = val + + with tmp_state_file.open("w", encoding="utf-8") as f: + json.dump(completed_steps, f, indent=4) + tmp_state_file.rename(state_file) + return val def prepare_uprev_from_json( - obj: Any) -> Optional[Tuple[RustVersion, str, RustVersion]]: - if not obj: - return None - version, ebuild_path, bootstrap_version = obj - return RustVersion(*version), ebuild_path, RustVersion(*bootstrap_version) - - -def create_rust_uprev(rust_version: RustVersion, - maybe_template_version: Optional[RustVersion], - skip_compile: bool, run_step: Callable[[], T]) -> None: - template_version, template_ebuild, old_bootstrap_version = run_step( - 'prepare uprev', - lambda: prepare_uprev(rust_version, maybe_template_version), - result_from_json=prepare_uprev_from_json, - ) - if template_ebuild is None: - return - - # The fetch steps will fail (on purpose) if the files they check for - # are not available on the mirror. To make them pass, fetch the - # required files yourself, verify their checksums, then upload them - # to the mirror. - run_step( - 'fetch bootstrap distfiles', lambda: fetch_bootstrap_distfiles( - old_bootstrap_version, template_version)) - run_step('fetch rust distfiles', lambda: fetch_rust_distfiles(rust_version)) - run_step('update bootstrap ebuild', lambda: update_bootstrap_ebuild( - template_version)) - run_step( - 'update bootstrap manifest', lambda: update_manifest(rust_bootstrap_path( - ).joinpath(f'rust-bootstrap-{template_version}.ebuild'))) - run_step('copy patches', lambda: copy_patches(RUST_PATH, template_version, - rust_version)) - ebuild_file = run_step( - 'create ebuild', lambda: create_ebuild(template_ebuild, rust_version)) - run_step( - 'update ebuild', lambda: update_ebuild(ebuild_file, template_version)) - run_step('update manifest to add new version', lambda: update_manifest( - Path(ebuild_file))) - if not skip_compile: + obj: Any, +) -> Optional[Tuple[RustVersion, str, RustVersion]]: + if not obj: + return None + version, ebuild_path, bootstrap_version = obj + return RustVersion(*version), ebuild_path, RustVersion(*bootstrap_version) + + +def create_rust_uprev( + rust_version: RustVersion, + maybe_template_version: Optional[RustVersion], + skip_compile: bool, + run_step: Callable[[], T], +) -> None: + template_version, template_ebuild, old_bootstrap_version = run_step( + "prepare uprev", + lambda: prepare_uprev(rust_version, maybe_template_version), + result_from_json=prepare_uprev_from_json, + ) + if template_ebuild is None: + return + + # The fetch steps will fail (on purpose) if the files they check for + # are not available on the mirror. To make them pass, fetch the + # required files yourself, verify their checksums, then upload them + # to the mirror. run_step( - 'emerge rust', lambda: subprocess.check_call( - ['sudo', 'emerge', 'dev-lang/rust'])) - run_step('insert version into rust packages', lambda: update_rust_packages( - rust_version, add=True)) - run_step('upgrade virtual/rust', lambda: update_virtual_rust( - template_version, rust_version)) + "fetch bootstrap distfiles", + lambda: fetch_bootstrap_distfiles( + old_bootstrap_version, template_version + ), + ) + run_step("fetch rust distfiles", lambda: fetch_rust_distfiles(rust_version)) + run_step( + "update bootstrap ebuild", + lambda: update_bootstrap_ebuild(template_version), + ) + run_step( + "update bootstrap manifest", + lambda: update_manifest( + rust_bootstrap_path().joinpath( + f"rust-bootstrap-{template_version}.ebuild" + ) + ), + ) + run_step( + "update bootstrap version", + lambda: update_bootstrap_version( + EBUILD_PREFIX.joinpath("eclass/cros-rustc.eclass"), template_version + ), + ) + run_step( + "copy patches", + lambda: copy_patches(RUST_PATH, template_version, rust_version), + ) + template_host_ebuild = EBUILD_PREFIX.joinpath( + f"dev-lang/rust-host/rust-host-{template_version}.ebuild" + ) + host_file = run_step( + "create host ebuild", + lambda: create_ebuild( + template_host_ebuild, "dev-lang/rust-host", rust_version + ), + ) + run_step( + "update host manifest to add new version", + lambda: update_manifest(Path(host_file)), + ) + target_file = run_step( + "create target ebuild", + lambda: create_ebuild(template_ebuild, "dev-lang/rust", rust_version), + ) + run_step( + "update target manifest to add new version", + lambda: update_manifest(Path(target_file)), + ) + if not skip_compile: + run_step("build packages", lambda: rebuild_packages(rust_version)) + run_step( + "insert host version into rust packages", + lambda: update_rust_packages( + "dev-lang/rust-host", rust_version, add=True + ), + ) + run_step( + "insert target version into rust packages", + lambda: update_rust_packages("dev-lang/rust", rust_version, add=True), + ) + run_step( + "upgrade virtual/rust", + lambda: update_virtual_rust(template_version, rust_version), + ) def find_rust_versions_in_chroot() -> List[Tuple[RustVersion, str]]: - return [(RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) - for x in os.listdir(RUST_PATH) if x.endswith('.ebuild')] + return [ + (RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) + for x in os.listdir(RUST_PATH) + if x.endswith(".ebuild") + ] -def find_oldest_rust_version_in_chroot() -> Tuple[RustVersion, str]: - rust_versions = find_rust_versions_in_chroot() - if len(rust_versions) <= 1: - raise RuntimeError('Expect to find more than one Rust versions') - return min(rust_versions) +def find_oldest_rust_version_in_chroot() -> RustVersion: + rust_versions = find_rust_versions_in_chroot() + if len(rust_versions) <= 1: + raise RuntimeError("Expect to find more than one Rust versions") + return min(rust_versions)[0] def find_ebuild_for_rust_version(version: RustVersion) -> str: - rust_ebuilds = [ - ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version - ] - if not rust_ebuilds: - raise ValueError(f'No Rust ebuilds found matching {version}') - if len(rust_ebuilds) > 1: - raise ValueError(f'Multiple Rust ebuilds found matching {version}: ' - f'{rust_ebuilds}') - return rust_ebuilds[0] + rust_ebuilds = [ + ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version + ] + if not rust_ebuilds: + raise ValueError(f"No Rust ebuilds found matching {version}") + if len(rust_ebuilds) > 1: + raise ValueError( + f"Multiple Rust ebuilds found matching {version}: " + f"{rust_ebuilds}" + ) + return rust_ebuilds[0] + + +def rebuild_packages(version: RustVersion): + """Rebuild packages modified by this script.""" + # Remove all packages we modify to avoid depending on preinstalled + # versions. This ensures that the packages can really be built. + packages = [ + "dev-lang/rust", + "dev-lang/rust-host", + "dev-lang/rust-bootstrap", + ] + for pkg in packages: + unmerge_package_if_installed(pkg) + # Mention only dev-lang/rust explicitly, so that others are pulled + # in as dependencies (letting us detect dependency errors). + # Packages we modify are listed in --usepkg-exclude to ensure they + # are built from source. + try: + subprocess.check_call( + [ + "sudo", + "emerge", + "--quiet-build", + "--usepkg-exclude", + " ".join(packages), + f"=dev-lang/rust-{version}", + ] + ) + except: + logging.warning( + "Failed to build dev-lang/rust or one of its dependencies." + " If necessary, you can restore rust and rust-host from" + " binary packages:\n sudo emerge --getbinpkgonly dev-lang/rust" + ) + raise + + +def remove_ebuild_version(path: os.PathLike, name: str, version: RustVersion): + """Remove the specified version of an ebuild. + + Removes {path}/{name}-{version}.ebuild and {path}/{name}-{version}-*.ebuild + using git rm. + + Args: + path: The directory in which the ebuild files are. + name: The name of the package (e.g. 'rust'). + version: The version of the ebuild to remove. + """ + path = Path(path) + pattern = f"{name}-{version}-*.ebuild" + matches = list(path.glob(pattern)) + ebuild = path / f"{name}-{version}.ebuild" + if ebuild.exists(): + matches.append(ebuild) + if not matches: + logging.warning( + "No ebuilds matching %s version %s in %r", name, version, str(path) + ) + for m in matches: + remove_files(m.name, path) def remove_files(filename: str, path: str) -> None: - subprocess.check_call(['git', 'rm', filename], cwd=path) - - -def remove_rust_bootstrap_version(version: RustVersion, - run_step: Callable[[], T]) -> None: - prefix = f'rust-bootstrap-{version}' - run_step('remove old bootstrap ebuild', lambda: remove_files( - f'{prefix}*.ebuild', rust_bootstrap_path())) - ebuild_file = find_ebuild_for_package('rust-bootstrap') - run_step('update bootstrap manifest to delete old version', lambda: - update_manifest(ebuild_file)) - - -def remove_rust_uprev(rust_version: Optional[RustVersion], - run_step: Callable[[], T]) -> None: - def find_desired_rust_version(): - if rust_version: - return rust_version, find_ebuild_for_rust_version(rust_version) - return find_oldest_rust_version_in_chroot() - - def find_desired_rust_version_from_json(obj: Any) -> Tuple[RustVersion, str]: - version, ebuild_path = obj - return RustVersion(*version), ebuild_path - - delete_version, delete_ebuild = run_step( - 'find rust version to delete', - find_desired_rust_version, - result_from_json=find_desired_rust_version_from_json, - ) - run_step( - 'remove patches', lambda: remove_files( - f'files/rust-{delete_version}-*.patch', RUST_PATH)) - run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH)) - ebuild_file = find_ebuild_for_package('rust') - run_step('update manifest to delete old version', lambda: update_manifest( - ebuild_file)) - run_step('remove version from rust packages', lambda: update_rust_packages( - delete_version, add=False)) - run_step('remove virtual/rust', lambda: remove_virtual_rust(delete_version)) + subprocess.check_call(["git", "rm", filename], cwd=path) + + +def remove_rust_bootstrap_version( + version: RustVersion, run_step: Callable[[], T] +) -> None: + run_step( + "remove old bootstrap ebuild", + lambda: remove_ebuild_version( + rust_bootstrap_path(), "rust-bootstrap", version + ), + ) + ebuild_file = find_ebuild_for_package("rust-bootstrap") + run_step( + "update bootstrap manifest to delete old version", + lambda: update_manifest(ebuild_file), + ) + + +def remove_rust_uprev( + rust_version: Optional[RustVersion], run_step: Callable[[], T] +) -> None: + def find_desired_rust_version() -> RustVersion: + if rust_version: + return rust_version + return find_oldest_rust_version_in_chroot() + + def find_desired_rust_version_from_json(obj: Any) -> RustVersion: + return RustVersion(*obj) + + delete_version = run_step( + "find rust version to delete", + find_desired_rust_version, + result_from_json=find_desired_rust_version_from_json, + ) + run_step( + "remove patches", + lambda: remove_files(f"files/rust-{delete_version}-*.patch", RUST_PATH), + ) + run_step( + "remove target ebuild", + lambda: remove_ebuild_version(RUST_PATH, "rust", delete_version), + ) + run_step( + "remove host ebuild", + lambda: remove_ebuild_version( + EBUILD_PREFIX.joinpath("dev-lang/rust-host"), + "rust-host", + delete_version, + ), + ) + target_file = find_ebuild_for_package("rust") + run_step( + "update target manifest to delete old version", + lambda: update_manifest(target_file), + ) + run_step( + "remove target version from rust packages", + lambda: update_rust_packages( + "dev-lang/rust", delete_version, add=False + ), + ) + host_file = find_ebuild_for_package("rust-host") + run_step( + "update host manifest to delete old version", + lambda: update_manifest(host_file), + ) + run_step( + "remove host version from rust packages", + lambda: update_rust_packages( + "dev-lang/rust-host", delete_version, add=False + ), + ) + run_step("remove virtual/rust", lambda: remove_virtual_rust(delete_version)) def remove_virtual_rust(delete_version: RustVersion) -> None: - ebuild = find_ebuild_path(RUST_PATH.joinpath('../../virtual/rust'), 'rust', - delete_version) - subprocess.check_call(['git', 'rm', str(ebuild.name)], cwd=ebuild.parent) + remove_ebuild_version( + EBUILD_PREFIX.joinpath("virtual/rust"), "rust", delete_version + ) def rust_bootstrap_path() -> Path: - return RUST_PATH.joinpath('../rust-bootstrap') + return EBUILD_PREFIX.joinpath("dev-lang/rust-bootstrap") def create_new_repo(rust_version: RustVersion) -> None: - output = get_command_output(['git', 'status', '--porcelain'], cwd=RUST_PATH) - if output: - raise RuntimeError( - f'{RUST_PATH} has uncommitted changes, please either discard them ' - 'or commit them.') - git.CreateBranch(RUST_PATH, f'rust-to-{rust_version}') + output = get_command_output( + ["git", "status", "--porcelain"], cwd=EBUILD_PREFIX + ) + if output: + raise RuntimeError( + f"{EBUILD_PREFIX} has uncommitted changes, please either discard " + "them or commit them." + ) + git.CreateBranch(EBUILD_PREFIX, f"rust-to-{rust_version}") def build_cross_compiler() -> None: - # Get target triples in ebuild - rust_ebuild = find_ebuild_for_package('rust') - with open(rust_ebuild, encoding='utf-8') as f: - contents = f.read() - - target_triples_re = re.compile(r'RUSTC_TARGET_TRIPLES=\(([^)]+)\)') - m = target_triples_re.search(contents) - assert m, 'RUST_TARGET_TRIPLES not found in rust ebuild' - target_triples = m.group(1).strip().split('\n') - - compiler_targets_to_install = [ - target.strip() for target in target_triples if 'cros-' in target - ] - for target in target_triples: - if 'cros-' not in target: - continue - target = target.strip() - - # We also always need arm-none-eabi, though it's not mentioned in - # RUSTC_TARGET_TRIPLES. - compiler_targets_to_install.append('arm-none-eabi') - - logging.info('Emerging cross compilers %s', compiler_targets_to_install) - subprocess.check_call( - ['sudo', 'emerge', '-j', '-G'] + - [f'cross-{target}/gcc' for target in compiler_targets_to_install]) + # Get target triples in ebuild + rust_ebuild = find_ebuild_for_package("rust") + with open(rust_ebuild, encoding="utf-8") as f: + contents = f.read() + + target_triples_re = re.compile(r"RUSTC_TARGET_TRIPLES=\(([^)]+)\)") + m = target_triples_re.search(contents) + assert m, "RUST_TARGET_TRIPLES not found in rust ebuild" + target_triples = m.group(1).strip().split("\n") + + compiler_targets_to_install = [ + target.strip() for target in target_triples if "cros-" in target + ] + for target in target_triples: + if "cros-" not in target: + continue + target = target.strip() + + # We also always need arm-none-eabi, though it's not mentioned in + # RUSTC_TARGET_TRIPLES. + compiler_targets_to_install.append("arm-none-eabi") + + logging.info("Emerging cross compilers %s", compiler_targets_to_install) + subprocess.check_call( + ["sudo", "emerge", "-j", "-G"] + + [f"cross-{target}/gcc" for target in compiler_targets_to_install] + ) def create_new_commit(rust_version: RustVersion) -> None: - subprocess.check_call(['git', 'add', '-A'], cwd=RUST_PATH) - messages = [ - f'[DO NOT SUBMIT] dev-lang/rust: upgrade to Rust {rust_version}', - '', - 'This CL is created by rust_uprev tool automatically.' - '', - 'BUG=None', - 'TEST=Use CQ to test the new Rust version', - ] - git.UploadChanges(RUST_PATH, f'rust-to-{rust_version}', messages) + subprocess.check_call(["git", "add", "-A"], cwd=EBUILD_PREFIX) + messages = [ + f"[DO NOT SUBMIT] dev-lang/rust: upgrade to Rust {rust_version}", + "", + "This CL is created by rust_uprev tool automatically." "", + "BUG=None", + "TEST=Use CQ to test the new Rust version", + ] + git.UploadChanges(EBUILD_PREFIX, f"rust-to-{rust_version}", messages) def main() -> None: - if not chroot.InChroot(): - raise RuntimeError('This script must be executed inside chroot') - - logging.basicConfig(level=logging.INFO) - - args = parse_commandline_args() - - state_file = pathlib.Path(args.state_file) - tmp_state_file = state_file.with_suffix('.tmp') - - try: - with state_file.open(encoding='utf-8') as f: - completed_steps = json.load(f) - except FileNotFoundError: - completed_steps = {} - - def run_step( - step_name: str, - step_fn: Callable[[], T], - result_from_json: Optional[Callable[[Any], T]] = None, - result_to_json: Optional[Callable[[T], Any]] = None, - ) -> T: - return perform_step(state_file, tmp_state_file, completed_steps, step_name, - step_fn, result_from_json, result_to_json) - - if args.subparser_name == 'create': - create_rust_uprev(args.rust_version, args.template, args.skip_compile, - run_step) - elif args.subparser_name == 'remove': - remove_rust_uprev(args.rust_version, run_step) - elif args.subparser_name == 'remove-bootstrap': - remove_rust_bootstrap_version(args.version, run_step) - else: - # If you have added more subparser_name, please also add the handlers above - assert args.subparser_name == 'roll' - run_step('create new repo', lambda: create_new_repo(args.uprev)) - if not args.skip_cross_compiler: - run_step('build cross compiler', build_cross_compiler) - create_rust_uprev(args.uprev, args.template, args.skip_compile, run_step) - remove_rust_uprev(args.remove, run_step) - bootstrap_version = prepare_uprev_from_json( - completed_steps['prepare uprev'])[2] - remove_rust_bootstrap_version(bootstrap_version, run_step) - if not args.no_upload: - run_step('create rust uprev CL', lambda: create_new_commit(args.uprev)) - - -if __name__ == '__main__': - sys.exit(main()) + if not chroot.InChroot(): + raise RuntimeError("This script must be executed inside chroot") + + logging.basicConfig(level=logging.INFO) + + args = parse_commandline_args() + + state_file = pathlib.Path(args.state_file) + tmp_state_file = state_file.with_suffix(".tmp") + + try: + with state_file.open(encoding="utf-8") as f: + completed_steps = json.load(f) + except FileNotFoundError: + completed_steps = {} + + def run_step( + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, + ) -> T: + return perform_step( + state_file, + tmp_state_file, + completed_steps, + step_name, + step_fn, + result_from_json, + result_to_json, + ) + + if args.subparser_name == "create": + create_rust_uprev( + args.rust_version, args.template, args.skip_compile, run_step + ) + elif args.subparser_name == "remove": + remove_rust_uprev(args.rust_version, run_step) + elif args.subparser_name == "remove-bootstrap": + remove_rust_bootstrap_version(args.version, run_step) + else: + # If you have added more subparser_name, please also add the handlers above + assert args.subparser_name == "roll" + run_step("create new repo", lambda: create_new_repo(args.uprev)) + if not args.skip_cross_compiler: + run_step("build cross compiler", build_cross_compiler) + create_rust_uprev( + args.uprev, args.template, args.skip_compile, run_step + ) + remove_rust_uprev(args.remove, run_step) + bootstrap_version = prepare_uprev_from_json( + completed_steps["prepare uprev"] + )[2] + remove_rust_bootstrap_version(bootstrap_version, run_step) + if not args.no_upload: + run_step( + "create rust uprev CL", lambda: create_new_commit(args.uprev) + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py index 00761391..0c4c91ed 100755 --- a/rust_tools/rust_uprev_test.py +++ b/rust_tools/rust_uprev_test.py @@ -1,313 +1,403 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for rust_uprev.py""" import os +from pathlib import Path import shutil import subprocess import tempfile import unittest -from pathlib import Path from unittest import mock from llvm_tools import git - import rust_uprev from rust_uprev import RustVersion def _fail_command(cmd, *_args, **_kwargs): - err = subprocess.CalledProcessError(returncode=1, cmd=cmd) - err.stderr = b'mock failure' - raise err + err = subprocess.CalledProcessError(returncode=1, cmd=cmd) + err.stderr = b"mock failure" + raise err class FetchDistfileTest(unittest.TestCase): - """Tests rust_uprev.fetch_distfile_from_mirror()""" - - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', side_effect=_fail_command) - def test_fetch_difstfile_fail(self, *_args) -> None: - with self.assertRaises(subprocess.CalledProcessError): - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object(rust_uprev, - 'get_command_output_unchecked', - return_value='AccessDeniedException: Access denied.') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_access_denied(self, *_args) -> None: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object( - rust_uprev, - 'get_command_output_unchecked', - return_value='[ { "entity": "allUsers", "role": "READER" } ]') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_ok(self, *_args) -> None: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object( - rust_uprev, - 'get_command_output_unchecked', - return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_wrong(self, *_args) -> None: - with self.assertRaisesRegex(Exception, 'allUsers.*READER'): - with self.assertLogs(level='ERROR') as log: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - self.assertIn( - '[ { "entity": "___fake@google.com", "role": "OWNER" } ]', - '\n'.join(log.output)) + """Tests rust_uprev.fetch_distfile_from_mirror()""" + + @mock.patch.object( + rust_uprev, "get_distdir", return_value="/fake/distfiles" + ) + @mock.patch.object(subprocess, "call", side_effect=_fail_command) + def test_fetch_difstfile_fail(self, *_args) -> None: + with self.assertRaises(subprocess.CalledProcessError): + rust_uprev.fetch_distfile_from_mirror("test_distfile.tar.gz") + + @mock.patch.object( + rust_uprev, + "get_command_output_unchecked", + return_value="AccessDeniedException: Access denied.", + ) + @mock.patch.object( + rust_uprev, "get_distdir", return_value="/fake/distfiles" + ) + @mock.patch.object(subprocess, "call", return_value=0) + def test_fetch_distfile_acl_access_denied(self, *_args) -> None: + rust_uprev.fetch_distfile_from_mirror("test_distfile.tar.gz") + + @mock.patch.object( + rust_uprev, + "get_command_output_unchecked", + return_value='[ { "entity": "allUsers", "role": "READER" } ]', + ) + @mock.patch.object( + rust_uprev, "get_distdir", return_value="/fake/distfiles" + ) + @mock.patch.object(subprocess, "call", return_value=0) + def test_fetch_distfile_acl_ok(self, *_args) -> None: + rust_uprev.fetch_distfile_from_mirror("test_distfile.tar.gz") + + @mock.patch.object( + rust_uprev, + "get_command_output_unchecked", + return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]', + ) + @mock.patch.object( + rust_uprev, "get_distdir", return_value="/fake/distfiles" + ) + @mock.patch.object(subprocess, "call", return_value=0) + def test_fetch_distfile_acl_wrong(self, *_args) -> None: + with self.assertRaisesRegex(Exception, "allUsers.*READER"): + with self.assertLogs(level="ERROR") as log: + rust_uprev.fetch_distfile_from_mirror("test_distfile.tar.gz") + self.assertIn( + '[ { "entity": "___fake@google.com", "role": "OWNER" } ]', + "\n".join(log.output), + ) class FindEbuildPathTest(unittest.TestCase): - """Tests for rust_uprev.find_ebuild_path()""" - - def test_exact_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.3.4.ebuild') - ebuild.touch() - Path(tmpdir, 'test-1.2.3.ebuild').touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test', - rust_uprev.RustVersion(1, 3, 4)) - self.assertEqual(result, ebuild) - - def test_no_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.2.3.ebuild') - ebuild.touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test') - self.assertEqual(result, ebuild) - - def test_patch_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.3.4-r3.ebuild') - ebuild.touch() - Path(tmpdir, 'test-1.2.3.ebuild').touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test', - rust_uprev.RustVersion(1, 3, 4)) - self.assertEqual(result, ebuild) + """Tests for rust_uprev.find_ebuild_path()""" + + def test_exact_version(self): + with tempfile.TemporaryDirectory() as t: + tmpdir = Path(t) + ebuild = tmpdir / "test-1.3.4.ebuild" + ebuild.touch() + (tmpdir / "test-1.2.3.ebuild").touch() + result = rust_uprev.find_ebuild_path( + tmpdir, "test", rust_uprev.RustVersion(1, 3, 4) + ) + self.assertEqual(result, ebuild) + + def test_no_version(self): + with tempfile.TemporaryDirectory() as t: + tmpdir = Path(t) + ebuild = tmpdir / "test-1.2.3.ebuild" + ebuild.touch() + result = rust_uprev.find_ebuild_path(tmpdir, "test") + self.assertEqual(result, ebuild) + + def test_patch_version(self): + with tempfile.TemporaryDirectory() as t: + tmpdir = Path(t) + ebuild = tmpdir / "test-1.3.4-r3.ebuild" + ebuild.touch() + (tmpdir / "test-1.2.3.ebuild").touch() + result = rust_uprev.find_ebuild_path( + tmpdir, "test", rust_uprev.RustVersion(1, 3, 4) + ) + self.assertEqual(result, ebuild) + + def test_multiple_versions(self): + with tempfile.TemporaryDirectory() as t: + tmpdir = Path(t) + (tmpdir / "test-1.3.4-r3.ebuild").touch() + (tmpdir / "test-1.3.5.ebuild").touch() + with self.assertRaises(AssertionError): + rust_uprev.find_ebuild_path(tmpdir, "test") + + def test_selected_version(self): + with tempfile.TemporaryDirectory() as t: + tmpdir = Path(t) + ebuild = tmpdir / "test-1.3.4-r3.ebuild" + ebuild.touch() + (tmpdir / "test-1.3.5.ebuild").touch() + result = rust_uprev.find_ebuild_path( + tmpdir, "test", rust_uprev.RustVersion(1, 3, 4) + ) + self.assertEqual(result, ebuild) + + def test_symlink(self): + # Symlinks to ebuilds in the same directory are allowed, and the return + # value is the regular file. + with tempfile.TemporaryDirectory() as t: + tmpdir = Path(t) + ebuild = tmpdir / "test-1.3.4.ebuild" + ebuild.touch() + (tmpdir / "test-1.3.4-r1.ebuild").symlink_to("test-1.3.4.ebuild") + result = rust_uprev.find_ebuild_path(tmpdir, "test") + self.assertEqual(result, ebuild) + + +class RemoveEbuildVersionTest(unittest.TestCase): + """Tests for rust_uprev.remove_ebuild_version()""" + + @mock.patch.object(subprocess, "check_call") + def test_single(self, check_call): + with tempfile.TemporaryDirectory() as tmpdir: + ebuild_dir = Path(tmpdir, "test-ebuilds") + ebuild_dir.mkdir() + ebuild = Path(ebuild_dir, "test-3.1.4.ebuild") + ebuild.touch() + Path(ebuild_dir, "unrelated-1.0.0.ebuild").touch() + rust_uprev.remove_ebuild_version( + ebuild_dir, "test", rust_uprev.RustVersion(3, 1, 4) + ) + check_call.assert_called_once_with( + ["git", "rm", "test-3.1.4.ebuild"], cwd=ebuild_dir + ) + + @mock.patch.object(subprocess, "check_call") + def test_symlink(self, check_call): + with tempfile.TemporaryDirectory() as tmpdir: + ebuild_dir = Path(tmpdir, "test-ebuilds") + ebuild_dir.mkdir() + ebuild = Path(ebuild_dir, "test-3.1.4.ebuild") + ebuild.touch() + symlink = Path(ebuild_dir, "test-3.1.4-r5.ebuild") + symlink.symlink_to(ebuild.name) + Path(ebuild_dir, "unrelated-1.0.0.ebuild").touch() + rust_uprev.remove_ebuild_version( + ebuild_dir, "test", rust_uprev.RustVersion(3, 1, 4) + ) + check_call.assert_has_calls( + [ + mock.call( + ["git", "rm", "test-3.1.4.ebuild"], cwd=ebuild_dir + ), + mock.call( + ["git", "rm", "test-3.1.4-r5.ebuild"], cwd=ebuild_dir + ), + ], + any_order=True, + ) class RustVersionTest(unittest.TestCase): - """Tests for RustVersion class""" + """Tests for RustVersion class""" - def test_str(self): - obj = rust_uprev.RustVersion(major=1, minor=2, patch=3) - self.assertEqual(str(obj), '1.2.3') + def test_str(self): + obj = rust_uprev.RustVersion(major=1, minor=2, patch=3) + self.assertEqual(str(obj), "1.2.3") - def test_parse_version_only(self): - expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) - actual = rust_uprev.RustVersion.parse('1.2.3') - self.assertEqual(expected, actual) + def test_parse_version_only(self): + expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) + actual = rust_uprev.RustVersion.parse("1.2.3") + self.assertEqual(expected, actual) - def test_parse_ebuild_name(self): - expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) - actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3.ebuild') - self.assertEqual(expected, actual) + def test_parse_ebuild_name(self): + expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) + actual = rust_uprev.RustVersion.parse_from_ebuild("rust-1.2.3.ebuild") + self.assertEqual(expected, actual) - actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3-r1.ebuild') - self.assertEqual(expected, actual) + actual = rust_uprev.RustVersion.parse_from_ebuild( + "rust-1.2.3-r1.ebuild" + ) + self.assertEqual(expected, actual) - def test_parse_fail(self): - with self.assertRaises(AssertionError) as context: - rust_uprev.RustVersion.parse('invalid-rust-1.2.3') - self.assertEqual("failed to parse 'invalid-rust-1.2.3'", - str(context.exception)) + def test_parse_fail(self): + with self.assertRaises(AssertionError) as context: + rust_uprev.RustVersion.parse("invalid-rust-1.2.3") + self.assertEqual( + "failed to parse 'invalid-rust-1.2.3'", str(context.exception) + ) class PrepareUprevTest(unittest.TestCase): - """Tests for prepare_uprev step in rust_uprev""" - - def setUp(self): - self.bootstrap_version = rust_uprev.RustVersion(1, 1, 0) - self.version_old = rust_uprev.RustVersion(1, 2, 3) - self.version_new = rust_uprev.RustVersion(1, 3, 5) - - @mock.patch.object(rust_uprev, - 'find_ebuild_for_rust_version', - return_value='/path/to/ebuild') - @mock.patch.object(rust_uprev, 'find_ebuild_path') - @mock.patch.object(rust_uprev, 'get_command_output') - def test_success_with_template(self, mock_command, mock_find_ebuild, - _ebuild_for_version): - bootstrap_ebuild_path = Path( - '/path/to/rust-bootstrap/', - f'rust-bootstrap-{self.bootstrap_version}.ebuild') - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, '/path/to/ebuild', self.bootstrap_version) - actual = rust_uprev.prepare_uprev(rust_version=self.version_new, - template=self.version_old) - self.assertEqual(expected, actual) - mock_command.assert_not_called() - - @mock.patch.object(rust_uprev, - 'find_ebuild_for_rust_version', - return_value='/path/to/ebuild') - @mock.patch.object(rust_uprev, - 'get_rust_bootstrap_version', - return_value=RustVersion(0, 41, 12)) - @mock.patch.object(rust_uprev, 'get_command_output') - def test_return_none_with_template_larger_than_input(self, mock_command, - *_args): - ret = rust_uprev.prepare_uprev(rust_version=self.version_old, - template=self.version_new) - self.assertIsNone(ret) - mock_command.assert_not_called() - - @mock.patch.object(rust_uprev, 'find_ebuild_path') - @mock.patch.object(os.path, 'exists') - @mock.patch.object(rust_uprev, 'get_command_output') - def test_success_without_template(self, mock_command, mock_exists, - mock_find_ebuild): - rust_ebuild_path = f'/path/to/rust/rust-{self.version_old}-r3.ebuild' - mock_command.return_value = rust_ebuild_path - bootstrap_ebuild_path = Path( - '/path/to/rust-bootstrap', - f'rust-bootstrap-{self.bootstrap_version}.ebuild') - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, rust_ebuild_path, self.bootstrap_version) - actual = rust_uprev.prepare_uprev(rust_version=self.version_new, - template=None) - self.assertEqual(expected, actual) - mock_command.assert_called_once_with(['equery', 'w', 'rust']) - mock_exists.assert_not_called() - - @mock.patch.object(rust_uprev, - 'get_rust_bootstrap_version', - return_value=RustVersion(0, 41, 12)) - @mock.patch.object(os.path, 'exists') - @mock.patch.object(rust_uprev, 'get_command_output') - def test_return_none_with_ebuild_larger_than_input(self, mock_command, - mock_exists, *_args): - mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild' - ret = rust_uprev.prepare_uprev(rust_version=self.version_old, - template=None) - self.assertIsNone(ret) - mock_exists.assert_not_called() - - def test_prepare_uprev_from_json(self): - ebuild_path = '/path/to/the/ebuild' - json_result = (list(self.version_new), ebuild_path, - list(self.bootstrap_version)) - expected = (self.version_new, ebuild_path, self.bootstrap_version) - actual = rust_uprev.prepare_uprev_from_json(json_result) - self.assertEqual(expected, actual) - - -class UpdateEbuildTest(unittest.TestCase): - """Tests for update_ebuild step in rust_uprev""" - ebuild_file_before = """ + """Tests for prepare_uprev step in rust_uprev""" + + def setUp(self): + self.bootstrap_version = rust_uprev.RustVersion(1, 1, 0) + self.version_old = rust_uprev.RustVersion(1, 2, 3) + self.version_new = rust_uprev.RustVersion(1, 3, 5) + + @mock.patch.object( + rust_uprev, + "find_ebuild_for_rust_version", + return_value="/path/to/ebuild", + ) + @mock.patch.object(rust_uprev, "find_ebuild_path") + @mock.patch.object(rust_uprev, "get_command_output") + def test_success_with_template( + self, mock_command, mock_find_ebuild, _ebuild_for_version + ): + bootstrap_ebuild_path = Path( + "/path/to/rust-bootstrap/", + f"rust-bootstrap-{self.bootstrap_version}.ebuild", + ) + mock_find_ebuild.return_value = bootstrap_ebuild_path + expected = (self.version_old, "/path/to/ebuild", self.bootstrap_version) + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=self.version_old + ) + self.assertEqual(expected, actual) + mock_command.assert_not_called() + + @mock.patch.object( + rust_uprev, + "find_ebuild_for_rust_version", + return_value="/path/to/ebuild", + ) + @mock.patch.object( + rust_uprev, + "get_rust_bootstrap_version", + return_value=RustVersion(0, 41, 12), + ) + @mock.patch.object(rust_uprev, "get_command_output") + def test_return_none_with_template_larger_than_input( + self, mock_command, *_args + ): + ret = rust_uprev.prepare_uprev( + rust_version=self.version_old, template=self.version_new + ) + self.assertIsNone(ret) + mock_command.assert_not_called() + + @mock.patch.object(rust_uprev, "find_ebuild_path") + @mock.patch.object(os.path, "exists") + @mock.patch.object(rust_uprev, "get_command_output") + def test_success_without_template( + self, mock_command, mock_exists, mock_find_ebuild + ): + rust_ebuild_path = f"/path/to/rust/rust-{self.version_old}-r3.ebuild" + mock_command.return_value = rust_ebuild_path + bootstrap_ebuild_path = Path( + "/path/to/rust-bootstrap", + f"rust-bootstrap-{self.bootstrap_version}.ebuild", + ) + mock_find_ebuild.return_value = bootstrap_ebuild_path + expected = (self.version_old, rust_ebuild_path, self.bootstrap_version) + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=None + ) + self.assertEqual(expected, actual) + mock_command.assert_called_once_with(["equery", "w", "rust"]) + mock_exists.assert_not_called() + + @mock.patch.object( + rust_uprev, + "get_rust_bootstrap_version", + return_value=RustVersion(0, 41, 12), + ) + @mock.patch.object(os.path, "exists") + @mock.patch.object(rust_uprev, "get_command_output") + def test_return_none_with_ebuild_larger_than_input( + self, mock_command, mock_exists, *_args + ): + mock_command.return_value = ( + f"/path/to/rust/rust-{self.version_new}.ebuild" + ) + ret = rust_uprev.prepare_uprev( + rust_version=self.version_old, template=None + ) + self.assertIsNone(ret) + mock_exists.assert_not_called() + + def test_prepare_uprev_from_json(self): + ebuild_path = "/path/to/the/ebuild" + json_result = ( + list(self.version_new), + ebuild_path, + list(self.bootstrap_version), + ) + expected = (self.version_new, ebuild_path, self.bootstrap_version) + actual = rust_uprev.prepare_uprev_from_json(json_result) + self.assertEqual(expected, actual) + + +class UpdateBootstrapVersionTest(unittest.TestCase): + """Tests for update_bootstrap_version step in rust_uprev""" + + ebuild_file_before = """ BOOTSTRAP_VERSION="1.2.0" """ - ebuild_file_after = """ + ebuild_file_after = """ BOOTSTRAP_VERSION="1.3.6" """ - def test_success(self): - mock_open = mock.mock_open(read_data=self.ebuild_file_before) - # ebuild_file and new bootstrap version are deliberately different - ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' - with mock.patch('builtins.open', mock_open): - rust_uprev.update_ebuild(ebuild_file, - rust_uprev.RustVersion.parse('1.3.6')) - mock_open.return_value.__enter__().write.assert_called_once_with( - self.ebuild_file_after) - - def test_fail_when_ebuild_misses_a_variable(self): - mock_open = mock.mock_open(read_data='') - ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' - with mock.patch('builtins.open', mock_open): - with self.assertRaises(RuntimeError) as context: - rust_uprev.update_ebuild(ebuild_file, - rust_uprev.RustVersion.parse('1.2.0')) - self.assertEqual('BOOTSTRAP_VERSION not found in rust ebuild', - str(context.exception)) + def test_success(self): + mock_open = mock.mock_open(read_data=self.ebuild_file_before) + # ebuild_file and new bootstrap version are deliberately different + ebuild_file = "/path/to/rust/cros-rustc.eclass" + with mock.patch("builtins.open", mock_open): + rust_uprev.update_bootstrap_version( + ebuild_file, rust_uprev.RustVersion.parse("1.3.6") + ) + mock_open.return_value.__enter__().write.assert_called_once_with( + self.ebuild_file_after + ) + + def test_fail_when_ebuild_misses_a_variable(self): + mock_open = mock.mock_open(read_data="") + ebuild_file = "/path/to/rust/rust-1.3.5.ebuild" + with mock.patch("builtins.open", mock_open): + with self.assertRaises(RuntimeError) as context: + rust_uprev.update_bootstrap_version( + ebuild_file, rust_uprev.RustVersion.parse("1.2.0") + ) + self.assertEqual( + "BOOTSTRAP_VERSION not found in /path/to/rust/rust-1.3.5.ebuild", + str(context.exception), + ) class UpdateManifestTest(unittest.TestCase): - """Tests for update_manifest step in rust_uprev""" - - # pylint: disable=protected-access - def _run_test_flip_mirror(self, before, after, add, expect_write): - mock_open = mock.mock_open(read_data=f'RESTRICT="{before}"') - with mock.patch('builtins.open', mock_open): - rust_uprev.flip_mirror_in_ebuild('', add=add) - if expect_write: - mock_open.return_value.__enter__().write.assert_called_once_with( - f'RESTRICT="{after}"') - - def test_add_mirror_in_ebuild(self): - self._run_test_flip_mirror(before='variable1 variable2', - after='variable1 variable2 mirror', - add=True, - expect_write=True) - - def test_remove_mirror_in_ebuild(self): - self._run_test_flip_mirror(before='variable1 variable2 mirror', - after='variable1 variable2', - add=False, - expect_write=True) - - def test_add_mirror_when_exists(self): - self._run_test_flip_mirror(before='variable1 variable2 mirror', - after='variable1 variable2 mirror', - add=True, - expect_write=False) - - def test_remove_mirror_when_not_exists(self): - self._run_test_flip_mirror(before='variable1 variable2', - after='variable1 variable2', - add=False, - expect_write=False) - - @mock.patch.object(rust_uprev, 'flip_mirror_in_ebuild') - @mock.patch.object(rust_uprev, 'ebuild_actions') - def test_update_manifest(self, mock_run, mock_flip): - ebuild_file = Path('/path/to/rust/rust-1.1.1.ebuild') - rust_uprev.update_manifest(ebuild_file) - mock_run.assert_called_once_with('rust', ['manifest']) - mock_flip.assert_has_calls( - [mock.call(ebuild_file, add=True), - mock.call(ebuild_file, add=False)]) + """Tests for update_manifest step in rust_uprev""" + + @mock.patch.object(rust_uprev, "ebuild_actions") + def test_update_manifest(self, mock_run): + ebuild_file = Path("/path/to/rust/rust-1.1.1.ebuild") + rust_uprev.update_manifest(ebuild_file) + mock_run.assert_called_once_with("rust", ["manifest"]) class UpdateBootstrapEbuildTest(unittest.TestCase): - """Tests for rust_uprev.update_bootstrap_ebuild()""" - - def test_update_bootstrap_ebuild(self): - # The update should do two things: - # 1. Create a copy of rust-bootstrap's ebuild with the new version number. - # 2. Add the old PV to RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE. - with tempfile.TemporaryDirectory() as tmpdir_str, \ - mock.patch.object(rust_uprev, 'find_ebuild_path') as mock_find_ebuild: - tmpdir = Path(tmpdir_str) - bootstrapdir = Path.joinpath(tmpdir, 'rust-bootstrap') - bootstrapdir.mkdir() - old_ebuild = bootstrapdir.joinpath('rust-bootstrap-1.45.2.ebuild') - old_ebuild.write_text(encoding='utf-8', - data=""" + """Tests for rust_uprev.update_bootstrap_ebuild()""" + + def test_update_bootstrap_ebuild(self): + # The update should do two things: + # 1. Create a copy of rust-bootstrap's ebuild with the new version number. + # 2. Add the old PV to RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE. + with tempfile.TemporaryDirectory() as tmpdir_str, mock.patch.object( + rust_uprev, "find_ebuild_path" + ) as mock_find_ebuild: + tmpdir = Path(tmpdir_str) + bootstrapdir = Path.joinpath(tmpdir, "rust-bootstrap") + bootstrapdir.mkdir() + old_ebuild = bootstrapdir.joinpath("rust-bootstrap-1.45.2.ebuild") + old_ebuild.write_text( + encoding="utf-8", + data=""" some text RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( \t1.43.1 \t1.44.1 ) some more text -""") - mock_find_ebuild.return_value = old_ebuild - rust_uprev.update_bootstrap_ebuild(rust_uprev.RustVersion(1, 46, 0)) - new_ebuild = bootstrapdir.joinpath('rust-bootstrap-1.46.0.ebuild') - self.assertTrue(new_ebuild.exists()) - text = new_ebuild.read_text() - self.assertEqual( - text, """ +""", + ) + mock_find_ebuild.return_value = old_ebuild + rust_uprev.update_bootstrap_ebuild(rust_uprev.RustVersion(1, 46, 0)) + new_ebuild = bootstrapdir.joinpath("rust-bootstrap-1.46.0.ebuild") + self.assertTrue(new_ebuild.exists()) + text = new_ebuild.read_text() + self.assertEqual( + text, + """ some text RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( \t1.43.1 @@ -315,186 +405,294 @@ RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( \t1.45.2 ) some more text -""") +""", + ) class UpdateRustPackagesTests(unittest.TestCase): - """Tests for update_rust_packages step.""" - - def setUp(self): - self.old_version = rust_uprev.RustVersion(1, 1, 0) - self.current_version = rust_uprev.RustVersion(1, 2, 3) - self.new_version = rust_uprev.RustVersion(1, 3, 5) - self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, - 'rust-{self.new_version}.ebuild') - - def test_add_new_rust_packages(self): - package_before = (f'dev-lang/rust-{self.old_version}\n' - f'dev-lang/rust-{self.current_version}') - package_after = (f'dev-lang/rust-{self.old_version}\n' - f'dev-lang/rust-{self.current_version}\n' - f'dev-lang/rust-{self.new_version}') - mock_open = mock.mock_open(read_data=package_before) - with mock.patch('builtins.open', mock_open): - rust_uprev.update_rust_packages(self.new_version, add=True) - mock_open.return_value.__enter__().write.assert_called_once_with( - package_after) - - def test_remove_old_rust_packages(self): - package_before = (f'dev-lang/rust-{self.old_version}\n' - f'dev-lang/rust-{self.current_version}\n' - f'dev-lang/rust-{self.new_version}') - package_after = (f'dev-lang/rust-{self.current_version}\n' - f'dev-lang/rust-{self.new_version}') - mock_open = mock.mock_open(read_data=package_before) - with mock.patch('builtins.open', mock_open): - rust_uprev.update_rust_packages(self.old_version, add=False) - mock_open.return_value.__enter__().write.assert_called_once_with( - package_after) + """Tests for update_rust_packages step.""" + + def setUp(self): + self.old_version = rust_uprev.RustVersion(1, 1, 0) + self.current_version = rust_uprev.RustVersion(1, 2, 3) + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.ebuild_file = os.path.join( + rust_uprev.RUST_PATH, "rust-{self.new_version}.ebuild" + ) + + def test_add_new_rust_packages(self): + package_before = ( + f"dev-lang/rust-{self.old_version}\n" + f"dev-lang/rust-{self.current_version}" + ) + package_after = ( + f"dev-lang/rust-{self.old_version}\n" + f"dev-lang/rust-{self.current_version}\n" + f"dev-lang/rust-{self.new_version}" + ) + mock_open = mock.mock_open(read_data=package_before) + with mock.patch("builtins.open", mock_open): + rust_uprev.update_rust_packages( + "dev-lang/rust", self.new_version, add=True + ) + mock_open.return_value.__enter__().write.assert_called_once_with( + package_after + ) + + def test_remove_old_rust_packages(self): + package_before = ( + f"dev-lang/rust-{self.old_version}\n" + f"dev-lang/rust-{self.current_version}\n" + f"dev-lang/rust-{self.new_version}" + ) + package_after = ( + f"dev-lang/rust-{self.current_version}\n" + f"dev-lang/rust-{self.new_version}" + ) + mock_open = mock.mock_open(read_data=package_before) + with mock.patch("builtins.open", mock_open): + rust_uprev.update_rust_packages( + "dev-lang/rust", self.old_version, add=False + ) + mock_open.return_value.__enter__().write.assert_called_once_with( + package_after + ) class RustUprevOtherStagesTests(unittest.TestCase): - """Tests for other steps in rust_uprev""" - - def setUp(self): - self.old_version = rust_uprev.RustVersion(1, 1, 0) - self.current_version = rust_uprev.RustVersion(1, 2, 3) - self.new_version = rust_uprev.RustVersion(1, 3, 5) - self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, - 'rust-{self.new_version}.ebuild') - - @mock.patch.object(shutil, 'copyfile') - @mock.patch.object(os, 'listdir') - @mock.patch.object(subprocess, 'check_call') - def test_copy_patches(self, mock_call, mock_ls, mock_copy): - mock_ls.return_value = [ - f'rust-{self.old_version}-patch-1.patch', - f'rust-{self.old_version}-patch-2-old.patch', - f'rust-{self.current_version}-patch-1.patch', - f'rust-{self.current_version}-patch-2-new.patch' - ] - rust_uprev.copy_patches(rust_uprev.RUST_PATH, self.current_version, - self.new_version) - mock_copy.assert_has_calls([ - mock.call( - os.path.join(rust_uprev.RUST_PATH, 'files', - f'rust-{self.current_version}-patch-1.patch'), - os.path.join(rust_uprev.RUST_PATH, 'files', - f'rust-{self.new_version}-patch-1.patch'), - ), - mock.call( - os.path.join(rust_uprev.RUST_PATH, 'files', - f'rust-{self.current_version}-patch-2-new.patch'), - os.path.join(rust_uprev.RUST_PATH, 'files', - f'rust-{self.new_version}-patch-2-new.patch')) - ]) - mock_call.assert_called_once_with( - ['git', 'add', f'rust-{self.new_version}-*.patch'], - cwd=rust_uprev.RUST_PATH.joinpath('files')) - - @mock.patch.object(shutil, 'copyfile') - @mock.patch.object(subprocess, 'check_call') - def test_create_ebuild(self, mock_call, mock_copy): - template_ebuild = f'/path/to/rust-{self.current_version}-r2.ebuild' - rust_uprev.create_ebuild(template_ebuild, self.new_version) - mock_copy.assert_called_once_with( - template_ebuild, - rust_uprev.RUST_PATH.joinpath(f'rust-{self.new_version}.ebuild')) - mock_call.assert_called_once_with( - ['git', 'add', f'rust-{self.new_version}.ebuild'], - cwd=rust_uprev.RUST_PATH) - - @mock.patch.object(rust_uprev, 'find_ebuild_for_package') - @mock.patch.object(subprocess, 'check_call') - def test_remove_rust_bootstrap_version(self, mock_call, *_args): - bootstrap_path = os.path.join(rust_uprev.RUST_PATH, '..', 'rust-bootstrap') - rust_uprev.remove_rust_bootstrap_version(self.old_version, lambda *x: ()) - mock_call.has_calls([ - [ - 'git', 'rm', - os.path.join(bootstrap_path, 'files', - f'rust-bootstrap-{self.old_version}-*.patch') - ], - [ - 'git', 'rm', - os.path.join(bootstrap_path, - f'rust-bootstrap-{self.old_version}.ebuild') - ], - ]) - - @mock.patch.object(rust_uprev, 'find_ebuild_path') - @mock.patch.object(subprocess, 'check_call') - def test_remove_virtual_rust(self, mock_call, mock_find_ebuild): - ebuild_path = Path( - f'/some/dir/virtual/rust/rust-{self.old_version}.ebuild') - mock_find_ebuild.return_value = Path(ebuild_path) - rust_uprev.remove_virtual_rust(self.old_version) - mock_call.assert_called_once_with( - ['git', 'rm', str(ebuild_path.name)], cwd=ebuild_path.parent) - - @mock.patch.object(rust_uprev, 'find_ebuild_path') - @mock.patch.object(shutil, 'copyfile') - @mock.patch.object(subprocess, 'check_call') - def test_update_virtual_rust(self, mock_call, mock_copy, mock_find_ebuild): - ebuild_path = Path( - f'/some/dir/virtual/rust/rust-{self.current_version}.ebuild') - mock_find_ebuild.return_value = Path(ebuild_path) - rust_uprev.update_virtual_rust(self.current_version, self.new_version) - mock_call.assert_called_once_with( - ['git', 'add', f'rust-{self.new_version}.ebuild'], - cwd=ebuild_path.parent) - mock_copy.assert_called_once_with( - ebuild_path.parent.joinpath(f'rust-{self.current_version}.ebuild'), - ebuild_path.parent.joinpath(f'rust-{self.new_version}.ebuild')) - - @mock.patch.object(os, 'listdir') - def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls): - oldest_version_name = f'rust-{self.old_version}.ebuild' - mock_ls.return_value = [ - oldest_version_name, f'rust-{self.current_version}.ebuild', - f'rust-{self.new_version}.ebuild' - ] - actual = rust_uprev.find_oldest_rust_version_in_chroot() - expected = (self.old_version, - os.path.join(rust_uprev.RUST_PATH, oldest_version_name)) - self.assertEqual(expected, actual) - - @mock.patch.object(os, 'listdir') - def test_find_oldest_rust_version_in_chroot_fail_with_only_one_ebuild( - self, mock_ls): - mock_ls.return_value = [f'rust-{self.new_version}.ebuild'] - with self.assertRaises(RuntimeError) as context: - rust_uprev.find_oldest_rust_version_in_chroot() - self.assertEqual('Expect to find more than one Rust versions', - str(context.exception)) - - @mock.patch.object(rust_uprev, 'get_command_output') - @mock.patch.object(git, 'CreateBranch') - def test_create_new_repo(self, mock_branch, mock_output): - mock_output.return_value = '' - rust_uprev.create_new_repo(self.new_version) - mock_branch.assert_called_once_with(rust_uprev.RUST_PATH, - f'rust-to-{self.new_version}') - - @mock.patch.object(rust_uprev, 'get_command_output') - @mock.patch.object(subprocess, 'check_call') - def test_build_cross_compiler(self, mock_call, mock_output): - mock_output.return_value = f'rust-{self.new_version}.ebuild' - cros_targets = [ - 'x86_64-cros-linux-gnu', - 'armv7a-cros-linux-gnueabihf', - 'aarch64-cros-linux-gnu', - ] - all_triples = ['x86_64-pc-linux-gnu'] + cros_targets - rust_ebuild = 'RUSTC_TARGET_TRIPLES=(' + '\n\t'.join(all_triples) + ')' - mock_open = mock.mock_open(read_data=rust_ebuild) - with mock.patch('builtins.open', mock_open): - rust_uprev.build_cross_compiler() - - mock_call.assert_called_once_with( - ['sudo', 'emerge', '-j', '-G'] + - [f'cross-{x}/gcc' for x in cros_targets + ['arm-none-eabi']]) - - -if __name__ == '__main__': - unittest.main() + """Tests for other steps in rust_uprev""" + + def setUp(self): + self.old_version = rust_uprev.RustVersion(1, 1, 0) + self.current_version = rust_uprev.RustVersion(1, 2, 3) + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.ebuild_file = os.path.join( + rust_uprev.RUST_PATH, "rust-{self.new_version}.ebuild" + ) + + @mock.patch.object(shutil, "copyfile") + @mock.patch.object(os, "listdir") + @mock.patch.object(subprocess, "check_call") + def test_copy_patches(self, mock_call, mock_ls, mock_copy): + mock_ls.return_value = [ + f"rust-{self.old_version}-patch-1.patch", + f"rust-{self.old_version}-patch-2-old.patch", + f"rust-{self.current_version}-patch-1.patch", + f"rust-{self.current_version}-patch-2-new.patch", + ] + rust_uprev.copy_patches( + rust_uprev.RUST_PATH, self.current_version, self.new_version + ) + mock_copy.assert_has_calls( + [ + mock.call( + os.path.join( + rust_uprev.RUST_PATH, + "files", + f"rust-{self.current_version}-patch-1.patch", + ), + os.path.join( + rust_uprev.RUST_PATH, + "files", + f"rust-{self.new_version}-patch-1.patch", + ), + ), + mock.call( + os.path.join( + rust_uprev.RUST_PATH, + "files", + f"rust-{self.current_version}-patch-2-new.patch", + ), + os.path.join( + rust_uprev.RUST_PATH, + "files", + f"rust-{self.new_version}-patch-2-new.patch", + ), + ), + ] + ) + mock_call.assert_called_once_with( + ["git", "add", f"rust-{self.new_version}-*.patch"], + cwd=rust_uprev.RUST_PATH.joinpath("files"), + ) + + @mock.patch.object(shutil, "copyfile") + @mock.patch.object(subprocess, "check_call") + def test_create_rust_ebuild(self, mock_call, mock_copy): + template_ebuild = f"/path/to/rust-{self.current_version}-r2.ebuild" + rust_uprev.create_ebuild( + template_ebuild, "dev-lang/rust", self.new_version + ) + mock_copy.assert_called_once_with( + template_ebuild, + rust_uprev.RUST_PATH.joinpath(f"rust-{self.new_version}.ebuild"), + ) + mock_call.assert_called_once_with( + ["git", "add", f"rust-{self.new_version}.ebuild"], + cwd=rust_uprev.RUST_PATH, + ) + + @mock.patch.object(shutil, "copyfile") + @mock.patch.object(subprocess, "check_call") + def test_create_rust_host_ebuild(self, mock_call, mock_copy): + template_ebuild = f"/path/to/rust-host-{self.current_version}-r2.ebuild" + rust_uprev.create_ebuild( + template_ebuild, "dev-lang/rust-host", self.new_version + ) + mock_copy.assert_called_once_with( + template_ebuild, + rust_uprev.EBUILD_PREFIX.joinpath( + f"dev-lang/rust-host/rust-host-{self.new_version}.ebuild" + ), + ) + mock_call.assert_called_once_with( + ["git", "add", f"rust-host-{self.new_version}.ebuild"], + cwd=rust_uprev.EBUILD_PREFIX.joinpath("dev-lang/rust-host"), + ) + + @mock.patch.object(rust_uprev, "find_ebuild_for_package") + @mock.patch.object(subprocess, "check_call") + def test_remove_rust_bootstrap_version(self, mock_call, *_args): + bootstrap_path = os.path.join( + rust_uprev.RUST_PATH, "..", "rust-bootstrap" + ) + rust_uprev.remove_rust_bootstrap_version( + self.old_version, lambda *x: () + ) + mock_call.has_calls( + [ + [ + "git", + "rm", + os.path.join( + bootstrap_path, + "files", + f"rust-bootstrap-{self.old_version}-*.patch", + ), + ], + [ + "git", + "rm", + os.path.join( + bootstrap_path, + f"rust-bootstrap-{self.old_version}.ebuild", + ), + ], + ] + ) + + @mock.patch.object(subprocess, "check_call") + def test_remove_virtual_rust(self, mock_call): + with tempfile.TemporaryDirectory() as tmpdir: + ebuild_path = Path( + tmpdir, f"virtual/rust/rust-{self.old_version}.ebuild" + ) + os.makedirs(ebuild_path.parent) + ebuild_path.touch() + with mock.patch("rust_uprev.EBUILD_PREFIX", Path(tmpdir)): + rust_uprev.remove_virtual_rust(self.old_version) + mock_call.assert_called_once_with( + ["git", "rm", str(ebuild_path.name)], cwd=ebuild_path.parent + ) + + @mock.patch.object(subprocess, "check_call") + def test_remove_virtual_rust_with_symlink(self, mock_call): + with tempfile.TemporaryDirectory() as tmpdir: + ebuild_path = Path( + tmpdir, f"virtual/rust/rust-{self.old_version}.ebuild" + ) + symlink_path = Path( + tmpdir, f"virtual/rust/rust-{self.old_version}-r14.ebuild" + ) + os.makedirs(ebuild_path.parent) + ebuild_path.touch() + symlink_path.symlink_to(ebuild_path.name) + with mock.patch("rust_uprev.EBUILD_PREFIX", Path(tmpdir)): + rust_uprev.remove_virtual_rust(self.old_version) + mock_call.assert_has_calls( + [ + mock.call( + ["git", "rm", ebuild_path.name], + cwd=ebuild_path.parent, + ), + mock.call( + ["git", "rm", symlink_path.name], + cwd=ebuild_path.parent, + ), + ], + any_order=True, + ) + + @mock.patch.object(rust_uprev, "find_ebuild_path") + @mock.patch.object(shutil, "copyfile") + @mock.patch.object(subprocess, "check_call") + def test_update_virtual_rust(self, mock_call, mock_copy, mock_find_ebuild): + ebuild_path = Path( + f"/some/dir/virtual/rust/rust-{self.current_version}.ebuild" + ) + mock_find_ebuild.return_value = Path(ebuild_path) + rust_uprev.update_virtual_rust(self.current_version, self.new_version) + mock_call.assert_called_once_with( + ["git", "add", f"rust-{self.new_version}.ebuild"], + cwd=ebuild_path.parent, + ) + mock_copy.assert_called_once_with( + ebuild_path.parent.joinpath(f"rust-{self.current_version}.ebuild"), + ebuild_path.parent.joinpath(f"rust-{self.new_version}.ebuild"), + ) + + @mock.patch.object(os, "listdir") + def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls): + oldest_version_name = f"rust-{self.old_version}.ebuild" + mock_ls.return_value = [ + oldest_version_name, + f"rust-{self.current_version}.ebuild", + f"rust-{self.new_version}.ebuild", + ] + actual = rust_uprev.find_oldest_rust_version_in_chroot() + expected = self.old_version + self.assertEqual(expected, actual) + + @mock.patch.object(os, "listdir") + def test_find_oldest_rust_version_in_chroot_fail_with_only_one_ebuild( + self, mock_ls + ): + mock_ls.return_value = [f"rust-{self.new_version}.ebuild"] + with self.assertRaises(RuntimeError) as context: + rust_uprev.find_oldest_rust_version_in_chroot() + self.assertEqual( + "Expect to find more than one Rust versions", str(context.exception) + ) + + @mock.patch.object(rust_uprev, "get_command_output") + @mock.patch.object(git, "CreateBranch") + def test_create_new_repo(self, mock_branch, mock_output): + mock_output.return_value = "" + rust_uprev.create_new_repo(self.new_version) + mock_branch.assert_called_once_with( + rust_uprev.EBUILD_PREFIX, f"rust-to-{self.new_version}" + ) + + @mock.patch.object(rust_uprev, "get_command_output") + @mock.patch.object(subprocess, "check_call") + def test_build_cross_compiler(self, mock_call, mock_output): + mock_output.return_value = f"rust-{self.new_version}.ebuild" + cros_targets = [ + "x86_64-cros-linux-gnu", + "armv7a-cros-linux-gnueabihf", + "aarch64-cros-linux-gnu", + ] + all_triples = ["x86_64-pc-linux-gnu"] + cros_targets + rust_ebuild = "RUSTC_TARGET_TRIPLES=(" + "\n\t".join(all_triples) + ")" + mock_open = mock.mock_open(read_data=rust_ebuild) + with mock.patch("builtins.open", mock_open): + rust_uprev.build_cross_compiler() + + mock_call.assert_called_once_with( + ["sudo", "emerge", "-j", "-G"] + + [f"cross-{x}/gcc" for x in cros_targets + ["arm-none-eabi"]] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index c347d2c6..dff239f3 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -21,362 +21,391 @@ import sys import time from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple -from cros_utils import bugs, email_sender, tiny_render +from cros_utils import bugs +from cros_utils import email_sender +from cros_utils import tiny_render def gentoo_sha_to_link(sha: str) -> str: - """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" - return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}' + """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" + return f"https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}" def send_email(subject: str, body: List[tiny_render.Piece]) -> None: - """Sends an email with the given title and body to... whoever cares.""" - email_sender.EmailSender().SendX20Email( - subject=subject, - identifier='rust-watch', - well_known_recipients=['cros-team'], - text_body=tiny_render.render_text_pieces(body), - html_body=tiny_render.render_html_pieces(body), - ) + """Sends an email with the given title and body to... whoever cares.""" + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier="rust-watch", + well_known_recipients=["cros-team"], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) class RustReleaseVersion(NamedTuple): - """Represents a version of Rust's stable compiler.""" - major: int - minor: int - patch: int + """Represents a version of Rust's stable compiler.""" + + major: int + minor: int + patch: int - @staticmethod - def from_string(version_string: str) -> 'RustReleaseVersion': - m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string) - if not m: - raise ValueError(f"{version_string!r} isn't a valid version string") - return RustReleaseVersion(*[int(x) for x in m.groups()]) + @staticmethod + def from_string(version_string: str) -> "RustReleaseVersion": + m = re.match(r"(\d+)\.(\d+)\.(\d+)", version_string) + if not m: + raise ValueError(f"{version_string!r} isn't a valid version string") + return RustReleaseVersion(*[int(x) for x in m.groups()]) - def __str__(self) -> str: - return f'{self.major}.{self.minor}.{self.patch}' + def __str__(self) -> str: + return f"{self.major}.{self.minor}.{self.patch}" - def to_json(self) -> str: - return str(self) + def to_json(self) -> str: + return str(self) - @staticmethod - def from_json(s: str) -> 'RustReleaseVersion': - return RustReleaseVersion.from_string(s) + @staticmethod + def from_json(s: str) -> "RustReleaseVersion": + return RustReleaseVersion.from_string(s) class State(NamedTuple): - """State that we keep around from run to run.""" - # The last Rust release tag that we've seen. - last_seen_release: RustReleaseVersion - - # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen - # that updates it. - last_gentoo_sha: str - - def to_json(self) -> Dict[str, Any]: - return { - 'last_seen_release': self.last_seen_release.to_json(), - 'last_gentoo_sha': self.last_gentoo_sha, - } - - @staticmethod - def from_json(s: Dict[str, Any]) -> 'State': - return State( - last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']), - last_gentoo_sha=s['last_gentoo_sha'], - ) + """State that we keep around from run to run.""" + + # The last Rust release tag that we've seen. + last_seen_release: RustReleaseVersion + + # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen + # that updates it. + last_gentoo_sha: str + + def to_json(self) -> Dict[str, Any]: + return { + "last_seen_release": self.last_seen_release.to_json(), + "last_gentoo_sha": self.last_gentoo_sha, + } + + @staticmethod + def from_json(s: Dict[str, Any]) -> "State": + return State( + last_seen_release=RustReleaseVersion.from_json( + s["last_seen_release"] + ), + last_gentoo_sha=s["last_gentoo_sha"], + ) def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: - """Parses `git ls-remote --tags` output into Rust stable release versions.""" - refs_tags = 'refs/tags/' - for line in lines: - _sha, tag = line.split(None, 1) - tag = tag.strip() - # Each tag has an associated 'refs/tags/name^{}', which is the actual - # object that the tag points to. That's irrelevant to us. - if tag.endswith('^{}'): - continue - - if not tag.startswith(refs_tags): - continue - - short_tag = tag[len(refs_tags):] - # There are a few old versioning schemes. Ignore them. - if short_tag.startswith('0.') or short_tag.startswith('release-'): - continue - yield RustReleaseVersion.from_string(short_tag) + """Parses `git ls-remote --tags` output into Rust stable release versions.""" + refs_tags = "refs/tags/" + for line in lines: + _sha, tag = line.split(None, 1) + tag = tag.strip() + # Each tag has an associated 'refs/tags/name^{}', which is the actual + # object that the tag points to. That's irrelevant to us. + if tag.endswith("^{}"): + continue + + if not tag.startswith(refs_tags): + continue + + short_tag = tag[len(refs_tags) :] + # There are a few old versioning schemes. Ignore them. + if short_tag.startswith("0.") or short_tag.startswith("release-"): + continue + yield RustReleaseVersion.from_string(short_tag) def fetch_most_recent_release() -> RustReleaseVersion: - """Fetches the most recent stable `rustc` version.""" - result = subprocess.run( - ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'], - check=True, - stdin=None, - capture_output=True, - encoding='utf-8', - ) - tag_lines = result.stdout.strip().splitlines() - return max(parse_release_tags(tag_lines)) + """Fetches the most recent stable `rustc` version.""" + result = subprocess.run( + ["git", "ls-remote", "--tags", "https://github.com/rust-lang/rust"], + check=True, + stdin=None, + capture_output=True, + encoding="utf-8", + ) + tag_lines = result.stdout.strip().splitlines() + return max(parse_release_tags(tag_lines)) class GitCommit(NamedTuple): - """Represents a single git commit.""" - sha: str - subject: str + """Represents a single git commit.""" + + sha: str + subject: str def update_git_repo(git_dir: pathlib.Path) -> None: - """Updates the repo at `git_dir`, retrying a few times on failure.""" - for i in itertools.count(start=1): - result = subprocess.run( - ['git', 'fetch', 'origin'], + """Updates the repo at `git_dir`, retrying a few times on failure.""" + for i in itertools.count(start=1): + result = subprocess.run( + ["git", "fetch", "origin"], + check=False, + cwd=str(git_dir), + stdin=None, + ) + + if not result.returncode: + break + + if i == 5: + # 5 attempts is too many. Something else may be wrong. + result.check_returncode() + + sleep_time = 60 * i + logging.error( + "Failed updating gentoo's repo; will try again in %ds...", + sleep_time, + ) + time.sleep(sleep_time) + + +def get_new_gentoo_commits( + git_dir: pathlib.Path, most_recent_sha: str +) -> List[GitCommit]: + """Gets commits to dev-lang/rust since `most_recent_sha`. + + Older commits come earlier in the returned list. + """ + commits = subprocess.run( + [ + "git", + "log", + "--format=%H %s", + f"{most_recent_sha}..origin/master", # nocheck + "--", + "dev-lang/rust", + ], + capture_output=True, check=False, cwd=str(git_dir), - stdin=None, + encoding="utf-8", ) - if not result.returncode: - break - - if i == 5: - # 5 attempts is too many. Something else may be wrong. - result.check_returncode() - - sleep_time = 60 * i - logging.error("Failed updating gentoo's repo; will try again in %ds...", - sleep_time) - time.sleep(sleep_time) - - -def get_new_gentoo_commits(git_dir: pathlib.Path, - most_recent_sha: str) -> List[GitCommit]: - """Gets commits to dev-lang/rust since `most_recent_sha`. - - Older commits come earlier in the returned list. - """ - commits = subprocess.run( - [ - 'git', - 'log', - '--format=%H %s', - f'{most_recent_sha}..origin/master', # nocheck - '--', - 'dev-lang/rust', - ], - capture_output=True, - check=False, - cwd=str(git_dir), - encoding='utf-8', - ) - - if commits.returncode: - logging.error('Error getting new gentoo commits; stderr:\n%s', - commits.stderr) - commits.check_returncode() - - results = [] - for line in commits.stdout.strip().splitlines(): - sha, subject = line.strip().split(None, 1) - results.append(GitCommit(sha=sha, subject=subject)) - - # `git log` outputs things in newest -> oldest order. - results.reverse() - return results + if commits.returncode: + logging.error( + "Error getting new gentoo commits; stderr:\n%s", commits.stderr + ) + commits.check_returncode() + + results = [] + for line in commits.stdout.strip().splitlines(): + sha, subject = line.strip().split(None, 1) + results.append(GitCommit(sha=sha, subject=subject)) + + # `git log` outputs things in newest -> oldest order. + results.reverse() + return results def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: - """Sets up a gentoo git repo at the given directory. Returns HEAD.""" - subprocess.run( - [ - 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git', - str(git_dir) - ], - stdin=None, - check=True, - ) - - head_rev = subprocess.run( - ['git', 'rev-parse', 'HEAD'], - cwd=str(git_dir), - check=True, - stdin=None, - capture_output=True, - encoding='utf-8', - ) - return head_rev.stdout.strip() + """Sets up a gentoo git repo at the given directory. Returns HEAD.""" + subprocess.run( + [ + "git", + "clone", + "https://anongit.gentoo.org/git/repo/gentoo.git", + str(git_dir), + ], + stdin=None, + check=True, + ) + + head_rev = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=str(git_dir), + check=True, + stdin=None, + capture_output=True, + encoding="utf-8", + ) + return head_rev.stdout.strip() def read_state(state_file: pathlib.Path) -> State: - """Reads state from the given file.""" - with state_file.open(encoding='utf-8') as f: - return State.from_json(json.load(f)) + """Reads state from the given file.""" + with state_file.open(encoding="utf-8") as f: + return State.from_json(json.load(f)) def atomically_write_state(state_file: pathlib.Path, state: State) -> None: - """Writes state to the given file.""" - temp_file = pathlib.Path(str(state_file) + '.new') - with temp_file.open('w', encoding='utf-8') as f: - json.dump(state.to_json(), f) - temp_file.rename(state_file) + """Writes state to the given file.""" + temp_file = pathlib.Path(str(state_file) + ".new") + with temp_file.open("w", encoding="utf-8") as f: + json.dump(state.to_json(), f) + temp_file.rename(state_file) def file_bug(title: str, body: str) -> None: - """Files a bug against gbiv@ with the given title/body.""" - bugs.CreateNewBug( - bugs.WellKnownComponents.CrOSToolchainPublic, - title, - body, - # To either take or reassign depending on the rotation. - assignee='gbiv@google.com', - ) + """Files a bug against gbiv@ with the given title/body.""" + bugs.CreateNewBug( + bugs.WellKnownComponents.CrOSToolchainPublic, + title, + body, + # To either take or reassign depending on the rotation. + assignee="gbiv@google.com", + ) def maybe_compose_bug( old_state: State, newest_release: RustReleaseVersion, ) -> Optional[Tuple[str, str]]: - """Creates a bug to file about the new release, if doing is desired.""" - if newest_release == old_state.last_seen_release: - return None - - title = f'[Rust] Update to {newest_release}' - body = ('A new release has been detected; we should probably roll to it. ' - "Please see go/crostc-rust-rotation for who's turn it is.") - return title, body + """Creates a bug to file about the new release, if doing is desired.""" + if newest_release == old_state.last_seen_release: + return None + + title = f"[Rust] Update to {newest_release}" + body = ( + "A new release has been detected; we should probably roll to it. " + "Please see go/crostc-rust-rotation for who's turn it is." + ) + return title, body def maybe_compose_email( - new_gentoo_commits: List[GitCommit] + new_gentoo_commits: List[GitCommit], ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: - """Creates an email given our new state, if doing so is appropriate.""" - if not new_gentoo_commits: - return None - - subject_pieces = [] - body_pieces = [] - - # Separate the sections a bit for prettier output. - if body_pieces: - body_pieces += [tiny_render.line_break, tiny_render.line_break] - - if len(new_gentoo_commits) == 1: - subject_pieces.append('new rust ebuild commit detected') - body_pieces.append('commit:') - else: - subject_pieces.append('new rust ebuild commits detected') - body_pieces.append('commits (newest first):') - - commit_lines = [] - for commit in new_gentoo_commits: - commit_lines.append([ - tiny_render.Link( - gentoo_sha_to_link(commit.sha), - commit.sha[:12], - ), - f': {commit.subject}', - ]) + """Creates an email given our new state, if doing so is appropriate.""" + if not new_gentoo_commits: + return None + + subject_pieces = [] + body_pieces = [] + + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] + + if len(new_gentoo_commits) == 1: + subject_pieces.append("new rust ebuild commit detected") + body_pieces.append("commit:") + else: + subject_pieces.append("new rust ebuild commits detected") + body_pieces.append("commits (newest first):") + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append( + [ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f": {commit.subject}", + ] + ) - body_pieces.append(tiny_render.UnorderedList(commit_lines)) + body_pieces.append(tiny_render.UnorderedList(commit_lines)) - subject = '[rust-watch] ' + '; '.join(subject_pieces) - return subject, body_pieces + subject = "[rust-watch] " + "; ".join(subject_pieces) + return subject, body_pieces def main(argv: List[str]) -> None: - logging.basicConfig(level=logging.INFO) - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--state_dir', - required=True, - help='Directory to store state in.') - parser.add_argument('--skip_side_effects', - action='store_true', - help="Don't send an email or file a bug.") - parser.add_argument( - '--skip_state_update', - action='store_true', - help="Don't update the state file. Doesn't apply to initial setup.") - opts = parser.parse_args(argv) - - state_dir = pathlib.Path(opts.state_dir) - state_file = state_dir / 'state.json' - gentoo_subdir = state_dir / 'upstream-gentoo' - if not state_file.exists(): - logging.info("state_dir isn't fully set up; doing that now.") - - # Could be in a partially set-up state. - if state_dir.exists(): - logging.info('incomplete state_dir detected; removing.') - shutil.rmtree(str(state_dir)) - - state_dir.mkdir(parents=True) + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--state_dir", required=True, help="Directory to store state in." + ) + parser.add_argument( + "--skip_side_effects", + action="store_true", + help="Don't send an email or file a bug.", + ) + parser.add_argument( + "--skip_state_update", + action="store_true", + help="Don't update the state file. Doesn't apply to initial setup.", + ) + opts = parser.parse_args(argv) + + state_dir = pathlib.Path(opts.state_dir) + state_file = state_dir / "state.json" + gentoo_subdir = state_dir / "upstream-gentoo" + if not state_file.exists(): + logging.info("state_dir isn't fully set up; doing that now.") + + # Could be in a partially set-up state. + if state_dir.exists(): + logging.info("incomplete state_dir detected; removing.") + shutil.rmtree(str(state_dir)) + + state_dir.mkdir(parents=True) + most_recent_release = fetch_most_recent_release() + most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=most_recent_gentoo_commit, + ), + ) + # Running through this _should_ be a nop, but do it anyway. Should make any + # bugs more obvious on the first run of the script. + + prior_state = read_state(state_file) + logging.info("Last state was %r", prior_state) + most_recent_release = fetch_most_recent_release() - most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + logging.info("Most recent Rust release is %s", most_recent_release) + + logging.info("Fetching new commits from Gentoo") + update_git_repo(gentoo_subdir) + new_commits = get_new_gentoo_commits( + gentoo_subdir, prior_state.last_gentoo_sha + ) + logging.info("New commits: %r", new_commits) + + maybe_bug = maybe_compose_bug(prior_state, most_recent_release) + maybe_email = maybe_compose_email(new_commits) + + if maybe_bug is None: + logging.info("No bug to file") + else: + title, body = maybe_bug + if opts.skip_side_effects: + logging.info( + "Skipping sending bug with title %r and contents\n%s", + title, + body, + ) + else: + logging.info("Writing new bug") + file_bug(title, body) + + if maybe_email is None: + logging.info("No email to send") + else: + title, body = maybe_email + if opts.skip_side_effects: + logging.info( + "Skipping sending email with title %r and contents\n%s", + title, + tiny_render.render_html_pieces(body), + ) + else: + logging.info("Sending email") + send_email(title, body) + + if opts.skip_state_update: + logging.info("Skipping state update, as requested") + return + + newest_sha = ( + new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha + ) atomically_write_state( state_file, State( last_seen_release=most_recent_release, - last_gentoo_sha=most_recent_gentoo_commit, + last_gentoo_sha=newest_sha, ), ) - # Running through this _should_ be a nop, but do it anyway. Should make any - # bugs more obvious on the first run of the script. - - prior_state = read_state(state_file) - logging.info('Last state was %r', prior_state) - - most_recent_release = fetch_most_recent_release() - logging.info('Most recent Rust release is %s', most_recent_release) - - logging.info('Fetching new commits from Gentoo') - update_git_repo(gentoo_subdir) - new_commits = get_new_gentoo_commits(gentoo_subdir, - prior_state.last_gentoo_sha) - logging.info('New commits: %r', new_commits) - - maybe_bug = maybe_compose_bug(prior_state, most_recent_release) - maybe_email = maybe_compose_email(new_commits) - - if maybe_bug is None: - logging.info('No bug to file') - else: - title, body = maybe_bug - if opts.skip_side_effects: - logging.info('Skipping sending bug with title %r and contents\n%s', - title, body) - else: - logging.info('Writing new bug') - file_bug(title, body) - - if maybe_email is None: - logging.info('No email to send') - else: - title, body = maybe_email - if opts.skip_side_effects: - logging.info('Skipping sending email with title %r and contents\n%s', - title, tiny_render.render_html_pieces(body)) - else: - logging.info('Sending email') - send_email(title, body) - - if opts.skip_state_update: - logging.info('Skipping state update, as requested') - return - - newest_sha = (new_commits[-1].sha - if new_commits else prior_state.last_gentoo_sha) - atomically_write_state( - state_file, - State( - last_seen_release=most_recent_release, - last_gentoo_sha=newest_sha, - ), - ) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py index 583a9125..1e6aec51 100755 --- a/rust_tools/rust_watch_test.py +++ b/rust_tools/rust_watch_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -14,154 +14,172 @@ import unittest import unittest.mock from cros_utils import tiny_render - import rust_watch class Test(unittest.TestCase): - """Tests.""" - def _silence_logs(self): - """Silences all log output until the end of the current test.""" - def should_log(_record): - return 0 - - logger = logging.root - logger.addFilter(should_log) - self.addCleanup(logger.removeFilter, should_log) - - def test_release_version_parsing(self): - self.assertEqual( - rust_watch.RustReleaseVersion.from_string('1.2.3'), - rust_watch.RustReleaseVersion(1, 2, 3), - ) - - def test_release_version_json_round_trips(self): - ver = rust_watch.RustReleaseVersion(1, 2, 3) - self.assertEqual(rust_watch.RustReleaseVersion.from_json(ver.to_json()), - ver) - - def test_state_json_round_trips(self): - state = rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 2, 3), - last_gentoo_sha='abc123', - ) - - self.assertEqual(rust_watch.State.from_json(state.to_json()), state) - - @unittest.mock.patch.object(subprocess, 'run') - @unittest.mock.patch.object(time, 'sleep') - def test_update_git_repo_tries_again_on_failure(self, sleep_mock, run_mock): - self._silence_logs() - - oh_no_error = ValueError('oh no') - - def check_returncode(): - raise oh_no_error - - run_call_count = 0 - - def run_sideeffect(*_args, **_kwargs): - nonlocal run_call_count - run_call_count += 1 - result = unittest.mock.Mock() - result.returncode = 1 - result.check_returncode = check_returncode - return result - - run_mock.side_effect = run_sideeffect - - with self.assertRaises(ValueError) as raised: - rust_watch.update_git_repo(pathlib.Path('/does/not/exist/at/all')) - - self.assertIs(raised.exception, oh_no_error) - self.assertEqual(run_call_count, 5) - - sleep_timings = [unittest.mock.call(60 * i) for i in range(1, 5)] - self.assertEqual(sleep_mock.mock_calls, sleep_timings) - - @unittest.mock.patch.object(subprocess, 'run') - def test_get_new_gentoo_commits_functions(self, run_mock): - returned = unittest.mock.Mock() - returned.returncode = 0 - returned.stdout = '\n'.join(( - 'abc123 newer commit', - 'abcdef and an older commit', - )) - run_mock.return_value = returned - results = rust_watch.get_new_gentoo_commits( - pathlib.Path('/does/not/exist/at/all'), 'defabc') - self.assertEqual(results, [ - rust_watch.GitCommit('abcdef', 'and an older commit'), - rust_watch.GitCommit('abc123', 'newer commit'), - ]) - - def test_compose_email_on_a_new_gentoo_commit(self): - sha_a = 'a' * 40 - new_commit = rust_watch.maybe_compose_email(new_gentoo_commits=[ - rust_watch.GitCommit( - sha=sha_a, - subject='summary_a', - ), - ], ) - - self.assertEqual(new_commit, - ('[rust-watch] new rust ebuild commit detected', [ - 'commit:', - tiny_render.UnorderedList([ - [ - tiny_render.Link( - rust_watch.gentoo_sha_to_link(sha_a), - sha_a[:12], - ), - ': summary_a', - ], - ]) - ])) - - def test_compose_email_composes_nothing_when_no_new_updates_exist(self): - self.assertIsNone(rust_watch.maybe_compose_email(new_gentoo_commits=())) - - def test_compose_bug_creates_bugs_on_new_versions(self): - title, body = rust_watch.maybe_compose_bug( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(1, 0, 1), - ) - self.assertEqual(title, '[Rust] Update to 1.0.1') - self.assertTrue(body.startswith('A new release has been detected;')) - - title, body = rust_watch.maybe_compose_bug( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(1, 1, 0), - ) - self.assertEqual(title, '[Rust] Update to 1.1.0') - self.assertTrue(body.startswith('A new release has been detected;')) - - title, body = rust_watch.maybe_compose_bug( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(2, 0, 0), - ) - self.assertEqual(title, '[Rust] Update to 2.0.0') - self.assertTrue(body.startswith('A new release has been detected;')) - - def test_compose_bug_does_nothing_when_no_new_updates_exist(self): - self.assertIsNone( - rust_watch.maybe_compose_bug( + """Tests.""" + + def _silence_logs(self): + """Silences all log output until the end of the current test.""" + + def should_log(_record): + return 0 + + logger = logging.root + logger.addFilter(should_log) + self.addCleanup(logger.removeFilter, should_log) + + def test_release_version_parsing(self): + self.assertEqual( + rust_watch.RustReleaseVersion.from_string("1.2.3"), + rust_watch.RustReleaseVersion(1, 2, 3), + ) + + def test_release_version_json_round_trips(self): + ver = rust_watch.RustReleaseVersion(1, 2, 3) + self.assertEqual( + rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver + ) + + def test_state_json_round_trips(self): + state = rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 2, 3), + last_gentoo_sha="abc123", + ) + + self.assertEqual(rust_watch.State.from_json(state.to_json()), state) + + @unittest.mock.patch.object(subprocess, "run") + @unittest.mock.patch.object(time, "sleep") + def test_update_git_repo_tries_again_on_failure(self, sleep_mock, run_mock): + self._silence_logs() + + oh_no_error = ValueError("oh no") + + def check_returncode(): + raise oh_no_error + + run_call_count = 0 + + def run_sideeffect(*_args, **_kwargs): + nonlocal run_call_count + run_call_count += 1 + result = unittest.mock.Mock() + result.returncode = 1 + result.check_returncode = check_returncode + return result + + run_mock.side_effect = run_sideeffect + + with self.assertRaises(ValueError) as raised: + rust_watch.update_git_repo(pathlib.Path("/does/not/exist/at/all")) + + self.assertIs(raised.exception, oh_no_error) + self.assertEqual(run_call_count, 5) + + sleep_timings = [unittest.mock.call(60 * i) for i in range(1, 5)] + self.assertEqual(sleep_mock.mock_calls, sleep_timings) + + @unittest.mock.patch.object(subprocess, "run") + def test_get_new_gentoo_commits_functions(self, run_mock): + returned = unittest.mock.Mock() + returned.returncode = 0 + returned.stdout = "\n".join( + ( + "abc123 newer commit", + "abcdef and an older commit", + ) + ) + run_mock.return_value = returned + results = rust_watch.get_new_gentoo_commits( + pathlib.Path("/does/not/exist/at/all"), "defabc" + ) + self.assertEqual( + results, + [ + rust_watch.GitCommit("abcdef", "and an older commit"), + rust_watch.GitCommit("abc123", "newer commit"), + ], + ) + + def test_compose_email_on_a_new_gentoo_commit(self): + sha_a = "a" * 40 + new_commit = rust_watch.maybe_compose_email( + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject="summary_a", + ), + ], + ) + + self.assertEqual( + new_commit, + ( + "[rust-watch] new rust ebuild commit detected", + [ + "commit:", + tiny_render.UnorderedList( + [ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ": summary_a", + ], + ] + ), + ], + ), + ) + + def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + self.assertIsNone(rust_watch.maybe_compose_email(new_gentoo_commits=())) + + def test_compose_bug_creates_bugs_on_new_versions(self): + title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', + last_gentoo_sha="", ), - newest_release=rust_watch.RustReleaseVersion(1, 0, 0), - )) + newest_release=rust_watch.RustReleaseVersion(1, 0, 1), + ) + self.assertEqual(title, "[Rust] Update to 1.0.1") + self.assertTrue(body.startswith("A new release has been detected;")) + title, body = rust_watch.maybe_compose_bug( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha="", + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + ) + self.assertEqual(title, "[Rust] Update to 1.1.0") + self.assertTrue(body.startswith("A new release has been detected;")) -if __name__ == '__main__': - unittest.main() + title, body = rust_watch.maybe_compose_bug( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha="", + ), + newest_release=rust_watch.RustReleaseVersion(2, 0, 0), + ) + self.assertEqual(title, "[Rust] Update to 2.0.0") + self.assertTrue(body.startswith("A new release has been detected;")) + + def test_compose_bug_does_nothing_when_no_new_updates_exist(self): + self.assertIsNone( + rust_watch.maybe_compose_bug( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha="", + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + ) + ) + + +if __name__ == "__main__": + unittest.main() |