diff options
Diffstat (limited to 'rust_tools')
| -rwxr-xr-x | rust_tools/rust_uprev.py | 673 | ||||
| -rwxr-xr-x | rust_tools/rust_uprev_test.py | 455 | ||||
| -rwxr-xr-x | rust_tools/rust_watch.py | 351 | ||||
| -rwxr-xr-x | rust_tools/rust_watch_test.py | 202 |
4 files changed, 1681 insertions, 0 deletions
diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py new file mode 100755 index 00000000..3c0ad012 --- /dev/null +++ b/rust_tools/rust_uprev.py @@ -0,0 +1,673 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tool to automatically generate a new Rust uprev CL. + +This tool is intended to automatically generate a CL to uprev Rust to a +newer version in Chrome OS, including creating a new Rust version or +removing an old version. It's based on +src/third_party/chromiumos-overlay/dev-lang/rust/UPGRADE.md. When using +the tool, the progress can be saved to a JSON file, so the user can resume +the process after a failing step is fixed. Example usage to create a new +version: + +1. (inside chroot) $ ./rust_tools/rust_uprev.py + --state_file /tmp/state-file.json + create --rust_version 1.45.0 +2. Step "compile rust" failed due to the patches can't apply to new version +3. Manually fix the patches +4. Execute the command in step 1 again. +5. Iterate 1-4 for each failed step until the tool passes. + +Replace `create --rust_version 1.45.0` with `remove --rust_version 1.43.0` +if you want to remove all 1.43.0 related stuff in the same CL. Remember to +use a different state file if you choose to run different subcommands. + +If you want a hammer that can do everything for you, use the subcommand +`roll`. It can create a Rust uprev CL with `create` and `remove` and upload +the CL to chromium code review. + +See `--help` for all available options. +""" + +# pylint: disable=cros-logging-import + +import argparse +import pathlib +import json +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple + +from llvm_tools import chroot, git +RUST_PATH = '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust' + + +def get_command_output(command: List[str], *args, **kwargs) -> str: + return subprocess.check_output( + command, encoding='utf-8', *args, **kwargs).strip() + + +class RustVersion(NamedTuple): + """NamedTuple represents a Rust version""" + major: int + minor: int + patch: int + + def __str__(self): + return f'{self.major}.{self.minor}.{self.patch}' + + @staticmethod + def parse_from_ebuild(ebuild_name: str) -> 'RustVersion': + input_re = re.compile(r'^rust-' + r'(?P<major>\d+)\.' + r'(?P<minor>\d+)\.' + r'(?P<patch>\d+)' + r'(:?-r\d+)?' + r'\.ebuild$') + m = input_re.match(ebuild_name) + assert m, f'failed to parse {ebuild_name!r}' + return RustVersion( + int(m.group('major')), int(m.group('minor')), int(m.group('patch'))) + + @staticmethod + def parse(x: str) -> 'RustVersion': + input_re = re.compile(r'^(?:rust-)?' + r'(?P<major>\d+)\.' + r'(?P<minor>\d+)\.' + r'(?P<patch>\d+)' + r'(?:.ebuild)?$') + m = input_re.match(x) + assert m, f'failed to parse {x!r}' + return RustVersion( + int(m.group('major')), int(m.group('minor')), int(m.group('patch'))) + + +def parse_commandline_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--state_file', + required=True, + help='A state file to hold previous completed steps. If the file ' + 'exists, it needs to be used together with --continue or --restart. ' + 'If not exist (do not use --continue in this case), we will create a ' + 'file for you.', + ) + parser.add_argument( + '--restart', + action='store_true', + help='Restart from the first step. Ignore the completed steps in ' + 'the state file', + ) + parser.add_argument( + '--continue', + dest='cont', + action='store_true', + help='Continue the steps from the state file', + ) + + create_parser_template = argparse.ArgumentParser(add_help=False) + create_parser_template.add_argument( + '--template', + type=RustVersion.parse, + default=None, + help='A template to use for creating a Rust uprev from, in the form ' + 'a.b.c The ebuild has to exist in the chroot. If not specified, the ' + 'tool will use the current Rust version in the chroot as template.', + ) + create_parser_template.add_argument( + '--skip_compile', + action='store_true', + help='Skip compiling rust to test the tool. Only for testing', + ) + + subparsers = parser.add_subparsers(dest='subparser_name') + subparser_names = [] + subparser_names.append('create') + create_parser = subparsers.add_parser( + 'create', + parents=[create_parser_template], + help='Create changes uprevs Rust to a new version', + ) + create_parser.add_argument( + '--rust_version', + type=RustVersion.parse, + required=True, + help='Rust version to uprev to, in the form a.b.c', + ) + + subparser_names.append('remove') + remove_parser = subparsers.add_parser( + 'remove', + help='Clean up old Rust version from chroot', + ) + remove_parser.add_argument( + '--rust_version', + type=RustVersion.parse, + default=None, + help='Rust version to remove, in the form a.b.c If not ' + 'specified, the tool will remove the oldest version in the chroot', + ) + + subparser_names.append('roll') + roll_parser = subparsers.add_parser( + 'roll', + parents=[create_parser_template], + help='A command can create and upload a Rust uprev CL, including ' + 'preparing the repo, creating new Rust uprev, deleting old uprev, ' + 'and upload a CL to crrev.', + ) + roll_parser.add_argument( + '--uprev', + type=RustVersion.parse, + required=True, + help='Rust version to uprev to, in the form a.b.c', + ) + roll_parser.add_argument( + '--remove', + type=RustVersion.parse, + default=None, + help='Rust version to remove, in the form a.b.c If not ' + 'specified, the tool will remove the oldest version in the chroot', + ) + roll_parser.add_argument( + '--skip_cross_compiler', + action='store_true', + help='Skip updating cross-compiler in the chroot', + ) + roll_parser.add_argument( + '--no_upload', + action='store_true', + help='If specified, the tool will not upload the CL for review', + ) + + args = parser.parse_args() + if args.subparser_name not in subparser_names: + parser.error('one of %s must be specified' % subparser_names) + + if args.cont and args.restart: + parser.error('Please select either --continue or --restart') + + if os.path.exists(args.state_file): + if not args.cont and not args.restart: + parser.error('State file exists, so you should either --continue ' + 'or --restart') + if args.cont and not os.path.exists(args.state_file): + parser.error('Indicate --continue but the state file does not exist') + + if args.restart and os.path.exists(args.state_file): + os.remove(args.state_file) + + return args + + +def parse_stage0_file(new_version: RustVersion) -> Tuple[str, str, str]: + # Find stage0 date, rustc and cargo + stage0_file = get_command_output([ + 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/' + f'{new_version}/src/stage0.txt' + ]) + regexp = re.compile(r'date:\s*(?P<date>\d+-\d+-\d+)\s+' + r'rustc:\s*(?P<rustc>\d+\.\d+\.\d+)\s+' + r'cargo:\s*(?P<cargo>\d+\.\d+\.\d+)') + m = regexp.search(stage0_file) + assert m, 'failed to parse stage0.txt file' + stage0_date, stage0_rustc, stage0_cargo = m.groups() + logging.info('Found stage0 file has date: %s, rustc: %s, cargo: %s', + stage0_date, stage0_rustc, stage0_cargo) + return stage0_date, stage0_rustc, stage0_cargo + + +def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion] + ) -> Optional[Tuple[RustVersion, str]]: + if template is None: + ebuild_path = get_command_output(['equery', 'w', 'rust']) + ebuild_name = os.path.basename(ebuild_path) + template_version = RustVersion.parse_from_ebuild(ebuild_name) + else: + ebuild_path = find_ebuild_for_rust_version(template) + template_version = template + + if rust_version <= template_version: + logging.info( + 'Requested version %s is not newer than the template version %s.', + rust_version, template_version) + return None + + logging.info('Template Rust version is %s (ebuild: %r)', template_version, + ebuild_path) + return template_version, ebuild_path + + +def copy_patches(template_version: RustVersion, + new_version: RustVersion) -> None: + patch_path = os.path.join(RUST_PATH, 'files') + for f in os.listdir(patch_path): + if f'rust-{template_version}' not in f: + continue + logging.info('Rename patch %s to new version', f) + new_name = f.replace(str(template_version), str(new_version)) + shutil.copyfile( + os.path.join(patch_path, f), + os.path.join(patch_path, new_name), + ) + + subprocess.check_call(['git', 'add', f'files/rust-{new_version}-*.patch'], + cwd=RUST_PATH) + + +def create_ebuild(template_ebuild: str, new_version: RustVersion) -> str: + shutil.copyfile(template_ebuild, + os.path.join(RUST_PATH, f'rust-{new_version}.ebuild')) + subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], + cwd=RUST_PATH) + return os.path.join(RUST_PATH, f'rust-{new_version}.ebuild') + + +def update_ebuild(ebuild_file: str, stage0_info: Tuple[str, str, str]) -> None: + stage0_date, stage0_rustc, stage0_cargo = stage0_info + with open(ebuild_file, encoding='utf-8') as f: + contents = f.read() + # Update STAGE0_DATE in the ebuild + stage0_date_re = re.compile(r'STAGE0_DATE="(\d+-\d+-\d+)"') + if not stage0_date_re.search(contents): + raise RuntimeError('STAGE0_DATE not found in rust ebuild') + new_contents = stage0_date_re.sub(f'STAGE0_DATE="{stage0_date}"', contents) + + # Update STAGE0_VERSION in the ebuild + stage0_rustc_re = re.compile(r'STAGE0_VERSION="[^"]*"') + if not stage0_rustc_re.search(new_contents): + raise RuntimeError('STAGE0_VERSION not found in rust ebuild') + new_contents = stage0_rustc_re.sub(f'STAGE0_VERSION="{stage0_rustc}"', + new_contents) + + # Update STAGE0_VERSION_CARGO in the ebuild + stage0_cargo_re = re.compile(r'STAGE0_VERSION_CARGO="[^"]*"') + if not stage0_cargo_re.search(new_contents): + raise RuntimeError('STAGE0_VERSION_CARGO not found in rust ebuild') + new_contents = stage0_cargo_re.sub(f'STAGE0_VERSION_CARGO="{stage0_cargo}"', + new_contents) + with open(ebuild_file, 'w', encoding='utf-8') as f: + f.write(new_contents) + logging.info( + 'Rust ebuild file has STAGE0_DATE, STAGE0_VERSION, STAGE0_VERSION_CARGO ' + 'updated to %s, %s, %s respectively', stage0_date, stage0_rustc, + stage0_cargo) + + +def flip_mirror_in_ebuild(ebuild_file: str, add: bool) -> None: + restrict_re = re.compile( + r'(?P<before>RESTRICT=")(?P<values>"[^"]*"|.*)(?P<after>")') + with open(ebuild_file, encoding='utf-8') as f: + contents = f.read() + m = restrict_re.search(contents) + assert m, 'failed to find RESTRICT variable in Rust ebuild' + values = m.group('values') + if add: + if 'mirror' in values: + return + values += ' mirror' + else: + if 'mirror' not in values: + return + values = values.replace(' mirror', '') + new_contents = restrict_re.sub(r'\g<before>%s\g<after>' % values, contents) + with open(ebuild_file, 'w', encoding='utf-8') as f: + f.write(new_contents) + + +def rust_ebuild_actions(actions: List[str], sudo: bool = False) -> None: + ebuild_path_inchroot = get_command_output(['equery', 'w', 'rust']) + cmd = ['ebuild', ebuild_path_inchroot] + actions + if sudo: + cmd = ['sudo'] + cmd + subprocess.check_call(cmd) + + +def update_manifest(ebuild_file: str) -> None: + logging.info('Added "mirror" to RESTRICT to Rust ebuild') + flip_mirror_in_ebuild(ebuild_file, add=True) + rust_ebuild_actions(['manifest']) + logging.info('Removed "mirror" to RESTRICT from Rust ebuild') + flip_mirror_in_ebuild(ebuild_file, add=False) + + +def update_rust_packages(rust_version: RustVersion, add: bool) -> None: + package_file = os.path.join( + RUST_PATH, '../../profiles/targets/chromeos/package.provided') + with open(package_file, encoding='utf-8') as f: + contents = f.read() + if add: + rust_packages_re = re.compile(r'dev-lang/rust-(\d+\.\d+\.\d+)') + rust_packages = rust_packages_re.findall(contents) + # Assume all the rust packages are in alphabetical order, so insert the new + # version to the place after the last rust_packages + new_str = f'dev-lang/rust-{rust_version}' + new_contents = contents.replace(rust_packages[-1], + f'{rust_packages[-1]}\n{new_str}') + logging.info('%s has been inserted into package.provided', new_str) + else: + old_str = f'dev-lang/rust-{rust_version}\n' + assert old_str in contents, f'{old_str!r} not found in package.provided' + new_contents = contents.replace(old_str, '') + logging.info('%s has been removed from package.provided', old_str) + + with open(package_file, 'w', encoding='utf-8') as f: + f.write(new_contents) + + +def update_virtual_rust(template_version: RustVersion, + new_version: RustVersion) -> None: + virtual_rust_dir = os.path.join(RUST_PATH, '../../virtual/rust') + assert os.path.exists(virtual_rust_dir) + shutil.copyfile( + os.path.join(virtual_rust_dir, f'rust-{template_version}.ebuild'), + os.path.join(virtual_rust_dir, f'rust-{new_version}.ebuild')) + subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], + cwd=virtual_rust_dir) + + +def upload_single_tarball(rust_url: str, tarfile_name: str, + tempdir: str) -> None: + rust_src = f'{rust_url}/{tarfile_name}' + gsutil_location = f'gs://chromeos-localmirror/distfiles/{tarfile_name}' + + missing_file = subprocess.call( + ['gsutil', 'ls', gsutil_location], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if not missing_file: + logging.info('Rust artifact at %s already exists; skipping download', + gsutil_location) + return + + logging.info('Downloading Rust artifact from %s', rust_src) + + # Download Rust's source + rust_file = os.path.join(tempdir, tarfile_name) + subprocess.check_call(['curl', '-f', '-o', rust_file, rust_src]) + + # Verify the signature of the source + sig_file = os.path.join(tempdir, 'rustc_sig.asc') + subprocess.check_call(['curl', '-f', '-o', sig_file, f'{rust_src}.asc']) + try: + subprocess.check_output(['gpg', '--verify', sig_file, rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + if "gpg: Can't check signature" not in e.output: + raise RuntimeError(f'Failed to execute `gpg --verify`, {e.output}') + + # If it fails to verify the signature, try import rustc key, and retry. + keys = get_command_output( + ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc']) + subprocess.run(['gpg', '--import'], + input=keys, + encoding='utf-8', + check=True) + subprocess.check_call(['gpg', '--verify', sig_file, rust_file]) + + # Since we are using `-n` to skip an item if it already exists, there's no + # need to check if the file exists on GS bucket or not. + subprocess.check_call( + ['gsutil', 'cp', '-n', '-a', 'public-read', rust_file, gsutil_location]) + + +def upload_to_localmirror(tempdir: str, rust_version: RustVersion, + stage0_info: Tuple[str, str, str]) -> None: + stage0_date, stage0_rustc, stage0_cargo = stage0_info + rust_url = 'https://static.rust-lang.org/dist' + # Upload rustc source + upload_single_tarball( + rust_url, + f'rustc-{rust_version}-src.tar.gz', + tempdir, + ) + # Upload stage0 toolchain + upload_single_tarball( + f'{rust_url}/{stage0_date}', + f'rust-std-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + # Upload stage0 source + upload_single_tarball( + rust_url, + f'rustc-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + # Upload stage0 cargo + upload_single_tarball( + rust_url, + f'cargo-{stage0_cargo}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + + +def perform_step(state_file: pathlib.Path, + tmp_state_file: pathlib.Path, + completed_steps: Dict[str, Any], + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None) -> T: + if step_name in completed_steps: + logging.info('Skipping previously completed step %s', step_name) + if result_from_json: + return result_from_json(completed_steps[step_name]) + return completed_steps[step_name] + + logging.info('Running step %s', step_name) + val = step_fn() + logging.info('Step %s complete', step_name) + if result_to_json: + completed_steps[step_name] = result_to_json(val) + else: + completed_steps[step_name] = val + + with tmp_state_file.open('w', encoding='utf-8') as f: + json.dump(completed_steps, f, indent=4) + tmp_state_file.rename(state_file) + return val + + +def prepare_uprev_from_json(obj: Any) -> Optional[Tuple[RustVersion, str]]: + if not obj: + return None + version, ebuild_path = obj + return RustVersion(*version), ebuild_path + + +def create_rust_uprev(rust_version: RustVersion, + maybe_template_version: Optional[RustVersion], + skip_compile: bool, run_step: Callable[[], T]) -> None: + stage0_info = run_step( + 'parse stage0 file', lambda: parse_stage0_file(rust_version)) + template_version, template_ebuild = run_step( + 'prepare uprev', + lambda: prepare_uprev(rust_version, maybe_template_version), + result_from_json=prepare_uprev_from_json, + ) + if template_ebuild is None: + return + + run_step('copy patches', lambda: copy_patches(template_version, rust_version)) + ebuild_file = run_step( + 'create ebuild', lambda: create_ebuild(template_ebuild, rust_version)) + run_step('update ebuild', lambda: update_ebuild(ebuild_file, stage0_info)) + with tempfile.TemporaryDirectory(dir='/tmp') as tempdir: + run_step('upload_to_localmirror', lambda: upload_to_localmirror( + tempdir, rust_version, stage0_info)) + run_step('update manifest to add new version', lambda: update_manifest( + ebuild_file)) + if not skip_compile: + run_step('emerge rust', lambda: subprocess.check_call( + ['sudo', 'emerge', 'dev-lang/rust'])) + run_step('insert version into rust packages', lambda: update_rust_packages( + rust_version, add=True)) + run_step('upgrade virtual/rust', lambda: update_virtual_rust( + template_version, rust_version)) + + +def find_rust_versions_in_chroot() -> List[Tuple[RustVersion, str]]: + return [(RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) + for x in os.listdir(RUST_PATH) + if x.endswith('.ebuild')] + + +def find_oldest_rust_version_in_chroot() -> Tuple[RustVersion, str]: + rust_versions = find_rust_versions_in_chroot() + if len(rust_versions) <= 1: + raise RuntimeError('Expect to find more than one Rust versions') + return min(rust_versions) + + +def find_ebuild_for_rust_version(version: RustVersion) -> str: + rust_ebuilds = [ + ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version + ] + if not rust_ebuilds: + raise ValueError(f'No Rust ebuilds found matching {version}') + if len(rust_ebuilds) > 1: + raise ValueError(f'Multiple Rust ebuilds found matching {version}: ' + f'{rust_ebuilds}') + return rust_ebuilds[0] + + +def remove_files(filename: str, path: str) -> None: + subprocess.check_call(['git', 'rm', filename], cwd=path) + + +def remove_rust_uprev(rust_version: Optional[RustVersion], + run_step: Callable[[], T]) -> None: + + def find_desired_rust_version(): + if rust_version: + return rust_version, find_ebuild_for_rust_version(rust_version) + return find_oldest_rust_version_in_chroot() + + delete_version, delete_ebuild = run_step( + 'find rust version to delete', + find_desired_rust_version, + result_from_json=prepare_uprev_from_json, + ) + run_step( + 'remove patches', lambda: remove_files( + f'files/rust-{delete_version}-*.patch', RUST_PATH)) + run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH)) + ebuild_file = get_command_output(['equery', 'w', 'rust']) + run_step('update manifest to delete old version', lambda: update_manifest( + ebuild_file)) + run_step('remove version from rust packages', lambda: update_rust_packages( + delete_version, add=False)) + run_step( + 'remove virtual/rust', lambda: remove_files( + f'rust-{delete_version}.ebuild', + os.path.join(RUST_PATH, '../../virtual/rust'))) + + +def create_new_repo(rust_version: RustVersion) -> None: + output = get_command_output(['git', 'status', '--porcelain'], cwd=RUST_PATH) + if output: + raise RuntimeError( + f'{RUST_PATH} has uncommitted changes, please either discard them ' + 'or commit them.') + git.CreateBranch(RUST_PATH, f'rust-to-{rust_version}') + + +def build_cross_compiler() -> None: + # Get target triples in ebuild + rust_ebuild = get_command_output(['equery', 'w', 'rust']) + with open(rust_ebuild, encoding='utf-8') as f: + contents = f.read() + + target_triples_re = re.compile(r'RUSTC_TARGET_TRIPLES=\(([^)]+)\)') + m = target_triples_re.search(contents) + assert m, 'RUST_TARGET_TRIPLES not found in rust ebuild' + target_triples = m.group(1).strip().split('\n') + + compiler_targets_to_install = [ + target.strip() for target in target_triples if 'cros-' in target + ] + for target in target_triples: + if 'cros-' not in target: + continue + target = target.strip() + + # We also always need arm-none-eabi, though it's not mentioned in + # RUSTC_TARGET_TRIPLES. + compiler_targets_to_install.append('arm-none-eabi') + + logging.info('Emerging cross compilers %s', compiler_targets_to_install) + subprocess.check_call( + ['sudo', 'emerge', '-j', '-G'] + + [f'cross-{target}/gcc' for target in compiler_targets_to_install]) + + +def create_new_commit(rust_version: RustVersion) -> None: + subprocess.check_call(['git', 'add', '-A'], cwd=RUST_PATH) + messages = [ + f'[DO NOT SUBMIT] dev-lang/rust: upgrade to Rust {rust_version}', + '', + 'This CL is created by rust_uprev tool automatically.' + '', + 'BUG=None', + 'TEST=Use CQ to test the new Rust version', + ] + git.UploadChanges(RUST_PATH, f'rust-to-{rust_version}', messages) + + +def main() -> None: + if not chroot.InChroot(): + raise RuntimeError('This script must be executed inside chroot') + + logging.basicConfig(level=logging.INFO) + + args = parse_commandline_args() + + state_file = pathlib.Path(args.state_file) + tmp_state_file = state_file.with_suffix('.tmp') + + try: + with state_file.open(encoding='utf-8') as f: + completed_steps = json.load(f) + except FileNotFoundError: + completed_steps = {} + + def run_step( + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, + ) -> T: + return perform_step(state_file, tmp_state_file, completed_steps, step_name, + step_fn, result_from_json, result_to_json) + + if args.subparser_name == 'create': + create_rust_uprev(args.rust_version, args.template, args.skip_compile, + run_step) + elif args.subparser_name == 'remove': + remove_rust_uprev(args.rust_version, run_step) + else: + # If you have added more subparser_name, please also add the handlers above + assert args.subparser_name == 'roll' + run_step('create new repo', lambda: create_new_repo(args.uprev)) + if not args.skip_cross_compiler: + run_step('build cross compiler', build_cross_compiler) + create_rust_uprev(args.uprev, args.template, args.skip_compile, run_step) + remove_rust_uprev(args.remove, run_step) + if not args.no_upload: + run_step('create rust uprev CL', lambda: create_new_commit(args.uprev)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py new file mode 100755 index 00000000..fc506004 --- /dev/null +++ b/rust_tools/rust_uprev_test.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for rust_uprev.py""" + +# pylint: disable=cros-logging-import +import os +import shutil +import subprocess +import unittest +from unittest import mock + +from llvm_tools import git + +import rust_uprev + + +class RustVersionTest(unittest.TestCase): + """Tests for RustVersion class""" + + def test_str(self): + obj = rust_uprev.RustVersion(major=1, minor=2, patch=3) + self.assertEqual(str(obj), '1.2.3') + + def test_parse_version_only(self): + expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) + actual = rust_uprev.RustVersion.parse('1.2.3') + self.assertEqual(expected, actual) + + def test_parse_ebuild_name(self): + expected = rust_uprev.RustVersion(major=1, minor=2, patch=3) + actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3.ebuild') + self.assertEqual(expected, actual) + + actual = rust_uprev.RustVersion.parse_from_ebuild('rust-1.2.3-r1.ebuild') + self.assertEqual(expected, actual) + + def test_parse_fail(self): + with self.assertRaises(AssertionError) as context: + rust_uprev.RustVersion.parse('invalid-rust-1.2.3') + self.assertEqual("failed to parse 'invalid-rust-1.2.3'", + str(context.exception)) + + +class PrepareUprevTest(unittest.TestCase): + """Tests for prepare_uprev step in rust_uprev""" + + def setUp(self): + self.version_old = rust_uprev.RustVersion(1, 2, 3) + self.version_new = rust_uprev.RustVersion(1, 3, 5) + + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_success_with_template(self, mock_command, mock_find_ebuild): + expected = (self.version_old, '/path/to/ebuild') + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=self.version_old) + self.assertEqual(expected, actual) + mock_command.assert_not_called() + + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_return_none_with_template_larger_than_input(self, mock_command, + _mock_find_ebuild): + ret = rust_uprev.prepare_uprev( + rust_version=self.version_old, template=self.version_new) + self.assertIsNone(ret) + mock_command.assert_not_called() + + @mock.patch.object(os.path, 'exists') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_success_without_template(self, mock_command, mock_exists): + rust_ebuild_path = f'/path/to/rust/rust-{self.version_old}-r3.ebuild' + mock_command.return_value = rust_ebuild_path + expected = (self.version_old, rust_ebuild_path) + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=None) + self.assertEqual(expected, actual) + mock_command.assert_called_once_with(['equery', 'w', 'rust']) + mock_exists.assert_not_called() + + @mock.patch.object(os.path, 'exists') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_return_none_with_ebuild_larger_than_input(self, mock_command, + mock_exists): + mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild' + ret = rust_uprev.prepare_uprev(rust_version=self.version_old, template=None) + self.assertIsNone(ret) + mock_exists.assert_not_called() + + def test_prepare_uprev_from_json(self): + ebuild_path = '/path/to/the/ebuild' + json_result = (list(self.version_new), ebuild_path) + expected = (self.version_new, ebuild_path) + actual = rust_uprev.prepare_uprev_from_json(json_result) + self.assertEqual(expected, actual) + + +class UpdateEbuildTest(unittest.TestCase): + """Tests for update_ebuild step in rust_uprev""" + ebuild_file_before = """ + STAGE0_DATE="2019-01-01" + STAGE0_VERSION="any.random.(number)" + STAGE0_VERSION_CARGO="0.0.0" + """ + ebuild_file_after = """ + STAGE0_DATE="2020-01-01" + STAGE0_VERSION="1.1.1" + STAGE0_VERSION_CARGO="0.1.0" + """ + + def test_success(self): + mock_open = mock.mock_open(read_data=self.ebuild_file_before) + ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' + with mock.patch('builtins.open', mock_open): + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) + mock_open.return_value.__enter__().write.assert_called_once_with( + self.ebuild_file_after) + + def test_fail_when_ebuild_misses_a_variable(self): + ebuild_file = 'STAGE0_DATE="2019-01-01"' + mock_open = mock.mock_open(read_data=ebuild_file) + ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' + with mock.patch('builtins.open', mock_open): + with self.assertRaises(RuntimeError) as context: + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) + self.assertEqual('STAGE0_VERSION not found in rust ebuild', + str(context.exception)) + + +class UpdateManifestTest(unittest.TestCase): + """Tests for update_manifest step in rust_uprev""" + + # pylint: disable=protected-access + def _run_test_flip_mirror(self, before, after, add, expect_write): + mock_open = mock.mock_open(read_data=f'RESTRICT="{before}"') + with mock.patch('builtins.open', mock_open): + rust_uprev.flip_mirror_in_ebuild('', add=add) + if expect_write: + mock_open.return_value.__enter__().write.assert_called_once_with( + f'RESTRICT="{after}"') + + def test_add_mirror_in_ebuild(self): + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2 mirror', + add=True, + expect_write=True) + + def test_remove_mirror_in_ebuild(self): + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2', + add=False, + expect_write=True) + + def test_add_mirror_when_exists(self): + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2 mirror', + add=True, + expect_write=False) + + def test_remove_mirror_when_not_exists(self): + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2', + add=False, + expect_write=False) + + @mock.patch.object(rust_uprev, 'flip_mirror_in_ebuild') + @mock.patch.object(rust_uprev, 'rust_ebuild_actions') + def test_update_manifest(self, mock_run, mock_flip): + ebuild_file = '/path/to/rust/rust-1.1.1.ebuild' + rust_uprev.update_manifest(ebuild_file) + mock_run.assert_called_once_with(['manifest']) + mock_flip.assert_has_calls( + [mock.call(ebuild_file, add=True), + mock.call(ebuild_file, add=False)]) + + +class UpdateRustPackagesTests(unittest.TestCase): + """Tests for update_rust_packages step.""" + + def setUp(self): + self.old_version = rust_uprev.RustVersion(1, 1, 0) + self.current_version = rust_uprev.RustVersion(1, 2, 3) + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, + 'rust-{self.new_version}.ebuild') + + def test_add_new_rust_packages(self): + package_before = (f'dev-lang/rust-{self.old_version}\n' + f'dev-lang/rust-{self.current_version}') + package_after = (f'dev-lang/rust-{self.old_version}\n' + f'dev-lang/rust-{self.current_version}\n' + f'dev-lang/rust-{self.new_version}') + mock_open = mock.mock_open(read_data=package_before) + with mock.patch('builtins.open', mock_open): + rust_uprev.update_rust_packages(self.new_version, add=True) + mock_open.return_value.__enter__().write.assert_called_once_with( + package_after) + + def test_remove_old_rust_packages(self): + package_before = (f'dev-lang/rust-{self.old_version}\n' + f'dev-lang/rust-{self.current_version}\n' + f'dev-lang/rust-{self.new_version}') + package_after = (f'dev-lang/rust-{self.current_version}\n' + f'dev-lang/rust-{self.new_version}') + mock_open = mock.mock_open(read_data=package_before) + with mock.patch('builtins.open', mock_open): + rust_uprev.update_rust_packages(self.old_version, add=False) + mock_open.return_value.__enter__().write.assert_called_once_with( + package_after) + + +class UploadToLocalmirrorTests(unittest.TestCase): + """Tests for upload_to_localmirror""" + + def setUp(self): + self.tempdir = '/tmp/any/dir' + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.rust_url = 'https://static.rust-lang.org/dist' + self.tarfile_name = f'rustc-{self.new_version}-src.tar.gz' + self.rust_src = f'https://static.rust-lang.org/dist/{self.tarfile_name}' + self.gsurl = f'gs://chromeos-localmirror/distfiles/{self.tarfile_name}' + self.rust_file = os.path.join(self.tempdir, self.tarfile_name) + self.sig_file = os.path.join(self.tempdir, 'rustc_sig.asc') + + @mock.patch.object(subprocess, 'call', return_value=1) + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + def test_pass_without_retry(self, mock_run, mock_output, mock_call, + mock_raw_call): + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_output.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + mock_run.assert_not_called() + + @mock.patch.object(subprocess, 'call') + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_pass_with_retry(self, mock_output, mock_run, mock_check, mock_call, + mock_raw_call): + mock_check.side_effect = subprocess.CalledProcessError( + returncode=2, cmd=None, output="gpg: Can't check signature") + mock_output.return_value = 'some_gpg_keys' + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_check.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_output.assert_called_once_with( + ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc']) + mock_run.assert_called_once_with(['gpg', '--import'], + input='some_gpg_keys', + encoding='utf-8', + check=True) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call(['gpg', '--verify', self.sig_file, self.rust_file]), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + + @mock.patch.object(rust_uprev, 'upload_single_tarball') + def test_upload_to_mirror(self, mock_upload): + stage0_info = '2020-01-01', '1.1.1', '0.1.0' + rust_uprev.upload_to_localmirror(self.tempdir, self.new_version, + stage0_info) + mock_upload.assert_has_calls([ + mock.call(self.rust_url, f'rustc-{self.new_version}-src.tar.gz', + self.tempdir), + mock.call(f'{self.rust_url}/{stage0_info[0]}', + f'rust-std-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'rustc-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'cargo-{stage0_info[2]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + ]) + + +class RustUprevOtherStagesTests(unittest.TestCase): + """Tests for other steps in rust_uprev""" + + def setUp(self): + self.old_version = rust_uprev.RustVersion(1, 1, 0) + self.current_version = rust_uprev.RustVersion(1, 2, 3) + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, + 'rust-{self.new_version}.ebuild') + + @mock.patch.object(rust_uprev, 'get_command_output') + def test_parse_stage0_file(self, mock_get): + stage0_file = """ + unrelated stuff before + date: 2020-01-01 + rustc: 1.1.1 + cargo: 0.1.0 + unrelated stuff after + """ + mock_get.return_value = stage0_file + expected = '2020-01-01', '1.1.1', '0.1.0' + rust_version = rust_uprev.RustVersion(1, 2, 3) + actual = rust_uprev.parse_stage0_file(rust_version) + self.assertEqual(expected, actual) + mock_get.assert_called_once_with([ + 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/' + f'{rust_version}/src/stage0.txt' + ]) + + @mock.patch.object(shutil, 'copyfile') + @mock.patch.object(os, 'listdir') + @mock.patch.object(subprocess, 'check_call') + def test_copy_patches(self, mock_call, mock_ls, mock_copy): + mock_ls.return_value = [ + f'rust-{self.old_version}-patch-1.patch', + f'rust-{self.old_version}-patch-2-old.patch', + f'rust-{self.current_version}-patch-1.patch', + f'rust-{self.current_version}-patch-2-new.patch' + ] + rust_uprev.copy_patches(self.current_version, self.new_version) + mock_copy.assert_has_calls([ + mock.call( + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.current_version}-patch-1.patch'), + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.new_version}-patch-1.patch'), + ), + mock.call( + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.current_version}-patch-2-new.patch'), + os.path.join(rust_uprev.RUST_PATH, 'files', + f'rust-{self.new_version}-patch-2-new.patch')) + ]) + mock_call.assert_called_once_with( + ['git', 'add', f'files/rust-{self.new_version}-*.patch'], + cwd=rust_uprev.RUST_PATH) + + @mock.patch.object(shutil, 'copyfile') + @mock.patch.object(subprocess, 'check_call') + def test_create_ebuild(self, mock_call, mock_copy): + template_ebuild = f'/path/to/rust-{self.current_version}-r2.ebuild' + rust_uprev.create_ebuild(template_ebuild, self.new_version) + mock_copy.assert_called_once_with( + template_ebuild, + os.path.join(rust_uprev.RUST_PATH, f'rust-{self.new_version}.ebuild')) + mock_call.assert_called_once_with( + ['git', 'add', f'rust-{self.new_version}.ebuild'], + cwd=rust_uprev.RUST_PATH) + + @mock.patch.object(os.path, 'exists', return_value=True) + @mock.patch.object(shutil, 'copyfile') + @mock.patch.object(subprocess, 'check_call') + def test_update_virtual_rust(self, mock_call, mock_copy, mock_exists): + virtual_rust_dir = os.path.join(rust_uprev.RUST_PATH, '../../virtual/rust') + rust_uprev.update_virtual_rust(self.current_version, self.new_version) + mock_call.assert_called_once_with( + ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=virtual_rust_dir) + mock_copy.assert_called_once_with( + os.path.join(virtual_rust_dir, f'rust-{self.current_version}.ebuild'), + os.path.join(virtual_rust_dir, f'rust-{self.new_version}.ebuild')) + mock_exists.assert_called_once_with(virtual_rust_dir) + + @mock.patch.object(os, 'listdir') + def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls): + oldest_version_name = f'rust-{self.old_version}.ebuild' + mock_ls.return_value = [ + oldest_version_name, f'rust-{self.current_version}.ebuild', + f'rust-{self.new_version}.ebuild' + ] + actual = rust_uprev.find_oldest_rust_version_in_chroot() + expected = (self.old_version, + os.path.join(rust_uprev.RUST_PATH, oldest_version_name)) + self.assertEqual(expected, actual) + + @mock.patch.object(os, 'listdir') + def test_find_oldest_rust_version_in_chroot_fail_with_only_one_ebuild( + self, mock_ls): + mock_ls.return_value = [f'rust-{self.new_version}.ebuild'] + with self.assertRaises(RuntimeError) as context: + rust_uprev.find_oldest_rust_version_in_chroot() + self.assertEqual('Expect to find more than one Rust versions', + str(context.exception)) + + @mock.patch.object(rust_uprev, 'get_command_output') + @mock.patch.object(git, 'CreateBranch') + def test_create_new_repo(self, mock_branch, mock_output): + mock_output.return_value = '' + rust_uprev.create_new_repo(self.new_version) + mock_branch.assert_called_once_with(rust_uprev.RUST_PATH, + f'rust-to-{self.new_version}') + + @mock.patch.object(rust_uprev, 'get_command_output') + @mock.patch.object(subprocess, 'check_call') + def test_build_cross_compiler(self, mock_call, mock_output): + mock_output.return_value = f'rust-{self.new_version}.ebuild' + cros_targets = [ + 'x86_64-cros-linux-gnu', + 'armv7a-cros-linux-gnueabihf', + 'aarch64-cros-linux-gnu', + ] + all_triples = ['x86_64-pc-linux-gnu'] + cros_targets + rust_ebuild = 'RUSTC_TARGET_TRIPLES=(' + '\n\t'.join(all_triples) + ')' + mock_open = mock.mock_open(read_data=rust_ebuild) + with mock.patch('builtins.open', mock_open): + rust_uprev.build_cross_compiler() + + mock_call.assert_called_once_with( + ['sudo', 'emerge', '-j', '-G'] + + [f'cross-{x}/gcc' for x in cros_targets + ['arm-none-eabi']]) + + +if __name__ == '__main__': + unittest.main() diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py new file mode 100755 index 00000000..b9ad7b82 --- /dev/null +++ b/rust_tools/rust_watch.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Checks for various upstream events with the Rust toolchain. + +Sends an email if something interesting (probably) happened. +""" + +# pylint: disable=cros-logging-import + +import argparse +import itertools +import json +import logging +import pathlib +import re +import shutil +import subprocess +import sys +import time +from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple + +from cros_utils import email_sender +from cros_utils import tiny_render + + +def gentoo_sha_to_link(sha: str) -> str: + """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" + return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}' + + +def send_email(subject: str, body: List[tiny_render.Piece]) -> None: + """Sends an email with the given title and body to... whoever cares.""" + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier='rust-watch', + well_known_recipients=['cros-team'], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) + + +class RustReleaseVersion(NamedTuple): + """Represents a version of Rust's stable compiler.""" + major: int + minor: int + patch: int + + @staticmethod + def from_string(version_string: str) -> 'RustReleaseVersion': + m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string) + if not m: + raise ValueError(f"{version_string!r} isn't a valid version string") + return RustReleaseVersion(*[int(x) for x in m.groups()]) + + def __str__(self) -> str: + return f'{self.major}.{self.minor}.{self.patch}' + + def to_json(self) -> str: + return str(self) + + @staticmethod + def from_json(s: str) -> 'RustReleaseVersion': + return RustReleaseVersion.from_string(s) + + +class State(NamedTuple): + """State that we keep around from run to run.""" + # The last Rust release tag that we've seen. + last_seen_release: RustReleaseVersion + + # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen + # that updates it. + last_gentoo_sha: str + + def to_json(self) -> Dict[str, Any]: + return { + 'last_seen_release': self.last_seen_release.to_json(), + 'last_gentoo_sha': self.last_gentoo_sha, + } + + @staticmethod + def from_json(s: Dict[str, Any]) -> 'State': + return State( + last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']), + last_gentoo_sha=s['last_gentoo_sha'], + ) + + +def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: + """Parses `git ls-remote --tags` output into Rust stable release versions.""" + refs_tags = 'refs/tags/' + for line in lines: + _sha, tag = line.split(None, 1) + tag = tag.strip() + # Each tag has an associated 'refs/tags/name^{}', which is the actual + # object that the tag points to. That's irrelevant to us. + if tag.endswith('^{}'): + continue + + if not tag.startswith(refs_tags): + continue + + short_tag = tag[len(refs_tags):] + # There are a few old versioning schemes. Ignore them. + if short_tag.startswith('0.') or short_tag.startswith('release-'): + continue + yield RustReleaseVersion.from_string(short_tag) + + +def fetch_most_recent_release() -> RustReleaseVersion: + """Fetches the most recent stable `rustc` version.""" + result = subprocess.run( + ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'], + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + tag_lines = result.stdout.strip().splitlines() + return max(parse_release_tags(tag_lines)) + + +class GitCommit(NamedTuple): + """Represents a single git commit.""" + sha: str + subject: str + + +def update_git_repo(git_dir: pathlib.Path) -> None: + """Updates the repo at `git_dir`, retrying a few times on failure.""" + for i in itertools.count(start=1): + result = subprocess.run( + ['git', 'fetch', 'origin'], + check=False, + cwd=str(git_dir), + stdin=None, + ) + + if not result.returncode: + break + + if i == 5: + # 5 attempts is too many. Something else may be wrong. + result.check_returncode() + + sleep_time = 60 * i + logging.error("Failed updating gentoo's repo; will try again in %ds...", + sleep_time) + time.sleep(sleep_time) + + +def get_new_gentoo_commits(git_dir: pathlib.Path, + most_recent_sha: str) -> List[GitCommit]: + """Gets commits to dev-lang/rust since `most_recent_sha`. + + Older commits come earlier in the returned list. + """ + commits = subprocess.run( + [ + 'git', + 'log', + '--format=%H %s', + f'{most_recent_sha}..origin/master', + '--', + 'dev-lang/rust', + ], + capture_output=True, + check=False, + cwd=str(git_dir), + encoding='utf-8', + ) + + if commits.returncode: + logging.error('Error getting new gentoo commits; stderr:\n%s', + commits.stderr) + commits.check_returncode() + + results = [] + for line in commits.stdout.strip().splitlines(): + sha, subject = line.strip().split(None, 1) + results.append(GitCommit(sha=sha, subject=subject)) + + # `git log` outputs things in newest -> oldest order. + results.reverse() + return results + + +def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: + """Sets up a gentoo git repo at the given directory. Returns HEAD.""" + subprocess.run( + [ + 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git', + str(git_dir) + ], + stdin=None, + check=True, + ) + + head_rev = subprocess.run( + ['git', 'rev-parse', 'HEAD'], + cwd=str(git_dir), + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + return head_rev.stdout.strip() + + +def read_state(state_file: pathlib.Path) -> State: + """Reads state from the given file.""" + with state_file.open(encoding='utf-8') as f: + return State.from_json(json.load(f)) + + +def atomically_write_state(state_file: pathlib.Path, state: State) -> None: + """Writes state to the given file.""" + temp_file = pathlib.Path(str(state_file) + '.new') + with temp_file.open('w', encoding='utf-8') as f: + json.dump(state.to_json(), f) + temp_file.rename(state_file) + + +def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, + new_gentoo_commits: List[GitCommit] + ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: + """Creates an email given our new state, if doing so is appropriate.""" + subject_pieces = [] + body_pieces = [] + + if newest_release > old_state.last_seen_release: + subject_pieces.append('new rustc release detected') + body_pieces.append(f'Rustc tag for v{newest_release} was found.') + + if new_gentoo_commits: + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] + + if len(new_gentoo_commits) == 1: + subject_pieces.append('new rust ebuild commit detected') + body_pieces.append('commit:') + else: + subject_pieces.append('new rust ebuild commits detected') + body_pieces.append('commits (newest first):') + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append([ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f': {commit.subject}', + ]) + + body_pieces.append(tiny_render.UnorderedList(commit_lines)) + + if not subject_pieces: + return None + + subject = '[rust-watch] ' + '; '.join(subject_pieces) + return subject, body_pieces + + +def main(argv: List[str]) -> None: + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--state_dir', required=True, help='Directory to store state in.') + parser.add_argument( + '--skip_email', action='store_true', help="Don't send an email.") + parser.add_argument( + '--skip_state_update', + action='store_true', + help="Don't update the state file. Doesn't apply to initial setup.") + opts = parser.parse_args(argv) + + state_dir = pathlib.Path(opts.state_dir) + state_file = state_dir / 'state.json' + gentoo_subdir = state_dir / 'upstream-gentoo' + if not state_file.exists(): + logging.info("state_dir isn't fully set up; doing that now.") + + # Could be in a partially set-up state. + if state_dir.exists(): + logging.info('incomplete state_dir detected; removing.') + shutil.rmtree(str(state_dir)) + + state_dir.mkdir(parents=True) + most_recent_release = fetch_most_recent_release() + most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=most_recent_gentoo_commit, + ), + ) + # Running through this _should_ be a nop, but do it anyway. Should make any + # bugs more obvious on the first run of the script. + + prior_state = read_state(state_file) + logging.info('Last state was %r', prior_state) + + most_recent_release = fetch_most_recent_release() + logging.info('Most recent Rust release is %s', most_recent_release) + + logging.info('Fetching new commits from Gentoo') + update_git_repo(gentoo_subdir) + new_commits = get_new_gentoo_commits(gentoo_subdir, + prior_state.last_gentoo_sha) + logging.info('New commits: %r', new_commits) + + maybe_email = maybe_compose_email(prior_state, most_recent_release, + new_commits) + + if maybe_email is None: + logging.info('No updates to send') + else: + title, body = maybe_email + if opts.skip_email: + logging.info('Skipping sending email with title %r and contents\n%s', + title, tiny_render.render_html_pieces(body)) + else: + logging.info('Sending email') + send_email(title, body) + + if opts.skip_state_update: + logging.info('Skipping state update, as requested') + return + + newest_sha = ( + new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=newest_sha, + ), + ) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py new file mode 100755 index 00000000..97d111fc --- /dev/null +++ b/rust_tools/rust_watch_test.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for rust_watch.py.""" + +# pylint: disable=cros-logging-import + +import logging +import pathlib +import subprocess +import time +import unittest +import unittest.mock + +import rust_watch +from cros_utils import tiny_render + + +class Test(unittest.TestCase): + """Tests.""" + + def _silence_logs(self): + """Silences all log output until the end of the current test.""" + + def should_log(_record): + return 0 + + logger = logging.root + logger.addFilter(should_log) + self.addCleanup(logger.removeFilter, should_log) + + def test_release_version_parsing(self): + self.assertEqual( + rust_watch.RustReleaseVersion.from_string('1.2.3'), + rust_watch.RustReleaseVersion(1, 2, 3), + ) + + def test_release_version_json_round_trips(self): + ver = rust_watch.RustReleaseVersion(1, 2, 3) + self.assertEqual( + rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver) + + def test_state_json_round_trips(self): + state = rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 2, 3), + last_gentoo_sha='abc123', + ) + + self.assertEqual(rust_watch.State.from_json(state.to_json()), state) + + @unittest.mock.patch.object(subprocess, 'run') + @unittest.mock.patch.object(time, 'sleep') + def test_update_git_repo_tries_again_on_failure(self, sleep_mock, run_mock): + self._silence_logs() + + oh_no_error = ValueError('oh no') + + def check_returncode(): + raise oh_no_error + + run_call_count = 0 + + def run_sideeffect(*_args, **_kwargs): + nonlocal run_call_count + run_call_count += 1 + result = unittest.mock.Mock() + result.returncode = 1 + result.check_returncode = check_returncode + return result + + run_mock.side_effect = run_sideeffect + + with self.assertRaises(ValueError) as raised: + rust_watch.update_git_repo(pathlib.Path('/does/not/exist/at/all')) + + self.assertIs(raised.exception, oh_no_error) + self.assertEqual(run_call_count, 5) + + sleep_timings = [unittest.mock.call(60 * i) for i in range(1, 5)] + self.assertEqual(sleep_mock.mock_calls, sleep_timings) + + @unittest.mock.patch.object(subprocess, 'run') + def test_get_new_gentoo_commits_functions(self, run_mock): + returned = unittest.mock.Mock() + returned.returncode = 0 + returned.stdout = '\n'.join(( + 'abc123 newer commit', + 'abcdef and an older commit', + )) + run_mock.return_value = returned + results = rust_watch.get_new_gentoo_commits( + pathlib.Path('/does/not/exist/at/all'), 'defabc') + self.assertEqual(results, [ + rust_watch.GitCommit('abcdef', 'and an older commit'), + rust_watch.GitCommit('abc123', 'newer commit'), + ]) + + def test_compose_email_on_a_new_release(self): + new_release = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[], + ) + + self.assertEqual(new_release, ('[rust-watch] new rustc release detected', + ['Rustc tag for v1.1.0 was found.'])) + + def test_compose_email_on_a_new_gentoo_commit(self): + sha_a = 'a' * 40 + new_commit = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], + ) + + self.assertEqual(new_commit, + ('[rust-watch] new rust ebuild commit detected', [ + 'commit:', + tiny_render.UnorderedList([ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ': summary_a', + ], + ]) + ])) + + def test_compose_email_on_multiple_events(self): + sha_a = 'a' * 40 + new_commit_and_release = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], + ) + + self.assertEqual( + new_commit_and_release, + ('[rust-watch] new rustc release detected; new rust ebuild commit ' + 'detected', [ + 'Rustc tag for v1.1.0 was found.', + tiny_render.line_break, + tiny_render.line_break, + 'commit:', + tiny_render.UnorderedList([ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ': summary_a', + ], + ]), + ])) + + def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + self.assertIsNone( + rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], + )) + + self.assertIsNone( + rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], + )) + + +if __name__ == '__main__': + unittest.main() |