diff options
Diffstat (limited to 'rust_tools')
| -rwxr-xr-x | rust_tools/rust_uprev.py | 479 | ||||
| -rwxr-xr-x | rust_tools/rust_uprev_test.py | 415 | ||||
| -rwxr-xr-x | rust_tools/rust_watch.py | 127 | ||||
| -rwxr-xr-x | rust_tools/rust_watch_test.py | 109 |
4 files changed, 493 insertions, 637 deletions
diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py index 011639df..3c0ad012 100755 --- a/rust_tools/rust_uprev.py +++ b/rust_tools/rust_uprev.py @@ -33,6 +33,8 @@ the CL to chromium code review. See `--help` for all available options. """ +# pylint: disable=cros-logging-import + import argparse import pathlib import json @@ -42,30 +44,16 @@ import re import shutil import subprocess import sys -from pathlib import Path +import tempfile from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple from llvm_tools import chroot, git - -EQUERY = 'equery' -GSUTIL = 'gsutil.py' -MIRROR_PATH = 'gs://chromeos-localmirror/distfiles' -RUST_PATH = Path( - '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust') +RUST_PATH = '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust' def get_command_output(command: List[str], *args, **kwargs) -> str: - return subprocess.check_output(command, encoding='utf-8', *args, - **kwargs).strip() - - -def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str: - return subprocess.run(command, - check=False, - stdout=subprocess.PIPE, - encoding='utf-8', - *args, - **kwargs).stdout.strip() + return subprocess.check_output( + command, encoding='utf-8', *args, **kwargs).strip() class RustVersion(NamedTuple): @@ -87,8 +75,8 @@ class RustVersion(NamedTuple): r'\.ebuild$') m = input_re.match(ebuild_name) assert m, f'failed to parse {ebuild_name!r}' - return RustVersion(int(m.group('major')), int(m.group('minor')), - int(m.group('patch'))) + return RustVersion( + int(m.group('major')), int(m.group('minor')), int(m.group('patch'))) @staticmethod def parse(x: str) -> 'RustVersion': @@ -99,54 +87,13 @@ class RustVersion(NamedTuple): r'(?:.ebuild)?$') m = input_re.match(x) assert m, f'failed to parse {x!r}' - return RustVersion(int(m.group('major')), int(m.group('minor')), - int(m.group('patch'))) - - -def compute_rustc_src_name(version: RustVersion) -> str: - return f'rustc-{version}-src.tar.gz' - - -def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str: - return f'rust-bootstrap-{version}.tbz2' - - -def find_ebuild_for_package(name: str) -> os.PathLike: - """Returns the path to the ebuild for the named package.""" - return get_command_output([EQUERY, 'w', name]) - - -def find_ebuild_path(directory: Path, - name: str, - version: Optional[RustVersion] = None) -> Path: - """Finds an ebuild in a directory. - - Returns the path to the ebuild file. Asserts if there is not - exactly one match. The match is constrained by name and optionally - by version, but can match any patch level. E.g. "rust" version - 1.3.4 can match rust-1.3.4.ebuild but also rust-1.3.4-r6.ebuild. - """ - if version: - pattern = f'{name}-{version}*.ebuild' - else: - pattern = f'{name}-*.ebuild' - matches = list(Path(directory).glob(pattern)) - assert len(matches) == 1, matches - return matches[0] - - -def get_rust_bootstrap_version(): - """Get the version of the current rust-bootstrap package.""" - bootstrap_ebuild = find_ebuild_path(rust_bootstrap_path(), 'rust-bootstrap') - m = re.match(r'^rust-bootstrap-(\d+).(\d+).(\d+)', bootstrap_ebuild.name) - assert m, bootstrap_ebuild.name - return RustVersion(int(m.group(1)), int(m.group(2)), int(m.group(3))) + return RustVersion( + int(m.group('major')), int(m.group('minor')), int(m.group('patch'))) def parse_commandline_args() -> argparse.Namespace: parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '--state_file', required=True, @@ -211,18 +158,6 @@ def parse_commandline_args() -> argparse.Namespace: 'specified, the tool will remove the oldest version in the chroot', ) - subparser_names.append('remove-bootstrap') - remove_bootstrap_parser = subparsers.add_parser( - 'remove-bootstrap', - help='Remove an old rust-bootstrap version', - ) - remove_bootstrap_parser.add_argument( - '--version', - type=RustVersion.parse, - required=True, - help='rust-bootstrap version to remove', - ) - subparser_names.append('roll') roll_parser = subparsers.add_parser( 'roll', @@ -275,18 +210,33 @@ def parse_commandline_args() -> argparse.Namespace: return args +def parse_stage0_file(new_version: RustVersion) -> Tuple[str, str, str]: + # Find stage0 date, rustc and cargo + stage0_file = get_command_output([ + 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/' + f'{new_version}/src/stage0.txt' + ]) + regexp = re.compile(r'date:\s*(?P<date>\d+-\d+-\d+)\s+' + r'rustc:\s*(?P<rustc>\d+\.\d+\.\d+)\s+' + r'cargo:\s*(?P<cargo>\d+\.\d+\.\d+)') + m = regexp.search(stage0_file) + assert m, 'failed to parse stage0.txt file' + stage0_date, stage0_rustc, stage0_cargo = m.groups() + logging.info('Found stage0 file has date: %s, rustc: %s, cargo: %s', + stage0_date, stage0_rustc, stage0_cargo) + return stage0_date, stage0_rustc, stage0_cargo + + def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion] - ) -> Optional[Tuple[RustVersion, str, RustVersion]]: + ) -> Optional[Tuple[RustVersion, str]]: if template is None: - ebuild_path = find_ebuild_for_package('rust') + ebuild_path = get_command_output(['equery', 'w', 'rust']) ebuild_name = os.path.basename(ebuild_path) template_version = RustVersion.parse_from_ebuild(ebuild_name) else: ebuild_path = find_ebuild_for_rust_version(template) template_version = template - bootstrap_version = get_rust_bootstrap_version() - if rust_version <= template_version: logging.info( 'Requested version %s is not newer than the template version %s.', @@ -295,69 +245,66 @@ def prepare_uprev(rust_version: RustVersion, template: Optional[RustVersion] logging.info('Template Rust version is %s (ebuild: %r)', template_version, ebuild_path) - logging.info('rust-bootstrap version is %s', bootstrap_version) - - return template_version, ebuild_path, bootstrap_version + return template_version, ebuild_path -def copy_patches(directory: Path, template_version: RustVersion, +def copy_patches(template_version: RustVersion, new_version: RustVersion) -> None: - patch_path = directory.joinpath('files') - prefix = '%s-%s-' % (directory.name, template_version) - new_prefix = '%s-%s-' % (directory.name, new_version) + patch_path = os.path.join(RUST_PATH, 'files') for f in os.listdir(patch_path): - if not f.startswith(prefix): + if f'rust-{template_version}' not in f: continue - logging.info('Copy patch %s to new version', f) + logging.info('Rename patch %s to new version', f) new_name = f.replace(str(template_version), str(new_version)) shutil.copyfile( os.path.join(patch_path, f), os.path.join(patch_path, new_name), ) - subprocess.check_call(['git', 'add', f'{new_prefix}*.patch'], cwd=patch_path) + subprocess.check_call(['git', 'add', f'files/rust-{new_version}-*.patch'], + cwd=RUST_PATH) def create_ebuild(template_ebuild: str, new_version: RustVersion) -> str: shutil.copyfile(template_ebuild, - RUST_PATH.joinpath(f'rust-{new_version}.ebuild')) + os.path.join(RUST_PATH, f'rust-{new_version}.ebuild')) subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], cwd=RUST_PATH) return os.path.join(RUST_PATH, f'rust-{new_version}.ebuild') -def update_bootstrap_ebuild(new_bootstrap_version: RustVersion) -> None: - old_ebuild = find_ebuild_path(rust_bootstrap_path(), 'rust-bootstrap') - m = re.match(r'^rust-bootstrap-(\d+).(\d+).(\d+)', old_ebuild.name) - assert m, old_ebuild.name - old_version = RustVersion(m.group(1), m.group(2), m.group(3)) - new_ebuild = old_ebuild.parent.joinpath( - f'rust-bootstrap-{new_bootstrap_version}.ebuild') - old_text = old_ebuild.read_text(encoding='utf-8') - new_text, changes = re.subn(r'(RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=\([^)]*)', - f'\\1\t{old_version}\n', - old_text, - flags=re.MULTILINE) - assert changes == 1, 'Failed to update RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE' - new_ebuild.write_text(new_text, encoding='utf-8') - - -def update_ebuild(ebuild_file: str, - new_bootstrap_version: RustVersion) -> None: - contents = open(ebuild_file, encoding='utf-8').read() - contents, subs = re.subn(r'^BOOTSTRAP_VERSION=.*$', - 'BOOTSTRAP_VERSION="%s"' % - (new_bootstrap_version, ), - contents, - flags=re.MULTILINE) - if not subs: - raise RuntimeError('BOOTSTRAP_VERSION not found in rust ebuild') - open(ebuild_file, 'w', encoding='utf-8').write(contents) - logging.info('Rust ebuild file has BOOTSTRAP_VERSION updated to %s', - new_bootstrap_version) - - -def flip_mirror_in_ebuild(ebuild_file: Path, add: bool) -> None: +def update_ebuild(ebuild_file: str, stage0_info: Tuple[str, str, str]) -> None: + stage0_date, stage0_rustc, stage0_cargo = stage0_info + with open(ebuild_file, encoding='utf-8') as f: + contents = f.read() + # Update STAGE0_DATE in the ebuild + stage0_date_re = re.compile(r'STAGE0_DATE="(\d+-\d+-\d+)"') + if not stage0_date_re.search(contents): + raise RuntimeError('STAGE0_DATE not found in rust ebuild') + new_contents = stage0_date_re.sub(f'STAGE0_DATE="{stage0_date}"', contents) + + # Update STAGE0_VERSION in the ebuild + stage0_rustc_re = re.compile(r'STAGE0_VERSION="[^"]*"') + if not stage0_rustc_re.search(new_contents): + raise RuntimeError('STAGE0_VERSION not found in rust ebuild') + new_contents = stage0_rustc_re.sub(f'STAGE0_VERSION="{stage0_rustc}"', + new_contents) + + # Update STAGE0_VERSION_CARGO in the ebuild + stage0_cargo_re = re.compile(r'STAGE0_VERSION_CARGO="[^"]*"') + if not stage0_cargo_re.search(new_contents): + raise RuntimeError('STAGE0_VERSION_CARGO not found in rust ebuild') + new_contents = stage0_cargo_re.sub(f'STAGE0_VERSION_CARGO="{stage0_cargo}"', + new_contents) + with open(ebuild_file, 'w', encoding='utf-8') as f: + f.write(new_contents) + logging.info( + 'Rust ebuild file has STAGE0_DATE, STAGE0_VERSION, STAGE0_VERSION_CARGO ' + 'updated to %s, %s, %s respectively', stage0_date, stage0_rustc, + stage0_cargo) + + +def flip_mirror_in_ebuild(ebuild_file: str, add: bool) -> None: restrict_re = re.compile( r'(?P<before>RESTRICT=")(?P<values>"[^"]*"|.*)(?P<after>")') with open(ebuild_file, encoding='utf-8') as f: @@ -378,116 +325,25 @@ def flip_mirror_in_ebuild(ebuild_file: Path, add: bool) -> None: f.write(new_contents) -def ebuild_actions(package: str, actions: List[str], - sudo: bool = False) -> None: - ebuild_path_inchroot = find_ebuild_for_package(package) +def rust_ebuild_actions(actions: List[str], sudo: bool = False) -> None: + ebuild_path_inchroot = get_command_output(['equery', 'w', 'rust']) cmd = ['ebuild', ebuild_path_inchroot] + actions if sudo: cmd = ['sudo'] + cmd subprocess.check_call(cmd) -def fetch_distfile_from_mirror(name: str) -> None: - """Gets the named file from the local mirror. - - This ensures that the file exists on the mirror, and - that we can read it. We overwrite any existing distfile - to ensure the checksums that update_manifest() records - match the file as it exists on the mirror. - - This function also attempts to verify the ACL for - the file (which is expected to have READER permission - for allUsers). We can only see the ACL if the user - gsutil runs with is the owner of the file. If not, - we get an access denied error. We also count this - as a success, because it means we were able to fetch - the file even though we don't own it. - """ - mirror_file = MIRROR_PATH + '/' + name - local_file = Path(get_distdir(), name) - cmd = [GSUTIL, 'cp', mirror_file, local_file] - logging.info('Running %r', cmd) - rc = subprocess.call(cmd) - if rc != 0: - logging.error( - """Could not fetch %s - -If the file does not yet exist at %s -please download the file, verify its integrity -with something like: - -curl -O https://static.rust-lang.org/dist/%s -gpg --verify %s.asc - -You may need to import the signing key first, e.g.: - -gpg --recv-keys 85AB96E6FA1BE5FE - -Once you have verify the integrity of the file, upload -it to the local mirror using gsutil cp. -""", mirror_file, MIRROR_PATH, name, name) - raise Exception(f'Could not fetch {mirror_file}') - # Check that the ACL allows allUsers READER access. - # If we get an AccessDeniedAcception here, that also - # counts as a success, because we were able to fetch - # the file as a non-owner. - cmd = [GSUTIL, 'acl', 'get', mirror_file] - logging.info('Running %r', cmd) - output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT) - acl_verified = False - if 'AccessDeniedException:' in output: - acl_verified = True - else: - acl = json.loads(output) - for x in acl: - if x['entity'] == 'allUsers' and x['role'] == 'READER': - acl_verified = True - break - if not acl_verified: - logging.error('Output from acl get:\n%s', output) - raise Exception('Could not verify that allUsers has READER permission') - - -def fetch_bootstrap_distfiles(old_version: RustVersion, - new_version: RustVersion) -> None: - """Fetches rust-bootstrap distfiles from the local mirror - - Fetches the distfiles for a rust-bootstrap ebuild to ensure they - are available on the mirror and the local copies are the same as - the ones on the mirror. - """ - fetch_distfile_from_mirror(compute_rust_bootstrap_prebuilt_name(old_version)) - fetch_distfile_from_mirror(compute_rustc_src_name(new_version)) - - -def fetch_rust_distfiles(version: RustVersion) -> None: - """Fetches rust distfiles from the local mirror - - Fetches the distfiles for a rust ebuild to ensure they - are available on the mirror and the local copies are - the same as the ones on the mirror. - """ - fetch_distfile_from_mirror(compute_rustc_src_name(version)) - - -def get_distdir() -> os.PathLike: - """Returns portage's distdir.""" - return get_command_output(['portageq', 'distdir']) - - -def update_manifest(ebuild_file: os.PathLike) -> None: - """Updates the MANIFEST for the ebuild at the given path.""" - ebuild = Path(ebuild_file) - logging.info('Added "mirror" to RESTRICT to %s', ebuild.name) - flip_mirror_in_ebuild(ebuild, add=True) - ebuild_actions(ebuild.parent.name, ['manifest']) - logging.info('Removed "mirror" to RESTRICT from %s', ebuild.name) - flip_mirror_in_ebuild(ebuild, add=False) +def update_manifest(ebuild_file: str) -> None: + logging.info('Added "mirror" to RESTRICT to Rust ebuild') + flip_mirror_in_ebuild(ebuild_file, add=True) + rust_ebuild_actions(['manifest']) + logging.info('Removed "mirror" to RESTRICT from Rust ebuild') + flip_mirror_in_ebuild(ebuild_file, add=False) def update_rust_packages(rust_version: RustVersion, add: bool) -> None: - package_file = RUST_PATH.joinpath( - '../../profiles/targets/chromeos/package.provided') + package_file = os.path.join( + RUST_PATH, '../../profiles/targets/chromeos/package.provided') with open(package_file, encoding='utf-8') as f: contents = f.read() if add: @@ -511,13 +367,90 @@ def update_rust_packages(rust_version: RustVersion, add: bool) -> None: def update_virtual_rust(template_version: RustVersion, new_version: RustVersion) -> None: - template_ebuild = find_ebuild_path(RUST_PATH.joinpath('../../virtual/rust'), - 'rust', template_version) - virtual_rust_dir = template_ebuild.parent - new_name = f'rust-{new_version}.ebuild' - new_ebuild = virtual_rust_dir.joinpath(new_name) - shutil.copyfile(template_ebuild, new_ebuild) - subprocess.check_call(['git', 'add', new_name], cwd=virtual_rust_dir) + virtual_rust_dir = os.path.join(RUST_PATH, '../../virtual/rust') + assert os.path.exists(virtual_rust_dir) + shutil.copyfile( + os.path.join(virtual_rust_dir, f'rust-{template_version}.ebuild'), + os.path.join(virtual_rust_dir, f'rust-{new_version}.ebuild')) + subprocess.check_call(['git', 'add', f'rust-{new_version}.ebuild'], + cwd=virtual_rust_dir) + + +def upload_single_tarball(rust_url: str, tarfile_name: str, + tempdir: str) -> None: + rust_src = f'{rust_url}/{tarfile_name}' + gsutil_location = f'gs://chromeos-localmirror/distfiles/{tarfile_name}' + + missing_file = subprocess.call( + ['gsutil', 'ls', gsutil_location], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if not missing_file: + logging.info('Rust artifact at %s already exists; skipping download', + gsutil_location) + return + + logging.info('Downloading Rust artifact from %s', rust_src) + + # Download Rust's source + rust_file = os.path.join(tempdir, tarfile_name) + subprocess.check_call(['curl', '-f', '-o', rust_file, rust_src]) + + # Verify the signature of the source + sig_file = os.path.join(tempdir, 'rustc_sig.asc') + subprocess.check_call(['curl', '-f', '-o', sig_file, f'{rust_src}.asc']) + try: + subprocess.check_output(['gpg', '--verify', sig_file, rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + if "gpg: Can't check signature" not in e.output: + raise RuntimeError(f'Failed to execute `gpg --verify`, {e.output}') + + # If it fails to verify the signature, try import rustc key, and retry. + keys = get_command_output( + ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc']) + subprocess.run(['gpg', '--import'], + input=keys, + encoding='utf-8', + check=True) + subprocess.check_call(['gpg', '--verify', sig_file, rust_file]) + + # Since we are using `-n` to skip an item if it already exists, there's no + # need to check if the file exists on GS bucket or not. + subprocess.check_call( + ['gsutil', 'cp', '-n', '-a', 'public-read', rust_file, gsutil_location]) + + +def upload_to_localmirror(tempdir: str, rust_version: RustVersion, + stage0_info: Tuple[str, str, str]) -> None: + stage0_date, stage0_rustc, stage0_cargo = stage0_info + rust_url = 'https://static.rust-lang.org/dist' + # Upload rustc source + upload_single_tarball( + rust_url, + f'rustc-{rust_version}-src.tar.gz', + tempdir, + ) + # Upload stage0 toolchain + upload_single_tarball( + f'{rust_url}/{stage0_date}', + f'rust-std-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + # Upload stage0 source + upload_single_tarball( + rust_url, + f'rustc-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) + # Upload stage0 cargo + upload_single_tarball( + rust_url, + f'cargo-{stage0_cargo}-x86_64-unknown-linux-gnu.tar.gz', + tempdir, + ) def perform_step(state_file: pathlib.Path, @@ -547,18 +480,19 @@ def perform_step(state_file: pathlib.Path, return val -def prepare_uprev_from_json( - obj: Any) -> Optional[Tuple[RustVersion, str, RustVersion]]: +def prepare_uprev_from_json(obj: Any) -> Optional[Tuple[RustVersion, str]]: if not obj: return None - version, ebuild_path, bootstrap_version = obj - return RustVersion(*version), ebuild_path, RustVersion(*bootstrap_version) + version, ebuild_path = obj + return RustVersion(*version), ebuild_path def create_rust_uprev(rust_version: RustVersion, maybe_template_version: Optional[RustVersion], skip_compile: bool, run_step: Callable[[], T]) -> None: - template_version, template_ebuild, old_bootstrap_version = run_step( + stage0_info = run_step( + 'parse stage0 file', lambda: parse_stage0_file(rust_version)) + template_version, template_ebuild = run_step( 'prepare uprev', lambda: prepare_uprev(rust_version, maybe_template_version), result_from_json=prepare_uprev_from_json, @@ -566,31 +500,18 @@ def create_rust_uprev(rust_version: RustVersion, if template_ebuild is None: return - # The fetch steps will fail (on purpose) if the files they check for - # are not available on the mirror. To make them pass, fetch the - # required files yourself, verify their checksums, then upload them - # to the mirror. - run_step( - 'fetch bootstrap distfiles', lambda: fetch_bootstrap_distfiles( - old_bootstrap_version, template_version)) - run_step('fetch rust distfiles', lambda: fetch_rust_distfiles(rust_version)) - run_step('update bootstrap ebuild', lambda: update_bootstrap_ebuild( - template_version)) - run_step( - 'update bootstrap manifest', lambda: update_manifest(rust_bootstrap_path( - ).joinpath(f'rust-bootstrap-{template_version}.ebuild'))) - run_step('copy patches', lambda: copy_patches(RUST_PATH, template_version, - rust_version)) + run_step('copy patches', lambda: copy_patches(template_version, rust_version)) ebuild_file = run_step( 'create ebuild', lambda: create_ebuild(template_ebuild, rust_version)) - run_step( - 'update ebuild', lambda: update_ebuild(ebuild_file, template_version)) + run_step('update ebuild', lambda: update_ebuild(ebuild_file, stage0_info)) + with tempfile.TemporaryDirectory(dir='/tmp') as tempdir: + run_step('upload_to_localmirror', lambda: upload_to_localmirror( + tempdir, rust_version, stage0_info)) run_step('update manifest to add new version', lambda: update_manifest( - Path(ebuild_file))) + ebuild_file)) if not skip_compile: - run_step( - 'emerge rust', lambda: subprocess.check_call( - ['sudo', 'emerge', 'dev-lang/rust'])) + run_step('emerge rust', lambda: subprocess.check_call( + ['sudo', 'emerge', 'dev-lang/rust'])) run_step('insert version into rust packages', lambda: update_rust_packages( rust_version, add=True)) run_step('upgrade virtual/rust', lambda: update_virtual_rust( @@ -599,7 +520,8 @@ def create_rust_uprev(rust_version: RustVersion, def find_rust_versions_in_chroot() -> List[Tuple[RustVersion, str]]: return [(RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) - for x in os.listdir(RUST_PATH) if x.endswith('.ebuild')] + for x in os.listdir(RUST_PATH) + if x.endswith('.ebuild')] def find_oldest_rust_version_in_chroot() -> Tuple[RustVersion, str]: @@ -625,52 +547,32 @@ def remove_files(filename: str, path: str) -> None: subprocess.check_call(['git', 'rm', filename], cwd=path) -def remove_rust_bootstrap_version(version: RustVersion, - run_step: Callable[[], T]) -> None: - prefix = f'rust-bootstrap-{version}' - run_step('remove old bootstrap ebuild', lambda: remove_files( - f'{prefix}*.ebuild', rust_bootstrap_path())) - ebuild_file = find_ebuild_for_package('rust-bootstrap') - run_step('update bootstrap manifest to delete old version', lambda: - update_manifest(ebuild_file)) - - def remove_rust_uprev(rust_version: Optional[RustVersion], run_step: Callable[[], T]) -> None: + def find_desired_rust_version(): if rust_version: return rust_version, find_ebuild_for_rust_version(rust_version) return find_oldest_rust_version_in_chroot() - def find_desired_rust_version_from_json(obj: Any) -> Tuple[RustVersion, str]: - version, ebuild_path = obj - return RustVersion(*version), ebuild_path - delete_version, delete_ebuild = run_step( 'find rust version to delete', find_desired_rust_version, - result_from_json=find_desired_rust_version_from_json, + result_from_json=prepare_uprev_from_json, ) run_step( 'remove patches', lambda: remove_files( f'files/rust-{delete_version}-*.patch', RUST_PATH)) run_step('remove ebuild', lambda: remove_files(delete_ebuild, RUST_PATH)) - ebuild_file = find_ebuild_for_package('rust') + ebuild_file = get_command_output(['equery', 'w', 'rust']) run_step('update manifest to delete old version', lambda: update_manifest( ebuild_file)) run_step('remove version from rust packages', lambda: update_rust_packages( delete_version, add=False)) - run_step('remove virtual/rust', lambda: remove_virtual_rust(delete_version)) - - -def remove_virtual_rust(delete_version: RustVersion) -> None: - ebuild = find_ebuild_path(RUST_PATH.joinpath('../../virtual/rust'), 'rust', - delete_version) - subprocess.check_call(['git', 'rm', str(ebuild.name)], cwd=ebuild.parent) - - -def rust_bootstrap_path() -> Path: - return RUST_PATH.joinpath('../rust-bootstrap') + run_step( + 'remove virtual/rust', lambda: remove_files( + f'rust-{delete_version}.ebuild', + os.path.join(RUST_PATH, '../../virtual/rust'))) def create_new_repo(rust_version: RustVersion) -> None: @@ -684,7 +586,7 @@ def create_new_repo(rust_version: RustVersion) -> None: def build_cross_compiler() -> None: # Get target triples in ebuild - rust_ebuild = find_ebuild_for_package('rust') + rust_ebuild = get_command_output(['equery', 'w', 'rust']) with open(rust_ebuild, encoding='utf-8') as f: contents = f.read() @@ -755,8 +657,6 @@ def main() -> None: run_step) elif args.subparser_name == 'remove': remove_rust_uprev(args.rust_version, run_step) - elif args.subparser_name == 'remove-bootstrap': - remove_rust_bootstrap_version(args.version, run_step) else: # If you have added more subparser_name, please also add the handlers above assert args.subparser_name == 'roll' @@ -765,9 +665,6 @@ def main() -> None: run_step('build cross compiler', build_cross_compiler) create_rust_uprev(args.uprev, args.template, args.skip_compile, run_step) remove_rust_uprev(args.remove, run_step) - bootstrap_version = prepare_uprev_from_json( - completed_steps['prepare uprev'])[2] - remove_rust_bootstrap_version(bootstrap_version, run_step) if not args.no_upload: run_step('create rust uprev CL', lambda: create_new_commit(args.uprev)) diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py index 00761391..fc506004 100755 --- a/rust_tools/rust_uprev_test.py +++ b/rust_tools/rust_uprev_test.py @@ -6,94 +6,16 @@ """Tests for rust_uprev.py""" +# pylint: disable=cros-logging-import import os import shutil import subprocess -import tempfile import unittest -from pathlib import Path from unittest import mock from llvm_tools import git import rust_uprev -from rust_uprev import RustVersion - - -def _fail_command(cmd, *_args, **_kwargs): - err = subprocess.CalledProcessError(returncode=1, cmd=cmd) - err.stderr = b'mock failure' - raise err - - -class FetchDistfileTest(unittest.TestCase): - """Tests rust_uprev.fetch_distfile_from_mirror()""" - - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', side_effect=_fail_command) - def test_fetch_difstfile_fail(self, *_args) -> None: - with self.assertRaises(subprocess.CalledProcessError): - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object(rust_uprev, - 'get_command_output_unchecked', - return_value='AccessDeniedException: Access denied.') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_access_denied(self, *_args) -> None: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object( - rust_uprev, - 'get_command_output_unchecked', - return_value='[ { "entity": "allUsers", "role": "READER" } ]') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_ok(self, *_args) -> None: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object( - rust_uprev, - 'get_command_output_unchecked', - return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_wrong(self, *_args) -> None: - with self.assertRaisesRegex(Exception, 'allUsers.*READER'): - with self.assertLogs(level='ERROR') as log: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - self.assertIn( - '[ { "entity": "___fake@google.com", "role": "OWNER" } ]', - '\n'.join(log.output)) - - -class FindEbuildPathTest(unittest.TestCase): - """Tests for rust_uprev.find_ebuild_path()""" - - def test_exact_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.3.4.ebuild') - ebuild.touch() - Path(tmpdir, 'test-1.2.3.ebuild').touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test', - rust_uprev.RustVersion(1, 3, 4)) - self.assertEqual(result, ebuild) - - def test_no_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.2.3.ebuild') - ebuild.touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test') - self.assertEqual(result, ebuild) - - def test_patch_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.3.4-r3.ebuild') - ebuild.touch() - Path(tmpdir, 'test-1.2.3.ebuild').touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test', - rust_uprev.RustVersion(1, 3, 4)) - self.assertEqual(result, ebuild) class RustVersionTest(unittest.TestCase): @@ -127,77 +49,58 @@ class PrepareUprevTest(unittest.TestCase): """Tests for prepare_uprev step in rust_uprev""" def setUp(self): - self.bootstrap_version = rust_uprev.RustVersion(1, 1, 0) self.version_old = rust_uprev.RustVersion(1, 2, 3) self.version_new = rust_uprev.RustVersion(1, 3, 5) - @mock.patch.object(rust_uprev, - 'find_ebuild_for_rust_version', - return_value='/path/to/ebuild') - @mock.patch.object(rust_uprev, 'find_ebuild_path') + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') @mock.patch.object(rust_uprev, 'get_command_output') - def test_success_with_template(self, mock_command, mock_find_ebuild, - _ebuild_for_version): - bootstrap_ebuild_path = Path( - '/path/to/rust-bootstrap/', - f'rust-bootstrap-{self.bootstrap_version}.ebuild') - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, '/path/to/ebuild', self.bootstrap_version) - actual = rust_uprev.prepare_uprev(rust_version=self.version_new, - template=self.version_old) + def test_success_with_template(self, mock_command, mock_find_ebuild): + expected = (self.version_old, '/path/to/ebuild') + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=self.version_old) self.assertEqual(expected, actual) mock_command.assert_not_called() - @mock.patch.object(rust_uprev, - 'find_ebuild_for_rust_version', - return_value='/path/to/ebuild') - @mock.patch.object(rust_uprev, - 'get_rust_bootstrap_version', - return_value=RustVersion(0, 41, 12)) + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') @mock.patch.object(rust_uprev, 'get_command_output') def test_return_none_with_template_larger_than_input(self, mock_command, - *_args): - ret = rust_uprev.prepare_uprev(rust_version=self.version_old, - template=self.version_new) + _mock_find_ebuild): + ret = rust_uprev.prepare_uprev( + rust_version=self.version_old, template=self.version_new) self.assertIsNone(ret) mock_command.assert_not_called() - @mock.patch.object(rust_uprev, 'find_ebuild_path') @mock.patch.object(os.path, 'exists') @mock.patch.object(rust_uprev, 'get_command_output') - def test_success_without_template(self, mock_command, mock_exists, - mock_find_ebuild): + def test_success_without_template(self, mock_command, mock_exists): rust_ebuild_path = f'/path/to/rust/rust-{self.version_old}-r3.ebuild' mock_command.return_value = rust_ebuild_path - bootstrap_ebuild_path = Path( - '/path/to/rust-bootstrap', - f'rust-bootstrap-{self.bootstrap_version}.ebuild') - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, rust_ebuild_path, self.bootstrap_version) - actual = rust_uprev.prepare_uprev(rust_version=self.version_new, - template=None) + expected = (self.version_old, rust_ebuild_path) + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=None) self.assertEqual(expected, actual) mock_command.assert_called_once_with(['equery', 'w', 'rust']) mock_exists.assert_not_called() - @mock.patch.object(rust_uprev, - 'get_rust_bootstrap_version', - return_value=RustVersion(0, 41, 12)) @mock.patch.object(os.path, 'exists') @mock.patch.object(rust_uprev, 'get_command_output') def test_return_none_with_ebuild_larger_than_input(self, mock_command, - mock_exists, *_args): + mock_exists): mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild' - ret = rust_uprev.prepare_uprev(rust_version=self.version_old, - template=None) + ret = rust_uprev.prepare_uprev(rust_version=self.version_old, template=None) self.assertIsNone(ret) mock_exists.assert_not_called() def test_prepare_uprev_from_json(self): ebuild_path = '/path/to/the/ebuild' - json_result = (list(self.version_new), ebuild_path, - list(self.bootstrap_version)) - expected = (self.version_new, ebuild_path, self.bootstrap_version) + json_result = (list(self.version_new), ebuild_path) + expected = (self.version_new, ebuild_path) actual = rust_uprev.prepare_uprev_from_json(json_result) self.assertEqual(expected, actual) @@ -205,30 +108,32 @@ class PrepareUprevTest(unittest.TestCase): class UpdateEbuildTest(unittest.TestCase): """Tests for update_ebuild step in rust_uprev""" ebuild_file_before = """ -BOOTSTRAP_VERSION="1.2.0" + STAGE0_DATE="2019-01-01" + STAGE0_VERSION="any.random.(number)" + STAGE0_VERSION_CARGO="0.0.0" """ ebuild_file_after = """ -BOOTSTRAP_VERSION="1.3.6" + STAGE0_DATE="2020-01-01" + STAGE0_VERSION="1.1.1" + STAGE0_VERSION_CARGO="0.1.0" """ def test_success(self): mock_open = mock.mock_open(read_data=self.ebuild_file_before) - # ebuild_file and new bootstrap version are deliberately different ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' with mock.patch('builtins.open', mock_open): - rust_uprev.update_ebuild(ebuild_file, - rust_uprev.RustVersion.parse('1.3.6')) + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) mock_open.return_value.__enter__().write.assert_called_once_with( self.ebuild_file_after) def test_fail_when_ebuild_misses_a_variable(self): - mock_open = mock.mock_open(read_data='') + ebuild_file = 'STAGE0_DATE="2019-01-01"' + mock_open = mock.mock_open(read_data=ebuild_file) ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' with mock.patch('builtins.open', mock_open): with self.assertRaises(RuntimeError) as context: - rust_uprev.update_ebuild(ebuild_file, - rust_uprev.RustVersion.parse('1.2.0')) - self.assertEqual('BOOTSTRAP_VERSION not found in rust ebuild', + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) + self.assertEqual('STAGE0_VERSION not found in rust ebuild', str(context.exception)) @@ -245,79 +150,44 @@ class UpdateManifestTest(unittest.TestCase): f'RESTRICT="{after}"') def test_add_mirror_in_ebuild(self): - self._run_test_flip_mirror(before='variable1 variable2', - after='variable1 variable2 mirror', - add=True, - expect_write=True) + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2 mirror', + add=True, + expect_write=True) def test_remove_mirror_in_ebuild(self): - self._run_test_flip_mirror(before='variable1 variable2 mirror', - after='variable1 variable2', - add=False, - expect_write=True) + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2', + add=False, + expect_write=True) def test_add_mirror_when_exists(self): - self._run_test_flip_mirror(before='variable1 variable2 mirror', - after='variable1 variable2 mirror', - add=True, - expect_write=False) + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2 mirror', + add=True, + expect_write=False) def test_remove_mirror_when_not_exists(self): - self._run_test_flip_mirror(before='variable1 variable2', - after='variable1 variable2', - add=False, - expect_write=False) + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2', + add=False, + expect_write=False) @mock.patch.object(rust_uprev, 'flip_mirror_in_ebuild') - @mock.patch.object(rust_uprev, 'ebuild_actions') + @mock.patch.object(rust_uprev, 'rust_ebuild_actions') def test_update_manifest(self, mock_run, mock_flip): - ebuild_file = Path('/path/to/rust/rust-1.1.1.ebuild') + ebuild_file = '/path/to/rust/rust-1.1.1.ebuild' rust_uprev.update_manifest(ebuild_file) - mock_run.assert_called_once_with('rust', ['manifest']) + mock_run.assert_called_once_with(['manifest']) mock_flip.assert_has_calls( [mock.call(ebuild_file, add=True), mock.call(ebuild_file, add=False)]) -class UpdateBootstrapEbuildTest(unittest.TestCase): - """Tests for rust_uprev.update_bootstrap_ebuild()""" - - def test_update_bootstrap_ebuild(self): - # The update should do two things: - # 1. Create a copy of rust-bootstrap's ebuild with the new version number. - # 2. Add the old PV to RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE. - with tempfile.TemporaryDirectory() as tmpdir_str, \ - mock.patch.object(rust_uprev, 'find_ebuild_path') as mock_find_ebuild: - tmpdir = Path(tmpdir_str) - bootstrapdir = Path.joinpath(tmpdir, 'rust-bootstrap') - bootstrapdir.mkdir() - old_ebuild = bootstrapdir.joinpath('rust-bootstrap-1.45.2.ebuild') - old_ebuild.write_text(encoding='utf-8', - data=""" -some text -RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( -\t1.43.1 -\t1.44.1 -) -some more text -""") - mock_find_ebuild.return_value = old_ebuild - rust_uprev.update_bootstrap_ebuild(rust_uprev.RustVersion(1, 46, 0)) - new_ebuild = bootstrapdir.joinpath('rust-bootstrap-1.46.0.ebuild') - self.assertTrue(new_ebuild.exists()) - text = new_ebuild.read_text() - self.assertEqual( - text, """ -some text -RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( -\t1.43.1 -\t1.44.1 -\t1.45.2 -) -some more text -""") - - class UpdateRustPackagesTests(unittest.TestCase): """Tests for update_rust_packages step.""" @@ -353,6 +223,103 @@ class UpdateRustPackagesTests(unittest.TestCase): package_after) +class UploadToLocalmirrorTests(unittest.TestCase): + """Tests for upload_to_localmirror""" + + def setUp(self): + self.tempdir = '/tmp/any/dir' + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.rust_url = 'https://static.rust-lang.org/dist' + self.tarfile_name = f'rustc-{self.new_version}-src.tar.gz' + self.rust_src = f'https://static.rust-lang.org/dist/{self.tarfile_name}' + self.gsurl = f'gs://chromeos-localmirror/distfiles/{self.tarfile_name}' + self.rust_file = os.path.join(self.tempdir, self.tarfile_name) + self.sig_file = os.path.join(self.tempdir, 'rustc_sig.asc') + + @mock.patch.object(subprocess, 'call', return_value=1) + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + def test_pass_without_retry(self, mock_run, mock_output, mock_call, + mock_raw_call): + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_output.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + mock_run.assert_not_called() + + @mock.patch.object(subprocess, 'call') + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_pass_with_retry(self, mock_output, mock_run, mock_check, mock_call, + mock_raw_call): + mock_check.side_effect = subprocess.CalledProcessError( + returncode=2, cmd=None, output="gpg: Can't check signature") + mock_output.return_value = 'some_gpg_keys' + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_check.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_output.assert_called_once_with( + ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc']) + mock_run.assert_called_once_with(['gpg', '--import'], + input='some_gpg_keys', + encoding='utf-8', + check=True) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call(['gpg', '--verify', self.sig_file, self.rust_file]), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + + @mock.patch.object(rust_uprev, 'upload_single_tarball') + def test_upload_to_mirror(self, mock_upload): + stage0_info = '2020-01-01', '1.1.1', '0.1.0' + rust_uprev.upload_to_localmirror(self.tempdir, self.new_version, + stage0_info) + mock_upload.assert_has_calls([ + mock.call(self.rust_url, f'rustc-{self.new_version}-src.tar.gz', + self.tempdir), + mock.call(f'{self.rust_url}/{stage0_info[0]}', + f'rust-std-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'rustc-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'cargo-{stage0_info[2]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + ]) + + class RustUprevOtherStagesTests(unittest.TestCase): """Tests for other steps in rust_uprev""" @@ -363,6 +330,25 @@ class RustUprevOtherStagesTests(unittest.TestCase): self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, 'rust-{self.new_version}.ebuild') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_parse_stage0_file(self, mock_get): + stage0_file = """ + unrelated stuff before + date: 2020-01-01 + rustc: 1.1.1 + cargo: 0.1.0 + unrelated stuff after + """ + mock_get.return_value = stage0_file + expected = '2020-01-01', '1.1.1', '0.1.0' + rust_version = rust_uprev.RustVersion(1, 2, 3) + actual = rust_uprev.parse_stage0_file(rust_version) + self.assertEqual(expected, actual) + mock_get.assert_called_once_with([ + 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/' + f'{rust_version}/src/stage0.txt' + ]) + @mock.patch.object(shutil, 'copyfile') @mock.patch.object(os, 'listdir') @mock.patch.object(subprocess, 'check_call') @@ -373,8 +359,7 @@ class RustUprevOtherStagesTests(unittest.TestCase): f'rust-{self.current_version}-patch-1.patch', f'rust-{self.current_version}-patch-2-new.patch' ] - rust_uprev.copy_patches(rust_uprev.RUST_PATH, self.current_version, - self.new_version) + rust_uprev.copy_patches(self.current_version, self.new_version) mock_copy.assert_has_calls([ mock.call( os.path.join(rust_uprev.RUST_PATH, 'files', @@ -389,8 +374,8 @@ class RustUprevOtherStagesTests(unittest.TestCase): f'rust-{self.new_version}-patch-2-new.patch')) ]) mock_call.assert_called_once_with( - ['git', 'add', f'rust-{self.new_version}-*.patch'], - cwd=rust_uprev.RUST_PATH.joinpath('files')) + ['git', 'add', f'files/rust-{self.new_version}-*.patch'], + cwd=rust_uprev.RUST_PATH) @mock.patch.object(shutil, 'copyfile') @mock.patch.object(subprocess, 'check_call') @@ -399,53 +384,23 @@ class RustUprevOtherStagesTests(unittest.TestCase): rust_uprev.create_ebuild(template_ebuild, self.new_version) mock_copy.assert_called_once_with( template_ebuild, - rust_uprev.RUST_PATH.joinpath(f'rust-{self.new_version}.ebuild')) + os.path.join(rust_uprev.RUST_PATH, f'rust-{self.new_version}.ebuild')) mock_call.assert_called_once_with( ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=rust_uprev.RUST_PATH) - @mock.patch.object(rust_uprev, 'find_ebuild_for_package') - @mock.patch.object(subprocess, 'check_call') - def test_remove_rust_bootstrap_version(self, mock_call, *_args): - bootstrap_path = os.path.join(rust_uprev.RUST_PATH, '..', 'rust-bootstrap') - rust_uprev.remove_rust_bootstrap_version(self.old_version, lambda *x: ()) - mock_call.has_calls([ - [ - 'git', 'rm', - os.path.join(bootstrap_path, 'files', - f'rust-bootstrap-{self.old_version}-*.patch') - ], - [ - 'git', 'rm', - os.path.join(bootstrap_path, - f'rust-bootstrap-{self.old_version}.ebuild') - ], - ]) - - @mock.patch.object(rust_uprev, 'find_ebuild_path') - @mock.patch.object(subprocess, 'check_call') - def test_remove_virtual_rust(self, mock_call, mock_find_ebuild): - ebuild_path = Path( - f'/some/dir/virtual/rust/rust-{self.old_version}.ebuild') - mock_find_ebuild.return_value = Path(ebuild_path) - rust_uprev.remove_virtual_rust(self.old_version) - mock_call.assert_called_once_with( - ['git', 'rm', str(ebuild_path.name)], cwd=ebuild_path.parent) - - @mock.patch.object(rust_uprev, 'find_ebuild_path') + @mock.patch.object(os.path, 'exists', return_value=True) @mock.patch.object(shutil, 'copyfile') @mock.patch.object(subprocess, 'check_call') - def test_update_virtual_rust(self, mock_call, mock_copy, mock_find_ebuild): - ebuild_path = Path( - f'/some/dir/virtual/rust/rust-{self.current_version}.ebuild') - mock_find_ebuild.return_value = Path(ebuild_path) + def test_update_virtual_rust(self, mock_call, mock_copy, mock_exists): + virtual_rust_dir = os.path.join(rust_uprev.RUST_PATH, '../../virtual/rust') rust_uprev.update_virtual_rust(self.current_version, self.new_version) mock_call.assert_called_once_with( - ['git', 'add', f'rust-{self.new_version}.ebuild'], - cwd=ebuild_path.parent) + ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=virtual_rust_dir) mock_copy.assert_called_once_with( - ebuild_path.parent.joinpath(f'rust-{self.current_version}.ebuild'), - ebuild_path.parent.joinpath(f'rust-{self.new_version}.ebuild')) + os.path.join(virtual_rust_dir, f'rust-{self.current_version}.ebuild'), + os.path.join(virtual_rust_dir, f'rust-{self.new_version}.ebuild')) + mock_exists.assert_called_once_with(virtual_rust_dir) @mock.patch.object(os, 'listdir') def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls): diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index c347d2c6..b9ad7b82 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -9,6 +9,8 @@ Sends an email if something interesting (probably) happened. """ +# pylint: disable=cros-logging-import + import argparse import itertools import json @@ -19,9 +21,10 @@ import shutil import subprocess import sys import time -from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple -from cros_utils import bugs, email_sender, tiny_render +from cros_utils import email_sender +from cros_utils import tiny_render def gentoo_sha_to_link(sha: str) -> str: @@ -161,7 +164,7 @@ def get_new_gentoo_commits(git_dir: pathlib.Path, 'git', 'log', '--format=%H %s', - f'{most_recent_sha}..origin/master', # nocheck + f'{most_recent_sha}..origin/master', '--', 'dev-lang/rust', ], @@ -222,63 +225,43 @@ def atomically_write_state(state_file: pathlib.Path, state: State) -> None: temp_file.rename(state_file) -def file_bug(title: str, body: str) -> None: - """Files a bug against gbiv@ with the given title/body.""" - bugs.CreateNewBug( - bugs.WellKnownComponents.CrOSToolchainPublic, - title, - body, - # To either take or reassign depending on the rotation. - assignee='gbiv@google.com', - ) - - -def maybe_compose_bug( - old_state: State, - newest_release: RustReleaseVersion, -) -> Optional[Tuple[str, str]]: - """Creates a bug to file about the new release, if doing is desired.""" - if newest_release == old_state.last_seen_release: - return None - - title = f'[Rust] Update to {newest_release}' - body = ('A new release has been detected; we should probably roll to it. ' - "Please see go/crostc-rust-rotation for who's turn it is.") - return title, body - - -def maybe_compose_email( - new_gentoo_commits: List[GitCommit] -) -> Optional[Tuple[str, List[tiny_render.Piece]]]: +def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, + new_gentoo_commits: List[GitCommit] + ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: """Creates an email given our new state, if doing so is appropriate.""" - if not new_gentoo_commits: - return None - subject_pieces = [] body_pieces = [] - # Separate the sections a bit for prettier output. - if body_pieces: - body_pieces += [tiny_render.line_break, tiny_render.line_break] + if newest_release > old_state.last_seen_release: + subject_pieces.append('new rustc release detected') + body_pieces.append(f'Rustc tag for v{newest_release} was found.') - if len(new_gentoo_commits) == 1: - subject_pieces.append('new rust ebuild commit detected') - body_pieces.append('commit:') - else: - subject_pieces.append('new rust ebuild commits detected') - body_pieces.append('commits (newest first):') - - commit_lines = [] - for commit in new_gentoo_commits: - commit_lines.append([ - tiny_render.Link( - gentoo_sha_to_link(commit.sha), - commit.sha[:12], - ), - f': {commit.subject}', - ]) + if new_gentoo_commits: + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] - body_pieces.append(tiny_render.UnorderedList(commit_lines)) + if len(new_gentoo_commits) == 1: + subject_pieces.append('new rust ebuild commit detected') + body_pieces.append('commit:') + else: + subject_pieces.append('new rust ebuild commits detected') + body_pieces.append('commits (newest first):') + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append([ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f': {commit.subject}', + ]) + + body_pieces.append(tiny_render.UnorderedList(commit_lines)) + + if not subject_pieces: + return None subject = '[rust-watch] ' + '; '.join(subject_pieces) return subject, body_pieces @@ -288,14 +271,11 @@ def main(argv: List[str]) -> None: logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--state_dir', - required=True, - help='Directory to store state in.') - parser.add_argument('--skip_side_effects', - action='store_true', - help="Don't send an email or file a bug.") + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--state_dir', required=True, help='Directory to store state in.') + parser.add_argument( + '--skip_email', action='store_true', help="Don't send an email.") parser.add_argument( '--skip_state_update', action='store_true', @@ -338,25 +318,14 @@ def main(argv: List[str]) -> None: prior_state.last_gentoo_sha) logging.info('New commits: %r', new_commits) - maybe_bug = maybe_compose_bug(prior_state, most_recent_release) - maybe_email = maybe_compose_email(new_commits) - - if maybe_bug is None: - logging.info('No bug to file') - else: - title, body = maybe_bug - if opts.skip_side_effects: - logging.info('Skipping sending bug with title %r and contents\n%s', - title, body) - else: - logging.info('Writing new bug') - file_bug(title, body) + maybe_email = maybe_compose_email(prior_state, most_recent_release, + new_commits) if maybe_email is None: - logging.info('No email to send') + logging.info('No updates to send') else: title, body = maybe_email - if opts.skip_side_effects: + if opts.skip_email: logging.info('Skipping sending email with title %r and contents\n%s', title, tiny_render.render_html_pieces(body)) else: @@ -367,8 +336,8 @@ def main(argv: List[str]) -> None: logging.info('Skipping state update, as requested') return - newest_sha = (new_commits[-1].sha - if new_commits else prior_state.last_gentoo_sha) + newest_sha = ( + new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha) atomically_write_state( state_file, State( diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py index 583a9125..97d111fc 100755 --- a/rust_tools/rust_watch_test.py +++ b/rust_tools/rust_watch_test.py @@ -6,6 +6,8 @@ """Tests for rust_watch.py.""" +# pylint: disable=cros-logging-import + import logging import pathlib import subprocess @@ -13,15 +15,16 @@ import time import unittest import unittest.mock -from cros_utils import tiny_render - import rust_watch +from cros_utils import tiny_render class Test(unittest.TestCase): """Tests.""" + def _silence_logs(self): """Silences all log output until the end of the current test.""" + def should_log(_record): return 0 @@ -37,8 +40,8 @@ class Test(unittest.TestCase): def test_release_version_json_round_trips(self): ver = rust_watch.RustReleaseVersion(1, 2, 3) - self.assertEqual(rust_watch.RustReleaseVersion.from_json(ver.to_json()), - ver) + self.assertEqual( + rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver) def test_state_json_round_trips(self): state = rust_watch.State( @@ -95,14 +98,34 @@ class Test(unittest.TestCase): rust_watch.GitCommit('abc123', 'newer commit'), ]) + def test_compose_email_on_a_new_release(self): + new_release = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[], + ) + + self.assertEqual(new_release, ('[rust-watch] new rustc release detected', + ['Rustc tag for v1.1.0 was found.'])) + def test_compose_email_on_a_new_gentoo_commit(self): sha_a = 'a' * 40 - new_commit = rust_watch.maybe_compose_email(new_gentoo_commits=[ - rust_watch.GitCommit( - sha=sha_a, - subject='summary_a', + new_commit = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', ), - ], ) + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], + ) self.assertEqual(new_commit, ('[rust-watch] new rust ebuild commit detected', [ @@ -118,48 +141,60 @@ class Test(unittest.TestCase): ]) ])) - def test_compose_email_composes_nothing_when_no_new_updates_exist(self): - self.assertIsNone(rust_watch.maybe_compose_email(new_gentoo_commits=())) - - def test_compose_bug_creates_bugs_on_new_versions(self): - title, body = rust_watch.maybe_compose_bug( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(1, 0, 1), - ) - self.assertEqual(title, '[Rust] Update to 1.0.1') - self.assertTrue(body.startswith('A new release has been detected;')) - - title, body = rust_watch.maybe_compose_bug( + def test_compose_email_on_multiple_events(self): + sha_a = 'a' * 40 + new_commit_and_release = rust_watch.maybe_compose_email( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), last_gentoo_sha='', ), newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], ) - self.assertEqual(title, '[Rust] Update to 1.1.0') - self.assertTrue(body.startswith('A new release has been detected;')) - title, body = rust_watch.maybe_compose_bug( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(2, 0, 0), - ) - self.assertEqual(title, '[Rust] Update to 2.0.0') - self.assertTrue(body.startswith('A new release has been detected;')) + self.assertEqual( + new_commit_and_release, + ('[rust-watch] new rustc release detected; new rust ebuild commit ' + 'detected', [ + 'Rustc tag for v1.1.0 was found.', + tiny_render.line_break, + tiny_render.line_break, + 'commit:', + tiny_render.UnorderedList([ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ': summary_a', + ], + ]), + ])) - def test_compose_bug_does_nothing_when_no_new_updates_exist(self): + def test_compose_email_composes_nothing_when_no_new_updates_exist(self): self.assertIsNone( - rust_watch.maybe_compose_bug( + rust_watch.maybe_compose_email( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), last_gentoo_sha='', ), newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], + )) + + self.assertIsNone( + rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], )) |