diff options
Diffstat (limited to 'rust_tools')
| -rwxr-xr-x | rust_tools/auto_update_rust_bootstrap.py | 857 | ||||
| -rwxr-xr-x | rust_tools/auto_update_rust_bootstrap_test.py | 501 | ||||
| -rwxr-xr-x | rust_tools/copy_rust_bootstrap.py | 298 | ||||
| -rwxr-xr-x | rust_tools/rust_uprev.py | 741 | ||||
| -rwxr-xr-x | rust_tools/rust_uprev_test.py | 625 | ||||
| -rwxr-xr-x | rust_tools/rust_watch.py | 67 | ||||
| -rwxr-xr-x | rust_tools/rust_watch_test.py | 7 |
7 files changed, 2433 insertions, 663 deletions
diff --git a/rust_tools/auto_update_rust_bootstrap.py b/rust_tools/auto_update_rust_bootstrap.py new file mode 100755 index 00000000..c41fcd03 --- /dev/null +++ b/rust_tools/auto_update_rust_bootstrap.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python3 +# Copyright 2023 The ChromiumOS Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Automatically maintains the rust-bootstrap package. + +This script is responsible for: + - uploading new rust-bootstrap prebuilts + - adding new versions of rust-bootstrap to keep up with dev-lang/rust + - removing versions of rust-bootstrap that are no longer depended on by + dev-lang/rust + +It's capable of (and intended to primarily be used for) uploading CLs to do +these things on its own, so it can easily be regularly run by Chrotomation. +""" + +import argparse +import collections +import dataclasses +import functools +import glob +import logging +import os +from pathlib import Path +import re +import subprocess +import sys +import textwrap +from typing import Dict, Iterable, List, Optional, Tuple, Union + +import copy_rust_bootstrap + + +# The bug to tag in all commit messages. +TRACKING_BUG = "b:315473495" + +# Reviewers for all CLs uploaded. +DEFAULT_CL_REVIEWERS = ( + "gbiv@chromium.org", + "inglorion@chromium.org", +) + + +@dataclasses.dataclass(frozen=True, eq=True, order=True) +class EbuildVersion: + """Represents an ebuild version, simplified for rust-bootstrap versions. + + "Simplified," means that no `_pre`/etc suffixes have to be accounted for. + """ + + major: int + minor: int + patch: int + rev: int + + def prior_minor_version(self) -> "EbuildVersion": + """Returns an EbuildVersion with just the major/minor from this one.""" + return dataclasses.replace(self, minor=self.minor - 1) + + def major_minor_only(self) -> "EbuildVersion": + """Returns an EbuildVersion with just the major/minor from this one.""" + if not self.rev and not self.patch: + return self + return EbuildVersion( + major=self.major, + minor=self.minor, + patch=0, + rev=0, + ) + + def without_rev(self) -> "EbuildVersion": + if not self.rev: + return self + return dataclasses.replace(self, rev=0) + + def __str__(self): + result = f"{self.major}.{self.minor}.{self.patch}" + if self.rev: + result += f"-r{self.rev}" + return result + + +def find_raw_bootstrap_sequence_lines( + ebuild_lines: List[str], +) -> Tuple[int, int]: + """Returns the start/end lines of RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE.""" + for i, line in enumerate(ebuild_lines): + if line.startswith("RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=("): + start = i + break + else: + raise ValueError("No bootstrap sequence start found in text") + + for i, line in enumerate(ebuild_lines[i + 1 :], i + 1): + if line.rstrip() == ")": + return start, i + raise ValueError("No bootstrap sequence end found in text") + + +def read_bootstrap_sequence_from_ebuild( + rust_bootstrap_ebuild: Path, +) -> List[EbuildVersion]: + """Returns a list of EbuildVersions from the given ebuild.""" + ebuild_lines = rust_bootstrap_ebuild.read_text( + encoding="utf-8" + ).splitlines() + start, end = find_raw_bootstrap_sequence_lines(ebuild_lines) + results = [] + for line in ebuild_lines[start + 1 : end]: + # Ignore comments. + line = line.split("#", 1)[0].strip() + if not line: + continue + assert len(line.split()) == 1, f"Unexpected line: {line!r}" + results.append(parse_raw_ebuild_version(line.strip())) + return results + + +def version_listed_in_bootstrap_sequence( + ebuild: Path, rust_bootstrap_version: EbuildVersion +) -> bool: + ebuild_lines = ebuild.read_text(encoding="utf-8").splitlines() + start, end = find_raw_bootstrap_sequence_lines(ebuild_lines) + str_version = str(rust_bootstrap_version.without_rev()) + return any( + line.strip() == str_version for line in ebuild_lines[start + 1 : end] + ) + + +@functools.lru_cache(1) +def fetch_most_recent_sdk_version() -> str: + """Fetches the most recent official SDK version from gs://.""" + latest_file_loc = "gs://chromiumos-sdk/cros-sdk-latest.conf" + sdk_latest_file = subprocess.run( + ["gsutil", "cat", latest_file_loc], + check=True, + encoding="utf-8", + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + ).stdout.strip() + + latest_sdk_re = re.compile(r'^LATEST_SDK="([0-9\.]+)"$') + for line in sdk_latest_file.splitlines(): + m = latest_sdk_re.match(line) + if m: + latest_version = m.group(1) + logging.info("Detected latest SDK version: %r", latest_version) + return latest_version + raise ValueError(f"Could not find LATEST_SDK in {latest_file_loc}") + + +def find_rust_bootstrap_prebuilt(version: EbuildVersion) -> Optional[str]: + """Returns a URL to a prebuilt for `version` of rust-bootstrap.""" + # Searching chroot-* is generally unsafe, because some uploads might + # include SDK artifacts built by CQ+1 runs, so just use the most recent + # verified SDK version. + sdk_version = fetch_most_recent_sdk_version() + + # Search for all rust-bootstrap versions rather than specifically + # `version`, since gsutil will exit(1) if no matches are found. exit(1) is + # desirable if _no rust boostrap artifacts at all exist_, but substantially + # less so if this function seeks to just `return False`. + gs_glob = ( + f"gs://chromeos-prebuilt/board/amd64-host/chroot-{sdk_version}" + "/packages/dev-lang/rust-bootstrap-*tbz2" + ) + + logging.info("Searching %s for rust-bootstrap version %s", gs_glob, version) + results = subprocess.run( + ["gsutil", "ls", gs_glob], + check=True, + encoding="utf-8", + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + ).stdout.strip() + + binpkg_name_re = re.compile( + r"rust-bootstrap-" + re.escape(str(version)) + r"\.tbz2$" + ) + result_lines = results.splitlines() + for line in result_lines: + result = line.strip() + if binpkg_name_re.search(result): + logging.info("Found rust-bootstrap prebuilt: %s", result) + return result + logging.info("Skipped rust-bootstrap prebuilt: %s", result) + + logging.info( + "No rust-bootstrap for %s found (regex: %s); options: %s", + version, + binpkg_name_re, + result_lines, + ) + return None + + +def parse_raw_ebuild_version(raw_ebuild_version: str) -> EbuildVersion: + """Parses an ebuild version without the ${PN} prefix or .ebuild suffix. + + >>> parse_raw_ebuild_version("1.70.0-r2") + EbuildVersion(major=1, minor=70, patch=0, rev=2) + """ + version_re = re.compile(r"(\d+)\.(\d+)\.(\d+)(?:-r(\d+))?") + m = version_re.match(raw_ebuild_version) + if not m: + raise ValueError(f"Version {raw_ebuild_version} can't be recognized.") + + major, minor, patch, rev_str = m.groups() + rev = 0 if not rev_str else int(rev_str) + return EbuildVersion( + major=int(major), minor=int(minor), patch=int(patch), rev=rev + ) + + +def parse_ebuild_version(ebuild_name: str) -> EbuildVersion: + """Parses the version from an ebuild. + + Raises: + ValueError if the `ebuild_name` doesn't contain a parseable version. + Notably, version suffixes like `_pre`, `_beta`, etc are unexpected in + Rust-y ebuilds, so they're not handled here. + + >>> parse_ebuild_version("rust-bootstrap-1.70.0-r2.ebuild") + EbuildVersion(major=1, minor=70, patch=0, rev=2) + """ + version_re = re.compile(r"(\d+)\.(\d+)\.(\d+)(?:-r(\d+))?\.ebuild$") + m = version_re.search(ebuild_name) + if not m: + raise ValueError(f"Ebuild {ebuild_name} has no obvious version") + + major, minor, patch, rev_str = m.groups() + rev = 0 if not rev_str else int(rev_str) + return EbuildVersion( + major=int(major), minor=int(minor), patch=int(patch), rev=rev + ) + + +def collect_ebuilds_by_version( + ebuild_dir: Path, +) -> List[Tuple[EbuildVersion, Path]]: + """Returns the latest ebuilds grouped by version.without_rev. + + Result is always sorted by version, latest versions are last. + """ + ebuilds = ebuild_dir.glob("*.ebuild") + versioned_ebuilds: Dict[EbuildVersion, Tuple[EbuildVersion, Path]] = {} + for ebuild in ebuilds: + version = parse_ebuild_version(ebuild.name) + version_no_rev = version.without_rev() + other = versioned_ebuilds.get(version_no_rev) + this_is_newer = other is None or other[0] < version + if this_is_newer: + versioned_ebuilds[version_no_rev] = (version, ebuild) + + return sorted(versioned_ebuilds.values()) + + +def maybe_copy_prebuilt_to_localmirror( + copy_rust_bootstrap_script: Path, prebuilt_gs_path: str, dry_run: bool +) -> bool: + upload_to = copy_rust_bootstrap.determine_target_path(prebuilt_gs_path) + result = subprocess.run( + ["gsutil", "ls", upload_to], + check=False, + encoding="utf-8", + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + if not result.returncode: + logging.info("Artifact at %s already exists", upload_to) + return False + + cmd: List[Union[Path, str]] = [ + copy_rust_bootstrap_script, + prebuilt_gs_path, + ] + + if dry_run: + cmd.append("--dry-run") + + subprocess.run( + cmd, + check=True, + stdin=subprocess.DEVNULL, + ) + return True + + +def add_version_to_bootstrap_sequence( + ebuild: Path, version: EbuildVersion, dry_run: bool +): + ebuild_lines = ebuild.read_text(encoding="utf-8").splitlines(keepends=True) + _, end = find_raw_bootstrap_sequence_lines(ebuild_lines) + # `end` is the final paren. Since we _need_ prebuilts for all preceding + # versions, always put this a line before the end. + ebuild_lines.insert(end, f"\t{version}\n") + if not dry_run: + ebuild.write_text("".join(ebuild_lines), encoding="utf-8") + + +def is_ebuild_linked_to_in_dir(root_ebuild_path: Path) -> bool: + """Returns whether symlinks point to `root_ebuild_path`. + + The only directory checked is the directory that contains + `root_ebuild_path`. + """ + assert ( + root_ebuild_path.is_absolute() + ), f"{root_ebuild_path} should be an absolute path." + in_dir = root_ebuild_path.parent + for ebuild in in_dir.glob("*.ebuild"): + if ebuild == root_ebuild_path or not ebuild.is_symlink(): + continue + + points_to = Path(os.path.normpath(in_dir / os.readlink(ebuild))) + if points_to == root_ebuild_path: + return True + return False + + +def uprev_ebuild(ebuild: Path, version: EbuildVersion, dry_run: bool) -> Path: + assert ebuild.is_absolute(), f"{ebuild} should be an absolute path." + + new_version = dataclasses.replace(version, rev=version.rev + 1) + new_ebuild = ebuild.parent / f"rust-bootstrap-{new_version}.ebuild" + if dry_run: + logging.info( + "Skipping rename of %s -> %s; dry-run specified", ebuild, new_ebuild + ) + return new_ebuild + + # This condition tries to follow CrOS best practices. Namely: + # - If the ebuild is a symlink, move it. + # - Otherwise, if the ebuild is a normal file, symlink to it as long as + # it has no revision. + # + # Since rust-bootstrap's functionality relies heavily on `${PV}`, it's + # completely expected for cross-${PV} symlinks to exist. + uprev_via_rename = ( + version.rev != 0 or ebuild.is_symlink() + ) and not is_ebuild_linked_to_in_dir(ebuild) + + if uprev_via_rename: + logging.info("Moving %s -> %s", ebuild, new_ebuild) + ebuild.rename(new_ebuild) + else: + logging.info("Symlinking %s to %s", new_ebuild, ebuild) + new_ebuild.symlink_to(ebuild.relative_to(ebuild.parent)) + return new_ebuild + + +def update_ebuild_manifest(rust_bootstrap_ebuild: Path): + subprocess.run( + ["ebuild", rust_bootstrap_ebuild, "manifest"], + check=True, + stdin=subprocess.DEVNULL, + ) + + +def commit_all_changes( + git_dir: Path, rust_bootstrap_dir: Path, commit_message: str +): + subprocess.run( + ["git", "add", rust_bootstrap_dir.relative_to(git_dir)], + cwd=git_dir, + check=True, + stdin=subprocess.DEVNULL, + ) + subprocess.run( + ["git", "commit", "-m", commit_message], + cwd=git_dir, + check=True, + stdin=subprocess.DEVNULL, + ) + + +def scrape_git_push_cl_id_strs(git_push_output: str) -> List[str]: + id_regex = re.compile( + r"^remote:\s+https://chromium-review\S+/\+/(\d+)\s", re.MULTILINE + ) + results = id_regex.findall(git_push_output) + if not results: + raise ValueError( + f"Found 0 matches of {id_regex} in {git_push_output!r}; expected " + "at least 1." + ) + return results + + +def upload_changes(git_dir: Path): + logging.info("Uploading changes") + result = subprocess.run( + ["git", "push", "cros", "HEAD:refs/for/main"], + check=True, + cwd=git_dir, + encoding="utf-8", + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + # Print this in case anyone's looking at the output. + print(result.stdout, end=None) + result.check_returncode() + + cl_ids = scrape_git_push_cl_id_strs(result.stdout) + logging.info( + "Uploaded %s successfully!", [f"crrev.com/c/{x}" for x in cl_ids] + ) + for cl_id in cl_ids: + gerrit_commands = ( + ["gerrit", "label-v", cl_id, "1"], + ["gerrit", "label-cq", cl_id, "1"], + ["gerrit", "label-as", cl_id, "1"], + ["gerrit", "reviewers", cl_id] + list(DEFAULT_CL_REVIEWERS), + ["gerrit", "ready", cl_id], + ) + for command in gerrit_commands: + logging.info("Running gerrit command: %s", command) + subprocess.run( + command, + check=True, + stdin=subprocess.DEVNULL, + ) + + +def maybe_add_newest_prebuilts( + copy_rust_bootstrap_script: Path, + chromiumos_overlay: Path, + rust_bootstrap_dir: Path, + dry_run: bool, +) -> bool: + """Ensures that prebuilts in rust-bootstrap ebuilds are up-to-date. + + If dry_run is True, no changes will be made on disk. Otherwise, changes + will be committed to git locally. + + Returns: + True if changes were made (or would've been made, in the case of + dry_run being True). False otherwise. + """ + # A list of (version, maybe_prebuilt_location). + versions_updated: List[Tuple[EbuildVersion, Optional[str]]] = [] + for version, ebuild in collect_ebuilds_by_version(rust_bootstrap_dir): + logging.info("Inspecting %s...", ebuild) + if version.without_rev() in read_bootstrap_sequence_from_ebuild(ebuild): + logging.info("Prebuilt already exists for %s.", ebuild) + continue + + logging.info("Prebuilt isn't in ebuild; checking remotely.") + prebuilt = find_rust_bootstrap_prebuilt(version) + if not prebuilt: + # `find_rust_bootstrap_prebuilt` handles logging in this case. + continue + + # Houston, we have prebuilt. + uploaded = maybe_copy_prebuilt_to_localmirror( + copy_rust_bootstrap_script, prebuilt, dry_run + ) + add_version_to_bootstrap_sequence(ebuild, version, dry_run) + uprevved_ebuild = uprev_ebuild(ebuild, version, dry_run) + versions_updated.append((version, prebuilt if uploaded else None)) + + if not versions_updated: + logging.info("No updates made; exiting cleanly.") + return False + + if dry_run: + logging.info("Dry-run specified; quit.") + return True + + # Just pick an arbitrary ebuild to run `ebuild ... manifest` on; it always + # updates for all ebuilds in the same package. + update_ebuild_manifest(uprevved_ebuild) + + pretty_artifact_lines = [] + for version, maybe_gs_path in versions_updated: + if maybe_gs_path: + pretty_artifact_lines.append( + f"- rust-bootstrap-{version.without_rev()} => {maybe_gs_path}" + ) + else: + pretty_artifact_lines.append( + f"- rust-bootstrap-{version.without_rev()} was already on " + "localmirror" + ) + + pretty_artifacts = "\n".join(pretty_artifact_lines) + + logging.info("Committing changes.") + commit_all_changes( + chromiumos_overlay, + rust_bootstrap_dir, + commit_message=textwrap.dedent( + f"""\ + rust-bootstrap: use prebuilts + + This CL used the following rust-bootstrap artifacts: + {pretty_artifacts} + + BUG={TRACKING_BUG} + TEST=CQ + """ + ), + ) + return True + + +def rust_dir_from_rust_bootstrap(rust_bootstrap_dir: Path) -> Path: + """Derives dev-lang/rust's dir from dev-lang/rust-bootstrap's dir.""" + return rust_bootstrap_dir.parent / "rust" + + +class MissingRustBootstrapPrebuiltError(Exception): + """Raised when rust-bootstrap can't be landed due to a missing prebuilt.""" + + +def maybe_add_new_rust_bootstrap_version( + chromiumos_overlay: Path, + rust_bootstrap_dir: Path, + dry_run: bool, + commit: bool = True, +) -> bool: + """Ensures that there's a rust-bootstrap-${N} ebuild matching rust-${N}. + + Args: + chromiumos_overlay: Path to chromiumos-overlay. + rust_bootstrap_dir: Path to rust-bootstrap's directory. + dry_run: if True, don't commit to git or write changes to disk. + Otherwise, write changes to disk. + commit: if True, commit changes to git. This value is meaningless if + `dry_run` is True. + + Returns: + True if changes were made (or would've been made, in the case of + dry_run being True). False otherwise. + + Raises: + MissingRustBootstrapPrebuiltError if the creation of a new + rust-bootstrap ebuild wouldn't be buildable, since there's no + rust-bootstrap prebuilt of the prior version for it to sync. + """ + # These are always returned in sorted error, so taking the last is the same + # as `max()`. + ( + newest_bootstrap_version, + newest_bootstrap_ebuild, + ) = collect_ebuilds_by_version(rust_bootstrap_dir)[-1] + + logging.info( + "Detected newest rust-bootstrap version: %s", newest_bootstrap_version + ) + + rust_dir = rust_dir_from_rust_bootstrap(rust_bootstrap_dir) + newest_rust_version, _ = collect_ebuilds_by_version(rust_dir)[-1] + logging.info("Detected newest rust version: %s", newest_rust_version) + + # Generally speaking, we don't care about keeping up with new patch + # releases for rust-bootstrap. It's OK to _initially land_ e.g., + # rust-bootstrap-1.73.1, but upgrades from rust-bootstrap-1.73.0 to + # rust-bootstrap-1.73.1 are rare, and have added complexity, so should be + # done manually. Hence, only check for major/minor version inequality. + if ( + newest_rust_version.major_minor_only() + <= newest_bootstrap_version.major_minor_only() + ): + logging.info("No missing rust-bootstrap versions detected.") + return False + + available_prebuilts = read_bootstrap_sequence_from_ebuild( + newest_bootstrap_ebuild + ) + need_prebuilt = newest_rust_version.major_minor_only().prior_minor_version() + + if all(x.major_minor_only() != need_prebuilt for x in available_prebuilts): + raise MissingRustBootstrapPrebuiltError( + f"want version {need_prebuilt}; " + f"available versions: {available_prebuilts}" + ) + + # Ensure the rust-bootstrap ebuild we're landing is a regular file. This + # makes cleanup of the old files trivial, since they're dead symlinks. + prior_ebuild_resolved = newest_bootstrap_ebuild.resolve() + new_ebuild = ( + rust_bootstrap_dir + / f"rust-bootstrap-{newest_rust_version.without_rev()}.ebuild" + ) + if dry_run: + logging.info("Would move %s to %s.", prior_ebuild_resolved, new_ebuild) + return True + + logging.info( + "Moving %s to %s, and creating symlink at the old location", + prior_ebuild_resolved, + new_ebuild, + ) + prior_ebuild_resolved.rename(new_ebuild) + prior_ebuild_resolved.symlink_to(new_ebuild.relative_to(rust_bootstrap_dir)) + + update_ebuild_manifest(new_ebuild) + if commit: + newest_no_rev = newest_rust_version.without_rev() + commit_all_changes( + chromiumos_overlay, + rust_bootstrap_dir, + commit_message=textwrap.dedent( + f"""\ + rust-bootstrap: add version {newest_no_rev} + + Rust is now at {newest_no_rev}; add a rust-bootstrap version so + prebuilts can be generated early. + + BUG={TRACKING_BUG} + TEST=CQ + """ + ), + ) + return True + + +class OldEbuildIsLinkedToError(Exception): + """Raised when a would-be-removed ebuild has symlinks to it.""" + + +def find_external_links_to_files_in_dir( + in_dir: Path, files: Iterable[Path] +) -> Dict[Path, List[Path]]: + """Returns all symlinks to `files` in `in_dir`, excluding from `files`. + + Essentially, if this returns an empty dict, nothing in `in_dir` symlinks to + any of `files`, _except potentially_ things in `files`. + """ + files_set = {x.absolute() for x in files} + linked_to = collections.defaultdict(list) + for f in in_dir.iterdir(): + if f not in files_set and f.is_symlink(): + target = f.parent / os.readlink(f) + if target in files_set: + linked_to[target].append(f) + return linked_to + + +def maybe_delete_old_rust_bootstrap_ebuilds( + chromiumos_overlay: Path, + rust_bootstrap_dir: Path, + dry_run: bool, + commit: bool = True, +) -> bool: + """Deletes versions of rust-bootstrap ebuilds that seem unneeded. + + "Unneeded", in this case, is specifically only referencing whether + dev-lang/rust (or similar) obviously relies on the ebuild, or whether it's + likely that a future version of dev-lang/rust will rely on it. + + Args: + chromiumos_overlay: Path to chromiumos-overlay. + rust_bootstrap_dir: Path to rust-bootstrap's directory. + dry_run: if True, don't commit to git or write changes to disk. + Otherwise, write changes to disk. + commit: if True, commit changes to git. This value is meaningless if + `dry_run` is True. + + Returns: + True if changes were made (or would've been made, in the case of + dry_run being True). False otherwise. + + Raises: + OldEbuildIsLinkedToError if the deletion of an ebuild was blocked by + other ebuilds linking to it. It's still 'needed' in this case, but with + some human intervention, it can be removed. + """ + rust_bootstrap_versions = collect_ebuilds_by_version(rust_bootstrap_dir) + logging.info( + "Current rust-bootstrap versions: %s", + [x for x, _ in rust_bootstrap_versions], + ) + rust_versions = collect_ebuilds_by_version( + rust_dir_from_rust_bootstrap(rust_bootstrap_dir) + ) + # rust_versions is sorted, so taking the last is the same as max(). + newest_rust_version = rust_versions[-1][0].major_minor_only() + need_rust_bootstrap_versions = { + rust_ver.major_minor_only().prior_minor_version() + for rust_ver, _ in rust_versions + } + logging.info( + "Needed rust-bootstrap versions (major/minor only): %s", + sorted(need_rust_bootstrap_versions), + ) + + discardable_bootstrap_versions = [ + (version, ebuild) + for version, ebuild in rust_bootstrap_versions + # Don't remove newer rust-bootstrap versions, since they're likely to + # be needed in the future (& may have been generated by this very + # script). + if version.major_minor_only() not in need_rust_bootstrap_versions + and version.major_minor_only() < newest_rust_version + ] + + if not discardable_bootstrap_versions: + logging.info("No versions of rust-bootstrap are unneeded.") + return False + + discardable_ebuilds = [] + for version, ebuild in discardable_bootstrap_versions: + # We may have other files with the same version at different revisions. + # Include those, as well. + escaped_ver = glob.escape(str(version.without_rev())) + discardable = list( + ebuild.parent.glob(f"rust-bootstrap-{escaped_ver}*.ebuild") + ) + assert ebuild in discardable, discardable + discardable_ebuilds += discardable + + # We can end up in a case where rust-bootstrap versions are unneeded, but + # the ebuild is still required. For example, consider a case where + # rust-bootstrap-1.73.0.ebuild is considered 'old', but + # rust-bootstrap-1.74.0.ebuild is required. If rust-bootstrap-1.74.0.ebuild + # is a symlink to rust-bootstrap-1.73.0.ebuild, the 'old' ebuild can't be + # deleted until rust-bootstrap-1.74.0.ebuild is fixed up. + # + # These cases are expected to be rare (this script should never push + # changes that gets us into this case, but human edits can), but uploading + # obviously-broken changes isn't a great UX. Opt to detect these and + # `raise` on them, since repairing can get complicated in instances where + # symlinks link to symlinks, etc. + has_links = find_external_links_to_files_in_dir( + rust_bootstrap_dir, discardable_ebuilds + ) + if has_links: + raise OldEbuildIsLinkedToError(str(has_links)) + + logging.info("Plan to remove ebuilds: %s", discardable_ebuilds) + if dry_run: + logging.info("Dry-run specified; removal skipped.") + return True + + for ebuild in discardable_ebuilds: + ebuild.unlink() + + remaining_ebuild = next( + ebuild + for _, ebuild in rust_bootstrap_versions + if ebuild not in discardable_ebuilds + ) + update_ebuild_manifest(remaining_ebuild) + if commit: + many = len(discardable_ebuilds) > 1 + message_lines = [ + "Rust has moved on in ChromeOS, so", + "these ebuilds are" if many else "this ebuild is", + "no longer needed.", + ] + message = textwrap.fill("\n".join(message_lines)) + commit_all_changes( + chromiumos_overlay, + rust_bootstrap_dir, + commit_message=textwrap.dedent( + f"""\ + rust-bootstrap: remove unused ebuild{"s" if many else ""} + + {message} + + BUG={TRACKING_BUG} + TEST=CQ + """ + ), + ) + return True + + +def main(argv: List[str]): + logging.basicConfig( + format=">> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: " + "%(message)s", + level=logging.INFO, + ) + + my_dir = Path(__file__).parent.resolve() + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--chromiumos-overlay", + type=Path, + default=my_dir.parent.parent / "chromiumos-overlay", + ) + parser.add_argument( + "action", + choices=("dry-run", "commit", "upload"), + help=""" + What to do. `dry-run` makes no changes, `commit` commits changes + locally, and `upload` commits changes and uploads the result to Gerrit, + and sets a few labels for convenience (reviewers, CQ+1, etc). + """, + ) + opts = parser.parse_args(argv) + + if opts.action == "dry-run": + dry_run = True + upload = False + elif opts.action == "commit": + dry_run = False + upload = False + else: + assert opts.action == "upload" + dry_run = False + upload = True + + rust_bootstrap_dir = opts.chromiumos_overlay / "dev-lang/rust-bootstrap" + copy_rust_bootstrap_script = my_dir / "copy_rust_bootstrap.py" + + had_recoverable_error = False + # Ensure prebuilts are up to date first, since it allows + # `ensure_newest_rust_bootstrap_ebuild_exists` to succeed in edge cases. + made_changes = maybe_add_newest_prebuilts( + copy_rust_bootstrap_script, + opts.chromiumos_overlay, + rust_bootstrap_dir, + dry_run, + ) + + try: + made_changes |= maybe_add_new_rust_bootstrap_version( + opts.chromiumos_overlay, rust_bootstrap_dir, dry_run + ) + except MissingRustBootstrapPrebuiltError: + logging.exception( + "Ensuring newest rust-bootstrap ebuild exists failed." + ) + had_recoverable_error = True + + try: + made_changes |= maybe_delete_old_rust_bootstrap_ebuilds( + opts.chromiumos_overlay, rust_bootstrap_dir, dry_run + ) + except OldEbuildIsLinkedToError: + logging.exception("An old ebuild is linked to; can't remove it") + had_recoverable_error = True + + if upload: + if made_changes: + upload_changes(opts.chromiumos_overlay) + logging.info("Changes uploaded successfully.") + else: + logging.info("No changes were made; uploading skipped.") + + if had_recoverable_error: + sys.exit("Exiting uncleanly due to above error(s).") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/rust_tools/auto_update_rust_bootstrap_test.py b/rust_tools/auto_update_rust_bootstrap_test.py new file mode 100755 index 00000000..0578539d --- /dev/null +++ b/rust_tools/auto_update_rust_bootstrap_test.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +# Copyright 2023 The ChromiumOS Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for auto_update_rust_bootstrap.""" + +import os +from pathlib import Path +import shutil +import tempfile +import textwrap +import unittest +from unittest import mock + +import auto_update_rust_bootstrap + + +_GIT_PUSH_OUTPUT = r""" +remote: Waiting for private key checker: 2/2 objects left +remote: +remote: Processing changes: new: 1 (\) +remote: Processing changes: new: 1 (|) +remote: Processing changes: new: 1 (/) +remote: Processing changes: refs: 1, new: 1 (/) +remote: Processing changes: refs: 1, new: 1 (/) +remote: Processing changes: refs: 1, new: 1 (/) +remote: Processing changes: refs: 1, new: 1, done +remote: +remote: SUCCESS +remote: +remote: https://chromium-review.googlesource.com/c/chromiumos/overlays/chromiumos-overlay/+/5018826 rust-bootstrap: use prebuilts [WIP] [NEW] +remote: +To https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay + * [new reference] HEAD -> refs/for/main +""" + +_GIT_PUSH_MULTI_CL_OUTPUT = r""" +remote: Waiting for private key checker: 2/2 objects left +remote: +remote: Processing changes: new: 1 (\) +remote: Processing changes: new: 1 (|) +remote: Processing changes: new: 1 (/) +remote: Processing changes: refs: 1, new: 1 (/) +remote: Processing changes: refs: 1, new: 1 (/) +remote: Processing changes: refs: 1, new: 1 (/) +remote: Processing changes: refs: 1, new: 1, done +remote: +remote: SUCCESS +remote: +remote: https://chromium-review.googlesource.com/c/chromiumos/overlays/chromiumos-overlay/+/5339923 rust-bootstrap: add version 1.75.0 [NEW] +remote: https://chromium-review.googlesource.com/c/chromiumos/overlays/chromiumos-overlay/+/5339924 rust-bootstrap: remove unused ebuilds [NEW] +remote: +To https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay + * [new reference] HEAD -> refs/for/main +""" + + +class Test(unittest.TestCase): + """Tests for auto_update_rust_bootstrap.""" + + def make_tempdir(self) -> Path: + tempdir = Path( + tempfile.mkdtemp(prefix="auto_update_rust_bootstrap_test_") + ) + self.addCleanup(shutil.rmtree, tempdir) + return tempdir + + def test_git_cl_id_scraping(self): + self.assertEqual( + auto_update_rust_bootstrap.scrape_git_push_cl_id_strs( + _GIT_PUSH_OUTPUT + ), + ["5018826"], + ) + + self.assertEqual( + auto_update_rust_bootstrap.scrape_git_push_cl_id_strs( + _GIT_PUSH_MULTI_CL_OUTPUT + ), + ["5339923", "5339924"], + ) + + def test_ebuild_linking_logic_handles_direct_relative_symlinks(self): + tempdir = self.make_tempdir() + target = tempdir / "target.ebuild" + target.touch() + (tempdir / "symlink.ebuild").symlink_to(target.name) + self.assertTrue( + auto_update_rust_bootstrap.is_ebuild_linked_to_in_dir(target) + ) + + def test_ebuild_linking_logic_handles_direct_absolute_symlinks(self): + tempdir = self.make_tempdir() + target = tempdir / "target.ebuild" + target.touch() + (tempdir / "symlink.ebuild").symlink_to(target) + self.assertTrue( + auto_update_rust_bootstrap.is_ebuild_linked_to_in_dir(target) + ) + + def test_ebuild_linking_logic_handles_indirect_relative_symlinks(self): + tempdir = self.make_tempdir() + target = tempdir / "target.ebuild" + target.touch() + (tempdir / "symlink.ebuild").symlink_to( + Path("..") / tempdir.name / target.name + ) + self.assertTrue( + auto_update_rust_bootstrap.is_ebuild_linked_to_in_dir(target) + ) + + def test_ebuild_linking_logic_handles_broken_symlinks(self): + tempdir = self.make_tempdir() + target = tempdir / "target.ebuild" + target.touch() + (tempdir / "symlink.ebuild").symlink_to("doesnt_exist.ebuild") + self.assertFalse( + auto_update_rust_bootstrap.is_ebuild_linked_to_in_dir(target) + ) + + def test_ebuild_linking_logic_only_steps_through_one_symlink(self): + tempdir = self.make_tempdir() + target = tempdir / "target.ebuild" + target.symlink_to("doesnt_exist.ebuild") + (tempdir / "symlink.ebuild").symlink_to(target.name) + self.assertTrue( + auto_update_rust_bootstrap.is_ebuild_linked_to_in_dir(target) + ) + + def test_raw_bootstrap_seq_finding_functions(self): + ebuild_contents = textwrap.dedent( + """\ + # Some copyright + FOO=bar + # Comment about RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( + RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( # another comment + 1.2.3 # (with a comment with parens) + 4.5.6 + ) + """ + ) + + ebuild_lines = ebuild_contents.splitlines() + ( + start, + end, + ) = auto_update_rust_bootstrap.find_raw_bootstrap_sequence_lines( + ebuild_lines + ) + self.assertEqual(start, len(ebuild_lines) - 4) + self.assertEqual(end, len(ebuild_lines) - 1) + + def test_collect_ebuilds_by_version_ignores_older_versions(self): + tempdir = self.make_tempdir() + ebuild_170 = tempdir / "rust-bootstrap-1.70.0.ebuild" + ebuild_170.touch() + ebuild_170_r1 = tempdir / "rust-bootstrap-1.70.0-r1.ebuild" + ebuild_170_r1.touch() + ebuild_171_r2 = tempdir / "rust-bootstrap-1.71.1-r2.ebuild" + ebuild_171_r2.touch() + + self.assertEqual( + auto_update_rust_bootstrap.collect_ebuilds_by_version(tempdir), + [ + ( + auto_update_rust_bootstrap.EbuildVersion( + major=1, minor=70, patch=0, rev=1 + ), + ebuild_170_r1, + ), + ( + auto_update_rust_bootstrap.EbuildVersion( + major=1, minor=71, patch=1, rev=2 + ), + ebuild_171_r2, + ), + ], + ) + + def test_has_prebuilt_works(self): + tempdir = self.make_tempdir() + ebuild = tempdir / "rust-bootstrap-1.70.0.ebuild" + ebuild.write_text( + textwrap.dedent( + """\ + # Some copyright + FOO=bar + # Comment about RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( + RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( # another comment + 1.67.0 + 1.68.1 + 1.69.0 + ) + """ + ), + encoding="utf-8", + ) + + self.assertTrue( + auto_update_rust_bootstrap.version_listed_in_bootstrap_sequence( + ebuild, + auto_update_rust_bootstrap.EbuildVersion( + major=1, + minor=69, + patch=0, + rev=0, + ), + ) + ) + + self.assertFalse( + auto_update_rust_bootstrap.version_listed_in_bootstrap_sequence( + ebuild, + auto_update_rust_bootstrap.EbuildVersion( + major=1, + minor=70, + patch=0, + rev=0, + ), + ) + ) + + def test_ebuild_updating_works(self): + tempdir = self.make_tempdir() + ebuild = tempdir / "rust-bootstrap-1.70.0.ebuild" + ebuild.write_text( + textwrap.dedent( + """\ + # Some copyright + FOO=bar + RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( + \t1.67.0 + \t1.68.1 + \t1.69.0 + ) + """ + ), + encoding="utf-8", + ) + + auto_update_rust_bootstrap.add_version_to_bootstrap_sequence( + ebuild, + auto_update_rust_bootstrap.EbuildVersion( + major=1, + minor=70, + patch=1, + rev=2, + ), + dry_run=False, + ) + + self.assertEqual( + ebuild.read_text(encoding="utf-8"), + textwrap.dedent( + """\ + # Some copyright + FOO=bar + RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( + \t1.67.0 + \t1.68.1 + \t1.69.0 + \t1.70.1-r2 + ) + """ + ), + ) + + def test_ebuild_version_parsing_works(self): + self.assertEqual( + auto_update_rust_bootstrap.parse_ebuild_version( + "rust-bootstrap-1.70.0-r2.ebuild" + ), + auto_update_rust_bootstrap.EbuildVersion( + major=1, minor=70, patch=0, rev=2 + ), + ) + + self.assertEqual( + auto_update_rust_bootstrap.parse_ebuild_version( + "rust-bootstrap-2.80.3.ebuild" + ), + auto_update_rust_bootstrap.EbuildVersion( + major=2, minor=80, patch=3, rev=0 + ), + ) + + with self.assertRaises(ValueError): + auto_update_rust_bootstrap.parse_ebuild_version( + "rust-bootstrap-2.80.3_pre1234.ebuild" + ) + + def test_raw_ebuild_version_parsing_works(self): + self.assertEqual( + auto_update_rust_bootstrap.parse_raw_ebuild_version("1.70.0-r2"), + auto_update_rust_bootstrap.EbuildVersion( + major=1, minor=70, patch=0, rev=2 + ), + ) + + with self.assertRaises(ValueError): + auto_update_rust_bootstrap.parse_ebuild_version("2.80.3_pre1234") + + def test_ensure_newest_version_does_nothing_if_no_new_rust_version(self): + tempdir = self.make_tempdir() + rust = tempdir / "rust" + rust.mkdir() + (rust / "rust-1.70.0-r1.ebuild").touch() + rust_bootstrap = tempdir / "rust-bootstrap" + rust_bootstrap.mkdir() + (rust_bootstrap / "rust-bootstrap-1.70.0.ebuild").touch() + + self.assertFalse( + auto_update_rust_bootstrap.maybe_add_new_rust_bootstrap_version( + tempdir, rust_bootstrap, dry_run=True + ) + ) + + @mock.patch.object(auto_update_rust_bootstrap, "update_ebuild_manifest") + def test_ensure_newest_version_upgrades_rust_bootstrap_properly( + self, update_ebuild_manifest + ): + tempdir = self.make_tempdir() + rust = tempdir / "rust" + rust.mkdir() + (rust / "rust-1.71.0-r1.ebuild").touch() + rust_bootstrap = tempdir / "rust-bootstrap" + rust_bootstrap.mkdir() + rust_bootstrap_1_70 = rust_bootstrap / "rust-bootstrap-1.70.0-r2.ebuild" + + rust_bootstrap_contents = textwrap.dedent( + """\ + # Some copyright + FOO=bar + RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( + \t1.67.0 + \t1.68.1 + \t1.69.0 + \t1.70.0-r1 + ) + """ + ) + rust_bootstrap_1_70.write_text( + rust_bootstrap_contents, encoding="utf-8" + ) + + self.assertTrue( + auto_update_rust_bootstrap.maybe_add_new_rust_bootstrap_version( + tempdir, rust_bootstrap, dry_run=False, commit=False + ) + ) + update_ebuild_manifest.assert_called_once() + rust_bootstrap_1_71 = rust_bootstrap / "rust-bootstrap-1.71.0.ebuild" + + self.assertTrue(rust_bootstrap_1_70.is_symlink()) + self.assertEqual( + os.readlink(rust_bootstrap_1_70), + rust_bootstrap_1_71.name, + ) + self.assertFalse(rust_bootstrap_1_71.is_symlink()) + self.assertEqual( + rust_bootstrap_1_71.read_text(encoding="utf-8"), + rust_bootstrap_contents, + ) + + def test_ensure_newest_version_breaks_if_prebuilt_is_not_available(self): + tempdir = self.make_tempdir() + rust = tempdir / "rust" + rust.mkdir() + (rust / "rust-1.71.0-r1.ebuild").touch() + rust_bootstrap = tempdir / "rust-bootstrap" + rust_bootstrap.mkdir() + rust_bootstrap_1_70 = rust_bootstrap / "rust-bootstrap-1.70.0-r2.ebuild" + + rust_bootstrap_contents = textwrap.dedent( + """\ + # Some copyright + FOO=bar + RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( + \t1.67.0 + \t1.68.1 + \t1.69.0 + # Note: Missing 1.70.0 for rust-bootstrap-1.71.1 + ) + """ + ) + rust_bootstrap_1_70.write_text( + rust_bootstrap_contents, encoding="utf-8" + ) + + with self.assertRaises( + auto_update_rust_bootstrap.MissingRustBootstrapPrebuiltError + ): + auto_update_rust_bootstrap.maybe_add_new_rust_bootstrap_version( + tempdir, rust_bootstrap, dry_run=True + ) + + def test_version_deletion_does_nothing_if_all_versions_are_needed(self): + tempdir = self.make_tempdir() + rust = tempdir / "rust" + rust.mkdir() + (rust / "rust-1.71.0-r1.ebuild").touch() + rust_bootstrap = tempdir / "rust-bootstrap" + rust_bootstrap.mkdir() + (rust_bootstrap / "rust-bootstrap-1.70.0-r2.ebuild").touch() + + self.assertFalse( + auto_update_rust_bootstrap.maybe_delete_old_rust_bootstrap_ebuilds( + tempdir, rust_bootstrap, dry_run=True + ) + ) + + def test_version_deletion_ignores_newer_than_needed_versions(self): + tempdir = self.make_tempdir() + rust = tempdir / "rust" + rust.mkdir() + (rust / "rust-1.71.0-r1.ebuild").touch() + rust_bootstrap = tempdir / "rust-bootstrap" + rust_bootstrap.mkdir() + (rust_bootstrap / "rust-bootstrap-1.70.0-r2.ebuild").touch() + (rust_bootstrap / "rust-bootstrap-1.71.0-r1.ebuild").touch() + (rust_bootstrap / "rust-bootstrap-1.72.0.ebuild").touch() + + self.assertFalse( + auto_update_rust_bootstrap.maybe_delete_old_rust_bootstrap_ebuilds( + tempdir, rust_bootstrap, dry_run=True + ) + ) + + @mock.patch.object(auto_update_rust_bootstrap, "update_ebuild_manifest") + def test_version_deletion_deletes_old_files(self, update_ebuild_manifest): + tempdir = self.make_tempdir() + rust = tempdir / "rust" + rust.mkdir() + (rust / "rust-1.71.0-r1.ebuild").touch() + rust_bootstrap = tempdir / "rust-bootstrap" + rust_bootstrap.mkdir() + needed_rust_bootstrap = ( + rust_bootstrap / "rust-bootstrap-1.70.0-r2.ebuild" + ) + needed_rust_bootstrap.touch() + + # There are quite a few of these, so corner-cases are tested. + + # Symlink to outside of the group of files to delete. + bootstrap_1_68_symlink = rust_bootstrap / "rust-bootstrap-1.68.0.ebuild" + bootstrap_1_68_symlink.symlink_to(needed_rust_bootstrap.name) + # Ensure that absolute symlinks are caught. + bootstrap_1_68_symlink_abs = ( + rust_bootstrap / "rust-bootstrap-1.68.0-r1.ebuild" + ) + bootstrap_1_68_symlink_abs.symlink_to(needed_rust_bootstrap) + # Regular files should be no issue. + bootstrap_1_69_regular = rust_bootstrap / "rust-bootstrap-1.69.0.ebuild" + bootstrap_1_69_regular.touch() + # Symlinks linking back into the set of files to delete should also be + # no issue. + bootstrap_1_69_symlink = ( + rust_bootstrap / "rust-bootstrap-1.69.0-r2.ebuild" + ) + bootstrap_1_69_symlink.symlink_to(bootstrap_1_69_regular.name) + + self.assertTrue( + auto_update_rust_bootstrap.maybe_delete_old_rust_bootstrap_ebuilds( + tempdir, + rust_bootstrap, + dry_run=False, + commit=False, + ) + ) + update_ebuild_manifest.assert_called_once() + + self.assertFalse(bootstrap_1_68_symlink.exists()) + self.assertFalse(bootstrap_1_68_symlink_abs.exists()) + self.assertFalse(bootstrap_1_69_regular.exists()) + self.assertFalse(bootstrap_1_69_symlink.exists()) + self.assertTrue(needed_rust_bootstrap.exists()) + + def test_version_deletion_raises_when_old_file_has_dep(self): + tempdir = self.make_tempdir() + rust = tempdir / "rust" + rust.mkdir() + (rust / "rust-1.71.0-r1.ebuild").touch() + rust_bootstrap = tempdir / "rust-bootstrap" + rust_bootstrap.mkdir() + old_rust_bootstrap = rust_bootstrap / "rust-bootstrap-1.69.0-r1.ebuild" + old_rust_bootstrap.touch() + (rust_bootstrap / "rust-bootstrap-1.70.0-r2.ebuild").symlink_to( + old_rust_bootstrap.name + ) + + with self.assertRaises( + auto_update_rust_bootstrap.OldEbuildIsLinkedToError + ): + auto_update_rust_bootstrap.maybe_delete_old_rust_bootstrap_ebuilds( + tempdir, rust_bootstrap, dry_run=True + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/rust_tools/copy_rust_bootstrap.py b/rust_tools/copy_rust_bootstrap.py index 5da8007f..fd6770f7 100755 --- a/rust_tools/copy_rust_bootstrap.py +++ b/rust_tools/copy_rust_bootstrap.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright 2022 The ChromiumOS Authors. +# Copyright 2022 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Copies rust-bootstrap artifacts from an SDK build to localmirror. We use localmirror to host these artifacts, but they've changed a bit over -time, so simply `gsutil.py cp $FROM $TO` doesn't work. This script allows the +time, so simply `gsutil cp $FROM $TO` doesn't work. This script allows the convenience of the old `cp` command. """ @@ -21,172 +21,184 @@ import tempfile from typing import List -_LOCALMIRROR_ROOT = 'gs://chromeos-localmirror/distfiles/' +_LOCALMIRROR_ROOT = "gs://chromeos-localmirror/distfiles/" def _is_in_chroot() -> bool: - return Path('/etc/cros_chroot_version').exists() + return Path("/etc/cros_chroot_version").exists() -def _ensure_pbzip2_is_installed(): - if shutil.which('pbzip2'): - return +def _ensure_lbzip2_is_installed(): + if shutil.which("lbzip2"): + return - logging.info('Auto-installing pbzip2...') - subprocess.run(['sudo', 'emerge', '-G', 'pbzip2'], check=True) + logging.info("Auto-installing lbzip2...") + subprocess.run(["sudo", "emerge", "-g", "lbzip2"], check=True) -def _determine_target_path(sdk_path: str) -> str: - """Determine where `sdk_path` should sit in localmirror.""" - gs_prefix = 'gs://' - if not sdk_path.startswith(gs_prefix): - raise ValueError(f'Invalid GS path: {sdk_path!r}') +def determine_target_path(sdk_path: str) -> str: + """Determine where `sdk_path` should sit in localmirror.""" + gs_prefix = "gs://" + if not sdk_path.startswith(gs_prefix): + raise ValueError(f"Invalid GS path: {sdk_path!r}") - file_name = Path(sdk_path[len(gs_prefix):]).name - return _LOCALMIRROR_ROOT + file_name + file_name = Path(sdk_path[len(gs_prefix) :]).name + return _LOCALMIRROR_ROOT + file_name def _download(remote_path: str, local_file: Path): - """Downloads the given gs:// path to the given local file.""" - logging.info('Downloading %s -> %s', remote_path, local_file) - subprocess.run( - ['gsutil.py', 'cp', remote_path, - str(local_file)], - check=True, - ) - - -def _debinpkgify(binpkg_file: Path) -> Path: - """Converts a binpkg into the files it installs. - - Note that this function makes temporary files in the same directory as - `binpkg_file`. It makes no attempt to clean them up. - """ - logging.info('Converting %s from a binpkg...', binpkg_file) - - # The SDK builder produces binary packages: - # https://wiki.gentoo.org/wiki/Binary_package_guide - # - # Which means that `binpkg_file` is in the XPAK format. We want to split - # that out, and recompress it from zstd (which is the compression format - # that CrOS uses) to bzip2 (which is what we've historically used, and - # which is what our ebuild expects). - tmpdir = binpkg_file.parent - - def _mkstemp(suffix=None) -> str: - fd, file_path = tempfile.mkstemp(dir=tmpdir, suffix=suffix) - os.close(fd) - return Path(file_path) - - # First, split the actual artifacts that land in the chroot out to - # `temp_file`. - artifacts_file = _mkstemp() - logging.info('Extracting artifacts from %s into %s...', binpkg_file, - artifacts_file) - with artifacts_file.open('wb') as f: + """Downloads the given gs:// path to the given local file.""" + logging.info("Downloading %s -> %s", remote_path, local_file) subprocess.run( - [ - 'qtbz2', - '-s', - '-t', - '-O', - str(binpkg_file), - ], + ["gsutil", "cp", remote_path, str(local_file)], check=True, - stdout=f, + stdin=subprocess.DEVNULL, ) - decompressed_artifacts_file = _mkstemp() - decompressed_artifacts_file.unlink() - logging.info('Decompressing artifacts from %s to %s...', artifacts_file, - decompressed_artifacts_file) - subprocess.run( - [ - 'zstd', - '-d', - str(artifacts_file), - '-o', - str(decompressed_artifacts_file), - ], - check=True, - ) - - # Finally, recompress it as a tbz2. - tbz2_file = _mkstemp('.tbz2') - logging.info( - 'Recompressing artifacts from %s to %s (this may take a while)...', - decompressed_artifacts_file, tbz2_file) - with tbz2_file.open('wb') as f: + +def _debinpkgify(binpkg_file: Path) -> Path: + """Converts a binpkg into the files it installs. + + Note that this function makes temporary files in the same directory as + `binpkg_file`. It makes no attempt to clean them up. + """ + logging.info("Converting %s from a binpkg...", binpkg_file) + + # The SDK builder produces binary packages: + # https://wiki.gentoo.org/wiki/Binary_package_guide + # + # Which means that `binpkg_file` is in the XPAK format. We want to split + # that out, and recompress it from zstd (which is the compression format + # that CrOS uses) to bzip2 (which is what we've historically used, and + # which is what our ebuild expects). + tmpdir = binpkg_file.parent + + def _mkstemp(suffix=None) -> Path: + fd, file_path = tempfile.mkstemp(dir=tmpdir, suffix=suffix) + os.close(fd) + return Path(file_path) + + # First, split the actual artifacts that land in the chroot out to + # `temp_file`. + artifacts_file = _mkstemp() + logging.info( + "Extracting artifacts from %s into %s...", binpkg_file, artifacts_file + ) + with artifacts_file.open("wb") as f: + subprocess.run( + [ + "qtbz2", + "-s", + "-t", + "-O", + str(binpkg_file), + ], + check=True, + stdout=f, + ) + + decompressed_artifacts_file = _mkstemp() + decompressed_artifacts_file.unlink() + logging.info( + "Decompressing artifacts from %s to %s...", + artifacts_file, + decompressed_artifacts_file, + ) subprocess.run( [ - 'pbzip2', - '-9', - '-c', + "zstd", + "-d", + str(artifacts_file), + "-o", str(decompressed_artifacts_file), ], check=True, - stdout=f, ) - return tbz2_file + + # Finally, recompress it as a tbz2. + tbz2_file = _mkstemp(".tbz2") + logging.info( + "Recompressing artifacts from %s to %s (this may take a while)...", + decompressed_artifacts_file, + tbz2_file, + ) + with tbz2_file.open("wb") as f: + subprocess.run( + [ + "lbzip2", + "-9", + "-c", + str(decompressed_artifacts_file), + ], + check=True, + stdout=f, + ) + return tbz2_file def _upload(local_file: Path, remote_path: str, force: bool): - """Uploads the local file to the given gs:// path.""" - logging.info('Uploading %s -> %s', local_file, remote_path) - cmd_base = ['gsutil.py', 'cp', '-a', 'public-read'] - if not force: - cmd_base.append('-n') - subprocess.run( - cmd_base + [str(local_file), remote_path], - check=True, - stdin=subprocess.DEVNULL, - ) + """Uploads the local file to the given gs:// path.""" + logging.info("Uploading %s -> %s", local_file, remote_path) + cmd_base = ["gsutil", "cp", "-a", "public-read"] + if not force: + cmd_base.append("-n") + subprocess.run( + cmd_base + [str(local_file), remote_path], + check=True, + stdin=subprocess.DEVNULL, + ) def main(argv: List[str]): - logging.basicConfig( - format='>> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: ' - '%(message)s', - level=logging.INFO, - ) - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument( - 'sdk_artifact', - help='Path to the SDK rust-bootstrap artifact to copy. e.g., ' - 'gs://chromeos-prebuilt/host/amd64/amd64-host/' - 'chroot-2022.07.12.134334/packages/dev-lang/' - 'rust-bootstrap-1.59.0.tbz2.') - parser.add_argument( - '-n', - '--dry-run', - action='store_true', - help='Do everything except actually uploading the artifact.') - parser.add_argument( - '--force', - action='store_true', - help='Upload the artifact even if one exists in localmirror already.') - opts = parser.parse_args(argv) - - if not _is_in_chroot(): - parser.error('Run me from within the chroot.') - _ensure_pbzip2_is_installed() - - target_path = _determine_target_path(opts.sdk_artifact) - with tempfile.TemporaryDirectory() as tempdir: - download_path = Path(tempdir) / 'sdk_artifact' - _download(opts.sdk_artifact, download_path) - file_to_upload = _debinpkgify(download_path) - if opts.dry_run: - logging.info('--dry-run specified; skipping upload of %s to %s', - file_to_upload, target_path) - else: - _upload(file_to_upload, target_path, opts.force) - - -if __name__ == '__main__': - main(sys.argv[1:]) + logging.basicConfig( + format=">> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: " + "%(message)s", + level=logging.INFO, + ) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "sdk_artifact", + help="Path to the SDK rust-bootstrap artifact to copy. e.g., " + "gs://chromeos-prebuilt/host/amd64/amd64-host/" + "chroot-2022.07.12.134334/packages/dev-lang/" + "rust-bootstrap-1.59.0.tbz2.", + ) + parser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="Do everything except actually uploading the artifact.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Upload the artifact even if one exists in localmirror already.", + ) + opts = parser.parse_args(argv) + + if not _is_in_chroot(): + parser.error("Run me from within the chroot.") + _ensure_lbzip2_is_installed() + + target_path = determine_target_path(opts.sdk_artifact) + with tempfile.TemporaryDirectory() as tempdir: + download_path = Path(tempdir) / "sdk_artifact" + _download(opts.sdk_artifact, download_path) + file_to_upload = _debinpkgify(download_path) + if opts.dry_run: + logging.info( + "--dry-run specified; skipping upload of %s to %s", + file_to_upload, + target_path, + ) + else: + _upload(file_to_upload, target_path, opts.force) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py index 382d991a..9845c7c7 100755 --- a/rust_tools/rust_uprev.py +++ b/rust_tools/rust_uprev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,7 +11,7 @@ removing an old version. When using the tool, the progress can be saved to a JSON file, so the user can resume the process after a failing step is fixed. Example usage to create a new version: -1. (inside chroot) $ ./rust_tools/rust_uprev.py \\ +1. (outside chroot) $ ./rust_tools/rust_uprev.py \\ --state_file /tmp/rust-to-1.60.0.json \\ roll --uprev 1.60.0 2. Step "compile rust" failed due to the patches can't apply to new version. @@ -27,6 +26,7 @@ See `--help` for all available options. """ import argparse +import functools import json import logging import os @@ -36,34 +36,115 @@ import re import shlex import shutil import subprocess -import sys -from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple +import threading +import time +from typing import ( + Any, + Callable, + Dict, + List, + NamedTuple, + Optional, + Protocol, + Sequence, + Tuple, + TypeVar, + Union, +) +import urllib.request from llvm_tools import chroot from llvm_tools import git -EQUERY = "equery" -GSUTIL = "gsutil.py" -MIRROR_PATH = "gs://chromeos-localmirror/distfiles" -EBUILD_PREFIX = Path("/mnt/host/source/src/third_party/chromiumos-overlay") -RUST_PATH = Path(EBUILD_PREFIX, "dev-lang", "rust") +T = TypeVar("T") +Command = Sequence[Union[str, os.PathLike]] +PathOrStr = Union[str, os.PathLike] + + +class RunStepFn(Protocol): + """Protocol that corresponds to run_step's type. + + This can be used as the type of a function parameter that accepts + run_step as its value. + """ + + def __call__( + self, + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, + ) -> T: + ... -def get_command_output(command: List[str], *args, **kwargs) -> str: +def get_command_output(command: Command, *args, **kwargs) -> str: return subprocess.check_output( command, encoding="utf-8", *args, **kwargs ).strip() -def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str: +def _get_source_root() -> Path: + """Returns the path to the chromiumos directory.""" + return Path(get_command_output(["repo", "--show-toplevel"])) + + +SOURCE_ROOT = _get_source_root() +EQUERY = "equery" +GPG = "gpg" +GSUTIL = "gsutil.py" +MIRROR_PATH = "gs://chromeos-localmirror/distfiles" +EBUILD_PREFIX = SOURCE_ROOT / "src/third_party/chromiumos-overlay" +CROS_RUSTC_ECLASS = EBUILD_PREFIX / "eclass/cros-rustc.eclass" +# Keyserver to use with GPG. Not all keyservers have Rust's signing key; +# this must be set to a keyserver that does. +GPG_KEYSERVER = "keyserver.ubuntu.com" +PGO_RUST = Path( + "/mnt/host/source" + "/src/third_party/toolchain-utils/pgo_tools_rust/pgo_rust.py" +) +RUST_PATH = Path(EBUILD_PREFIX, "dev-lang", "rust") +# This is the signing key used by upstream Rust as of 2023-08-09. +# If the project switches to a different key, this will have to be updated. +# We require the key to be updated manually so that we have an opportunity +# to verify that the key change is legitimate. +RUST_SIGNING_KEY = "85AB96E6FA1BE5FE" +RUST_SRC_BASE_URI = "https://static.rust-lang.org/dist/" +# Packages that need to be processed like dev-lang/rust. +RUST_PACKAGES = ( + ("dev-lang", "rust-host"), + ("dev-lang", "rust"), +) + + +class SignatureVerificationError(Exception): + """Error that indicates verification of a downloaded file failed. + + Attributes: + message: explanation of why the verification failed. + path: the path to the file whose integrity was being verified. + """ + + def __init__(self, message: str, path: Path): + super(SignatureVerificationError, self).__init__() + self.message = message + self.path = path + + +def get_command_output_unchecked(command: Command, *args, **kwargs) -> str: + # pylint: disable=subprocess-run-check return subprocess.run( command, - check=False, - stdout=subprocess.PIPE, - encoding="utf-8", *args, - **kwargs, + **dict( + { + "check": False, + "stdout": subprocess.PIPE, + "encoding": "utf-8", + }, + **kwargs, + ), ).stdout.strip() @@ -78,7 +159,7 @@ class RustVersion(NamedTuple): return f"{self.major}.{self.minor}.{self.patch}" @staticmethod - def parse_from_ebuild(ebuild_name: str) -> "RustVersion": + def parse_from_ebuild(ebuild_name: PathOrStr) -> "RustVersion": input_re = re.compile( r"^rust-" r"(?P<major>\d+)\." @@ -87,7 +168,7 @@ class RustVersion(NamedTuple): r"(:?-r\d+)?" r"\.ebuild$" ) - m = input_re.match(ebuild_name) + m = input_re.match(Path(ebuild_name).name) assert m, f"failed to parse {ebuild_name!r}" return RustVersion( int(m.group("major")), int(m.group("minor")), int(m.group("patch")) @@ -109,17 +190,26 @@ class RustVersion(NamedTuple): ) -def compute_rustc_src_name(version: RustVersion) -> str: - return f"rustc-{version}-src.tar.gz" +class PreparedUprev(NamedTuple): + """Container for the information returned by prepare_uprev.""" + + template_version: RustVersion + +def compute_ebuild_path(category: str, name: str, version: RustVersion) -> Path: + return EBUILD_PREFIX / category / name / f"{name}-{version}.ebuild" -def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str: - return f"rust-bootstrap-{version}.tbz2" + +def compute_rustc_src_name(version: RustVersion) -> str: + return f"rustc-{version}-src.tar.gz" -def find_ebuild_for_package(name: str) -> os.PathLike: +def find_ebuild_for_package(name: str) -> str: """Returns the path to the ebuild for the named package.""" - return get_command_output([EQUERY, "w", name]) + return run_in_chroot( + [EQUERY, "w", name], + stdout=subprocess.PIPE, + ).stdout.strip() def find_ebuild_path( @@ -235,18 +325,6 @@ def parse_commandline_args() -> argparse.Namespace: "specified, the tool will remove the oldest version in the chroot", ) - subparser_names.append("remove-bootstrap") - remove_bootstrap_parser = subparsers.add_parser( - "remove-bootstrap", - help="Remove an old rust-bootstrap version", - ) - remove_bootstrap_parser.add_argument( - "--version", - type=RustVersion.parse, - required=True, - help="rust-bootstrap version to remove", - ) - subparser_names.append("roll") roll_parser = subparsers.add_parser( "roll", @@ -302,90 +380,73 @@ def parse_commandline_args() -> argparse.Namespace: def prepare_uprev( - rust_version: RustVersion, template: Optional[RustVersion] -) -> Optional[Tuple[RustVersion, str, RustVersion]]: - if template is None: - ebuild_path = find_ebuild_for_package("rust") - ebuild_name = os.path.basename(ebuild_path) - template_version = RustVersion.parse_from_ebuild(ebuild_name) - else: - ebuild_path = find_ebuild_for_rust_version(template) - template_version = template - - bootstrap_version = get_rust_bootstrap_version() + rust_version: RustVersion, template: RustVersion +) -> Optional[PreparedUprev]: + ebuild_path = find_ebuild_for_rust_version(template) - if rust_version <= template_version: + if rust_version <= template: logging.info( "Requested version %s is not newer than the template version %s.", rust_version, - template_version, + template, ) return None logging.info( - "Template Rust version is %s (ebuild: %r)", - template_version, + "Template Rust version is %s (ebuild: %s)", + template, ebuild_path, ) - logging.info("rust-bootstrap version is %s", bootstrap_version) - return template_version, ebuild_path, bootstrap_version + return PreparedUprev(template) -def copy_patches( - directory: Path, template_version: RustVersion, new_version: RustVersion +def create_ebuild( + category: str, + name: str, + template_version: RustVersion, + new_version: RustVersion, ) -> None: - patch_path = directory / "files" - prefix = "%s-%s-" % (directory.name, template_version) - new_prefix = "%s-%s-" % (directory.name, new_version) - for f in os.listdir(patch_path): - if not f.startswith(prefix): - continue - logging.info("Copy patch %s to new version", f) - new_name = f.replace(str(template_version), str(new_version)) - shutil.copyfile( - os.path.join(patch_path, f), - os.path.join(patch_path, new_name), - ) - + template_ebuild = compute_ebuild_path(category, name, template_version) + new_ebuild = compute_ebuild_path(category, name, new_version) + shutil.copyfile(template_ebuild, new_ebuild) subprocess.check_call( - ["git", "add", f"{new_prefix}*.patch"], cwd=patch_path + ["git", "add", new_ebuild.name], cwd=new_ebuild.parent ) -def create_ebuild( - template_ebuild: str, pkgatom: str, new_version: RustVersion -) -> str: - filename = f"{Path(pkgatom).name}-{new_version}.ebuild" - ebuild = EBUILD_PREFIX.joinpath(f"{pkgatom}/{filename}") - shutil.copyfile(template_ebuild, ebuild) - subprocess.check_call(["git", "add", filename], cwd=ebuild.parent) - return str(ebuild) - - -def update_bootstrap_ebuild(new_bootstrap_version: RustVersion) -> None: - old_ebuild = find_ebuild_path(rust_bootstrap_path(), "rust-bootstrap") - m = re.match(r"^rust-bootstrap-(\d+).(\d+).(\d+)", old_ebuild.name) - assert m, old_ebuild.name - old_version = RustVersion(m.group(1), m.group(2), m.group(3)) - new_ebuild = old_ebuild.parent.joinpath( - f"rust-bootstrap-{new_bootstrap_version}.ebuild" - ) - old_text = old_ebuild.read_text(encoding="utf-8") - new_text, changes = re.subn( - r"(RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=\([^)]*)", - f"\\1\t{old_version}\n", - old_text, +def set_include_profdata_src(ebuild_path: os.PathLike, include: bool) -> None: + """Changes an ebuild file to include or omit profile data from SRC_URI. + + If include is True, the ebuild file will be rewritten to include + profile data in SRC_URI. + + If include is False, the ebuild file will be rewritten to omit profile + data from SRC_URI. + """ + if include: + old = "" + new = "yes" + else: + old = "yes" + new = "" + contents = Path(ebuild_path).read_text(encoding="utf-8") + contents, subs = re.subn( + f"^INCLUDE_PROFDATA_IN_SRC_URI={old}$", + f"INCLUDE_PROFDATA_IN_SRC_URI={new}", + contents, flags=re.MULTILINE, ) - assert changes == 1, "Failed to update RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE" - new_ebuild.write_text(new_text, encoding="utf-8") + # We expect exactly one substitution. + assert subs == 1, "Failed to update INCLUDE_PROFDATA_IN_SRC_URI" + Path(ebuild_path).write_text(contents, encoding="utf-8") def update_bootstrap_version( - path: str, new_bootstrap_version: RustVersion + path: PathOrStr, new_bootstrap_version: RustVersion ) -> None: - contents = open(path, encoding="utf-8").read() + path = Path(path) + contents = path.read_text(encoding="utf-8") contents, subs = re.subn( r"^BOOTSTRAP_VERSION=.*$", 'BOOTSTRAP_VERSION="%s"' % (new_bootstrap_version,), @@ -394,7 +455,7 @@ def update_bootstrap_version( ) if not subs: raise RuntimeError(f"BOOTSTRAP_VERSION not found in {path}") - open(path, "w", encoding="utf-8").write(contents) + path.write_text(contents, encoding="utf-8") logging.info("Rust BOOTSTRAP_VERSION updated to %s", new_bootstrap_version) @@ -405,7 +466,7 @@ def ebuild_actions( cmd = ["ebuild", ebuild_path_inchroot] + actions if sudo: cmd = ["sudo"] + cmd - subprocess.check_call(cmd) + run_in_chroot(cmd) def fetch_distfile_from_mirror(name: str) -> None: @@ -413,7 +474,7 @@ def fetch_distfile_from_mirror(name: str) -> None: This ensures that the file exists on the mirror, and that we can read it. We overwrite any existing distfile - to ensure the checksums that update_manifest() records + to ensure the checksums that `ebuild manifest` records match the file as it exists on the mirror. This function also attempts to verify the ACL for @@ -425,8 +486,8 @@ def fetch_distfile_from_mirror(name: str) -> None: the file even though we don't own it. """ mirror_file = MIRROR_PATH + "/" + name - local_file = Path(get_distdir(), name) - cmd = [GSUTIL, "cp", mirror_file, local_file] + local_file = get_distdir() / name + cmd: Command = [GSUTIL, "cp", mirror_file, local_file] logging.info("Running %r", cmd) rc = subprocess.call(cmd) if rc != 0: @@ -474,19 +535,14 @@ it to the local mirror using gsutil cp. raise Exception("Could not verify that allUsers has READER permission") -def fetch_bootstrap_distfiles( - old_version: RustVersion, new_version: RustVersion -) -> None: +def fetch_bootstrap_distfiles(version: RustVersion) -> None: """Fetches rust-bootstrap distfiles from the local mirror Fetches the distfiles for a rust-bootstrap ebuild to ensure they are available on the mirror and the local copies are the same as the ones on the mirror. """ - fetch_distfile_from_mirror( - compute_rust_bootstrap_prebuilt_name(old_version) - ) - fetch_distfile_from_mirror(compute_rustc_src_name(new_version)) + fetch_distfile_from_mirror(compute_rustc_src_name(version)) def fetch_rust_distfiles(version: RustVersion) -> None: @@ -499,15 +555,113 @@ def fetch_rust_distfiles(version: RustVersion) -> None: fetch_distfile_from_mirror(compute_rustc_src_name(version)) -def get_distdir() -> os.PathLike: - """Returns portage's distdir.""" - return get_command_output(["portageq", "distdir"]) +def fetch_rust_src_from_upstream(uri: str, local_path: Path) -> None: + """Fetches Rust sources from upstream. + + This downloads the source distribution and the .asc file + containing the signatures. It then verifies that the sources + have the expected signature and have been signed by + the expected key. + """ + subprocess.run( + [GPG, "--keyserver", GPG_KEYSERVER, "--recv-keys", RUST_SIGNING_KEY], + check=True, + ) + subprocess.run( + [GPG, "--keyserver", GPG_KEYSERVER, "--refresh-keys", RUST_SIGNING_KEY], + check=True, + ) + asc_uri = uri + ".asc" + local_asc_path = Path(local_path.parent, local_path.name + ".asc") + logging.info("Fetching %s", uri) + urllib.request.urlretrieve(uri, local_path) + logging.info("%s fetched", uri) + + # Raise SignatureVerificationError if we cannot get the signature. + try: + logging.info("Fetching %s", asc_uri) + urllib.request.urlretrieve(asc_uri, local_asc_path) + logging.info("%s fetched", asc_uri) + except Exception as e: + raise SignatureVerificationError( + f"error fetching signature file {asc_uri}", + local_path, + ) from e + + # Raise SignatureVerificationError if verifying the signature + # failed. + try: + output = get_command_output( + [GPG, "--verify", "--status-fd", "1", local_asc_path] + ) + except subprocess.CalledProcessError as e: + raise SignatureVerificationError( + f"error verifying signature. GPG output:\n{e.stdout}", + local_path, + ) from e + + # Raise SignatureVerificationError if the file was not signed + # with the expected key. + if f"GOODSIG {RUST_SIGNING_KEY}" not in output: + message = f"GOODSIG {RUST_SIGNING_KEY} not found in output" + if f"REVKEYSIG {RUST_SIGNING_KEY}" in output: + message = "signing key has been revoked" + elif f"EXPKEYSIG {RUST_SIGNING_KEY}" in output: + message = "signing key has expired" + elif f"EXPSIG {RUST_SIGNING_KEY}" in output: + message = "signature has expired" + raise SignatureVerificationError( + f"{message}. GPG output:\n{output}", + local_path, + ) + + +def get_distdir() -> Path: + """Returns portage's distdir outside the chroot.""" + return SOURCE_ROOT / ".cache/distfiles" + + +def mirror_has_file(name: str) -> bool: + """Checks if the mirror has the named file.""" + mirror_file = MIRROR_PATH + "/" + name + cmd: Command = [GSUTIL, "ls", mirror_file] + proc = subprocess.run( + cmd, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + ) + if "URLs matched no objects" in proc.stdout: + return False + elif proc.returncode == 0: + return True + + raise Exception( + "Unexpected result from gsutil ls:" + f" rc {proc.returncode} output:\n{proc.stdout}" + ) + +def mirror_rust_source(version: RustVersion) -> None: + """Ensures source code for a Rust version is on the local mirror. -def update_manifest(ebuild_file: os.PathLike) -> None: - """Updates the MANIFEST for the ebuild at the given path.""" - ebuild = Path(ebuild_file) - ebuild_actions(ebuild.parent.name, ["manifest"]) + If the source code is not found on the mirror, it is fetched + from upstream, its integrity is verified, and it is uploaded + to the mirror. + """ + filename = compute_rustc_src_name(version) + if mirror_has_file(filename): + logging.info("%s is present on the mirror", filename) + return + uri = f"{RUST_SRC_BASE_URI}{filename}" + local_path = get_distdir() / filename + mirror_path = f"{MIRROR_PATH}/{filename}" + fetch_rust_src_from_upstream(uri, local_path) + subprocess.run( + [GSUTIL, "cp", "-a", "public-read", local_path, mirror_path], + check=True, + ) def update_rust_packages( @@ -556,14 +710,14 @@ def update_virtual_rust( def unmerge_package_if_installed(pkgatom: str) -> None: """Unmerges a package if it is installed.""" shpkg = shlex.quote(pkgatom) - subprocess.check_call( + run_in_chroot( [ "sudo", "bash", "-c", f"! emerge --pretend --quiet --unmerge {shpkg}" f" || emerge --rage-clean {shpkg}", - ] + ], ) @@ -596,28 +750,51 @@ def perform_step( return val -def prepare_uprev_from_json( - obj: Any, -) -> Optional[Tuple[RustVersion, str, RustVersion]]: +def prepare_uprev_from_json(obj: Any) -> Optional[PreparedUprev]: if not obj: return None - version, ebuild_path, bootstrap_version = obj - return RustVersion(*version), ebuild_path, RustVersion(*bootstrap_version) + version = obj[0] + return PreparedUprev( + RustVersion(*version), + ) + + +def prepare_uprev_to_json( + prepared_uprev: Optional[PreparedUprev], +) -> Optional[Tuple[RustVersion]]: + if prepared_uprev is None: + return None + return (prepared_uprev.template_version,) def create_rust_uprev( rust_version: RustVersion, - maybe_template_version: Optional[RustVersion], + template_version: RustVersion, skip_compile: bool, - run_step: Callable[[], T], + run_step: RunStepFn, ) -> None: - template_version, template_ebuild, old_bootstrap_version = run_step( + prepared = run_step( "prepare uprev", - lambda: prepare_uprev(rust_version, maybe_template_version), + lambda: prepare_uprev(rust_version, template_version), result_from_json=prepare_uprev_from_json, + result_to_json=prepare_uprev_to_json, ) - if template_ebuild is None: + if prepared is None: return + template_version = prepared.template_version + + run_step( + "mirror bootstrap sources", + lambda: mirror_rust_source( + template_version, + ), + ) + run_step( + "mirror rust sources", + lambda: mirror_rust_source( + rust_version, + ), + ) # The fetch steps will fail (on purpose) if the files they check for # are not available on the mirror. To make them pass, fetch the @@ -625,53 +802,56 @@ def create_rust_uprev( # to the mirror. run_step( "fetch bootstrap distfiles", - lambda: fetch_bootstrap_distfiles( - old_bootstrap_version, template_version - ), + lambda: fetch_bootstrap_distfiles(template_version), ) run_step("fetch rust distfiles", lambda: fetch_rust_distfiles(rust_version)) run_step( - "update bootstrap ebuild", - lambda: update_bootstrap_ebuild(template_version), + "update bootstrap version", + lambda: update_bootstrap_version(CROS_RUSTC_ECLASS, template_version), ) run_step( - "update bootstrap manifest", - lambda: update_manifest( - rust_bootstrap_path().joinpath( - f"rust-bootstrap-{template_version}.ebuild" - ) - ), + "turn off profile data sources in cros-rustc.eclass", + lambda: set_include_profdata_src(CROS_RUSTC_ECLASS, include=False), ) + + for category, name in RUST_PACKAGES: + run_step( + f"create new {category}/{name} ebuild", + functools.partial( + create_ebuild, + category, + name, + template_version, + rust_version, + ), + ) + run_step( - "update bootstrap version", - lambda: update_bootstrap_version( - EBUILD_PREFIX.joinpath("eclass/cros-rustc.eclass"), template_version - ), + "update dev-lang/rust-host manifest to add new version", + lambda: ebuild_actions("dev-lang/rust-host", ["manifest"]), ) + run_step( - "copy patches", - lambda: copy_patches(RUST_PATH, template_version, rust_version), - ) - template_host_ebuild = EBUILD_PREFIX.joinpath( - f"dev-lang/rust-host/rust-host-{template_version}.ebuild" - ) - host_file = run_step( - "create host ebuild", - lambda: create_ebuild( - template_host_ebuild, "dev-lang/rust-host", rust_version - ), + "generate profile data for rustc", + lambda: run_in_chroot([PGO_RUST, "generate"]), + # Avoid returning subprocess.CompletedProcess, which cannot be + # serialized to JSON. + result_to_json=lambda _x: None, ) run_step( - "update host manifest to add new version", - lambda: update_manifest(Path(host_file)), + "upload profile data for rustc", + lambda: run_in_chroot([PGO_RUST, "upload-profdata"]), + # Avoid returning subprocess.CompletedProcess, which cannot be + # serialized to JSON. + result_to_json=lambda _x: None, ) - target_file = run_step( - "create target ebuild", - lambda: create_ebuild(template_ebuild, "dev-lang/rust", rust_version), + run_step( + "turn on profile data sources in cros-rustc.eclass", + lambda: set_include_profdata_src(CROS_RUSTC_ECLASS, include=True), ) run_step( - "update target manifest to add new version", - lambda: update_manifest(Path(target_file)), + "update dev-lang/rust-host manifest to add profile data", + lambda: ebuild_actions("dev-lang/rust-host", ["manifest"]), ) if not skip_compile: run_step("build packages", lambda: rebuild_packages(rust_version)) @@ -691,44 +871,38 @@ def create_rust_uprev( ) -def find_rust_versions_in_chroot() -> List[Tuple[RustVersion, str]]: +def find_rust_versions() -> List[Tuple[RustVersion, Path]]: + """Returns (RustVersion, ebuild_path) for base versions of dev-lang/rust. + + This excludes symlinks to ebuilds, so if rust-1.34.0.ebuild and + rust-1.34.0-r1.ebuild both exist and -r1 is a symlink to the other, + only rust-1.34.0.ebuild will be in the return value. + """ return [ - (RustVersion.parse_from_ebuild(x), os.path.join(RUST_PATH, x)) - for x in os.listdir(RUST_PATH) - if x.endswith(".ebuild") + (RustVersion.parse_from_ebuild(ebuild), ebuild) + for ebuild in RUST_PATH.iterdir() + if ebuild.suffix == ".ebuild" and not ebuild.is_symlink() ] -def find_oldest_rust_version_in_chroot() -> RustVersion: - rust_versions = find_rust_versions_in_chroot() +def find_oldest_rust_version() -> RustVersion: + """Returns the RustVersion of the oldest dev-lang/rust ebuild.""" + rust_versions = find_rust_versions() if len(rust_versions) <= 1: raise RuntimeError("Expect to find more than one Rust versions") return min(rust_versions)[0] -def find_ebuild_for_rust_version(version: RustVersion) -> str: - rust_ebuilds = [ - ebuild for x, ebuild in find_rust_versions_in_chroot() if x == version - ] - if not rust_ebuilds: - raise ValueError(f"No Rust ebuilds found matching {version}") - if len(rust_ebuilds) > 1: - raise ValueError( - f"Multiple Rust ebuilds found matching {version}: " - f"{rust_ebuilds}" - ) - return rust_ebuilds[0] +def find_ebuild_for_rust_version(version: RustVersion) -> Path: + """Returns the path of the ebuild for the given version of dev-lang/rust.""" + return find_ebuild_path(RUST_PATH, "rust", version) def rebuild_packages(version: RustVersion): """Rebuild packages modified by this script.""" # Remove all packages we modify to avoid depending on preinstalled # versions. This ensures that the packages can really be built. - packages = [ - "dev-lang/rust", - "dev-lang/rust-host", - "dev-lang/rust-bootstrap", - ] + packages = [f"{category}/{name}" for category, name in RUST_PACKAGES] for pkg in packages: unmerge_package_if_installed(pkg) # Mention only dev-lang/rust explicitly, so that others are pulled @@ -736,7 +910,7 @@ def rebuild_packages(version: RustVersion): # Packages we modify are listed in --usepkg-exclude to ensure they # are built from source. try: - subprocess.check_call( + run_in_chroot( [ "sudo", "emerge", @@ -744,7 +918,7 @@ def rebuild_packages(version: RustVersion): "--usepkg-exclude", " ".join(packages), f"=dev-lang/rust-{version}", - ] + ], ) except: logging.warning( @@ -755,16 +929,16 @@ def rebuild_packages(version: RustVersion): raise -def remove_ebuild_version(path: os.PathLike, name: str, version: RustVersion): +def remove_ebuild_version(path: PathOrStr, name: str, version: RustVersion): """Remove the specified version of an ebuild. Removes {path}/{name}-{version}.ebuild and {path}/{name}-{version}-*.ebuild using git rm. Args: - path: The directory in which the ebuild files are. - name: The name of the package (e.g. 'rust'). - version: The version of the ebuild to remove. + path: The directory in which the ebuild files are. + name: The name of the package (e.g. 'rust'). + version: The version of the ebuild to remove. """ path = Path(path) pattern = f"{name}-{version}-*.ebuild" @@ -780,33 +954,18 @@ def remove_ebuild_version(path: os.PathLike, name: str, version: RustVersion): remove_files(m.name, path) -def remove_files(filename: str, path: str) -> None: +def remove_files(filename: PathOrStr, path: PathOrStr) -> None: subprocess.check_call(["git", "rm", filename], cwd=path) -def remove_rust_bootstrap_version( - version: RustVersion, run_step: Callable[[], T] -) -> None: - run_step( - "remove old bootstrap ebuild", - lambda: remove_ebuild_version( - rust_bootstrap_path(), "rust-bootstrap", version - ), - ) - ebuild_file = find_ebuild_for_package("rust-bootstrap") - run_step( - "update bootstrap manifest to delete old version", - lambda: update_manifest(ebuild_file), - ) - - def remove_rust_uprev( - rust_version: Optional[RustVersion], run_step: Callable[[], T] + rust_version: Optional[RustVersion], + run_step: RunStepFn, ) -> None: def find_desired_rust_version() -> RustVersion: if rust_version: return rust_version - return find_oldest_rust_version_in_chroot() + return find_oldest_rust_version() def find_desired_rust_version_from_json(obj: Any) -> RustVersion: return RustVersion(*obj) @@ -816,26 +975,21 @@ def remove_rust_uprev( find_desired_rust_version, result_from_json=find_desired_rust_version_from_json, ) + + for category, name in RUST_PACKAGES: + run_step( + f"remove old {name} ebuild", + functools.partial( + remove_ebuild_version, + EBUILD_PREFIX / category / name, + name, + delete_version, + ), + ) + run_step( - "remove patches", - lambda: remove_files(f"files/rust-{delete_version}-*.patch", RUST_PATH), - ) - run_step( - "remove target ebuild", - lambda: remove_ebuild_version(RUST_PATH, "rust", delete_version), - ) - run_step( - "remove host ebuild", - lambda: remove_ebuild_version( - EBUILD_PREFIX.joinpath("dev-lang/rust-host"), - "rust-host", - delete_version, - ), - ) - target_file = find_ebuild_for_package("rust") - run_step( - "update target manifest to delete old version", - lambda: update_manifest(target_file), + "update dev-lang/rust-host manifest to delete old version", + lambda: ebuild_actions("dev-lang/rust-host", ["manifest"]), ) run_step( "remove target version from rust packages", @@ -843,11 +997,6 @@ def remove_rust_uprev( "dev-lang/rust", delete_version, add=False ), ) - host_file = find_ebuild_for_package("rust-host") - run_step( - "update host manifest to delete old version", - lambda: update_manifest(host_file), - ) run_step( "remove host version from rust packages", lambda: update_rust_packages( @@ -879,11 +1028,10 @@ def create_new_repo(rust_version: RustVersion) -> None: git.CreateBranch(EBUILD_PREFIX, f"rust-to-{rust_version}") -def build_cross_compiler() -> None: +def build_cross_compiler(template_version: RustVersion) -> None: # Get target triples in ebuild - rust_ebuild = find_ebuild_for_package("rust") - with open(rust_ebuild, encoding="utf-8") as f: - contents = f.read() + rust_ebuild = find_ebuild_path(RUST_PATH, "rust", template_version) + contents = rust_ebuild.read_text(encoding="utf-8") target_triples_re = re.compile(r"RUSTC_TARGET_TRIPLES=\(([^)]+)\)") m = target_triples_re.search(contents) @@ -903,9 +1051,9 @@ def build_cross_compiler() -> None: compiler_targets_to_install.append("arm-none-eabi") logging.info("Emerging cross compilers %s", compiler_targets_to_install) - subprocess.check_call( + run_in_chroot( ["sudo", "emerge", "-j", "-G"] - + [f"cross-{target}/gcc" for target in compiler_targets_to_install] + + [f"cross-{target}/gcc" for target in compiler_targets_to_install], ) @@ -918,17 +1066,95 @@ def create_new_commit(rust_version: RustVersion) -> None: "BUG=None", "TEST=Use CQ to test the new Rust version", ] - git.UploadChanges(EBUILD_PREFIX, f"rust-to-{rust_version}", messages) + branch = f"rust-to-{rust_version}" + git.CommitChanges(EBUILD_PREFIX, messages) + git.UploadChanges(EBUILD_PREFIX, branch) -def main() -> None: - if not chroot.InChroot(): - raise RuntimeError("This script must be executed inside chroot") +def run_in_chroot(cmd: Command, *args, **kwargs) -> subprocess.CompletedProcess: + """Runs a command in the ChromiumOS chroot. - logging.basicConfig(level=logging.INFO) + This takes the same arguments as subprocess.run(). By default, + it uses check=True, encoding="utf-8". If needed, these can be + overridden by keyword arguments passed to run_in_chroot(). + """ + full_kwargs = dict( + { + "check": True, + "encoding": "utf-8", + }, + **kwargs, + ) + full_cmd = ["cros_sdk", "--"] + list(cmd) + logging.info("Running %s", shlex.join(str(x) for x in full_cmd)) + # pylint: disable=subprocess-run-check + # (check is actually set above; it defaults to True) + return subprocess.run(full_cmd, *args, **full_kwargs) + + +def sudo_keepalive() -> None: + """Ensures we have sudo credentials, and keeps them up-to-date. + + Some operations (notably run_in_chroot) run sudo, which may require + user interaction. To avoid blocking progress while we sit waiting + for that interaction, sudo_keepalive checks that we have cached + sudo credentials, gets them if necessary, then keeps them up-to-date + so that the rest of the script can run without needing to get + sudo credentials again. + """ + logging.info( + "Caching sudo credentials for running commands inside the chroot" + ) + # Exits successfully if cached credentials exist. Otherwise, tries + # created cached credentials, prompting for authentication if necessary. + subprocess.run(["sudo", "true"], check=True) + + def sudo_keepalive_loop() -> None: + # Between credential refreshes, we sleep so that we don't + # unnecessarily burn CPU cycles. The sleep time must be shorter + # than sudo's configured cached credential expiration time, which + # is 15 minutes by default. + sleep_seconds = 10 * 60 + # So as to not keep credentials cached forever, we limit the number + # of times we will refresh them. + max_seconds = 16 * 3600 + max_refreshes = max_seconds // sleep_seconds + for _x in range(max_refreshes): + # Refreshes cached credentials if they exist, but never prompts + # for anything. If cached credentials do not exist, this + # command exits with an error. We ignore that error to keep the + # loop going, so that cached credentials will be kept fresh + # again once we have them (e.g. after the next cros_sdk command + # successfully authenticates the user). + # + # The standard file descriptors are all redirected to/from + # /dev/null to prevent this command from consuming any input + # or mixing its output with that of the other commands rust_uprev + # runs (which could be confusing, for example making it look like + # errors occurred during a build when they are actually in a + # separate task). + # + # Note: The command specifically uses "true" and not "-v", because + # it turns out that "-v" actually will prompt for a password when + # sudo is configured with NOPASSWD=all, even though in that case + # no password is required to run actual commands. + subprocess.run( + ["sudo", "-n", "true"], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + time.sleep(sleep_seconds) + + # daemon=True causes the thread to be killed when the script exits. + threading.Thread(target=sudo_keepalive_loop, daemon=True).start() - args = parse_commandline_args() +def main() -> None: + chroot.VerifyOutsideChroot() + logging.basicConfig(level=logging.INFO) + args = parse_commandline_args() state_file = pathlib.Path(args.state_file) tmp_state_file = state_file.with_suffix(".tmp") @@ -955,27 +1181,36 @@ def main() -> None: ) if args.subparser_name == "create": + sudo_keepalive() create_rust_uprev( args.rust_version, args.template, args.skip_compile, run_step ) elif args.subparser_name == "remove": remove_rust_uprev(args.rust_version, run_step) - elif args.subparser_name == "remove-bootstrap": - remove_rust_bootstrap_version(args.version, run_step) else: - # If you have added more subparser_name, please also add the handlers above + # If you have added more subparser_name, please also add the handlers + # above assert args.subparser_name == "roll" + + sudo_keepalive() + # Determine the template version, if not given. + template_version = args.template + if template_version is None: + rust_ebuild = find_ebuild_for_package("dev-lang/rust") + template_version = RustVersion.parse_from_ebuild(rust_ebuild) + run_step("create new repo", lambda: create_new_repo(args.uprev)) if not args.skip_cross_compiler: - run_step("build cross compiler", build_cross_compiler) + run_step( + "build cross compiler", + lambda: build_cross_compiler(template_version), + ) create_rust_uprev( - args.uprev, args.template, args.skip_compile, run_step + args.uprev, template_version, args.skip_compile, run_step ) remove_rust_uprev(args.remove, run_step) - bootstrap_version = prepare_uprev_from_json( - completed_steps["prepare uprev"] - )[2] - remove_rust_bootstrap_version(bootstrap_version, run_step) + prepared = prepare_uprev_from_json(completed_steps["prepare uprev"]) + assert prepared is not None, "no prepared uprev decoded from JSON" if not args.no_upload: run_step( "create rust uprev CL", lambda: create_new_commit(args.uprev) @@ -983,4 +1218,4 @@ def main() -> None: if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py index 0c4c91ed..a90f3d10 100755 --- a/rust_tools/rust_uprev_test.py +++ b/rust_tools/rust_uprev_test.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -15,8 +14,13 @@ import unittest from unittest import mock from llvm_tools import git -import rust_uprev -from rust_uprev import RustVersion + + +# rust_uprev sets SOURCE_ROOT to the output of `repo --show-toplevel`. +# The mock below makes us not actually run repo but use a fake value +# instead. +with mock.patch("subprocess.check_output", return_value="/fake/chromiumos"): + import rust_uprev def _fail_command(cmd, *_args, **_kwargs): @@ -25,11 +29,28 @@ def _fail_command(cmd, *_args, **_kwargs): raise err +def start_mock(obj, *args, **kwargs): + """Creates a patcher, starts it, and registers a cleanup to stop it. + + Args: + obj: + the object to attach the cleanup to + *args: + passed to mock.patch() + **kwargs: + passsed to mock.patch() + """ + patcher = mock.patch(*args, **kwargs) + val = patcher.start() + obj.addCleanup(patcher.stop) + return val + + class FetchDistfileTest(unittest.TestCase): """Tests rust_uprev.fetch_distfile_from_mirror()""" @mock.patch.object( - rust_uprev, "get_distdir", return_value="/fake/distfiles" + rust_uprev, "get_distdir", return_value=Path("/fake/distfiles") ) @mock.patch.object(subprocess, "call", side_effect=_fail_command) def test_fetch_difstfile_fail(self, *_args) -> None: @@ -42,7 +63,7 @@ class FetchDistfileTest(unittest.TestCase): return_value="AccessDeniedException: Access denied.", ) @mock.patch.object( - rust_uprev, "get_distdir", return_value="/fake/distfiles" + rust_uprev, "get_distdir", return_value=Path("/fake/distfiles") ) @mock.patch.object(subprocess, "call", return_value=0) def test_fetch_distfile_acl_access_denied(self, *_args) -> None: @@ -54,7 +75,7 @@ class FetchDistfileTest(unittest.TestCase): return_value='[ { "entity": "allUsers", "role": "READER" } ]', ) @mock.patch.object( - rust_uprev, "get_distdir", return_value="/fake/distfiles" + rust_uprev, "get_distdir", return_value=Path("/fake/distfiles") ) @mock.patch.object(subprocess, "call", return_value=0) def test_fetch_distfile_acl_ok(self, *_args) -> None: @@ -66,7 +87,7 @@ class FetchDistfileTest(unittest.TestCase): return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]', ) @mock.patch.object( - rust_uprev, "get_distdir", return_value="/fake/distfiles" + rust_uprev, "get_distdir", return_value=Path("/fake/distfiles") ) @mock.patch.object(subprocess, "call", return_value=0) def test_fetch_distfile_acl_wrong(self, *_args) -> None: @@ -79,6 +100,151 @@ class FetchDistfileTest(unittest.TestCase): ) +class FetchRustSrcFromUpstreamTest(unittest.TestCase): + """Tests for rust_uprev.fetch_rust_src_from_upstream.""" + + def setUp(self) -> None: + self._mock_get_distdir = start_mock( + self, + "rust_uprev.get_distdir", + return_value=Path("/fake/distfiles"), + ) + + self._mock_gpg = start_mock( + self, + "subprocess.run", + side_effect=self.fake_gpg, + ) + + self._mock_urlretrieve = start_mock( + self, + "urllib.request.urlretrieve", + side_effect=self.fake_urlretrieve, + ) + + self._mock_rust_signing_key = start_mock( + self, + "rust_uprev.RUST_SIGNING_KEY", + "1234567", + ) + + @staticmethod + def fake_urlretrieve(src: str, dest: Path) -> None: + pass + + @staticmethod + def fake_gpg(cmd, **_kwargs): + val = mock.Mock() + val.returncode = 0 + val.stdout = "" + if "--verify" in cmd: + val.stdout = "GOODSIG 1234567" + return val + + def test_success(self): + with mock.patch("rust_uprev.GPG", "gnupg"): + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self._mock_urlretrieve.has_calls( + [ + ( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ), + ( + "fakehttps://rustc-1.60.3-src.tar.gz.asc", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz.asc"), + ), + ] + ) + self._mock_gpg.has_calls( + [ + (["gnupg", "--refresh-keys", "1234567"], {"check": True}), + ] + ) + + def test_no_signature_file(self): + def _urlretrieve(src, dest): + if src.endswith(".asc"): + raise Exception("404 not found") + return self.fake_urlretrieve(src, dest) + + self._mock_urlretrieve.side_effect = _urlretrieve + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("error fetching signature file", ctx.exception.message) + + def test_key_expired(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "EXPKEYSIG 1234567" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("key has expired", ctx.exception.message) + + def test_key_revoked(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "REVKEYSIG 1234567" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("key has been revoked", ctx.exception.message) + + def test_signature_expired(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "EXPSIG 1234567" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("signature has expired", ctx.exception.message) + + def test_wrong_key(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "GOODSIG 0000000" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("1234567 not found", ctx.exception.message) + + class FindEbuildPathTest(unittest.TestCase): """Tests for rust_uprev.find_ebuild_path()""" @@ -143,6 +309,100 @@ class FindEbuildPathTest(unittest.TestCase): self.assertEqual(result, ebuild) +class FindRustVersionsTest(unittest.TestCase): + """Tests for rust_uprev.find_rust_versions.""" + + def test_with_symlinks(self): + with tempfile.TemporaryDirectory() as t: + tmpdir = Path(t) + rust_1_49_1_ebuild = tmpdir / "rust-1.49.1.ebuild" + rust_1_50_0_ebuild = tmpdir / "rust-1.50.0.ebuild" + rust_1_50_0_r1_ebuild = tmpdir / "rust-1.50.0-r1.ebuild" + rust_1_49_1_ebuild.touch() + rust_1_50_0_ebuild.touch() + rust_1_50_0_r1_ebuild.symlink_to(rust_1_50_0_ebuild) + with mock.patch("rust_uprev.RUST_PATH", tmpdir): + actual = rust_uprev.find_rust_versions() + expected = [ + (rust_uprev.RustVersion(1, 49, 1), rust_1_49_1_ebuild), + (rust_uprev.RustVersion(1, 50, 0), rust_1_50_0_ebuild), + ] + self.assertEqual(actual, expected) + + +class MirrorHasFileTest(unittest.TestCase): + """Tests for rust_uprev.mirror_has_file.""" + + @mock.patch.object(subprocess, "run") + def test_no(self, mock_run): + mock_run.return_value = mock.Mock( + returncode=1, + stdout="CommandException: One or more URLs matched no objects.", + ) + self.assertFalse(rust_uprev.mirror_has_file("rustc-1.69.0-src.tar.gz")) + + @mock.patch.object(subprocess, "run") + def test_yes(self, mock_run): + mock_run.return_value = mock.Mock( + returncode=0, + # pylint: disable=line-too-long + stdout="gs://chromeos-localmirror/distfiles/rustc-1.69.0-src.tar.gz", + ) + self.assertTrue(rust_uprev.mirror_has_file("rustc-1.69.0-src.tar.gz")) + + +class MirrorRustSourceTest(unittest.TestCase): + """Tests for rust_uprev.mirror_rust_source.""" + + def setUp(self) -> None: + start_mock(self, "rust_uprev.GSUTIL", "gsutil") + start_mock(self, "rust_uprev.MIRROR_PATH", "fakegs://fakemirror/") + start_mock( + self, "rust_uprev.get_distdir", return_value=Path("/fake/distfiles") + ) + self.mock_mirror_has_file = start_mock( + self, + "rust_uprev.mirror_has_file", + ) + self.mock_fetch_rust_src_from_upstream = start_mock( + self, + "rust_uprev.fetch_rust_src_from_upstream", + ) + self.mock_subprocess_run = start_mock( + self, + "subprocess.run", + ) + + def test_already_present(self): + self.mock_mirror_has_file.return_value = True + rust_uprev.mirror_rust_source( + rust_uprev.RustVersion.parse("1.67.3"), + ) + self.mock_fetch_rust_src_from_upstream.assert_not_called() + self.mock_subprocess_run.assert_not_called() + + def test_fetch_and_upload(self): + self.mock_mirror_has_file.return_value = False + rust_uprev.mirror_rust_source( + rust_uprev.RustVersion.parse("1.67.3"), + ) + self.mock_fetch_rust_src_from_upstream.called_once() + self.mock_subprocess_run.has_calls( + [ + ( + [ + "gsutil", + "cp", + "-a", + "public-read", + "/fake/distdir/rustc-1.67.3-src.tar.gz", + "fakegs://fakemirror/rustc-1.67.3-src.tar.gz", + ] + ), + ] + ) + + class RemoveEbuildVersionTest(unittest.TestCase): """Tests for rust_uprev.remove_ebuild_version()""" @@ -221,26 +481,17 @@ class PrepareUprevTest(unittest.TestCase): """Tests for prepare_uprev step in rust_uprev""" def setUp(self): - self.bootstrap_version = rust_uprev.RustVersion(1, 1, 0) self.version_old = rust_uprev.RustVersion(1, 2, 3) self.version_new = rust_uprev.RustVersion(1, 3, 5) @mock.patch.object( rust_uprev, "find_ebuild_for_rust_version", - return_value="/path/to/ebuild", + return_value=Path("/path/to/ebuild"), ) - @mock.patch.object(rust_uprev, "find_ebuild_path") @mock.patch.object(rust_uprev, "get_command_output") - def test_success_with_template( - self, mock_command, mock_find_ebuild, _ebuild_for_version - ): - bootstrap_ebuild_path = Path( - "/path/to/rust-bootstrap/", - f"rust-bootstrap-{self.bootstrap_version}.ebuild", - ) - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, "/path/to/ebuild", self.bootstrap_version) + def test_success_with_template(self, mock_command, _ebuild_for_version): + expected = rust_uprev.PreparedUprev(self.version_old) actual = rust_uprev.prepare_uprev( rust_version=self.version_new, template=self.version_old ) @@ -252,11 +503,6 @@ class PrepareUprevTest(unittest.TestCase): "find_ebuild_for_rust_version", return_value="/path/to/ebuild", ) - @mock.patch.object( - rust_uprev, - "get_rust_bootstrap_version", - return_value=RustVersion(0, 41, 12), - ) @mock.patch.object(rust_uprev, "get_command_output") def test_return_none_with_template_larger_than_input( self, mock_command, *_args @@ -267,58 +513,70 @@ class PrepareUprevTest(unittest.TestCase): self.assertIsNone(ret) mock_command.assert_not_called() - @mock.patch.object(rust_uprev, "find_ebuild_path") - @mock.patch.object(os.path, "exists") - @mock.patch.object(rust_uprev, "get_command_output") - def test_success_without_template( - self, mock_command, mock_exists, mock_find_ebuild - ): - rust_ebuild_path = f"/path/to/rust/rust-{self.version_old}-r3.ebuild" - mock_command.return_value = rust_ebuild_path - bootstrap_ebuild_path = Path( - "/path/to/rust-bootstrap", - f"rust-bootstrap-{self.bootstrap_version}.ebuild", - ) - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, rust_ebuild_path, self.bootstrap_version) - actual = rust_uprev.prepare_uprev( - rust_version=self.version_new, template=None - ) - self.assertEqual(expected, actual) - mock_command.assert_called_once_with(["equery", "w", "rust"]) - mock_exists.assert_not_called() - - @mock.patch.object( - rust_uprev, - "get_rust_bootstrap_version", - return_value=RustVersion(0, 41, 12), - ) - @mock.patch.object(os.path, "exists") - @mock.patch.object(rust_uprev, "get_command_output") - def test_return_none_with_ebuild_larger_than_input( - self, mock_command, mock_exists, *_args - ): - mock_command.return_value = ( - f"/path/to/rust/rust-{self.version_new}.ebuild" - ) - ret = rust_uprev.prepare_uprev( - rust_version=self.version_old, template=None - ) - self.assertIsNone(ret) - mock_exists.assert_not_called() - def test_prepare_uprev_from_json(self): - ebuild_path = "/path/to/the/ebuild" - json_result = ( - list(self.version_new), - ebuild_path, - list(self.bootstrap_version), + json_result = (list(self.version_new),) + expected = rust_uprev.PreparedUprev( + self.version_new, ) - expected = (self.version_new, ebuild_path, self.bootstrap_version) actual = rust_uprev.prepare_uprev_from_json(json_result) self.assertEqual(expected, actual) +class ToggleProfileData(unittest.TestCase): + """Tests functionality to include or exclude profile data from SRC_URI.""" + + ebuild_with_profdata = """ +# Some text here. +INCLUDE_PROFDATA_IN_SRC_URI=yes +some code here +""" + + ebuild_without_profdata = """ +# Some text here. +INCLUDE_PROFDATA_IN_SRC_URI= +some code here +""" + + ebuild_unexpected_content = """ +# Does not contain OMIT_PROFDATA_FROM_SRC_URI assignment +""" + + def setUp(self): + self.mock_read_text = start_mock(self, "pathlib.Path.read_text") + + def test_turn_off_profdata(self): + # Test that a file with profdata on is rewritten to a file with + # profdata off. + self.mock_read_text.return_value = self.ebuild_with_profdata + ebuild_file = "/path/to/eclass/cros-rustc.eclass" + with mock.patch("pathlib.Path.write_text") as mock_write_text: + rust_uprev.set_include_profdata_src(ebuild_file, include=False) + mock_write_text.assert_called_once_with( + self.ebuild_without_profdata, encoding="utf-8" + ) + + def test_turn_on_profdata(self): + # Test that a file with profdata off is rewritten to a file with + # profdata on. + self.mock_read_text.return_value = self.ebuild_without_profdata + ebuild_file = "/path/to/eclass/cros-rustc.eclass" + with mock.patch("pathlib.Path.write_text") as mock_write_text: + rust_uprev.set_include_profdata_src(ebuild_file, include=True) + mock_write_text.assert_called_once_with( + self.ebuild_with_profdata, encoding="utf-8" + ) + + def test_turn_on_profdata_fails_if_no_assignment(self): + # Test that if the string the code expects to find is not found, + # this causes an exception and the file is not overwritten. + self.mock_read_text.return_value = self.ebuild_unexpected_content + ebuild_file = "/path/to/eclass/cros-rustc.eclass" + with mock.patch("pathlib.Path.write_text") as mock_write_text: + with self.assertRaises(Exception): + rust_uprev.set_include_profdata_src(ebuild_file, include=False) + mock_write_text.assert_not_called() + + class UpdateBootstrapVersionTest(unittest.TestCase): """Tests for update_bootstrap_version step in rust_uprev""" @@ -329,86 +587,34 @@ BOOTSTRAP_VERSION="1.2.0" BOOTSTRAP_VERSION="1.3.6" """ + def setUp(self): + self.mock_read_text = start_mock(self, "pathlib.Path.read_text") + def test_success(self): - mock_open = mock.mock_open(read_data=self.ebuild_file_before) + self.mock_read_text.return_value = self.ebuild_file_before # ebuild_file and new bootstrap version are deliberately different ebuild_file = "/path/to/rust/cros-rustc.eclass" - with mock.patch("builtins.open", mock_open): + with mock.patch("pathlib.Path.write_text") as mock_write_text: rust_uprev.update_bootstrap_version( ebuild_file, rust_uprev.RustVersion.parse("1.3.6") ) - mock_open.return_value.__enter__().write.assert_called_once_with( - self.ebuild_file_after - ) + mock_write_text.assert_called_once_with( + self.ebuild_file_after, encoding="utf-8" + ) def test_fail_when_ebuild_misses_a_variable(self): - mock_open = mock.mock_open(read_data="") + self.mock_read_text.return_value = "" ebuild_file = "/path/to/rust/rust-1.3.5.ebuild" - with mock.patch("builtins.open", mock_open): - with self.assertRaises(RuntimeError) as context: - rust_uprev.update_bootstrap_version( - ebuild_file, rust_uprev.RustVersion.parse("1.2.0") - ) + with self.assertRaises(RuntimeError) as context: + rust_uprev.update_bootstrap_version( + ebuild_file, rust_uprev.RustVersion.parse("1.2.0") + ) self.assertEqual( "BOOTSTRAP_VERSION not found in /path/to/rust/rust-1.3.5.ebuild", str(context.exception), ) -class UpdateManifestTest(unittest.TestCase): - """Tests for update_manifest step in rust_uprev""" - - @mock.patch.object(rust_uprev, "ebuild_actions") - def test_update_manifest(self, mock_run): - ebuild_file = Path("/path/to/rust/rust-1.1.1.ebuild") - rust_uprev.update_manifest(ebuild_file) - mock_run.assert_called_once_with("rust", ["manifest"]) - - -class UpdateBootstrapEbuildTest(unittest.TestCase): - """Tests for rust_uprev.update_bootstrap_ebuild()""" - - def test_update_bootstrap_ebuild(self): - # The update should do two things: - # 1. Create a copy of rust-bootstrap's ebuild with the new version number. - # 2. Add the old PV to RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE. - with tempfile.TemporaryDirectory() as tmpdir_str, mock.patch.object( - rust_uprev, "find_ebuild_path" - ) as mock_find_ebuild: - tmpdir = Path(tmpdir_str) - bootstrapdir = Path.joinpath(tmpdir, "rust-bootstrap") - bootstrapdir.mkdir() - old_ebuild = bootstrapdir.joinpath("rust-bootstrap-1.45.2.ebuild") - old_ebuild.write_text( - encoding="utf-8", - data=""" -some text -RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( -\t1.43.1 -\t1.44.1 -) -some more text -""", - ) - mock_find_ebuild.return_value = old_ebuild - rust_uprev.update_bootstrap_ebuild(rust_uprev.RustVersion(1, 46, 0)) - new_ebuild = bootstrapdir.joinpath("rust-bootstrap-1.46.0.ebuild") - self.assertTrue(new_ebuild.exists()) - text = new_ebuild.read_text() - self.assertEqual( - text, - """ -some text -RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( -\t1.43.1 -\t1.44.1 -\t1.45.2 -) -some more text -""", - ) - - class UpdateRustPackagesTests(unittest.TestCase): """Tests for update_rust_packages step.""" @@ -471,114 +677,49 @@ class RustUprevOtherStagesTests(unittest.TestCase): ) @mock.patch.object(shutil, "copyfile") - @mock.patch.object(os, "listdir") @mock.patch.object(subprocess, "check_call") - def test_copy_patches(self, mock_call, mock_ls, mock_copy): - mock_ls.return_value = [ - f"rust-{self.old_version}-patch-1.patch", - f"rust-{self.old_version}-patch-2-old.patch", - f"rust-{self.current_version}-patch-1.patch", - f"rust-{self.current_version}-patch-2-new.patch", - ] - rust_uprev.copy_patches( - rust_uprev.RUST_PATH, self.current_version, self.new_version - ) - mock_copy.assert_has_calls( - [ - mock.call( - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.current_version}-patch-1.patch", - ), - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.new_version}-patch-1.patch", - ), - ), - mock.call( - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.current_version}-patch-2-new.patch", - ), - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.new_version}-patch-2-new.patch", - ), - ), - ] + def test_create_rust_ebuild(self, mock_call, mock_copy): + template_ebuild = ( + rust_uprev.EBUILD_PREFIX + / f"dev-lang/rust/rust-{self.current_version}.ebuild" ) - mock_call.assert_called_once_with( - ["git", "add", f"rust-{self.new_version}-*.patch"], - cwd=rust_uprev.RUST_PATH.joinpath("files"), + new_ebuild = ( + rust_uprev.EBUILD_PREFIX + / f"dev-lang/rust/rust-{self.new_version}.ebuild" ) - - @mock.patch.object(shutil, "copyfile") - @mock.patch.object(subprocess, "check_call") - def test_create_rust_ebuild(self, mock_call, mock_copy): - template_ebuild = f"/path/to/rust-{self.current_version}-r2.ebuild" rust_uprev.create_ebuild( - template_ebuild, "dev-lang/rust", self.new_version + "dev-lang", "rust", self.current_version, self.new_version ) mock_copy.assert_called_once_with( template_ebuild, - rust_uprev.RUST_PATH.joinpath(f"rust-{self.new_version}.ebuild"), + new_ebuild, ) mock_call.assert_called_once_with( ["git", "add", f"rust-{self.new_version}.ebuild"], - cwd=rust_uprev.RUST_PATH, + cwd=new_ebuild.parent, ) @mock.patch.object(shutil, "copyfile") @mock.patch.object(subprocess, "check_call") def test_create_rust_host_ebuild(self, mock_call, mock_copy): - template_ebuild = f"/path/to/rust-host-{self.current_version}-r2.ebuild" + template_ebuild = ( + rust_uprev.EBUILD_PREFIX + / f"dev-lang/rust-host/rust-host-{self.current_version}.ebuild" + ) + new_ebuild = ( + rust_uprev.EBUILD_PREFIX + / f"dev-lang/rust-host/rust-host-{self.new_version}.ebuild" + ) rust_uprev.create_ebuild( - template_ebuild, "dev-lang/rust-host", self.new_version + "dev-lang", "rust-host", self.current_version, self.new_version ) mock_copy.assert_called_once_with( template_ebuild, - rust_uprev.EBUILD_PREFIX.joinpath( - f"dev-lang/rust-host/rust-host-{self.new_version}.ebuild" - ), + new_ebuild, ) mock_call.assert_called_once_with( ["git", "add", f"rust-host-{self.new_version}.ebuild"], - cwd=rust_uprev.EBUILD_PREFIX.joinpath("dev-lang/rust-host"), - ) - - @mock.patch.object(rust_uprev, "find_ebuild_for_package") - @mock.patch.object(subprocess, "check_call") - def test_remove_rust_bootstrap_version(self, mock_call, *_args): - bootstrap_path = os.path.join( - rust_uprev.RUST_PATH, "..", "rust-bootstrap" - ) - rust_uprev.remove_rust_bootstrap_version( - self.old_version, lambda *x: () - ) - mock_call.has_calls( - [ - [ - "git", - "rm", - os.path.join( - bootstrap_path, - "files", - f"rust-bootstrap-{self.old_version}-*.patch", - ), - ], - [ - "git", - "rm", - os.path.join( - bootstrap_path, - f"rust-bootstrap-{self.old_version}.ebuild", - ), - ], - ] + cwd=new_ebuild.parent, ) @mock.patch.object(subprocess, "check_call") @@ -641,25 +782,27 @@ class RustUprevOtherStagesTests(unittest.TestCase): ebuild_path.parent.joinpath(f"rust-{self.new_version}.ebuild"), ) - @mock.patch.object(os, "listdir") - def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls): + @mock.patch("rust_uprev.find_rust_versions") + def test_find_oldest_rust_version_pass(self, rust_versions): oldest_version_name = f"rust-{self.old_version}.ebuild" - mock_ls.return_value = [ - oldest_version_name, - f"rust-{self.current_version}.ebuild", - f"rust-{self.new_version}.ebuild", + rust_versions.return_value = [ + (self.old_version, oldest_version_name), + (self.current_version, f"rust-{self.current_version}.ebuild"), + (self.new_version, f"rust-{self.new_version}.ebuild"), ] - actual = rust_uprev.find_oldest_rust_version_in_chroot() + actual = rust_uprev.find_oldest_rust_version() expected = self.old_version self.assertEqual(expected, actual) - @mock.patch.object(os, "listdir") - def test_find_oldest_rust_version_in_chroot_fail_with_only_one_ebuild( - self, mock_ls + @mock.patch("rust_uprev.find_rust_versions") + def test_find_oldest_rust_version_fail_with_only_one_ebuild( + self, rust_versions ): - mock_ls.return_value = [f"rust-{self.new_version}.ebuild"] + rust_versions.return_value = [ + (self.new_version, f"rust-{self.new_version}.ebuild"), + ] with self.assertRaises(RuntimeError) as context: - rust_uprev.find_oldest_rust_version_in_chroot() + rust_uprev.find_oldest_rust_version() self.assertEqual( "Expect to find more than one Rust versions", str(context.exception) ) @@ -673,10 +816,8 @@ class RustUprevOtherStagesTests(unittest.TestCase): rust_uprev.EBUILD_PREFIX, f"rust-to-{self.new_version}" ) - @mock.patch.object(rust_uprev, "get_command_output") - @mock.patch.object(subprocess, "check_call") - def test_build_cross_compiler(self, mock_call, mock_output): - mock_output.return_value = f"rust-{self.new_version}.ebuild" + @mock.patch.object(rust_uprev, "run_in_chroot") + def test_build_cross_compiler(self, mock_run_in_chroot): cros_targets = [ "x86_64-cros-linux-gnu", "armv7a-cros-linux-gnueabihf", @@ -684,11 +825,13 @@ class RustUprevOtherStagesTests(unittest.TestCase): ] all_triples = ["x86_64-pc-linux-gnu"] + cros_targets rust_ebuild = "RUSTC_TARGET_TRIPLES=(" + "\n\t".join(all_triples) + ")" - mock_open = mock.mock_open(read_data=rust_ebuild) - with mock.patch("builtins.open", mock_open): - rust_uprev.build_cross_compiler() + with mock.patch("rust_uprev.find_ebuild_path") as mock_find_ebuild_path: + mock_path = mock.Mock() + mock_path.read_text.return_value = rust_ebuild + mock_find_ebuild_path.return_value = mock_path + rust_uprev.build_cross_compiler(rust_uprev.RustVersion(7, 3, 31)) - mock_call.assert_called_once_with( + mock_run_in_chroot.assert_called_once_with( ["sudo", "emerge", "-j", "-G"] + [f"cross-{x}/gcc" for x in cros_targets + ["arm-none-eabi"]] ) diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index dff239f3..ba760e78 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -94,7 +93,7 @@ class State(NamedTuple): def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: - """Parses `git ls-remote --tags` output into Rust stable release versions.""" + """Parses `git ls-remote --tags` output into Rust stable versions.""" refs_tags = "refs/tags/" for line in lines: _sha, tag = line.split(None, 1) @@ -236,14 +235,24 @@ def atomically_write_state(state_file: pathlib.Path, state: State) -> None: def file_bug(title: str, body: str) -> None: - """Files a bug against gbiv@ with the given title/body.""" - bugs.CreateNewBug( - bugs.WellKnownComponents.CrOSToolchainPublic, - title, - body, - # To either take or reassign depending on the rotation. - assignee="gbiv@google.com", + """Files update bugs with the given title/body.""" + # (component, optional_assignee) + targets = ( + (bugs.WellKnownComponents.CrOSToolchainPublic, "gbiv@google.com"), + # b/269170429: Some Android folks said they wanted this before, and + # figuring out the correct way to apply permissions has been a pain. No + # one seems to be missing these notifications & the Android Rust folks + # are keeping on top of their toolchain, so ignore this for now. + # (bugs.WellKnownComponents.AndroidRustToolchain, None), ) + for component, assignee in targets: + bugs.CreateNewBug( + component, + title, + body, + assignee, + parent_bug=bugs.RUST_MAINTENANCE_METABUG, + ) def maybe_compose_bug( @@ -256,8 +265,20 @@ def maybe_compose_bug( title = f"[Rust] Update to {newest_release}" body = ( - "A new release has been detected; we should probably roll to it. " - "Please see go/crostc-rust-rotation for who's turn it is." + "A new Rust stable release has been detected; we should probably roll " + "to it.\n" + "\n" + "The regression-from-stable-to-stable tag might be interesting to " + "keep an eye on: https://github.com/rust-lang/rust/labels/" + "regression-from-stable-to-stable\n" + "\n" + "If you notice any bugs or issues you'd like to share, please " + "also note them on go/shared-rust-update-notes.\n" + "\n" + "See go/crostc-rust-rotation for the current rotation schedule.\n" + "\n" + "For questions about this bot, please contact chromeos-toolchain@ and " + "CC gbiv@." ) return title, body @@ -270,7 +291,7 @@ def maybe_compose_email( return None subject_pieces = [] - body_pieces = [] + body_pieces: List[tiny_render.Piece] = [] # Separate the sections a bit for prettier output. if body_pieces: @@ -344,8 +365,8 @@ def main(argv: List[str]) -> None: last_gentoo_sha=most_recent_gentoo_commit, ), ) - # Running through this _should_ be a nop, but do it anyway. Should make any - # bugs more obvious on the first run of the script. + # Running through this _should_ be a nop, but do it anyway. Should make + # any bugs more obvious on the first run of the script. prior_state = read_state(state_file) logging.info("Last state was %r", prior_state) @@ -366,30 +387,30 @@ def main(argv: List[str]) -> None: if maybe_bug is None: logging.info("No bug to file") else: - title, body = maybe_bug + bug_title, bug_body = maybe_bug if opts.skip_side_effects: logging.info( "Skipping sending bug with title %r and contents\n%s", - title, - body, + bug_title, + bug_body, ) else: logging.info("Writing new bug") - file_bug(title, body) + file_bug(bug_title, bug_body) if maybe_email is None: logging.info("No email to send") else: - title, body = maybe_email + email_title, email_body = maybe_email if opts.skip_side_effects: logging.info( "Skipping sending email with title %r and contents\n%s", - title, - tiny_render.render_html_pieces(body), + email_title, + tiny_render.render_html_pieces(email_body), ) else: logging.info("Sending email") - send_email(title, body) + send_email(email_title, email_body) if opts.skip_state_update: logging.info("Skipping state update, as requested") @@ -408,4 +429,4 @@ def main(argv: List[str]) -> None: if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py index 1e6aec51..79006f35 100755 --- a/rust_tools/rust_watch_test.py +++ b/rust_tools/rust_watch_test.py @@ -139,6 +139,7 @@ class Test(unittest.TestCase): self.assertIsNone(rust_watch.maybe_compose_email(new_gentoo_commits=())) def test_compose_bug_creates_bugs_on_new_versions(self): + bug_body_start = "A new Rust stable release has been detected;" title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), @@ -147,7 +148,7 @@ class Test(unittest.TestCase): newest_release=rust_watch.RustReleaseVersion(1, 0, 1), ) self.assertEqual(title, "[Rust] Update to 1.0.1") - self.assertTrue(body.startswith("A new release has been detected;")) + self.assertTrue(body.startswith(bug_body_start)) title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( @@ -157,7 +158,7 @@ class Test(unittest.TestCase): newest_release=rust_watch.RustReleaseVersion(1, 1, 0), ) self.assertEqual(title, "[Rust] Update to 1.1.0") - self.assertTrue(body.startswith("A new release has been detected;")) + self.assertTrue(body.startswith(bug_body_start)) title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( @@ -167,7 +168,7 @@ class Test(unittest.TestCase): newest_release=rust_watch.RustReleaseVersion(2, 0, 0), ) self.assertEqual(title, "[Rust] Update to 2.0.0") - self.assertTrue(body.startswith("A new release has been detected;")) + self.assertTrue(body.startswith(bug_body_start)) def test_compose_bug_does_nothing_when_no_new_updates_exist(self): self.assertIsNone( |