diff options
Diffstat (limited to 'rust_tools/rust_watch.py')
| -rwxr-xr-x | rust_tools/rust_watch.py | 617 |
1 files changed, 323 insertions, 294 deletions
diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index c347d2c6..dff239f3 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -21,362 +21,391 @@ import sys import time from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple -from cros_utils import bugs, email_sender, tiny_render +from cros_utils import bugs +from cros_utils import email_sender +from cros_utils import tiny_render def gentoo_sha_to_link(sha: str) -> str: - """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" - return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}' + """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" + return f"https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}" def send_email(subject: str, body: List[tiny_render.Piece]) -> None: - """Sends an email with the given title and body to... whoever cares.""" - email_sender.EmailSender().SendX20Email( - subject=subject, - identifier='rust-watch', - well_known_recipients=['cros-team'], - text_body=tiny_render.render_text_pieces(body), - html_body=tiny_render.render_html_pieces(body), - ) + """Sends an email with the given title and body to... whoever cares.""" + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier="rust-watch", + well_known_recipients=["cros-team"], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) class RustReleaseVersion(NamedTuple): - """Represents a version of Rust's stable compiler.""" - major: int - minor: int - patch: int + """Represents a version of Rust's stable compiler.""" + + major: int + minor: int + patch: int - @staticmethod - def from_string(version_string: str) -> 'RustReleaseVersion': - m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string) - if not m: - raise ValueError(f"{version_string!r} isn't a valid version string") - return RustReleaseVersion(*[int(x) for x in m.groups()]) + @staticmethod + def from_string(version_string: str) -> "RustReleaseVersion": + m = re.match(r"(\d+)\.(\d+)\.(\d+)", version_string) + if not m: + raise ValueError(f"{version_string!r} isn't a valid version string") + return RustReleaseVersion(*[int(x) for x in m.groups()]) - def __str__(self) -> str: - return f'{self.major}.{self.minor}.{self.patch}' + def __str__(self) -> str: + return f"{self.major}.{self.minor}.{self.patch}" - def to_json(self) -> str: - return str(self) + def to_json(self) -> str: + return str(self) - @staticmethod - def from_json(s: str) -> 'RustReleaseVersion': - return RustReleaseVersion.from_string(s) + @staticmethod + def from_json(s: str) -> "RustReleaseVersion": + return RustReleaseVersion.from_string(s) class State(NamedTuple): - """State that we keep around from run to run.""" - # The last Rust release tag that we've seen. - last_seen_release: RustReleaseVersion - - # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen - # that updates it. - last_gentoo_sha: str - - def to_json(self) -> Dict[str, Any]: - return { - 'last_seen_release': self.last_seen_release.to_json(), - 'last_gentoo_sha': self.last_gentoo_sha, - } - - @staticmethod - def from_json(s: Dict[str, Any]) -> 'State': - return State( - last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']), - last_gentoo_sha=s['last_gentoo_sha'], - ) + """State that we keep around from run to run.""" + + # The last Rust release tag that we've seen. + last_seen_release: RustReleaseVersion + + # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen + # that updates it. + last_gentoo_sha: str + + def to_json(self) -> Dict[str, Any]: + return { + "last_seen_release": self.last_seen_release.to_json(), + "last_gentoo_sha": self.last_gentoo_sha, + } + + @staticmethod + def from_json(s: Dict[str, Any]) -> "State": + return State( + last_seen_release=RustReleaseVersion.from_json( + s["last_seen_release"] + ), + last_gentoo_sha=s["last_gentoo_sha"], + ) def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: - """Parses `git ls-remote --tags` output into Rust stable release versions.""" - refs_tags = 'refs/tags/' - for line in lines: - _sha, tag = line.split(None, 1) - tag = tag.strip() - # Each tag has an associated 'refs/tags/name^{}', which is the actual - # object that the tag points to. That's irrelevant to us. - if tag.endswith('^{}'): - continue - - if not tag.startswith(refs_tags): - continue - - short_tag = tag[len(refs_tags):] - # There are a few old versioning schemes. Ignore them. - if short_tag.startswith('0.') or short_tag.startswith('release-'): - continue - yield RustReleaseVersion.from_string(short_tag) + """Parses `git ls-remote --tags` output into Rust stable release versions.""" + refs_tags = "refs/tags/" + for line in lines: + _sha, tag = line.split(None, 1) + tag = tag.strip() + # Each tag has an associated 'refs/tags/name^{}', which is the actual + # object that the tag points to. That's irrelevant to us. + if tag.endswith("^{}"): + continue + + if not tag.startswith(refs_tags): + continue + + short_tag = tag[len(refs_tags) :] + # There are a few old versioning schemes. Ignore them. + if short_tag.startswith("0.") or short_tag.startswith("release-"): + continue + yield RustReleaseVersion.from_string(short_tag) def fetch_most_recent_release() -> RustReleaseVersion: - """Fetches the most recent stable `rustc` version.""" - result = subprocess.run( - ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'], - check=True, - stdin=None, - capture_output=True, - encoding='utf-8', - ) - tag_lines = result.stdout.strip().splitlines() - return max(parse_release_tags(tag_lines)) + """Fetches the most recent stable `rustc` version.""" + result = subprocess.run( + ["git", "ls-remote", "--tags", "https://github.com/rust-lang/rust"], + check=True, + stdin=None, + capture_output=True, + encoding="utf-8", + ) + tag_lines = result.stdout.strip().splitlines() + return max(parse_release_tags(tag_lines)) class GitCommit(NamedTuple): - """Represents a single git commit.""" - sha: str - subject: str + """Represents a single git commit.""" + + sha: str + subject: str def update_git_repo(git_dir: pathlib.Path) -> None: - """Updates the repo at `git_dir`, retrying a few times on failure.""" - for i in itertools.count(start=1): - result = subprocess.run( - ['git', 'fetch', 'origin'], + """Updates the repo at `git_dir`, retrying a few times on failure.""" + for i in itertools.count(start=1): + result = subprocess.run( + ["git", "fetch", "origin"], + check=False, + cwd=str(git_dir), + stdin=None, + ) + + if not result.returncode: + break + + if i == 5: + # 5 attempts is too many. Something else may be wrong. + result.check_returncode() + + sleep_time = 60 * i + logging.error( + "Failed updating gentoo's repo; will try again in %ds...", + sleep_time, + ) + time.sleep(sleep_time) + + +def get_new_gentoo_commits( + git_dir: pathlib.Path, most_recent_sha: str +) -> List[GitCommit]: + """Gets commits to dev-lang/rust since `most_recent_sha`. + + Older commits come earlier in the returned list. + """ + commits = subprocess.run( + [ + "git", + "log", + "--format=%H %s", + f"{most_recent_sha}..origin/master", # nocheck + "--", + "dev-lang/rust", + ], + capture_output=True, check=False, cwd=str(git_dir), - stdin=None, + encoding="utf-8", ) - if not result.returncode: - break - - if i == 5: - # 5 attempts is too many. Something else may be wrong. - result.check_returncode() - - sleep_time = 60 * i - logging.error("Failed updating gentoo's repo; will try again in %ds...", - sleep_time) - time.sleep(sleep_time) - - -def get_new_gentoo_commits(git_dir: pathlib.Path, - most_recent_sha: str) -> List[GitCommit]: - """Gets commits to dev-lang/rust since `most_recent_sha`. - - Older commits come earlier in the returned list. - """ - commits = subprocess.run( - [ - 'git', - 'log', - '--format=%H %s', - f'{most_recent_sha}..origin/master', # nocheck - '--', - 'dev-lang/rust', - ], - capture_output=True, - check=False, - cwd=str(git_dir), - encoding='utf-8', - ) - - if commits.returncode: - logging.error('Error getting new gentoo commits; stderr:\n%s', - commits.stderr) - commits.check_returncode() - - results = [] - for line in commits.stdout.strip().splitlines(): - sha, subject = line.strip().split(None, 1) - results.append(GitCommit(sha=sha, subject=subject)) - - # `git log` outputs things in newest -> oldest order. - results.reverse() - return results + if commits.returncode: + logging.error( + "Error getting new gentoo commits; stderr:\n%s", commits.stderr + ) + commits.check_returncode() + + results = [] + for line in commits.stdout.strip().splitlines(): + sha, subject = line.strip().split(None, 1) + results.append(GitCommit(sha=sha, subject=subject)) + + # `git log` outputs things in newest -> oldest order. + results.reverse() + return results def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: - """Sets up a gentoo git repo at the given directory. Returns HEAD.""" - subprocess.run( - [ - 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git', - str(git_dir) - ], - stdin=None, - check=True, - ) - - head_rev = subprocess.run( - ['git', 'rev-parse', 'HEAD'], - cwd=str(git_dir), - check=True, - stdin=None, - capture_output=True, - encoding='utf-8', - ) - return head_rev.stdout.strip() + """Sets up a gentoo git repo at the given directory. Returns HEAD.""" + subprocess.run( + [ + "git", + "clone", + "https://anongit.gentoo.org/git/repo/gentoo.git", + str(git_dir), + ], + stdin=None, + check=True, + ) + + head_rev = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=str(git_dir), + check=True, + stdin=None, + capture_output=True, + encoding="utf-8", + ) + return head_rev.stdout.strip() def read_state(state_file: pathlib.Path) -> State: - """Reads state from the given file.""" - with state_file.open(encoding='utf-8') as f: - return State.from_json(json.load(f)) + """Reads state from the given file.""" + with state_file.open(encoding="utf-8") as f: + return State.from_json(json.load(f)) def atomically_write_state(state_file: pathlib.Path, state: State) -> None: - """Writes state to the given file.""" - temp_file = pathlib.Path(str(state_file) + '.new') - with temp_file.open('w', encoding='utf-8') as f: - json.dump(state.to_json(), f) - temp_file.rename(state_file) + """Writes state to the given file.""" + temp_file = pathlib.Path(str(state_file) + ".new") + with temp_file.open("w", encoding="utf-8") as f: + json.dump(state.to_json(), f) + temp_file.rename(state_file) def file_bug(title: str, body: str) -> None: - """Files a bug against gbiv@ with the given title/body.""" - bugs.CreateNewBug( - bugs.WellKnownComponents.CrOSToolchainPublic, - title, - body, - # To either take or reassign depending on the rotation. - assignee='gbiv@google.com', - ) + """Files a bug against gbiv@ with the given title/body.""" + bugs.CreateNewBug( + bugs.WellKnownComponents.CrOSToolchainPublic, + title, + body, + # To either take or reassign depending on the rotation. + assignee="gbiv@google.com", + ) def maybe_compose_bug( old_state: State, newest_release: RustReleaseVersion, ) -> Optional[Tuple[str, str]]: - """Creates a bug to file about the new release, if doing is desired.""" - if newest_release == old_state.last_seen_release: - return None - - title = f'[Rust] Update to {newest_release}' - body = ('A new release has been detected; we should probably roll to it. ' - "Please see go/crostc-rust-rotation for who's turn it is.") - return title, body + """Creates a bug to file about the new release, if doing is desired.""" + if newest_release == old_state.last_seen_release: + return None + + title = f"[Rust] Update to {newest_release}" + body = ( + "A new release has been detected; we should probably roll to it. " + "Please see go/crostc-rust-rotation for who's turn it is." + ) + return title, body def maybe_compose_email( - new_gentoo_commits: List[GitCommit] + new_gentoo_commits: List[GitCommit], ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: - """Creates an email given our new state, if doing so is appropriate.""" - if not new_gentoo_commits: - return None - - subject_pieces = [] - body_pieces = [] - - # Separate the sections a bit for prettier output. - if body_pieces: - body_pieces += [tiny_render.line_break, tiny_render.line_break] - - if len(new_gentoo_commits) == 1: - subject_pieces.append('new rust ebuild commit detected') - body_pieces.append('commit:') - else: - subject_pieces.append('new rust ebuild commits detected') - body_pieces.append('commits (newest first):') - - commit_lines = [] - for commit in new_gentoo_commits: - commit_lines.append([ - tiny_render.Link( - gentoo_sha_to_link(commit.sha), - commit.sha[:12], - ), - f': {commit.subject}', - ]) + """Creates an email given our new state, if doing so is appropriate.""" + if not new_gentoo_commits: + return None + + subject_pieces = [] + body_pieces = [] + + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] + + if len(new_gentoo_commits) == 1: + subject_pieces.append("new rust ebuild commit detected") + body_pieces.append("commit:") + else: + subject_pieces.append("new rust ebuild commits detected") + body_pieces.append("commits (newest first):") + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append( + [ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f": {commit.subject}", + ] + ) - body_pieces.append(tiny_render.UnorderedList(commit_lines)) + body_pieces.append(tiny_render.UnorderedList(commit_lines)) - subject = '[rust-watch] ' + '; '.join(subject_pieces) - return subject, body_pieces + subject = "[rust-watch] " + "; ".join(subject_pieces) + return subject, body_pieces def main(argv: List[str]) -> None: - logging.basicConfig(level=logging.INFO) - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--state_dir', - required=True, - help='Directory to store state in.') - parser.add_argument('--skip_side_effects', - action='store_true', - help="Don't send an email or file a bug.") - parser.add_argument( - '--skip_state_update', - action='store_true', - help="Don't update the state file. Doesn't apply to initial setup.") - opts = parser.parse_args(argv) - - state_dir = pathlib.Path(opts.state_dir) - state_file = state_dir / 'state.json' - gentoo_subdir = state_dir / 'upstream-gentoo' - if not state_file.exists(): - logging.info("state_dir isn't fully set up; doing that now.") - - # Could be in a partially set-up state. - if state_dir.exists(): - logging.info('incomplete state_dir detected; removing.') - shutil.rmtree(str(state_dir)) - - state_dir.mkdir(parents=True) + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--state_dir", required=True, help="Directory to store state in." + ) + parser.add_argument( + "--skip_side_effects", + action="store_true", + help="Don't send an email or file a bug.", + ) + parser.add_argument( + "--skip_state_update", + action="store_true", + help="Don't update the state file. Doesn't apply to initial setup.", + ) + opts = parser.parse_args(argv) + + state_dir = pathlib.Path(opts.state_dir) + state_file = state_dir / "state.json" + gentoo_subdir = state_dir / "upstream-gentoo" + if not state_file.exists(): + logging.info("state_dir isn't fully set up; doing that now.") + + # Could be in a partially set-up state. + if state_dir.exists(): + logging.info("incomplete state_dir detected; removing.") + shutil.rmtree(str(state_dir)) + + state_dir.mkdir(parents=True) + most_recent_release = fetch_most_recent_release() + most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=most_recent_gentoo_commit, + ), + ) + # Running through this _should_ be a nop, but do it anyway. Should make any + # bugs more obvious on the first run of the script. + + prior_state = read_state(state_file) + logging.info("Last state was %r", prior_state) + most_recent_release = fetch_most_recent_release() - most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + logging.info("Most recent Rust release is %s", most_recent_release) + + logging.info("Fetching new commits from Gentoo") + update_git_repo(gentoo_subdir) + new_commits = get_new_gentoo_commits( + gentoo_subdir, prior_state.last_gentoo_sha + ) + logging.info("New commits: %r", new_commits) + + maybe_bug = maybe_compose_bug(prior_state, most_recent_release) + maybe_email = maybe_compose_email(new_commits) + + if maybe_bug is None: + logging.info("No bug to file") + else: + title, body = maybe_bug + if opts.skip_side_effects: + logging.info( + "Skipping sending bug with title %r and contents\n%s", + title, + body, + ) + else: + logging.info("Writing new bug") + file_bug(title, body) + + if maybe_email is None: + logging.info("No email to send") + else: + title, body = maybe_email + if opts.skip_side_effects: + logging.info( + "Skipping sending email with title %r and contents\n%s", + title, + tiny_render.render_html_pieces(body), + ) + else: + logging.info("Sending email") + send_email(title, body) + + if opts.skip_state_update: + logging.info("Skipping state update, as requested") + return + + newest_sha = ( + new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha + ) atomically_write_state( state_file, State( last_seen_release=most_recent_release, - last_gentoo_sha=most_recent_gentoo_commit, + last_gentoo_sha=newest_sha, ), ) - # Running through this _should_ be a nop, but do it anyway. Should make any - # bugs more obvious on the first run of the script. - - prior_state = read_state(state_file) - logging.info('Last state was %r', prior_state) - - most_recent_release = fetch_most_recent_release() - logging.info('Most recent Rust release is %s', most_recent_release) - - logging.info('Fetching new commits from Gentoo') - update_git_repo(gentoo_subdir) - new_commits = get_new_gentoo_commits(gentoo_subdir, - prior_state.last_gentoo_sha) - logging.info('New commits: %r', new_commits) - - maybe_bug = maybe_compose_bug(prior_state, most_recent_release) - maybe_email = maybe_compose_email(new_commits) - - if maybe_bug is None: - logging.info('No bug to file') - else: - title, body = maybe_bug - if opts.skip_side_effects: - logging.info('Skipping sending bug with title %r and contents\n%s', - title, body) - else: - logging.info('Writing new bug') - file_bug(title, body) - - if maybe_email is None: - logging.info('No email to send') - else: - title, body = maybe_email - if opts.skip_side_effects: - logging.info('Skipping sending email with title %r and contents\n%s', - title, tiny_render.render_html_pieces(body)) - else: - logging.info('Sending email') - send_email(title, body) - - if opts.skip_state_update: - logging.info('Skipping state update, as requested') - return - - newest_sha = (new_commits[-1].sha - if new_commits else prior_state.last_gentoo_sha) - atomically_write_state( - state_file, - State( - last_seen_release=most_recent_release, - last_gentoo_sha=newest_sha, - ), - ) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) |