diff options
Diffstat (limited to 'rust_tools/rust_watch.py')
| -rwxr-xr-x | rust_tools/rust_watch.py | 351 |
1 files changed, 351 insertions, 0 deletions
diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py new file mode 100755 index 00000000..b9ad7b82 --- /dev/null +++ b/rust_tools/rust_watch.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Checks for various upstream events with the Rust toolchain. + +Sends an email if something interesting (probably) happened. +""" + +# pylint: disable=cros-logging-import + +import argparse +import itertools +import json +import logging +import pathlib +import re +import shutil +import subprocess +import sys +import time +from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple + +from cros_utils import email_sender +from cros_utils import tiny_render + + +def gentoo_sha_to_link(sha: str) -> str: + """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" + return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}' + + +def send_email(subject: str, body: List[tiny_render.Piece]) -> None: + """Sends an email with the given title and body to... whoever cares.""" + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier='rust-watch', + well_known_recipients=['cros-team'], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) + + +class RustReleaseVersion(NamedTuple): + """Represents a version of Rust's stable compiler.""" + major: int + minor: int + patch: int + + @staticmethod + def from_string(version_string: str) -> 'RustReleaseVersion': + m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string) + if not m: + raise ValueError(f"{version_string!r} isn't a valid version string") + return RustReleaseVersion(*[int(x) for x in m.groups()]) + + def __str__(self) -> str: + return f'{self.major}.{self.minor}.{self.patch}' + + def to_json(self) -> str: + return str(self) + + @staticmethod + def from_json(s: str) -> 'RustReleaseVersion': + return RustReleaseVersion.from_string(s) + + +class State(NamedTuple): + """State that we keep around from run to run.""" + # The last Rust release tag that we've seen. + last_seen_release: RustReleaseVersion + + # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen + # that updates it. + last_gentoo_sha: str + + def to_json(self) -> Dict[str, Any]: + return { + 'last_seen_release': self.last_seen_release.to_json(), + 'last_gentoo_sha': self.last_gentoo_sha, + } + + @staticmethod + def from_json(s: Dict[str, Any]) -> 'State': + return State( + last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']), + last_gentoo_sha=s['last_gentoo_sha'], + ) + + +def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: + """Parses `git ls-remote --tags` output into Rust stable release versions.""" + refs_tags = 'refs/tags/' + for line in lines: + _sha, tag = line.split(None, 1) + tag = tag.strip() + # Each tag has an associated 'refs/tags/name^{}', which is the actual + # object that the tag points to. That's irrelevant to us. + if tag.endswith('^{}'): + continue + + if not tag.startswith(refs_tags): + continue + + short_tag = tag[len(refs_tags):] + # There are a few old versioning schemes. Ignore them. + if short_tag.startswith('0.') or short_tag.startswith('release-'): + continue + yield RustReleaseVersion.from_string(short_tag) + + +def fetch_most_recent_release() -> RustReleaseVersion: + """Fetches the most recent stable `rustc` version.""" + result = subprocess.run( + ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'], + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + tag_lines = result.stdout.strip().splitlines() + return max(parse_release_tags(tag_lines)) + + +class GitCommit(NamedTuple): + """Represents a single git commit.""" + sha: str + subject: str + + +def update_git_repo(git_dir: pathlib.Path) -> None: + """Updates the repo at `git_dir`, retrying a few times on failure.""" + for i in itertools.count(start=1): + result = subprocess.run( + ['git', 'fetch', 'origin'], + check=False, + cwd=str(git_dir), + stdin=None, + ) + + if not result.returncode: + break + + if i == 5: + # 5 attempts is too many. Something else may be wrong. + result.check_returncode() + + sleep_time = 60 * i + logging.error("Failed updating gentoo's repo; will try again in %ds...", + sleep_time) + time.sleep(sleep_time) + + +def get_new_gentoo_commits(git_dir: pathlib.Path, + most_recent_sha: str) -> List[GitCommit]: + """Gets commits to dev-lang/rust since `most_recent_sha`. + + Older commits come earlier in the returned list. + """ + commits = subprocess.run( + [ + 'git', + 'log', + '--format=%H %s', + f'{most_recent_sha}..origin/master', + '--', + 'dev-lang/rust', + ], + capture_output=True, + check=False, + cwd=str(git_dir), + encoding='utf-8', + ) + + if commits.returncode: + logging.error('Error getting new gentoo commits; stderr:\n%s', + commits.stderr) + commits.check_returncode() + + results = [] + for line in commits.stdout.strip().splitlines(): + sha, subject = line.strip().split(None, 1) + results.append(GitCommit(sha=sha, subject=subject)) + + # `git log` outputs things in newest -> oldest order. + results.reverse() + return results + + +def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: + """Sets up a gentoo git repo at the given directory. Returns HEAD.""" + subprocess.run( + [ + 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git', + str(git_dir) + ], + stdin=None, + check=True, + ) + + head_rev = subprocess.run( + ['git', 'rev-parse', 'HEAD'], + cwd=str(git_dir), + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + return head_rev.stdout.strip() + + +def read_state(state_file: pathlib.Path) -> State: + """Reads state from the given file.""" + with state_file.open(encoding='utf-8') as f: + return State.from_json(json.load(f)) + + +def atomically_write_state(state_file: pathlib.Path, state: State) -> None: + """Writes state to the given file.""" + temp_file = pathlib.Path(str(state_file) + '.new') + with temp_file.open('w', encoding='utf-8') as f: + json.dump(state.to_json(), f) + temp_file.rename(state_file) + + +def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, + new_gentoo_commits: List[GitCommit] + ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: + """Creates an email given our new state, if doing so is appropriate.""" + subject_pieces = [] + body_pieces = [] + + if newest_release > old_state.last_seen_release: + subject_pieces.append('new rustc release detected') + body_pieces.append(f'Rustc tag for v{newest_release} was found.') + + if new_gentoo_commits: + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] + + if len(new_gentoo_commits) == 1: + subject_pieces.append('new rust ebuild commit detected') + body_pieces.append('commit:') + else: + subject_pieces.append('new rust ebuild commits detected') + body_pieces.append('commits (newest first):') + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append([ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f': {commit.subject}', + ]) + + body_pieces.append(tiny_render.UnorderedList(commit_lines)) + + if not subject_pieces: + return None + + subject = '[rust-watch] ' + '; '.join(subject_pieces) + return subject, body_pieces + + +def main(argv: List[str]) -> None: + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--state_dir', required=True, help='Directory to store state in.') + parser.add_argument( + '--skip_email', action='store_true', help="Don't send an email.") + parser.add_argument( + '--skip_state_update', + action='store_true', + help="Don't update the state file. Doesn't apply to initial setup.") + opts = parser.parse_args(argv) + + state_dir = pathlib.Path(opts.state_dir) + state_file = state_dir / 'state.json' + gentoo_subdir = state_dir / 'upstream-gentoo' + if not state_file.exists(): + logging.info("state_dir isn't fully set up; doing that now.") + + # Could be in a partially set-up state. + if state_dir.exists(): + logging.info('incomplete state_dir detected; removing.') + shutil.rmtree(str(state_dir)) + + state_dir.mkdir(parents=True) + most_recent_release = fetch_most_recent_release() + most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=most_recent_gentoo_commit, + ), + ) + # Running through this _should_ be a nop, but do it anyway. Should make any + # bugs more obvious on the first run of the script. + + prior_state = read_state(state_file) + logging.info('Last state was %r', prior_state) + + most_recent_release = fetch_most_recent_release() + logging.info('Most recent Rust release is %s', most_recent_release) + + logging.info('Fetching new commits from Gentoo') + update_git_repo(gentoo_subdir) + new_commits = get_new_gentoo_commits(gentoo_subdir, + prior_state.last_gentoo_sha) + logging.info('New commits: %r', new_commits) + + maybe_email = maybe_compose_email(prior_state, most_recent_release, + new_commits) + + if maybe_email is None: + logging.info('No updates to send') + else: + title, body = maybe_email + if opts.skip_email: + logging.info('Skipping sending email with title %r and contents\n%s', + title, tiny_render.render_html_pieces(body)) + else: + logging.info('Sending email') + send_email(title, body) + + if opts.skip_state_update: + logging.info('Skipping state update, as requested') + return + + newest_sha = ( + new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=newest_sha, + ), + ) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) |