diff options
author | George Burgess IV <gbiv@google.com> | 2020-07-22 15:48:59 -0700 |
---|---|---|
committer | George Burgess <gbiv@chromium.org> | 2020-07-24 18:09:04 +0000 |
commit | 120da3fec7feb6e2f85c0443406cbf753b16b448 (patch) | |
tree | fa99452efc86a823e4280072e10dd1fc3f2df87a | |
parent | cb46500d29085ee65b524981f0fbf4af3badc0bb (diff) | |
download | toolchain-utils-120da3fec7feb6e2f85c0443406cbf753b16b448.tar.gz |
rust_tools: add a monitor for upstream gentoo commits + rust releases
Apparently it's useful to watch upstream Gentoo's patches to
dev-lang/rust, and it'd be nice in general if we were pinged for new
Rust stable releases.
This CL adds a tool to handle the alerting of all of that for us.
BUG=None
TEST=Ran in a few different ways with `--skip_email`
Change-Id: Ie972872d597870c715266f88df50812b55e30c75
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2314745
Reviewed-by: Tiancong Wang <tcwang@google.com>
Tested-by: George Burgess <gbiv@chromium.org>
-rwxr-xr-x | rust_tools/rust_watch.py | 351 | ||||
-rwxr-xr-x | rust_tools/rust_watch_test.py | 202 |
2 files changed, 553 insertions, 0 deletions
diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py new file mode 100755 index 00000000..b9ad7b82 --- /dev/null +++ b/rust_tools/rust_watch.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Checks for various upstream events with the Rust toolchain. + +Sends an email if something interesting (probably) happened. +""" + +# pylint: disable=cros-logging-import + +import argparse +import itertools +import json +import logging +import pathlib +import re +import shutil +import subprocess +import sys +import time +from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple + +from cros_utils import email_sender +from cros_utils import tiny_render + + +def gentoo_sha_to_link(sha: str) -> str: + """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" + return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}' + + +def send_email(subject: str, body: List[tiny_render.Piece]) -> None: + """Sends an email with the given title and body to... whoever cares.""" + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier='rust-watch', + well_known_recipients=['cros-team'], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) + + +class RustReleaseVersion(NamedTuple): + """Represents a version of Rust's stable compiler.""" + major: int + minor: int + patch: int + + @staticmethod + def from_string(version_string: str) -> 'RustReleaseVersion': + m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string) + if not m: + raise ValueError(f"{version_string!r} isn't a valid version string") + return RustReleaseVersion(*[int(x) for x in m.groups()]) + + def __str__(self) -> str: + return f'{self.major}.{self.minor}.{self.patch}' + + def to_json(self) -> str: + return str(self) + + @staticmethod + def from_json(s: str) -> 'RustReleaseVersion': + return RustReleaseVersion.from_string(s) + + +class State(NamedTuple): + """State that we keep around from run to run.""" + # The last Rust release tag that we've seen. + last_seen_release: RustReleaseVersion + + # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen + # that updates it. + last_gentoo_sha: str + + def to_json(self) -> Dict[str, Any]: + return { + 'last_seen_release': self.last_seen_release.to_json(), + 'last_gentoo_sha': self.last_gentoo_sha, + } + + @staticmethod + def from_json(s: Dict[str, Any]) -> 'State': + return State( + last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']), + last_gentoo_sha=s['last_gentoo_sha'], + ) + + +def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: + """Parses `git ls-remote --tags` output into Rust stable release versions.""" + refs_tags = 'refs/tags/' + for line in lines: + _sha, tag = line.split(None, 1) + tag = tag.strip() + # Each tag has an associated 'refs/tags/name^{}', which is the actual + # object that the tag points to. That's irrelevant to us. + if tag.endswith('^{}'): + continue + + if not tag.startswith(refs_tags): + continue + + short_tag = tag[len(refs_tags):] + # There are a few old versioning schemes. Ignore them. + if short_tag.startswith('0.') or short_tag.startswith('release-'): + continue + yield RustReleaseVersion.from_string(short_tag) + + +def fetch_most_recent_release() -> RustReleaseVersion: + """Fetches the most recent stable `rustc` version.""" + result = subprocess.run( + ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'], + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + tag_lines = result.stdout.strip().splitlines() + return max(parse_release_tags(tag_lines)) + + +class GitCommit(NamedTuple): + """Represents a single git commit.""" + sha: str + subject: str + + +def update_git_repo(git_dir: pathlib.Path) -> None: + """Updates the repo at `git_dir`, retrying a few times on failure.""" + for i in itertools.count(start=1): + result = subprocess.run( + ['git', 'fetch', 'origin'], + check=False, + cwd=str(git_dir), + stdin=None, + ) + + if not result.returncode: + break + + if i == 5: + # 5 attempts is too many. Something else may be wrong. + result.check_returncode() + + sleep_time = 60 * i + logging.error("Failed updating gentoo's repo; will try again in %ds...", + sleep_time) + time.sleep(sleep_time) + + +def get_new_gentoo_commits(git_dir: pathlib.Path, + most_recent_sha: str) -> List[GitCommit]: + """Gets commits to dev-lang/rust since `most_recent_sha`. + + Older commits come earlier in the returned list. + """ + commits = subprocess.run( + [ + 'git', + 'log', + '--format=%H %s', + f'{most_recent_sha}..origin/master', + '--', + 'dev-lang/rust', + ], + capture_output=True, + check=False, + cwd=str(git_dir), + encoding='utf-8', + ) + + if commits.returncode: + logging.error('Error getting new gentoo commits; stderr:\n%s', + commits.stderr) + commits.check_returncode() + + results = [] + for line in commits.stdout.strip().splitlines(): + sha, subject = line.strip().split(None, 1) + results.append(GitCommit(sha=sha, subject=subject)) + + # `git log` outputs things in newest -> oldest order. + results.reverse() + return results + + +def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: + """Sets up a gentoo git repo at the given directory. Returns HEAD.""" + subprocess.run( + [ + 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git', + str(git_dir) + ], + stdin=None, + check=True, + ) + + head_rev = subprocess.run( + ['git', 'rev-parse', 'HEAD'], + cwd=str(git_dir), + check=True, + stdin=None, + capture_output=True, + encoding='utf-8', + ) + return head_rev.stdout.strip() + + +def read_state(state_file: pathlib.Path) -> State: + """Reads state from the given file.""" + with state_file.open(encoding='utf-8') as f: + return State.from_json(json.load(f)) + + +def atomically_write_state(state_file: pathlib.Path, state: State) -> None: + """Writes state to the given file.""" + temp_file = pathlib.Path(str(state_file) + '.new') + with temp_file.open('w', encoding='utf-8') as f: + json.dump(state.to_json(), f) + temp_file.rename(state_file) + + +def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, + new_gentoo_commits: List[GitCommit] + ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: + """Creates an email given our new state, if doing so is appropriate.""" + subject_pieces = [] + body_pieces = [] + + if newest_release > old_state.last_seen_release: + subject_pieces.append('new rustc release detected') + body_pieces.append(f'Rustc tag for v{newest_release} was found.') + + if new_gentoo_commits: + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] + + if len(new_gentoo_commits) == 1: + subject_pieces.append('new rust ebuild commit detected') + body_pieces.append('commit:') + else: + subject_pieces.append('new rust ebuild commits detected') + body_pieces.append('commits (newest first):') + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append([ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f': {commit.subject}', + ]) + + body_pieces.append(tiny_render.UnorderedList(commit_lines)) + + if not subject_pieces: + return None + + subject = '[rust-watch] ' + '; '.join(subject_pieces) + return subject, body_pieces + + +def main(argv: List[str]) -> None: + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--state_dir', required=True, help='Directory to store state in.') + parser.add_argument( + '--skip_email', action='store_true', help="Don't send an email.") + parser.add_argument( + '--skip_state_update', + action='store_true', + help="Don't update the state file. Doesn't apply to initial setup.") + opts = parser.parse_args(argv) + + state_dir = pathlib.Path(opts.state_dir) + state_file = state_dir / 'state.json' + gentoo_subdir = state_dir / 'upstream-gentoo' + if not state_file.exists(): + logging.info("state_dir isn't fully set up; doing that now.") + + # Could be in a partially set-up state. + if state_dir.exists(): + logging.info('incomplete state_dir detected; removing.') + shutil.rmtree(str(state_dir)) + + state_dir.mkdir(parents=True) + most_recent_release = fetch_most_recent_release() + most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=most_recent_gentoo_commit, + ), + ) + # Running through this _should_ be a nop, but do it anyway. Should make any + # bugs more obvious on the first run of the script. + + prior_state = read_state(state_file) + logging.info('Last state was %r', prior_state) + + most_recent_release = fetch_most_recent_release() + logging.info('Most recent Rust release is %s', most_recent_release) + + logging.info('Fetching new commits from Gentoo') + update_git_repo(gentoo_subdir) + new_commits = get_new_gentoo_commits(gentoo_subdir, + prior_state.last_gentoo_sha) + logging.info('New commits: %r', new_commits) + + maybe_email = maybe_compose_email(prior_state, most_recent_release, + new_commits) + + if maybe_email is None: + logging.info('No updates to send') + else: + title, body = maybe_email + if opts.skip_email: + logging.info('Skipping sending email with title %r and contents\n%s', + title, tiny_render.render_html_pieces(body)) + else: + logging.info('Sending email') + send_email(title, body) + + if opts.skip_state_update: + logging.info('Skipping state update, as requested') + return + + newest_sha = ( + new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha) + atomically_write_state( + state_file, + State( + last_seen_release=most_recent_release, + last_gentoo_sha=newest_sha, + ), + ) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py new file mode 100755 index 00000000..97d111fc --- /dev/null +++ b/rust_tools/rust_watch_test.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for rust_watch.py.""" + +# pylint: disable=cros-logging-import + +import logging +import pathlib +import subprocess +import time +import unittest +import unittest.mock + +import rust_watch +from cros_utils import tiny_render + + +class Test(unittest.TestCase): + """Tests.""" + + def _silence_logs(self): + """Silences all log output until the end of the current test.""" + + def should_log(_record): + return 0 + + logger = logging.root + logger.addFilter(should_log) + self.addCleanup(logger.removeFilter, should_log) + + def test_release_version_parsing(self): + self.assertEqual( + rust_watch.RustReleaseVersion.from_string('1.2.3'), + rust_watch.RustReleaseVersion(1, 2, 3), + ) + + def test_release_version_json_round_trips(self): + ver = rust_watch.RustReleaseVersion(1, 2, 3) + self.assertEqual( + rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver) + + def test_state_json_round_trips(self): + state = rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 2, 3), + last_gentoo_sha='abc123', + ) + + self.assertEqual(rust_watch.State.from_json(state.to_json()), state) + + @unittest.mock.patch.object(subprocess, 'run') + @unittest.mock.patch.object(time, 'sleep') + def test_update_git_repo_tries_again_on_failure(self, sleep_mock, run_mock): + self._silence_logs() + + oh_no_error = ValueError('oh no') + + def check_returncode(): + raise oh_no_error + + run_call_count = 0 + + def run_sideeffect(*_args, **_kwargs): + nonlocal run_call_count + run_call_count += 1 + result = unittest.mock.Mock() + result.returncode = 1 + result.check_returncode = check_returncode + return result + + run_mock.side_effect = run_sideeffect + + with self.assertRaises(ValueError) as raised: + rust_watch.update_git_repo(pathlib.Path('/does/not/exist/at/all')) + + self.assertIs(raised.exception, oh_no_error) + self.assertEqual(run_call_count, 5) + + sleep_timings = [unittest.mock.call(60 * i) for i in range(1, 5)] + self.assertEqual(sleep_mock.mock_calls, sleep_timings) + + @unittest.mock.patch.object(subprocess, 'run') + def test_get_new_gentoo_commits_functions(self, run_mock): + returned = unittest.mock.Mock() + returned.returncode = 0 + returned.stdout = '\n'.join(( + 'abc123 newer commit', + 'abcdef and an older commit', + )) + run_mock.return_value = returned + results = rust_watch.get_new_gentoo_commits( + pathlib.Path('/does/not/exist/at/all'), 'defabc') + self.assertEqual(results, [ + rust_watch.GitCommit('abcdef', 'and an older commit'), + rust_watch.GitCommit('abc123', 'newer commit'), + ]) + + def test_compose_email_on_a_new_release(self): + new_release = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[], + ) + + self.assertEqual(new_release, ('[rust-watch] new rustc release detected', + ['Rustc tag for v1.1.0 was found.'])) + + def test_compose_email_on_a_new_gentoo_commit(self): + sha_a = 'a' * 40 + new_commit = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], + ) + + self.assertEqual(new_commit, + ('[rust-watch] new rust ebuild commit detected', [ + 'commit:', + tiny_render.UnorderedList([ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ': summary_a', + ], + ]) + ])) + + def test_compose_email_on_multiple_events(self): + sha_a = 'a' * 40 + new_commit_and_release = rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 1, 0), + new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', + ), + ], + ) + + self.assertEqual( + new_commit_and_release, + ('[rust-watch] new rustc release detected; new rust ebuild commit ' + 'detected', [ + 'Rustc tag for v1.1.0 was found.', + tiny_render.line_break, + tiny_render.line_break, + 'commit:', + tiny_render.UnorderedList([ + [ + tiny_render.Link( + rust_watch.gentoo_sha_to_link(sha_a), + sha_a[:12], + ), + ': summary_a', + ], + ]), + ])) + + def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + self.assertIsNone( + rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], + )) + + self.assertIsNone( + rust_watch.maybe_compose_email( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 0), + new_gentoo_commits=[], + )) + + +if __name__ == '__main__': + unittest.main() |