aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGeorge Burgess IV <gbiv@google.com>2020-07-22 15:48:59 -0700
committerGeorge Burgess <gbiv@chromium.org>2020-07-24 18:09:04 +0000
commit120da3fec7feb6e2f85c0443406cbf753b16b448 (patch)
treefa99452efc86a823e4280072e10dd1fc3f2df87a
parentcb46500d29085ee65b524981f0fbf4af3badc0bb (diff)
downloadtoolchain-utils-120da3fec7feb6e2f85c0443406cbf753b16b448.tar.gz
rust_tools: add a monitor for upstream gentoo commits + rust releases
Apparently it's useful to watch upstream Gentoo's patches to dev-lang/rust, and it'd be nice in general if we were pinged for new Rust stable releases. This CL adds a tool to handle the alerting of all of that for us. BUG=None TEST=Ran in a few different ways with `--skip_email` Change-Id: Ie972872d597870c715266f88df50812b55e30c75 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2314745 Reviewed-by: Tiancong Wang <tcwang@google.com> Tested-by: George Burgess <gbiv@chromium.org>
-rwxr-xr-xrust_tools/rust_watch.py351
-rwxr-xr-xrust_tools/rust_watch_test.py202
2 files changed, 553 insertions, 0 deletions
diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py
new file mode 100755
index 00000000..b9ad7b82
--- /dev/null
+++ b/rust_tools/rust_watch.py
@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Checks for various upstream events with the Rust toolchain.
+
+Sends an email if something interesting (probably) happened.
+"""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import itertools
+import json
+import logging
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+import time
+from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple
+
+from cros_utils import email_sender
+from cros_utils import tiny_render
+
+
+def gentoo_sha_to_link(sha: str) -> str:
+ """Gets a URL to a webpage that shows the Gentoo commit at `sha`."""
+ return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}'
+
+
+def send_email(subject: str, body: List[tiny_render.Piece]) -> None:
+ """Sends an email with the given title and body to... whoever cares."""
+ email_sender.EmailSender().SendX20Email(
+ subject=subject,
+ identifier='rust-watch',
+ well_known_recipients=['cros-team'],
+ text_body=tiny_render.render_text_pieces(body),
+ html_body=tiny_render.render_html_pieces(body),
+ )
+
+
+class RustReleaseVersion(NamedTuple):
+ """Represents a version of Rust's stable compiler."""
+ major: int
+ minor: int
+ patch: int
+
+ @staticmethod
+ def from_string(version_string: str) -> 'RustReleaseVersion':
+ m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string)
+ if not m:
+ raise ValueError(f"{version_string!r} isn't a valid version string")
+ return RustReleaseVersion(*[int(x) for x in m.groups()])
+
+ def __str__(self) -> str:
+ return f'{self.major}.{self.minor}.{self.patch}'
+
+ def to_json(self) -> str:
+ return str(self)
+
+ @staticmethod
+ def from_json(s: str) -> 'RustReleaseVersion':
+ return RustReleaseVersion.from_string(s)
+
+
+class State(NamedTuple):
+ """State that we keep around from run to run."""
+ # The last Rust release tag that we've seen.
+ last_seen_release: RustReleaseVersion
+
+ # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen
+ # that updates it.
+ last_gentoo_sha: str
+
+ def to_json(self) -> Dict[str, Any]:
+ return {
+ 'last_seen_release': self.last_seen_release.to_json(),
+ 'last_gentoo_sha': self.last_gentoo_sha,
+ }
+
+ @staticmethod
+ def from_json(s: Dict[str, Any]) -> 'State':
+ return State(
+ last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']),
+ last_gentoo_sha=s['last_gentoo_sha'],
+ )
+
+
+def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]:
+ """Parses `git ls-remote --tags` output into Rust stable release versions."""
+ refs_tags = 'refs/tags/'
+ for line in lines:
+ _sha, tag = line.split(None, 1)
+ tag = tag.strip()
+ # Each tag has an associated 'refs/tags/name^{}', which is the actual
+ # object that the tag points to. That's irrelevant to us.
+ if tag.endswith('^{}'):
+ continue
+
+ if not tag.startswith(refs_tags):
+ continue
+
+ short_tag = tag[len(refs_tags):]
+ # There are a few old versioning schemes. Ignore them.
+ if short_tag.startswith('0.') or short_tag.startswith('release-'):
+ continue
+ yield RustReleaseVersion.from_string(short_tag)
+
+
+def fetch_most_recent_release() -> RustReleaseVersion:
+ """Fetches the most recent stable `rustc` version."""
+ result = subprocess.run(
+ ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'],
+ check=True,
+ stdin=None,
+ capture_output=True,
+ encoding='utf-8',
+ )
+ tag_lines = result.stdout.strip().splitlines()
+ return max(parse_release_tags(tag_lines))
+
+
+class GitCommit(NamedTuple):
+ """Represents a single git commit."""
+ sha: str
+ subject: str
+
+
+def update_git_repo(git_dir: pathlib.Path) -> None:
+ """Updates the repo at `git_dir`, retrying a few times on failure."""
+ for i in itertools.count(start=1):
+ result = subprocess.run(
+ ['git', 'fetch', 'origin'],
+ check=False,
+ cwd=str(git_dir),
+ stdin=None,
+ )
+
+ if not result.returncode:
+ break
+
+ if i == 5:
+ # 5 attempts is too many. Something else may be wrong.
+ result.check_returncode()
+
+ sleep_time = 60 * i
+ logging.error("Failed updating gentoo's repo; will try again in %ds...",
+ sleep_time)
+ time.sleep(sleep_time)
+
+
+def get_new_gentoo_commits(git_dir: pathlib.Path,
+ most_recent_sha: str) -> List[GitCommit]:
+ """Gets commits to dev-lang/rust since `most_recent_sha`.
+
+ Older commits come earlier in the returned list.
+ """
+ commits = subprocess.run(
+ [
+ 'git',
+ 'log',
+ '--format=%H %s',
+ f'{most_recent_sha}..origin/master',
+ '--',
+ 'dev-lang/rust',
+ ],
+ capture_output=True,
+ check=False,
+ cwd=str(git_dir),
+ encoding='utf-8',
+ )
+
+ if commits.returncode:
+ logging.error('Error getting new gentoo commits; stderr:\n%s',
+ commits.stderr)
+ commits.check_returncode()
+
+ results = []
+ for line in commits.stdout.strip().splitlines():
+ sha, subject = line.strip().split(None, 1)
+ results.append(GitCommit(sha=sha, subject=subject))
+
+ # `git log` outputs things in newest -> oldest order.
+ results.reverse()
+ return results
+
+
+def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str:
+ """Sets up a gentoo git repo at the given directory. Returns HEAD."""
+ subprocess.run(
+ [
+ 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git',
+ str(git_dir)
+ ],
+ stdin=None,
+ check=True,
+ )
+
+ head_rev = subprocess.run(
+ ['git', 'rev-parse', 'HEAD'],
+ cwd=str(git_dir),
+ check=True,
+ stdin=None,
+ capture_output=True,
+ encoding='utf-8',
+ )
+ return head_rev.stdout.strip()
+
+
+def read_state(state_file: pathlib.Path) -> State:
+ """Reads state from the given file."""
+ with state_file.open(encoding='utf-8') as f:
+ return State.from_json(json.load(f))
+
+
+def atomically_write_state(state_file: pathlib.Path, state: State) -> None:
+ """Writes state to the given file."""
+ temp_file = pathlib.Path(str(state_file) + '.new')
+ with temp_file.open('w', encoding='utf-8') as f:
+ json.dump(state.to_json(), f)
+ temp_file.rename(state_file)
+
+
+def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion,
+ new_gentoo_commits: List[GitCommit]
+ ) -> Optional[Tuple[str, List[tiny_render.Piece]]]:
+ """Creates an email given our new state, if doing so is appropriate."""
+ subject_pieces = []
+ body_pieces = []
+
+ if newest_release > old_state.last_seen_release:
+ subject_pieces.append('new rustc release detected')
+ body_pieces.append(f'Rustc tag for v{newest_release} was found.')
+
+ if new_gentoo_commits:
+ # Separate the sections a bit for prettier output.
+ if body_pieces:
+ body_pieces += [tiny_render.line_break, tiny_render.line_break]
+
+ if len(new_gentoo_commits) == 1:
+ subject_pieces.append('new rust ebuild commit detected')
+ body_pieces.append('commit:')
+ else:
+ subject_pieces.append('new rust ebuild commits detected')
+ body_pieces.append('commits (newest first):')
+
+ commit_lines = []
+ for commit in new_gentoo_commits:
+ commit_lines.append([
+ tiny_render.Link(
+ gentoo_sha_to_link(commit.sha),
+ commit.sha[:12],
+ ),
+ f': {commit.subject}',
+ ])
+
+ body_pieces.append(tiny_render.UnorderedList(commit_lines))
+
+ if not subject_pieces:
+ return None
+
+ subject = '[rust-watch] ' + '; '.join(subject_pieces)
+ return subject, body_pieces
+
+
+def main(argv: List[str]) -> None:
+ logging.basicConfig(level=logging.INFO)
+
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument(
+ '--state_dir', required=True, help='Directory to store state in.')
+ parser.add_argument(
+ '--skip_email', action='store_true', help="Don't send an email.")
+ parser.add_argument(
+ '--skip_state_update',
+ action='store_true',
+ help="Don't update the state file. Doesn't apply to initial setup.")
+ opts = parser.parse_args(argv)
+
+ state_dir = pathlib.Path(opts.state_dir)
+ state_file = state_dir / 'state.json'
+ gentoo_subdir = state_dir / 'upstream-gentoo'
+ if not state_file.exists():
+ logging.info("state_dir isn't fully set up; doing that now.")
+
+ # Could be in a partially set-up state.
+ if state_dir.exists():
+ logging.info('incomplete state_dir detected; removing.')
+ shutil.rmtree(str(state_dir))
+
+ state_dir.mkdir(parents=True)
+ most_recent_release = fetch_most_recent_release()
+ most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir)
+ atomically_write_state(
+ state_file,
+ State(
+ last_seen_release=most_recent_release,
+ last_gentoo_sha=most_recent_gentoo_commit,
+ ),
+ )
+ # Running through this _should_ be a nop, but do it anyway. Should make any
+ # bugs more obvious on the first run of the script.
+
+ prior_state = read_state(state_file)
+ logging.info('Last state was %r', prior_state)
+
+ most_recent_release = fetch_most_recent_release()
+ logging.info('Most recent Rust release is %s', most_recent_release)
+
+ logging.info('Fetching new commits from Gentoo')
+ update_git_repo(gentoo_subdir)
+ new_commits = get_new_gentoo_commits(gentoo_subdir,
+ prior_state.last_gentoo_sha)
+ logging.info('New commits: %r', new_commits)
+
+ maybe_email = maybe_compose_email(prior_state, most_recent_release,
+ new_commits)
+
+ if maybe_email is None:
+ logging.info('No updates to send')
+ else:
+ title, body = maybe_email
+ if opts.skip_email:
+ logging.info('Skipping sending email with title %r and contents\n%s',
+ title, tiny_render.render_html_pieces(body))
+ else:
+ logging.info('Sending email')
+ send_email(title, body)
+
+ if opts.skip_state_update:
+ logging.info('Skipping state update, as requested')
+ return
+
+ newest_sha = (
+ new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha)
+ atomically_write_state(
+ state_file,
+ State(
+ last_seen_release=most_recent_release,
+ last_gentoo_sha=newest_sha,
+ ),
+ )
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py
new file mode 100755
index 00000000..97d111fc
--- /dev/null
+++ b/rust_tools/rust_watch_test.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for rust_watch.py."""
+
+# pylint: disable=cros-logging-import
+
+import logging
+import pathlib
+import subprocess
+import time
+import unittest
+import unittest.mock
+
+import rust_watch
+from cros_utils import tiny_render
+
+
+class Test(unittest.TestCase):
+ """Tests."""
+
+ def _silence_logs(self):
+ """Silences all log output until the end of the current test."""
+
+ def should_log(_record):
+ return 0
+
+ logger = logging.root
+ logger.addFilter(should_log)
+ self.addCleanup(logger.removeFilter, should_log)
+
+ def test_release_version_parsing(self):
+ self.assertEqual(
+ rust_watch.RustReleaseVersion.from_string('1.2.3'),
+ rust_watch.RustReleaseVersion(1, 2, 3),
+ )
+
+ def test_release_version_json_round_trips(self):
+ ver = rust_watch.RustReleaseVersion(1, 2, 3)
+ self.assertEqual(
+ rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver)
+
+ def test_state_json_round_trips(self):
+ state = rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 2, 3),
+ last_gentoo_sha='abc123',
+ )
+
+ self.assertEqual(rust_watch.State.from_json(state.to_json()), state)
+
+ @unittest.mock.patch.object(subprocess, 'run')
+ @unittest.mock.patch.object(time, 'sleep')
+ def test_update_git_repo_tries_again_on_failure(self, sleep_mock, run_mock):
+ self._silence_logs()
+
+ oh_no_error = ValueError('oh no')
+
+ def check_returncode():
+ raise oh_no_error
+
+ run_call_count = 0
+
+ def run_sideeffect(*_args, **_kwargs):
+ nonlocal run_call_count
+ run_call_count += 1
+ result = unittest.mock.Mock()
+ result.returncode = 1
+ result.check_returncode = check_returncode
+ return result
+
+ run_mock.side_effect = run_sideeffect
+
+ with self.assertRaises(ValueError) as raised:
+ rust_watch.update_git_repo(pathlib.Path('/does/not/exist/at/all'))
+
+ self.assertIs(raised.exception, oh_no_error)
+ self.assertEqual(run_call_count, 5)
+
+ sleep_timings = [unittest.mock.call(60 * i) for i in range(1, 5)]
+ self.assertEqual(sleep_mock.mock_calls, sleep_timings)
+
+ @unittest.mock.patch.object(subprocess, 'run')
+ def test_get_new_gentoo_commits_functions(self, run_mock):
+ returned = unittest.mock.Mock()
+ returned.returncode = 0
+ returned.stdout = '\n'.join((
+ 'abc123 newer commit',
+ 'abcdef and an older commit',
+ ))
+ run_mock.return_value = returned
+ results = rust_watch.get_new_gentoo_commits(
+ pathlib.Path('/does/not/exist/at/all'), 'defabc')
+ self.assertEqual(results, [
+ rust_watch.GitCommit('abcdef', 'and an older commit'),
+ rust_watch.GitCommit('abc123', 'newer commit'),
+ ])
+
+ def test_compose_email_on_a_new_release(self):
+ new_release = rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 1, 0),
+ new_gentoo_commits=[],
+ )
+
+ self.assertEqual(new_release, ('[rust-watch] new rustc release detected',
+ ['Rustc tag for v1.1.0 was found.']))
+
+ def test_compose_email_on_a_new_gentoo_commit(self):
+ sha_a = 'a' * 40
+ new_commit = rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ new_gentoo_commits=[
+ rust_watch.GitCommit(
+ sha=sha_a,
+ subject='summary_a',
+ ),
+ ],
+ )
+
+ self.assertEqual(new_commit,
+ ('[rust-watch] new rust ebuild commit detected', [
+ 'commit:',
+ tiny_render.UnorderedList([
+ [
+ tiny_render.Link(
+ rust_watch.gentoo_sha_to_link(sha_a),
+ sha_a[:12],
+ ),
+ ': summary_a',
+ ],
+ ])
+ ]))
+
+ def test_compose_email_on_multiple_events(self):
+ sha_a = 'a' * 40
+ new_commit_and_release = rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 1, 0),
+ new_gentoo_commits=[
+ rust_watch.GitCommit(
+ sha=sha_a,
+ subject='summary_a',
+ ),
+ ],
+ )
+
+ self.assertEqual(
+ new_commit_and_release,
+ ('[rust-watch] new rustc release detected; new rust ebuild commit '
+ 'detected', [
+ 'Rustc tag for v1.1.0 was found.',
+ tiny_render.line_break,
+ tiny_render.line_break,
+ 'commit:',
+ tiny_render.UnorderedList([
+ [
+ tiny_render.Link(
+ rust_watch.gentoo_sha_to_link(sha_a),
+ sha_a[:12],
+ ),
+ ': summary_a',
+ ],
+ ]),
+ ]))
+
+ def test_compose_email_composes_nothing_when_no_new_updates_exist(self):
+ self.assertIsNone(
+ rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ new_gentoo_commits=[],
+ ))
+
+ self.assertIsNone(
+ rust_watch.maybe_compose_email(
+ old_state=rust_watch.State(
+ last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0),
+ last_gentoo_sha='',
+ ),
+ newest_release=rust_watch.RustReleaseVersion(1, 0, 0),
+ new_gentoo_commits=[],
+ ))
+
+
+if __name__ == '__main__':
+ unittest.main()