diff options
Diffstat (limited to 'pgo_tools')
| -rwxr-xr-x | pgo_tools/monitor_pgo_profiles.py | 150 | ||||
| -rwxr-xr-x | pgo_tools/monitor_pgo_profiles_unittest.py | 100 |
2 files changed, 250 insertions, 0 deletions
diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py new file mode 100755 index 00000000..86701675 --- /dev/null +++ b/pgo_tools/monitor_pgo_profiles.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Emails the mage if PGO profile generation hasn't succeeded recently.""" + +# pylint: disable=cros-logging-import + +import argparse +import datetime +import sys +import subprocess +import logging +from typing import List, NamedTuple, Optional, Tuple + +from cros_utils import email_sender +from cros_utils import tiny_render + +PGO_BUILDBOT_LINK = ('https://ci.chromium.org/p/chromeos/builders/toolchain/' + 'pgo-generate-llvm-next-orchestrator') + + +class ProfdataInfo(NamedTuple): + """Data about an llvm profdata in our gs:// bucket.""" + date: datetime.datetime + location: str + + +def parse_date(date: str) -> datetime.datetime: + time_format = '%Y-%m-%dT%H:%M:%SZ' + if not date.endswith('Z'): + time_format += '%z' + return datetime.datetime.strptime(date, time_format) + + +def fetch_most_recent_profdata(arch: str) -> ProfdataInfo: + result = subprocess.run( + [ + 'gsutil', + 'ls', + '-l', + f'gs://chromeos-toolchain-artifacts/llvm-pgo/{arch}/' + '*.profdata.tar.xz', + ], + check=True, + stdout=subprocess.PIPE, + encoding='utf-8', + ) + + # Each line will be a profdata; the last one is a summary, so drop it. + infos = [] + for rec in result.stdout.strip().splitlines()[:-1]: + _size, date, url = rec.strip().split() + infos.append(ProfdataInfo(date=parse_date(date), location=url)) + return max(infos) + + +def compose_complaint_email( + out_of_date_profiles: List[Tuple[datetime.datetime, ProfdataInfo]] +) -> Optional[Tuple[str, tiny_render.Piece]]: + if not out_of_date_profiles: + return None + + if len(out_of_date_profiles) == 1: + subject = '1 llvm profile is out of date' + body = ['out-of-date profile:'] + else: + subject = f'{len(out_of_date_profiles)} llvm profiles are out of date' + body = ['out-of-date profiles:'] + + out_of_date_items = [] + for arch, profdata_info in out_of_date_profiles: + out_of_date_items.append( + f'{arch} (most recent profile was from {profdata_info.date} at ' + f'{profdata_info.location!r})') + + body += [ + tiny_render.UnorderedList(out_of_date_items), + tiny_render.line_break, + tiny_render.line_break, + 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' + 'Their status can be found at ', + tiny_render.Link(href=PGO_BUILDBOT_LINK, inner=PGO_BUILDBOT_LINK), + '.', + ] + return subject, body + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--dry_run', + action='store_true', + help="Don't actually send an email", + ) + parser.add_argument( + '--max_age_days', + # These builders run ~weekly. If we fail to generate two in a row, + # something's probably wrong. + default=15, + type=int, + help='How old to let profiles get before complaining, in days', + ) + args = parser.parse_args() + + now = datetime.datetime.now() + logging.info('Start time is %r', now) + + max_age = datetime.timedelta(days=args.max_age_days) + out_of_date_profiles = [] + for arch in ('arm', 'arm64', 'amd64'): + logging.info('Fetching most recent profdata for %r', arch) + most_recent = fetch_most_recent_profdata(arch) + logging.info('Most recent profdata for %r is %r', arch, most_recent) + + age = now - most_recent.date + if age >= max_age: + out_of_date_profiles.append((arch, most_recent)) + + email = compose_complaint_email(out_of_date_profiles) + if not email: + logging.info('No email to send; quit') + return + + subject, body = email + + identifier = 'llvm-pgo-monitor' + subject = f'[{identifier}] {subject}' + + logging.info('Sending email with title %r', subject) + if args.dry_run: + logging.info('Dry run specified\nSubject: %s\nBody:\n%s', subject, + tiny_render.render_text_pieces(body)) + else: + email_sender.EmailSender().SendX20Email( + subject=subject, + identifier=identifier, + well_known_recipients=['mage'], + direct_recipients=['gbiv@google.com'], + text_body=tiny_render.render_text_pieces(body), + html_body=tiny_render.render_html_pieces(body), + ) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/pgo_tools/monitor_pgo_profiles_unittest.py b/pgo_tools/monitor_pgo_profiles_unittest.py new file mode 100755 index 00000000..b4e085ec --- /dev/null +++ b/pgo_tools/monitor_pgo_profiles_unittest.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for monitor_pgo_profiles.""" + +import datetime +import subprocess +import unittest +import unittest.mock + +import monitor_pgo_profiles +from cros_utils import tiny_render + + +class Test(unittest.TestCase): + """Tests for monitor_pgo_profiles.""" + + def test_compose_complaint_email_with_zero_out_of_date(self): + self.assertIsNone(monitor_pgo_profiles.compose_complaint_email([])) + + def test_compose_complaint_email_with_one_out_of_date(self): + profdata_info = monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 1, 2, 3, 4, 5), + location='gs://somewhere', + ) + result = monitor_pgo_profiles.compose_complaint_email([ + ('some_arch', profdata_info), + ]) + self.assertEqual(result, ('1 llvm profile is out of date', [ + 'out-of-date profile:', + tiny_render.UnorderedList([ + f'some_arch (most recent profile was from {profdata_info.date} at ' + f'{profdata_info.location!r})' + ]), + tiny_render.line_break, + tiny_render.line_break, + 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' + 'Their status can be found at ', + tiny_render.Link( + href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + ), + '.', + ])) + + def test_compose_complaint_email_with_two_out_of_date(self): + profdata_info_1 = monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 1, 2, 3, 4, 5), + location='gs://somewhere', + ) + profdata_info_2 = monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 3, 2, 1, 4, 5), + location='gs://somewhere-else', + ) + result = monitor_pgo_profiles.compose_complaint_email([ + ('some_arch', profdata_info_1), + ('some_other_arch', profdata_info_2), + ]) + self.assertEqual(result, ('2 llvm profiles are out of date', [ + 'out-of-date profiles:', + tiny_render.UnorderedList([ + f'some_arch (most recent profile was from {profdata_info_1.date} ' + f'at {profdata_info_1.location!r})', + f'some_other_arch (most recent profile was from ' + f'{profdata_info_2.date} at {profdata_info_2.location!r})' + ]), + tiny_render.line_break, + tiny_render.line_break, + 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' + 'Their status can be found at ', + tiny_render.Link( + href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, + ), + '.', + ])) + + @unittest.mock.patch.object(subprocess, 'run') + def test_fetching_profdata_functions(self, subprocess_run_mock): + ls_return_value = unittest.mock.MagicMock() + ls_return_value.stdout = '\n'.join(( + ' 1234 2020-06-26T05:26:40Z gs://bar', + ' 44 2020-06-23T05:26:40Z gs://foo', + ' 1234 2020-06-25T05:26:40Z gs://zzz', + )) + subprocess_run_mock.return_value = ls_return_value + + most_recent = monitor_pgo_profiles.fetch_most_recent_profdata('arm') + self.assertEqual( + most_recent, + monitor_pgo_profiles.ProfdataInfo( + date=datetime.datetime(2020, 6, 26, 5, 26, 40), + location='gs://bar', + )) + + +if __name__ == '__main__': + unittest.main() |