pgo_tools: Add a monitor for LLVM PGO profile freshness

This CL lands a monitor that'll poke our mage if we have no recent LLVM
PGO profiles sitting in our bucket. The intent is to run it every few
days on chrotomation3, just like the other monitoring infra we have.

BUG=chromium:978741
TEST=unittests; ran with a very low days value, and verified the email

Change-Id: Ie7a7d44d58c260dd4f17f0759c9fc84bbf8f120d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2346840
Tested-by: George Burgess <gbiv@chromium.org>
Reviewed-by: Tiancong Wang <tcwang@google.com>

2 files changed, 250 insertions, 0 deletions

diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py
new file mode 100755
index 00000000..86701675
--- /dev/null
+++ b/pgo_tools/monitor_pgo_profiles.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Emails the mage if PGO profile generation hasn't succeeded recently."""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import datetime
+import sys
+import subprocess
+import logging
+from typing import List, NamedTuple, Optional, Tuple
+
+from cros_utils import email_sender
+from cros_utils import tiny_render
+
+PGO_BUILDBOT_LINK = ('https://ci.chromium.org/p/chromeos/builders/toolchain/'
+                     'pgo-generate-llvm-next-orchestrator')
+
+
+class ProfdataInfo(NamedTuple):
+  """Data about an llvm profdata in our gs:// bucket."""
+  date: datetime.datetime
+  location: str
+
+
+def parse_date(date: str) -> datetime.datetime:
+  time_format = '%Y-%m-%dT%H:%M:%SZ'
+  if not date.endswith('Z'):
+    time_format += '%z'
+  return datetime.datetime.strptime(date, time_format)
+
+
+def fetch_most_recent_profdata(arch: str) -> ProfdataInfo:
+  result = subprocess.run(
+      [
+          'gsutil',
+          'ls',
+          '-l',
+          f'gs://chromeos-toolchain-artifacts/llvm-pgo/{arch}/'
+          '*.profdata.tar.xz',
+      ],
+      check=True,
+      stdout=subprocess.PIPE,
+      encoding='utf-8',
+  )
+
+  # Each line will be a profdata; the last one is a summary, so drop it.
+  infos = []
+  for rec in result.stdout.strip().splitlines()[:-1]:
+    _size, date, url = rec.strip().split()
+    infos.append(ProfdataInfo(date=parse_date(date), location=url))
+  return max(infos)
+
+
+def compose_complaint_email(
+    out_of_date_profiles: List[Tuple[datetime.datetime, ProfdataInfo]]
+) -> Optional[Tuple[str, tiny_render.Piece]]:
+  if not out_of_date_profiles:
+    return None
+
+  if len(out_of_date_profiles) == 1:
+    subject = '1 llvm profile is out of date'
+    body = ['out-of-date profile:']
+  else:
+    subject = f'{len(out_of_date_profiles)} llvm profiles are out of date'
+    body = ['out-of-date profiles:']
+
+  out_of_date_items = []
+  for arch, profdata_info in out_of_date_profiles:
+    out_of_date_items.append(
+        f'{arch} (most recent profile was from {profdata_info.date} at '
+        f'{profdata_info.location!r})')
+
+  body += [
+      tiny_render.UnorderedList(out_of_date_items),
+      tiny_render.line_break,
+      tiny_render.line_break,
+      'PTAL to see if the llvm-pgo-generate bots are functioning normally. '
+      'Their status can be found at ',
+      tiny_render.Link(href=PGO_BUILDBOT_LINK, inner=PGO_BUILDBOT_LINK),
+      '.',
+  ]
+  return subject, body
+
+
+def main() -> None:
+  logging.basicConfig(level=logging.INFO)
+
+  parser = argparse.ArgumentParser(
+      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+  parser.add_argument(
+      '--dry_run',
+      action='store_true',
+      help="Don't actually send an email",
+  )
+  parser.add_argument(
+      '--max_age_days',
+      # These builders run ~weekly. If we fail to generate two in a row,
+      # something's probably wrong.
+      default=15,
+      type=int,
+      help='How old to let profiles get before complaining, in days',
+  )
+  args = parser.parse_args()
+
+  now = datetime.datetime.now()
+  logging.info('Start time is %r', now)
+
+  max_age = datetime.timedelta(days=args.max_age_days)
+  out_of_date_profiles = []
+  for arch in ('arm', 'arm64', 'amd64'):
+    logging.info('Fetching most recent profdata for %r', arch)
+    most_recent = fetch_most_recent_profdata(arch)
+    logging.info('Most recent profdata for %r is %r', arch, most_recent)
+
+    age = now - most_recent.date
+    if age >= max_age:
+      out_of_date_profiles.append((arch, most_recent))
+
+  email = compose_complaint_email(out_of_date_profiles)
+  if not email:
+    logging.info('No email to send; quit')
+    return
+
+  subject, body = email
+
+  identifier = 'llvm-pgo-monitor'
+  subject = f'[{identifier}] {subject}'
+
+  logging.info('Sending email with title %r', subject)
+  if args.dry_run:
+    logging.info('Dry run specified\nSubject: %s\nBody:\n%s', subject,
+                 tiny_render.render_text_pieces(body))
+  else:
+    email_sender.EmailSender().SendX20Email(
+        subject=subject,
+        identifier=identifier,
+        well_known_recipients=['mage'],
+        direct_recipients=['gbiv@google.com'],
+        text_body=tiny_render.render_text_pieces(body),
+        html_body=tiny_render.render_html_pieces(body),
+    )
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/pgo_tools/monitor_pgo_profiles_unittest.py b/pgo_tools/monitor_pgo_profiles_unittest.py
new file mode 100755
index 00000000..b4e085ec
--- /dev/null
+++ b/pgo_tools/monitor_pgo_profiles_unittest.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for monitor_pgo_profiles."""
+
+import datetime
+import subprocess
+import unittest
+import unittest.mock
+
+import monitor_pgo_profiles
+from cros_utils import tiny_render
+
+
+class Test(unittest.TestCase):
+  """Tests for monitor_pgo_profiles."""
+
+  def test_compose_complaint_email_with_zero_out_of_date(self):
+    self.assertIsNone(monitor_pgo_profiles.compose_complaint_email([]))
+
+  def test_compose_complaint_email_with_one_out_of_date(self):
+    profdata_info = monitor_pgo_profiles.ProfdataInfo(
+        date=datetime.datetime(2020, 1, 2, 3, 4, 5),
+        location='gs://somewhere',
+    )
+    result = monitor_pgo_profiles.compose_complaint_email([
+        ('some_arch', profdata_info),
+    ])
+    self.assertEqual(result, ('1 llvm profile is out of date', [
+        'out-of-date profile:',
+        tiny_render.UnorderedList([
+            f'some_arch (most recent profile was from {profdata_info.date} at '
+            f'{profdata_info.location!r})'
+        ]),
+        tiny_render.line_break,
+        tiny_render.line_break,
+        'PTAL to see if the llvm-pgo-generate bots are functioning normally. '
+        'Their status can be found at ',
+        tiny_render.Link(
+            href=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+            inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+        ),
+        '.',
+    ]))
+
+  def test_compose_complaint_email_with_two_out_of_date(self):
+    profdata_info_1 = monitor_pgo_profiles.ProfdataInfo(
+        date=datetime.datetime(2020, 1, 2, 3, 4, 5),
+        location='gs://somewhere',
+    )
+    profdata_info_2 = monitor_pgo_profiles.ProfdataInfo(
+        date=datetime.datetime(2020, 3, 2, 1, 4, 5),
+        location='gs://somewhere-else',
+    )
+    result = monitor_pgo_profiles.compose_complaint_email([
+        ('some_arch', profdata_info_1),
+        ('some_other_arch', profdata_info_2),
+    ])
+    self.assertEqual(result, ('2 llvm profiles are out of date', [
+        'out-of-date profiles:',
+        tiny_render.UnorderedList([
+            f'some_arch (most recent profile was from {profdata_info_1.date} '
+            f'at {profdata_info_1.location!r})',
+            f'some_other_arch (most recent profile was from '
+            f'{profdata_info_2.date} at {profdata_info_2.location!r})'
+        ]),
+        tiny_render.line_break,
+        tiny_render.line_break,
+        'PTAL to see if the llvm-pgo-generate bots are functioning normally. '
+        'Their status can be found at ',
+        tiny_render.Link(
+            href=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+            inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK,
+        ),
+        '.',
+    ]))
+
+  @unittest.mock.patch.object(subprocess, 'run')
+  def test_fetching_profdata_functions(self, subprocess_run_mock):
+    ls_return_value = unittest.mock.MagicMock()
+    ls_return_value.stdout = '\n'.join((
+        '  1234 2020-06-26T05:26:40Z gs://bar',
+        '    44 2020-06-23T05:26:40Z gs://foo',
+        '  1234 2020-06-25T05:26:40Z gs://zzz',
+    ))
+    subprocess_run_mock.return_value = ls_return_value
+
+    most_recent = monitor_pgo_profiles.fetch_most_recent_profdata('arm')
+    self.assertEqual(
+        most_recent,
+        monitor_pgo_profiles.ProfdataInfo(
+            date=datetime.datetime(2020, 6, 26, 5, 26, 40),
+            location='gs://bar',
+        ))
+
+
+if __name__ == '__main__':
+  unittest.main()