1 files changed, 150 insertions, 0 deletions
diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py
new file mode 100755
index 00000000..cb41eb88
--- /dev/null
+++ b/pgo_tools/monitor_pgo_profiles.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Emails the mage if PGO profile generation hasn't succeeded recently."""
+
+# pylint: disable=cros-logging-import
+
+import argparse
+import datetime
+import sys
+import subprocess
+import logging
+from typing import List, NamedTuple, Optional, Tuple
+
+from cros_utils import email_sender
+from cros_utils import tiny_render
+
+PGO_BUILDBOT_LINK = ('https://ci.chromium.org/p/chromeos/builders/toolchain/'
+                     'pgo-generate-llvm-next-orchestrator')
+
+
+class ProfdataInfo(NamedTuple):
+  """Data about an llvm profdata in our gs:// bucket."""
+  date: datetime.datetime
+  location: str
+
+
+def parse_date(date: str) -> datetime.datetime:
+  time_format = '%Y-%m-%dT%H:%M:%SZ'
+  if not date.endswith('Z'):
+    time_format += '%z'
+  return datetime.datetime.strptime(date, time_format)
+
+
+def fetch_most_recent_profdata(arch: str) -> ProfdataInfo:
+  result = subprocess.run(
+      [
+          'gsutil.py',
+          'ls',
+          '-l',
+          f'gs://chromeos-toolchain-artifacts/llvm-pgo/{arch}/'
+          '*.profdata.tar.xz',
+      ],
+      check=True,
+      stdout=subprocess.PIPE,
+      encoding='utf-8',
+  )
+
+  # Each line will be a profdata; the last one is a summary, so drop it.
+  infos = []
+  for rec in result.stdout.strip().splitlines()[:-1]:
+    _size, date, url = rec.strip().split()
+    infos.append(ProfdataInfo(date=parse_date(date), location=url))
+  return max(infos)
+
+
+def compose_complaint_email(
+    out_of_date_profiles: List[Tuple[datetime.datetime, ProfdataInfo]]
+) -> Optional[Tuple[str, tiny_render.Piece]]:
+  if not out_of_date_profiles:
+    return None
+
+  if len(out_of_date_profiles) == 1:
+    subject = '1 llvm profile is out of date'
+    body = ['out-of-date profile:']
+  else:
+    subject = f'{len(out_of_date_profiles)} llvm profiles are out of date'
+    body = ['out-of-date profiles:']
+
+  out_of_date_items = []
+  for arch, profdata_info in out_of_date_profiles:
+    out_of_date_items.append(
+        f'{arch} (most recent profile was from {profdata_info.date} at '
+        f'{profdata_info.location!r})')
+
+  body += [
+      tiny_render.UnorderedList(out_of_date_items),
+      tiny_render.line_break,
+      tiny_render.line_break,
+      'PTAL to see if the llvm-pgo-generate bots are functioning normally. '
+      'Their status can be found at ',
+      tiny_render.Link(href=PGO_BUILDBOT_LINK, inner=PGO_BUILDBOT_LINK),
+      '.',
+  ]
+  return subject, body
+
+
+def main() -> None:
+  logging.basicConfig(level=logging.INFO)
+
+  parser = argparse.ArgumentParser(
+      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+  parser.add_argument(
+      '--dry_run',
+      action='store_true',
+      help="Don't actually send an email",
+  )
+  parser.add_argument(
+      '--max_age_days',
+      # These builders run ~weekly. If we fail to generate two in a row,
+      # something's probably wrong.
+      default=15,
+      type=int,
+      help='How old to let profiles get before complaining, in days',
+  )
+  args = parser.parse_args()
+
+  now = datetime.datetime.now()
+  logging.info('Start time is %r', now)
+
+  max_age = datetime.timedelta(days=args.max_age_days)
+  out_of_date_profiles = []
+  for arch in ('arm', 'arm64', 'amd64'):
+    logging.info('Fetching most recent profdata for %r', arch)
+    most_recent = fetch_most_recent_profdata(arch)
+    logging.info('Most recent profdata for %r is %r', arch, most_recent)
+
+    age = now - most_recent.date
+    if age >= max_age:
+      out_of_date_profiles.append((arch, most_recent))
+
+  email = compose_complaint_email(out_of_date_profiles)
+  if not email:
+    logging.info('No email to send; quit')
+    return
+
+  subject, body = email
+
+  identifier = 'llvm-pgo-monitor'
+  subject = f'[{identifier}] {subject}'
+
+  logging.info('Sending email with title %r', subject)
+  if args.dry_run:
+    logging.info('Dry run specified\nSubject: %s\nBody:\n%s', subject,
+                 tiny_render.render_text_pieces(body))
+  else:
+    email_sender.EmailSender().SendX20Email(
+        subject=subject,
+        identifier=identifier,
+        well_known_recipients=['mage'],
+        direct_recipients=['gbiv@google.com'],
+        text_body=tiny_render.render_text_pieces(body),
+        html_body=tiny_render.render_html_pieces(body),
+    )
+
+
+if __name__ == '__main__':
+  sys.exit(main())