1 files changed, 410 insertions, 0 deletions
diff --git a/new-generate-waterfall-reports.py b/new-generate-waterfall-reports.py
new file mode 100755
index 00000000..ef48f8be
--- /dev/null
+++ b/new-generate-waterfall-reports.py
@@ -0,0 +1,410 @@
+#!/usr/bin/env python2
+"""Generate summary report for ChromeOS toolchain waterfalls."""
+
+from __future__ import print_function
+
+import argparse
+import datetime
+import getpass
+import json
+import os
+import re
+import shutil
+import sys
+import time
+
+from cros_utils import command_executer
+
+# All the test suites whose data we might want for the reports.
+TESTS = (('bvt-inline', 'HWTest [bvt-inline]'), ('bvt-cq', 'HWTest [bvt-cq]'),
+         ('security', 'HWTest [security]'))
+
+# The main waterfall builders, IN THE ORDER IN WHICH WE WANT THEM
+# LISTED IN THE REPORT.
+WATERFALL_BUILDERS = [
+    'amd64-llvm-next-toolchain',
+    'arm-llvm-next-toolchain',
+    'arm64-llvm-next-toolchain',
+]
+
+DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/'
+ARCHIVE_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-reports/'
+DOWNLOAD_DIR = '/tmp/waterfall-logs'
+MAX_SAVE_RECORDS = 7
+BUILD_DATA_FILE = '%s/build-data.txt' % DATA_DIR
+LLVM_ROTATING_BUILDER = 'llvm_next_toolchain'
+ROTATING_BUILDERS = [LLVM_ROTATING_BUILDER]
+
+# For int-to-string date conversion.  Note, the index of the month in this
+# list needs to correspond to the month's integer value.  i.e. 'Sep' must
+# be as MONTHS[9].
+MONTHS = [
+    '', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
+    'Nov', 'Dec'
+]
+
+DAYS_PER_MONTH = {
+    1: 31,
+    2: 28,
+    3: 31,
+    4: 30,
+    5: 31,
+    6: 30,
+    7: 31,
+    8: 31,
+    9: 30,
+    10: 31,
+    11: 31,
+    12: 31
+}
+
+
+def format_date(int_date, use_int_month=False):
+  """Convert an integer date to a string date. YYYYMMDD -> YYYY-MMM-DD"""
+
+  if int_date == 0:
+    return 'today'
+
+  tmp_date = int_date
+  day = tmp_date % 100
+  tmp_date = tmp_date / 100
+  month = tmp_date % 100
+  year = tmp_date / 100
+
+  if use_int_month:
+    date_str = '%d-%02d-%02d' % (year, month, day)
+  else:
+    month_str = MONTHS[month]
+    date_str = '%d-%s-%d' % (year, month_str, day)
+  return date_str
+
+
+def EmailReport(report_file, report_type, date, email_to):
+  """Emails the report to the approprite address."""
+  subject = '%s Waterfall Summary report, %s' % (report_type, date)
+  sendgmr_path = '/google/data/ro/projects/gws-sre/sendgmr'
+  command = ('%s --to=%s --subject="%s" --body_file=%s' %
+             (sendgmr_path, email_to, subject, report_file))
+  command_executer.GetCommandExecuter().RunCommand(command)
+
+
+def GetColor(status):
+  """Given a job status string, returns appropriate color string."""
+  if status.strip() == 'pass':
+    color = 'green '
+  elif status.strip() == 'fail':
+    color = ' red  '
+  elif status.strip() == 'warning':
+    color = 'orange'
+  else:
+    color = '      '
+  return color
+
+
+def GenerateWaterfallReport(report_dict, waterfall_type, date):
+  """Write out the actual formatted report."""
+
+  filename = 'waterfall_report.%s_waterfall.%s.txt' % (waterfall_type, date)
+
+  date_string = ''
+  report_list = report_dict.keys()
+
+  with open(filename, 'w') as out_file:
+    # Write Report Header
+    out_file.write('\nStatus of %s Waterfall Builds from %s\n\n' %
+                   (waterfall_type, date_string))
+    out_file.write('                                                        \n')
+    out_file.write(
+        '                                         Build       bvt-    '
+        '     bvt-cq     '
+        ' security \n')
+    out_file.write(
+        '                                         status     inline   '
+        '              \n')
+
+    # Write daily waterfall status section.
+    for builder in report_list:
+      build_dict = report_dict[builder]
+      buildbucket_id = build_dict['buildbucket_id']
+      overall_status = build_dict['status']
+      if 'bvt-inline' in build_dict.keys():
+        inline_status = build_dict['bvt-inline']
+      else:
+        inline_status = '    '
+      if 'bvt-cq' in build_dict.keys():
+        cq_status = build_dict['bvt-cq']
+      else:
+        cq_status = '    '
+      if 'security' in build_dict.keys():
+        security_status = build_dict['security']
+      else:
+        security_status = '    '
+      inline_color = GetColor(inline_status)
+      cq_color = GetColor(cq_status)
+      security_color = GetColor(security_status)
+
+      out_file.write(
+          '%26s   %4s        %6s        %6s         %6s\n' %
+          (builder, overall_status, inline_color, cq_color, security_color))
+      if waterfall_type == 'main':
+        out_file.write('     build url: https://cros-goldeneye.corp.google.com/'
+                       'chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
+                       '\n' % buildbucket_id)
+      else:
+        out_file.write('     build url: https://ci.chromium.org/p/chromeos/'
+                       'builds/b%s \n' % buildbucket_id)
+        report_url = ('https://logs.chromium.org/v/?s=chromeos%2Fbuildbucket%2F'
+                      'cr-buildbucket.appspot.com%2F' + buildbucket_id +
+                      '%2F%2B%2Fsteps%2FReport%2F0%2Fstdout')
+        out_file.write('\n     report status url: %s\n' % report_url)
+      out_file.write('\n')
+
+    print('Report generated in %s.' % filename)
+    return filename
+
+
+def GetTryjobData(date, rotating_builds_dict):
+  """Read buildbucket id and board from stored file.
+
+  buildbot_test_llvm.py, when it launches the rotating builders,
+  records the buildbucket_id and board for each launch in a file.
+  This reads that data out of the file so we can find the right
+  tryjob data.
+  """
+
+  date_str = format_date(date, use_int_month=True)
+  fname = '%s.builds' % date_str
+  filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
+
+  if not os.path.exists(filename):
+    print('Cannot find file: %s' % filename)
+    print('Unable to generate rotating builder report for date %d.' % date)
+    return
+
+  with open(filename, 'r') as in_file:
+    lines = in_file.readlines()
+
+  for line in lines:
+    l = line.strip()
+    parts = l.split(',')
+    if len(parts) != 2:
+      print('Warning: Illegal line in data file.')
+      print('File: %s' % filename)
+      print('Line: %s' % l)
+      continue
+    buildbucket_id = parts[0]
+    board = parts[1]
+    rotating_builds_dict[board] = buildbucket_id
+
+  return
+
+
+def GetRotatingBuildData(date, report_dict, chromeos_root, board,
+                         buildbucket_id, ce):
+  """Gets rotating builder job results via 'cros buildresult'."""
+  path = os.path.join(chromeos_root, 'chromite')
+  save_dir = os.getcwd()
+  date_str = format_date(date, use_int_month=True)
+  os.chdir(path)
+
+  command = (
+      'cros buildresult --buildbucket-id %s --report json' % buildbucket_id)
+  _, out, _ = ce.RunCommandWOutput(command)
+  tmp_dict = json.loads(out)
+  results = tmp_dict[buildbucket_id]
+
+  board_dict = dict()
+  board_dict['buildbucket_id'] = buildbucket_id
+  stages_results = results['stages']
+  for test in TESTS:
+    key1 = test[0]
+    key2 = test[1]
+    if key2 in stages_results:
+      board_dict[key1] = stages_results[key2]
+  board_dict['status'] = results['status']
+  report_dict[board] = board_dict
+  os.chdir(save_dir)
+  return
+
+
+def GetMainWaterfallData(date, report_dict, chromeos_root, ce):
+  """Gets main waterfall job results via 'cros buildresult'."""
+  path = os.path.join(chromeos_root, 'chromite')
+  save_dir = os.getcwd()
+  date_str = format_date(date, use_int_month=True)
+  os.chdir(path)
+  for builder in WATERFALL_BUILDERS:
+    command = ('cros buildresult --build-config %s --date %s --report json' %
+               (builder, date_str))
+    _, out, _ = ce.RunCommandWOutput(command)
+    tmp_dict = json.loads(out)
+    builder_dict = dict()
+    for k in tmp_dict.keys():
+      buildbucket_id = k
+      results = tmp_dict[k]
+
+    builder_dict['buildbucket_id'] = buildbucket_id
+    builder_dict['status'] = results['status']
+    stages_results = results['stages']
+    for test in TESTS:
+      key1 = test[0]
+      key2 = test[1]
+      builder_dict[key1] = stages_results[key2]
+    report_dict[builder] = builder_dict
+  os.chdir(save_dir)
+  return
+
+
+# Check for prodaccess.
+def CheckProdAccess():
+  """Verifies prodaccess is current."""
+  status, output, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
+      'prodcertstatus')
+  if status != 0:
+    return False
+  # Verify that status is not expired
+  if 'expires' in output:
+    return True
+  return False
+
+
+def ValidDate(date):
+  """Ensures 'date' is a valid date."""
+  min_year = 2018
+
+  tmp_date = date
+  day = tmp_date % 100
+  tmp_date = tmp_date / 100
+  month = tmp_date % 100
+  year = tmp_date / 100
+
+  if day < 1 or month < 1 or year < min_year:
+    return False
+
+  cur_year = datetime.datetime.now().year
+  if year > cur_year:
+    return False
+
+  if month > 12:
+    return False
+
+  if month == 2 and cur_year % 4 == 0 and cur_year % 100 != 0:
+    max_day = 29
+  else:
+    max_day = DAYS_PER_MONTH[month]
+
+  if day > max_day:
+    return False
+
+  return True
+
+
+def ValidOptions(parser, options):
+  """Error-check the options passed to this script."""
+  too_many_options = False
+  if options.main:
+    if options.rotating:
+      too_many_options = True
+
+  if too_many_options:
+    parser.error('Can only specify one of --main, --rotating.')
+
+  if not os.path.exists(options.chromeos_root):
+    parser.error(
+        'Invalid chromeos root. Cannot find: %s' % options.chromeos_root)
+
+  email_ok = True
+  if options.email and options.email.find('@') == -1:
+    email_ok = False
+    parser.error('"%s" is not a valid email address; it must contain "@..."' %
+                 options.email)
+
+  valid_date = ValidDate(options.date)
+
+  return not too_many_options and valid_date and email_ok
+
+
+def Main(argv):
+  """Main function for this script."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--main',
+      dest='main',
+      default=False,
+      action='store_true',
+      help='Generate report only for main waterfall '
+      'builders.')
+  parser.add_argument(
+      '--rotating',
+      dest='rotating',
+      default=False,
+      action='store_true',
+      help='Generate report only for rotating builders.')
+  parser.add_argument(
+      '--date',
+      dest='date',
+      required=True,
+      type=int,
+      help='The date YYYYMMDD of waterfall report.')
+  parser.add_argument(
+      '--email',
+      dest='email',
+      default='',
+      help='Email address to use for sending the report.')
+  parser.add_argument(
+      '--chromeos_root',
+      dest='chromeos_root',
+      required=True,
+      help='Chrome OS root in which to run chroot commands.')
+
+  options = parser.parse_args(argv)
+
+  if not ValidOptions(parser, options):
+    return 1
+
+  main_only = options.main
+  rotating_only = options.rotating
+  date = options.date
+
+  prod_access = CheckProdAccess()
+  if not prod_access:
+    print('ERROR: Please run prodaccess first.')
+    return
+
+  waterfall_report_dict = dict()
+  rotating_report_dict = dict()
+
+  ce = command_executer.GetCommandExecuter()
+  if not rotating_only:
+    GetMainWaterfallData(date, waterfall_report_dict, options.chromeos_root, ce)
+
+  if not main_only:
+    rotating_builds_dict = dict()
+    GetTryjobData(date, rotating_builds_dict)
+    if len(rotating_builds_dict.keys()) > 0:
+      for board in rotating_builds_dict.keys():
+        buildbucket_id = rotating_builds_dict[board]
+        GetRotatingBuildData(date, rotating_report_dict, options.chromeos_root,
+                             board, buildbucket_id, ce)
+
+  if options.email:
+    email_to = options.email
+  else:
+    email_to = getpass.getuser()
+
+  if waterfall_report_dict and not rotating_only:
+    main_report = GenerateWaterfallReport(waterfall_report_dict, 'main', date)
+
+    EmailReport(main_report, 'Main', format_date(date), email_to)
+    shutil.copy(main_report, ARCHIVE_DIR)
+  if rotating_report_dict and not main_only:
+    rotating_report = GenerateWaterfallReport(rotating_report_dict, 'rotating',
+                                              date)
+
+    EmailReport(rotating_report, 'Rotating', format_date(date), email_to)
+    shutil.copy(rotating_report, ARCHIVE_DIR)
+
+
+if __name__ == '__main__':
+  Main(sys.argv[1:])
+  sys.exit(0)