[toolchain-utils] Update waterfall reports.

This CL updates the waterfall reports to use the results of
'cros buildresult' and the new URLS.  It removes the attempts to
track individual test failures, as the new URL system no longer
allows that.

BUG=chromium:829648
TEST=launched reports from my directory. They worked fine.
Tested buildbot_test_llvm.py changes with mobiletc-prebuild
on chrotomation2.

Change-Id: I12cf97c7960b032ec3caca636fe349f564369378
Reviewed-on: https://chromium-review.googlesource.com/1021990
Commit-Ready: Caroline Tice <cmtice@chromium.org>
Tested-by: Caroline Tice <cmtice@chromium.org>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>

2 files changed, 428 insertions, 3 deletions

diff --git a/buildbot_test_llvm.py b/buildbot_test_llvm.py
index 13140c80..5b9e6944 100755
--- a/buildbot_test_llvm.py
+++ b/buildbot_test_llvm.py
@@ -34,6 +34,7 @@ MAIL_PROGRAM = '~/var/bin/mail-sheriff'
 VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
 START_DATE = datetime.date(2016, 1, 1)
 TEST_PER_DAY = 4
+DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/'
 
 # Information about Rotating Boards
 #  Board        Arch     Reference    Platform      Kernel
@@ -124,14 +125,24 @@ class ToolchainVerifier(object):
     Launch trybot, get image names, create crosperf experiment file, run
     crosperf, and copy images into seven-day report directories.
     """
-    _ = buildbot_utils.GetTrybotImage(
+    buildbucket_id, _ = buildbot_utils.GetTrybotImage(
         self._chromeos_root,
         self._build,
         self._patches,
         tryjob_flags=['--hwtest'],
         async=True)
 
-    return 0
+    return buildbucket_id
+
+
+def WriteRotatingReportsData(results_dict, date):
+  """Write data for waterfall report."""
+  fname = '%d-%02d-%02d.builds' % (date.year, date.month, date.day)
+  filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
+  with open(filename, 'w') as out_file:
+    for board in results_dict.keys():
+      buildbucket_id = results_dict[board]
+      out_file.write('%s,%s\n' % (buildbucket_id, board))
 
 
 def Main(argv):
@@ -182,16 +193,20 @@ def Main(argv):
   days = delta.days
 
   start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
+  results_dict = dict()
   for i in range(TEST_PER_DAY):
     try:
       board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
       fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
                              options.patches, options.compiler)
-      fv.DoAll()
+      buildbucket_id = fv.DoAll()
+      if buildbucket_id:
+        results_dict[board] = buildbucket_id
     except SystemExit:
       logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
       with open(logfile, 'w') as f:
         f.write('Verifier got an exception, please check the log.\n')
+  WriteRotatingReportsData(results_dict, today)
 
 
 if __name__ == '__main__':
diff --git a/new-generate-waterfall-reports.py b/new-generate-waterfall-reports.py
new file mode 100755
index 00000000..ef48f8be
--- /dev/null
+++ b/new-generate-waterfall-reports.py
@@ -0,0 +1,410 @@
+#!/usr/bin/env python2
+"""Generate summary report for ChromeOS toolchain waterfalls."""
+
+from __future__ import print_function
+
+import argparse
+import datetime
+import getpass
+import json
+import os
+import re
+import shutil
+import sys
+import time
+
+from cros_utils import command_executer
+
+# All the test suites whose data we might want for the reports.
+TESTS = (('bvt-inline', 'HWTest [bvt-inline]'), ('bvt-cq', 'HWTest [bvt-cq]'),
+         ('security', 'HWTest [security]'))
+
+# The main waterfall builders, IN THE ORDER IN WHICH WE WANT THEM
+# LISTED IN THE REPORT.
+WATERFALL_BUILDERS = [
+    'amd64-llvm-next-toolchain',
+    'arm-llvm-next-toolchain',
+    'arm64-llvm-next-toolchain',
+]
+
+DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/'
+ARCHIVE_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-reports/'
+DOWNLOAD_DIR = '/tmp/waterfall-logs'
+MAX_SAVE_RECORDS = 7
+BUILD_DATA_FILE = '%s/build-data.txt' % DATA_DIR
+LLVM_ROTATING_BUILDER = 'llvm_next_toolchain'
+ROTATING_BUILDERS = [LLVM_ROTATING_BUILDER]
+
+# For int-to-string date conversion.  Note, the index of the month in this
+# list needs to correspond to the month's integer value.  i.e. 'Sep' must
+# be as MONTHS[9].
+MONTHS = [
+    '', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
+    'Nov', 'Dec'
+]
+
+DAYS_PER_MONTH = {
+    1: 31,
+    2: 28,
+    3: 31,
+    4: 30,
+    5: 31,
+    6: 30,
+    7: 31,
+    8: 31,
+    9: 30,
+    10: 31,
+    11: 31,
+    12: 31
+}
+
+
+def format_date(int_date, use_int_month=False):
+  """Convert an integer date to a string date. YYYYMMDD -> YYYY-MMM-DD"""
+
+  if int_date == 0:
+    return 'today'
+
+  tmp_date = int_date
+  day = tmp_date % 100
+  tmp_date = tmp_date / 100
+  month = tmp_date % 100
+  year = tmp_date / 100
+
+  if use_int_month:
+    date_str = '%d-%02d-%02d' % (year, month, day)
+  else:
+    month_str = MONTHS[month]
+    date_str = '%d-%s-%d' % (year, month_str, day)
+  return date_str
+
+
+def EmailReport(report_file, report_type, date, email_to):
+  """Emails the report to the approprite address."""
+  subject = '%s Waterfall Summary report, %s' % (report_type, date)
+  sendgmr_path = '/google/data/ro/projects/gws-sre/sendgmr'
+  command = ('%s --to=%s --subject="%s" --body_file=%s' %
+             (sendgmr_path, email_to, subject, report_file))
+  command_executer.GetCommandExecuter().RunCommand(command)
+
+
+def GetColor(status):
+  """Given a job status string, returns appropriate color string."""
+  if status.strip() == 'pass':
+    color = 'green '
+  elif status.strip() == 'fail':
+    color = ' red  '
+  elif status.strip() == 'warning':
+    color = 'orange'
+  else:
+    color = '      '
+  return color
+
+
+def GenerateWaterfallReport(report_dict, waterfall_type, date):
+  """Write out the actual formatted report."""
+
+  filename = 'waterfall_report.%s_waterfall.%s.txt' % (waterfall_type, date)
+
+  date_string = ''
+  report_list = report_dict.keys()
+
+  with open(filename, 'w') as out_file:
+    # Write Report Header
+    out_file.write('\nStatus of %s Waterfall Builds from %s\n\n' %
+                   (waterfall_type, date_string))
+    out_file.write('                                                        \n')
+    out_file.write(
+        '                                         Build       bvt-    '
+        '     bvt-cq     '
+        ' security \n')
+    out_file.write(
+        '                                         status     inline   '
+        '              \n')
+
+    # Write daily waterfall status section.
+    for builder in report_list:
+      build_dict = report_dict[builder]
+      buildbucket_id = build_dict['buildbucket_id']
+      overall_status = build_dict['status']
+      if 'bvt-inline' in build_dict.keys():
+        inline_status = build_dict['bvt-inline']
+      else:
+        inline_status = '    '
+      if 'bvt-cq' in build_dict.keys():
+        cq_status = build_dict['bvt-cq']
+      else:
+        cq_status = '    '
+      if 'security' in build_dict.keys():
+        security_status = build_dict['security']
+      else:
+        security_status = '    '
+      inline_color = GetColor(inline_status)
+      cq_color = GetColor(cq_status)
+      security_color = GetColor(security_status)
+
+      out_file.write(
+          '%26s   %4s        %6s        %6s         %6s\n' %
+          (builder, overall_status, inline_color, cq_color, security_color))
+      if waterfall_type == 'main':
+        out_file.write('     build url: https://cros-goldeneye.corp.google.com/'
+                       'chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
+                       '\n' % buildbucket_id)
+      else:
+        out_file.write('     build url: https://ci.chromium.org/p/chromeos/'
+                       'builds/b%s \n' % buildbucket_id)
+        report_url = ('https://logs.chromium.org/v/?s=chromeos%2Fbuildbucket%2F'
+                      'cr-buildbucket.appspot.com%2F' + buildbucket_id +
+                      '%2F%2B%2Fsteps%2FReport%2F0%2Fstdout')
+        out_file.write('\n     report status url: %s\n' % report_url)
+      out_file.write('\n')
+
+    print('Report generated in %s.' % filename)
+    return filename
+
+
+def GetTryjobData(date, rotating_builds_dict):
+  """Read buildbucket id and board from stored file.
+
+  buildbot_test_llvm.py, when it launches the rotating builders,
+  records the buildbucket_id and board for each launch in a file.
+  This reads that data out of the file so we can find the right
+  tryjob data.
+  """
+
+  date_str = format_date(date, use_int_month=True)
+  fname = '%s.builds' % date_str
+  filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
+
+  if not os.path.exists(filename):
+    print('Cannot find file: %s' % filename)
+    print('Unable to generate rotating builder report for date %d.' % date)
+    return
+
+  with open(filename, 'r') as in_file:
+    lines = in_file.readlines()
+
+  for line in lines:
+    l = line.strip()
+    parts = l.split(',')
+    if len(parts) != 2:
+      print('Warning: Illegal line in data file.')
+      print('File: %s' % filename)
+      print('Line: %s' % l)
+      continue
+    buildbucket_id = parts[0]
+    board = parts[1]
+    rotating_builds_dict[board] = buildbucket_id
+
+  return
+
+
+def GetRotatingBuildData(date, report_dict, chromeos_root, board,
+                         buildbucket_id, ce):
+  """Gets rotating builder job results via 'cros buildresult'."""
+  path = os.path.join(chromeos_root, 'chromite')
+  save_dir = os.getcwd()
+  date_str = format_date(date, use_int_month=True)
+  os.chdir(path)
+
+  command = (
+      'cros buildresult --buildbucket-id %s --report json' % buildbucket_id)
+  _, out, _ = ce.RunCommandWOutput(command)
+  tmp_dict = json.loads(out)
+  results = tmp_dict[buildbucket_id]
+
+  board_dict = dict()
+  board_dict['buildbucket_id'] = buildbucket_id
+  stages_results = results['stages']
+  for test in TESTS:
+    key1 = test[0]
+    key2 = test[1]
+    if key2 in stages_results:
+      board_dict[key1] = stages_results[key2]
+  board_dict['status'] = results['status']
+  report_dict[board] = board_dict
+  os.chdir(save_dir)
+  return
+
+
+def GetMainWaterfallData(date, report_dict, chromeos_root, ce):
+  """Gets main waterfall job results via 'cros buildresult'."""
+  path = os.path.join(chromeos_root, 'chromite')
+  save_dir = os.getcwd()
+  date_str = format_date(date, use_int_month=True)
+  os.chdir(path)
+  for builder in WATERFALL_BUILDERS:
+    command = ('cros buildresult --build-config %s --date %s --report json' %
+               (builder, date_str))
+    _, out, _ = ce.RunCommandWOutput(command)
+    tmp_dict = json.loads(out)
+    builder_dict = dict()
+    for k in tmp_dict.keys():
+      buildbucket_id = k
+      results = tmp_dict[k]
+
+    builder_dict['buildbucket_id'] = buildbucket_id
+    builder_dict['status'] = results['status']
+    stages_results = results['stages']
+    for test in TESTS:
+      key1 = test[0]
+      key2 = test[1]
+      builder_dict[key1] = stages_results[key2]
+    report_dict[builder] = builder_dict
+  os.chdir(save_dir)
+  return
+
+
+# Check for prodaccess.
+def CheckProdAccess():
+  """Verifies prodaccess is current."""
+  status, output, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
+      'prodcertstatus')
+  if status != 0:
+    return False
+  # Verify that status is not expired
+  if 'expires' in output:
+    return True
+  return False
+
+
+def ValidDate(date):
+  """Ensures 'date' is a valid date."""
+  min_year = 2018
+
+  tmp_date = date
+  day = tmp_date % 100
+  tmp_date = tmp_date / 100
+  month = tmp_date % 100
+  year = tmp_date / 100
+
+  if day < 1 or month < 1 or year < min_year:
+    return False
+
+  cur_year = datetime.datetime.now().year
+  if year > cur_year:
+    return False
+
+  if month > 12:
+    return False
+
+  if month == 2 and cur_year % 4 == 0 and cur_year % 100 != 0:
+    max_day = 29
+  else:
+    max_day = DAYS_PER_MONTH[month]
+
+  if day > max_day:
+    return False
+
+  return True
+
+
+def ValidOptions(parser, options):
+  """Error-check the options passed to this script."""
+  too_many_options = False
+  if options.main:
+    if options.rotating:
+      too_many_options = True
+
+  if too_many_options:
+    parser.error('Can only specify one of --main, --rotating.')
+
+  if not os.path.exists(options.chromeos_root):
+    parser.error(
+        'Invalid chromeos root. Cannot find: %s' % options.chromeos_root)
+
+  email_ok = True
+  if options.email and options.email.find('@') == -1:
+    email_ok = False
+    parser.error('"%s" is not a valid email address; it must contain "@..."' %
+                 options.email)
+
+  valid_date = ValidDate(options.date)
+
+  return not too_many_options and valid_date and email_ok
+
+
+def Main(argv):
+  """Main function for this script."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--main',
+      dest='main',
+      default=False,
+      action='store_true',
+      help='Generate report only for main waterfall '
+      'builders.')
+  parser.add_argument(
+      '--rotating',
+      dest='rotating',
+      default=False,
+      action='store_true',
+      help='Generate report only for rotating builders.')
+  parser.add_argument(
+      '--date',
+      dest='date',
+      required=True,
+      type=int,
+      help='The date YYYYMMDD of waterfall report.')
+  parser.add_argument(
+      '--email',
+      dest='email',
+      default='',
+      help='Email address to use for sending the report.')
+  parser.add_argument(
+      '--chromeos_root',
+      dest='chromeos_root',
+      required=True,
+      help='Chrome OS root in which to run chroot commands.')
+
+  options = parser.parse_args(argv)
+
+  if not ValidOptions(parser, options):
+    return 1
+
+  main_only = options.main
+  rotating_only = options.rotating
+  date = options.date
+
+  prod_access = CheckProdAccess()
+  if not prod_access:
+    print('ERROR: Please run prodaccess first.')
+    return
+
+  waterfall_report_dict = dict()
+  rotating_report_dict = dict()
+
+  ce = command_executer.GetCommandExecuter()
+  if not rotating_only:
+    GetMainWaterfallData(date, waterfall_report_dict, options.chromeos_root, ce)
+
+  if not main_only:
+    rotating_builds_dict = dict()
+    GetTryjobData(date, rotating_builds_dict)
+    if len(rotating_builds_dict.keys()) > 0:
+      for board in rotating_builds_dict.keys():
+        buildbucket_id = rotating_builds_dict[board]
+        GetRotatingBuildData(date, rotating_report_dict, options.chromeos_root,
+                             board, buildbucket_id, ce)
+
+  if options.email:
+    email_to = options.email
+  else:
+    email_to = getpass.getuser()
+
+  if waterfall_report_dict and not rotating_only:
+    main_report = GenerateWaterfallReport(waterfall_report_dict, 'main', date)
+
+    EmailReport(main_report, 'Main', format_date(date), email_to)
+    shutil.copy(main_report, ARCHIVE_DIR)
+  if rotating_report_dict and not main_only:
+    rotating_report = GenerateWaterfallReport(rotating_report_dict, 'rotating',
+                                              date)
+
+    EmailReport(rotating_report, 'Rotating', format_date(date), email_to)
+    shutil.copy(rotating_report, ARCHIVE_DIR)
+
+
+if __name__ == '__main__':
+  Main(sys.argv[1:])
+  sys.exit(0)