aboutsummaryrefslogtreecommitdiff
path: root/deprecated/new-generate-waterfall-reports.py
diff options
context:
space:
mode:
Diffstat (limited to 'deprecated/new-generate-waterfall-reports.py')
-rwxr-xr-xdeprecated/new-generate-waterfall-reports.py410
1 files changed, 0 insertions, 410 deletions
diff --git a/deprecated/new-generate-waterfall-reports.py b/deprecated/new-generate-waterfall-reports.py
deleted file mode 100755
index ef48f8be..00000000
--- a/deprecated/new-generate-waterfall-reports.py
+++ /dev/null
@@ -1,410 +0,0 @@
-#!/usr/bin/env python2
-"""Generate summary report for ChromeOS toolchain waterfalls."""
-
-from __future__ import print_function
-
-import argparse
-import datetime
-import getpass
-import json
-import os
-import re
-import shutil
-import sys
-import time
-
-from cros_utils import command_executer
-
-# All the test suites whose data we might want for the reports.
-TESTS = (('bvt-inline', 'HWTest [bvt-inline]'), ('bvt-cq', 'HWTest [bvt-cq]'),
- ('security', 'HWTest [security]'))
-
-# The main waterfall builders, IN THE ORDER IN WHICH WE WANT THEM
-# LISTED IN THE REPORT.
-WATERFALL_BUILDERS = [
- 'amd64-llvm-next-toolchain',
- 'arm-llvm-next-toolchain',
- 'arm64-llvm-next-toolchain',
-]
-
-DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/'
-ARCHIVE_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-reports/'
-DOWNLOAD_DIR = '/tmp/waterfall-logs'
-MAX_SAVE_RECORDS = 7
-BUILD_DATA_FILE = '%s/build-data.txt' % DATA_DIR
-LLVM_ROTATING_BUILDER = 'llvm_next_toolchain'
-ROTATING_BUILDERS = [LLVM_ROTATING_BUILDER]
-
-# For int-to-string date conversion. Note, the index of the month in this
-# list needs to correspond to the month's integer value. i.e. 'Sep' must
-# be as MONTHS[9].
-MONTHS = [
- '', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
- 'Nov', 'Dec'
-]
-
-DAYS_PER_MONTH = {
- 1: 31,
- 2: 28,
- 3: 31,
- 4: 30,
- 5: 31,
- 6: 30,
- 7: 31,
- 8: 31,
- 9: 30,
- 10: 31,
- 11: 31,
- 12: 31
-}
-
-
-def format_date(int_date, use_int_month=False):
- """Convert an integer date to a string date. YYYYMMDD -> YYYY-MMM-DD"""
-
- if int_date == 0:
- return 'today'
-
- tmp_date = int_date
- day = tmp_date % 100
- tmp_date = tmp_date / 100
- month = tmp_date % 100
- year = tmp_date / 100
-
- if use_int_month:
- date_str = '%d-%02d-%02d' % (year, month, day)
- else:
- month_str = MONTHS[month]
- date_str = '%d-%s-%d' % (year, month_str, day)
- return date_str
-
-
-def EmailReport(report_file, report_type, date, email_to):
- """Emails the report to the approprite address."""
- subject = '%s Waterfall Summary report, %s' % (report_type, date)
- sendgmr_path = '/google/data/ro/projects/gws-sre/sendgmr'
- command = ('%s --to=%s --subject="%s" --body_file=%s' %
- (sendgmr_path, email_to, subject, report_file))
- command_executer.GetCommandExecuter().RunCommand(command)
-
-
-def GetColor(status):
- """Given a job status string, returns appropriate color string."""
- if status.strip() == 'pass':
- color = 'green '
- elif status.strip() == 'fail':
- color = ' red '
- elif status.strip() == 'warning':
- color = 'orange'
- else:
- color = ' '
- return color
-
-
-def GenerateWaterfallReport(report_dict, waterfall_type, date):
- """Write out the actual formatted report."""
-
- filename = 'waterfall_report.%s_waterfall.%s.txt' % (waterfall_type, date)
-
- date_string = ''
- report_list = report_dict.keys()
-
- with open(filename, 'w') as out_file:
- # Write Report Header
- out_file.write('\nStatus of %s Waterfall Builds from %s\n\n' %
- (waterfall_type, date_string))
- out_file.write(' \n')
- out_file.write(
- ' Build bvt- '
- ' bvt-cq '
- ' security \n')
- out_file.write(
- ' status inline '
- ' \n')
-
- # Write daily waterfall status section.
- for builder in report_list:
- build_dict = report_dict[builder]
- buildbucket_id = build_dict['buildbucket_id']
- overall_status = build_dict['status']
- if 'bvt-inline' in build_dict.keys():
- inline_status = build_dict['bvt-inline']
- else:
- inline_status = ' '
- if 'bvt-cq' in build_dict.keys():
- cq_status = build_dict['bvt-cq']
- else:
- cq_status = ' '
- if 'security' in build_dict.keys():
- security_status = build_dict['security']
- else:
- security_status = ' '
- inline_color = GetColor(inline_status)
- cq_color = GetColor(cq_status)
- security_color = GetColor(security_status)
-
- out_file.write(
- '%26s %4s %6s %6s %6s\n' %
- (builder, overall_status, inline_color, cq_color, security_color))
- if waterfall_type == 'main':
- out_file.write(' build url: https://cros-goldeneye.corp.google.com/'
- 'chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
- '\n' % buildbucket_id)
- else:
- out_file.write(' build url: https://ci.chromium.org/p/chromeos/'
- 'builds/b%s \n' % buildbucket_id)
- report_url = ('https://logs.chromium.org/v/?s=chromeos%2Fbuildbucket%2F'
- 'cr-buildbucket.appspot.com%2F' + buildbucket_id +
- '%2F%2B%2Fsteps%2FReport%2F0%2Fstdout')
- out_file.write('\n report status url: %s\n' % report_url)
- out_file.write('\n')
-
- print('Report generated in %s.' % filename)
- return filename
-
-
-def GetTryjobData(date, rotating_builds_dict):
- """Read buildbucket id and board from stored file.
-
- buildbot_test_llvm.py, when it launches the rotating builders,
- records the buildbucket_id and board for each launch in a file.
- This reads that data out of the file so we can find the right
- tryjob data.
- """
-
- date_str = format_date(date, use_int_month=True)
- fname = '%s.builds' % date_str
- filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
-
- if not os.path.exists(filename):
- print('Cannot find file: %s' % filename)
- print('Unable to generate rotating builder report for date %d.' % date)
- return
-
- with open(filename, 'r') as in_file:
- lines = in_file.readlines()
-
- for line in lines:
- l = line.strip()
- parts = l.split(',')
- if len(parts) != 2:
- print('Warning: Illegal line in data file.')
- print('File: %s' % filename)
- print('Line: %s' % l)
- continue
- buildbucket_id = parts[0]
- board = parts[1]
- rotating_builds_dict[board] = buildbucket_id
-
- return
-
-
-def GetRotatingBuildData(date, report_dict, chromeos_root, board,
- buildbucket_id, ce):
- """Gets rotating builder job results via 'cros buildresult'."""
- path = os.path.join(chromeos_root, 'chromite')
- save_dir = os.getcwd()
- date_str = format_date(date, use_int_month=True)
- os.chdir(path)
-
- command = (
- 'cros buildresult --buildbucket-id %s --report json' % buildbucket_id)
- _, out, _ = ce.RunCommandWOutput(command)
- tmp_dict = json.loads(out)
- results = tmp_dict[buildbucket_id]
-
- board_dict = dict()
- board_dict['buildbucket_id'] = buildbucket_id
- stages_results = results['stages']
- for test in TESTS:
- key1 = test[0]
- key2 = test[1]
- if key2 in stages_results:
- board_dict[key1] = stages_results[key2]
- board_dict['status'] = results['status']
- report_dict[board] = board_dict
- os.chdir(save_dir)
- return
-
-
-def GetMainWaterfallData(date, report_dict, chromeos_root, ce):
- """Gets main waterfall job results via 'cros buildresult'."""
- path = os.path.join(chromeos_root, 'chromite')
- save_dir = os.getcwd()
- date_str = format_date(date, use_int_month=True)
- os.chdir(path)
- for builder in WATERFALL_BUILDERS:
- command = ('cros buildresult --build-config %s --date %s --report json' %
- (builder, date_str))
- _, out, _ = ce.RunCommandWOutput(command)
- tmp_dict = json.loads(out)
- builder_dict = dict()
- for k in tmp_dict.keys():
- buildbucket_id = k
- results = tmp_dict[k]
-
- builder_dict['buildbucket_id'] = buildbucket_id
- builder_dict['status'] = results['status']
- stages_results = results['stages']
- for test in TESTS:
- key1 = test[0]
- key2 = test[1]
- builder_dict[key1] = stages_results[key2]
- report_dict[builder] = builder_dict
- os.chdir(save_dir)
- return
-
-
-# Check for prodaccess.
-def CheckProdAccess():
- """Verifies prodaccess is current."""
- status, output, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
- 'prodcertstatus')
- if status != 0:
- return False
- # Verify that status is not expired
- if 'expires' in output:
- return True
- return False
-
-
-def ValidDate(date):
- """Ensures 'date' is a valid date."""
- min_year = 2018
-
- tmp_date = date
- day = tmp_date % 100
- tmp_date = tmp_date / 100
- month = tmp_date % 100
- year = tmp_date / 100
-
- if day < 1 or month < 1 or year < min_year:
- return False
-
- cur_year = datetime.datetime.now().year
- if year > cur_year:
- return False
-
- if month > 12:
- return False
-
- if month == 2 and cur_year % 4 == 0 and cur_year % 100 != 0:
- max_day = 29
- else:
- max_day = DAYS_PER_MONTH[month]
-
- if day > max_day:
- return False
-
- return True
-
-
-def ValidOptions(parser, options):
- """Error-check the options passed to this script."""
- too_many_options = False
- if options.main:
- if options.rotating:
- too_many_options = True
-
- if too_many_options:
- parser.error('Can only specify one of --main, --rotating.')
-
- if not os.path.exists(options.chromeos_root):
- parser.error(
- 'Invalid chromeos root. Cannot find: %s' % options.chromeos_root)
-
- email_ok = True
- if options.email and options.email.find('@') == -1:
- email_ok = False
- parser.error('"%s" is not a valid email address; it must contain "@..."' %
- options.email)
-
- valid_date = ValidDate(options.date)
-
- return not too_many_options and valid_date and email_ok
-
-
-def Main(argv):
- """Main function for this script."""
- parser = argparse.ArgumentParser()
- parser.add_argument(
- '--main',
- dest='main',
- default=False,
- action='store_true',
- help='Generate report only for main waterfall '
- 'builders.')
- parser.add_argument(
- '--rotating',
- dest='rotating',
- default=False,
- action='store_true',
- help='Generate report only for rotating builders.')
- parser.add_argument(
- '--date',
- dest='date',
- required=True,
- type=int,
- help='The date YYYYMMDD of waterfall report.')
- parser.add_argument(
- '--email',
- dest='email',
- default='',
- help='Email address to use for sending the report.')
- parser.add_argument(
- '--chromeos_root',
- dest='chromeos_root',
- required=True,
- help='Chrome OS root in which to run chroot commands.')
-
- options = parser.parse_args(argv)
-
- if not ValidOptions(parser, options):
- return 1
-
- main_only = options.main
- rotating_only = options.rotating
- date = options.date
-
- prod_access = CheckProdAccess()
- if not prod_access:
- print('ERROR: Please run prodaccess first.')
- return
-
- waterfall_report_dict = dict()
- rotating_report_dict = dict()
-
- ce = command_executer.GetCommandExecuter()
- if not rotating_only:
- GetMainWaterfallData(date, waterfall_report_dict, options.chromeos_root, ce)
-
- if not main_only:
- rotating_builds_dict = dict()
- GetTryjobData(date, rotating_builds_dict)
- if len(rotating_builds_dict.keys()) > 0:
- for board in rotating_builds_dict.keys():
- buildbucket_id = rotating_builds_dict[board]
- GetRotatingBuildData(date, rotating_report_dict, options.chromeos_root,
- board, buildbucket_id, ce)
-
- if options.email:
- email_to = options.email
- else:
- email_to = getpass.getuser()
-
- if waterfall_report_dict and not rotating_only:
- main_report = GenerateWaterfallReport(waterfall_report_dict, 'main', date)
-
- EmailReport(main_report, 'Main', format_date(date), email_to)
- shutil.copy(main_report, ARCHIVE_DIR)
- if rotating_report_dict and not main_only:
- rotating_report = GenerateWaterfallReport(rotating_report_dict, 'rotating',
- date)
-
- EmailReport(rotating_report, 'Rotating', format_date(date), email_to)
- shutil.copy(rotating_report, ARCHIVE_DIR)
-
-
-if __name__ == '__main__':
- Main(sys.argv[1:])
- sys.exit(0)
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-15 20:10:04 +0000