diff options
Diffstat (limited to 'buildbot_test_toolchains.py')
| -rwxr-xr-x | buildbot_test_toolchains.py | 356 |
1 files changed, 356 insertions, 0 deletions
diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py new file mode 100755 index 00000000..5bef28a4 --- /dev/null +++ b/buildbot_test_toolchains.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python2 +# +# Copyright 2016 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Script for running nightly compiler tests on ChromeOS. + +This script launches a buildbot to build ChromeOS with the latest compiler on +a particular board; then it finds and downloads the trybot image and the +corresponding official image, and runs crosperf performance tests comparing +the two. It then generates a report, emails it to the c-compiler-chrome, as +well as copying the images into the seven-day reports directory. +""" + +# Script to test different toolchains against ChromeOS benchmarks. + +from __future__ import print_function + +import argparse +import datetime +import os +import re +import sys +import time + +from cros_utils import command_executer +from cros_utils import logger + +from cros_utils import buildbot_utils + +# CL that updated GCC ebuilds to use 'next_gcc'. +USE_NEXT_GCC_PATCH = '230260' + +# CL that uses LLVM to build the peppy image. +USE_LLVM_PATCH = '295217' + +# CL that uses LLVM-Next to build the images (includes chrome). +USE_LLVM_NEXT_PATCH = '424123' + +CROSTC_ROOT = '/usr/local/google/crostc' +ROLE_ACCOUNT = 'mobiletc-prebuild' +TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) +MAIL_PROGRAM = '~/var/bin/mail-sheriff' +PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') +NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') + +IMAGE_DIR = '{board}-{image_type}' +IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}' +IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR +TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}' +PFQ_IMAGE_FS = IMAGE_FS + '-rc1' +IMAGE_RE_GROUPS = { + 'board': r'(?P<board>\S+)', + 'image_type': r'(?P<image_type>\S+)', + 'chrome_version': r'(?P<chrome_version>R\d+)', + 'tip': r'(?P<tip>\d+)', + 'branch': r'(?P<branch>\d+)', + 'branch_branch': r'(?P<branch_branch>\d+)', + 'build_id': r'(?P<build_id>b\d+)' +} +TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS) + + +class ToolchainComparator(object): + """Class for doing the nightly tests work.""" + + def __init__(self, + board, + remotes, + chromeos_root, + weekday, + patches, + noschedv2=False): + self._board = board + self._remotes = remotes + self._chromeos_root = chromeos_root + self._base_dir = os.getcwd() + self._ce = command_executer.GetCommandExecuter() + self._l = logger.GetLogger() + self._build = '%s-release' % board + self._patches = patches.split(',') + self._patches_string = '_'.join(str(p) for p in self._patches) + self._noschedv2 = noschedv2 + + if not weekday: + self._weekday = time.strftime('%a') + else: + self._weekday = weekday + timestamp = datetime.datetime.strftime(datetime.datetime.now(), + '%Y-%m-%d_%H:%M:%S') + self._reports_dir = os.path.join( + NIGHTLY_TESTS_DIR, + '%s.%s' % (timestamp, board),) + + def _GetVanillaImageName(self, trybot_image): + """Given a trybot artifact name, get latest vanilla image name. + + Args: + trybot_image: artifact name such as + 'trybot-daisy-release/R40-6394.0.0-b1389' + + Returns: + Latest official image name, e.g. 'daisy-release/R57-9089.0.0'. + """ + mo = re.search(TRYBOT_IMAGE_RE, trybot_image) + assert mo + dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict()) + version = buildbot_utils.GetGSContent(self._chromeos_root, + dirname + '/LATEST-master') + return dirname + '/' + version + + def _GetNonAFDOImageName(self, trybot_image): + """Given a trybot artifact name, get corresponding non-AFDO image name. + + We get the non-AFDO image from the PFQ builders. This image + is not generated for all the boards and, the closest PFQ image + was the one build for the previous ChromeOS version (the chrome + used in the current version is the one validated in the previous + version). + The previous ChromeOS does not always exist either. So, we try + a couple of versions before. + + Args: + trybot_image: artifact name such as + 'trybot-daisy-release/R40-6394.0.0-b1389' + + Returns: + Corresponding chrome PFQ image name, e.g. + 'daisy-chrome-pfq/R40-6393.0.0-rc1'. + """ + mo = re.search(TRYBOT_IMAGE_RE, trybot_image) + assert mo + image_dict = mo.groupdict() + image_dict['image_type'] = 'chrome-pfq' + for _ in xrange(2): + image_dict['tip'] = str(int(image_dict['tip']) - 1) + nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict) + if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image): + return nonafdo_image + return '' + + def _FinishSetup(self): + """Make sure testing_rsa file is properly set up.""" + # Fix protections on ssh key + command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' + '/chrome-src-internal/src/third_party/chromite/ssh_keys' + '/testing_rsa') + ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) + if ret_val != 0: + raise RuntimeError('chmod for testing_rsa failed') + + def _TestImages(self, trybot_image, vanilla_image, nonafdo_image): + """Create crosperf experiment file. + + Given the names of the trybot, vanilla and non-AFDO images, create the + appropriate crosperf experiment file and launch crosperf on it. + """ + experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday) + experiment_file_name = '%s_toolchain_experiment.txt' % self._board + + compiler_string = 'gcc' + if USE_LLVM_NEXT_PATCH in self._patches_string: + experiment_file_name = '%s_llvm_next_experiment.txt' % self._board + compiler_string = 'llvm_next' + elif USE_LLVM_PATCH in self._patches_string: + experiment_file_name = '%s_llvm_experiment.txt' % self._board + compiler_string = 'llvm' + + experiment_file = os.path.join(experiment_file_dir, experiment_file_name) + experiment_header = """ + board: %s + remote: %s + retries: 1 + """ % (self._board, self._remotes) + experiment_tests = """ + benchmark: all_toolchain_perf { + suite: telemetry_Crosperf + iterations: 3 + } + + benchmark: page_cycler_v2.typical_25 { + suite: telemetry_Crosperf + iterations: 2 + run_local: False + retries: 0 + } + """ + + with open(experiment_file, 'w') as f: + f.write(experiment_header) + f.write(experiment_tests) + + # Now add vanilla to test file. + official_image = """ + vanilla_image { + chromeos_root: %s + build: %s + compiler: gcc + } + """ % (self._chromeos_root, vanilla_image) + f.write(official_image) + + # Now add non-AFDO image to test file. + if nonafdo_image: + official_nonafdo_image = """ + nonafdo_image { + chromeos_root: %s + build: %s + compiler: gcc + } + """ % (self._chromeos_root, nonafdo_image) + f.write(official_nonafdo_image) + + label_string = '%s_trybot_image' % compiler_string + if USE_NEXT_GCC_PATCH in self._patches: + label_string = 'gcc_next_trybot_image' + + # Reuse autotest files from vanilla image for trybot images + autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files') + experiment_image = """ + %s { + chromeos_root: %s + build: %s + autotest_path: %s + compiler: %s + } + """ % (label_string, self._chromeos_root, trybot_image, + autotest_files, compiler_string) + f.write(experiment_image) + + crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf') + noschedv2_opts = '--noschedv2' if self._noschedv2 else '' + command = ('{crosperf} --no_email=True --results_dir={r_dir} ' + '--json_report=True {noschedv2_opts} {exp_file}').format( + crosperf=crosperf, + r_dir=self._reports_dir, + noschedv2_opts=noschedv2_opts, + exp_file=experiment_file) + + ret = self._ce.RunCommand(command) + if ret != 0: + raise RuntimeError('Crosperf execution error!') + else: + # Copy json report to pending archives directory. + command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) + ret = self._ce.RunCommand(command) + return + + def _SendEmail(self): + """Find email message generated by crosperf and send it.""" + filename = os.path.join(self._reports_dir, 'msg_body.html') + if (os.path.exists(filename) and + os.path.exists(os.path.expanduser(MAIL_PROGRAM))): + email_title = 'buildbot test results' + if USE_LLVM_NEXT_PATCH in self._patches_string: + email_title = 'buildbot llvm_next test results' + elif USE_LLVM_PATCH in self._patches_string: + email_title = 'buildbot llvm test results' + command = ('cat %s | %s -s "%s, %s" -team -html' % + (filename, MAIL_PROGRAM, email_title, self._board)) + self._ce.RunCommand(command) + + def DoAll(self): + """Main function inside ToolchainComparator class. + + Launch trybot, get image names, create crosperf experiment file, run + crosperf, and copy images into seven-day report directories. + """ + date_str = datetime.date.today() + description = 'master_%s_%s_%s' % (self._patches_string, self._build, + date_str) + build_id, trybot_image = buildbot_utils.GetTrybotImage( + self._chromeos_root, + self._build, + self._patches, + description, + other_flags=['--notests'], + build_toolchain=True) + + print('trybot_url: \ + https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/release/builds/%s' + % build_id) + if len(trybot_image) == 0: + self._l.LogError('Unable to find trybot_image for %s!' % description) + return 1 + + vanilla_image = self._GetVanillaImageName(trybot_image) + nonafdo_image = self._GetNonAFDOImageName(trybot_image) + + print('trybot_image: %s' % trybot_image) + print('vanilla_image: %s' % vanilla_image) + print('nonafdo_image: %s' % nonafdo_image) + + if os.getlogin() == ROLE_ACCOUNT: + self._FinishSetup() + + self._TestImages(trybot_image, vanilla_image, nonafdo_image) + self._SendEmail() + return 0 + + +def Main(argv): + """The main function.""" + + # Common initializations + command_executer.InitCommandExecuter() + parser = argparse.ArgumentParser() + parser.add_argument( + '--remote', dest='remote', help='Remote machines to run tests on.') + parser.add_argument( + '--board', dest='board', default='x86-zgb', help='The target board.') + parser.add_argument( + '--chromeos_root', + dest='chromeos_root', + help='The chromeos root from which to run tests.') + parser.add_argument( + '--weekday', + default='', + dest='weekday', + help='The day of the week for which to run tests.') + parser.add_argument( + '--patch', + dest='patches', + help='The patches to use for the testing, ' + "seprate the patch numbers with ',' " + 'for more than one patches.') + parser.add_argument( + '--noschedv2', + dest='noschedv2', + action='store_true', + default=False, + help='Pass --noschedv2 to crosperf.') + + options = parser.parse_args(argv[1:]) + if not options.board: + print('Please give a board.') + return 1 + if not options.remote: + print('Please give at least one remote machine.') + return 1 + if not options.chromeos_root: + print('Please specify the ChromeOS root directory.') + return 1 + if options.patches: + patches = options.patches + else: + patches = USE_NEXT_GCC_PATCH + + fc = ToolchainComparator(options.board, options.remote, options.chromeos_root, + options.weekday, patches, options.noschedv2) + return fc.DoAll() + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) |