diff options
Diffstat (limited to 'deprecated/test_toolchains.py')
| -rwxr-xr-x | deprecated/test_toolchains.py | 364 |
1 files changed, 0 insertions, 364 deletions
diff --git a/deprecated/test_toolchains.py b/deprecated/test_toolchains.py deleted file mode 100755 index 8684653f..00000000 --- a/deprecated/test_toolchains.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python2 - -# Script to test different toolchains against ChromeOS benchmarks. -"""Toolchain team nightly performance test script (local builds).""" - -from __future__ import print_function - -import argparse -import datetime -import os -import sys -import build_chromeos -import setup_chromeos -from cros_utils import command_executer -from cros_utils import misc -from cros_utils import logger - -CROSTC_ROOT = '/usr/local/google/crostc' -MAIL_PROGRAM = '~/var/bin/mail-sheriff' -PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') -NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') - - -class GCCConfig(object): - """GCC configuration class.""" - - def __init__(self, githash): - self.githash = githash - - -class ToolchainConfig(object): - """Toolchain configuration class.""" - - def __init__(self, gcc_config=None): - self.gcc_config = gcc_config - - -class ChromeOSCheckout(object): - """Main class for checking out, building and testing ChromeOS.""" - - def __init__(self, board, chromeos_root): - self._board = board - self._chromeos_root = chromeos_root - self._ce = command_executer.GetCommandExecuter() - self._l = logger.GetLogger() - self._build_num = None - - def _DeleteChroot(self): - command = 'cd %s; cros_sdk --delete' % self._chromeos_root - return self._ce.RunCommand(command) - - def _DeleteCcahe(self): - # crosbug.com/34956 - command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache') - return self._ce.RunCommand(command) - - def _GetBuildNumber(self): - """Get the build number of the ChromeOS image from the chroot. - - This function assumes a ChromeOS image has been built in the chroot. - It translates the 'latest' symlink in the - <chroot>/src/build/images/<board> directory, to find the actual - ChromeOS build number for the image that was built. For example, if - src/build/image/lumpy/latest -> R37-5982.0.2014_06_23_0454-a1, then - This function would parse it out and assign 'R37-5982' to self._build_num. - This is used to determine the official, vanilla build to use for - comparison tests. - """ - # Get the path to 'latest' - sym_path = os.path.join( - misc.GetImageDir(self._chromeos_root, self._board), 'latest') - # Translate the symlink to its 'real' path. - real_path = os.path.realpath(sym_path) - # Break up the path and get the last piece - # (e.g. 'R37-5982.0.2014_06_23_0454-a1" - path_pieces = real_path.split('/') - last_piece = path_pieces[-1] - # Break this piece into the image number + other pieces, and get the - # image number [ 'R37-5982', '0', '2014_06_23_0454-a1'] - image_parts = last_piece.split('.') - self._build_num = image_parts[0] - - def _BuildLabelName(self, config): - pieces = config.split('/') - compiler_version = pieces[-1] - label = compiler_version + '_tot_afdo' - return label - - def _BuildAndImage(self, label=''): - if (not label or - not misc.DoesLabelExist(self._chromeos_root, self._board, label)): - build_chromeos_args = [ - build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root, - '--board=%s' % self._board, '--rebuild' - ] - if self._public: - build_chromeos_args.append('--env=USE=-chrome_internal') - - ret = build_chromeos.Main(build_chromeos_args) - if ret != 0: - raise RuntimeError("Couldn't build ChromeOS!") - - if not self._build_num: - self._GetBuildNumber() - # Check to see if we need to create the symbolic link for the vanilla - # image, and do so if appropriate. - if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'): - build_name = '%s-release/%s.0.0' % (self._board, self._build_num) - full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root, - 'chroot/tmp', build_name) - misc.LabelLatestImage(self._chromeos_root, self._board, label, - full_vanilla_path) - else: - misc.LabelLatestImage(self._chromeos_root, self._board, label) - return label - - def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag): - env_string = misc.GetEnvStringFromDict(env_dict) - command = ('%s %s' % (env_string, misc.GetSetupBoardCommand( - self._board, usepkg=usepkg_flag, force=clobber_flag))) - ret = self._ce.ChrootRunCommand(self._chromeos_root, command) - error_str = "Could not setup board: '%s'" % command - assert ret == 0, error_str - - def _UnInstallToolchain(self): - command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' % - misc.GetCtargetFromBoard(self._board, self._chromeos_root)) - ret = self._ce.ChrootRunCommand(self._chromeos_root, command) - if ret != 0: - raise RuntimeError("Couldn't uninstall the toolchain!") - - def _CheckoutChromeOS(self): - # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot). - if not os.path.exists(self._chromeos_root): - setup_chromeos_args = ['--dir=%s' % self._chromeos_root] - if self._public: - setup_chromeos_args.append('--public') - ret = setup_chromeos.Main(setup_chromeos_args) - if ret != 0: - raise RuntimeError("Couldn't run setup_chromeos!") - - def _BuildToolchain(self, config): - # Call setup_board for basic, vanilla setup. - self._SetupBoard({}, usepkg_flag=True, clobber_flag=False) - # Now uninstall the vanilla compiler and setup/build our custom - # compiler. - self._UnInstallToolchain() - envdict = { - 'USE': 'git_gcc', - 'GCC_GITHASH': config.gcc_config.githash, - 'EMERGE_DEFAULT_OPTS': '--exclude=gcc' - } - self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False) - - -class ToolchainComparator(ChromeOSCheckout): - """Main class for running tests and generating reports.""" - - def __init__(self, - board, - remotes, - configs, - clean, - public, - force_mismatch, - noschedv2=False): - self._board = board - self._remotes = remotes - self._chromeos_root = 'chromeos' - self._configs = configs - self._clean = clean - self._public = public - self._force_mismatch = force_mismatch - self._ce = command_executer.GetCommandExecuter() - self._l = logger.GetLogger() - timestamp = datetime.datetime.strftime(datetime.datetime.now(), - '%Y-%m-%d_%H:%M:%S') - self._reports_dir = os.path.join( - NIGHTLY_TESTS_DIR, - '%s.%s' % (timestamp, board),) - self._noschedv2 = noschedv2 - ChromeOSCheckout.__init__(self, board, self._chromeos_root) - - def _FinishSetup(self): - # Get correct .boto file - current_dir = os.getcwd() - src = '/usr/local/google/home/mobiletc-prebuild/.boto' - dest = os.path.join(current_dir, self._chromeos_root, - 'src/private-overlays/chromeos-overlay/' - 'googlestorage_account.boto') - # Copy the file to the correct place - copy_cmd = 'cp %s %s' % (src, dest) - retv = self._ce.RunCommand(copy_cmd) - if retv != 0: - raise RuntimeError("Couldn't copy .boto file for google storage.") - - # Fix protections on ssh key - command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' - '/chrome-src-internal/src/third_party/chromite/ssh_keys' - '/testing_rsa') - retv = self._ce.ChrootRunCommand(self._chromeos_root, command) - if retv != 0: - raise RuntimeError('chmod for testing_rsa failed') - - def _TestLabels(self, labels): - experiment_file = 'toolchain_experiment.txt' - image_args = '' - if self._force_mismatch: - image_args = '--force-mismatch' - experiment_header = """ - board: %s - remote: %s - retries: 1 - """ % (self._board, self._remotes) - experiment_tests = """ - benchmark: all_toolchain_perf { - suite: telemetry_Crosperf - iterations: 3 - } - """ - - with open(experiment_file, 'w') as f: - f.write(experiment_header) - f.write(experiment_tests) - for label in labels: - # TODO(asharif): Fix crosperf so it accepts labels with symbols - crosperf_label = label - crosperf_label = crosperf_label.replace('-', '_') - crosperf_label = crosperf_label.replace('+', '_') - crosperf_label = crosperf_label.replace('.', '') - - # Use the official build instead of building vanilla ourselves. - if label == 'vanilla': - build_name = '%s-release/%s.0.0' % (self._board, self._build_num) - - # Now add 'official build' to test file. - official_image = """ - official_image { - chromeos_root: %s - build: %s - } - """ % (self._chromeos_root, build_name) - f.write(official_image) - - else: - experiment_image = """ - %s { - chromeos_image: %s - image_args: %s - } - """ % (crosperf_label, os.path.join( - misc.GetImageDir(self._chromeos_root, self._board), label, - 'chromiumos_test_image.bin'), image_args) - f.write(experiment_image) - - crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf') - noschedv2_opts = '--noschedv2' if self._noschedv2 else '' - command = ('{crosperf} --no_email=True --results_dir={r_dir} ' - '--json_report=True {noschedv2_opts} {exp_file}').format( - crosperf=crosperf, - r_dir=self._reports_dir, - noschedv2_opts=noschedv2_opts, - exp_file=experiment_file) - - ret = self._ce.RunCommand(command) - if ret != 0: - raise RuntimeError('Crosperf execution error!') - else: - # Copy json report to pending archives directory. - command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) - ret = self._ce.RunCommand(command) - return - - def _SendEmail(self): - """Find email msesage generated by crosperf and send it.""" - filename = os.path.join(self._reports_dir, 'msg_body.html') - if (os.path.exists(filename) and - os.path.exists(os.path.expanduser(MAIL_PROGRAM))): - command = ('cat %s | %s -s "Nightly test results, %s" -team -html' % - (filename, MAIL_PROGRAM, self._board)) - self._ce.RunCommand(command) - - def DoAll(self): - self._CheckoutChromeOS() - labels = [] - labels.append('vanilla') - for config in self._configs: - label = self._BuildLabelName(config.gcc_config.githash) - if not misc.DoesLabelExist(self._chromeos_root, self._board, label): - self._BuildToolchain(config) - label = self._BuildAndImage(label) - labels.append(label) - self._FinishSetup() - self._TestLabels(labels) - self._SendEmail() - if self._clean: - ret = self._DeleteChroot() - if ret != 0: - return ret - ret = self._DeleteCcahe() - if ret != 0: - return ret - return 0 - - -def Main(argv): - """The main function.""" - # Common initializations - ### command_executer.InitCommandExecuter(True) - command_executer.InitCommandExecuter() - parser = argparse.ArgumentParser() - parser.add_argument( - '--remote', dest='remote', help='Remote machines to run tests on.') - parser.add_argument( - '--board', dest='board', default='x86-alex', help='The target board.') - parser.add_argument( - '--githashes', - dest='githashes', - default='master', - help='The gcc githashes to test.') - parser.add_argument( - '--clean', - dest='clean', - default=False, - action='store_true', - help='Clean the chroot after testing.') - parser.add_argument( - '--public', - dest='public', - default=False, - action='store_true', - help='Use the public checkout/build.') - parser.add_argument( - '--force-mismatch', - dest='force_mismatch', - default='', - help='Force the image regardless of board mismatch') - parser.add_argument( - '--noschedv2', - dest='noschedv2', - action='store_true', - default=False, - help='Pass --noschedv2 to crosperf.') - options = parser.parse_args(argv) - if not options.board: - print('Please give a board.') - return 1 - if not options.remote: - print('Please give at least one remote machine.') - return 1 - toolchain_configs = [] - for githash in options.githashes.split(','): - gcc_config = GCCConfig(githash=githash) - toolchain_config = ToolchainConfig(gcc_config=gcc_config) - toolchain_configs.append(toolchain_config) - fc = ToolchainComparator(options.board, options.remote, toolchain_configs, - options.clean, options.public, - options.force_mismatch, options.noschedv2) - return fc.DoAll() - - -if __name__ == '__main__': - retval = Main(sys.argv[1:]) - sys.exit(retval) |