diff options
Diffstat (limited to 'deprecated/test_toolchains.py')
| -rwxr-xr-x | deprecated/test_toolchains.py | 364 |
1 files changed, 364 insertions, 0 deletions
diff --git a/deprecated/test_toolchains.py b/deprecated/test_toolchains.py new file mode 100755 index 00000000..8684653f --- /dev/null +++ b/deprecated/test_toolchains.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python2 + +# Script to test different toolchains against ChromeOS benchmarks. +"""Toolchain team nightly performance test script (local builds).""" + +from __future__ import print_function + +import argparse +import datetime +import os +import sys +import build_chromeos +import setup_chromeos +from cros_utils import command_executer +from cros_utils import misc +from cros_utils import logger + +CROSTC_ROOT = '/usr/local/google/crostc' +MAIL_PROGRAM = '~/var/bin/mail-sheriff' +PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') +NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') + + +class GCCConfig(object): + """GCC configuration class.""" + + def __init__(self, githash): + self.githash = githash + + +class ToolchainConfig(object): + """Toolchain configuration class.""" + + def __init__(self, gcc_config=None): + self.gcc_config = gcc_config + + +class ChromeOSCheckout(object): + """Main class for checking out, building and testing ChromeOS.""" + + def __init__(self, board, chromeos_root): + self._board = board + self._chromeos_root = chromeos_root + self._ce = command_executer.GetCommandExecuter() + self._l = logger.GetLogger() + self._build_num = None + + def _DeleteChroot(self): + command = 'cd %s; cros_sdk --delete' % self._chromeos_root + return self._ce.RunCommand(command) + + def _DeleteCcahe(self): + # crosbug.com/34956 + command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache') + return self._ce.RunCommand(command) + + def _GetBuildNumber(self): + """Get the build number of the ChromeOS image from the chroot. + + This function assumes a ChromeOS image has been built in the chroot. + It translates the 'latest' symlink in the + <chroot>/src/build/images/<board> directory, to find the actual + ChromeOS build number for the image that was built. For example, if + src/build/image/lumpy/latest -> R37-5982.0.2014_06_23_0454-a1, then + This function would parse it out and assign 'R37-5982' to self._build_num. + This is used to determine the official, vanilla build to use for + comparison tests. + """ + # Get the path to 'latest' + sym_path = os.path.join( + misc.GetImageDir(self._chromeos_root, self._board), 'latest') + # Translate the symlink to its 'real' path. + real_path = os.path.realpath(sym_path) + # Break up the path and get the last piece + # (e.g. 'R37-5982.0.2014_06_23_0454-a1" + path_pieces = real_path.split('/') + last_piece = path_pieces[-1] + # Break this piece into the image number + other pieces, and get the + # image number [ 'R37-5982', '0', '2014_06_23_0454-a1'] + image_parts = last_piece.split('.') + self._build_num = image_parts[0] + + def _BuildLabelName(self, config): + pieces = config.split('/') + compiler_version = pieces[-1] + label = compiler_version + '_tot_afdo' + return label + + def _BuildAndImage(self, label=''): + if (not label or + not misc.DoesLabelExist(self._chromeos_root, self._board, label)): + build_chromeos_args = [ + build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root, + '--board=%s' % self._board, '--rebuild' + ] + if self._public: + build_chromeos_args.append('--env=USE=-chrome_internal') + + ret = build_chromeos.Main(build_chromeos_args) + if ret != 0: + raise RuntimeError("Couldn't build ChromeOS!") + + if not self._build_num: + self._GetBuildNumber() + # Check to see if we need to create the symbolic link for the vanilla + # image, and do so if appropriate. + if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'): + build_name = '%s-release/%s.0.0' % (self._board, self._build_num) + full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root, + 'chroot/tmp', build_name) + misc.LabelLatestImage(self._chromeos_root, self._board, label, + full_vanilla_path) + else: + misc.LabelLatestImage(self._chromeos_root, self._board, label) + return label + + def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag): + env_string = misc.GetEnvStringFromDict(env_dict) + command = ('%s %s' % (env_string, misc.GetSetupBoardCommand( + self._board, usepkg=usepkg_flag, force=clobber_flag))) + ret = self._ce.ChrootRunCommand(self._chromeos_root, command) + error_str = "Could not setup board: '%s'" % command + assert ret == 0, error_str + + def _UnInstallToolchain(self): + command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' % + misc.GetCtargetFromBoard(self._board, self._chromeos_root)) + ret = self._ce.ChrootRunCommand(self._chromeos_root, command) + if ret != 0: + raise RuntimeError("Couldn't uninstall the toolchain!") + + def _CheckoutChromeOS(self): + # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot). + if not os.path.exists(self._chromeos_root): + setup_chromeos_args = ['--dir=%s' % self._chromeos_root] + if self._public: + setup_chromeos_args.append('--public') + ret = setup_chromeos.Main(setup_chromeos_args) + if ret != 0: + raise RuntimeError("Couldn't run setup_chromeos!") + + def _BuildToolchain(self, config): + # Call setup_board for basic, vanilla setup. + self._SetupBoard({}, usepkg_flag=True, clobber_flag=False) + # Now uninstall the vanilla compiler and setup/build our custom + # compiler. + self._UnInstallToolchain() + envdict = { + 'USE': 'git_gcc', + 'GCC_GITHASH': config.gcc_config.githash, + 'EMERGE_DEFAULT_OPTS': '--exclude=gcc' + } + self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False) + + +class ToolchainComparator(ChromeOSCheckout): + """Main class for running tests and generating reports.""" + + def __init__(self, + board, + remotes, + configs, + clean, + public, + force_mismatch, + noschedv2=False): + self._board = board + self._remotes = remotes + self._chromeos_root = 'chromeos' + self._configs = configs + self._clean = clean + self._public = public + self._force_mismatch = force_mismatch + self._ce = command_executer.GetCommandExecuter() + self._l = logger.GetLogger() + timestamp = datetime.datetime.strftime(datetime.datetime.now(), + '%Y-%m-%d_%H:%M:%S') + self._reports_dir = os.path.join( + NIGHTLY_TESTS_DIR, + '%s.%s' % (timestamp, board),) + self._noschedv2 = noschedv2 + ChromeOSCheckout.__init__(self, board, self._chromeos_root) + + def _FinishSetup(self): + # Get correct .boto file + current_dir = os.getcwd() + src = '/usr/local/google/home/mobiletc-prebuild/.boto' + dest = os.path.join(current_dir, self._chromeos_root, + 'src/private-overlays/chromeos-overlay/' + 'googlestorage_account.boto') + # Copy the file to the correct place + copy_cmd = 'cp %s %s' % (src, dest) + retv = self._ce.RunCommand(copy_cmd) + if retv != 0: + raise RuntimeError("Couldn't copy .boto file for google storage.") + + # Fix protections on ssh key + command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' + '/chrome-src-internal/src/third_party/chromite/ssh_keys' + '/testing_rsa') + retv = self._ce.ChrootRunCommand(self._chromeos_root, command) + if retv != 0: + raise RuntimeError('chmod for testing_rsa failed') + + def _TestLabels(self, labels): + experiment_file = 'toolchain_experiment.txt' + image_args = '' + if self._force_mismatch: + image_args = '--force-mismatch' + experiment_header = """ + board: %s + remote: %s + retries: 1 + """ % (self._board, self._remotes) + experiment_tests = """ + benchmark: all_toolchain_perf { + suite: telemetry_Crosperf + iterations: 3 + } + """ + + with open(experiment_file, 'w') as f: + f.write(experiment_header) + f.write(experiment_tests) + for label in labels: + # TODO(asharif): Fix crosperf so it accepts labels with symbols + crosperf_label = label + crosperf_label = crosperf_label.replace('-', '_') + crosperf_label = crosperf_label.replace('+', '_') + crosperf_label = crosperf_label.replace('.', '') + + # Use the official build instead of building vanilla ourselves. + if label == 'vanilla': + build_name = '%s-release/%s.0.0' % (self._board, self._build_num) + + # Now add 'official build' to test file. + official_image = """ + official_image { + chromeos_root: %s + build: %s + } + """ % (self._chromeos_root, build_name) + f.write(official_image) + + else: + experiment_image = """ + %s { + chromeos_image: %s + image_args: %s + } + """ % (crosperf_label, os.path.join( + misc.GetImageDir(self._chromeos_root, self._board), label, + 'chromiumos_test_image.bin'), image_args) + f.write(experiment_image) + + crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf') + noschedv2_opts = '--noschedv2' if self._noschedv2 else '' + command = ('{crosperf} --no_email=True --results_dir={r_dir} ' + '--json_report=True {noschedv2_opts} {exp_file}').format( + crosperf=crosperf, + r_dir=self._reports_dir, + noschedv2_opts=noschedv2_opts, + exp_file=experiment_file) + + ret = self._ce.RunCommand(command) + if ret != 0: + raise RuntimeError('Crosperf execution error!') + else: + # Copy json report to pending archives directory. + command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) + ret = self._ce.RunCommand(command) + return + + def _SendEmail(self): + """Find email msesage generated by crosperf and send it.""" + filename = os.path.join(self._reports_dir, 'msg_body.html') + if (os.path.exists(filename) and + os.path.exists(os.path.expanduser(MAIL_PROGRAM))): + command = ('cat %s | %s -s "Nightly test results, %s" -team -html' % + (filename, MAIL_PROGRAM, self._board)) + self._ce.RunCommand(command) + + def DoAll(self): + self._CheckoutChromeOS() + labels = [] + labels.append('vanilla') + for config in self._configs: + label = self._BuildLabelName(config.gcc_config.githash) + if not misc.DoesLabelExist(self._chromeos_root, self._board, label): + self._BuildToolchain(config) + label = self._BuildAndImage(label) + labels.append(label) + self._FinishSetup() + self._TestLabels(labels) + self._SendEmail() + if self._clean: + ret = self._DeleteChroot() + if ret != 0: + return ret + ret = self._DeleteCcahe() + if ret != 0: + return ret + return 0 + + +def Main(argv): + """The main function.""" + # Common initializations + ### command_executer.InitCommandExecuter(True) + command_executer.InitCommandExecuter() + parser = argparse.ArgumentParser() + parser.add_argument( + '--remote', dest='remote', help='Remote machines to run tests on.') + parser.add_argument( + '--board', dest='board', default='x86-alex', help='The target board.') + parser.add_argument( + '--githashes', + dest='githashes', + default='master', + help='The gcc githashes to test.') + parser.add_argument( + '--clean', + dest='clean', + default=False, + action='store_true', + help='Clean the chroot after testing.') + parser.add_argument( + '--public', + dest='public', + default=False, + action='store_true', + help='Use the public checkout/build.') + parser.add_argument( + '--force-mismatch', + dest='force_mismatch', + default='', + help='Force the image regardless of board mismatch') + parser.add_argument( + '--noschedv2', + dest='noschedv2', + action='store_true', + default=False, + help='Pass --noschedv2 to crosperf.') + options = parser.parse_args(argv) + if not options.board: + print('Please give a board.') + return 1 + if not options.remote: + print('Please give at least one remote machine.') + return 1 + toolchain_configs = [] + for githash in options.githashes.split(','): + gcc_config = GCCConfig(githash=githash) + toolchain_config = ToolchainConfig(gcc_config=gcc_config) + toolchain_configs.append(toolchain_config) + fc = ToolchainComparator(options.board, options.remote, toolchain_configs, + options.clean, options.public, + options.force_mismatch, options.noschedv2) + return fc.DoAll() + + +if __name__ == '__main__': + retval = Main(sys.argv[1:]) + sys.exit(retval) |