diff options
Diffstat (limited to 'crosperf/experiment_factory.py')
| -rw-r--r-- | crosperf/experiment_factory.py | 331 |
1 files changed, 331 insertions, 0 deletions
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py new file mode 100644 index 00000000..2278015b --- /dev/null +++ b/crosperf/experiment_factory.py @@ -0,0 +1,331 @@ +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""A module to generate experiments.""" + +from __future__ import print_function +import os +import re +import socket + +from benchmark import Benchmark +import config +from experiment import Experiment +from label import Label +from label import MockLabel +from results_cache import CacheConditions +import test_flag +import file_lock_machine + +# Users may want to run Telemetry tests either individually, or in +# specified sets. Here we define sets of tests that users may want +# to run together. + +telemetry_perfv2_tests = [ + 'dromaeo.domcoreattr', 'dromaeo.domcoremodify', 'dromaeo.domcorequery', + 'dromaeo.domcoretraverse', 'kraken', 'octane', 'robohornet_pro', 'sunspider' +] + +telemetry_pagecycler_tests = [ + 'page_cycler_v2.intl_ar_fa_he', + 'page_cycler_v2.intl_es_fr_pt-BR', + 'page_cycler_v2.intl_hi_ru', + 'page_cycler_v2.intl_ja_zh', + 'page_cycler_v2.intl_ko_th_vi', + # 'page_cycler_v2.morejs', + # 'page_cycler_v2.moz', + # 'page_cycler_v2.netsim.top_10', + 'page_cycler_v2.tough_layout_cases', + 'page_cycler_v2.typical_25' +] + +telemetry_toolchain_old_perf_tests = [ + 'dromaeo.domcoremodify', 'page_cycler_v2.intl_es_fr_pt-BR', + 'page_cycler_v2.intl_hi_ru', 'page_cycler_v2.intl_ja_zh', + 'page_cycler_v2.intl_ko_th_vi', 'page_cycler_v2.netsim.top_10', + 'page_cycler_v2.typical_25', 'robohornet_pro', 'spaceport', + 'tab_switching.top_10' +] +telemetry_toolchain_perf_tests = [ + 'octane', + 'kraken', + 'speedometer', + 'dromaeo.domcoreattr', + 'dromaeo.domcoremodify', + 'smoothness.tough_webgl_cases', +] +graphics_perf_tests = [ + 'graphics_GLBench', + 'graphics_GLMark2', + 'graphics_SanAngeles', + 'graphics_WebGLAquarium', + 'graphics_WebGLPerformance', +] +telemetry_crosbolt_perf_tests = [ + 'octane', + 'kraken', + 'speedometer', + 'jetstream', + 'startup.cold.blank_page', + 'smoothness.top_25_smooth', +] +crosbolt_perf_tests = [ + 'graphics_WebGLAquarium', + 'video_PlaybackPerf.h264', + 'video_PlaybackPerf.vp9', + 'video_WebRtcPerf', + 'BootPerfServerCrosPerf', + 'power_Resume', + 'video_PlaybackPerf.h264', + 'build_RootFilesystemSize', +# 'cheets_AntutuTest', +# 'cheets_PerfBootServer', +# 'cheets_CandyCrushTest', +# 'cheets_LinpackTest', +] + + +class ExperimentFactory(object): + """Factory class for building an Experiment, given an ExperimentFile as input. + + This factory is currently hardcoded to produce an experiment for running + ChromeOS benchmarks, but the idea is that in the future, other types + of experiments could be produced. + """ + + def AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args, + iterations, rm_chroot_tmp, perf_args, suite, + show_all_results, retries, run_local): + """Add all the tests in a set to the benchmarks list.""" + for test_name in benchmark_list: + telemetry_benchmark = Benchmark(test_name, test_name, test_args, + iterations, rm_chroot_tmp, perf_args, + suite, show_all_results, retries, + run_local) + benchmarks.append(telemetry_benchmark) + + def GetExperiment(self, experiment_file, working_directory, log_dir): + """Construct an experiment from an experiment file.""" + global_settings = experiment_file.GetGlobalSettings() + experiment_name = global_settings.GetField('name') + board = global_settings.GetField('board') + remote = global_settings.GetField('remote') + # This is used to remove the ",' from the remote if user + # add them to the remote string. + new_remote = [] + if remote: + for i in remote: + c = re.sub('["\']', '', i) + new_remote.append(c) + remote = new_remote + chromeos_root = global_settings.GetField('chromeos_root') + rm_chroot_tmp = global_settings.GetField('rm_chroot_tmp') + perf_args = global_settings.GetField('perf_args') + acquire_timeout = global_settings.GetField('acquire_timeout') + cache_dir = global_settings.GetField('cache_dir') + cache_only = global_settings.GetField('cache_only') + config.AddConfig('no_email', global_settings.GetField('no_email')) + share_cache = global_settings.GetField('share_cache') + results_dir = global_settings.GetField('results_dir') + use_file_locks = global_settings.GetField('use_file_locks') + locks_dir = global_settings.GetField('locks_dir') + # If we pass a blank locks_dir to the Experiment, it will use the AFE server + # lock mechanism. So if the user specified use_file_locks, but did not + # specify a locks dir, set the locks dir to the default locks dir in + # file_lock_machine. + if use_file_locks and not locks_dir: + locks_dir = file_lock_machine.Machine.LOCKS_DIR + chrome_src = global_settings.GetField('chrome_src') + show_all_results = global_settings.GetField('show_all_results') + log_level = global_settings.GetField('logging_level') + if log_level not in ('quiet', 'average', 'verbose'): + log_level = 'verbose' + # Default cache hit conditions. The image checksum in the cache and the + # computed checksum of the image must match. Also a cache file must exist. + cache_conditions = [ + CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH + ] + if global_settings.GetField('rerun_if_failed'): + cache_conditions.append(CacheConditions.RUN_SUCCEEDED) + if global_settings.GetField('rerun'): + cache_conditions.append(CacheConditions.FALSE) + if global_settings.GetField('same_machine'): + cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH) + if global_settings.GetField('same_specs'): + cache_conditions.append(CacheConditions.MACHINES_MATCH) + + # Construct benchmarks. + # Some fields are common with global settings. The values are + # inherited and/or merged with the global settings values. + benchmarks = [] + all_benchmark_settings = experiment_file.GetSettings('benchmark') + for benchmark_settings in all_benchmark_settings: + benchmark_name = benchmark_settings.name + test_name = benchmark_settings.GetField('test_name') + if not test_name: + test_name = benchmark_name + test_args = benchmark_settings.GetField('test_args') + iterations = benchmark_settings.GetField('iterations') + suite = benchmark_settings.GetField('suite') + retries = benchmark_settings.GetField('retries') + run_local = benchmark_settings.GetField('run_local') + + if suite == 'telemetry_Crosperf': + if test_name == 'all_perfv2': + self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, test_args, + iterations, rm_chroot_tmp, perf_args, suite, + show_all_results, retries, run_local) + elif test_name == 'all_pagecyclers': + self.AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local) + elif test_name == 'all_toolchain_perf': + self.AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local) + # Add non-telemetry toolchain-perf benchmarks: + benchmarks.append( + Benchmark( + 'graphics_WebGLAquarium', + 'graphics_WebGLAquarium', + '', + iterations, + rm_chroot_tmp, + perf_args, + '', + show_all_results, + retries, + run_local=False)) + elif test_name == 'all_toolchain_perf_old': + self.AppendBenchmarkSet(benchmarks, + telemetry_toolchain_old_perf_tests, test_args, + iterations, rm_chroot_tmp, perf_args, suite, + show_all_results, retries, run_local) + else: + benchmark = Benchmark(test_name, test_name, test_args, iterations, + rm_chroot_tmp, perf_args, suite, + show_all_results, retries, run_local) + benchmarks.append(benchmark) + else: + if test_name == 'all_graphics_perf': + self.AppendBenchmarkSet(benchmarks, + graphics_perf_tests, '', + iterations, rm_chroot_tmp, perf_args, '', + show_all_results, retries, run_local=False) + elif test_name == 'all_crosbolt_perf': + self.AppendBenchmarkSet(benchmarks, + telemetry_crosbolt_perf_tests, test_args, + iterations, rm_chroot_tmp, perf_args, + 'telemetry_Crosperf', show_all_results, + retries, run_local) + self.AppendBenchmarkSet(benchmarks, + crosbolt_perf_tests, '', + iterations, rm_chroot_tmp, perf_args, '', + show_all_results, retries, run_local=False) + else: + # Add the single benchmark. + benchmark = Benchmark( + benchmark_name, + test_name, + test_args, + iterations, + rm_chroot_tmp, + perf_args, + suite, + show_all_results, + retries, + run_local=False) + benchmarks.append(benchmark) + + if not benchmarks: + raise RuntimeError('No benchmarks specified') + + # Construct labels. + # Some fields are common with global settings. The values are + # inherited and/or merged with the global settings values. + labels = [] + all_label_settings = experiment_file.GetSettings('label') + all_remote = list(remote) + for label_settings in all_label_settings: + label_name = label_settings.name + image = label_settings.GetField('chromeos_image') + autotest_path = label_settings.GetField('autotest_path') + chromeos_root = label_settings.GetField('chromeos_root') + my_remote = label_settings.GetField('remote') + compiler = label_settings.GetField('compiler') + new_remote = [] + if my_remote: + for i in my_remote: + c = re.sub('["\']', '', i) + new_remote.append(c) + my_remote = new_remote + if image == '': + build = label_settings.GetField('build') + if len(build) == 0: + raise RuntimeError("Can not have empty 'build' field!") + image, autotest_path = label_settings.GetXbuddyPath(build, + autotest_path, + board, + chromeos_root, + log_level) + + cache_dir = label_settings.GetField('cache_dir') + chrome_src = label_settings.GetField('chrome_src') + + # TODO(yunlian): We should consolidate code in machine_manager.py + # to derermine whether we are running from within google or not + if ('corp.google.com' in socket.gethostname() and + (not my_remote or my_remote == remote and + global_settings.GetField('board') != board)): + my_remote = self.GetDefaultRemotes(board) + if global_settings.GetField('same_machine') and len(my_remote) > 1: + raise RuntimeError('Only one remote is allowed when same_machine ' + 'is turned on') + all_remote += my_remote + image_args = label_settings.GetField('image_args') + if test_flag.GetTestMode(): + # pylint: disable=too-many-function-args + label = MockLabel(label_name, image, autotest_path, chromeos_root, + board, my_remote, image_args, cache_dir, cache_only, + log_level, compiler, chrome_src) + else: + label = Label(label_name, image, autotest_path, chromeos_root, board, + my_remote, image_args, cache_dir, cache_only, log_level, + compiler, chrome_src) + labels.append(label) + + if not labels: + raise RuntimeError('No labels specified') + + email = global_settings.GetField('email') + all_remote += list(set(my_remote)) + all_remote = list(set(all_remote)) + experiment = Experiment(experiment_name, all_remote, working_directory, + chromeos_root, cache_conditions, labels, benchmarks, + experiment_file.Canonicalize(), email, + acquire_timeout, log_dir, log_level, share_cache, + results_dir, locks_dir) + + return experiment + + def GetDefaultRemotes(self, board): + default_remotes_file = os.path.join( + os.path.dirname(__file__), 'default_remotes') + try: + with open(default_remotes_file) as f: + for line in f: + key, v = line.split(':') + if key.strip() == board: + remotes = v.strip().split() + if remotes: + return remotes + else: + raise RuntimeError('There is no remote for {0}'.format(board)) + except IOError: + # TODO: rethrow instead of throwing different exception. + raise RuntimeError('IOError while reading file {0}' + .format(default_remotes_file)) + else: + raise RuntimeError('There is not remote for {0}'.format(board)) |