diff options
Diffstat (limited to 'crosperf')
-rw-r--r-- | crosperf/autotest_runner.py | 48 | ||||
-rw-r--r-- | crosperf/benchmark.py | 24 | ||||
-rw-r--r-- | crosperf/benchmark_run.py | 86 | ||||
-rwxr-xr-x | crosperf/benchmark_run_unittest.py | 6 | ||||
-rw-r--r-- | crosperf/experiment_factory.py | 20 | ||||
-rwxr-xr-x | crosperf/experiment_factory_unittest.py | 6 | ||||
-rw-r--r-- | crosperf/label.py | 11 | ||||
-rw-r--r-- | crosperf/machine_manager.py | 26 | ||||
-rw-r--r-- | crosperf/results_cache.py | 127 | ||||
-rw-r--r-- | crosperf/results_organizer.py | 17 | ||||
-rw-r--r-- | crosperf/results_report.py | 8 | ||||
-rw-r--r-- | crosperf/settings_factory.py | 29 | ||||
-rw-r--r-- | crosperf/suite_runner.py | 83 |
13 files changed, 314 insertions, 177 deletions
diff --git a/crosperf/autotest_runner.py b/crosperf/autotest_runner.py deleted file mode 100644 index 7385da5d..00000000 --- a/crosperf/autotest_runner.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2011 Google Inc. All Rights Reserved. - -import time - -from utils import command_executer - - -class AutotestRunner(object): - """ This defines the interface from crosperf to ./run_remote_tests.sh. - """ - def __init__(self, logger_to_use=None): - self._logger = logger_to_use - self._ce = command_executer.GetCommandExecuter(self._logger) - self._ct = command_executer.CommandTerminator() - - def Run(self, machine_name, chromeos_root, board, autotest_name, - autotest_args): - """Run the run_remote_test.""" - options = "" - if board: - options += " --board=%s" % board - if autotest_args: - options += " %s" % autotest_args - command = "rm -rf /usr/local/autotest/results/*" - self._ce.CrosRunCommand(command, machine=machine_name, username="root", - chromeos_root=chromeos_root) - - command ="reboot && exit" - self._ce.CrosRunCommand(command, machine=machine_name, - chromeos_root=chromeos_root) - time.sleep(60) - - command = ("./run_remote_tests.sh --remote=%s %s %s" % - (machine_name, options, autotest_name)) - return self._ce.ChrootRunCommand(chromeos_root, command, True, self._ct) - - def Terminate(self): - self._ct.Terminate() - - -class MockAutotestRunner(object): - def __init__(self): - pass - - def Run(self, *args): - return ["", "", 0] diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py index 8fe8a492..4a2d431e 100644 --- a/crosperf/benchmark.py +++ b/crosperf/benchmark.py @@ -1,26 +1,32 @@ #!/usr/bin/python -# Copyright 2011 Google Inc. All Rights Reserved. +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. class Benchmark(object): """Class representing a benchmark to be run. - Contains details of the autotest, arguments to pass to the autotest, - iterations to run the autotest and so on. Note that the benchmark name - can be different to the autotest name. For example, you may want to have - two different benchmarks which run the same autotest with different + Contains details of the benchmark suite, arguments to pass to the suite, + iterations to run the benchmark suite and so on. Note that the benchmark name + can be different to the test suite name. For example, you may want to have + two different benchmarks which run the same test_name with different arguments. """ - def __init__(self, name, autotest_name, autotest_args, iterations, - outlier_range, key_results_only, rm_chroot_tmp, perf_args): + def __init__(self, name, test_name, test_args, iterations, + outlier_range, key_results_only, rm_chroot_tmp, perf_args, + suite="pyauto"): self.name = name - self.autotest_name = autotest_name - self.autotest_args = autotest_args + #For telemetry, this is the benchmark name. + self.test_name = test_name + #For telemetry, this is the data. + self.test_args = test_args self.iterations = iterations self.outlier_range = outlier_range self.perf_args = perf_args self.key_results_only = key_results_only self.rm_chroot_tmp = rm_chroot_tmp self.iteration_adjusted = False + self.suite = suite diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py index 9197dc9e..0fcd0c61 100644 --- a/crosperf/benchmark_run.py +++ b/crosperf/benchmark_run.py @@ -13,9 +13,11 @@ import traceback from utils import command_executer from utils import timeline -from autotest_runner import AutotestRunner +from suite_runner import SuiteRunner from results_cache import Result from results_cache import ResultsCache +from results_cache import TelemetryResult + STATUS_FAILED = "FAILED" STATUS_SUCCEEDED = "SUCCEEDED" @@ -24,7 +26,6 @@ STATUS_RUNNING = "RUNNING" STATUS_WAITING = "WAITING" STATUS_PENDING = "PENDING" - class BenchmarkRun(threading.Thread): def __init__(self, name, benchmark, label, @@ -44,39 +45,43 @@ class BenchmarkRun(threading.Thread): self.retval = None self.run_completed = False self.machine_manager = machine_manager - self.cache = ResultsCache() - self.autotest_runner = AutotestRunner(self._logger) + self.suite_runner = SuiteRunner(self._logger) self.machine = None self.cache_conditions = cache_conditions self.runs_complete = 0 self.cache_hit = False self.failure_reason = "" - self.autotest_args = "%s %s" % (benchmark.autotest_args, + self.test_args = "%s %s" % (benchmark.test_args, self._GetExtraAutotestArgs()) self._ce = command_executer.GetCommandExecuter(self._logger) self.timeline = timeline.Timeline() self.timeline.Record(STATUS_PENDING) self.share_users = share_users + def ReadCache(self): + # Just use the first machine for running the cached version, + # without locking it. + self.cache = ResultsCache() + self.cache.Init(self.label.chromeos_image, + self.label.chromeos_root, + self.benchmark.test_name, + self.iteration, + self.test_args, + self.machine_manager, + self.label.board, + self.cache_conditions, + self._logger, + self.label, + self.share_users, + self.benchmark.suite + ) + + self.result = self.cache.ReadResult() + self.cache_hit = (self.result is not None) + def run(self): try: - # Just use the first machine for running the cached version, - # without locking it. - self.cache.Init(self.label.chromeos_image, - self.label.chromeos_root, - self.benchmark.autotest_name, - self.iteration, - self.autotest_args, - self.machine_manager, - self.label.board, - self.cache_conditions, - self._logger, - self.label, - self.share_users - ) - - self.result = self.cache.ReadResult() - self.cache_hit = (self.result is not None) + self.ReadCache() if self.result: self._logger.LogOutput("%s: Cache hit." % self.name) @@ -88,8 +93,9 @@ class BenchmarkRun(threading.Thread): self.timeline.Record(STATUS_WAITING) # Try to acquire a machine now. self.machine = self.AcquireMachine() - self.cache.remote = self.machine.name self.result = self.RunTest(self.machine) + + self.cache.remote = self.machine.name self.cache.StoreResult(self.result) if self.terminated: @@ -99,7 +105,7 @@ class BenchmarkRun(threading.Thread): self.timeline.Record(STATUS_SUCCEEDED) else: if self.timeline.GetLastEvent() != STATUS_FAILED: - self.failure_reason = "Return value of autotest was non-zero." + self.failure_reason = "Return value of test suite was non-zero." self.timeline.Record(STATUS_FAILED) except Exception, e: @@ -120,7 +126,7 @@ class BenchmarkRun(threading.Thread): def Terminate(self): self.terminated = True - self.autotest_runner.Terminate() + self.suite_runner.Terminate() if self.timeline.GetLastEvent() != STATUS_FAILED: self.timeline.Record(STATUS_FAILED) self.failure_reason = "Thread terminated." @@ -144,16 +150,19 @@ class BenchmarkRun(threading.Thread): return machine def _GetExtraAutotestArgs(self): + if self.benchmark.perf_args and self.benchmark.suite == "telemetry": + self._logger.LogError("Telemetry benchmark does not support profiler.") + if self.benchmark.perf_args: perf_args_list = self.benchmark.perf_args.split(" ") perf_args_list = [perf_args_list[0]] + ["-a"] + perf_args_list[1:] perf_args = " ".join(perf_args_list) if not perf_args_list[0] in ["record", "stat"]: raise Exception("perf_args must start with either record or stat") - extra_autotest_args = ["--profiler=custom_perf", + extra_test_args = ["--profiler=custom_perf", ("--profiler_args='perf_options=\"%s\"'" % perf_args)] - return " ".join(extra_autotest_args) + return " ".join(extra_test_args) else: return "" @@ -162,20 +171,17 @@ class BenchmarkRun(threading.Thread): self.machine_manager.ImageMachine(machine, self.label) self.timeline.Record(STATUS_RUNNING) - [retval, out, err] = self.autotest_runner.Run(machine.name, - self.label.chromeos_root, - self.label.board, - self.benchmark.autotest_name, - self.autotest_args) + [retval, out, err] = self.suite_runner.Run(machine.name, + self.label, + self.benchmark, + self.test_args) self.run_completed = True - return Result.CreateFromRun(self._logger, - self.label.chromeos_root, - self.label.board, - self.label.name, + self.label, out, err, - retval) + retval, + self.benchmark.suite) def SetCacheConditions(self, cache_conditions): self.cache_conditions = cache_conditions @@ -190,11 +196,11 @@ class MockBenchmarkRun(BenchmarkRun): self.machine_manager.ImageMachine(machine, self.label) self.timeline.Record(STATUS_RUNNING) - [retval, out, err] = self.autotest_runner.Run(machine.name, + [retval, out, err] = self.suite_runner.Run(machine.name, self.label.chromeos_root, self.label.board, - self.benchmark.autotest_name, - self.autotest_args) + self.benchmark.test_name, + self.test_args) self.run_completed = True rr = Result("Results placed in /tmp/test", "", 0) rr.out = out diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py index 47e027f4..c08d1893 100755 --- a/crosperf/benchmark_run_unittest.py +++ b/crosperf/benchmark_run_unittest.py @@ -1,6 +1,8 @@ #!/usr/bin/python -# Copyright 2011 Google Inc. All Rights Reserved. +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. """Testing of benchmark_run.""" @@ -8,7 +10,7 @@ import unittest from utils import logger -from autotest_runner import MockAutotestRunner +from suite_runner import MockSuiteRunner from benchmark_run import MockBenchmarkRun from label import MockLabel from benchmark import Benchmark diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index 811de268..1c5d271d 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -39,6 +39,7 @@ class ExperimentFactory(object): config.AddConfig("no_email", global_settings.GetField("no_email")) share_users = global_settings.GetField("share_users") results_dir = global_settings.GetField("results_dir") + chrome_src = global_settings.GetField("chrome_src") # Default cache hit conditions. The image checksum in the cache and the # computed checksum of the image must match. Also a cache file must exist. cache_conditions = [CacheConditions.CACHE_FILE_EXISTS, @@ -57,20 +58,21 @@ class ExperimentFactory(object): all_benchmark_settings = experiment_file.GetSettings("benchmark") for benchmark_settings in all_benchmark_settings: benchmark_name = benchmark_settings.name - autotest_name = benchmark_settings.GetField("autotest_name") - if not autotest_name: - autotest_name = benchmark_name - autotest_args = benchmark_settings.GetField("autotest_args") + test_name = benchmark_settings.GetField("test_name") + if not test_name: + test_name = benchmark_name + test_args = benchmark_settings.GetField("test_args") iterations = benchmark_settings.GetField("iterations") outlier_range = benchmark_settings.GetField("outlier_range") perf_args = benchmark_settings.GetField("perf_args") rm_chroot_tmp = benchmark_settings.GetField("rm_chroot_tmp") key_results_only = benchmark_settings.GetField("key_results_only") + suite = benchmark_settings.GetField("suite") - benchmark = Benchmark(benchmark_name, autotest_name, autotest_args, + benchmark = Benchmark(benchmark_name, test_name, test_args, iterations, outlier_range, key_results_only, rm_chroot_tmp, - perf_args) + perf_args, suite) benchmarks.append(benchmark) # Construct labels. @@ -85,6 +87,8 @@ class ExperimentFactory(object): my_remote = label_settings.GetField("remote") image_md5sum = label_settings.GetField("md5sum") cache_dir = label_settings.GetField("cache_dir") + chrome_src = label_settings.GetField("chrome_src") + # TODO(yunlian): We should consolidate code in machine_manager.py # to derermine whether we are running from within google or not if ("corp.google.com" in socket.gethostname() and @@ -99,10 +103,10 @@ class ExperimentFactory(object): image_args = label_settings.GetField("image_args") if test_flag.GetTestMode(): label = MockLabel(label_name, image, chromeos_root, board, my_remote, - image_args, image_md5sum, cache_dir) + image_args, image_md5sum, cache_dir, chrome_src) else: label = Label(label_name, image, chromeos_root, board, my_remote, - image_args, image_md5sum, cache_dir) + image_args, image_md5sum, cache_dir, chrome_src) labels.append(label) email = global_settings.GetField("email") diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 6cee6b74..4ff89f8e 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -1,6 +1,8 @@ #!/usr/bin/python -# Copyright 2011 Google Inc. All Rights Reserved. +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. import StringIO import unittest @@ -39,7 +41,7 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertEqual(len(experiment.benchmarks), 1) self.assertEqual(experiment.benchmarks[0].name, "PageCycler") - self.assertEqual(experiment.benchmarks[0].autotest_name, "PageCycler") + self.assertEqual(experiment.benchmarks[0].test_name, "PageCycler") self.assertEqual(experiment.benchmarks[0].iterations, 3) self.assertEqual(len(experiment.labels), 2) diff --git a/crosperf/label.py b/crosperf/label.py index be7a868e..e8193529 100644 --- a/crosperf/label.py +++ b/crosperf/label.py @@ -1,6 +1,8 @@ #!/usr/bin/python -# Copyright 2011 Google Inc. All Rights Reserved. +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. """The label of benchamrks.""" @@ -10,7 +12,7 @@ from utils.file_utils import FileUtils class Label(object): def __init__(self, name, chromeos_image, chromeos_root, board, remote, - image_args, image_md5sum, cache_dir): + image_args, image_md5sum, cache_dir, chrome_src=None): # Expand ~ chromeos_root = os.path.expanduser(chromeos_root) chromeos_image = os.path.expanduser(chromeos_image) @@ -36,6 +38,11 @@ class Label(object): % (name, chromeos_root)) self.chromeos_root = chromeos_root + if not chrome_src: + self.chrome_src = os.path.join(self.chromeos_root, + "chroot/var/cache/chromeos-chrome/chrome-src-internal/src") + else: + chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chrome_src) class MockLabel(object): diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index 957c8304..e12796ed 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -30,7 +30,7 @@ class CrosMachine(object): self.checksum = None self.locked = False self.released_time = time.time() - self.autotest_run = None + self.test_run = None self.chromeos_root = chromeos_root if not self.IsReachable(): self.machine_checksum = None @@ -55,7 +55,7 @@ class CrosMachine(object): def _ParseMemoryInfo(self): line = self.meminfo.splitlines()[0] usable_kbytes = int(line.split()[1]) - # This code is from src/third_party/autotest/files/client/bin/base_utils.py + # This code is from src/third_party/test/files/client/bin/base_utils.py # usable_kbytes is system's usable DRAM in kbytes, # as reported by memtotal() from device /proc/meminfo memtotal # after Linux deducts 1.5% to 9.5% for system table overhead @@ -303,13 +303,13 @@ class MachineManager(object): if not machine.locked]: if m.checksum == image_checksum: m.locked = True - m.autotest_run = threading.current_thread() + m.test_run = threading.current_thread() return m for m in [machine for machine in self.GetAvailableMachines(label) if not machine.locked]: if not m.checksum: m.locked = True - m.autotest_run = threading.current_thread() + m.test_run = threading.current_thread() return m # This logic ensures that threads waiting on a machine will get a machine # with a checksum equal to their image over other threads. This saves time @@ -321,7 +321,7 @@ class MachineManager(object): if not machine.locked]: if time.time() - m.released_time > 20: m.locked = True - m.autotest_run = threading.current_thread() + m.test_run = threading.current_thread() return m return None @@ -369,18 +369,18 @@ class MachineManager(object): "Checksum") table = [header] for m in self._machines: - if m.autotest_run: - autotest_name = m.autotest_run.name - autotest_status = m.autotest_run.timeline.GetLastEvent() + if m.test_run: + test_name = m.test_run.name + test_status = m.test_run.timeline.GetLastEvent() else: - autotest_name = "" - autotest_status = "" + test_name = "" + test_status = "" try: machine_string = stringify_fmt % (m.name, - autotest_name, + test_name, m.locked, - autotest_status, + test_status, m.checksum) except Exception: machine_string = "" @@ -414,7 +414,7 @@ class MockCrosMachine(CrosMachine): self.checksum = None self.locked = False self.released_time = time.time() - self.autotest_run = None + self.test_run = None self.chromeos_root = chromeos_root self.checksum_string = re.sub("\d", "", name) #In test, we assume "lumpy1", "lumpy2" are the same machine. diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index f3b3183b..4178b467 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -31,12 +31,15 @@ class Result(object): perf.report, etc. The key generation is handled by the ResultsCache class. """ - def __init__(self, chromeos_root, logger, label_name): - self._chromeos_root = chromeos_root + def __init__(self, logger, label): + self._chromeos_root = label.chromeos_root self._logger = logger self._ce = command_executer.GetCommandExecuter(self._logger) self._temp_dir = None - self.label_name = label_name + self.label = label + self.results_dir = None + self.perf_data_files = [] + self.perf_report_files = [] def _CopyFilesTo(self, dest_dir, files_to_copy): file_index = 0 @@ -91,6 +94,8 @@ class Result(object): raise Exception("Could not find results directory.") def _FindFilesInResultsDir(self, find_args): + if not self.results_dir: + return None command = "find %s %s" % (self.results_dir, find_args) ret, out, _ = self._ce.RunCommand(command, return_output=True) @@ -150,8 +155,8 @@ class Result(object): value = str(misc.UnitToNumber(num_events)) self.keyvals[key] = value - def _PopulateFromRun(self, board, out, err, retval): - self._board = board + def _PopulateFromRun(self, out, err, retval): + self._board = self.label.board self.out = out self.err = err self.retval = retval @@ -199,7 +204,7 @@ class Result(object): self._ProcessResults() def CleanUp(self, rm_chroot_tmp): - if rm_chroot_tmp: + if rm_chroot_tmp and self.results_dir: command = "rm -rf %s" % self.results_dir self._ce.RunCommand(command) if self._temp_dir: @@ -216,20 +221,21 @@ class Result(object): pickle.dump(self.err, f) pickle.dump(self.retval, f) - tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) - command = ("cd %s && " - "tar " - "--exclude=var/spool " - "--exclude=var/log " - "-cjf %s ." % (self.results_dir, tarball)) - ret = self._ce.RunCommand(command) - if ret: - raise Exception("Couldn't store autotest output directory.") + if self.results_dir: + tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) + command = ("cd %s && " + "tar " + "--exclude=var/spool " + "--exclude=var/log " + "-cjf %s ." % (self.results_dir, tarball)) + ret = self._ce.RunCommand(command) + if ret: + raise Exception("Couldn't store autotest output directory.") # Store machine info. # TODO(asharif): Make machine_manager a singleton, and don't pass it into # this function. with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: - f.write(machine_manager.machine_checksum_string[self.label_name]) + f.write(machine_manager.machine_checksum_string[self.label.name]) if os.path.exists(cache_dir): command = "rm -rf {0}".format(cache_dir) @@ -246,22 +252,72 @@ class Result(object): (temp_dir, cache_dir)) @classmethod - def CreateFromRun(cls, logger, chromeos_root, board, label_name, - out, err, retval): - result = cls(chromeos_root, logger, label_name) - result._PopulateFromRun(board, out, err, retval) + def CreateFromRun(cls, logger, label, out, err, retval, suite="pyauto"): + if suite == "telemetry": + result = TelemetryResult(logger, label) + else: + result = cls(logger, label) + result._PopulateFromRun(out, err, retval) return result @classmethod - def CreateFromCacheHit(cls, chromeos_root, logger, cache_dir, label_name): - result = cls(chromeos_root, logger, label_name) + def CreateFromCacheHit(cls, logger, label, cache_dir, + suite="pyauto"): + if suite == "telemetry": + result = TelemetryResult(logger, label) + else: + result = cls(logger, label) try: result._PopulateFromCacheDir(cache_dir) + except Exception as e: logger.LogError("Exception while using cache: %s" % e) return None return result +class TelemetryResult(Result): + + def __init__(self, logger, label): + super(TelemetryResult, self).__init__(logger, label) + + def _PopulateFromRun(self, out, err, retval): + self.out = out + self.err = err + self.retval = retval + + self._ProcessResults() + + def _ProcessResults(self): + # The output is: + # url,average_commit_time (ms),... + # www.google.com,33.4,21.2,... + # We need to convert to this format: + # {"www.google.com:average_commit_time (ms)": "33.4", + # "www.google.com:...": "21.2"} + + lines = self.out.splitlines() + self.keyvals = {} + + if not lines: + return + labels = lines[0].split(",") + for line in lines[1:]: + fields = line.split(",") + if (len(fields) != len(labels)): + continue + for i in range(1, len(labels)): + key = "%s %s" % (fields[0], labels[i]) + value = fields[i] + self.keyvals[key] = value + self.keyvals["retval"] = self.retval + + def _PopulateFromCacheDir(self, cache_dir): + with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: + self.out = pickle.load(f) + self.err = pickle.load(f) + self.retval = pickle.load(f) + self._ProcessResults() + class CacheConditions(object): # Cache hit only if the result file exists. @@ -295,14 +351,14 @@ class ResultsCache(object): """ CACHE_VERSION = 6 - def Init(self, chromeos_image, chromeos_root, autotest_name, iteration, - autotest_args, machine_manager, board, cache_conditions, - logger_to_use, label, share_users): + def Init(self, chromeos_image, chromeos_root, test_name, iteration, + test_args, machine_manager, board, cache_conditions, + logger_to_use, label, share_users, suite): self.chromeos_image = chromeos_image self.chromeos_root = chromeos_root - self.autotest_name = autotest_name + self.test_name = test_name self.iteration = iteration - self.autotest_args = autotest_args, + self.test_args = test_args, self.board = board self.cache_conditions = cache_conditions self.machine_manager = machine_manager @@ -310,6 +366,7 @@ class ResultsCache(object): self._ce = command_executer.GetCommandExecuter(self._logger) self.label = label self.share_users = share_users + self.suite = suite def _GetCacheDirForRead(self): matching_dirs = [] @@ -363,11 +420,11 @@ class ResultsCache(object): machine_id_checksum = machine.machine_id_checksum break - autotest_args_checksum = hashlib.md5( - "".join(self.autotest_args)).hexdigest() + test_args_checksum = hashlib.md5( + "".join(self.test_args)).hexdigest() return (image_path_checksum, - self.autotest_name, str(self.iteration), - autotest_args_checksum, + self.test_name, str(self.iteration), + test_args_checksum, checksum, machine_checksum, machine_id_checksum, @@ -385,10 +442,10 @@ class ResultsCache(object): return None self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir) - - result = Result.CreateFromCacheHit(self.chromeos_root, - self._logger, cache_dir, self.label.name) - + result = Result.CreateFromCacheHit(self._logger, + self.label, + cache_dir, + self.suite) if not result: return None diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py index 2e5c9296..6274a484 100644 --- a/crosperf/results_organizer.py +++ b/crosperf/results_organizer.py @@ -1,6 +1,9 @@ #!/usr/bin/python -# Copyright 2012 Google Inc. All Rights Reserved. +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Parse data from benchmark_runs for tabulator.""" import re @@ -50,15 +53,15 @@ class ResultOrganizer(object): continue benchmark = benchmark_run.benchmark key_filter_on = (benchmark.key_results_only and - "PyAutoPerfTest" in benchmark.name + benchmark.autotest_name and - "perf." not in benchmark.autotest_args) - for autotest_key in benchmark_run.result.keyvals: + "PyAutoPerfTest" in benchmark.name + benchmark.test_name + and "perf." not in benchmark.test_args) + for test_key in benchmark_run.result.keyvals: if (key_filter_on and - not any([key for key in self.key_filter if key in autotest_key]) + not any([key for key in self.key_filter if key in test_key]) ): continue - result_value = benchmark_run.result.keyvals[autotest_key] - cur_dict[autotest_key] = result_value + result_value = benchmark_run.result.keyvals[test_key] + cur_dict[test_key] = result_value self._DuplicatePass() def _DuplicatePass(self): diff --git a/crosperf/results_report.py b/crosperf/results_report.py index 61c67d5b..c56e25ea 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -1,6 +1,8 @@ #!/usr/bin/python -# Copyright 2011 Google Inc. All Rights Reserved. +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. from utils.tabulator import * @@ -479,8 +481,8 @@ pre { for i in range(2, len(data_table)): cur_row_data = data_table[i] - autotest_key = cur_row_data[0].string_value - title = "{0}: {1}".format(item, autotest_key.replace("/", "")) + test_key = cur_row_data[0].string_value + title = "{0}: {1}".format(item, test_key.replace("/", "")) chart = ColumnChart(title, 300, 200) chart.AddColumn("Label", "string") chart.AddColumn("Average", "number") diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 5cbae5df..365b8cbd 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -17,15 +17,15 @@ from settings import Settings class BenchmarkSettings(Settings): def __init__(self, name): super(BenchmarkSettings, self).__init__(name, "benchmark") - self.AddField(TextField("autotest_name", - description="The name of the autotest to run." + self.AddField(TextField("test_name", + description="The name of the test to run." "Defaults to the name of the benchmark.")) - self.AddField(TextField("autotest_args", + self.AddField(TextField("test_args", description="Arguments to be passed to the " - "autotest.")) + "test.")) self.AddField(IntegerField("iterations", default=1, description="Number of iterations to run the " - "autotest.")) + "test.")) self.AddField(FloatField("outlier_range", default=0.2, description="The percentage of highest/lowest " "values to omit when computing the average.")) @@ -40,6 +40,8 @@ class BenchmarkSettings(Settings): "enables perf commands to record perforamance " "related counters. It must start with perf " "command record or stat followed by arguments.")) + self.AddField(TextField("suite", default="pyauto", + description="The type of the benchmark")) class LabelSettings(Settings): @@ -66,6 +68,11 @@ class LabelSettings(Settings): "image_chromeos.py.")) self.AddField(TextField("cache_dir", default="", description="The cache dir for this image.")) + self.AddField(TextField("chrome_src", + description="The path to the source of chrome. " + "This is used to run telemetry benchmarks. " + "The default one is the src inside chroot.", + required=False, default="")) class GlobalSettings(Settings): @@ -80,7 +87,7 @@ class GlobalSettings(Settings): description="A comma-separated list of ip's of " "chromeos devices to run experiments on.")) self.AddField(BooleanField("rerun_if_failed", description="Whether to " - "re-run failed autotest runs or not.", + "re-run failed test runs or not.", default=False)) self.AddField(BooleanField("rm_chroot_tmp", default=False, description="Whether remove the run_remote_test" @@ -88,7 +95,7 @@ class GlobalSettings(Settings): self.AddField(ListField("email", description="Space-seperated" "list of email addresses to send email to.")) self.AddField(BooleanField("rerun", description="Whether to ignore the " - "cache and for autotests to be re-run.", + "cache and for tests to be re-run.", default=False)) self.AddField(BooleanField("same_specs", default=True, description="Ensure cached runs are run on the " @@ -99,7 +106,7 @@ class GlobalSettings(Settings): "exact the same remote")) self.AddField(IntegerField("iterations", default=1, description="Number of iterations to run all " - "autotests.")) + "tests.")) self.AddField(TextField("chromeos_root", description="The path to a chromeos checkout which " "contains a src/scripts directory. Defaults to " @@ -128,6 +135,12 @@ class GlobalSettings(Settings): "use. It accepts multiple users seperated by \",\"")) self.AddField(TextField("results_dir", default="", description="The results dir")) + self.AddField(TextField("chrome_src", + description="The path to the source of chrome. " + "This is used to run telemetry benchmarks. " + "The default one is the src inside chroot.", + + required=False, default="")) class SettingsFactory(object): diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py new file mode 100644 index 00000000..d89f307d --- /dev/null +++ b/crosperf/suite_runner.py @@ -0,0 +1,83 @@ +#!/usr/bin/python + +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import time + +from utils import command_executer + + +class SuiteRunner(object): + """ This defines the interface from crosperf to test script. + """ + def __init__(self, logger_to_use=None): + self._logger = logger_to_use + self._ce = command_executer.GetCommandExecuter(self._logger) + self._ct = command_executer.CommandTerminator() + + def Run(self, machine, label, benchmark, test_args): + if benchmark.suite == "telemetry": + return self.Telemetry_Run(machine, label, benchmark) + else: + return self.Pyauto_Run(machine, label, benchmark, test_args) + + def RebootMachine(self, machine_name, chromeos_root): + command ="reboot && exit" + self._ce.CrosRunCommand(command, machine=machine_name, + chromeos_root=chromeos_root) + time.sleep(60) + + + def Pyauto_Run(self, machine, label, benchmark, test_args): + """Run the run_remote_test.""" + options = "" + if label.board: + options += " --board=%s" % label.board + if test_args: + options += " %s" % test_args + command = "rm -rf /usr/local/autotest/results/*" + self._ce.CrosRunCommand(command, machine=machine, username="root", + chromeos_root=label.chromeos_root) + + self.RebootMachine(machine, label.chromeos_root) + + command = ("./run_remote_tests.sh --remote=%s %s %s" % + (machine, options, benchmark.test_name)) + return self._ce.ChrootRunCommand(label.chromeos_root, + command, + True, + self._ct) + + def Telemetry_Run(self, machine, label, benchmark): + if not os.path.isdir(label.chrome_src): + self._logger.GetLogger().LogFatal("Cannot find chrome src dir to" + "run telemetry.") + rsa_key = os.path.join(label.chromeos_root, + "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa") + + cmd = ("cd {0} && " + "./tools/perf/run_multipage_benchmarks " + "--browser=cros-chrome " + "--output-format=csv " + "--remote={1} " + "--identity {2} " + "{3} {4}".format(label.chrome_src, machine, + rsa_key, + benchmark.test_name, + benchmark.test_args)) + return self._ce.RunCommand(cmd, return_output=True, + print_to_console=False) + + def Terminate(self): + self._ct.Terminate() + + +class MockSuiteRunner(object): + def __init__(self): + pass + + def Run(self, *args): + return ["", "", 0] |