diff options
Diffstat (limited to 'crosperf/experiment_runner.py')
| -rw-r--r-- | crosperf/experiment_runner.py | 309 |
1 files changed, 309 insertions, 0 deletions
diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py new file mode 100644 index 00000000..b30c8bd5 --- /dev/null +++ b/crosperf/experiment_runner.py @@ -0,0 +1,309 @@ +# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""The experiment runner module.""" +from __future__ import print_function + +import getpass +import os +import shutil +import time + +import afe_lock_machine +import test_flag + +from cros_utils import command_executer +from cros_utils import logger +from cros_utils.email_sender import EmailSender +from cros_utils.file_utils import FileUtils + +import config +from experiment_status import ExperimentStatus +from results_cache import CacheConditions +from results_cache import ResultsCache +from results_report import HTMLResultsReport +from results_report import TextResultsReport +from results_report import JSONResultsReport +from schedv2 import Schedv2 + +def _WriteJSONReportToFile(experiment, results_dir, json_report): + """Writes a JSON report to a file in results_dir.""" + has_llvm = any('llvm' in l.compiler for l in experiment.labels) + compiler_string = 'llvm' if has_llvm else 'gcc' + board = experiment.labels[0].board + filename = 'report_%s_%s_%s.%s.json' % ( + board, json_report.date, json_report.time.replace(':', '.'), + compiler_string) + fullname = os.path.join(results_dir, filename) + report_text = json_report.GetReport() + with open(fullname, 'w') as out_file: + out_file.write(report_text) + + +class ExperimentRunner(object): + """ExperimentRunner Class.""" + + STATUS_TIME_DELAY = 30 + THREAD_MONITOR_DELAY = 2 + + def __init__(self, + experiment, + json_report, + using_schedv2=False, + log=None, + cmd_exec=None): + self._experiment = experiment + self.l = log or logger.GetLogger(experiment.log_dir) + self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l) + self._terminated = False + self.json_report = json_report + self.locked_machines = [] + if experiment.log_level != 'verbose': + self.STATUS_TIME_DELAY = 10 + + # Setting this to True will use crosperf sched v2 (feature in progress). + self._using_schedv2 = using_schedv2 + + def _GetMachineList(self): + """Return a list of all requested machines. + + Create a list of all the requested machines, both global requests and + label-specific requests, and return the list. + """ + machines = self._experiment.remote + # All Label.remote is a sublist of experiment.remote. + for l in self._experiment.labels: + for r in l.remote: + assert r in machines + return machines + + def _UpdateMachineList(self, locked_machines): + """Update machines lists to contain only locked machines. + + Go through all the lists of requested machines, both global and + label-specific requests, and remove any machine that we were not + able to lock. + + Args: + locked_machines: A list of the machines we successfully locked. + """ + for m in self._experiment.remote: + if m not in locked_machines: + self._experiment.remote.remove(m) + + for l in self._experiment.labels: + for m in l.remote: + if m not in locked_machines: + l.remote.remove(m) + + def _LockAllMachines(self, experiment): + """Attempt to globally lock all of the machines requested for run. + + This method will use the AFE server to globally lock all of the machines + requested for this crosperf run, to prevent any other crosperf runs from + being able to update/use the machines while this experiment is running. + """ + if test_flag.GetTestMode(): + self.locked_machines = self._GetMachineList() + self._experiment.locked_machines = self.locked_machines + else: + lock_mgr = afe_lock_machine.AFELockManager( + self._GetMachineList(), + '', + experiment.labels[0].chromeos_root, + None, + log=self.l,) + for m in lock_mgr.machines: + if not lock_mgr.MachineIsKnown(m): + lock_mgr.AddLocalMachine(m) + machine_states = lock_mgr.GetMachineStates('lock') + lock_mgr.CheckMachineLocks(machine_states, 'lock') + self.locked_machines = lock_mgr.UpdateMachines(True) + self._experiment.locked_machines = self.locked_machines + self._UpdateMachineList(self.locked_machines) + self._experiment.machine_manager.RemoveNonLockedMachines( + self.locked_machines) + if len(self.locked_machines) == 0: + raise RuntimeError('Unable to lock any machines.') + + def _UnlockAllMachines(self, experiment): + """Attempt to globally unlock all of the machines requested for run. + + The method will use the AFE server to globally unlock all of the machines + requested for this crosperf run. + """ + if not self.locked_machines or test_flag.GetTestMode(): + return + + lock_mgr = afe_lock_machine.AFELockManager( + self.locked_machines, + '', + experiment.labels[0].chromeos_root, + None, + log=self.l,) + machine_states = lock_mgr.GetMachineStates('unlock') + lock_mgr.CheckMachineLocks(machine_states, 'unlock') + lock_mgr.UpdateMachines(False) + + def _ClearCacheEntries(self, experiment): + for br in experiment.benchmark_runs: + cache = ResultsCache() + cache.Init(br.label.chromeos_image, br.label.chromeos_root, + br.benchmark.test_name, br.iteration, br.test_args, + br.profiler_args, br.machine_manager, br.machine, + br.label.board, br.cache_conditions, br._logger, br.log_level, + br.label, br.share_cache, br.benchmark.suite, + br.benchmark.show_all_results, br.benchmark.run_local) + cache_dir = cache.GetCacheDirForWrite() + if os.path.exists(cache_dir): + self.l.LogOutput('Removing cache dir: %s' % cache_dir) + shutil.rmtree(cache_dir) + + def _Run(self, experiment): + try: + if not experiment.locks_dir: + self._LockAllMachines(experiment) + if self._using_schedv2: + schedv2 = Schedv2(experiment) + experiment.set_schedv2(schedv2) + if CacheConditions.FALSE in experiment.cache_conditions: + self._ClearCacheEntries(experiment) + status = ExperimentStatus(experiment) + experiment.Run() + last_status_time = 0 + last_status_string = '' + try: + if experiment.log_level != 'verbose': + self.l.LogStartDots() + while not experiment.IsComplete(): + if last_status_time + self.STATUS_TIME_DELAY < time.time(): + last_status_time = time.time() + border = '==============================' + if experiment.log_level == 'verbose': + self.l.LogOutput(border) + self.l.LogOutput(status.GetProgressString()) + self.l.LogOutput(status.GetStatusString()) + self.l.LogOutput(border) + else: + current_status_string = status.GetStatusString() + if current_status_string != last_status_string: + self.l.LogEndDots() + self.l.LogOutput(border) + self.l.LogOutput(current_status_string) + self.l.LogOutput(border) + last_status_string = current_status_string + else: + self.l.LogAppendDot() + time.sleep(self.THREAD_MONITOR_DELAY) + except KeyboardInterrupt: + self._terminated = True + self.l.LogError('Ctrl-c pressed. Cleaning up...') + experiment.Terminate() + raise + except SystemExit: + self._terminated = True + self.l.LogError('Unexpected exit. Cleaning up...') + experiment.Terminate() + raise + finally: + if not experiment.locks_dir: + self._UnlockAllMachines(experiment) + + def _PrintTable(self, experiment): + self.l.LogOutput(TextResultsReport.FromExperiment(experiment).GetReport()) + + def _Email(self, experiment): + # Only email by default if a new run was completed. + send_mail = False + for benchmark_run in experiment.benchmark_runs: + if not benchmark_run.cache_hit: + send_mail = True + break + if (not send_mail and not experiment.email_to or + config.GetConfig('no_email')): + return + + label_names = [] + for label in experiment.labels: + label_names.append(label.name) + subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names)) + + text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() + text_report += ('\nResults are stored in %s.\n' % + experiment.results_directory) + text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report + html_report = HTMLResultsReport.FromExperiment(experiment).GetReport() + attachment = EmailSender.Attachment('report.html', html_report) + email_to = experiment.email_to or [] + email_to.append(getpass.getuser()) + EmailSender().SendEmail(email_to, + subject, + text_report, + attachments=[attachment], + msg_type='html') + + def _StoreResults(self, experiment): + if self._terminated: + return + results_directory = experiment.results_directory + FileUtils().RmDir(results_directory) + FileUtils().MkDirP(results_directory) + self.l.LogOutput('Storing experiment file in %s.' % results_directory) + experiment_file_path = os.path.join(results_directory, 'experiment.exp') + FileUtils().WriteFile(experiment_file_path, experiment.experiment_file) + + self.l.LogOutput('Storing results report in %s.' % results_directory) + results_table_path = os.path.join(results_directory, 'results.html') + report = HTMLResultsReport.FromExperiment(experiment).GetReport() + if self.json_report: + json_report = JSONResultsReport.FromExperiment(experiment, + json_args={'indent': 2}) + _WriteJSONReportToFile(experiment, results_directory, json_report) + + FileUtils().WriteFile(results_table_path, report) + + self.l.LogOutput('Storing email message body in %s.' % results_directory) + msg_file_path = os.path.join(results_directory, 'msg_body.html') + text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() + text_report += ('\nResults are stored in %s.\n' % + experiment.results_directory) + msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report + FileUtils().WriteFile(msg_file_path, msg_body) + + self.l.LogOutput('Storing results of each benchmark run.') + for benchmark_run in experiment.benchmark_runs: + if benchmark_run.result: + benchmark_run_name = filter(str.isalnum, benchmark_run.name) + benchmark_run_path = os.path.join(results_directory, benchmark_run_name) + benchmark_run.result.CopyResultsTo(benchmark_run_path) + benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp) + + def Run(self): + try: + self._Run(self._experiment) + finally: + # Always print the report at the end of the run. + self._PrintTable(self._experiment) + if not self._terminated: + self._StoreResults(self._experiment) + self._Email(self._experiment) + + +class MockExperimentRunner(ExperimentRunner): + """Mocked ExperimentRunner for testing.""" + + def __init__(self, experiment, json_report): + super(MockExperimentRunner, self).__init__(experiment, json_report) + + def _Run(self, experiment): + self.l.LogOutput("Would run the following experiment: '%s'." % + experiment.name) + + def _PrintTable(self, experiment): + self.l.LogOutput('Would print the experiment table.') + + def _Email(self, experiment): + self.l.LogOutput('Would send result email.') + + def _StoreResults(self, experiment): + self.l.LogOutput('Would store the results.') |