diff options
| -rw-r--r-- | crosperf/experiment.py | 3 | ||||
| -rw-r--r-- | crosperf/experiment_runner.py | 5 | ||||
| -rw-r--r-- | crosperf/suite_runner.py | 170 | ||||
| -rwxr-xr-x | crosperf/suite_runner_unittest.py | 141 |
4 files changed, 190 insertions, 129 deletions
diff --git a/crosperf/experiment.py b/crosperf/experiment.py index 80674b4b..1ce9b5aa 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -105,6 +105,9 @@ class Experiment(object): except BadChecksum: # Force same image on all machines, then we do checksum again. No # bailout if checksums still do not match. + # TODO: It's not a good idea to force flashing the image when we are + # running with skylab... Although for now it even helps us getting + # that machine... self.machine_manager.ForceSameImageToAllMachines(label) self.machine_manager.ComputeCommonCheckSum(label) diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index 9eff1fb5..159c86e4 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -2,6 +2,7 @@ # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """The experiment runner module.""" from __future__ import print_function @@ -166,7 +167,7 @@ class ExperimentRunner(object): def _Run(self, experiment): try: - if not experiment.locks_dir: + if not experiment.locks_dir and not experiment.skylab: self._LockAllMachines(experiment) if self._using_schedv2: schedv2 = Schedv2(experiment) @@ -211,7 +212,7 @@ class ExperimentRunner(object): experiment.Terminate() raise finally: - if not experiment.locks_dir: + if not experiment.locks_dir and not experiment.skylab: self._UnlockAllMachines(experiment) def _PrintTable(self, experiment): diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index ba1c3fa7..f61a90bf 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -9,11 +9,14 @@ from __future__ import print_function import os import shlex +import time from cros_utils import command_executer -import test_flag TEST_THAT_PATH = '/usr/bin/test_that' +# TODO: Need to check whether Skylab is installed and set up correctly. +SKYLAB_PATH = '/usr/local/bin/skylab' +GS_UTIL = 'chromium/tools/depot_tools/gsutil.py' AUTOTEST_DIR = '~/trunk/src/third_party/autotest/files' CHROME_MOUNT_DIR = '/tmp/chrome_root' @@ -62,21 +65,26 @@ class SuiteRunner(object): def Run(self, machine, label, benchmark, test_args, profiler_args): for i in range(0, benchmark.retries + 1): - # Unless the user turns on ASLR in the flag, we first disable ASLR - # before running the benchmarks - if not self.enable_aslr: - self.DisableASLR(machine, label.chromeos_root) - self.PinGovernorExecutionFrequencies(machine, label.chromeos_root) - if benchmark.suite == 'telemetry': - self.DecreaseWaitTime(machine, label.chromeos_root) - ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args) - elif benchmark.suite == 'telemetry_Crosperf': - self.DecreaseWaitTime(machine, label.chromeos_root) - ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark, - test_args, profiler_args) + if label.skylab: + # TODO: need to migrate DisableASLR and PinGovernorExecutionFrequencies + # since in skylab mode, we may not know the DUT until one is assigned + # to the test. For telemetry_Crosperf run, we can move them into the + # server test script, for client runs, need to figure out wrapper to do + # it before running. + ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args) else: - ret_tup = self.Test_That_Run(machine, label, benchmark, test_args, - profiler_args) + # Unless the user turns on ASLR in the flag, we first disable ASLR + # before running the benchmarks + if not self.enable_aslr: + self.DisableASLR(machine, label.chromeos_root) + self.PinGovernorExecutionFrequencies(machine, label.chromeos_root) + if benchmark.suite == 'telemetry_Crosperf': + self.DecreaseWaitTime(machine, label.chromeos_root) + ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark, + test_args, profiler_args) + else: + ret_tup = self.Test_That_Run(machine, label, benchmark, test_args, + profiler_args) if ret_tup[0] != 0: self.logger.LogOutput('benchmark %s failed. Retries left: %s' % @@ -226,6 +234,103 @@ class SuiteRunner(object): command_terminator=self._ct, cros_sdk_options='--no-ns-pid') + def DownloadResult(self, label, task_id): + gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL) + result_dir = 'gs://chromeos-autotest-results/swarming-%s' % task_id + download_path = os.path.join(label.chromeos_root, 'chroot/tmp') + ls_command = '%s ls %s' % (gsutil_cmd, + os.path.join(result_dir, 'autoserv_test')) + cp_command = '%s -mq cp -r %s %s' % (gsutil_cmd, result_dir, download_path) + + # Server sometimes will not be able to generate the result directory right + # after the test. Will try to access this gs location every 60s for 5 mins. + t = 0 + RETRY_LIMIT = 5 + while t < RETRY_LIMIT: + self.logger.LogOutput('Downloading test results.') + t += 1 + status = self._ce.RunCommand(ls_command, print_to_console=False) + if status == 0: + break + if t < RETRY_LIMIT: + self.logger.LogOutput('Result directory not generated yet, ' + 'retry (%d) in 60s.' % t) + time.sleep(60) + else: + self.logger.LogOutput('No result directory for task %s' % task_id) + return status + + # Wait for 60s to make sure server finished writing to gs location. + time.sleep(60) + + status = self._ce.RunCommand(cp_command) + if status != 0: + self.logger.LogOutput('Cannot download results from task %s' % task_id) + return status + + def Skylab_Run(self, label, benchmark, test_args, profiler_args): + """Run the test via skylab..""" + options = '' + if benchmark.suite != 'telemetry_Crosperf': + options += '-client-test' + if label.board: + options += ' -board=%s' % label.board + if label.build: + options += ' -image=%s' % label.build + # TODO: now only put quota pool here, user need to be able to specify which + # pool to use. Need to request feature to not use this option at all. + options += ' -pool=DUT_POOL_QUOTA' + if benchmark.suite == 'telemetry_Crosperf': + if test_args: + # Strip double quotes off args (so we can wrap them in single + # quotes, to pass through to Telemetry). + if test_args[0] == '"' and test_args[-1] == '"': + test_args = test_args[1:-1] + if profiler_args: + test_args += GetProfilerArgs(profiler_args) + test_args += ' run_local={} test={}'.format( + benchmark.run_local, + benchmark.test_name, + ) + else: + if profiler_args: + self.logger.LogFatal('Client tests do not support profiler.') + if test_args: + options += ' -test-args="%s"' % test_args + + dimensions = '' + for dut in label.remote: + dimensions += ' dut_name:%s' % dut.rstrip('.cros') + + command = (('%s create-test %s %s %s') % \ + (SKYLAB_PATH, options, benchmark.test_name, dimensions)) + + if self.log_level != 'verbose': + self.logger.LogOutput('Starting skylab test.') + self.logger.LogOutput('CMD: %s' % command) + ret_tup = self._ce.RunCommandWOutput(command, command_terminator=self._ct) + + if ret_tup[0] != 0: + self.logger.LogOutput('Skylab test not created successfully.') + return ret_tup + + # Std output of the command will look like: + # Created Swarming task https://chromeos-swarming.appspot.com/task?id=12345 + # We want to parse it and get the id number of the task. + task_id = ret_tup[1].strip().split('id=')[1] + + command = ('skylab wait-task %s' % (task_id)) + if self.log_level != 'verbose': + self.logger.LogOutput('Waiting for skylab test to finish.') + self.logger.LogOutput('CMD: %s' % command) + + ret_tup = self._ce.RunCommandWOutput(command, command_terminator=self._ct) + if '"success":true' in ret_tup[1]: + if self.DownloadResult(label, task_id) == 0: + result_dir = '\nResults placed in tmp/swarming-%s\n' % task_id + return (ret_tup[0], result_dir, ret_tup[2]) + return ret_tup + def RemoveTelemetryTempFile(self, machine, chromeos_root): filename = 'telemetry@%s' % machine fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename) @@ -284,41 +389,6 @@ class SuiteRunner(object): command_terminator=self._ct, cros_sdk_options=chrome_root_options) - def Telemetry_Run(self, machine, label, benchmark, profiler_args): - telemetry_run_path = '' - if not os.path.isdir(label.chrome_src): - self.logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.') - else: - telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf') - if not os.path.exists(telemetry_run_path): - self.logger.LogFatal('Cannot find %s directory.' % telemetry_run_path) - - if profiler_args: - self.logger.LogFatal('Telemetry does not support the perf profiler.') - - # Check for and remove temporary file that may have been left by - # previous telemetry runs (and which might prevent this run from - # working). - if not test_flag.GetTestMode(): - self.RemoveTelemetryTempFile(machine, label.chromeos_root) - - rsa_key = os.path.join( - label.chromeos_root, - 'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa') - - cmd = ('cd {0} && ' - './run_measurement ' - '--browser=cros-chrome ' - '--output-format=csv ' - '--remote={1} ' - '--identity {2} ' - '{3} {4}'.format(telemetry_run_path, machine, rsa_key, - benchmark.test_name, benchmark.test_args)) - if self.log_level != 'verbose': - self.logger.LogOutput('Running test.') - self.logger.LogOutput('CMD: %s' % cmd) - return self._ce.RunCommandWOutput(cmd, print_to_console=False) - def CommandTerminator(self): return self._ct diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index e3a34f1e..2fcb45ac 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -10,13 +10,13 @@ from __future__ import print_function import os.path +import time import mock import unittest import suite_runner import label -import test_flag from benchmark import Benchmark @@ -68,13 +68,14 @@ class SuiteRunnerTest(unittest.TestCase): self.call_test_that_run = False self.disable_aslr_args = [] self.pin_governor_args = [] + self.skylab_run_args = [] self.test_that_args = [] self.telemetry_run_args = [] self.telemetry_crosperf_args = [] + self.call_skylab_run = False self.call_telemetry_crosperf_run = False self.call_disable_aslr = False self.call_pin_governor = False - self.call_telemetry_run = False def setUp(self): self.runner = suite_runner.SuiteRunner( @@ -93,9 +94,10 @@ class SuiteRunnerTest(unittest.TestCase): def reset(): self.call_pin_governor = False self.call_test_that_run = False - self.call_telemetry_run = False + self.call_skylab_run = False self.call_telemetry_crosperf_run = False self.pin_governor_args = [] + self.skylab_run_args = [] self.test_that_args = [] self.telemetry_run_args = [] self.telemetry_crosperf_args = [] @@ -108,10 +110,10 @@ class SuiteRunnerTest(unittest.TestCase): self.call_pin_governor = True self.pin_governor_args = [machine, chroot] - def FakeTelemetryRun(machine, test_label, benchmark, profiler_args): - self.telemetry_run_args = [machine, test_label, benchmark, profiler_args] - self.call_telemetry_run = True - return 'Ran FakeTelemetryRun' + def FakeSkylabRun(test_label, benchmark, test_args, profiler_args): + self.skylab_run_args = [test_label, benchmark, test_args, profiler_args] + self.call_skylab_run = True + return 'Ran FakeSkylabRun' def FakeTelemetryCrosperfRun(machine, test_label, benchmark, test_args, profiler_args): @@ -131,31 +133,31 @@ class SuiteRunnerTest(unittest.TestCase): self.runner.DisableASLR = FakeDisableASLR self.runner.PinGovernorExecutionFrequencies = FakePinGovernor - self.runner.Telemetry_Run = FakeTelemetryRun + self.runner.Skylab_Run = FakeSkylabRun self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun self.runner.Test_That_Run = FakeTestThatRun - machine = 'fake_machine' test_args = '' profiler_args = '' + reset() + self.mock_label.skylab = True self.runner.Run(machine, self.mock_label, self.telemetry_bench, test_args, profiler_args) - self.assertTrue(self.call_disable_aslr) - self.assertTrue(self.call_pin_governor) - self.assertTrue(self.call_telemetry_run) + self.assertFalse(self.call_disable_aslr) + self.assertFalse(self.call_pin_governor) + self.assertTrue(self.call_skylab_run) self.assertFalse(self.call_test_that_run) self.assertFalse(self.call_telemetry_crosperf_run) - self.assertEqual( - self.telemetry_run_args, - ['fake_machine', self.mock_label, self.telemetry_bench, '']) + self.assertEqual(self.skylab_run_args, + [self.mock_label, self.telemetry_bench, '', '']) + self.mock_label.skylab = False reset() self.runner.Run(machine, self.mock_label, self.test_that_bench, test_args, profiler_args) self.assertTrue(self.call_disable_aslr) self.assertTrue(self.call_pin_governor) - self.assertFalse(self.call_telemetry_run) self.assertTrue(self.call_test_that_run) self.assertFalse(self.call_telemetry_crosperf_run) self.assertEqual( @@ -167,7 +169,6 @@ class SuiteRunnerTest(unittest.TestCase): test_args, profiler_args) self.assertTrue(self.call_disable_aslr) self.assertTrue(self.call_pin_governor) - self.assertFalse(self.call_telemetry_run) self.assertFalse(self.call_test_that_run) self.assertTrue(self.call_telemetry_crosperf_run) self.assertEqual(self.telemetry_crosperf_args, [ @@ -235,10 +236,6 @@ class SuiteRunnerTest(unittest.TestCase): 'ChrootRunCommandWOutput') def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd): - def FakeRebootMachine(machine, chroot): - if machine or chroot: - pass - def FakeLogMsg(fd, termfd, msg, flush=True): if fd or termfd or msg or flush: pass @@ -246,7 +243,6 @@ class SuiteRunnerTest(unittest.TestCase): save_log_msg = self.real_logger.LogMsg self.real_logger.LogMsg = FakeLogMsg self.runner.logger = self.real_logger - self.runner.RebootMachine = FakeRebootMachine raised_exception = False try: @@ -307,67 +303,58 @@ class SuiteRunnerTest(unittest.TestCase): self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) self.assertEqual(len(args_dict), 2) - @mock.patch.object(os.path, 'isdir') - @mock.patch.object(os.path, 'exists') @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') - def test_telemetry_run(self, mock_runcmd, mock_exists, mock_isdir): - - def FakeLogMsg(fd, termfd, msg, flush=True): - if fd or termfd or msg or flush: - pass + def test_skylab_run(self, mock_runcmd): - save_log_msg = self.real_logger.LogMsg - self.real_logger.LogMsg = FakeLogMsg - mock_runcmd.return_value = 0 + def FakeDownloadResult(l, task_id): + if l and task_id: + self.assertEqual(task_id, '12345') + return 0 + mock_runcmd.return_value = \ + (0, + '"success":true\nCreated Swarming task https://swarming/task?id=12345', + '') self.mock_cmd_exec.RunCommandWOutput = mock_runcmd - self.runner.logger = self.real_logger - - profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options' - '="record -a -e cycles,instructions"\'') - - raises_exception = False - mock_isdir.return_value = False - try: - self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, - self.telemetry_bench, '') - except SystemExit: - raises_exception = True - self.assertTrue(raises_exception) - - raises_exception = False - mock_isdir.return_value = True - mock_exists.return_value = False - try: - self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, - self.telemetry_bench, '') - except SystemExit: - raises_exception = True - self.assertTrue(raises_exception) - - raises_exception = False - mock_isdir.return_value = True - mock_exists.return_value = True - try: - self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, - self.telemetry_bench, profiler_args) - except SystemExit: - raises_exception = True - self.assertTrue(raises_exception) + self.mock_label.skylab = True + self.runner.DownloadResult = FakeDownloadResult + res = self.runner.Skylab_Run(self.mock_label, self.test_that_bench, '', '') + ret_tup = (0, '\nResults placed in tmp/swarming-12345\n', '') + self.assertEqual(res, ret_tup) + self.assertEqual(mock_runcmd.call_count, 2) + + args_list = mock_runcmd.call_args_list[0][0] + args_dict = mock_runcmd.call_args_list[0][1] + self.assertEqual(args_list[0], + ('/usr/local/bin/skylab create-test -client-test ' + '-board=lumpy -image=build -pool=DUT_POOL_QUOTA ' + 'octane dut_name:lumpy1 dut_name:lumpy.cros2')) + self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - test_flag.SetTestMode(True) - res = self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, - self.telemetry_bench, '') - self.assertEqual(res, 0) - self.assertEqual(mock_runcmd.call_count, 1) - self.assertEqual( - mock_runcmd.call_args_list[0][0], - (('cd src/tools/perf && ./run_measurement ' - '--browser=cros-chrome --output-format=csv ' - '--remote=lumpy1.cros --identity /tmp/chromeos/src/scripts' - '/mod_for_test_scripts/ssh_keys/testing_rsa octane '),)) + args_list = mock_runcmd.call_args_list[1][0] + self.assertEqual(args_list[0], ('skylab wait-task 12345')) + self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - self.real_logger.LogMsg = save_log_msg + @mock.patch.object(time, 'sleep') + @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') + def test_download_result(self, mock_runcmd, mock_sleep): + mock_runcmd.return_value = 0 + mock_sleep.return_value = 0 + self.mock_cmd_exec.RunCommand = mock_runcmd + + self.runner.DownloadResult(self.mock_label, '12345') + + self.assertEqual(mock_runcmd.call_count, 2) + cmd = mock_runcmd.call_args_list[0][0][0] + self.assertEqual(cmd, + ('/tmp/chromeos/chromium/tools/depot_tools/gsutil.py ls ' + 'gs://chromeos-autotest-results/swarming-12345/' + 'autoserv_test')) + cmd = mock_runcmd.call_args_list[1][0][0] + self.assertEqual(cmd, + ('/tmp/chromeos/chromium/tools/depot_tools/gsutil.py -mq ' + 'cp -r gs://chromeos-autotest-results/swarming-12345 ' + '/tmp/chromeos/chroot/tmp')) if __name__ == '__main__': |