diff options
| -rwxr-xr-x | cros_utils/device_setup_utils.py | 7 | ||||
| -rwxr-xr-x | cros_utils/device_setup_utils_unittest.py | 6 | ||||
| -rw-r--r-- | crosperf/suite_runner.py | 111 | ||||
| -rwxr-xr-x | crosperf/suite_runner_unittest.py | 107 |
4 files changed, 129 insertions, 102 deletions
diff --git a/cros_utils/device_setup_utils.py b/cros_utils/device_setup_utils.py index 3b2d1326..93e681bd 100755 --- a/cros_utils/device_setup_utils.py +++ b/cros_utils/device_setup_utils.py @@ -327,10 +327,13 @@ class DutWrapper(object): self.RunCommandOnDut(sed_command + FILE) def StopUI(self): - self.RunCommandOnDut('stop ui') + # Added "ignore_status" for the case when crosperf stops ui service which + # was already stopped. Command is going to fail with 1. + self.RunCommandOnDut('stop ui', ignore_status=True) def StartUI(self): - self.RunCommandOnDut('start ui') + # Similar to StopUI, `start ui` fails if the service is already started. + self.RunCommandOnDut('start ui', ignore_status=True) def KerncmdUpdateNeeded(self, intel_pstate): """Check whether kernel cmdline update is needed. diff --git a/cros_utils/device_setup_utils_unittest.py b/cros_utils/device_setup_utils_unittest.py index 162851ed..b2f76f7d 100755 --- a/cros_utils/device_setup_utils_unittest.py +++ b/cros_utils/device_setup_utils_unittest.py @@ -606,12 +606,14 @@ class DutWrapperTest(unittest.TestCase): def test_stop_ui(self): self.dw.RunCommandOnDut = mock.Mock(return_value=(0, '', '')) self.dw.StopUI() - self.dw.RunCommandOnDut.assert_called_once_with('stop ui') + self.dw.RunCommandOnDut.assert_called_once_with( + 'stop ui', ignore_status=True) def test_start_ui(self): self.dw.RunCommandOnDut = mock.Mock(return_value=(0, '', '')) self.dw.StartUI() - self.dw.RunCommandOnDut.assert_called_once_with('start ui') + self.dw.RunCommandOnDut.assert_called_once_with( + 'start ui', ignore_status=True) if __name__ == '__main__': diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index ddca65df..b5649e86 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -13,6 +13,8 @@ import os import shlex import time +from contextlib import contextmanager + from cros_utils import command_executer from cros_utils.device_setup_utils import DutWrapper @@ -112,59 +114,68 @@ class SuiteRunner(object): break return ret_tup + @contextmanager + def PauseUI(self, run_on_dut): + """Stop UI before and Start UI after the context block. + + Context manager will make sure UI is always resumed at the end. + """ + run_on_dut.StopUI() + try: + yield + + finally: + run_on_dut.StartUI() + def SetupDevice(self, run_on_dut, cros_machine): - # Stop UI before configuring the DUT. + # Pause UI while configuring the DUT. # This will accelerate setup (waiting for cooldown has x10 drop) # and help to reset a Chrome state left after the previous test. - run_on_dut.StopUI() - - # Unless the user turns on ASLR in the flag, we first disable ASLR - # before running the benchmarks - if not self.enable_aslr: - run_on_dut.DisableASLR() - - # CPU usage setup comes first where we enable/disable cores. - run_on_dut.SetupCpuUsage() - cpu_online_status = run_on_dut.GetCpuOnline() - # List of online cores of type int (core number). - online_cores = [ - core for core, status in cpu_online_status.items() if status - ] - if self.dut_config['cooldown_time']: - # Setup power conservative mode for effective cool down. - # Set ignore status since powersave may no be available - # on all platforms and we are going to handle it. - ret = run_on_dut.SetCpuGovernor('powersave', ignore_status=True) - if ret: - # "powersave" is not available, use "ondemand". - # Still not a fatal error if it fails. - ret = run_on_dut.SetCpuGovernor('ondemand', ignore_status=True) - # TODO(denik): Run comparison test for 'powersave' and 'ondemand' - # on scarlet and kevin64. - # We might have to consider reducing freq manually to the min - # if it helps to reduce waiting time. - wait_time = run_on_dut.WaitCooldown() - cros_machine.AddCooldownWaitTime(wait_time) - - # Setup CPU governor for the benchmark run. - # It overwrites the previous governor settings. - governor = self.dut_config['governor'] - # FIXME(denik): Pass online cores to governor setup. - run_on_dut.SetCpuGovernor(governor, ignore_status=False) - - # Disable Turbo and Setup CPU freq should ALWAYS proceed governor setup - # since governor may change: - # - frequency; - # - turbo/boost. - run_on_dut.DisableTurbo() - run_on_dut.SetupCpuFreq(online_cores) - # FIXME(denik): Currently we are not recovering the previous cpufreq - # settings since we do reboot/setup every time anyway. - # But it may change in the future and then we have to recover the - # settings. - - # DUT setup is done. Start a fresh new shiny UI. - run_on_dut.StartUI() + with self.PauseUI(run_on_dut): + # Unless the user turns on ASLR in the flag, we first disable ASLR + # before running the benchmarks + if not self.enable_aslr: + run_on_dut.DisableASLR() + + # CPU usage setup comes first where we enable/disable cores. + run_on_dut.SetupCpuUsage() + cpu_online_status = run_on_dut.GetCpuOnline() + # List of online cores of type int (core number). + online_cores = [ + core for core, status in cpu_online_status.items() if status + ] + if self.dut_config['cooldown_time']: + # Setup power conservative mode for effective cool down. + # Set ignore status since powersave may no be available + # on all platforms and we are going to handle it. + ret = run_on_dut.SetCpuGovernor('powersave', ignore_status=True) + if ret: + # "powersave" is not available, use "ondemand". + # Still not a fatal error if it fails. + ret = run_on_dut.SetCpuGovernor('ondemand', ignore_status=True) + # TODO(denik): Run comparison test for 'powersave' and 'ondemand' + # on scarlet and kevin64. + # We might have to consider reducing freq manually to the min + # if it helps to reduce waiting time. + wait_time = run_on_dut.WaitCooldown() + cros_machine.AddCooldownWaitTime(wait_time) + + # Setup CPU governor for the benchmark run. + # It overwrites the previous governor settings. + governor = self.dut_config['governor'] + # FIXME(denik): Pass online cores to governor setup. + run_on_dut.SetCpuGovernor(governor, ignore_status=False) + + # Disable Turbo and Setup CPU freq should ALWAYS proceed governor setup + # since governor may change: + # - frequency; + # - turbo/boost. + run_on_dut.DisableTurbo() + run_on_dut.SetupCpuFreq(online_cores) + # FIXME(denik): Currently we are not recovering the previous cpufreq + # settings since we do reboot/setup every time anyway. + # But it may change in the future and then we have to recover the + # settings. def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): """Run the test_that test..""" diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index 42b2392d..5f377d7c 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -178,36 +178,29 @@ class SuiteRunnerTest(unittest.TestCase): del command, ignore_status return 0, '', '' - DutWrapper.RunCommandOnDut = mock.Mock(return_value=FakeRunner) - DutWrapper.DisableASLR = mock.Mock() - DutWrapper.SetupCpuUsage = mock.Mock() - DutWrapper.SetupCpuFreq = mock.Mock() - DutWrapper.DisableTurbo = mock.Mock() - DutWrapper.SetCpuGovernor = mock.Mock() - DutWrapper.WaitCooldown = mock.Mock(return_value=0) - DutWrapper.GetCpuOnline = mock.Mock(return_value={0: 1, 1: 1, 2: 0}) - self.runner.dut_config['cooldown_time'] = 0 - self.runner.dut_config['governor'] = 'fake_governor' - self.runner.dut_config['cpu_freq_pct'] = 65 - machine = 'fake_machine' + mock_run_on_dut = mock.Mock(spec=DutWrapper) cros_machine = MockCrosMachine(machine, self.mock_label.chromeos_root, self.mock_logger) - mock_run_on_dut = DutWrapper( - self.mock_label.chromeos_root, - machine, - logger=self.mock_logger, - dut_config=self.runner.dut_config) + + mock_run_on_dut.RunCommandOnDut = mock.Mock(return_value=FakeRunner) + mock_run_on_dut.WaitCooldown = mock.Mock(return_value=0) + mock_run_on_dut.GetCpuOnline = mock.Mock(return_value={0: 1, 1: 1, 2: 0}) + self.runner.dut_config['cooldown_time'] = 0 + self.runner.dut_config['governor'] = 'fake_governor' + self.runner.dut_config['cpu_freq_pct'] = 65 self.runner.SetupDevice(mock_run_on_dut, cros_machine) - DutWrapper.SetupCpuUsage.assert_called_once_with() - DutWrapper.SetupCpuFreq.assert_called_once_with([0, 1]) - DutWrapper.GetCpuOnline.assert_called_once_with() - DutWrapper.SetCpuGovernor.assert_called_once_with( + mock_run_on_dut.SetupCpuUsage.assert_called_once_with() + mock_run_on_dut.SetupCpuFreq.assert_called_once_with([0, 1]) + mock_run_on_dut.GetCpuOnline.assert_called_once_with() + mock_run_on_dut.SetCpuGovernor.assert_called_once_with( 'fake_governor', ignore_status=False) - DutWrapper.DisableTurbo.assert_called_once_with() - DutWrapper.WaitCooldown.assert_not_called() + mock_run_on_dut.DisableTurbo.assert_called_once_with() + mock_run_on_dut.StopUI.assert_called_once_with() + mock_run_on_dut.StartUI.assert_called_once_with() + mock_run_on_dut.WaitCooldown.assert_not_called() def test_setup_device_with_cooldown(self): @@ -216,40 +209,58 @@ class SuiteRunnerTest(unittest.TestCase): del command, ignore_status return 0, '', '' - DutWrapper.RunCommandOnDut = mock.Mock(return_value=FakeRunner) - DutWrapper.DisableASLR = mock.Mock() - DutWrapper.DisableTurbo = mock.Mock() - DutWrapper.SetCpuGovernor = mock.Mock() - DutWrapper.SetupCpuUsage = mock.Mock() - DutWrapper.SetupCpuFreq = mock.Mock() - DutWrapper.WaitCooldown = mock.Mock(return_value=0) - DutWrapper.GetCpuOnline = mock.Mock(return_value={0: 0, 1: 1}) + machine = 'fake_machine' + mock_run_on_dut = mock.Mock(spec=DutWrapper) + cros_machine = MockCrosMachine(machine, self.mock_label.chromeos_root, + self.mock_logger) + + mock_run_on_dut.RunCommandOnDut = mock.Mock(return_value=FakeRunner) + mock_run_on_dut.WaitCooldown = mock.Mock(return_value=0) + mock_run_on_dut.GetCpuOnline = mock.Mock(return_value={0: 0, 1: 1}) self.runner.dut_config['cooldown_time'] = 10 self.runner.dut_config['governor'] = 'fake_governor' self.runner.dut_config['cpu_freq_pct'] = 75 + self.runner.SetupDevice(mock_run_on_dut, cros_machine) + + mock_run_on_dut.WaitCooldown.assert_called_once_with() + mock_run_on_dut.DisableASLR.assert_called_once() + mock_run_on_dut.DisableTurbo.assert_called_once_with() + mock_run_on_dut.SetupCpuUsage.assert_called_once_with() + mock_run_on_dut.SetupCpuFreq.assert_called_once_with([1]) + mock_run_on_dut.SetCpuGovernor.assert_called() + mock_run_on_dut.GetCpuOnline.assert_called_once_with() + mock_run_on_dut.StopUI.assert_called_once_with() + mock_run_on_dut.StartUI.assert_called_once_with() + self.assertGreater(mock_run_on_dut.SetCpuGovernor.call_count, 1) + self.assertEqual(mock_run_on_dut.SetCpuGovernor.call_args, + mock.call('fake_governor', ignore_status=False)) + + def test_setup_device_with_exception(self): + """Test SetupDevice with an exception.""" + machine = 'fake_machine' + mock_run_on_dut = mock.Mock(spec=DutWrapper) cros_machine = MockCrosMachine(machine, self.mock_label.chromeos_root, self.mock_logger) - mock_run_on_dut = DutWrapper( - self.mock_label.chromeos_root, - machine, - logger=self.mock_logger, - dut_config=self.runner.dut_config) - self.runner.SetupDevice(mock_run_on_dut, cros_machine) - - DutWrapper.WaitCooldown.assert_called_once_with() - DutWrapper.DisableASLR.assert_called_once() - DutWrapper.DisableTurbo.assert_called_once_with() - DutWrapper.SetupCpuUsage.assert_called_once_with() - DutWrapper.SetupCpuFreq.assert_called_once_with([1]) - DutWrapper.SetCpuGovernor.assert_called() - DutWrapper.GetCpuOnline.assert_called_once_with() - self.assertGreater(DutWrapper.SetCpuGovernor.call_count, 1) - self.assertEqual(DutWrapper.SetCpuGovernor.call_args, - mock.call('fake_governor', ignore_status=False)) + mock_run_on_dut.SetupCpuUsage = mock.Mock(side_effect=RuntimeError()) + + with self.assertRaises(RuntimeError): + self.runner.SetupDevice(mock_run_on_dut, cros_machine) + + # This called injected an exception. + mock_run_on_dut.SetupCpuUsage.assert_called_once_with() + # Calls following the expeption are skipped. + mock_run_on_dut.WaitCooldown.assert_not_called() + mock_run_on_dut.DisableTurbo.assert_not_called() + mock_run_on_dut.SetupCpuFreq.assert_not_called() + mock_run_on_dut.SetCpuGovernor.assert_not_called() + mock_run_on_dut.GetCpuOnline.assert_not_called() + # Check that Stop/Start UI are always called. + mock_run_on_dut.StopUI.assert_called_once_with() + mock_run_on_dut.StartUI.assert_called_once_with() @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') @mock.patch.object(command_executer.CommandExecuter, |