diff options
Diffstat (limited to 'crosperf/suite_runner_unittest.py')
| -rwxr-xr-x | crosperf/suite_runner_unittest.py | 213 |
1 files changed, 65 insertions, 148 deletions
diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index 7f3a7bc4..73fcb45b 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -10,8 +10,6 @@ from __future__ import print_function import json -import os.path -import time import unittest import unittest.mock as mock @@ -23,7 +21,6 @@ from benchmark import Benchmark from cros_utils import command_executer from cros_utils import logger -from cros_utils.device_setup_utils import DutWrapper from machine_manager import MockCrosMachine @@ -49,23 +46,21 @@ class SuiteRunnerTest(unittest.TestCase): 'telemetry_Crosperf', # suite True) # show_all_results - test_that_bench = Benchmark( + crosperf_wrapper_bench = Benchmark( 'b2_test', # name - 'octane', # test_name + 'webgl', # test_name '', # test_args 3, # iterations False, # rm_chroot_tmp - 'record -e cycles') # perf_args + '', # perf_args + 'crosperf_Wrapper') # suite def __init__(self, *args, **kwargs): super(SuiteRunnerTest, self).__init__(*args, **kwargs) - self.call_test_that_run = False self.skylab_run_args = [] self.test_that_args = [] - self.telemetry_run_args = [] - self.telemetry_crosperf_args = [] self.call_skylab_run = False - self.call_telemetry_crosperf_run = False + self.call_test_that_run = False def setUp(self): self.runner = suite_runner.SuiteRunner( @@ -79,30 +74,30 @@ class SuiteRunnerTest(unittest.TestCase): res = suite_runner.GetProfilerArgs(input_str) self.assertEqual(res, output_str) + def test_get_dut_config_args(self): + dut_config = {'enable_aslr': False, 'top_interval': 1.0} + output_str = ('dut_config=' + "'" + '{"enable_aslr": ' + 'false, "top_interval": 1.0}' + "'" + '') + res = suite_runner.GetDutConfigArgs(dut_config) + self.assertEqual(res, output_str) + def test_run(self): def reset(): + self.test_that_args = [] + self.skylab_run_args = [] self.call_test_that_run = False self.call_skylab_run = False - self.call_telemetry_crosperf_run = False - self.skylab_run_args = [] - self.test_that_args = [] - self.telemetry_run_args = [] - self.telemetry_crosperf_args = [] def FakeSkylabRun(test_label, benchmark, test_args, profiler_args): self.skylab_run_args = [test_label, benchmark, test_args, profiler_args] self.call_skylab_run = True return 'Ran FakeSkylabRun' - def FakeTelemetryCrosperfRun(machine, test_label, benchmark, test_args, - profiler_args): - self.telemetry_crosperf_args = [ - machine, test_label, benchmark, test_args, profiler_args - ] - self.call_telemetry_crosperf_run = True - return 'Ran FakeTelemetryCrosperfRun' - def FakeTestThatRun(machine, test_label, benchmark, test_args, profiler_args): self.test_that_args = [ @@ -111,16 +106,8 @@ class SuiteRunnerTest(unittest.TestCase): self.call_test_that_run = True return 'Ran FakeTestThatRun' - def FakeRunner(command, ignore_status=False): - # pylint fix for unused variable. - del command, ignore_status - return 0, '', '' - self.runner.Skylab_Run = FakeSkylabRun - self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun self.runner.Test_That_Run = FakeTestThatRun - DutWrapper.SetupDevice = mock.Mock(return_value=0) - DutWrapper.RunCommandOnDut = mock.Mock(return_value=FakeRunner) self.runner.dut_config['enable_aslr'] = False self.runner.dut_config['cooldown_time'] = 0 @@ -133,36 +120,63 @@ class SuiteRunnerTest(unittest.TestCase): test_args = '' profiler_args = '' - reset() + # Test skylab run for telemetry_Crosperf and crosperf_Wrapper benchmarks. self.mock_label.skylab = True - self.runner.Run(cros_machine, self.mock_label, self.test_that_bench, + reset() + self.runner.Run(cros_machine, self.mock_label, self.crosperf_wrapper_bench, test_args, profiler_args) self.assertTrue(self.call_skylab_run) self.assertFalse(self.call_test_that_run) - self.assertFalse(self.call_telemetry_crosperf_run) self.assertEqual(self.skylab_run_args, - [self.mock_label, self.test_that_bench, '', '']) - DutWrapper.SetupDevice.assert_called_once() - self.mock_label.skylab = False + [self.mock_label, self.crosperf_wrapper_bench, '', '']) + + reset() + self.runner.Run(cros_machine, self.mock_label, + self.telemetry_crosperf_bench, test_args, profiler_args) + self.assertTrue(self.call_skylab_run) + self.assertFalse(self.call_test_that_run) + self.assertEqual(self.skylab_run_args, + [self.mock_label, self.telemetry_crosperf_bench, '', '']) + # Test test_that run for telemetry_Crosperf and crosperf_Wrapper benchmarks. + self.mock_label.skylab = False reset() - self.runner.Run(cros_machine, self.mock_label, self.test_that_bench, + self.runner.Run(cros_machine, self.mock_label, self.crosperf_wrapper_bench, test_args, profiler_args) self.assertTrue(self.call_test_that_run) - self.assertFalse(self.call_telemetry_crosperf_run) + self.assertFalse(self.call_skylab_run) self.assertEqual( self.test_that_args, - ['fake_machine', self.mock_label, self.test_that_bench, '', '']) + ['fake_machine', self.mock_label, self.crosperf_wrapper_bench, '', '']) reset() self.runner.Run(cros_machine, self.mock_label, self.telemetry_crosperf_bench, test_args, profiler_args) - self.assertFalse(self.call_test_that_run) - self.assertTrue(self.call_telemetry_crosperf_run) - self.assertEqual(self.telemetry_crosperf_args, [ + self.assertTrue(self.call_test_that_run) + self.assertFalse(self.call_skylab_run) + self.assertEqual(self.test_that_args, [ 'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', '' ]) + def test_gen_test_args(self): + test_args = '--iterations=2' + perf_args = 'record -a -e cycles' + + # Test crosperf_Wrapper benchmarks arg list generation + args_list = ["test_args='--iterations=2'", "dut_config='{}'", 'test=webgl'] + res = self.runner.GenTestArgs(self.crosperf_wrapper_bench, test_args, '') + self.assertCountEqual(res, args_list) + + # Test telemetry_Crosperf benchmarks arg list generation + args_list = [ + "test_args='--iterations=2'", "dut_config='{}'", 'test=octane', + 'run_local=False' + ] + args_list.append(suite_runner.GetProfilerArgs(perf_args)) + res = self.runner.GenTestArgs(self.telemetry_crosperf_bench, test_args, + perf_args) + self.assertCountEqual(res, args_list) + @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommandWOutput') @@ -176,10 +190,12 @@ class SuiteRunnerTest(unittest.TestCase): self.real_logger.LogMsg = FakeLogMsg self.runner.logger = self.real_logger + # Test crosperf_Wrapper benchmarks cannot take perf_args raised_exception = False try: self.runner.Test_That_Run('lumpy1.cros', self.mock_label, - self.test_that_bench, '', 'record -a -e cycles') + self.crosperf_wrapper_bench, '', + 'record -a -e cycles') except SystemExit: raised_exception = True self.assertTrue(raised_exception) @@ -188,7 +204,8 @@ class SuiteRunnerTest(unittest.TestCase): self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd res = self.runner.Test_That_Run('lumpy1.cros', self.mock_label, - self.test_that_bench, '--iterations=2', '') + self.crosperf_wrapper_bench, + '--iterations=2', '') self.assertEqual(mock_cros_runcmd.call_count, 1) self.assertEqual(mock_chroot_runcmd.call_count, 1) self.assertEqual(res, 0) @@ -197,48 +214,9 @@ class SuiteRunnerTest(unittest.TestCase): args_list = mock_chroot_runcmd.call_args_list[0][0] args_dict = mock_chroot_runcmd.call_args_list[0][1] self.assertEqual(len(args_list), 2) - self.assertEqual(args_list[0], '/tmp/chromeos') - self.assertEqual(args_list[1], ('/usr/bin/test_that ' - '--fast --board=lumpy ' - '--iterations=2 lumpy1.cros octane')) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) self.real_logger.LogMsg = save_log_msg - @mock.patch.object(os.path, 'isdir') - @mock.patch.object(command_executer.CommandExecuter, - 'ChrootRunCommandWOutput') - def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir): - - mock_isdir.return_value = True - mock_chroot_runcmd.return_value = 0 - self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd - profiler_args = ("--profiler=custom_perf --profiler_args='perf_options" - '="record -a -e cycles,instructions"\'') - self.runner.dut_config['turbostat'] = True - self.runner.dut_config['top_interval'] = 3 - res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label, - self.telemetry_crosperf_bench, '', - profiler_args) - self.assertEqual(res, 0) - self.assertEqual(mock_chroot_runcmd.call_count, 1) - args_list = mock_chroot_runcmd.call_args_list[0][0] - args_dict = mock_chroot_runcmd.call_args_list[0][1] - self.assertEqual(args_list[0], '/tmp/chromeos') - self.assertEqual( - args_list[1], - ('/usr/bin/test_that --autotest_dir ' - '/mnt/host/source/src/third_party/autotest/files --fast ' - "--board=lumpy --args=' run_local=False test=octane " - 'dut_config=\'"\'"\'{"turbostat": true, "top_interval": 3}\'"\'"\' ' - 'profiler=custom_perf profiler_args=\'"\'"\'record -a -e ' - 'cycles,instructions\'"\'"\'\' lumpy1.cros telemetry_Crosperf')) - self.assertEqual(args_dict['cros_sdk_options'], - ('--no-ns-pid --chrome_root= ' - '--chrome_root_mount=/tmp/chrome_root ' - 'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root')) - self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - self.assertEqual(len(args_dict), 2) - @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') @mock.patch.object(json, 'loads') def test_skylab_run_client(self, mock_json_loads, mock_runcmd): @@ -265,82 +243,21 @@ class SuiteRunnerTest(unittest.TestCase): self.mock_label.skylab = True self.runner.DownloadResult = FakeDownloadResult - res = self.runner.Skylab_Run(self.mock_label, self.test_that_bench, '', '') + res = self.runner.Skylab_Run(self.mock_label, self.crosperf_wrapper_bench, + '', '') ret_tup = (0, '\nResults placed in tmp/swarming-12345\n', '') self.assertEqual(res, ret_tup) self.assertEqual(mock_runcmd.call_count, 2) args_list = mock_runcmd.call_args_list[0][0] args_dict = mock_runcmd.call_args_list[0][1] - self.assertEqual(args_list[0], - ('/usr/local/bin/skylab create-test ' - '-dim dut_name:lumpy1 -dim dut_name:lumpy.cros2 ' - '-board=lumpy -image=build ' - '-pool=toolchain octane')) + self.assertEqual(len(args_list), 1) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) args_list = mock_runcmd.call_args_list[1][0] self.assertEqual(args_list[0], ('skylab wait-task 12345')) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') - @mock.patch.object(json, 'loads') - def test_skylab_run_telemetry_Crosperf(self, mock_json_loads, mock_runcmd): - - def FakeDownloadResult(l, task_id): - if l and task_id: - self.assertEqual(task_id, '12345') - return None - - mock_runcmd.return_value = ( - 0, - 'Created Swarming task https://swarming/task/b12345', - '', - ) - self.mock_cmd_exec.RunCommandWOutput = mock_runcmd - - mock_json_loads.return_value = { - 'child-results': [{ - 'success': True, - 'task-run-url': 'https://swarming/task?id=12345' - }] - } - self.mock_json.loads = mock_json_loads - - self.mock_label.skylab = True - self.runner.DownloadResult = FakeDownloadResult - self.runner.Skylab_Run(self.mock_label, self.telemetry_crosperf_bench, '', - '') - args_list = mock_runcmd.call_args_list[0][0] - self.assertEqual( - args_list[0], - ('/usr/local/bin/skylab create-test ' - '-dim dut_name:lumpy1 -dim dut_name:lumpy.cros2 ' - '-board=lumpy -image=build ' - "-pool=toolchain -test-args='run_local=False test=octane' " - 'telemetry_Crosperf')) - - @mock.patch.object(time, 'sleep') - @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') - def test_download_result(self, mock_runcmd, mock_sleep): - mock_runcmd.return_value = 0 - mock_sleep.return_value = 0 - self.mock_cmd_exec.RunCommand = mock_runcmd - - self.runner.DownloadResult(self.mock_label, '12345') - - self.assertEqual(mock_runcmd.call_count, 2) - cmd = mock_runcmd.call_args_list[0][0][0] - self.assertEqual(cmd, - ('/tmp/chromeos/src/chromium/depot_tools/gsutil.py ls ' - 'gs://chromeos-autotest-results/swarming-12345/' - 'autoserv_test')) - cmd = mock_runcmd.call_args_list[1][0][0] - self.assertEqual(cmd, - ('/tmp/chromeos/src/chromium/depot_tools/gsutil.py -mq ' - 'cp -r gs://chromeos-autotest-results/swarming-12345 ' - '/tmp/chromeos/chroot/tmp')) - if __name__ == '__main__': unittest.main() |