diff options
-rw-r--r-- | cros_utils/tabulator.py | 2 | ||||
-rw-r--r-- | crosperf/default-telemetry-results.json | 3 | ||||
-rw-r--r-- | crosperf/results_cache.py | 5 | ||||
-rw-r--r-- | crosperf/suite_runner.py | 27 | ||||
-rwxr-xr-x | crosperf/suite_runner_unittest.py | 62 |
5 files changed, 82 insertions, 17 deletions
diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py index 3d5f2aed..2cf4ffd7 100644 --- a/cros_utils/tabulator.py +++ b/cros_utils/tabulator.py @@ -709,7 +709,7 @@ class KeyAwareComparisonResult(ComparisonResult): 'dropped_percent', '(ms)', '(seconds)', '--ms', '--average_num_missing_tiles', '--experimental_jank', '--experimental_mean_frame', '--experimental_median_frame_time', - '--total_deferred_image_decode_count', '--seconds', 'samples' + '--total_deferred_image_decode_count', '--seconds', 'samples', 'bytes' ] return any([l in key for l in lower_is_better_keys]) diff --git a/crosperf/default-telemetry-results.json b/crosperf/default-telemetry-results.json index c4fe0d44..3dd22f86 100644 --- a/crosperf/default-telemetry-results.json +++ b/crosperf/default-telemetry-results.json @@ -169,5 +169,8 @@ "rendering.desktop@@aquarium_20k$": [ "avg_surface_fps", "exp_avg_surface_fps" + ], + "platform.ReportDiskUsage": [ + "bytes_rootfs_prod__summary" ] } diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index a7fd4169..85c3c6e3 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -268,7 +268,10 @@ class Result(object): return [samples, u'samples'] def GetResultsDir(self): - mo = re.search(r'Results placed in (\S+)', self.out) + if self.suite == 'tast': + mo = re.search(r'Writing results to (\S+)', self.out) + else: + mo = re.search(r'Results placed in (\S+)', self.out) if mo: result = mo.group(1) return result diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index 62a85cc6..17e1ad73 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -17,7 +17,7 @@ import time from cros_utils import command_executer TEST_THAT_PATH = '/usr/bin/test_that' -# TODO: Need to check whether Skylab is installed and set up correctly. +TAST_PATH = '/usr/bin/tast' SKYLAB_PATH = '/usr/local/bin/skylab' GS_UTIL = 'src/chromium/depot_tools/gsutil.py' AUTOTEST_DIR = '/mnt/host/source/src/third_party/autotest/files' @@ -78,8 +78,11 @@ class SuiteRunner(object): if label.skylab: ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args) else: - ret_tup = self.Test_That_Run(machine_name, label, benchmark, test_args, - profiler_args) + if benchmark.suite == 'tast': + ret_tup = self.Tast_Run(machine_name, label, benchmark) + else: + ret_tup = self.Test_That_Run(machine_name, label, benchmark, + test_args, profiler_args) if ret_tup[0] != 0: self.logger.LogOutput('benchmark %s failed. Retries left: %s' % (benchmark.name, benchmark.retries - i)) @@ -127,6 +130,24 @@ class SuiteRunner(object): return args_list + # TODO(zhizhouy): Currently do not support passing arguments or running + # customized tast tests, as we do not have such requirements. + def Tast_Run(self, machine, label, benchmark): + # Remove existing tast results + command = 'rm -rf /usr/local/autotest/results/*' + self._ce.CrosRunCommand( + command, machine=machine, chromeos_root=label.chromeos_root) + + command = ' '.join( + [TAST_PATH, 'run', '-build=False', machine, benchmark.test_name]) + + if self.log_level != 'verbose': + self.logger.LogOutput('Running test.') + self.logger.LogOutput('CMD: %s' % command) + + return self._ce.ChrootRunCommandWOutput( + label.chromeos_root, command, command_terminator=self._ct) + def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): """Run the test_that test..""" diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index 73fcb45b..86e1ef19 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -26,8 +26,6 @@ from machine_manager import MockCrosMachine class SuiteRunnerTest(unittest.TestCase): """Class of SuiteRunner test.""" - real_logger = logger.GetLogger() - mock_json = mock.Mock(spec=json) mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator) @@ -55,12 +53,23 @@ class SuiteRunnerTest(unittest.TestCase): '', # perf_args 'crosperf_Wrapper') # suite + tast_bench = Benchmark( + 'b3_test', # name + 'platform.ReportDiskUsage', # test_name + '', # test_args + 1, # iterations + False, # rm_chroot_tmp + '', # perf_args + 'tast') # suite + def __init__(self, *args, **kwargs): super(SuiteRunnerTest, self).__init__(*args, **kwargs) self.skylab_run_args = [] self.test_that_args = [] + self.tast_args = [] self.call_skylab_run = False self.call_test_that_run = False + self.call_tast_run = False def setUp(self): self.runner = suite_runner.SuiteRunner( @@ -90,8 +99,10 @@ class SuiteRunnerTest(unittest.TestCase): def reset(): self.test_that_args = [] self.skylab_run_args = [] + self.tast_args = [] self.call_test_that_run = False self.call_skylab_run = False + self.call_tast_run = False def FakeSkylabRun(test_label, benchmark, test_args, profiler_args): self.skylab_run_args = [test_label, benchmark, test_args, profiler_args] @@ -106,8 +117,14 @@ class SuiteRunnerTest(unittest.TestCase): self.call_test_that_run = True return 'Ran FakeTestThatRun' + def FakeTastRun(machine, test_label, benchmark): + self.tast_args = [machine, test_label, benchmark] + self.call_tast_run = True + return 'Ran FakeTastRun' + self.runner.Skylab_Run = FakeSkylabRun self.runner.Test_That_Run = FakeTestThatRun + self.runner.Tast_Run = FakeTastRun self.runner.dut_config['enable_aslr'] = False self.runner.dut_config['cooldown_time'] = 0 @@ -158,6 +175,15 @@ class SuiteRunnerTest(unittest.TestCase): 'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', '' ]) + # Test tast run for tast benchmarks. + reset() + self.runner.Run(cros_machine, self.mock_label, self.tast_bench, '', '') + self.assertTrue(self.call_tast_run) + self.assertFalse(self.call_test_that_run) + self.assertFalse(self.call_skylab_run) + self.assertEqual(self.tast_args, + ['fake_machine', self.mock_label, self.tast_bench]) + def test_gen_test_args(self): test_args = '--iterations=2' perf_args = 'record -a -e cycles' @@ -180,16 +206,29 @@ class SuiteRunnerTest(unittest.TestCase): @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommandWOutput') - def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd): - - def FakeLogMsg(fd, termfd, msg, flush=True): - if fd or termfd or msg or flush: - pass - - save_log_msg = self.real_logger.LogMsg - self.real_logger.LogMsg = FakeLogMsg - self.runner.logger = self.real_logger + def test_tast_run(self, mock_chroot_runcmd, mock_cros_runcmd): + mock_chroot_runcmd.return_value = 0 + self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd + self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd + res = self.runner.Tast_Run('lumpy1.cros', self.mock_label, self.tast_bench) + self.assertEqual(mock_cros_runcmd.call_count, 1) + self.assertEqual(mock_chroot_runcmd.call_count, 1) + self.assertEqual(res, 0) + self.assertEqual(mock_cros_runcmd.call_args_list[0][0], + ('rm -rf /usr/local/autotest/results/*',)) + args_list = mock_chroot_runcmd.call_args_list[0][0] + args_dict = mock_chroot_runcmd.call_args_list[0][1] + self.assertEqual(len(args_list), 2) + self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) + @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + @mock.patch.object(logger.Logger, 'LogFatal') + def test_test_that_run(self, mock_log_fatal, mock_chroot_runcmd, + mock_cros_runcmd): + mock_log_fatal.side_effect = SystemExit() + self.runner.logger.LogFatal = mock_log_fatal # Test crosperf_Wrapper benchmarks cannot take perf_args raised_exception = False try: @@ -215,7 +254,6 @@ class SuiteRunnerTest(unittest.TestCase): args_dict = mock_chroot_runcmd.call_args_list[0][1] self.assertEqual(len(args_list), 2) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - self.real_logger.LogMsg = save_log_msg @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') @mock.patch.object(json, 'loads') |