diff options
Diffstat (limited to 'crosperf')
24 files changed, 817 insertions, 302 deletions
diff --git a/crosperf/crosperf.py b/crosperf/crosperf.py index ec07e7c7..f195b13a 100755 --- a/crosperf/crosperf.py +++ b/crosperf/crosperf.py @@ -27,6 +27,9 @@ from cros_utils import logger import test_flag +HAS_FAILURE = 1 +ALL_FAILED = 2 + def SetupParserOptions(parser): """Add all options to the parser.""" @@ -128,7 +131,11 @@ def RunCrosperf(argv): runner = ExperimentRunner( experiment, json_report, using_schedv2=(not options.noschedv2)) - runner.Run() + ret = runner.Run() + if ret == HAS_FAILURE: + raise RuntimeError('One or more benchmarks failed.') + if ret == ALL_FAILED: + raise RuntimeError('All benchmarks failed to run.') def Main(argv): diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py index ffd964a2..9c7d52a1 100755 --- a/crosperf/crosperf_unittest.py +++ b/crosperf/crosperf_unittest.py @@ -68,7 +68,7 @@ class CrosperfTest(unittest.TestCase): settings = crosperf.ConvertOptionsToSettings(options) self.assertIsNotNone(settings) self.assertIsInstance(settings, settings_factory.GlobalSettings) - self.assertEqual(len(settings.fields), 38) + self.assertEqual(len(settings.fields), 39) self.assertTrue(settings.GetField('rerun')) argv = ['crosperf/crosperf.py', 'temp.exp'] options, _ = parser.parse_known_args(argv) diff --git a/crosperf/default-telemetry-results.json b/crosperf/default-telemetry-results.json index c4fe0d44..3dd22f86 100644 --- a/crosperf/default-telemetry-results.json +++ b/crosperf/default-telemetry-results.json @@ -169,5 +169,8 @@ "rendering.desktop@@aquarium_20k$": [ "avg_surface_fps", "exp_avg_surface_fps" + ], + "platform.ReportDiskUsage": [ + "bytes_rootfs_prod__summary" ] } diff --git a/crosperf/default_remotes b/crosperf/default_remotes index 7b59c2af..f23fe21b 100644 --- a/crosperf/default_remotes +++ b/crosperf/default_remotes @@ -1,8 +1,9 @@ -elm : chromeos2-row9-rack8-host19.cros chromeos2-row9-rack8-host21.cros -bob : chromeos2-row9-rack7-host1.cros chromeos2-row9-rack7-host3.cros -chell : chromeos2-row9-rack8-host3.cros chromeos2-row9-rack8-host5.cros -kefka : chromeos2-row9-rack9-host21.cros chromeos2-row9-rack8-host1.cros -lulu : chromeos2-row9-rack8-host9.cros chromeos2-row9-rack8-host7.cros -nautilus : chromeos2-row9-rack7-host11.cros chromeos2-row9-rack7-host9.cros -snappy : chromeos2-row9-rack7-host5.cros chromeos2-row9-rack7-host7.cros -veyron_minnie : chromeos2-row9-rack8-host15.cros chromeos2-row9-rack8-host17.cros +bob : chromeos2-row10-rack9-host1.cros chromeos2-row10-rack9-host3.cros +coral : chromeos2-row9-rack9-host9.cros chromeos2-row9-rack9-host11.cros chromeos2-row9-rack9-host13.cros +elm : chromeos2-row10-rack9-host19.cros chromeos2-row10-rack9-host21.cros +chell : chromeos2-row9-rack9-host1.cros chromeos2-row9-rack9-host3.cros +kefka : chromeos2-row9-rack9-host21.cros chromeos2-row10-rack9-host13.cros +lulu : chromeos2-row9-rack9-host5.cros chromeos2-row9-rack9-host7.cros +nautilus : chromeos2-row10-rack9-host9.cros chromeos2-row10-rack9-host11.cros +snappy : chromeos2-row10-rack9-host5.cros chromeos2-row10-rack9-host7.cros +veyron_tiger : chromeos2-row9-rack9-host17.cros chromeos2-row9-rack9-host19.cros diff --git a/crosperf/experiment.py b/crosperf/experiment.py index 45a028ad..6e2efd45 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -28,8 +28,8 @@ class Experiment(object): def __init__(self, name, remote, working_directory, chromeos_root, cache_conditions, labels, benchmarks, experiment_file, email_to, acquire_timeout, log_dir, log_level, share_cache, - results_directory, locks_directory, cwp_dso, ignore_min_max, - skylab, dut_config): + results_directory, compress_results, locks_directory, cwp_dso, + ignore_min_max, skylab, dut_config): self.name = name self.working_directory = working_directory self.remote = remote @@ -42,6 +42,7 @@ class Experiment(object): self.name + '_results') else: self.results_directory = misc.CanonicalizePath(results_directory) + self.compress_results = compress_results self.log_dir = log_dir self.log_level = log_level self.labels = labels diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index 4527db5f..332f0357 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -145,6 +145,7 @@ class ExperimentFactory(object): config.AddConfig('no_email', global_settings.GetField('no_email')) share_cache = global_settings.GetField('share_cache') results_dir = global_settings.GetField('results_dir') + compress_results = global_settings.GetField('compress_results') # Warn user that option use_file_locks is deprecated. use_file_locks = global_settings.GetField('use_file_locks') if use_file_locks: @@ -229,8 +230,8 @@ class ExperimentFactory(object): iterations = benchmark_settings.GetField('iterations') if cwp_dso: - if cwp_dso_iterations != 0 and iterations != cwp_dso_iterations: - raise RuntimeError('Iterations of each benchmark run are not the ' \ + if cwp_dso_iterations not in (0, iterations): + raise RuntimeError('Iterations of each benchmark run are not the ' 'same') cwp_dso_iterations = iterations @@ -288,20 +289,37 @@ class ExperimentFactory(object): perf_args, suite, show_all_results, retries, run_local, cwp_dso, weight) # Add non-telemetry toolchain-perf benchmarks: + + # Tast test platform.ReportDiskUsage for image size. benchmarks.append( Benchmark( - 'graphics_WebGLAquarium', - 'graphics_WebGLAquarium', + 'platform.ReportDiskUsage', + 'platform.ReportDiskUsage', '', - iterations, + 1, # This is not a performance benchmark, only run once. rm_chroot_tmp, - perf_args, - 'crosperf_Wrapper', # Use client wrapper in Autotest + '', + 'tast', # Specify the suite to be 'tast' show_all_results, - retries, - run_local=False, - cwp_dso=cwp_dso, - weight=weight)) + retries)) + + # TODO: crbug.com/1057755 Do not enable graphics_WebGLAquarium until + # it gets fixed. + # + # benchmarks.append( + # Benchmark( + # 'graphics_WebGLAquarium', + # 'graphics_WebGLAquarium', + # '', + # iterations, + # rm_chroot_tmp, + # perf_args, + # 'crosperf_Wrapper', # Use client wrapper in Autotest + # show_all_results, + # retries, + # run_local=False, + # cwp_dso=cwp_dso, + # weight=weight)) elif test_name == 'all_toolchain_perf_old': self.AppendBenchmarkSet( benchmarks, telemetry_toolchain_old_perf_tests, test_args, @@ -421,8 +439,8 @@ class ExperimentFactory(object): chromeos_root, cache_conditions, labels, benchmarks, experiment_file.Canonicalize(), email, acquire_timeout, log_dir, log_level, share_cache, - results_dir, locks_dir, cwp_dso, ignore_min_max, - skylab, dut_config) + results_dir, compress_results, locks_dir, cwp_dso, + ignore_min_max, skylab, dut_config) return experiment diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index f5b17ee2..3528eb1f 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -248,17 +248,19 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] - ef.AppendBenchmarkSet( - bench_list, experiment_factory.telemetry_pagecycler_tests, '', 1, False, - '', 'telemetry_Crosperf', False, 0, False, '', 0) + ef.AppendBenchmarkSet(bench_list, + experiment_factory.telemetry_pagecycler_tests, '', 1, + False, '', 'telemetry_Crosperf', False, 0, False, '', + 0) self.assertEqual( len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] - ef.AppendBenchmarkSet( - bench_list, experiment_factory.telemetry_toolchain_perf_tests, '', 1, - False, '', 'telemetry_Crosperf', False, 0, False, '', 0) + ef.AppendBenchmarkSet(bench_list, + experiment_factory.telemetry_toolchain_perf_tests, '', + 1, False, '', 'telemetry_Crosperf', False, 0, False, + '', 0) self.assertEqual( len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) @@ -398,7 +400,7 @@ class ExperimentFactoryTest(unittest.TestCase): def test_get_default_remotes(self): board_list = [ 'elm', 'bob', 'chell', 'kefka', 'lulu', 'nautilus', 'snappy', - 'veyron_minnie' + 'veyron_tiger' ] ef = ExperimentFactory() diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py index 1d89edad..d2831bda 100644 --- a/crosperf/experiment_file.py +++ b/crosperf/experiment_file.py @@ -95,7 +95,8 @@ class ExperimentFile(object): if not line: continue - elif ExperimentFile._FIELD_VALUE_RE.match(line): + + if ExperimentFile._FIELD_VALUE_RE.match(line): field = self._ParseField(reader) settings.SetField(field[0], field[1], field[2]) elif ExperimentFile._CLOSE_SETTINGS_RE.match(line): @@ -113,7 +114,8 @@ class ExperimentFile(object): if not line: continue - elif ExperimentFile._OPEN_SETTINGS_RE.match(line): + + if ExperimentFile._OPEN_SETTINGS_RE.match(line): new_settings, settings_type = self._ParseSettings(reader) # We will allow benchmarks with duplicated settings name for now. # Further decision will be made when parsing benchmark details in diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index 39e3f863..8ba85a4c 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -35,8 +35,8 @@ def _WriteJSONReportToFile(experiment, results_dir, json_report): compiler_string = 'llvm' if has_llvm else 'gcc' board = experiment.labels[0].board filename = 'report_%s_%s_%s.%s.json' % (board, json_report.date, - json_report.time.replace(':', '.'), - compiler_string) + json_report.time.replace( + ':', '.'), compiler_string) fullname = os.path.join(results_dir, filename) report_text = json_report.GetReport() with open(fullname, 'w') as out_file: @@ -49,6 +49,10 @@ class ExperimentRunner(object): STATUS_TIME_DELAY = 30 THREAD_MONITOR_DELAY = 2 + SUCCEEDED = 0 + HAS_FAILURE = 1 + ALL_FAILED = 2 + def __init__(self, experiment, json_report, @@ -153,13 +157,13 @@ class ExperimentRunner(object): def _ClearCacheEntries(self, experiment): for br in experiment.benchmark_runs: cache = ResultsCache() - cache.Init( - br.label.chromeos_image, br.label.chromeos_root, - br.benchmark.test_name, br.iteration, br.test_args, br.profiler_args, - br.machine_manager, br.machine, br.label.board, br.cache_conditions, - br.logger(), br.log_level, br.label, br.share_cache, - br.benchmark.suite, br.benchmark.show_all_results, - br.benchmark.run_local, br.benchmark.cwp_dso) + cache.Init(br.label.chromeos_image, br.label.chromeos_root, + br.benchmark.test_name, br.iteration, br.test_args, + br.profiler_args, br.machine_manager, br.machine, + br.label.board, br.cache_conditions, br.logger(), br.log_level, + br.label, br.share_cache, br.benchmark.suite, + br.benchmark.show_all_results, br.benchmark.run_local, + br.benchmark.cwp_dso) cache_dir = cache.GetCacheDirForWrite() if os.path.exists(cache_dir): self.l.LogOutput('Removing cache dir: %s' % cache_dir) @@ -169,7 +173,7 @@ class ExperimentRunner(object): try: # We should not lease machines if tests are launched via `skylab # create-test`. This is because leasing DUT in skylab will create a - # dummy task on the DUT and new test created will be hanging there. + # no-op task on the DUT and new test created will be hanging there. # TODO(zhizhouy): Need to check whether machine is ready or not before # assigning a test to it. if not experiment.skylab: @@ -242,8 +246,8 @@ class ExperimentRunner(object): subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names)) text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() - text_report += ( - '\nResults are stored in %s.\n' % experiment.results_directory) + text_report += ('\nResults are stored in %s.\n' % + experiment.results_directory) text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report html_report = HTMLResultsReport.FromExperiment(experiment).GetReport() attachment = EmailSender.Attachment('report.html', html_report) @@ -258,7 +262,8 @@ class ExperimentRunner(object): def _StoreResults(self, experiment): if self._terminated: - return + return self.ALL_FAILED + results_directory = experiment.results_directory FileUtils().RmDir(results_directory) FileUtils().MkDirP(results_directory) @@ -266,6 +271,44 @@ class ExperimentRunner(object): experiment_file_path = os.path.join(results_directory, 'experiment.exp') FileUtils().WriteFile(experiment_file_path, experiment.experiment_file) + has_failure = False + all_failed = True + + topstats_file = os.path.join(results_directory, 'topstats.log') + self.l.LogOutput('Storing top statistics of each benchmark run into %s.' % + topstats_file) + with open(topstats_file, 'w') as top_fd: + for benchmark_run in experiment.benchmark_runs: + if benchmark_run.result: + # FIXME: Pylint has a bug suggesting the following change, which + # should be fixed in pylint 2.0. Resolve this after pylint >= 2.0. + # Bug: https://github.com/PyCQA/pylint/issues/1984 + # pylint: disable=simplifiable-if-statement + if benchmark_run.result.retval: + has_failure = True + else: + all_failed = False + # Header with benchmark run name. + top_fd.write('%s\n' % str(benchmark_run)) + # Formatted string with top statistics. + top_fd.write(benchmark_run.result.FormatStringTopCommands()) + top_fd.write('\n\n') + + if all_failed: + return self.ALL_FAILED + + self.l.LogOutput('Storing results of each benchmark run.') + for benchmark_run in experiment.benchmark_runs: + if benchmark_run.result: + benchmark_run_name = ''.join( + ch for ch in benchmark_run.name if ch.isalnum()) + benchmark_run_path = os.path.join(results_directory, benchmark_run_name) + if experiment.compress_results: + benchmark_run.result.CompressResultsTo(benchmark_run_path) + else: + benchmark_run.result.CopyResultsTo(benchmark_run_path) + benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp) + self.l.LogOutput('Storing results report in %s.' % results_directory) results_table_path = os.path.join(results_directory, 'results.html') report = HTMLResultsReport.FromExperiment(experiment).GetReport() @@ -279,31 +322,12 @@ class ExperimentRunner(object): self.l.LogOutput('Storing email message body in %s.' % results_directory) msg_file_path = os.path.join(results_directory, 'msg_body.html') text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() - text_report += ( - '\nResults are stored in %s.\n' % experiment.results_directory) + text_report += ('\nResults are stored in %s.\n' % + experiment.results_directory) msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report FileUtils().WriteFile(msg_file_path, msg_body) - self.l.LogOutput('Storing results of each benchmark run.') - for benchmark_run in experiment.benchmark_runs: - if benchmark_run.result: - benchmark_run_name = ''.join( - ch for ch in benchmark_run.name if ch.isalnum()) - benchmark_run_path = os.path.join(results_directory, benchmark_run_name) - benchmark_run.result.CopyResultsTo(benchmark_run_path) - benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp) - - topstats_file = os.path.join(results_directory, 'topstats.log') - self.l.LogOutput('Storing top5 statistics of each benchmark run into %s.' % - topstats_file) - with open(topstats_file, 'w') as top_fd: - for benchmark_run in experiment.benchmark_runs: - if benchmark_run.result: - # Header with benchmark run name. - top_fd.write('%s\n' % str(benchmark_run)) - # Formatted string with top statistics. - top_fd.write(benchmark_run.result.FormatStringTop5()) - top_fd.write('\n\n') + return self.SUCCEEDED if not has_failure else self.HAS_FAILURE def Run(self): try: @@ -311,9 +335,10 @@ class ExperimentRunner(object): finally: # Always print the report at the end of the run. self._PrintTable(self._experiment) - if not self._terminated: - self._StoreResults(self._experiment) + ret = self._StoreResults(self._experiment) + if ret != self.ALL_FAILED: self._Email(self._experiment) + return ret class MockExperimentRunner(ExperimentRunner): @@ -323,8 +348,8 @@ class MockExperimentRunner(ExperimentRunner): super(MockExperimentRunner, self).__init__(experiment, json_report) def _Run(self, experiment): - self.l.LogOutput( - "Would run the following experiment: '%s'." % experiment.name) + self.l.LogOutput("Would run the following experiment: '%s'." % + experiment.name) def _PrintTable(self, experiment): self.l.LogOutput('Would print the experiment table.') diff --git a/crosperf/experiment_runner_unittest.py b/crosperf/experiment_runner_unittest.py index 4905975c..31d02e71 100755 --- a/crosperf/experiment_runner_unittest.py +++ b/crosperf/experiment_runner_unittest.py @@ -406,13 +406,14 @@ class ExperimentRunnerTest(unittest.TestCase): @mock.patch.object(FileUtils, 'WriteFile') @mock.patch.object(HTMLResultsReport, 'FromExperiment') @mock.patch.object(TextResultsReport, 'FromExperiment') + @mock.patch.object(Result, 'CompressResultsTo') @mock.patch.object(Result, 'CopyResultsTo') @mock.patch.object(Result, 'CleanUp') - @mock.patch.object(Result, 'FormatStringTop5') + @mock.patch.object(Result, 'FormatStringTopCommands') @mock.patch('builtins.open', new_callable=mock.mock_open) - def test_store_results(self, mock_open, mock_top5, mock_cleanup, mock_copy, - _mock_text_report, mock_report, mock_writefile, - mock_mkdir, mock_rmdir): + def test_store_results(self, mock_open, mock_top_commands, mock_cleanup, + mock_copy, mock_compress, _mock_text_report, + mock_report, mock_writefile, mock_mkdir, mock_rmdir): self.mock_logger.Reset() self.exp.results_directory = '/usr/local/crosperf-results' @@ -433,13 +434,14 @@ class ExperimentRunnerTest(unittest.TestCase): er._StoreResults(self.exp) self.assertEqual(mock_cleanup.call_count, 0) self.assertEqual(mock_copy.call_count, 0) + self.assertEqual(mock_compress.call_count, 0) self.assertEqual(mock_report.call_count, 0) self.assertEqual(mock_writefile.call_count, 0) self.assertEqual(mock_mkdir.call_count, 0) self.assertEqual(mock_rmdir.call_count, 0) self.assertEqual(self.mock_logger.LogOutputCount, 0) self.assertEqual(mock_open.call_count, 0) - self.assertEqual(mock_top5.call_count, 0) + self.assertEqual(mock_top_commands.call_count, 0) # Test 2. _terminated is false; everything works properly. fake_result = Result(self.mock_logger, self.exp.labels[0], 'average', @@ -447,6 +449,7 @@ class ExperimentRunnerTest(unittest.TestCase): for r in self.exp.benchmark_runs: r.result = fake_result er._terminated = False + self.exp.compress_results = False er._StoreResults(self.exp) self.assertEqual(mock_cleanup.call_count, 6) mock_cleanup.assert_called_with(bench_run.benchmark.rm_chroot_tmp) @@ -467,11 +470,11 @@ class ExperimentRunnerTest(unittest.TestCase): self.assertEqual(self.mock_logger.LogOutputCount, 5) self.assertEqual(self.mock_logger.output_msgs, [ 'Storing experiment file in /usr/local/crosperf-results.', + 'Storing top statistics of each benchmark run into' + ' /usr/local/crosperf-results/topstats.log.', + 'Storing results of each benchmark run.', 'Storing results report in /usr/local/crosperf-results.', 'Storing email message body in /usr/local/crosperf-results.', - 'Storing results of each benchmark run.', - 'Storing top5 statistics of each benchmark run into' - ' /usr/local/crosperf-results/topstats.log.', ]) self.assertEqual(mock_open.call_count, 1) # Check write to a topstats.log file. @@ -479,9 +482,19 @@ class ExperimentRunnerTest(unittest.TestCase): 'w') mock_open().write.assert_called() - # Check top5 calls with no arguments. - top5calls = [mock.call()] * 6 - self.assertEqual(mock_top5.call_args_list, top5calls) + # Check top calls with no arguments. + topcalls = [mock.call()] * 6 + self.assertEqual(mock_top_commands.call_args_list, topcalls) + + # Test 3. Test compress_results. + self.exp.compress_results = True + mock_copy.call_count = 0 + mock_compress.call_count = 0 + er._StoreResults(self.exp) + self.assertEqual(mock_copy.call_count, 0) + mock_copy.assert_called_with(bench_path) + self.assertEqual(mock_compress.call_count, 6) + mock_compress.assert_called_with(bench_path) if __name__ == '__main__': diff --git a/crosperf/generate_report_unittest.py b/crosperf/generate_report_unittest.py index e19d4695..8c3510a9 100755 --- a/crosperf/generate_report_unittest.py +++ b/crosperf/generate_report_unittest.py @@ -109,9 +109,9 @@ class GenerateReportTests(unittest.TestCase): } results = generate_report.CutResultsInPlace( bench_data, max_keys=0, complain_on_update=False) - # Just reach into results assuming we know it otherwise outputs things - # sanely. If it doesn't, testCutResultsInPlace should give an indication as - # to what, exactly, is broken. + # Just reach into results assuming we know it otherwise outputs things in + # the expected way. If it doesn't, testCutResultsInPlace should give an + # indication as to what, exactly, is broken. self.assertEqual(list(results['foo'][0][0].items()), [('retval', 0)]) self.assertEqual(list(results['bar'][0][0].items()), [('retval', 1)]) self.assertEqual(list(results['baz'][0][0].items()), []) diff --git a/crosperf/label.py b/crosperf/label.py index b8122613..a55d663c 100644 --- a/crosperf/label.py +++ b/crosperf/label.py @@ -61,9 +61,9 @@ class Label(object): if self.image_type == 'local': chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image) if not chromeos_root: - raise RuntimeError( - "No ChromeOS root given for label '%s' and could " - "not determine one from image path: '%s'." % (name, chromeos_image)) + raise RuntimeError("No ChromeOS root given for label '%s' and could " + "not determine one from image path: '%s'." % + (name, chromeos_image)) else: chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chromeos_root) if not chromeos_root: @@ -72,17 +72,31 @@ class Label(object): self.chromeos_root = chromeos_root if not chrome_src: - self.chrome_src = os.path.join( - self.chromeos_root, '.cache/distfiles/target/chrome-src-internal') - if not os.path.exists(self.chrome_src): - self.chrome_src = os.path.join(self.chromeos_root, - '.cache/distfiles/target/chrome-src') + # Old and new chroots may have different chrome src locations. + # The path also depends on the chrome build flags. + # Give priority to chrome-src-internal. + chrome_src_rel_paths = [ + '.cache/distfiles/target/chrome-src-internal', + '.cache/distfiles/chrome-src-internal', + '.cache/distfiles/target/chrome-src', + '.cache/distfiles/chrome-src', + ] + for chrome_src_rel_path in chrome_src_rel_paths: + chrome_src_abs_path = os.path.join(self.chromeos_root, + chrome_src_rel_path) + if os.path.exists(chrome_src_abs_path): + chrome_src = chrome_src_abs_path + break + if not chrome_src: + raise RuntimeError('Can not find location of Chrome sources.\n' + f'Checked paths: {chrome_src_rel_paths}') else: - chromeos_src = misc.CanonicalizePath(chrome_src) - if not chromeos_src: + chrome_src = misc.CanonicalizePath(chrome_src) + # Make sure the path exists. + if not os.path.exists(chrome_src): raise RuntimeError("Invalid Chrome src given for label '%s': '%s'." % (name, chrome_src)) - self.chrome_src = chromeos_src + self.chrome_src = chrome_src self._SetupChecksum() diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index 7211662c..aaf09bf5 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -28,12 +28,10 @@ CHECKSUM_FILE = '/usr/local/osimage_checksum_file' class BadChecksum(Exception): """Raised if all machines for a label don't have the same checksum.""" - pass class BadChecksumString(Exception): """Raised if all machines for a label don't have the same checksum string.""" - pass class MissingLocksDirectory(Exception): @@ -143,7 +141,12 @@ class CrosMachine(object): def _ComputeMachineChecksumString(self): self.checksum_string = '' - exclude_lines_list = ['MHz', 'BogoMIPS', 'bogomips'] + # Some lines from cpuinfo have to be excluded because they are not + # persistent across DUTs. + # MHz, BogoMIPS are dynamically changing values. + # core id, apicid are identifiers assigned on startup + # and may differ on the same type of machine. + exclude_lines_list = ['MHz', 'BogoMIPS', 'bogomips', 'core id', 'apicid'] for line in self.cpuinfo.splitlines(): if not any(e in line for e in exclude_lines_list): self.checksum_string += line @@ -222,8 +225,8 @@ class MachineManager(object): self.logger = lgr or logger.GetLogger() if self.locks_dir and not os.path.isdir(self.locks_dir): - raise MissingLocksDirectory( - 'Cannot access locks directory: %s' % self.locks_dir) + raise MissingLocksDirectory('Cannot access locks directory: %s' % + self.locks_dir) self._initialized_machines = [] self.chromeos_root = chromeos_root @@ -244,8 +247,8 @@ class MachineManager(object): ret, version, _ = self.ce.CrosRunCommandWOutput( cmd, machine=machine.name, chromeos_root=self.chromeos_root) if ret != 0: - raise CrosCommandError( - "Couldn't get Chrome version from %s." % machine.name) + raise CrosCommandError("Couldn't get Chrome version from %s." % + machine.name) if ret != 0: version = '' @@ -298,8 +301,8 @@ class MachineManager(object): retval = image_chromeos.DoImage(image_chromeos_args) if retval: raise RuntimeError("Could not image machine: '%s'." % machine.name) - else: - self.num_reimages += 1 + + self.num_reimages += 1 machine.checksum = checksum machine.image = label.chromeos_image machine.label = label @@ -314,20 +317,33 @@ class MachineManager(object): # Since this is used for cache lookups before the machines have been # compared/verified, check here to make sure they all have the same # checksum (otherwise the cache lookup may not be valid). - common_checksum = None + base = None for machine in self.GetMachines(label): # Make sure the machine's checksums are calculated. if not machine.machine_checksum: machine.SetUpChecksumInfo() - cs = machine.machine_checksum - # If this is the first machine we've examined, initialize - # common_checksum. - if not common_checksum: - common_checksum = cs + # Use the first machine as the basis for comparison. + if not base: + base = machine # Make sure this machine's checksum matches our 'common' checksum. - if cs != common_checksum: - raise BadChecksum('Machine checksums do not match!') - self.machine_checksum[label.name] = common_checksum + if base.machine_checksum != machine.machine_checksum: + # Found a difference. Fatal error. + # Extract non-matching part and report it. + for mismatch_index in range(len(base.checksum_string)): + if (mismatch_index >= len(machine.checksum_string) or + base.checksum_string[mismatch_index] != + machine.checksum_string[mismatch_index]): + break + # We want to show some context after the mismatch. + end_ind = mismatch_index + 8 + # Print a mismatching string. + raise BadChecksum( + 'Machine checksums do not match!\n' + 'Diff:\n' + f'{base.name}: {base.checksum_string[:end_ind]}\n' + f'{machine.name}: {machine.checksum_string[:end_ind]}\n' + '\nCheck for matching /proc/cpuinfo and /proc/meminfo on DUTs.\n') + self.machine_checksum[label.name] = base.machine_checksum def ComputeCommonCheckSumString(self, label): # The assumption is that this function is only called AFTER @@ -371,8 +387,8 @@ class MachineManager(object): if self.log_level != 'verbose': self.logger.LogOutput('Setting up remote access to %s' % machine_name) - self.logger.LogOutput( - 'Checking machine characteristics for %s' % machine_name) + self.logger.LogOutput('Checking machine characteristics for %s' % + machine_name) cm = CrosMachine(machine_name, self.chromeos_root, self.log_level) if cm.machine_checksum: self._all_machines.append(cm) @@ -412,8 +428,8 @@ class MachineManager(object): if self.acquire_timeout < 0: self.logger.LogFatal('Could not acquire any of the ' - "following machines: '%s'" % ', '.join( - machine.name for machine in machines)) + "following machines: '%s'" % + ', '.join(machine.name for machine in machines)) ### for m in self._machines: @@ -666,8 +682,8 @@ class MockMachineManager(MachineManager): for m in self._all_machines: assert m.name != machine_name, 'Tried to double-add %s' % machine_name cm = MockCrosMachine(machine_name, self.chromeos_root, self.log_level) - assert cm.machine_checksum, ( - 'Could not find checksum for machine %s' % machine_name) + assert cm.machine_checksum, ('Could not find checksum for machine %s' % + machine_name) # In Original MachineManager, the test is 'if cm.machine_checksum:' - if a # machine is unreachable, then its machine_checksum is None. Here we # cannot do this, because machine_checksum is always faked, so we directly diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py index 26eacbd7..f47cc881 100755 --- a/crosperf/machine_manager_unittest.py +++ b/crosperf/machine_manager_unittest.py @@ -44,8 +44,8 @@ class MyMachineManager(machine_manager.MachineManager): assert m.name != machine_name, 'Tried to double-add %s' % machine_name cm = machine_manager.MockCrosMachine(machine_name, self.chromeos_root, 'average') - assert cm.machine_checksum, ( - 'Could not find checksum for machine %s' % machine_name) + assert cm.machine_checksum, ('Could not find checksum for machine %s' % + machine_name) self._all_machines.append(cm) @@ -87,9 +87,10 @@ class MachineManagerTest(unittest.TestCase): def setUp(self, mock_isdir): mock_isdir.return_value = True - self.mm = machine_manager.MachineManager( - '/usr/local/chromeos', 0, 'average', None, self.mock_cmd_exec, - self.mock_logger) + self.mm = machine_manager.MachineManager('/usr/local/chromeos', 0, + 'average', None, + self.mock_cmd_exec, + self.mock_logger) self.mock_lumpy1.name = 'lumpy1' self.mock_lumpy2.name = 'lumpy2' @@ -225,15 +226,14 @@ class MachineManagerTest(unittest.TestCase): self.assertEqual(mock_sleep.call_count, 0) def test_compute_common_checksum(self): - self.mm.machine_checksum = {} self.mm.ComputeCommonCheckSum(LABEL_LUMPY) self.assertEqual(self.mm.machine_checksum['lumpy'], 'lumpy123') self.assertEqual(len(self.mm.machine_checksum), 1) self.mm.machine_checksum = {} - self.assertRaises(machine_manager.BadChecksum, - self.mm.ComputeCommonCheckSum, LABEL_MIX) + self.assertRaisesRegex(machine_manager.BadChecksum, r'daisy.*\n.*lumpy', + self.mm.ComputeCommonCheckSum, LABEL_MIX) def test_compute_common_checksum_string(self): self.mm.machine_checksum_string = {} @@ -583,8 +583,8 @@ power management: CHECKSUM_STRING = ('processor: 0vendor_id: GenuineIntelcpu family: 6model: ' '42model name: Intel(R) Celeron(R) CPU 867 @ ' '1.30GHzstepping: 7microcode: 0x25cache size: 2048 ' - 'KBphysical id: 0siblings: 2core id: 0cpu cores: 2apicid: ' - '0initial apicid: 0fpu: yesfpu_exception: yescpuid level: ' + 'KBphysical id: 0siblings: 2cpu cores: 2' + 'fpu: yesfpu_exception: yescpuid level: ' '13wp: yesflags: fpu vme de pse tsc msr pae mce cx8 apic sep' ' mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse ' 'sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc ' @@ -597,8 +597,8 @@ CHECKSUM_STRING = ('processor: 0vendor_id: GenuineIntelcpu family: 6model: ' 'bits virtualpower management:processor: 1vendor_id: ' 'GenuineIntelcpu family: 6model: 42model name: Intel(R) ' 'Celeron(R) CPU 867 @ 1.30GHzstepping: 7microcode: 0x25cache' - ' size: 2048 KBphysical id: 0siblings: 2core id: 1cpu cores:' - ' 2apicid: 2initial apicid: 2fpu: yesfpu_exception: yescpuid' + ' size: 2048 KBphysical id: 0siblings: 2cpu cores:' + ' 2fpu: yesfpu_exception: yescpuid' ' level: 13wp: yesflags: fpu vme de pse tsc msr pae mce cx8 ' 'apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx ' 'fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm ' diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index 98062194..c5c85942 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -30,10 +30,19 @@ SCRATCH_DIR = os.path.expanduser('~/cros_scratch') RESULTS_FILE = 'results.txt' MACHINE_FILE = 'machine.txt' AUTOTEST_TARBALL = 'autotest.tbz2' +RESULTS_TARBALL = 'results.tbz2' PERF_RESULTS_FILE = 'perf-results.txt' CACHE_KEYS_FILE = 'cache_keys.txt' +class PidVerificationError(Exception): + """Error of perf PID verification in per-process mode.""" + + +class PerfDataReadError(Exception): + """Error of reading a perf.data header.""" + + class Result(object): """Class for holding the results of a single test run. @@ -57,6 +66,7 @@ class Result(object): self.results_file = [] self.turbostat_log_file = '' self.cpustats_log_file = '' + self.cpuinfo_file = '' self.top_log_file = '' self.wait_time_log_file = '' self.chrome_version = '' @@ -75,22 +85,34 @@ class Result(object): """Get the list of top commands consuming CPU on the machine.""" return self.top_cmds - def FormatStringTop5(self): - """Get formatted top5 string. + def FormatStringTopCommands(self): + """Get formatted string of top commands. - Get the formatted string with top5 commands consuming CPU on DUT machine. + Get the formatted string with top commands consuming CPU on DUT machine. + Number of "non-chrome" processes in the list is limited to 5. """ format_list = [ - 'Top 5 commands with highest CPU usage:', + 'Top commands with highest CPU usage:', # Header. '%20s %9s %6s %s' % ('COMMAND', 'AVG CPU%', 'COUNT', 'HIGHEST 5'), '-' * 50, ] if self.top_cmds: - for topcmd in self.top_cmds[:5]: - print_line = '%20s %9.2f %6s %s' % (topcmd['cmd'], topcmd['cpu_avg'], - topcmd['count'], topcmd['top5']) + # After switching to top processes we have to expand the list since there + # will be a lot of 'chrome' processes (up to 10, sometimes more) in the + # top. + # Let's limit the list size by the number of non-chrome processes. + limit_of_non_chrome_procs = 5 + num_of_non_chrome_procs = 0 + for topcmd in self.top_cmds: + print_line = '%20s %9.2f %6s %s' % ( + topcmd['cmd'], topcmd['cpu_use_avg'], topcmd['count'], + topcmd['top5_cpu_use']) format_list.append(print_line) + if not topcmd['cmd'].startswith('chrome'): + num_of_non_chrome_procs += 1 + if num_of_non_chrome_procs >= limit_of_non_chrome_procs: + break else: format_list.append('[NO DATA FROM THE TOP LOG]') format_list.append('-' * 50) @@ -109,10 +131,37 @@ class Result(object): raise IOError('Could not copy results file: %s' % file_to_copy) def CopyResultsTo(self, dest_dir): + self.CopyFilesTo(dest_dir, self.results_file) self.CopyFilesTo(dest_dir, self.perf_data_files) self.CopyFilesTo(dest_dir, self.perf_report_files) - if self.perf_data_files or self.perf_report_files: - self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) + extra_files = [] + if self.top_log_file: + extra_files.append(self.top_log_file) + if self.cpuinfo_file: + extra_files.append(self.cpuinfo_file) + if extra_files: + self.CopyFilesTo(dest_dir, extra_files) + if self.results_file or self.perf_data_files or self.perf_report_files: + self._logger.LogOutput('Results files stored in %s.' % dest_dir) + + def CompressResultsTo(self, dest_dir): + tarball = os.path.join(self.results_dir, RESULTS_TARBALL) + # Test_that runs hold all output under TEST_NAME_HASHTAG/results/, + # while tast runs hold output under TEST_NAME/. + # Both ensure to be unique. + result_dir_name = self.test_name if self.suite == 'tast' else 'results' + results_dir = self.FindFilesInResultsDir('-name %s' % + result_dir_name).split('\n')[0] + + if not results_dir: + self._logger.LogOutput('WARNING: No results dir matching %r found' % + result_dir_name) + return + + self.CreateTarball(results_dir, tarball) + self.CopyFilesTo(dest_dir, [tarball]) + if results_dir: + self._logger.LogOutput('Results files compressed into %s.' % dest_dir) def GetNewKeyvals(self, keyvals_dict): # Initialize 'units' dictionary. @@ -198,8 +247,8 @@ class Result(object): command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) self.ce.RunCommand(command, print_to_console=False) - command = ('./generate_test_report --no-color --csv %s' % (os.path.join( - '/tmp', os.path.basename(self.temp_dir)))) + command = ('./generate_test_report --no-color --csv %s' % + (os.path.join('/tmp', os.path.basename(self.temp_dir)))) _, out, _ = self.ce.ChrootRunCommandWOutput( self.chromeos_root, command, print_to_console=False) keyvals_dict = {} @@ -267,7 +316,10 @@ class Result(object): return [samples, u'samples'] def GetResultsDir(self): - mo = re.search(r'Results placed in (\S+)', self.out) + if self.suite == 'tast': + mo = re.search(r'Writing results to (\S+)', self.out) + else: + mo = re.search(r'Results placed in (\S+)', self.out) if mo: result = mo.group(1) return result @@ -313,6 +365,10 @@ class Result(object): """Get cpustats log path string.""" return self.FindFilesInResultsDir('-name cpustats.log').split('\n')[0] + def GetCpuinfoFile(self): + """Get cpustats log path string.""" + return self.FindFilesInResultsDir('-name cpuinfo.log').split('\n')[0] + def GetTopFile(self): """Get cpustats log path string.""" return self.FindFilesInResultsDir('-name top.log').split('\n')[0] @@ -342,8 +398,8 @@ class Result(object): perf_data_file) perf_report_file = '%s.report' % perf_data_file if os.path.exists(perf_report_file): - raise RuntimeError( - 'Perf report file already exists: %s' % perf_report_file) + raise RuntimeError('Perf report file already exists: %s' % + perf_report_file) chroot_perf_report_file = misc.GetInsideChrootPath( self.chromeos_root, perf_report_file) perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') @@ -381,8 +437,8 @@ class Result(object): if self.log_level != 'verbose': self._logger.LogOutput('Perf report generated successfully.') else: - raise RuntimeError( - 'Perf report not generated correctly. CMD: %s' % command) + raise RuntimeError('Perf report not generated correctly. CMD: %s' % + command) # Add a keyval to the dictionary for the events captured. perf_report_files.append( @@ -419,6 +475,7 @@ class Result(object): self.perf_report_files = self.GeneratePerfReportFiles() self.turbostat_log_file = self.GetTurbostatFile() self.cpustats_log_file = self.GetCpustatsFile() + self.cpuinfo_file = self.GetCpuinfoFile() self.top_log_file = self.GetTopFile() self.wait_time_log_file = self.GetWaitTimeFile() # TODO(asharif): Do something similar with perf stat. @@ -535,9 +592,9 @@ class Result(object): Returns: List of dictionaries with the following keyvals: 'cmd': command name (string), - 'cpu_avg': average cpu usage (float), + 'cpu_use_avg': average cpu usage (float), 'count': number of occurrences (int), - 'top5': up to 5 highest cpu usages (descending list of floats) + 'top5_cpu_use': up to 5 highest cpu usages (descending list of floats) Example of the top log: PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND @@ -588,7 +645,7 @@ class Result(object): process = { # NOTE: One command may be represented by multiple processes. 'cmd': match.group('cmd'), - 'pid': int(match.group('pid')), + 'pid': match.group('pid'), 'cpu_use': float(match.group('cpu_use')), } @@ -610,7 +667,10 @@ class Result(object): # not running. # On 1-core DUT 90% chrome cpu load occurs in 55%, 95% in 33% and 100% in 2% # of snapshots accordingly. - CHROME_HIGH_CPU_LOAD = 90 + # Threshold of "high load" is reduced to 70% (from 90) when we switched to + # topstats per process. From experiment data the rest 20% are distributed + # among other chrome processes. + CHROME_HIGH_CPU_LOAD = 70 # Number of snapshots where chrome is heavily used. high_load_snapshots = 0 # Total CPU use per process in ALL active snapshots. @@ -621,35 +681,41 @@ class Result(object): topcmds = [] for snapshot_processes in snapshots: - # CPU usage per command in one snapshot. - cmd_cpu_use_per_snapshot = collections.defaultdict(float) + # CPU usage per command, per PID in one snapshot. + cmd_cpu_use_per_snapshot = collections.defaultdict(dict) for process in snapshot_processes: cmd = process['cmd'] cpu_use = process['cpu_use'] + pid = process['pid'] + cmd_cpu_use_per_snapshot[cmd][pid] = cpu_use - # Collect CPU usage per command. - cmd_cpu_use_per_snapshot[cmd] += cpu_use + # Chrome processes, pid: cpu_usage. + chrome_processes = cmd_cpu_use_per_snapshot.get('chrome', {}) + chrome_cpu_use_list = chrome_processes.values() - if cmd_cpu_use_per_snapshot.setdefault('chrome', - 0.0) > CHROME_HIGH_CPU_LOAD: - # Combined CPU usage of "chrome" command exceeds "High load" threshold - # which means DUT is busy running a benchmark. + if chrome_cpu_use_list and max( + chrome_cpu_use_list) > CHROME_HIGH_CPU_LOAD: + # CPU usage of any of the "chrome" processes exceeds "High load" + # threshold which means DUT is busy running a benchmark. high_load_snapshots += 1 - for cmd, cpu_use in cmd_cpu_use_per_snapshot.items(): - # Update total CPU usage. - cmd_total_cpu_use[cmd] += cpu_use + for cmd, cpu_use_per_pid in cmd_cpu_use_per_snapshot.items(): + for pid, cpu_use in cpu_use_per_pid.items(): + # Append PID to the name of the command. + cmd_with_pid = cmd + '-' + pid + cmd_total_cpu_use[cmd_with_pid] += cpu_use - # Add cpu_use into command top cpu usages, sorted in descending order. - heapq.heappush(cmd_top5_cpu_use[cmd], round(cpu_use, 1)) + # Add cpu_use into command top cpu usages, sorted in descending + # order. + heapq.heappush(cmd_top5_cpu_use[cmd_with_pid], round(cpu_use, 1)) for consumer, usage in sorted( cmd_total_cpu_use.items(), key=lambda x: x[1], reverse=True): # Iterate through commands by descending order of total CPU usage. topcmd = { 'cmd': consumer, - 'cpu_avg': usage / high_load_snapshots, + 'cpu_use_avg': usage / high_load_snapshots, 'count': len(cmd_top5_cpu_use[consumer]), - 'top5': heapq.nlargest(5, cmd_top5_cpu_use[consumer]), + 'top5_cpu_use': heapq.nlargest(5, cmd_top5_cpu_use[consumer]), } topcmds.append(topcmd) @@ -786,6 +852,88 @@ class Result(object): keyvals[key] = [result, unit] return keyvals + def ReadPidFromPerfData(self): + """Read PIDs from perf.data files. + + Extract PID from perf.data if "perf record" was running per process, + i.e. with "-p <PID>" and no "-a". + + Returns: + pids: list of PIDs. + + Raises: + PerfDataReadError when perf.data header reading fails. + """ + cmd = ['/usr/bin/perf', 'report', '--header-only', '-i'] + pids = [] + + for perf_data_path in self.perf_data_files: + perf_data_path_in_chroot = misc.GetInsideChrootPath( + self.chromeos_root, perf_data_path) + path_str = ' '.join(cmd + [perf_data_path_in_chroot]) + status, output, _ = self.ce.ChrootRunCommandWOutput( + self.chromeos_root, path_str) + if status: + # Error of reading a perf.data profile is fatal. + raise PerfDataReadError(f'Failed to read perf.data profile: {path_str}') + + # Pattern to search a line with "perf record" command line: + # # cmdline : /usr/bin/perf record -e instructions -p 123" + cmdline_regex = re.compile( + r'^\#\scmdline\s:\s+(?P<cmd>.*perf\s+record\s+.*)$') + # Pattern to search PID in a command line. + pid_regex = re.compile(r'^.*\s-p\s(?P<pid>\d+)\s*.*$') + for line in output.splitlines(): + cmd_match = cmdline_regex.match(line) + if cmd_match: + # Found a perf command line. + cmdline = cmd_match.group('cmd') + # '-a' is a system-wide mode argument. + if '-a' not in cmdline.split(): + # It can be that perf was attached to PID and was still running in + # system-wide mode. + # We filter out this case here since it's not per-process. + pid_match = pid_regex.match(cmdline) + if pid_match: + pids.append(pid_match.group('pid')) + # Stop the search and move to the next perf.data file. + break + else: + # cmdline wasn't found in the header. It's a fatal error. + raise PerfDataReadError(f'Perf command line is not found in {path_str}') + return pids + + def VerifyPerfDataPID(self): + """Verify PIDs in per-process perf.data profiles. + + Check that at list one top process is profiled if perf was running in + per-process mode. + + Raises: + PidVerificationError if PID verification of per-process perf.data profiles + fail. + """ + perf_data_pids = self.ReadPidFromPerfData() + if not perf_data_pids: + # In system-wide mode there are no PIDs. + self._logger.LogOutput('System-wide perf mode. Skip verification.') + return + + # PIDs will be present only in per-process profiles. + # In this case we need to verify that profiles are collected on the + # hottest processes. + top_processes = [top_cmd['cmd'] for top_cmd in self.top_cmds] + # top_process structure: <cmd>-<pid> + top_pids = [top_process.split('-')[-1] for top_process in top_processes] + for top_pid in top_pids: + if top_pid in perf_data_pids: + self._logger.LogOutput('PID verification passed! ' + f'Top process {top_pid} is profiled.') + return + raise PidVerificationError( + f'top processes {top_processes} are missing in perf.data traces with' + f' PID: {perf_data_pids}.') + def ProcessResults(self, use_cache=False): # Note that this function doesn't know anything about whether there is a # cache hit or miss. It should process results agnostic of the cache hit @@ -824,6 +972,9 @@ class Result(object): cpustats = self.ProcessCpustatsResults() if self.top_log_file: self.top_cmds = self.ProcessTopResults() + # Verify that PID in non system-wide perf.data and top_cmds are matching. + if self.perf_data_files and self.top_cmds: + self.VerifyPerfDataPID() if self.wait_time_log_file: with open(self.wait_time_log_file) as f: wait_time = f.readline().strip() @@ -902,6 +1053,19 @@ class Result(object): command = 'rm -rf %s' % self.temp_dir self.ce.RunCommand(command) + def CreateTarball(self, results_dir, tarball): + if not results_dir.strip(): + raise ValueError('Refusing to `tar` an empty results_dir: %r' % + results_dir) + + ret = self.ce.RunCommand('cd %s && ' + 'tar ' + '--exclude=var/spool ' + '--exclude=var/log ' + '-cjf %s .' % (results_dir, tarball)) + if ret: + raise RuntimeError("Couldn't compress test output directory.") + def StoreToCacheDir(self, cache_dir, machine_manager, key_list): # Create the dir if it doesn't exist. temp_dir = tempfile.mkdtemp() @@ -923,14 +1087,8 @@ class Result(object): if self.results_dir: tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) - command = ('cd %s && ' - 'tar ' - '--exclude=var/spool ' - '--exclude=var/log ' - '-cjf %s .' % (self.results_dir, tarball)) - ret = self.ce.RunCommand(command) - if ret: - raise RuntimeError("Couldn't store autotest output directory.") + self.CreateTarball(self.results_dir, tarball) + # Store machine info. # TODO(asharif): Make machine_manager a singleton, and don't pass it into # this function. @@ -948,8 +1106,8 @@ class Result(object): if ret: command = 'rm -rf {0}'.format(temp_dir) self.ce.RunCommand(command) - raise RuntimeError( - 'Could not move dir %s to dir %s' % (temp_dir, cache_dir)) + raise RuntimeError('Could not move dir %s to dir %s' % + (temp_dir, cache_dir)) @classmethod def CreateFromRun(cls, diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index 1e7f04a1..91ceed22 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -21,6 +21,8 @@ import test_flag from label import MockLabel from results_cache import CacheConditions +from results_cache import PerfDataReadError +from results_cache import PidVerificationError from results_cache import Result from results_cache import ResultsCache from results_cache import TelemetryResult @@ -158,6 +160,34 @@ keyvals = { 'b_string_strstr___abcdefghijklmnopqrstuvwxyz__': '0.0134553343333' } +PERF_DATA_HEADER = """ +# ======== +# captured on : Thu Jan 01 00:00:00 1980 +# header version : 1 +# data offset : 536 +# data size : 737678672 +# feat offset : 737679208 +# hostname : localhost +# os release : 5.4.61 +# perf version : +# arch : aarch64 +# nrcpus online : 8 +# nrcpus avail : 8 +# total memory : 5911496 kB +# cmdline : /usr/bin/perf record -e instructions -p {pid} +# event : name = instructions, , id = ( 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193 ), type = 8, size = 112 +# event : name = dummy:u, , id = ( 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204 ), type = 1, size = 112, config = 0x9 +# CPU_TOPOLOGY info available, use -I to display +# pmu mappings: software = 1, uprobe = 6, cs_etm = 8, breakpoint = 5, tracepoint = 2, armv8_pmuv3 = 7 +# contains AUX area data (e.g. instruction trace) +# time of first sample : 0.000000 +# time of last sample : 0.000000 +# sample duration : 0.000 ms +# missing features: TRACING_DATA CPUDESC CPUID NUMA_TOPOLOGY BRANCH_STACK GROUP_DESC STAT CACHE MEM_TOPOLOGY CLOCKID DIR_FORMAT +# ======== +# +""" + TURBOSTAT_LOG_OUTPUT = \ """CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ CoreTmp - 329 12.13 2723 2393 10975 77 @@ -223,40 +253,52 @@ TOP_LOG = \ """ TOP_DATA = [ { - 'cmd': 'chrome', - 'cpu_avg': 124.75, + 'cmd': 'chrome-5745', + 'cpu_use_avg': 115.35, 'count': 2, - 'top5': [125.7, 123.8], + 'top5_cpu_use': [122.8, 107.9], }, { - 'cmd': 'irq/cros-ec', - 'cpu_avg': 1.0, + 'cmd': 'chrome-5713', + 'cpu_use_avg': 8.9, 'count': 1, - 'top5': [2.0], + 'top5_cpu_use': [17.8] }, { - 'cmd': 'spi5', - 'cpu_avg': 0.5, + 'cmd': 'irq/cros-ec-912', + 'cpu_use_avg': 1.0, 'count': 1, - 'top5': [1.0], + 'top5_cpu_use': [2.0], }, { - 'cmd': 'sshd', - 'cpu_avg': 0.5, + 'cmd': 'chrome-5205', + 'cpu_use_avg': 0.5, 'count': 1, - 'top5': [1.0], + 'top5_cpu_use': [1.0] }, { - 'cmd': 'rcu_preempt', - 'cpu_avg': 0.5, + 'cmd': 'spi5-121', + 'cpu_use_avg': 0.5, 'count': 1, - 'top5': [1.0], + 'top5_cpu_use': [1.0], }, { - 'cmd': 'kworker/4:2', - 'cpu_avg': 0.5, + 'cmd': 'sshd-4811', + 'cpu_use_avg': 0.5, 'count': 1, - 'top5': [1.0], + 'top5_cpu_use': [1.0], + }, + { + 'cmd': 'rcu_preempt-7', + 'cpu_use_avg': 0.5, + 'count': 1, + 'top5_cpu_use': [1.0], + }, + { + 'cmd': 'kworker/4:2-855', + 'cpu_use_avg': 0.5, + 'count': 1, + 'top5_cpu_use': [1.0], }, ] TOP_OUTPUT = \ @@ -433,6 +475,7 @@ class ResultTest(unittest.TestCase): self.callGetTurbostatFile = False self.callGetCpustatsFile = False self.callGetTopFile = False + self.callGetCpuinfoFile = False self.callGetWaitTimeFile = False self.args = None self.callGatherPerfResults = False @@ -457,6 +500,7 @@ class ResultTest(unittest.TestCase): def setUp(self): self.result = Result(self.mock_logger, self.mock_label, 'average', self.mock_cmd_exec) + self.result.chromeos_root = '/tmp/chromeos' @mock.patch.object(os.path, 'isdir') @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') @@ -501,6 +545,9 @@ class ResultTest(unittest.TestCase): @mock.patch.object(Result, 'CopyFilesTo') def test_copy_results_to(self, mockCopyFilesTo): + results_file = [ + '/tmp/result.json.0', '/tmp/result.json.1', '/tmp/result.json.2' + ] perf_data_files = [ '/tmp/perf.data.0', '/tmp/perf.data.1', '/tmp/perf.data.2' ] @@ -508,16 +555,19 @@ class ResultTest(unittest.TestCase): '/tmp/perf.report.0', '/tmp/perf.report.1', '/tmp/perf.report.2' ] + self.result.results_file = results_file self.result.perf_data_files = perf_data_files self.result.perf_report_files = perf_report_files self.result.CopyFilesTo = mockCopyFilesTo self.result.CopyResultsTo('/tmp/results/') - self.assertEqual(mockCopyFilesTo.call_count, 2) - self.assertEqual(len(mockCopyFilesTo.call_args_list), 2) + self.assertEqual(mockCopyFilesTo.call_count, 3) + self.assertEqual(len(mockCopyFilesTo.call_args_list), 3) self.assertEqual(mockCopyFilesTo.call_args_list[0][0], - ('/tmp/results/', perf_data_files)) + ('/tmp/results/', results_file)) self.assertEqual(mockCopyFilesTo.call_args_list[1][0], + ('/tmp/results/', perf_data_files)) + self.assertEqual(mockCopyFilesTo.call_args_list[2][0], ('/tmp/results/', perf_report_files)) def test_get_new_keyvals(self): @@ -688,7 +738,8 @@ class ResultTest(unittest.TestCase): self.assertEqual(mock_chrootruncmd.call_count, 1) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], - ('/tmp', ('./generate_test_report --no-color --csv %s') % TMP_DIR1)) + (self.result.chromeos_root, + ('./generate_test_report --no-color --csv %s') % TMP_DIR1)) self.assertEqual(mock_getpath.call_count, 1) self.assertEqual(mock_mkdtemp.call_count, 1) self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) @@ -861,6 +912,15 @@ class ResultTest(unittest.TestCase): self.assertEqual(found_no_logs, '') @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') + def test_get_cpuinfo_file_finds_single_log(self, mock_runcmd): + """Expected behavior when a single cpuinfo file found.""" + self.result.results_dir = '/tmp/test_results' + self.result.ce.RunCommandWOutput = mock_runcmd + mock_runcmd.return_value = (0, 'some/long/path/cpuinfo.log', '') + found_single_log = self.result.GetCpuinfoFile() + self.assertEqual(found_single_log, 'some/long/path/cpuinfo.log') + + @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') def test_get_cpustats_file_finds_single_log(self, mock_runcmd): """Expected behavior when a single log file found.""" self.result.results_dir = '/tmp/test_results' @@ -888,6 +948,101 @@ class ResultTest(unittest.TestCase): found_no_logs = self.result.GetCpustatsFile() self.assertEqual(found_no_logs, '') + def test_verify_perf_data_pid_ok(self): + """Verify perf PID which is present in TOP_DATA.""" + self.result.top_cmds = TOP_DATA + # pid is present in TOP_DATA. + with mock.patch.object( + Result, 'ReadPidFromPerfData', return_value=['5713']): + self.result.VerifyPerfDataPID() + + def test_verify_perf_data_pid_fail(self): + """Test perf PID missing in top raises the error.""" + self.result.top_cmds = TOP_DATA + # pid is not in the list of top processes. + with mock.patch.object( + Result, 'ReadPidFromPerfData', return_value=['9999']): + with self.assertRaises(PidVerificationError): + self.result.VerifyPerfDataPID() + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_ok(self, mock_runcmd): + """Test perf header parser, normal flow.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + exp_pid = '12345' + mock_runcmd.return_value = (0, PERF_DATA_HEADER.format(pid=exp_pid), '') + pids = self.result.ReadPidFromPerfData() + self.assertEqual(pids, [exp_pid]) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_mult_profiles(self, mock_runcmd): + """Test multiple perf.data files with PID.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + # self.result.chromeos_root = '/tmp/chromeos' + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data.0', + '/tmp/chromeos/chroot/tmp/results/perf.data.1', + ] + # There is '-p <pid>' in command line but it's still system-wide: '-a'. + cmd_line = '# cmdline : /usr/bin/perf record -e instructions -p {pid}' + exp_perf_pids = ['1111', '2222'] + mock_runcmd.side_effect = [ + (0, cmd_line.format(pid=exp_perf_pids[0]), ''), + (0, cmd_line.format(pid=exp_perf_pids[1]), ''), + ] + pids = self.result.ReadPidFromPerfData() + self.assertEqual(pids, exp_perf_pids) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_no_pid(self, mock_runcmd): + """Test perf.data without PID.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + cmd_line = '# cmdline : /usr/bin/perf record -e instructions' + mock_runcmd.return_value = (0, cmd_line, '') + pids = self.result.ReadPidFromPerfData() + # pids is empty. + self.assertEqual(pids, []) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_system_wide(self, mock_runcmd): + """Test reading from system-wide profile with PID.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + # There is '-p <pid>' in command line but it's still system-wide: '-a'. + cmd_line = '# cmdline : /usr/bin/perf record -e instructions -a -p 1234' + mock_runcmd.return_value = (0, cmd_line, '') + pids = self.result.ReadPidFromPerfData() + # pids should be empty since it's not a per-process profiling. + self.assertEqual(pids, []) + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_read_fail(self, mock_runcmd): + """Failure to read perf.data raises the error.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + # Error status of the profile read. + mock_runcmd.return_value = (1, '', '') + with self.assertRaises(PerfDataReadError): + self.result.ReadPidFromPerfData() + + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + def test_read_pid_from_perf_data_fail(self, mock_runcmd): + """Failure to find cmdline in perf.data header raises the error.""" + self.result.ce.ChrootRunCommandWOutput = mock_runcmd + self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + # Empty output. + mock_runcmd.return_value = (0, '', '') + with self.assertRaises(PerfDataReadError): + self.result.ReadPidFromPerfData() + def test_process_turbostat_results_with_valid_data(self): """Normal case when log exists and contains valid data.""" self.result.turbostat_log_file = '/tmp/somelogfile.log' @@ -988,69 +1143,84 @@ class ResultTest(unittest.TestCase): mo.assert_has_calls(calls) self.assertEqual(topcalls, []) - def test_format_string_top5_cmds(self): - """Test formatted string with top5 commands.""" + def test_format_string_top_cmds(self): + """Test formatted string with top commands.""" self.result.top_cmds = [ { - 'cmd': 'chrome', - 'cpu_avg': 119.753453465, + 'cmd': 'chrome-111', + 'cpu_use_avg': 119.753453465, 'count': 44444, - 'top5': [222.8, 217.9, 217.8, 191.0, 189.9], + 'top5_cpu_use': [222.8, 217.9, 217.8, 191.0, 189.9], + }, + { + 'cmd': 'chrome-222', + 'cpu_use_avg': 100, + 'count': 33333, + 'top5_cpu_use': [200.0, 195.0, 190.0, 185.0, 180.0], }, { 'cmd': 'irq/230-cros-ec', - 'cpu_avg': 10.000000000000001, + 'cpu_use_avg': 10.000000000000001, 'count': 1000, - 'top5': [11.5, 11.4, 11.3, 11.2, 11.1], + 'top5_cpu_use': [11.5, 11.4, 11.3, 11.2, 11.1], }, { 'cmd': 'powerd', - 'cpu_avg': 2.0, + 'cpu_use_avg': 2.0, 'count': 2, - 'top5': [3.0, 1.0] + 'top5_cpu_use': [3.0, 1.0] }, { - 'cmd': 'cmd1', - 'cpu_avg': 1.0, + 'cmd': 'cmd3', + 'cpu_use_avg': 1.0, 'count': 1, - 'top5': [1.0], + 'top5_cpu_use': [1.0], }, { - 'cmd': 'cmd2', - 'cpu_avg': 1.0, + 'cmd': 'cmd4', + 'cpu_use_avg': 1.0, 'count': 1, - 'top5': [1.0], + 'top5_cpu_use': [1.0], }, { - 'cmd': 'not_for_print', + 'cmd': 'cmd5', + 'cpu_use_avg': 1.0, + 'count': 1, + 'top5_cpu_use': [1.0], + }, + { + 'cmd': 'cmd6_not_for_print', 'cpu_avg': 1.0, 'count': 1, 'top5': [1.0], }, ] - form_str = self.result.FormatStringTop5() + form_str = self.result.FormatStringTopCommands() self.assertEqual( form_str, '\n'.join([ - 'Top 5 commands with highest CPU usage:', + 'Top commands with highest CPU usage:', ' COMMAND AVG CPU% COUNT HIGHEST 5', '-' * 50, - ' chrome 119.75 44444 ' + ' chrome-111 119.75 44444 ' '[222.8, 217.9, 217.8, 191.0, 189.9]', + ' chrome-222 100.00 33333 ' + '[200.0, 195.0, 190.0, 185.0, 180.0]', ' irq/230-cros-ec 10.00 1000 ' '[11.5, 11.4, 11.3, 11.2, 11.1]', ' powerd 2.00 2 [3.0, 1.0]', - ' cmd1 1.00 1 [1.0]', - ' cmd2 1.00 1 [1.0]', + ' cmd3 1.00 1 [1.0]', + ' cmd4 1.00 1 [1.0]', + ' cmd5 1.00 1 [1.0]', '-' * 50, ])) - def test_format_string_top5_calls_no_data(self): - """Test formatted string of top5 with no data.""" + def test_format_string_top_calls_no_data(self): + """Test formatted string of top with no data.""" self.result.top_cmds = [] - form_str = self.result.FormatStringTop5() + form_str = self.result.FormatStringTopCommands() self.assertEqual( form_str, '\n'.join([ - 'Top 5 commands with highest CPU usage:', + 'Top commands with highest CPU usage:', ' COMMAND AVG CPU% COUNT HIGHEST 5', '-' * 50, '[NO DATA FROM THE TOP LOG]', @@ -1069,10 +1239,11 @@ class ResultTest(unittest.TestCase): # Debug path not found self.result.label.debug_path = '' tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file]) + self.assertEqual(tmp, ['/tmp/chromeos/chroot%s' % fake_file]) self.assertEqual(mock_chrootruncmd.call_args_list[0][0], - ('/tmp', ('/usr/sbin/perf report -n ' - '-i %s --stdio > %s') % (fake_file, fake_file))) + (self.result.chromeos_root, + ('/usr/sbin/perf report -n ' + '-i %s --stdio > %s') % (fake_file, fake_file))) @mock.patch.object(misc, 'GetInsideChrootPath') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') @@ -1087,11 +1258,12 @@ class ResultTest(unittest.TestCase): # Debug path found self.result.label.debug_path = '/tmp/debug' tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file]) + self.assertEqual(tmp, ['/tmp/chromeos/chroot%s' % fake_file]) self.assertEqual(mock_chrootruncmd.call_args_list[0][0], - ('/tmp', ('/usr/sbin/perf report -n --symfs /tmp/debug ' - '--vmlinux /tmp/debug/boot/vmlinux ' - '-i %s --stdio > %s') % (fake_file, fake_file))) + (self.result.chromeos_root, + ('/usr/sbin/perf report -n --symfs /tmp/debug ' + '--vmlinux /tmp/debug/boot/vmlinux ' + '-i %s --stdio > %s') % (fake_file, fake_file))) @mock.patch.object(misc, 'GetOutsideChrootPath') def test_populate_from_run(self, mock_getpath): @@ -1124,6 +1296,10 @@ class ResultTest(unittest.TestCase): self.callGetTopFile = True return [] + def FakeGetCpuinfoFile(): + self.callGetCpuinfoFile = True + return [] + def FakeGetWaitTimeFile(): self.callGetWaitTimeFile = True return [] @@ -1136,7 +1312,6 @@ class ResultTest(unittest.TestCase): if mock_getpath: pass mock.get_path = '/tmp/chromeos/tmp/results_dir' - self.result.chromeos_root = '/tmp/chromeos' self.callGetResultsDir = False self.callGetResultsFile = False @@ -1145,6 +1320,7 @@ class ResultTest(unittest.TestCase): self.callGetTurbostatFile = False self.callGetCpustatsFile = False self.callGetTopFile = False + self.callGetCpuinfoFile = False self.callGetWaitTimeFile = False self.callProcessResults = False @@ -1155,6 +1331,7 @@ class ResultTest(unittest.TestCase): self.result.GetTurbostatFile = FakeGetTurbostatFile self.result.GetCpustatsFile = FakeGetCpustatsFile self.result.GetTopFile = FakeGetTopFile + self.result.GetCpuinfoFile = FakeGetCpuinfoFile self.result.GetWaitTimeFile = FakeGetWaitTimeFile self.result.ProcessResults = FakeProcessResults @@ -1167,6 +1344,7 @@ class ResultTest(unittest.TestCase): self.assertTrue(self.callGetTurbostatFile) self.assertTrue(self.callGetCpustatsFile) self.assertTrue(self.callGetTopFile) + self.assertTrue(self.callGetCpuinfoFile) self.assertTrue(self.callGetWaitTimeFile) self.assertTrue(self.callProcessResults) @@ -1403,8 +1581,7 @@ class ResultTest(unittest.TestCase): u'crypto-md5__crypto-md5': [10.5, u'ms'], u'string-tagcloud__string-tagcloud': [52.8, u'ms'], u'access-nbody__access-nbody': [8.5, u'ms'], - 'retval': - 0, + 'retval': 0, u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'], u'math-cordic__math-cordic': [8.7, u'ms'], u'access-binary-trees__access-binary-trees': [4.5, u'ms'], @@ -1442,8 +1619,7 @@ class ResultTest(unittest.TestCase): u'crypto-md5__crypto-md5': [10.5, u'ms'], u'string-tagcloud__string-tagcloud': [52.8, u'ms'], u'access-nbody__access-nbody': [8.5, u'ms'], - 'retval': - 0, + 'retval': 0, u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'], u'math-cordic__math-cordic': [8.7, u'ms'], u'access-binary-trees__access-binary-trees': [4.5, u'ms'], @@ -1672,8 +1848,9 @@ class TelemetryResultTest(unittest.TestCase): 'autotest_dir', 'debug_dir', '/tmp', 'lumpy', 'remote', 'image_args', 'cache_dir', 'average', 'gcc', False, None) - self.mock_machine = machine_manager.MockCrosMachine( - 'falco.cros', '/tmp/chromeos', 'average') + self.mock_machine = machine_manager.MockCrosMachine('falco.cros', + '/tmp/chromeos', + 'average') def test_populate_from_run(self): @@ -1753,10 +1930,12 @@ class ResultsCacheTest(unittest.TestCase): def FakeGetMachines(label): if label: pass - m1 = machine_manager.MockCrosMachine( - 'lumpy1.cros', self.results_cache.chromeos_root, 'average') - m2 = machine_manager.MockCrosMachine( - 'lumpy2.cros', self.results_cache.chromeos_root, 'average') + m1 = machine_manager.MockCrosMachine('lumpy1.cros', + self.results_cache.chromeos_root, + 'average') + m2 = machine_manager.MockCrosMachine('lumpy2.cros', + self.results_cache.chromeos_root, + 'average') return [m1, m2] mock_checksum.return_value = 'FakeImageChecksumabc123' @@ -1798,10 +1977,12 @@ class ResultsCacheTest(unittest.TestCase): def FakeGetMachines(label): if label: pass - m1 = machine_manager.MockCrosMachine( - 'lumpy1.cros', self.results_cache.chromeos_root, 'average') - m2 = machine_manager.MockCrosMachine( - 'lumpy2.cros', self.results_cache.chromeos_root, 'average') + m1 = machine_manager.MockCrosMachine('lumpy1.cros', + self.results_cache.chromeos_root, + 'average') + m2 = machine_manager.MockCrosMachine('lumpy2.cros', + self.results_cache.chromeos_root, + 'average') return [m1, m2] mock_checksum.return_value = 'FakeImageChecksumabc123' diff --git a/crosperf/results_report.py b/crosperf/results_report.py index ff6c4f96..dc80b53b 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -418,8 +418,8 @@ class TextResultsReport(ResultsReport): cpu_info = experiment.machine_manager.GetAllCPUInfo(experiment.labels) sections.append(self._MakeSection('CPUInfo', cpu_info)) - totaltime = ( - time.time() - experiment.start_time) if experiment.start_time else 0 + totaltime = (time.time() - + experiment.start_time) if experiment.start_time else 0 totaltime_str = 'Total experiment time:\n%d min' % (totaltime // 60) cooldown_waittime_list = ['Cooldown wait time:'] # When running experiment on multiple DUTs cooldown wait time may vary @@ -430,8 +430,9 @@ class TextResultsReport(ResultsReport): cooldown_waittime_list.append('DUT %s: %d min' % (dut, waittime // 60)) cooldown_waittime_str = '\n'.join(cooldown_waittime_list) sections.append( - self._MakeSection('Duration', '\n\n'.join( - [totaltime_str, cooldown_waittime_str]))) + self._MakeSection('Duration', + '\n\n'.join([totaltime_str, + cooldown_waittime_str]))) return '\n'.join(sections) @@ -505,7 +506,7 @@ class HTMLResultsReport(ResultsReport): experiment_file = '' if self.experiment is not None: experiment_file = self.experiment.experiment_file - # Use kwargs for sanity, and so that testing is a bit easier. + # Use kwargs for code readability, and so that testing is a bit easier. return templates.GenerateHTMLPage( perf_table=perf_table, chart_js=chart_javascript, diff --git a/crosperf/results_report_templates.py b/crosperf/results_report_templates.py index 3c5258c9..ea411e21 100644 --- a/crosperf/results_report_templates.py +++ b/crosperf/results_report_templates.py @@ -6,7 +6,7 @@ """Text templates used by various parts of results_report.""" from __future__ import print_function -import cgi +import html from string import Template _TabMenuTemplate = Template(""" @@ -19,7 +19,7 @@ _TabMenuTemplate = Template(""" def _GetTabMenuHTML(table_name): # N.B. cgi.escape does some very basic HTML escaping. Nothing more. - escaped = cgi.escape(table_name, quote=True) + escaped = html.escape(table_name) return _TabMenuTemplate.substitute(table_name=escaped) @@ -35,7 +35,7 @@ _ExperimentFileHTML = """ def _GetExperimentFileHTML(experiment_file_text): if not experiment_file_text: return '' - return _ExperimentFileHTML % (cgi.escape(experiment_file_text),) + return _ExperimentFileHTML % (html.escape(experiment_file_text, quote=False),) _ResultsSectionHTML = Template(""" diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py index e03ea431..1e96ef97 100755 --- a/crosperf/results_report_unittest.py +++ b/crosperf/results_report_unittest.py @@ -57,7 +57,8 @@ class FreeFunctionsTest(unittest.TestCase): ParseChromeosImage(os.path.dirname(buildbot_case)), ('', os.path.dirname(buildbot_img))) - # Ensure we don't act completely insanely given a few mildly insane paths. + # Ensure we do something reasonable when giving paths that don't quite + # match the expected pattern. fun_case = '/chromiumos_test_image.bin' self.assertEqual(ParseChromeosImage(fun_case), ('', fun_case)) diff --git a/crosperf/settings.py b/crosperf/settings.py index 41530d97..75c8d9ec 100644 --- a/crosperf/settings.py +++ b/crosperf/settings.py @@ -75,7 +75,7 @@ class Settings(object): prefix = 'remote' l = logger.GetLogger() if (path_str.find('trybot') < 0 and path_str.find('toolchain') < 0 and - path_str.find(board) < 0): + path_str.find(board) < 0 and path_str.find(board.replace('_', '-'))): xbuddy_path = '%s/%s/%s' % (prefix, board, path_str) else: xbuddy_path = '%s/%s' % (prefix, path_str) diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 20ab2ad2..7033a3e8 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -291,6 +291,12 @@ class GlobalSettings(Settings): self.AddField( TextField('results_dir', default='', description='The results dir.')) self.AddField( + BooleanField( + 'compress_results', + default=True, + description='Whether to compress all test results other than ' + 'reports into a tarball to save disk space.')) + self.AddField( TextField( 'locks_dir', default='', @@ -344,16 +350,16 @@ class GlobalSettings(Settings): TextField( 'intel_pstate', description='Intel Pstate mode.\n' - 'Supported modes: passive, no_hwp.\n' - 'By default kernel works in active HWP mode if HWP is supported' - " by CPU. This corresponds to a default intel_pstate=''", + 'Supported modes: "active", "passive", "no_hwp".\n' + 'Default is "no_hwp" which disables hardware pstates to avoid ' + 'noise in benchmarks.', required=False, - default='')) + default='no_hwp')) self.AddField( BooleanField( 'turbostat', description='Run turbostat process in the background' - ' of a benchmark', + ' of a benchmark. Enabled by default.', required=False, default=True)) self.AddField( @@ -365,10 +371,11 @@ class GlobalSettings(Settings): ' data.\n' 'With 0 - do not run top.\n' 'NOTE: Running top with interval 1-5 sec has insignificant' - ' performance impact (performance degradation does not exceed 0.3%,' - ' measured on x86_64, ARM32, and ARM64).', + ' performance impact (performance degradation does not exceed' + ' 0.3%%, measured on x86_64, ARM32, and ARM64). ' + 'The default value is 1.', required=False, - default=0)) + default=1)) self.AddField( IntegerField( 'cooldown_temp', @@ -376,14 +383,16 @@ class GlobalSettings(Settings): default=40, description='Wait until CPU temperature goes down below' ' specified temperature in Celsius' - ' prior starting a benchmark.')) + ' prior starting a benchmark. ' + 'By default the value is set to 40 degrees.')) self.AddField( IntegerField( 'cooldown_time', required=False, - default=0, + default=10, description='Wait specified time in minutes allowing' - ' CPU to cool down. Zero value disables cooldown.')) + ' CPU to cool down. Zero value disables cooldown. ' + 'The default value is 10 minutes.')) self.AddField( EnumField( 'governor', @@ -401,7 +410,8 @@ class GlobalSettings(Settings): required=False, description='Setup CPU governor for all cores.\n' 'For more details refer to:\n' - 'https://www.kernel.org/doc/Documentation/cpu-freq/governors.txt')) + 'https://www.kernel.org/doc/Documentation/cpu-freq/governors.txt. ' + 'Default is "performance" governor.')) self.AddField( EnumField( 'cpu_usage', @@ -413,19 +423,22 @@ class GlobalSettings(Settings): ], default='all', required=False, - description='Restrict usage CPUs to decrease CPU interference.\n' - 'all - no restrictions;\n' - 'big-only, little-only - enable only big/little cores,' + description='Restrict usage of CPUs to decrease CPU interference.\n' + '"all" - no restrictions;\n' + '"big-only", "little-only" - enable only big/little cores,' ' applicable only on ARM;\n' - 'exclusive-cores - (for future use)' - ' isolate cores for exclusive use of benchmark processes.')) + '"exclusive-cores" - (for future use)' + ' isolate cores for exclusive use of benchmark processes. ' + 'By default use all CPUs.')) self.AddField( IntegerField( 'cpu_freq_pct', required=False, - default=100, + default=95, description='Setup CPU frequency to a supported value less than' - ' or equal to a percent of max_freq.')) + ' or equal to a percent of max_freq. ' + 'CPU frequency is reduced to 95%% by default to reduce thermal ' + 'throttling.')) class SettingsFactory(object): diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index a6339034..bc107110 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -50,7 +50,7 @@ class GlobalSettingsTest(unittest.TestCase): def test_init(self): res = settings_factory.GlobalSettings('g_settings') self.assertIsNotNone(res) - self.assertEqual(len(res.fields), 38) + self.assertEqual(len(res.fields), 39) self.assertEqual(res.GetField('name'), '') self.assertEqual(res.GetField('board'), '') self.assertEqual(res.GetField('skylab'), False) @@ -73,18 +73,19 @@ class GlobalSettingsTest(unittest.TestCase): self.assertEqual(res.GetField('show_all_results'), False) self.assertEqual(res.GetField('share_cache'), '') self.assertEqual(res.GetField('results_dir'), '') + self.assertEqual(res.GetField('compress_results'), True) self.assertEqual(res.GetField('chrome_src'), '') self.assertEqual(res.GetField('cwp_dso'), '') self.assertEqual(res.GetField('enable_aslr'), False) self.assertEqual(res.GetField('ignore_min_max'), False) - self.assertEqual(res.GetField('intel_pstate'), '') + self.assertEqual(res.GetField('intel_pstate'), 'no_hwp') self.assertEqual(res.GetField('turbostat'), True) - self.assertEqual(res.GetField('top_interval'), 0) - self.assertEqual(res.GetField('cooldown_time'), 0) + self.assertEqual(res.GetField('top_interval'), 1) + self.assertEqual(res.GetField('cooldown_time'), 10) self.assertEqual(res.GetField('cooldown_temp'), 40) self.assertEqual(res.GetField('governor'), 'performance') self.assertEqual(res.GetField('cpu_usage'), 'all') - self.assertEqual(res.GetField('cpu_freq_pct'), 100) + self.assertEqual(res.GetField('cpu_freq_pct'), 95) class SettingsFactoryTest(unittest.TestCase): @@ -107,7 +108,7 @@ class SettingsFactoryTest(unittest.TestCase): g_settings = settings_factory.SettingsFactory().GetSettings( 'global', 'global') self.assertIsInstance(g_settings, settings_factory.GlobalSettings) - self.assertEqual(len(g_settings.fields), 38) + self.assertEqual(len(g_settings.fields), 39) if __name__ == '__main__': diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index 79ace20d..17e1ad73 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -17,7 +17,7 @@ import time from cros_utils import command_executer TEST_THAT_PATH = '/usr/bin/test_that' -# TODO: Need to check whether Skylab is installed and set up correctly. +TAST_PATH = '/usr/bin/tast' SKYLAB_PATH = '/usr/local/bin/skylab' GS_UTIL = 'src/chromium/depot_tools/gsutil.py' AUTOTEST_DIR = '/mnt/host/source/src/third_party/autotest/files' @@ -78,8 +78,11 @@ class SuiteRunner(object): if label.skylab: ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args) else: - ret_tup = self.Test_That_Run(machine_name, label, benchmark, test_args, - profiler_args) + if benchmark.suite == 'tast': + ret_tup = self.Tast_Run(machine_name, label, benchmark) + else: + ret_tup = self.Test_That_Run(machine_name, label, benchmark, + test_args, profiler_args) if ret_tup[0] != 0: self.logger.LogOutput('benchmark %s failed. Retries left: %s' % (benchmark.name, benchmark.retries - i)) @@ -127,6 +130,24 @@ class SuiteRunner(object): return args_list + # TODO(zhizhouy): Currently do not support passing arguments or running + # customized tast tests, as we do not have such requirements. + def Tast_Run(self, machine, label, benchmark): + # Remove existing tast results + command = 'rm -rf /usr/local/autotest/results/*' + self._ce.CrosRunCommand( + command, machine=machine, chromeos_root=label.chromeos_root) + + command = ' '.join( + [TAST_PATH, 'run', '-build=False', machine, benchmark.test_name]) + + if self.log_level != 'verbose': + self.logger.LogOutput('Running test.') + self.logger.LogOutput('CMD: %s' % command) + + return self._ce.ChrootRunCommandWOutput( + label.chromeos_root, command, command_terminator=self._ct) + def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): """Run the test_that test..""" @@ -166,11 +187,10 @@ class SuiteRunner(object): # process namespace and we can kill process created easily by their # process group. chrome_root_options = ('--no-ns-pid ' - '--chrome_root={} --chrome_root_mount={} ' + '--chrome_root={0} --chrome_root_mount={1} ' 'FEATURES="-usersandbox" ' - 'CHROME_ROOT={}'.format(label.chrome_src, - CHROME_MOUNT_DIR, - CHROME_MOUNT_DIR)) + 'CHROME_ROOT={1}'.format(label.chrome_src, + CHROME_MOUNT_DIR)) if self.log_level != 'verbose': self.logger.LogOutput('Running test.') diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py index 73fcb45b..86e1ef19 100755 --- a/crosperf/suite_runner_unittest.py +++ b/crosperf/suite_runner_unittest.py @@ -26,8 +26,6 @@ from machine_manager import MockCrosMachine class SuiteRunnerTest(unittest.TestCase): """Class of SuiteRunner test.""" - real_logger = logger.GetLogger() - mock_json = mock.Mock(spec=json) mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator) @@ -55,12 +53,23 @@ class SuiteRunnerTest(unittest.TestCase): '', # perf_args 'crosperf_Wrapper') # suite + tast_bench = Benchmark( + 'b3_test', # name + 'platform.ReportDiskUsage', # test_name + '', # test_args + 1, # iterations + False, # rm_chroot_tmp + '', # perf_args + 'tast') # suite + def __init__(self, *args, **kwargs): super(SuiteRunnerTest, self).__init__(*args, **kwargs) self.skylab_run_args = [] self.test_that_args = [] + self.tast_args = [] self.call_skylab_run = False self.call_test_that_run = False + self.call_tast_run = False def setUp(self): self.runner = suite_runner.SuiteRunner( @@ -90,8 +99,10 @@ class SuiteRunnerTest(unittest.TestCase): def reset(): self.test_that_args = [] self.skylab_run_args = [] + self.tast_args = [] self.call_test_that_run = False self.call_skylab_run = False + self.call_tast_run = False def FakeSkylabRun(test_label, benchmark, test_args, profiler_args): self.skylab_run_args = [test_label, benchmark, test_args, profiler_args] @@ -106,8 +117,14 @@ class SuiteRunnerTest(unittest.TestCase): self.call_test_that_run = True return 'Ran FakeTestThatRun' + def FakeTastRun(machine, test_label, benchmark): + self.tast_args = [machine, test_label, benchmark] + self.call_tast_run = True + return 'Ran FakeTastRun' + self.runner.Skylab_Run = FakeSkylabRun self.runner.Test_That_Run = FakeTestThatRun + self.runner.Tast_Run = FakeTastRun self.runner.dut_config['enable_aslr'] = False self.runner.dut_config['cooldown_time'] = 0 @@ -158,6 +175,15 @@ class SuiteRunnerTest(unittest.TestCase): 'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', '' ]) + # Test tast run for tast benchmarks. + reset() + self.runner.Run(cros_machine, self.mock_label, self.tast_bench, '', '') + self.assertTrue(self.call_tast_run) + self.assertFalse(self.call_test_that_run) + self.assertFalse(self.call_skylab_run) + self.assertEqual(self.tast_args, + ['fake_machine', self.mock_label, self.tast_bench]) + def test_gen_test_args(self): test_args = '--iterations=2' perf_args = 'record -a -e cycles' @@ -180,16 +206,29 @@ class SuiteRunnerTest(unittest.TestCase): @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommandWOutput') - def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd): - - def FakeLogMsg(fd, termfd, msg, flush=True): - if fd or termfd or msg or flush: - pass - - save_log_msg = self.real_logger.LogMsg - self.real_logger.LogMsg = FakeLogMsg - self.runner.logger = self.real_logger + def test_tast_run(self, mock_chroot_runcmd, mock_cros_runcmd): + mock_chroot_runcmd.return_value = 0 + self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd + self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd + res = self.runner.Tast_Run('lumpy1.cros', self.mock_label, self.tast_bench) + self.assertEqual(mock_cros_runcmd.call_count, 1) + self.assertEqual(mock_chroot_runcmd.call_count, 1) + self.assertEqual(res, 0) + self.assertEqual(mock_cros_runcmd.call_args_list[0][0], + ('rm -rf /usr/local/autotest/results/*',)) + args_list = mock_chroot_runcmd.call_args_list[0][0] + args_dict = mock_chroot_runcmd.call_args_list[0][1] + self.assertEqual(len(args_list), 2) + self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) + @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') + @mock.patch.object(command_executer.CommandExecuter, + 'ChrootRunCommandWOutput') + @mock.patch.object(logger.Logger, 'LogFatal') + def test_test_that_run(self, mock_log_fatal, mock_chroot_runcmd, + mock_cros_runcmd): + mock_log_fatal.side_effect = SystemExit() + self.runner.logger.LogFatal = mock_log_fatal # Test crosperf_Wrapper benchmarks cannot take perf_args raised_exception = False try: @@ -215,7 +254,6 @@ class SuiteRunnerTest(unittest.TestCase): args_dict = mock_chroot_runcmd.call_args_list[0][1] self.assertEqual(len(args_list), 2) self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) - self.real_logger.LogMsg = save_log_msg @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') @mock.patch.object(json, 'loads') |