diff options
Diffstat (limited to 'crosperf/results_report_unittest.py')
| -rwxr-xr-x | crosperf/results_report_unittest.py | 819 |
1 files changed, 429 insertions, 390 deletions
diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py index 1e96ef97..4ce654d0 100755 --- a/crosperf/results_report_unittest.py +++ b/crosperf/results_report_unittest.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# Copyright 2016 The Chromium OS Authors. All rights reserved. +# Copyright 2016 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Unittest for the results reporter.""" -from __future__ import division -from __future__ import print_function import collections import io @@ -16,8 +14,6 @@ import os import unittest import unittest.mock as mock -import test_flag - from benchmark_run import MockBenchmarkRun from cros_utils import logger from experiment_factory import ExperimentFactory @@ -31,39 +27,46 @@ from results_report import JSONResultsReport from results_report import ParseChromeosImage from results_report import ParseStandardPerfReport from results_report import TextResultsReport +import test_flag class FreeFunctionsTest(unittest.TestCase): - """Tests for any free functions in results_report.""" - - def testParseChromeosImage(self): - # N.B. the cases with blank versions aren't explicitly supported by - # ParseChromeosImage. I'm not sure if they need to be supported, but the - # goal of this was to capture existing functionality as much as possible. - base_case = '/my/chroot/src/build/images/x86-generic/R01-1.0.date-time' \ - '/chromiumos_test_image.bin' - self.assertEqual(ParseChromeosImage(base_case), ('R01-1.0', base_case)) - - dir_base_case = os.path.dirname(base_case) - self.assertEqual(ParseChromeosImage(dir_base_case), ('', dir_base_case)) - - buildbot_case = '/my/chroot/chroot/tmp/buildbot-build/R02-1.0.date-time' \ - '/chromiumos_test_image.bin' - buildbot_img = buildbot_case.split('/chroot/tmp')[1] - - self.assertEqual( - ParseChromeosImage(buildbot_case), ('R02-1.0', buildbot_img)) - self.assertEqual( - ParseChromeosImage(os.path.dirname(buildbot_case)), - ('', os.path.dirname(buildbot_img))) - - # Ensure we do something reasonable when giving paths that don't quite - # match the expected pattern. - fun_case = '/chromiumos_test_image.bin' - self.assertEqual(ParseChromeosImage(fun_case), ('', fun_case)) - - fun_case2 = 'chromiumos_test_image.bin' - self.assertEqual(ParseChromeosImage(fun_case2), ('', fun_case2)) + """Tests for any free functions in results_report.""" + + def testParseChromeosImage(self): + # N.B. the cases with blank versions aren't explicitly supported by + # ParseChromeosImage. I'm not sure if they need to be supported, but the + # goal of this was to capture existing functionality as much as possible. + base_case = ( + "/my/chroot/src/build/images/x86-generic/R01-1.0.date-time" + "/chromiumos_test_image.bin" + ) + self.assertEqual(ParseChromeosImage(base_case), ("R01-1.0", base_case)) + + dir_base_case = os.path.dirname(base_case) + self.assertEqual(ParseChromeosImage(dir_base_case), ("", dir_base_case)) + + buildbot_case = ( + "/my/chroot/chroot/tmp/buildbot-build/R02-1.0.date-time" + "/chromiumos_test_image.bin" + ) + buildbot_img = buildbot_case.split("/chroot/tmp")[1] + + self.assertEqual( + ParseChromeosImage(buildbot_case), ("R02-1.0", buildbot_img) + ) + self.assertEqual( + ParseChromeosImage(os.path.dirname(buildbot_case)), + ("", os.path.dirname(buildbot_img)), + ) + + # Ensure we do something reasonable when giving paths that don't quite + # match the expected pattern. + fun_case = "/chromiumos_test_image.bin" + self.assertEqual(ParseChromeosImage(fun_case), ("", fun_case)) + + fun_case2 = "chromiumos_test_image.bin" + self.assertEqual(ParseChromeosImage(fun_case2), ("", fun_case2)) # There are many ways for this to be done better, but the linter complains @@ -72,19 +75,20 @@ _fake_path_number = [0] def FakePath(ext): - """Makes a unique path that shouldn't exist on the host system. + """Makes a unique path that shouldn't exist on the host system. - Each call returns a different path, so if said path finds its way into an - error message, it may be easier to track it to its source. - """ - _fake_path_number[0] += 1 - prefix = '/tmp/should/not/exist/%d/' % (_fake_path_number[0],) - return os.path.join(prefix, ext) + Each call returns a different path, so if said path finds its way into an + error message, it may be easier to track it to its source. + """ + _fake_path_number[0] += 1 + prefix = "/tmp/should/not/exist/%d/" % (_fake_path_number[0],) + return os.path.join(prefix, ext) -def MakeMockExperiment(compiler='gcc'): - """Mocks an experiment using the given compiler.""" - mock_experiment_file = io.StringIO(""" +def MakeMockExperiment(compiler="gcc"): + """Mocks an experiment using the given compiler.""" + mock_experiment_file = io.StringIO( + """ board: x86-alex remote: 127.0.0.1 locks_dir: /tmp @@ -101,363 +105,398 @@ def MakeMockExperiment(compiler='gcc'): remote: 127.0.0.2 chromeos_image: %s } - """ % (FakePath('cros_image1.bin'), FakePath('cros_image2.bin'))) - efile = ExperimentFile(mock_experiment_file) - experiment = ExperimentFactory().GetExperiment(efile, - FakePath('working_directory'), - FakePath('log_dir')) - for label in experiment.labels: - label.compiler = compiler - return experiment + """ + % (FakePath("cros_image1.bin"), FakePath("cros_image2.bin")) + ) + efile = ExperimentFile(mock_experiment_file) + experiment = ExperimentFactory().GetExperiment( + efile, FakePath("working_directory"), FakePath("log_dir") + ) + for label in experiment.labels: + label.compiler = compiler + return experiment def _InjectSuccesses(experiment, how_many, keyvals, for_benchmark=0): - """Injects successful experiment runs (for each label) into the experiment.""" - # Defensive copy of keyvals, so if it's modified, we'll know. - keyvals = dict(keyvals) - num_configs = len(experiment.benchmarks) * len(experiment.labels) - num_runs = len(experiment.benchmark_runs) // num_configs - - # TODO(gbiv): Centralize the mocking of these, maybe? (It's also done in - # benchmark_run_unittest) - bench = experiment.benchmarks[for_benchmark] - cache_conditions = [] - log_level = 'average' - share_cache = '' - locks_dir = '' - log = logger.GetLogger() - machine_manager = MockMachineManager( - FakePath('chromeos_root'), 0, log_level, locks_dir) - machine_manager.AddMachine('testing_machine') - machine = next( - m for m in machine_manager.GetMachines() if m.name == 'testing_machine') - - def MakeSuccessfulRun(n, label): - run = MockBenchmarkRun('mock_success%d' % (n,), bench, label, - 1 + n + num_runs, cache_conditions, machine_manager, - log, log_level, share_cache, {}) - mock_result = MockResult(log, label, log_level, machine) - mock_result.keyvals = keyvals - run.result = mock_result - return run - - for label in experiment.labels: - experiment.benchmark_runs.extend( - MakeSuccessfulRun(n, label) for n in range(how_many)) - return experiment + """Injects successful experiment runs (for each label) into the experiment.""" + # Defensive copy of keyvals, so if it's modified, we'll know. + keyvals = dict(keyvals) + num_configs = len(experiment.benchmarks) * len(experiment.labels) + num_runs = len(experiment.benchmark_runs) // num_configs + + # TODO(gbiv): Centralize the mocking of these, maybe? (It's also done in + # benchmark_run_unittest) + bench = experiment.benchmarks[for_benchmark] + cache_conditions = [] + log_level = "average" + share_cache = "" + locks_dir = "" + log = logger.GetLogger() + machine_manager = MockMachineManager( + FakePath("chromeos_root"), 0, log_level, locks_dir + ) + machine_manager.AddMachine("testing_machine") + machine = next( + m for m in machine_manager.GetMachines() if m.name == "testing_machine" + ) + + def MakeSuccessfulRun(n, label): + run = MockBenchmarkRun( + "mock_success%d" % (n,), + bench, + label, + 1 + n + num_runs, + cache_conditions, + machine_manager, + log, + log_level, + share_cache, + {}, + ) + mock_result = MockResult(log, label, log_level, machine) + mock_result.keyvals = keyvals + run.result = mock_result + return run + + for label in experiment.labels: + experiment.benchmark_runs.extend( + MakeSuccessfulRun(n, label) for n in range(how_many) + ) + return experiment class TextResultsReportTest(unittest.TestCase): - """Tests that the output of a text report contains the things we pass in. - - At the moment, this doesn't care deeply about the format in which said - things are displayed. It just cares that they're present. - """ - - def _checkReport(self, mock_getcooldown, email): - num_success = 2 - success_keyvals = {'retval': 0, 'machine': 'some bot', 'a_float': 3.96} - experiment = _InjectSuccesses(MakeMockExperiment(), num_success, - success_keyvals) - SECONDS_IN_MIN = 60 - mock_getcooldown.return_value = { - experiment.remote[0]: 12 * SECONDS_IN_MIN, - experiment.remote[1]: 8 * SECONDS_IN_MIN - } - - text_report = TextResultsReport.FromExperiment( - experiment, email=email).GetReport() - self.assertIn(str(success_keyvals['a_float']), text_report) - self.assertIn(success_keyvals['machine'], text_report) - self.assertIn(MockCrosMachine.CPUINFO_STRING, text_report) - self.assertIn('\nDuration\n', text_report) - self.assertIn('Total experiment time:\n', text_report) - self.assertIn('Cooldown wait time:\n', text_report) - self.assertIn('DUT %s: %d min' % (experiment.remote[0], 12), text_report) - self.assertIn('DUT %s: %d min' % (experiment.remote[1], 8), text_report) - return text_report - - @mock.patch.object(TextResultsReport, 'GetTotalWaitCooldownTime') - def testOutput(self, mock_getcooldown): - email_report = self._checkReport(mock_getcooldown, email=True) - text_report = self._checkReport(mock_getcooldown, email=False) - - # Ensure that the reports somehow different. Otherwise, having the - # distinction is useless. - self.assertNotEqual(email_report, text_report) - - def test_get_totalwait_cooldowntime(self): - experiment = MakeMockExperiment() - cros_machines = experiment.machine_manager.GetMachines() - cros_machines[0].AddCooldownWaitTime(120) - cros_machines[1].AddCooldownWaitTime(240) - text_results = TextResultsReport.FromExperiment(experiment, email=False) - total = text_results.GetTotalWaitCooldownTime() - self.assertEqual(total[experiment.remote[0]], 120) - self.assertEqual(total[experiment.remote[1]], 240) + """Tests that the output of a text report contains the things we pass in. + + At the moment, this doesn't care deeply about the format in which said + things are displayed. It just cares that they're present. + """ + + def _checkReport(self, mock_getcooldown, email): + num_success = 2 + success_keyvals = {"retval": 0, "machine": "some bot", "a_float": 3.96} + experiment = _InjectSuccesses( + MakeMockExperiment(), num_success, success_keyvals + ) + SECONDS_IN_MIN = 60 + mock_getcooldown.return_value = { + experiment.remote[0]: 12 * SECONDS_IN_MIN, + experiment.remote[1]: 8 * SECONDS_IN_MIN, + } + + text_report = TextResultsReport.FromExperiment( + experiment, email=email + ).GetReport() + self.assertIn(str(success_keyvals["a_float"]), text_report) + self.assertIn(success_keyvals["machine"], text_report) + self.assertIn(MockCrosMachine.CPUINFO_STRING, text_report) + self.assertIn("\nDuration\n", text_report) + self.assertIn("Total experiment time:\n", text_report) + self.assertIn("Cooldown wait time:\n", text_report) + self.assertIn( + "DUT %s: %d min" % (experiment.remote[0], 12), text_report + ) + self.assertIn("DUT %s: %d min" % (experiment.remote[1], 8), text_report) + return text_report + + @mock.patch.object(TextResultsReport, "GetTotalWaitCooldownTime") + def testOutput(self, mock_getcooldown): + email_report = self._checkReport(mock_getcooldown, email=True) + text_report = self._checkReport(mock_getcooldown, email=False) + + # Ensure that the reports somehow different. Otherwise, having the + # distinction is useless. + self.assertNotEqual(email_report, text_report) + + def test_get_totalwait_cooldowntime(self): + experiment = MakeMockExperiment() + cros_machines = experiment.machine_manager.GetMachines() + cros_machines[0].AddCooldownWaitTime(120) + cros_machines[1].AddCooldownWaitTime(240) + text_results = TextResultsReport.FromExperiment(experiment, email=False) + total = text_results.GetTotalWaitCooldownTime() + self.assertEqual(total[experiment.remote[0]], 120) + self.assertEqual(total[experiment.remote[1]], 240) class HTMLResultsReportTest(unittest.TestCase): - """Tests that the output of a HTML report contains the things we pass in. - - At the moment, this doesn't care deeply about the format in which said - things are displayed. It just cares that they're present. - """ - - _TestOutput = collections.namedtuple('TestOutput', [ - 'summary_table', 'perf_html', 'chart_js', 'charts', 'full_table', - 'experiment_file' - ]) - - @staticmethod - def _GetTestOutput(perf_table, chart_js, summary_table, print_table, - chart_divs, full_table, experiment_file): - # N.B. Currently we don't check chart_js; it's just passed through because - # cros lint complains otherwise. - summary_table = print_table(summary_table, 'HTML') - perf_html = print_table(perf_table, 'HTML') - full_table = print_table(full_table, 'HTML') - return HTMLResultsReportTest._TestOutput( - summary_table=summary_table, - perf_html=perf_html, - chart_js=chart_js, - charts=chart_divs, - full_table=full_table, - experiment_file=experiment_file) - - def _GetOutput(self, experiment=None, benchmark_results=None): - with mock.patch('results_report_templates.GenerateHTMLPage') as standin: - if experiment is not None: - HTMLResultsReport.FromExperiment(experiment).GetReport() - else: - HTMLResultsReport(benchmark_results).GetReport() - mod_mock = standin - self.assertEqual(mod_mock.call_count, 1) - # call_args[0] is positional args, call_args[1] is kwargs. - self.assertEqual(mod_mock.call_args[0], tuple()) - fmt_args = mod_mock.call_args[1] - return self._GetTestOutput(**fmt_args) - - def testNoSuccessOutput(self): - output = self._GetOutput(MakeMockExperiment()) - self.assertIn('no result', output.summary_table) - self.assertIn('no result', output.full_table) - self.assertEqual(output.charts, '') - self.assertNotEqual(output.experiment_file, '') - - def testSuccessfulOutput(self): - num_success = 2 - success_keyvals = {'retval': 0, 'a_float': 3.96} - output = self._GetOutput( - _InjectSuccesses(MakeMockExperiment(), num_success, success_keyvals)) - - self.assertNotIn('no result', output.summary_table) - # self.assertIn(success_keyvals['machine'], output.summary_table) - self.assertIn('a_float', output.summary_table) - self.assertIn(str(success_keyvals['a_float']), output.summary_table) - self.assertIn('a_float', output.full_table) - # The _ in a_float is filtered out when we're generating HTML. - self.assertIn('afloat', output.charts) - # And make sure we have our experiment file... - self.assertNotEqual(output.experiment_file, '') - - def testBenchmarkResultFailure(self): - labels = ['label1'] - benchmark_names_and_iterations = [('bench1', 1)] - benchmark_keyvals = {'bench1': [[]]} - results = BenchmarkResults(labels, benchmark_names_and_iterations, - benchmark_keyvals) - output = self._GetOutput(benchmark_results=results) - self.assertIn('no result', output.summary_table) - self.assertEqual(output.charts, '') - self.assertEqual(output.experiment_file, '') - - def testBenchmarkResultSuccess(self): - labels = ['label1'] - benchmark_names_and_iterations = [('bench1', 1)] - benchmark_keyvals = {'bench1': [[{'retval': 1, 'foo': 2.0}]]} - results = BenchmarkResults(labels, benchmark_names_and_iterations, - benchmark_keyvals) - output = self._GetOutput(benchmark_results=results) - self.assertNotIn('no result', output.summary_table) - self.assertIn('bench1', output.summary_table) - self.assertIn('bench1', output.full_table) - self.assertNotEqual(output.charts, '') - self.assertEqual(output.experiment_file, '') + """Tests that the output of a HTML report contains the things we pass in. + + At the moment, this doesn't care deeply about the format in which said + things are displayed. It just cares that they're present. + """ + + _TestOutput = collections.namedtuple( + "TestOutput", + [ + "summary_table", + "perf_html", + "chart_js", + "charts", + "full_table", + "experiment_file", + ], + ) + + @staticmethod + def _GetTestOutput( + perf_table, + chart_js, + summary_table, + print_table, + chart_divs, + full_table, + experiment_file, + ): + # N.B. Currently we don't check chart_js; it's just passed through because + # cros lint complains otherwise. + summary_table = print_table(summary_table, "HTML") + perf_html = print_table(perf_table, "HTML") + full_table = print_table(full_table, "HTML") + return HTMLResultsReportTest._TestOutput( + summary_table=summary_table, + perf_html=perf_html, + chart_js=chart_js, + charts=chart_divs, + full_table=full_table, + experiment_file=experiment_file, + ) + + def _GetOutput(self, experiment=None, benchmark_results=None): + with mock.patch("results_report_templates.GenerateHTMLPage") as standin: + if experiment is not None: + HTMLResultsReport.FromExperiment(experiment).GetReport() + else: + HTMLResultsReport(benchmark_results).GetReport() + mod_mock = standin + self.assertEqual(mod_mock.call_count, 1) + # call_args[0] is positional args, call_args[1] is kwargs. + self.assertEqual(mod_mock.call_args[0], tuple()) + fmt_args = mod_mock.call_args[1] + return self._GetTestOutput(**fmt_args) + + def testNoSuccessOutput(self): + output = self._GetOutput(MakeMockExperiment()) + self.assertIn("no result", output.summary_table) + self.assertIn("no result", output.full_table) + self.assertEqual(output.charts, "") + self.assertNotEqual(output.experiment_file, "") + + def testSuccessfulOutput(self): + num_success = 2 + success_keyvals = {"retval": 0, "a_float": 3.96} + output = self._GetOutput( + _InjectSuccesses(MakeMockExperiment(), num_success, success_keyvals) + ) + + self.assertNotIn("no result", output.summary_table) + # self.assertIn(success_keyvals['machine'], output.summary_table) + self.assertIn("a_float", output.summary_table) + self.assertIn(str(success_keyvals["a_float"]), output.summary_table) + self.assertIn("a_float", output.full_table) + # The _ in a_float is filtered out when we're generating HTML. + self.assertIn("afloat", output.charts) + # And make sure we have our experiment file... + self.assertNotEqual(output.experiment_file, "") + + def testBenchmarkResultFailure(self): + labels = ["label1"] + benchmark_names_and_iterations = [("bench1", 1)] + benchmark_keyvals = {"bench1": [[]]} + results = BenchmarkResults( + labels, benchmark_names_and_iterations, benchmark_keyvals + ) + output = self._GetOutput(benchmark_results=results) + self.assertIn("no result", output.summary_table) + self.assertEqual(output.charts, "") + self.assertEqual(output.experiment_file, "") + + def testBenchmarkResultSuccess(self): + labels = ["label1"] + benchmark_names_and_iterations = [("bench1", 1)] + benchmark_keyvals = {"bench1": [[{"retval": 1, "foo": 2.0}]]} + results = BenchmarkResults( + labels, benchmark_names_and_iterations, benchmark_keyvals + ) + output = self._GetOutput(benchmark_results=results) + self.assertNotIn("no result", output.summary_table) + self.assertIn("bench1", output.summary_table) + self.assertIn("bench1", output.full_table) + self.assertNotEqual(output.charts, "") + self.assertEqual(output.experiment_file, "") class JSONResultsReportTest(unittest.TestCase): - """Tests JSONResultsReport.""" - - REQUIRED_REPORT_KEYS = ('date', 'time', 'label', 'test_name', 'pass') - EXPERIMENT_REPORT_KEYS = ('board', 'chromeos_image', 'chromeos_version', - 'chrome_version', 'compiler') - - @staticmethod - def _GetRequiredKeys(is_experiment): - required_keys = JSONResultsReportTest.REQUIRED_REPORT_KEYS - if is_experiment: - required_keys += JSONResultsReportTest.EXPERIMENT_REPORT_KEYS - return required_keys - - def _CheckRequiredKeys(self, test_output, is_experiment): - required_keys = self._GetRequiredKeys(is_experiment) - for output in test_output: - for key in required_keys: - self.assertIn(key, output) - - def testAllFailedJSONReportOutput(self): - experiment = MakeMockExperiment() - results = JSONResultsReport.FromExperiment(experiment).GetReportObject() - self._CheckRequiredKeys(results, is_experiment=True) - # Nothing succeeded; we don't send anything more than what's required. - required_keys = self._GetRequiredKeys(is_experiment=True) - for result in results: - self.assertCountEqual(result.keys(), required_keys) - - def testJSONReportOutputWithSuccesses(self): - success_keyvals = { - 'retval': 0, - 'a_float': '2.3', - 'many_floats': [['1.0', '2.0'], ['3.0']], - 'machine': "i'm a pirate" - } - - # 2 is arbitrary. - num_success = 2 - experiment = _InjectSuccesses(MakeMockExperiment(), num_success, - success_keyvals) - results = JSONResultsReport.FromExperiment(experiment).GetReportObject() - self._CheckRequiredKeys(results, is_experiment=True) - - num_passes = num_success * len(experiment.labels) - non_failures = [r for r in results if r['pass']] - self.assertEqual(num_passes, len(non_failures)) - - # TODO(gbiv): ...Is the 3.0 *actually* meant to be dropped? - expected_detailed = {'a_float': 2.3, 'many_floats': [1.0, 2.0]} - for pass_ in non_failures: - self.assertIn('detailed_results', pass_) - self.assertDictEqual(expected_detailed, pass_['detailed_results']) - self.assertIn('machine', pass_) - self.assertEqual(success_keyvals['machine'], pass_['machine']) - - def testFailedJSONReportOutputWithoutExperiment(self): - labels = ['label1'] - # yapf:disable - benchmark_names_and_iterations = [('bench1', 1), ('bench2', 2), - ('bench3', 1), ('bench4', 0)] - # yapf:enable - - benchmark_keyvals = { - 'bench1': [[{ - 'retval': 1, - 'foo': 2.0 - }]], - 'bench2': [[{ - 'retval': 1, - 'foo': 4.0 - }, { - 'retval': -1, - 'bar': 999 - }]], - # lack of retval is considered a failure. - 'bench3': [[{}]], - 'bench4': [[]] - } - bench_results = BenchmarkResults(labels, benchmark_names_and_iterations, - benchmark_keyvals) - results = JSONResultsReport(bench_results).GetReportObject() - self._CheckRequiredKeys(results, is_experiment=False) - self.assertFalse(any(r['pass'] for r in results)) - - def testJSONGetReportObeysJSONSettings(self): - labels = ['label1'] - benchmark_names_and_iterations = [('bench1', 1)] - # These can be anything, really. So long as they're distinctive. - separators = (',\t\n\t', ':\t\n\t') - benchmark_keyvals = {'bench1': [[{'retval': 0, 'foo': 2.0}]]} - bench_results = BenchmarkResults(labels, benchmark_names_and_iterations, - benchmark_keyvals) - reporter = JSONResultsReport( - bench_results, json_args={'separators': separators}) - result_str = reporter.GetReport() - self.assertIn(separators[0], result_str) - self.assertIn(separators[1], result_str) - - def testSuccessfulJSONReportOutputWithoutExperiment(self): - labels = ['label1'] - benchmark_names_and_iterations = [('bench1', 1), ('bench2', 2)] - benchmark_keyvals = { - 'bench1': [[{ - 'retval': 0, - 'foo': 2.0 - }]], - 'bench2': [[{ - 'retval': 0, - 'foo': 4.0 - }, { - 'retval': 0, - 'bar': 999 - }]] - } - bench_results = BenchmarkResults(labels, benchmark_names_and_iterations, - benchmark_keyvals) - results = JSONResultsReport(bench_results).GetReportObject() - self._CheckRequiredKeys(results, is_experiment=False) - self.assertTrue(all(r['pass'] for r in results)) - # Enforce that the results have *some* deterministic order. - keyfn = lambda r: (r['test_name'], r['detailed_results'].get('foo', 5.0)) - sorted_results = sorted(results, key=keyfn) - detailed_results = [r['detailed_results'] for r in sorted_results] - bench1, bench2_foo, bench2_bar = detailed_results - self.assertEqual(bench1['foo'], 2.0) - self.assertEqual(bench2_foo['foo'], 4.0) - self.assertEqual(bench2_bar['bar'], 999) - self.assertNotIn('bar', bench1) - self.assertNotIn('bar', bench2_foo) - self.assertNotIn('foo', bench2_bar) + """Tests JSONResultsReport.""" + + REQUIRED_REPORT_KEYS = ("date", "time", "label", "test_name", "pass") + EXPERIMENT_REPORT_KEYS = ( + "board", + "chromeos_image", + "chromeos_version", + "chrome_version", + "compiler", + ) + + @staticmethod + def _GetRequiredKeys(is_experiment): + required_keys = JSONResultsReportTest.REQUIRED_REPORT_KEYS + if is_experiment: + required_keys += JSONResultsReportTest.EXPERIMENT_REPORT_KEYS + return required_keys + + def _CheckRequiredKeys(self, test_output, is_experiment): + required_keys = self._GetRequiredKeys(is_experiment) + for output in test_output: + for key in required_keys: + self.assertIn(key, output) + + def testAllFailedJSONReportOutput(self): + experiment = MakeMockExperiment() + results = JSONResultsReport.FromExperiment(experiment).GetReportObject() + self._CheckRequiredKeys(results, is_experiment=True) + # Nothing succeeded; we don't send anything more than what's required. + required_keys = self._GetRequiredKeys(is_experiment=True) + for result in results: + self.assertCountEqual(result.keys(), required_keys) + + def testJSONReportOutputWithSuccesses(self): + success_keyvals = { + "retval": 0, + "a_float": "2.3", + "many_floats": [["1.0", "2.0"], ["3.0"]], + "machine": "i'm a pirate", + } + + # 2 is arbitrary. + num_success = 2 + experiment = _InjectSuccesses( + MakeMockExperiment(), num_success, success_keyvals + ) + results = JSONResultsReport.FromExperiment(experiment).GetReportObject() + self._CheckRequiredKeys(results, is_experiment=True) + + num_passes = num_success * len(experiment.labels) + non_failures = [r for r in results if r["pass"]] + self.assertEqual(num_passes, len(non_failures)) + + # TODO(gbiv): ...Is the 3.0 *actually* meant to be dropped? + expected_detailed = {"a_float": 2.3, "many_floats": [1.0, 2.0]} + for pass_ in non_failures: + self.assertIn("detailed_results", pass_) + self.assertDictEqual(expected_detailed, pass_["detailed_results"]) + self.assertIn("machine", pass_) + self.assertEqual(success_keyvals["machine"], pass_["machine"]) + + def testFailedJSONReportOutputWithoutExperiment(self): + labels = ["label1"] + # yapf:disable + benchmark_names_and_iterations = [ + ("bench1", 1), + ("bench2", 2), + ("bench3", 1), + ("bench4", 0), + ] + # yapf:enable + + benchmark_keyvals = { + "bench1": [[{"retval": 1, "foo": 2.0}]], + "bench2": [[{"retval": 1, "foo": 4.0}, {"retval": -1, "bar": 999}]], + # lack of retval is considered a failure. + "bench3": [[{}]], + "bench4": [[]], + } + bench_results = BenchmarkResults( + labels, benchmark_names_and_iterations, benchmark_keyvals + ) + results = JSONResultsReport(bench_results).GetReportObject() + self._CheckRequiredKeys(results, is_experiment=False) + self.assertFalse(any(r["pass"] for r in results)) + + def testJSONGetReportObeysJSONSettings(self): + labels = ["label1"] + benchmark_names_and_iterations = [("bench1", 1)] + # These can be anything, really. So long as they're distinctive. + separators = (",\t\n\t", ":\t\n\t") + benchmark_keyvals = {"bench1": [[{"retval": 0, "foo": 2.0}]]} + bench_results = BenchmarkResults( + labels, benchmark_names_and_iterations, benchmark_keyvals + ) + reporter = JSONResultsReport( + bench_results, json_args={"separators": separators} + ) + result_str = reporter.GetReport() + self.assertIn(separators[0], result_str) + self.assertIn(separators[1], result_str) + + def testSuccessfulJSONReportOutputWithoutExperiment(self): + labels = ["label1"] + benchmark_names_and_iterations = [("bench1", 1), ("bench2", 2)] + benchmark_keyvals = { + "bench1": [[{"retval": 0, "foo": 2.0}]], + "bench2": [[{"retval": 0, "foo": 4.0}, {"retval": 0, "bar": 999}]], + } + bench_results = BenchmarkResults( + labels, benchmark_names_and_iterations, benchmark_keyvals + ) + results = JSONResultsReport(bench_results).GetReportObject() + self._CheckRequiredKeys(results, is_experiment=False) + self.assertTrue(all(r["pass"] for r in results)) + # Enforce that the results have *some* deterministic order. + keyfn = lambda r: ( + r["test_name"], + r["detailed_results"].get("foo", 5.0), + ) + sorted_results = sorted(results, key=keyfn) + detailed_results = [r["detailed_results"] for r in sorted_results] + bench1, bench2_foo, bench2_bar = detailed_results + self.assertEqual(bench1["foo"], 2.0) + self.assertEqual(bench2_foo["foo"], 4.0) + self.assertEqual(bench2_bar["bar"], 999) + self.assertNotIn("bar", bench1) + self.assertNotIn("bar", bench2_foo) + self.assertNotIn("foo", bench2_bar) class PerfReportParserTest(unittest.TestCase): - """Tests for the perf report parser in results_report.""" - - @staticmethod - def _ReadRealPerfReport(): - my_dir = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(my_dir, 'perf_files/perf.data.report.0')) as f: - return f.read() - - def testParserParsesRealWorldPerfReport(self): - report = ParseStandardPerfReport(self._ReadRealPerfReport()) - self.assertCountEqual(['cycles', 'instructions'], list(report.keys())) - - # Arbitrarily selected known percentages from the perf report. - known_cycles_percentages = { - '0xffffffffa4a1f1c9': 0.66, - '0x0000115bb7ba9b54': 0.47, - '0x0000000000082e08': 0.00, - '0xffffffffa4a13e63': 0.00, - } - report_cycles = report['cycles'] - self.assertEqual(len(report_cycles), 214) - for k, v in known_cycles_percentages.items(): - self.assertIn(k, report_cycles) - self.assertEqual(v, report_cycles[k]) - - known_instrunctions_percentages = { - '0x0000115bb6c35d7a': 1.65, - '0x0000115bb7ba9b54': 0.67, - '0x0000000000024f56': 0.00, - '0xffffffffa4a0ee03': 0.00, - } - report_instructions = report['instructions'] - self.assertEqual(len(report_instructions), 492) - for k, v in known_instrunctions_percentages.items(): - self.assertIn(k, report_instructions) - self.assertEqual(v, report_instructions[k]) - - -if __name__ == '__main__': - test_flag.SetTestMode(True) - unittest.main() + """Tests for the perf report parser in results_report.""" + + @staticmethod + def _ReadRealPerfReport(): + my_dir = os.path.dirname(os.path.realpath(__file__)) + with open(os.path.join(my_dir, "perf_files/perf.data.report.0")) as f: + return f.read() + + def testParserParsesRealWorldPerfReport(self): + report = ParseStandardPerfReport(self._ReadRealPerfReport()) + self.assertCountEqual(["cycles", "instructions"], list(report.keys())) + + # Arbitrarily selected known percentages from the perf report. + known_cycles_percentages = { + "0xffffffffa4a1f1c9": 0.66, + "0x0000115bb7ba9b54": 0.47, + "0x0000000000082e08": 0.00, + "0xffffffffa4a13e63": 0.00, + } + report_cycles = report["cycles"] + self.assertEqual(len(report_cycles), 214) + for k, v in known_cycles_percentages.items(): + self.assertIn(k, report_cycles) + self.assertEqual(v, report_cycles[k]) + + known_instrunctions_percentages = { + "0x0000115bb6c35d7a": 1.65, + "0x0000115bb7ba9b54": 0.67, + "0x0000000000024f56": 0.00, + "0xffffffffa4a0ee03": 0.00, + } + report_instructions = report["instructions"] + self.assertEqual(len(report_instructions), 492) + for k, v in known_instrunctions_percentages.items(): + self.assertIn(k, report_instructions) + self.assertEqual(v, report_instructions[k]) + + +if __name__ == "__main__": + test_flag.SetTestMode(True) + unittest.main() |