diff options
Diffstat (limited to 'crosperf')
-rw-r--r-- | crosperf/results_organizer.py | 278 | ||||
-rwxr-xr-x | crosperf/results_organizer_unittest.py | 6 | ||||
-rw-r--r-- | crosperf/results_report.py | 29 |
3 files changed, 178 insertions, 135 deletions
diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py index 39554c41..097c744d 100644 --- a/crosperf/results_organizer.py +++ b/crosperf/results_organizer.py @@ -5,6 +5,7 @@ from __future__ import print_function +import errno import json import os import re @@ -12,10 +13,129 @@ import sys from cros_utils import misc -TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json' +_TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json' +_DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}') -class ResultOrganizer(object): +def _AdjustIteration(benchmarks, max_dup, bench): + """Adjust the interation numbers if they have keys like ABCD{i}.""" + for benchmark in benchmarks: + if benchmark.name != bench or benchmark.iteration_adjusted: + continue + benchmark.iteration_adjusted = True + benchmark.iterations *= (max_dup + 1) + + +def _GetMaxDup(data): + """Find the maximum i inside ABCD{i}. + + data should be a [[[Key]]], where Key is a string that may look like + ABCD{i}. + """ + max_dup = 0 + for label in data: + for run in label: + for key in run: + match = _DUP_KEY_REGEX.match(key) + if match: + max_dup = max(max_dup, int(match.group(2))) + return max_dup + + +def _Repeat(func, times): + """Returns the result of running func() n times.""" + return [func() for _ in xrange(times)] + + +def _GetNonDupLabel(max_dup, runs): + """Create new list for the runs of the same label. + + Specifically, this will split out keys like foo{0}, foo{1} from one run into + their own runs. For example, given a run like: + {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5} + + You'll get: + [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}] + + Hands back the lists of transformed runs, all concatenated together. + """ + new_runs = [] + for run in runs: + new_run = {} + added_runs = _Repeat(dict, max_dup) + for key, value in run.iteritems(): + match = _DUP_KEY_REGEX.match(key) + if not match: + new_run[key] = value + else: + new_key, index_str = match.groups() + added_runs[int(index_str)-1][new_key] = str(value) + new_runs.append(new_run) + new_runs += added_runs + return new_runs + + +def _DuplicatePass(result, benchmarks): + """Properly expands keys like `foo{1}` in `result`.""" + for bench, data in result.iteritems(): + max_dup = _GetMaxDup(data) + # If there's nothing to expand, there's nothing to do. + if not max_dup: + continue + for i, runs in enumerate(data): + data[i] = _GetNonDupLabel(max_dup, runs) + _AdjustIteration(benchmarks, max_dup, bench) + + +def _ReadSummaryFile(filename): + """Reads the summary file at filename.""" + dirname, _ = misc.GetRoot(filename) + fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE) + try: + # Slurp the summary file into a dictionary. The keys in the dictionary are + # the benchmark names. The value for a key is a list containing the names + # of all the result fields that should be returned in a 'default' report. + with open(fullname) as in_file: + return json.load(in_file) + except IOError as e: + # ENOENT means "no such file or directory" + if e.errno == errno.ENOENT: + return {} + raise + + +def _MakeOrganizeResultOutline(benchmark_runs, labels): + """Creates the "outline" of the OrganizeResults result for a set of runs. + + Report generation returns lists of different sizes, depending on the input + data. Depending on the order in which we iterate through said input data, we + may populate the Nth index of a list, then the N-1st, then the N+Mth, ... + + It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't + have to worry about resizing while computing results. + """ + # Count how many iterations exist for each benchmark run. + # We can't simply count up, since we may be given an incomplete set of + # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3]) + iteration_count = {} + for run in benchmark_runs: + name = run.benchmark.name + old_iterations = iteration_count.get(name, -1) + # N.B. run.iteration starts at 1, not 0. + iteration_count[name] = max(old_iterations, run.iteration) + + # Result structure: {benchmark_name: [[{key: val}]]} + result = {} + for run in benchmark_runs: + name = run.benchmark.name + num_iterations = iteration_count[name] + # default param makes cros lint be quiet about defining num_iterations in a + # loop. + make_dicts = lambda n=num_iterations: _Repeat(dict, n) + result[name] = _Repeat(make_dicts, len(labels)) + return result + +def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False): """Create a dict from benchmark_runs. The structure of the output dict is as follows: @@ -29,122 +149,44 @@ class ResultOrganizer(object): [ ]}. """ + result = _MakeOrganizeResultOutline(benchmark_runs, labels) + label_names = [label.name for label in labels] + label_indices = {name: i for i, name in enumerate(label_names)} + summary_file = _ReadSummaryFile(sys.argv[0]) + if benchmarks is None: + benchmarks = [] - def __init__(self, - benchmark_runs, - labels, - benchmarks=None, - json_report=False): - self.result = {} - self.labels = [] - self.prog = re.compile(r'(\w+)\{(\d+)\}') - self.benchmarks = benchmarks - if not self.benchmarks: - self.benchmarks = [] - for label in labels: - self.labels.append(label.name) - for benchmark_run in benchmark_runs: - benchmark_name = benchmark_run.benchmark.name - if json_report: - show_all_results = True + for benchmark_run in benchmark_runs: + if not benchmark_run.result: + continue + benchmark = benchmark_run.benchmark + label_index = label_indices[benchmark_run.label.name] + cur_label_list = result[benchmark.name][label_index] + cur_dict = cur_label_list[benchmark_run.iteration - 1] + + show_all_results = json_report or benchmark.show_all_results + if not show_all_results: + summary_list = summary_file.get(benchmark.test_name) + if summary_list: + summary_list.append('retval') else: - show_all_results = benchmark_run.benchmark.show_all_results - if benchmark_name not in self.result: - self.result[benchmark_name] = [] - while len(self.result[benchmark_name]) < len(labels): - self.result[benchmark_name].append([]) - label_index = self.labels.index(benchmark_run.label.name) - cur_table = self.result[benchmark_name][label_index] - index = benchmark_run.iteration - 1 - while index >= len(cur_table): - cur_table.append({}) - cur_dict = cur_table[index] - if not benchmark_run.result: - continue - benchmark = benchmark_run.benchmark - if not show_all_results: - summary_list = self._GetSummaryResults(benchmark.test_name) - if len(summary_list) > 0: - summary_list.append('retval') - else: - # Did not find test_name in json file; therefore show everything. - show_all_results = True - for test_key in benchmark_run.result.keyvals: - if not show_all_results and not test_key in summary_list: - continue + # Did not find test_name in json file; show everything. + show_all_results = True + for test_key in benchmark_run.result.keyvals: + if show_all_results or test_key in summary_list: cur_dict[test_key] = benchmark_run.result.keyvals[test_key] - # Occasionally Telemetry tests will not fail but they will not return a - # result, either. Look for those cases, and force them to be a fail. - # (This can happen if, for example, the test has been disabled.) - if len(cur_dict) == 1 and cur_dict['retval'] == 0: - cur_dict['retval'] = 1 - # TODO: This output should be sent via logger. - print("WARNING: Test '%s' appears to have succeeded but returned" - ' no results.' % benchmark_name, - file=sys.stderr) - if json_report and benchmark_run.machine: - cur_dict['machine'] = benchmark_run.machine.name - cur_dict['machine_checksum'] = benchmark_run.machine.checksum - cur_dict['machine_string'] = benchmark_run.machine.checksum_string - self._DuplicatePass() - - def _GetSummaryResults(self, test_name): - dirname, _ = misc.GetRoot(sys.argv[0]) - fullname = os.path.join(dirname, TELEMETRY_RESULT_DEFAULTS_FILE) - if os.path.exists(fullname): - # Slurp the file into a dictionary. The keys in the dictionary are - # the benchmark names. The value for a key is a list containing the - # names of all the result fields that should be returned in a 'default' - # report. - result_defaults = json.load(open(fullname)) - # Check to see if the current benchmark test actually has an entry in - # the dictionary. - if test_name in result_defaults: - return result_defaults[test_name] - else: - return [] - - def _DuplicatePass(self): - for bench, data in self.result.items(): - max_dup = self._GetMaxDup(data) - if not max_dup: - continue - for label in data: - index = data.index(label) - data[index] = self._GetNonDupLabel(max_dup, label) - self._AdjustIteration(max_dup, bench) - - def _GetMaxDup(self, data): - """Find the maximum i inside ABCD{i}.""" - max_dup = 0 - for label in data: - for run in label: - for key in run: - if re.match(self.prog, key): - max_dup = max(max_dup, int(re.search(self.prog, key).group(2))) - return max_dup - - def _GetNonDupLabel(self, max_dup, label): - """Create new list for the runs of the same label.""" - new_label = [] - for run in label: - start_index = len(new_label) - new_label.append(dict(run)) - for _ in range(max_dup): - new_label.append({}) - new_run = new_label[start_index] - for key, value in new_run.items(): - if re.match(self.prog, key): - new_key = re.search(self.prog, key).group(1) - index = int(re.search(self.prog, key).group(2)) - new_label[start_index + index][new_key] = str(value) - del new_run[key] - return new_label - - def _AdjustIteration(self, max_dup, bench): - """Adjust the interation numbers if the have keys like ABCD{i}.""" - for benchmark in self.benchmarks: - if benchmark.name == bench: - if not benchmark.iteration_adjusted: - benchmark.iteration_adjusted = True - benchmark.iterations *= (max_dup + 1) + # Occasionally Telemetry tests will not fail but they will not return a + # result, either. Look for those cases, and force them to be a fail. + # (This can happen if, for example, the test has been disabled.) + if len(cur_dict) == 1 and cur_dict['retval'] == 0: + cur_dict['retval'] = 1 + # TODO: This output should be sent via logger. + print("WARNING: Test '%s' appears to have succeeded but returned" + ' no results.' % benchmark.name, + file=sys.stderr) + if json_report and benchmark_run.machine: + cur_dict['machine'] = benchmark_run.machine.name + cur_dict['machine_checksum'] = benchmark_run.machine.checksum + cur_dict['machine_string'] = benchmark_run.machine.checksum_string + _DuplicatePass(result, benchmarks) + return result diff --git a/crosperf/results_organizer_unittest.py b/crosperf/results_organizer_unittest.py index 74d5c23a..0d214124 100755 --- a/crosperf/results_organizer_unittest.py +++ b/crosperf/results_organizer_unittest.py @@ -16,7 +16,7 @@ import unittest from benchmark_run import BenchmarkRun from results_cache import Result -from results_organizer import ResultOrganizer +from results_organizer import OrganizeResults import mock_instance @@ -101,8 +101,8 @@ class ResultOrganizerTest(unittest.TestCase): b.result.keyvals = mock_instance.keyval[i] i += 1 - ro = ResultOrganizer(benchmark_runs, labels, benchmarks) - self.assertEqual(ro.result, result) + organized = OrganizeResults(benchmark_runs, labels, benchmarks) + self.assertEqual(organized, result) if __name__ == '__main__': diff --git a/crosperf/results_report.py b/crosperf/results_report.py index adb85874..825d1d7c 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -30,7 +30,7 @@ from cros_utils.tabulator import TablePrinter from update_telemetry_defaults import TelemetryDefaults from column_chart import ColumnChart -from results_organizer import ResultOrganizer +from results_organizer import OrganizeResults from perf_table import PerfTable @@ -154,9 +154,8 @@ class ResultsReport(object): def _GetTables(self, labels, benchmark_runs, columns, table_type): tables = [] - ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks) - result = ro.result - label_name = ro.labels + result = OrganizeResults(benchmark_runs, labels, self.benchmarks) + label_name = [label.name for label in labels] for item in result: benchmark = None runs = result[item] @@ -500,10 +499,10 @@ pre { def _GetCharts(self, labels, benchmark_runs): charts = [] - ro = ResultOrganizer(benchmark_runs, labels) - result = ro.result + result = OrganizeResults(benchmark_runs, labels) + label_names = [label.name for label in labels] for item, runs in result.iteritems(): - tg = TableGenerator(runs, ro.labels) + tg = TableGenerator(runs, label_names) table = tg.GetTable() columns = [Column(AmeanResult(), Format()), Column(MinResult(), Format()), Column(MaxResult(), Format())] @@ -522,7 +521,7 @@ pre { chart.AddSeries('Min', 'line', 'black') chart.AddSeries('Max', 'line', 'black') cur_index = 1 - for label in ro.labels: + for label in label_names: chart.AddRow([label, cur_row_data[cur_index].value, cur_row_data[ cur_index + 1].value, cur_row_data[cur_index + 2].value]) if isinstance(cur_row_data[cur_index].value, str): @@ -545,10 +544,11 @@ class JSONResultsReport(ResultsReport): def __init__(self, experiment, date=None, time=None): super(JSONResultsReport, self).__init__(experiment) - self.ro = ResultOrganizer(experiment.benchmark_runs, - experiment.labels, - experiment.benchmarks, - json_report=True) + self.label_names = [label.name for label in experiment.labels] + self.organized_result = OrganizeResults(experiment.benchmark_runs, + experiment.labels, + experiment.benchmarks, + json_report=True) self.date = date self.time = time self.defaults = TelemetryDefaults() @@ -567,8 +567,9 @@ class JSONResultsReport(ResultsReport): final_results = [] board = self.experiment.labels[0].board compiler_string = 'gcc' - for test, test_results in self.ro.result.iteritems(): - for label, label_results in itertools.izip(self.ro.labels, test_results): + for test, test_results in self.organized_result.iteritems(): + for label, label_results in itertools.izip(self.label_names, + test_results): for iter_results in label_results: json_results = { 'date': self.date, |