diff options
author | George Burgess IV <gbiv@google.com> | 2016-08-25 11:57:01 -0700 |
---|---|---|
committer | chrome-bot <chrome-bot@chromium.org> | 2016-08-26 17:09:03 -0700 |
commit | afb8cc77e82c35faedfe541d097fc01fd1d7ca3d (patch) | |
tree | cd2473b3f517ca25afe48ff7ef8d9b741d68045e /crosperf/results_organizer.py | |
parent | 2f4a226d58efdf4eabe560a64e711c16599db08d (diff) | |
download | toolchain-utils-afb8cc77e82c35faedfe541d097fc01fd1d7ca3d.tar.gz |
crosperf: refactor results_organizer; NFC.
As part of playing with results_report (so I could make it less
ChromeOS-dependent), I started refactoring results_organizer, which is
only used by results_report.
This was the result. The goal was to make it easier to understand what,
exactly, results_organizer is trying to do.
BUG=chromium:641098
TEST=./test_all.sh passes.
Change-Id: I8e53175fbc1ded6f23631550223905b3321376ca
Reviewed-on: https://chrome-internal-review.googlesource.com/280815
Commit-Ready: George Burgess <gbiv@google.com>
Tested-by: George Burgess <gbiv@google.com>
Reviewed-by: Caroline Tice <cmtice@google.com>
Diffstat (limited to 'crosperf/results_organizer.py')
-rw-r--r-- | crosperf/results_organizer.py | 278 |
1 files changed, 160 insertions, 118 deletions
diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py index 39554c41..097c744d 100644 --- a/crosperf/results_organizer.py +++ b/crosperf/results_organizer.py @@ -5,6 +5,7 @@ from __future__ import print_function +import errno import json import os import re @@ -12,10 +13,129 @@ import sys from cros_utils import misc -TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json' +_TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json' +_DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}') -class ResultOrganizer(object): +def _AdjustIteration(benchmarks, max_dup, bench): + """Adjust the interation numbers if they have keys like ABCD{i}.""" + for benchmark in benchmarks: + if benchmark.name != bench or benchmark.iteration_adjusted: + continue + benchmark.iteration_adjusted = True + benchmark.iterations *= (max_dup + 1) + + +def _GetMaxDup(data): + """Find the maximum i inside ABCD{i}. + + data should be a [[[Key]]], where Key is a string that may look like + ABCD{i}. + """ + max_dup = 0 + for label in data: + for run in label: + for key in run: + match = _DUP_KEY_REGEX.match(key) + if match: + max_dup = max(max_dup, int(match.group(2))) + return max_dup + + +def _Repeat(func, times): + """Returns the result of running func() n times.""" + return [func() for _ in xrange(times)] + + +def _GetNonDupLabel(max_dup, runs): + """Create new list for the runs of the same label. + + Specifically, this will split out keys like foo{0}, foo{1} from one run into + their own runs. For example, given a run like: + {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5} + + You'll get: + [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}] + + Hands back the lists of transformed runs, all concatenated together. + """ + new_runs = [] + for run in runs: + new_run = {} + added_runs = _Repeat(dict, max_dup) + for key, value in run.iteritems(): + match = _DUP_KEY_REGEX.match(key) + if not match: + new_run[key] = value + else: + new_key, index_str = match.groups() + added_runs[int(index_str)-1][new_key] = str(value) + new_runs.append(new_run) + new_runs += added_runs + return new_runs + + +def _DuplicatePass(result, benchmarks): + """Properly expands keys like `foo{1}` in `result`.""" + for bench, data in result.iteritems(): + max_dup = _GetMaxDup(data) + # If there's nothing to expand, there's nothing to do. + if not max_dup: + continue + for i, runs in enumerate(data): + data[i] = _GetNonDupLabel(max_dup, runs) + _AdjustIteration(benchmarks, max_dup, bench) + + +def _ReadSummaryFile(filename): + """Reads the summary file at filename.""" + dirname, _ = misc.GetRoot(filename) + fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE) + try: + # Slurp the summary file into a dictionary. The keys in the dictionary are + # the benchmark names. The value for a key is a list containing the names + # of all the result fields that should be returned in a 'default' report. + with open(fullname) as in_file: + return json.load(in_file) + except IOError as e: + # ENOENT means "no such file or directory" + if e.errno == errno.ENOENT: + return {} + raise + + +def _MakeOrganizeResultOutline(benchmark_runs, labels): + """Creates the "outline" of the OrganizeResults result for a set of runs. + + Report generation returns lists of different sizes, depending on the input + data. Depending on the order in which we iterate through said input data, we + may populate the Nth index of a list, then the N-1st, then the N+Mth, ... + + It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't + have to worry about resizing while computing results. + """ + # Count how many iterations exist for each benchmark run. + # We can't simply count up, since we may be given an incomplete set of + # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3]) + iteration_count = {} + for run in benchmark_runs: + name = run.benchmark.name + old_iterations = iteration_count.get(name, -1) + # N.B. run.iteration starts at 1, not 0. + iteration_count[name] = max(old_iterations, run.iteration) + + # Result structure: {benchmark_name: [[{key: val}]]} + result = {} + for run in benchmark_runs: + name = run.benchmark.name + num_iterations = iteration_count[name] + # default param makes cros lint be quiet about defining num_iterations in a + # loop. + make_dicts = lambda n=num_iterations: _Repeat(dict, n) + result[name] = _Repeat(make_dicts, len(labels)) + return result + +def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False): """Create a dict from benchmark_runs. The structure of the output dict is as follows: @@ -29,122 +149,44 @@ class ResultOrganizer(object): [ ]}. """ + result = _MakeOrganizeResultOutline(benchmark_runs, labels) + label_names = [label.name for label in labels] + label_indices = {name: i for i, name in enumerate(label_names)} + summary_file = _ReadSummaryFile(sys.argv[0]) + if benchmarks is None: + benchmarks = [] - def __init__(self, - benchmark_runs, - labels, - benchmarks=None, - json_report=False): - self.result = {} - self.labels = [] - self.prog = re.compile(r'(\w+)\{(\d+)\}') - self.benchmarks = benchmarks - if not self.benchmarks: - self.benchmarks = [] - for label in labels: - self.labels.append(label.name) - for benchmark_run in benchmark_runs: - benchmark_name = benchmark_run.benchmark.name - if json_report: - show_all_results = True + for benchmark_run in benchmark_runs: + if not benchmark_run.result: + continue + benchmark = benchmark_run.benchmark + label_index = label_indices[benchmark_run.label.name] + cur_label_list = result[benchmark.name][label_index] + cur_dict = cur_label_list[benchmark_run.iteration - 1] + + show_all_results = json_report or benchmark.show_all_results + if not show_all_results: + summary_list = summary_file.get(benchmark.test_name) + if summary_list: + summary_list.append('retval') else: - show_all_results = benchmark_run.benchmark.show_all_results - if benchmark_name not in self.result: - self.result[benchmark_name] = [] - while len(self.result[benchmark_name]) < len(labels): - self.result[benchmark_name].append([]) - label_index = self.labels.index(benchmark_run.label.name) - cur_table = self.result[benchmark_name][label_index] - index = benchmark_run.iteration - 1 - while index >= len(cur_table): - cur_table.append({}) - cur_dict = cur_table[index] - if not benchmark_run.result: - continue - benchmark = benchmark_run.benchmark - if not show_all_results: - summary_list = self._GetSummaryResults(benchmark.test_name) - if len(summary_list) > 0: - summary_list.append('retval') - else: - # Did not find test_name in json file; therefore show everything. - show_all_results = True - for test_key in benchmark_run.result.keyvals: - if not show_all_results and not test_key in summary_list: - continue + # Did not find test_name in json file; show everything. + show_all_results = True + for test_key in benchmark_run.result.keyvals: + if show_all_results or test_key in summary_list: cur_dict[test_key] = benchmark_run.result.keyvals[test_key] - # Occasionally Telemetry tests will not fail but they will not return a - # result, either. Look for those cases, and force them to be a fail. - # (This can happen if, for example, the test has been disabled.) - if len(cur_dict) == 1 and cur_dict['retval'] == 0: - cur_dict['retval'] = 1 - # TODO: This output should be sent via logger. - print("WARNING: Test '%s' appears to have succeeded but returned" - ' no results.' % benchmark_name, - file=sys.stderr) - if json_report and benchmark_run.machine: - cur_dict['machine'] = benchmark_run.machine.name - cur_dict['machine_checksum'] = benchmark_run.machine.checksum - cur_dict['machine_string'] = benchmark_run.machine.checksum_string - self._DuplicatePass() - - def _GetSummaryResults(self, test_name): - dirname, _ = misc.GetRoot(sys.argv[0]) - fullname = os.path.join(dirname, TELEMETRY_RESULT_DEFAULTS_FILE) - if os.path.exists(fullname): - # Slurp the file into a dictionary. The keys in the dictionary are - # the benchmark names. The value for a key is a list containing the - # names of all the result fields that should be returned in a 'default' - # report. - result_defaults = json.load(open(fullname)) - # Check to see if the current benchmark test actually has an entry in - # the dictionary. - if test_name in result_defaults: - return result_defaults[test_name] - else: - return [] - - def _DuplicatePass(self): - for bench, data in self.result.items(): - max_dup = self._GetMaxDup(data) - if not max_dup: - continue - for label in data: - index = data.index(label) - data[index] = self._GetNonDupLabel(max_dup, label) - self._AdjustIteration(max_dup, bench) - - def _GetMaxDup(self, data): - """Find the maximum i inside ABCD{i}.""" - max_dup = 0 - for label in data: - for run in label: - for key in run: - if re.match(self.prog, key): - max_dup = max(max_dup, int(re.search(self.prog, key).group(2))) - return max_dup - - def _GetNonDupLabel(self, max_dup, label): - """Create new list for the runs of the same label.""" - new_label = [] - for run in label: - start_index = len(new_label) - new_label.append(dict(run)) - for _ in range(max_dup): - new_label.append({}) - new_run = new_label[start_index] - for key, value in new_run.items(): - if re.match(self.prog, key): - new_key = re.search(self.prog, key).group(1) - index = int(re.search(self.prog, key).group(2)) - new_label[start_index + index][new_key] = str(value) - del new_run[key] - return new_label - - def _AdjustIteration(self, max_dup, bench): - """Adjust the interation numbers if the have keys like ABCD{i}.""" - for benchmark in self.benchmarks: - if benchmark.name == bench: - if not benchmark.iteration_adjusted: - benchmark.iteration_adjusted = True - benchmark.iterations *= (max_dup + 1) + # Occasionally Telemetry tests will not fail but they will not return a + # result, either. Look for those cases, and force them to be a fail. + # (This can happen if, for example, the test has been disabled.) + if len(cur_dict) == 1 and cur_dict['retval'] == 0: + cur_dict['retval'] = 1 + # TODO: This output should be sent via logger. + print("WARNING: Test '%s' appears to have succeeded but returned" + ' no results.' % benchmark.name, + file=sys.stderr) + if json_report and benchmark_run.machine: + cur_dict['machine'] = benchmark_run.machine.name + cur_dict['machine_checksum'] = benchmark_run.machine.checksum + cur_dict['machine_string'] = benchmark_run.machine.checksum_string + _DuplicatePass(result, benchmarks) + return result |