crosperf: refactor results_organizer; NFC.

As part of playing with results_report (so I could make it less
ChromeOS-dependent), I started refactoring results_organizer, which is
only used by results_report.

This was the result. The goal was to make it easier to understand what,
exactly, results_organizer is trying to do.

BUG=chromium:641098
TEST=./test_all.sh passes.

Change-Id: I8e53175fbc1ded6f23631550223905b3321376ca
Reviewed-on: https://chrome-internal-review.googlesource.com/280815
Commit-Ready: George Burgess <gbiv@google.com>
Tested-by: George Burgess <gbiv@google.com>
Reviewed-by: Caroline Tice <cmtice@google.com>

1 files changed, 160 insertions, 118 deletions

diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py
index 39554c41..097c744d 100644
--- a/crosperf/results_organizer.py
+++ b/crosperf/results_organizer.py
@@ -5,6 +5,7 @@
 
 from __future__ import print_function
 
+import errno
 import json
 import os
 import re
@@ -12,10 +13,129 @@ import sys
 
 from cros_utils import misc
 
-TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json'
+_TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json'
+_DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}')
 
 
-class ResultOrganizer(object):
+def _AdjustIteration(benchmarks, max_dup, bench):
+  """Adjust the interation numbers if they have keys like ABCD{i}."""
+  for benchmark in benchmarks:
+    if benchmark.name != bench or benchmark.iteration_adjusted:
+      continue
+    benchmark.iteration_adjusted = True
+    benchmark.iterations *= (max_dup + 1)
+
+
+def _GetMaxDup(data):
+  """Find the maximum i inside ABCD{i}.
+
+  data should be a [[[Key]]], where Key is a string that may look like
+  ABCD{i}.
+  """
+  max_dup = 0
+  for label in data:
+    for run in label:
+      for key in run:
+        match = _DUP_KEY_REGEX.match(key)
+        if match:
+          max_dup = max(max_dup, int(match.group(2)))
+  return max_dup
+
+
+def _Repeat(func, times):
+  """Returns the result of running func() n times."""
+  return [func() for _ in xrange(times)]
+
+
+def _GetNonDupLabel(max_dup, runs):
+  """Create new list for the runs of the same label.
+
+  Specifically, this will split out keys like foo{0}, foo{1} from one run into
+  their own runs. For example, given a run like:
+    {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5}
+
+  You'll get:
+    [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}]
+
+  Hands back the lists of transformed runs, all concatenated together.
+  """
+  new_runs = []
+  for run in runs:
+    new_run = {}
+    added_runs = _Repeat(dict, max_dup)
+    for key, value in run.iteritems():
+      match = _DUP_KEY_REGEX.match(key)
+      if not match:
+        new_run[key] = value
+      else:
+        new_key, index_str = match.groups()
+        added_runs[int(index_str)-1][new_key] = str(value)
+    new_runs.append(new_run)
+    new_runs += added_runs
+  return new_runs
+
+
+def _DuplicatePass(result, benchmarks):
+  """Properly expands keys like `foo{1}` in `result`."""
+  for bench, data in result.iteritems():
+    max_dup = _GetMaxDup(data)
+    # If there's nothing to expand, there's nothing to do.
+    if not max_dup:
+      continue
+    for i, runs in enumerate(data):
+      data[i] = _GetNonDupLabel(max_dup, runs)
+    _AdjustIteration(benchmarks, max_dup, bench)
+
+
+def _ReadSummaryFile(filename):
+  """Reads the summary file at filename."""
+  dirname, _ = misc.GetRoot(filename)
+  fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE)
+  try:
+    # Slurp the summary file into a dictionary. The keys in the dictionary are
+    # the benchmark names. The value for a key is a list containing the names
+    # of all the result fields that should be returned in a 'default' report.
+    with open(fullname) as in_file:
+      return json.load(in_file)
+  except IOError as e:
+    # ENOENT means "no such file or directory"
+    if e.errno == errno.ENOENT:
+      return {}
+    raise
+
+
+def _MakeOrganizeResultOutline(benchmark_runs, labels):
+  """Creates the "outline" of the OrganizeResults result for a set of runs.
+
+  Report generation returns lists of different sizes, depending on the input
+  data. Depending on the order in which we iterate through said input data, we
+  may populate the Nth index of a list, then the N-1st, then the N+Mth, ...
+
+  It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't
+  have to worry about resizing while computing results.
+  """
+  # Count how many iterations exist for each benchmark run.
+  # We can't simply count up, since we may be given an incomplete set of
+  # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3])
+  iteration_count = {}
+  for run in benchmark_runs:
+    name = run.benchmark.name
+    old_iterations = iteration_count.get(name, -1)
+    # N.B. run.iteration starts at 1, not 0.
+    iteration_count[name] = max(old_iterations, run.iteration)
+
+  # Result structure: {benchmark_name: [[{key: val}]]}
+  result = {}
+  for run in benchmark_runs:
+    name = run.benchmark.name
+    num_iterations = iteration_count[name]
+    # default param makes cros lint be quiet about defining num_iterations in a
+    # loop.
+    make_dicts = lambda n=num_iterations: _Repeat(dict, n)
+    result[name] = _Repeat(make_dicts, len(labels))
+  return result
+
+def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False):
   """Create a dict from benchmark_runs.
 
   The structure of the output dict is as follows:
@@ -29,122 +149,44 @@ class ResultOrganizer(object):
     [
     ]}.
   """
+  result = _MakeOrganizeResultOutline(benchmark_runs, labels)
+  label_names = [label.name for label in labels]
+  label_indices = {name: i for i, name in enumerate(label_names)}
+  summary_file = _ReadSummaryFile(sys.argv[0])
+  if benchmarks is None:
+    benchmarks = []
 
-  def __init__(self,
-               benchmark_runs,
-               labels,
-               benchmarks=None,
-               json_report=False):
-    self.result = {}
-    self.labels = []
-    self.prog = re.compile(r'(\w+)\{(\d+)\}')
-    self.benchmarks = benchmarks
-    if not self.benchmarks:
-      self.benchmarks = []
-    for label in labels:
-      self.labels.append(label.name)
-    for benchmark_run in benchmark_runs:
-      benchmark_name = benchmark_run.benchmark.name
-      if json_report:
-        show_all_results = True
+  for benchmark_run in benchmark_runs:
+    if not benchmark_run.result:
+      continue
+    benchmark = benchmark_run.benchmark
+    label_index = label_indices[benchmark_run.label.name]
+    cur_label_list = result[benchmark.name][label_index]
+    cur_dict = cur_label_list[benchmark_run.iteration - 1]
+
+    show_all_results = json_report or benchmark.show_all_results
+    if not show_all_results:
+      summary_list = summary_file.get(benchmark.test_name)
+      if summary_list:
+        summary_list.append('retval')
       else:
-        show_all_results = benchmark_run.benchmark.show_all_results
-      if benchmark_name not in self.result:
-        self.result[benchmark_name] = []
-        while len(self.result[benchmark_name]) < len(labels):
-          self.result[benchmark_name].append([])
-      label_index = self.labels.index(benchmark_run.label.name)
-      cur_table = self.result[benchmark_name][label_index]
-      index = benchmark_run.iteration - 1
-      while index >= len(cur_table):
-        cur_table.append({})
-      cur_dict = cur_table[index]
-      if not benchmark_run.result:
-        continue
-      benchmark = benchmark_run.benchmark
-      if not show_all_results:
-        summary_list = self._GetSummaryResults(benchmark.test_name)
-        if len(summary_list) > 0:
-          summary_list.append('retval')
-        else:
-          # Did not find test_name in json file; therefore show everything.
-          show_all_results = True
-      for test_key in benchmark_run.result.keyvals:
-        if not show_all_results and not test_key in summary_list:
-          continue
+        # Did not find test_name in json file; show everything.
+        show_all_results = True
+    for test_key in benchmark_run.result.keyvals:
+      if show_all_results or test_key in summary_list:
         cur_dict[test_key] = benchmark_run.result.keyvals[test_key]
-      # Occasionally Telemetry tests will not fail but they will not return a
-      # result, either.  Look for those cases, and force them to be a fail.
-      # (This can happen if, for example, the test has been disabled.)
-      if len(cur_dict) == 1 and cur_dict['retval'] == 0:
-        cur_dict['retval'] = 1
-        # TODO: This output should be sent via logger.
-        print("WARNING: Test '%s' appears to have succeeded but returned"
-              ' no results.' % benchmark_name,
-              file=sys.stderr)
-      if json_report and benchmark_run.machine:
-        cur_dict['machine'] = benchmark_run.machine.name
-        cur_dict['machine_checksum'] = benchmark_run.machine.checksum
-        cur_dict['machine_string'] = benchmark_run.machine.checksum_string
-    self._DuplicatePass()
-
-  def _GetSummaryResults(self, test_name):
-    dirname, _ = misc.GetRoot(sys.argv[0])
-    fullname = os.path.join(dirname, TELEMETRY_RESULT_DEFAULTS_FILE)
-    if os.path.exists(fullname):
-      # Slurp the file into a dictionary.  The keys in the dictionary are
-      # the benchmark names.  The value for a key is a list containing the
-      # names of all the result fields that should be returned in a 'default'
-      # report.
-      result_defaults = json.load(open(fullname))
-      # Check to see if the current benchmark test actually has an entry in
-      # the dictionary.
-      if test_name in result_defaults:
-        return result_defaults[test_name]
-      else:
-        return []
-
-  def _DuplicatePass(self):
-    for bench, data in self.result.items():
-      max_dup = self._GetMaxDup(data)
-      if not max_dup:
-        continue
-      for label in data:
-        index = data.index(label)
-        data[index] = self._GetNonDupLabel(max_dup, label)
-      self._AdjustIteration(max_dup, bench)
-
-  def _GetMaxDup(self, data):
-    """Find the maximum i inside ABCD{i}."""
-    max_dup = 0
-    for label in data:
-      for run in label:
-        for key in run:
-          if re.match(self.prog, key):
-            max_dup = max(max_dup, int(re.search(self.prog, key).group(2)))
-    return max_dup
-
-  def _GetNonDupLabel(self, max_dup, label):
-    """Create new list for the runs of the same label."""
-    new_label = []
-    for run in label:
-      start_index = len(new_label)
-      new_label.append(dict(run))
-      for _ in range(max_dup):
-        new_label.append({})
-      new_run = new_label[start_index]
-      for key, value in new_run.items():
-        if re.match(self.prog, key):
-          new_key = re.search(self.prog, key).group(1)
-          index = int(re.search(self.prog, key).group(2))
-          new_label[start_index + index][new_key] = str(value)
-          del new_run[key]
-    return new_label
-
-  def _AdjustIteration(self, max_dup, bench):
-    """Adjust the interation numbers if the have keys like ABCD{i}."""
-    for benchmark in self.benchmarks:
-      if benchmark.name == bench:
-        if not benchmark.iteration_adjusted:
-          benchmark.iteration_adjusted = True
-          benchmark.iterations *= (max_dup + 1)
+    # Occasionally Telemetry tests will not fail but they will not return a
+    # result, either.  Look for those cases, and force them to be a fail.
+    # (This can happen if, for example, the test has been disabled.)
+    if len(cur_dict) == 1 and cur_dict['retval'] == 0:
+      cur_dict['retval'] = 1
+      # TODO: This output should be sent via logger.
+      print("WARNING: Test '%s' appears to have succeeded but returned"
+            ' no results.' % benchmark.name,
+            file=sys.stderr)
+    if json_report and benchmark_run.machine:
+      cur_dict['machine'] = benchmark_run.machine.name
+      cur_dict['machine_checksum'] = benchmark_run.machine.checksum
+      cur_dict['machine_string'] = benchmark_run.machine.checksum_string
+  _DuplicatePass(result, benchmarks)
+  return result