Add json report archiving.

This CL adds a new output, in json format, for archiving test
results.  It also adds a new flag, --json_report, to crosperf to
tell it to generate the json output file in the usual results
directory.  The json output is always in addition to the other
reports that Crosperf generates.  This CL also make some minor
changes to the tool that maintains the telemetry default results
file, so it can be used by the json report generator.

BUG=None
TEST=Tested Crosperf with & without new flag; it did what it should.

Change-Id: Id0e476716ceee208d33e6f9427ea95235fbf11d4
Reviewed-on: https://chrome-internal-review.googlesource.com/228325
Reviewed-by: Han Shen <shenhan@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>

4 files changed, 127 insertions, 2 deletions

diff --git a/crosperf/crosperf.py b/crosperf/crosperf.py
index b0f0b0c9..1e934e87 100755
--- a/crosperf/crosperf.py
+++ b/crosperf/crosperf.py
@@ -100,12 +100,15 @@ def Main(argv):
                                                  working_directory,
                                                  log_dir)
 
+  json_report = experiment_file.GetGlobalSettings().GetField("json_report")
+
   atexit.register(Cleanup, experiment)
 
   if options.dry_run:
     runner = MockExperimentRunner(experiment)
   else:
-    runner = ExperimentRunner(experiment, using_schedv2=options.schedv2)
+    runner = ExperimentRunner(experiment, json_report,
+                              using_schedv2=options.schedv2)
 
   runner.Run()
 
diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py
index 90c07e77..0d7a0262 100644
--- a/crosperf/experiment_runner.py
+++ b/crosperf/experiment_runner.py
@@ -25,6 +25,7 @@ import config
 from experiment_status import ExperimentStatus
 from results_report import HTMLResultsReport
 from results_report import TextResultsReport
+from results_report import JSONResultsReport
 
 
 class ExperimentRunner(object):
@@ -33,11 +34,13 @@ class ExperimentRunner(object):
   STATUS_TIME_DELAY = 30
   THREAD_MONITOR_DELAY = 2
 
-  def __init__(self, experiment, using_schedv2=False, log=None, cmd_exec=None):
+  def __init__(self, experiment, json_report, using_schedv2=False, log=None,
+               cmd_exec=None):
     self._experiment = experiment
     self.l = log or logger.GetLogger(experiment.log_dir)
     self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l)
     self._terminated = False
+    self.json_report = json_report
     self.locked_machines = []
     if experiment.log_level != "verbose":
       self.STATUS_TIME_DELAY = 10
@@ -211,6 +214,8 @@ class ExperimentRunner(object):
     self.l.LogOutput("Storing results report in %s." % results_directory)
     results_table_path = os.path.join(results_directory, "results.html")
     report = HTMLResultsReport(experiment).GetReport()
+    if self.json_report:
+      JSONResultsReport(experiment).GetReport(results_directory)
     FileUtils().WriteFile(results_table_path, report)
 
     self.l.LogOutput("Storing email message body in %s." % results_directory)
diff --git a/crosperf/results_report.py b/crosperf/results_report.py
index e74e3b4b..97d15a54 100644
--- a/crosperf/results_report.py
+++ b/crosperf/results_report.py
@@ -4,8 +4,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import datetime
+import json
+import os
+
 from utils.tabulator import *
 
+from update_telemetry_defaults import TelemetryDefaults
 from column_chart import ColumnChart
 from results_organizer import ResultOrganizer
 from perf_table import PerfTable
@@ -511,3 +516,112 @@ pre {
         if chart:
           charts.append(chart)
     return charts
+
+class JSONResultsReport(ResultsReport):
+
+  def __init__(self, experiment, date=None, time=None):
+    super(JSONResultsReport, self).__init__(experiment)
+    self.ro = ResultOrganizer(experiment.benchmark_runs,
+                              experiment.labels,
+                              experiment.benchmarks)
+    self.date = date
+    self.time = time
+    self.defaults = TelemetryDefaults()
+    if not self.date:
+      timestamp = datetime.datetime.strftime(datetime.datetime.now(),
+                                           "%Y-%m-%d %H:%M:%S")
+      date, time = timestamp.split(" ")
+      self.date = date
+      self.time = time
+
+
+  def _ParseChromeosImage(self, chromeos_image):
+    """Parse the chromeos_image string for the image and version.
+
+    The chromeos_image string will probably be in one of two formats:
+    1:  <path-to-chroot>/src/build/images/<board>/<ChromeOS-version>.<datetime>/chromiumos_test_image.bin
+    2: <path-to-chroot>/chroot/tmp/<buildbot-build>/<ChromeOS-version>/chromiumos_test_image.bin
+
+    We parse these strings to find the 'chromeos_version' to store in the
+    json archive (without the .datatime bit in the first case); and also
+    the 'chromeos_image', which would be all of the first case, but only the
+    part after '/chroot/tmp' in the second case.
+
+    Args:
+      chromeos_image:  String containing the path to the chromeos_image that
+        crosperf used for the test.
+
+    Returns:
+      version, image:  The results of parsing the input string, as explained
+        above.
+    """
+    version = ''
+    real_file = os.path.realpath(os.path.expanduser(chromeos_image))
+    pieces = real_file.split('/')
+    # Find the Chromeos Version, e.g. R45-2345.0.0.....
+    # chromeos_image should have been something like:
+    # <path>/<board-trybot-release>/<chromeos-version>/chromiumos_test_image.bin"
+    num_pieces = len(pieces)
+    if pieces[num_pieces-1] == "chromiumos_test_image.bin":
+      version = pieces[num_pieces-2]
+      # Find last '.' in the version and chop it off (removing the .datatime
+      # piece from local builds).
+      loc = version.rfind('.')
+      version = version[:loc]
+    # Find the chromeos image.  If it's somewhere in .../chroot/tmp/..., then
+    # it's an official image that got downloaded, so chop off the download path
+    # to make the official image name more clear.
+    loc = real_file.find('/chroot/tmp')
+    if loc != -1:
+      loc += len('/chroot/tmp')
+      real_file = real_file[loc:]
+    image = real_file
+    return version,image
+
+  def GetReport(self, results_dir):
+    self.defaults.ReadDefaultsFile()
+    final_results = []
+    board = self.experiment.labels[0].board
+    for test, test_results in self.ro.result.iteritems():
+      for i, label in enumerate(self.ro.labels):
+        label_results = test_results[i]
+        for j, iter_Results in enumerate(label_results):
+          iter_results = label_results[j]
+          json_results = dict()
+          json_results['date'] = self.date
+          json_results['time'] = self.time
+          json_results['board'] = board
+          for l in self.experiment.labels:
+            if l.name == label:
+              ver, img = self._ParseChromeosImage(l.chromeos_image)
+              json_results['chromeos_image'] = img
+              json_results['chromeos_version'] = ver
+              break
+          json_results['test_name'] = test
+          if iter_results['retval'] != 0:
+            json_results['pass'] = False
+          else:
+            json_results['pass'] = True
+            # Get overall results.
+            if test in self.defaults._defaults:
+              default_result_fields = self.defaults._defaults[test]
+              value = []
+              for f in default_result_fields:
+                item = (f, float(iter_results[f][0]))
+                value.append(item)
+              json_results['overall_result'] = value
+            # Get detailed results.
+            detail_results = dict()
+            for k in iter_results.keys():
+              if k != 'retval' and type(iter_results[k]) == list:
+                v_list = iter_results[k]
+                v = v_list[0]
+                detail_results[k] = float(v)
+            json_results['detailed_results'] = detail_results
+          final_results.append(json_results)
+
+    filename = "report_%s_%s_%s.json" % (board, self.date,
+                                         self.time.replace(':','.'))
+    fullname = os.path.join(results_dir, filename)
+    with open(fullname, "w") as fp:
+      json.dump(final_results, fp, indent=2)
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index bf15719a..22849ba9 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -135,6 +135,9 @@ class GlobalSettings(Settings):
     self.AddField(BooleanField("no_email", default=False,
                                description="Whether to disable the email to "
                                "user after crosperf finishes."))
+    self.AddField(BooleanField("json_report", default=False,
+                               description="Whether to generate a json version"
+                               " of the report, for archiving."))
     self.AddField(BooleanField("show_all_results", default=False,
                                description="When running Telemetry tests, "
                                "whether to all the results, instead of just "