crosperf: Add cpu statistics into results report

Check for turbostat.log file in results directory and if file exists
parse the context and extract measurements of CPU Frequency and
temperature for ALL cores (not each).
Keys added to keyvals: 'cpufreq_min', 'cpufreq_max', 'cpufreq_avg',
'cputemp'.

BUG=chromium:966514
TEST=tested all crosbolt perf benchmarks

Change-Id: If90d26664c70ccb73750f17573b89933fdb048f4
Reviewed-on: https://chromium-review.googlesource.com/1709824
Tested-by: Denis Nikitin <denik@chromium.org>
Commit-Ready: Denis Nikitin <denik@chromium.org>
Legacy-Commit-Queue: Commit Bot <commit-bot@chromium.org>
Reviewed-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>

1 files changed, 80 insertions, 1 deletions

diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 4db15901..178c0860 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -55,6 +55,7 @@ class Result(object):
     self.perf_data_files = []
     self.perf_report_files = []
     self.results_file = []
+    self.turbostat_log_file = []
     self.chrome_version = ''
     self.err = None
     self.chroot_results_dir = ''
@@ -65,6 +66,8 @@ class Result(object):
     self.cwp_dso = ''
     self.retval = None
     self.out = None
+    self.cpufreq = []
+    self.cputemp = []
 
   def CopyFilesTo(self, dest_dir, files_to_copy):
     file_index = 0
@@ -245,7 +248,7 @@ class Result(object):
 
   def FindFilesInResultsDir(self, find_args):
     if not self.results_dir:
-      return None
+      return ''
 
     command = 'find %s %s' % (self.results_dir, find_args)
     ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False)
@@ -277,6 +280,9 @@ class Result(object):
             self.FindFilesInResultsDir('-name results-chart.json').splitlines()
     return result
 
+  def GetTurbostatFile(self):
+    return self.FindFilesInResultsDir('-name turbostat.log').split('\n')[0]
+
   def _CheckDebugPath(self, option, path):
     relative_path = path[1:]
     out_chroot_path = os.path.join(self.chromeos_root, 'chroot', relative_path)
@@ -373,6 +379,7 @@ class Result(object):
     self.perf_data_files = self.GetPerfDataFiles()
     # Include all perf.report data in table.
     self.perf_report_files = self.GeneratePerfReportFiles()
+    self.turbostat_log_file = self.GetTurbostatFile()
     # TODO(asharif): Do something similar with perf stat.
 
     # Grab keyvals from the directory.
@@ -416,6 +423,69 @@ class Result(object):
           keyvals[keyname] = new_value
     return keyvals
 
+  def ProcessTurbostatResults(self):
+    """Given turbostat_log_file parse cpu stats from file."""
+    if not self.turbostat_log_file:
+      self._logger.LogError('Turbostat output file not found.')
+      return {}
+
+    cpustats = {}
+    read_data = ''
+    with open(self.turbostat_log_file) as f:
+      read_data = f.readlines()
+
+    if not read_data:
+      self._logger.LogError('Turbostat output file is empty.')
+      return {}
+
+    # Gather CPU statistics from turbostat output.
+    # Example of the output.
+    # ----------------------
+    # CPU     Avg_MHz Busy%   Bzy_MHz TSC_MHz IRQ     CoreTmp
+    # -       329     12.13   2723    2393    10975   77
+    # 0       336     12.41   2715    2393    6328    77
+    # 2       323     11.86   2731    2393    4647    69
+    # CPU     Avg_MHz Busy%   Bzy_MHz TSC_MHz IRQ     CoreTmp
+    # -       1940    67.46   2884    2393    39920   83
+    # 0       1827    63.70   2877    2393    21184   83
+
+    # First line always contains the header.
+    stats = read_data[0].split()
+
+    # Mandatory parameters.
+    if 'CPU' not in stats:
+      self._logger.LogError('Missing data for CPU# in Turbostat output.')
+      return {}
+    if 'Bzy_MHz' not in stats:
+      self._logger.LogError('Missing data for Bzy_MHz in Turbostat output.')
+      return {}
+    cpu_index = stats.index('CPU')
+    cpufreq_index = stats.index('Bzy_MHz')
+    cpustats['freq'] = []
+
+    # Optional parameters.
+    cputemp_index = -1
+    if 'CoreTmp' in stats:
+      cputemp_index = stats.index('CoreTmp')
+      cpustats['temp'] = []
+
+    # Parse data starting from the second line ignoring repeating headers.
+    for st in read_data[1:]:
+      # Data represented by int or float separated by spaces.
+      numbers = st.split()
+      if not all(word.replace('.', '', 1).isdigit() for word in numbers[1:]):
+        # Skip the line if data mismatch.
+        continue
+      if numbers[cpu_index] != '-':
+        # Ignore Core-specific statistics which starts with Core number.
+        # Combined statistics for all core has "-" CPU identifier.
+        continue
+
+      cpustats['freq'].append(int(numbers[cpufreq_index]))
+      if cputemp_index != -1:
+        cpustats['temp'].append(int(numbers[cputemp_index]))
+    return cpustats
+
   def ProcessHistogramsResults(self):
     # Open and parse the json results file generated by telemetry/test_that.
     if not self.results_file:
@@ -479,6 +549,15 @@ class Result(object):
     # Generate report from all perf.data files.
     # Now parse all perf report files and include them in keyvals.
     self.GatherPerfResults()
+    cpustats = self.ProcessTurbostatResults()
+    if 'freq' in cpustats:
+      self.cpufreq = cpustats['freq']
+      self.keyvals['cpufreq_avg'] = sum(self.cpufreq) / len(self.cpufreq)
+      self.keyvals['cpufreq_min'] = min(self.cpufreq)
+      self.keyvals['cpufreq_max'] = max(self.cpufreq)
+    if 'temp' in cpustats:
+      self.cputemp = cpustats['temp']
+      self.keyvals['cputemp'] = sum(self.cputemp) / len(self.cputemp)
 
   def GetChromeVersionFromCache(self, cache_dir):
     # Read chrome_version from keys file, if present.