crosperf: Add ARM CPU stats in crosperf report

Check for cpustats.log file in results_dir and extract
data with CPU frequncy and temperature.
Calculate avg/min/max values from the measurements and
add to result keyvals.

The parser attempts to read turbostat.log first (which comes from
Intel devices along with cpustats.log) and only if it doesn't exist
or empty switches to cpustats.log (usually ARM devices).

BUG=chromium:966514
TEST=tested locally on veyron, scarlet, kevin64 (arm), eve (intel)

Change-Id: I12991b39a0f1acb6197ca663a6876b0f013d2026
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1753164
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Denis Nikitin <denik@chromium.org>

3 files changed, 379 insertions, 75 deletions

diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 178c0860..80727f6f 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -55,7 +55,8 @@ class Result(object):
     self.perf_data_files = []
     self.perf_report_files = []
     self.results_file = []
-    self.turbostat_log_file = []
+    self.turbostat_log_file = ''
+    self.cpustats_log_file = ''
     self.chrome_version = ''
     self.err = None
     self.chroot_results_dir = ''
@@ -66,8 +67,6 @@ class Result(object):
     self.cwp_dso = ''
     self.retval = None
     self.out = None
-    self.cpufreq = []
-    self.cputemp = []
 
   def CopyFilesTo(self, dest_dir, files_to_copy):
     file_index = 0
@@ -281,8 +280,13 @@ class Result(object):
     return result
 
   def GetTurbostatFile(self):
+    """Get turbostat log path string."""
     return self.FindFilesInResultsDir('-name turbostat.log').split('\n')[0]
 
+  def GetCpustatsFile(self):
+    """Get cpustats log path string."""
+    return self.FindFilesInResultsDir('-name cpustats.log').split('\n')[0]
+
   def _CheckDebugPath(self, option, path):
     relative_path = path[1:]
     out_chroot_path = os.path.join(self.chromeos_root, 'chroot', relative_path)
@@ -380,6 +384,7 @@ class Result(object):
     # Include all perf.report data in table.
     self.perf_report_files = self.GeneratePerfReportFiles()
     self.turbostat_log_file = self.GetTurbostatFile()
+    self.cpustats_log_file = self.GetCpustatsFile()
     # TODO(asharif): Do something similar with perf stat.
 
     # Grab keyvals from the directory.
@@ -400,7 +405,7 @@ class Result(object):
       raw_dict = json.load(f)
       if 'charts' in raw_dict:
         raw_dict = raw_dict['charts']
-      for k, field_dict in raw_dict.iteritems():
+      for k, field_dict in raw_dict.items():
         for item in field_dict:
           keyname = k + '__' + item
           value_dict = field_dict[item]
@@ -424,11 +429,22 @@ class Result(object):
     return keyvals
 
   def ProcessTurbostatResults(self):
-    """Given turbostat_log_file parse cpu stats from file."""
-    if not self.turbostat_log_file:
-      self._logger.LogError('Turbostat output file not found.')
-      return {}
-
+    """Given turbostat_log_file non-null parse cpu stats from file.
+
+    Returns:
+      Dictionary of 'cpufreq', 'cputemp' where each
+      includes dictionary 'all': [list_of_values]
+
+    Example of the output of turbostat_log.
+    ----------------------
+    CPU     Avg_MHz Busy%   Bzy_MHz TSC_MHz IRQ     CoreTmp
+    -       329     12.13   2723    2393    10975   77
+    0       336     12.41   2715    2393    6328    77
+    2       323     11.86   2731    2393    4647    69
+    CPU     Avg_MHz Busy%   Bzy_MHz TSC_MHz IRQ     CoreTmp
+    -       1940    67.46   2884    2393    39920   83
+    0       1827    63.70   2877    2393    21184   83
+    """
     cpustats = {}
     read_data = ''
     with open(self.turbostat_log_file) as f:
@@ -438,17 +454,6 @@ class Result(object):
       self._logger.LogError('Turbostat output file is empty.')
       return {}
 
-    # Gather CPU statistics from turbostat output.
-    # Example of the output.
-    # ----------------------
-    # CPU     Avg_MHz Busy%   Bzy_MHz TSC_MHz IRQ     CoreTmp
-    # -       329     12.13   2723    2393    10975   77
-    # 0       336     12.41   2715    2393    6328    77
-    # 2       323     11.86   2731    2393    4647    69
-    # CPU     Avg_MHz Busy%   Bzy_MHz TSC_MHz IRQ     CoreTmp
-    # -       1940    67.46   2884    2393    39920   83
-    # 0       1827    63.70   2877    2393    21184   83
-
     # First line always contains the header.
     stats = read_data[0].split()
 
@@ -461,13 +466,13 @@ class Result(object):
       return {}
     cpu_index = stats.index('CPU')
     cpufreq_index = stats.index('Bzy_MHz')
-    cpustats['freq'] = []
+    cpufreq = cpustats.setdefault('cpufreq', {'all': []})
 
     # Optional parameters.
     cputemp_index = -1
     if 'CoreTmp' in stats:
       cputemp_index = stats.index('CoreTmp')
-      cpustats['temp'] = []
+      cputemp = cpustats.setdefault('cputemp', {'all': []})
 
     # Parse data starting from the second line ignoring repeating headers.
     for st in read_data[1:]:
@@ -481,11 +486,81 @@ class Result(object):
         # Combined statistics for all core has "-" CPU identifier.
         continue
 
-      cpustats['freq'].append(int(numbers[cpufreq_index]))
+      cpufreq['all'].append(int(numbers[cpufreq_index]))
       if cputemp_index != -1:
-        cpustats['temp'].append(int(numbers[cputemp_index]))
+        cputemp['all'].append(int(numbers[cputemp_index]))
     return cpustats
 
+  def ProcessCpustatsResults(self):
+    """Given cpustats_log_file non-null parse cpu data from file.
+
+    Returns:
+      Dictionary of 'cpufreq', 'cputemp' where each
+      includes dictionary of parameter: [list_of_values]
+
+    Example of cpustats.log output.
+    ----------------------
+    /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 1512000
+    /sys/devices/system/cpu/cpu2/cpufreq/cpuinfo_cur_freq 2016000
+    little-cpu 41234
+    big-cpu 51234
+
+    If cores share the same policy their frequencies may always match
+    on some devices.
+    To make report concise we should eliminate redundancy in the output.
+    Function removes cpuN data if it duplicates data from other cores.
+    """
+
+    cpustats = {}
+    read_data = ''
+    with open(self.cpustats_log_file) as f:
+      read_data = f.readlines()
+
+    if not read_data:
+      self._logger.LogError('Cpustats output file is empty.')
+      return {}
+
+    cpufreq_regex = re.compile(r'^[/\S]+/(cpu\d+)/[/\S]+\s+(\d+)$')
+    cputemp_regex = re.compile(r'^([^/\s]+)\s+(\d+)$')
+
+    for st in read_data:
+      match = cpufreq_regex.match(st)
+      if match:
+        cpu = match.group(1)
+        # CPU frequency comes in kHz.
+        freq_khz = int(match.group(2))
+        freq_mhz = freq_khz / 1000
+        # cpufreq represents a dictionary with CPU frequency-related
+        # data from cpustats.log.
+        cpufreq = cpustats.setdefault('cpufreq', {})
+        cpu_n_freq = cpufreq.setdefault(cpu, [])
+        cpu_n_freq.append(freq_mhz)
+      else:
+        match = cputemp_regex.match(st)
+        if match:
+          therm_type = match.group(1)
+          # The value is int, uCelsius unit.
+          temp_uc = float(match.group(2))
+          # Round to XX.X float.
+          temp_c = round(temp_uc / 1000, 1)
+          # cputemp represents a dictionary with temperature measurements
+          # from cpustats.log.
+          cputemp = cpustats.setdefault('cputemp', {})
+          therm_type = cputemp.setdefault(therm_type, [])
+          therm_type.append(temp_c)
+
+    # Remove duplicate statistics from cpustats.
+    pruned_stats = {}
+    for cpukey, cpuparam in cpustats.items():
+      # Copy 'cpufreq' and 'cputemp'.
+      pruned_params = pruned_stats.setdefault(cpukey, {})
+      for paramkey, paramvalue in sorted(cpuparam.items()):
+        # paramvalue is list of all measured data.
+        if paramvalue not in pruned_params.values():
+          pruned_params[paramkey] = paramvalue
+
+    return pruned_stats
+
   def ProcessHistogramsResults(self):
     # Open and parse the json results file generated by telemetry/test_that.
     if not self.results_file:
@@ -549,15 +624,28 @@ class Result(object):
     # Generate report from all perf.data files.
     # Now parse all perf report files and include them in keyvals.
     self.GatherPerfResults()
-    cpustats = self.ProcessTurbostatResults()
-    if 'freq' in cpustats:
-      self.cpufreq = cpustats['freq']
-      self.keyvals['cpufreq_avg'] = sum(self.cpufreq) / len(self.cpufreq)
-      self.keyvals['cpufreq_min'] = min(self.cpufreq)
-      self.keyvals['cpufreq_max'] = max(self.cpufreq)
-    if 'temp' in cpustats:
-      self.cputemp = cpustats['temp']
-      self.keyvals['cputemp'] = sum(self.cputemp) / len(self.cputemp)
+
+    cpustats = {}
+    # Turbostat output has higher priority of processing.
+    if self.turbostat_log_file:
+      cpustats = self.ProcessTurbostatResults()
+    # Process cpustats output only if turbostat has no data.
+    if not cpustats and self.cpustats_log_file:
+      cpustats = self.ProcessCpustatsResults()
+
+    for param_key, param in cpustats.items():
+      for param_type, param_values in param.items():
+        val_avg = sum(param_values) / len(param_values)
+        val_min = min(param_values)
+        val_max = max(param_values)
+        # Average data is always included.
+        self.keyvals['_'.join([param_key, param_type, 'avg'])] = val_avg
+        # Insert min/max results only if they deviate
+        # from average.
+        if val_min != val_avg:
+          self.keyvals['_'.join([param_key, param_type, 'min'])] = val_min
+        if val_max != val_avg:
+          self.keyvals['_'.join([param_key, param_type, 'max'])] = val_max
 
   def GetChromeVersionFromCache(self, cache_dir):
     # Read chrome_version from keys file, if present.
diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py
index fee67491..a5d36383 100755
--- a/crosperf/results_cache_unittest.py
+++ b/crosperf/results_cache_unittest.py
@@ -186,9 +186,62 @@ CPU     Avg_MHz Busy%   Bzy_MHz TSC_MHz IRQ     CoreTmp
 0       827     29.35   2826    2393    16093   47
 2       858     30.31   2838    2393    12068   46
 """
-TURBOSTAT_CPUSTATS = {
-    'freq': [2723, 2884, 2927, 2937, 2932, 2933, 2832],
-    'temp': [77, 83, 84, 72, 75, 46, 47]
+TURBOSTAT_DATA = {
+    'cpufreq': {
+        'all': [2723, 2884, 2927, 2937, 2932, 2933, 2832]
+    },
+    'cputemp': {
+        'all': [77, 83, 84, 72, 75, 46, 47]
+    },
+}
+
+CPUSTATS_UNIQ_OUTPUT = \
+"""
+/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 1512000
+/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_cur_freq 1512000
+/sys/devices/system/cpu/cpu3/cpufreq/cpuinfo_cur_freq 2016000
+soc-thermal 44444
+little-cpu 41234
+big-cpu 51234
+/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 1500000
+/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_cur_freq 1600000
+/sys/devices/system/cpu/cpu3/cpufreq/cpuinfo_cur_freq 2012000
+soc-thermal 45456
+little-cpu 42555
+big-cpu 61724
+"""
+CPUSTATS_UNIQ_DATA = {
+    'cpufreq': {
+        'cpu0': [1512, 1500],
+        'cpu1': [1512, 1600],
+        'cpu3': [2016, 2012]
+    },
+    'cputemp': {
+        'soc-thermal': [44.4, 45.5],
+        'little-cpu': [41.2, 42.6],
+        'big-cpu': [51.2, 61.7]
+    }
+}
+CPUSTATS_DUPL_OUTPUT = \
+"""
+/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 1512000
+/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_cur_freq 1512000
+/sys/devices/system/cpu/cpu2/cpufreq/cpuinfo_cur_freq 1512000
+/sys/devices/system/cpu/cpu3/cpufreq/cpuinfo_cur_freq 2016000
+/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 1500000
+/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_cur_freq 1500000
+/sys/devices/system/cpu/cpu2/cpufreq/cpuinfo_cur_freq 1500000
+/sys/devices/system/cpu/cpu3/cpufreq/cpuinfo_cur_freq 2016000
+/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 1614000
+/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_cur_freq 1614000
+/sys/devices/system/cpu/cpu2/cpufreq/cpuinfo_cur_freq 1614000
+/sys/devices/system/cpu/cpu3/cpufreq/cpuinfo_cur_freq 1982000
+"""
+CPUSTATS_DUPL_DATA = {
+    'cpufreq': {
+        'cpu0': [1512, 1500, 1614],
+        'cpu3': [2016, 2016, 1982]
+    },
 }
 
 TMP_DIR1 = '/tmp/tmpAbcXyz'
@@ -226,9 +279,9 @@ class ResultTest(unittest.TestCase):
     self.callGetResultsFile = False
     self.callGetPerfDataFiles = False
     self.callGetTurbostatFile = False
+    self.callGetCpustatsFile = False
     self.args = None
     self.callGatherPerfResults = False
-    self.callProcessTutbostatResults = False
     self.mock_logger = mock.Mock(spec=logger.Logger)
     self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
     self.mock_label = MockLabel('mock_label', 'build', 'chromeos_image',
@@ -627,6 +680,34 @@ class ResultTest(unittest.TestCase):
     with self.assertRaises(RuntimeError):
       self.result.GetTurbostatFile()
 
+  @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
+  def test_get_cpustats_file_finds_single_log(self, mock_runcmd):
+    """Expected behavior when a single log file found."""
+    self.result.results_dir = '/tmp/test_results'
+    self.result.ce.RunCommandWOutput = mock_runcmd
+    mock_runcmd.return_value = (0, 'some/long/path/cpustats.log', '')
+    found_single_log = self.result.GetCpustatsFile()
+    self.assertEqual(found_single_log, 'some/long/path/cpustats.log')
+
+  @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
+  def test_get_cpustats_file_finds_multiple_logs(self, mock_runcmd):
+    """The case when multiple files found."""
+    self.result.results_dir = '/tmp/test_results'
+    self.result.ce.RunCommandWOutput = mock_runcmd
+    mock_runcmd.return_value = (0, 'some/long/path/cpustats.log\ncpustats.log',
+                                '')
+    found_first_logs = self.result.GetCpustatsFile()
+    self.assertEqual(found_first_logs, 'some/long/path/cpustats.log')
+
+  @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
+  def test_get_cpustats_file_finds_no_logs(self, mock_runcmd):
+    """Error case when no log file found."""
+    self.result.results_dir = '/tmp/test_results'
+    self.result.ce.RunCommandWOutput = mock_runcmd
+    mock_runcmd.return_value = (0, '', '')
+    found_no_logs = self.result.GetCpustatsFile()
+    self.assertEqual(found_no_logs, '')
+
   def test_process_turbostat_results_with_valid_data(self):
     """Normal case when log exists and contains valid data."""
     self.result.turbostat_log_file = '/tmp/somelogfile.log'
@@ -636,7 +717,7 @@ class ResultTest(unittest.TestCase):
       # Check that the log got opened and data were read/parsed.
       calls = [mock.call('/tmp/somelogfile.log')]
       mo.assert_has_calls(calls)
-      self.assertEqual(cpustats, TURBOSTAT_CPUSTATS)
+      self.assertEqual(cpustats, TURBOSTAT_DATA)
 
   def test_process_turbostat_results_from_empty_file(self):
     """Error case when log exists but file is empty."""
@@ -649,15 +730,6 @@ class ResultTest(unittest.TestCase):
       mo.assert_has_calls(calls)
       self.assertEqual(cpustats, {})
 
-  def test_process_turbostat_results_with_no_filename(self):
-    """Error case when no log file name provided."""
-    self.result.turbostat_log_file = ''
-    with mock.patch('__builtin__.open', mock.mock_open()) as mo:
-      cpustats = self.result.ProcessTurbostatResults()
-      # Check no attempt to open a log and empty data returned.
-      mo.assert_not_called()
-      self.assertEqual(cpustats, {})
-
   def test_process_turbostat_results_when_file_doesnt_exist(self):
     """Error case when file does not exist."""
     nonexistinglog = '/tmp/1'
@@ -669,6 +741,52 @@ class ResultTest(unittest.TestCase):
     with self.assertRaises(IOError):
       self.result.ProcessTurbostatResults()
 
+  def test_process_cpustats_results_with_uniq_data(self):
+    """Process cpustats log which has freq unique to each core.
+
+    Testing normal case when frequency data vary between
+    different cores.
+    Expecting that data for all cores will be present in
+    returned cpustats.
+    """
+    self.maxDiff = None
+    self.result.cpustats_log_file = '/tmp/somelogfile.log'
+    with mock.patch('__builtin__.open',
+                    mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT)) as mo:
+      cpustats = self.result.ProcessCpustatsResults()
+      # Check that the log got opened and data were read/parsed.
+      calls = [mock.call('/tmp/somelogfile.log')]
+      mo.assert_has_calls(calls)
+      self.assertEqual(cpustats, CPUSTATS_UNIQ_DATA)
+
+  def test_process_cpustats_results_with_dupl_data(self):
+    """Process cpustats log where cores have duplicate freq.
+
+    Testing normal case when frequency data on some cores
+    are duplicated.
+    Expecting that duplicated data is discarded in
+    returned cpustats.
+    """
+    self.result.cpustats_log_file = '/tmp/somelogfile.log'
+    with mock.patch('__builtin__.open',
+                    mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT)) as mo:
+      cpustats = self.result.ProcessCpustatsResults()
+      # Check that the log got opened and data were read/parsed.
+      calls = [mock.call('/tmp/somelogfile.log')]
+      mo.assert_has_calls(calls)
+      self.assertEqual(cpustats, CPUSTATS_DUPL_DATA)
+
+  def test_process_cpustats_results_from_empty_file(self):
+    """Error case when log exists but file is empty."""
+    self.result.cpustats_log_file = '/tmp/emptylogfile.log'
+    with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo:
+      cpustats = self.result.ProcessCpustatsResults()
+      # Check that the log got opened and parsed successfully and empty data
+      # returned.
+      calls = [mock.call('/tmp/emptylogfile.log')]
+      mo.assert_has_calls(calls)
+      self.assertEqual(cpustats, {})
+
   @mock.patch.object(misc, 'GetInsideChrootPath')
   @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand')
   def test_generate_perf_report_files(self, mock_chrootruncmd, mock_getpath):
@@ -728,6 +846,10 @@ class ResultTest(unittest.TestCase):
       self.callGetTurbostatFile = True
       return []
 
+    def FakeGetCpustatsFile():
+      self.callGetCpustatsFile = True
+      return []
+
     def FakeProcessResults(show_results=False):
       if show_results:
         pass
@@ -743,6 +865,7 @@ class ResultTest(unittest.TestCase):
     self.callGetPerfDataFiles = False
     self.callGetPerfReportFiles = False
     self.callGetTurbostatFile = False
+    self.callGetCpustatsFile = False
     self.callProcessResults = False
 
     self.result.GetResultsDir = FakeGetResultsDir
@@ -750,6 +873,7 @@ class ResultTest(unittest.TestCase):
     self.result.GetPerfDataFiles = FakeGetPerfDataFiles
     self.result.GeneratePerfReportFiles = FakeGetPerfReportFiles
     self.result.GetTurbostatFile = FakeGetTurbostatFile
+    self.result.GetCpustatsFile = FakeGetCpustatsFile
     self.result.ProcessResults = FakeProcessResults
 
     self.result.PopulateFromRun(OUTPUT, '', 0, 'test', 'telemetry_Crosperf',
@@ -759,15 +883,16 @@ class ResultTest(unittest.TestCase):
     self.assertTrue(self.callGetPerfDataFiles)
     self.assertTrue(self.callGetPerfReportFiles)
     self.assertTrue(self.callGetTurbostatFile)
+    self.assertTrue(self.callGetCpustatsFile)
     self.assertTrue(self.callProcessResults)
 
-  def test_process_results(self):
+  def FakeGetKeyvals(self, show_all=False):
+    if show_all:
+      return {'first_time': 680, 'Total': 10}
+    else:
+      return {'Total': 10}
 
-    def FakeGetKeyvals(show_all=False):
-      if show_all:
-        return {'first_time': 680, 'Total': 10}
-      else:
-        return {'Total': 10}
+  def test_process_results(self):
 
     def FakeGatherPerfResults():
       self.callGatherPerfResults = True
@@ -775,21 +900,10 @@ class ResultTest(unittest.TestCase):
     def FakeGetSamples():
       return 1
 
-    def FakeProcessTurbostatResults():
-      self.callProcessTutbostatResults = True
-      res = {}
-      if self.result.turbostat_log_file:
-        res['freq'] = [1, 2, 3]
-        res['temp'] = [5, 6, 7]
-      return res
-
     self.callGatherPerfResults = False
-    self.callProcessTutbostatResults = False
-    self.result.turbostat_log_file = ''
 
-    self.result.GetKeyvals = FakeGetKeyvals
+    self.result.GetKeyvals = self.FakeGetKeyvals
     self.result.GatherPerfResults = FakeGatherPerfResults
-    self.result.ProcessTurbostatResults = FakeProcessTurbostatResults
 
     self.result.retval = 0
     self.result.ProcessResults()
@@ -812,23 +926,123 @@ class ResultTest(unittest.TestCase):
         'samples': 1,
         'retval': 0
     })
-    self.result.cwp_dso = ''
+
+  @mock.patch.object(Result, 'ProcessCpustatsResults')
+  @mock.patch.object(Result, 'ProcessTurbostatResults')
+  def test_process_results_with_turbostat_log(self, mock_proc_turbo,
+                                              mock_proc_cpustats):
+    self.result.GetKeyvals = self.FakeGetKeyvals
+
+    self.result.retval = 0
+    self.result.turbostat_log_file = '/tmp/turbostat.log'
+    mock_proc_turbo.return_value = {
+        'cpufreq': {
+            'all': [1, 2, 3]
+        },
+        'cputemp': {
+            'all': [5.0, 6.0, 7.0]
+        }
+    }
+    self.result.ProcessResults()
+    mock_proc_turbo.assert_has_calls([mock.call()])
+    mock_proc_cpustats.assert_not_called()
+    self.assertEqual(len(self.result.keyvals), 8)
+    self.assertEqual(
+        self.result.keyvals, {
+            'Total': 10,
+            'cpufreq_all_avg': 2,
+            'cpufreq_all_max': 3,
+            'cpufreq_all_min': 1,
+            'cputemp_all_avg': 6.0,
+            'cputemp_all_min': 5.0,
+            'cputemp_all_max': 7.0,
+            'retval': 0
+        })
+
+  @mock.patch.object(Result, 'ProcessCpustatsResults')
+  @mock.patch.object(Result, 'ProcessTurbostatResults')
+  def test_process_results_with_cpustats_log(self, mock_proc_turbo,
+                                             mock_proc_cpustats):
+    self.result.GetKeyvals = self.FakeGetKeyvals
+
+    self.result.retval = 0
+    self.result.cpustats_log_file = '/tmp/cpustats.log'
+    mock_proc_cpustats.return_value = {
+        'cpufreq': {
+            'cpu0': [100, 100, 100],
+            'cpu1': [4, 5, 6]
+        },
+        'cputemp': {
+            'little': [20.2, 20.2, 20.2],
+            'big': [55.2, 66.1, 77.3]
+        }
+    }
+    self.result.ProcessResults()
+    mock_proc_turbo.assert_not_called()
+    mock_proc_cpustats.assert_has_calls([mock.call()])
+    self.assertEqual(len(self.result.keyvals), 10)
+    self.assertEqual(
+        self.result.keyvals, {
+            'Total': 10,
+            'cpufreq_cpu0_avg': 100,
+            'cpufreq_cpu1_avg': 5,
+            'cpufreq_cpu1_max': 6,
+            'cpufreq_cpu1_min': 4,
+            'cputemp_big_avg': 66.2,
+            'cputemp_big_max': 77.3,
+            'cputemp_big_min': 55.2,
+            'cputemp_little_avg': 20.2,
+            'retval': 0
+        })
+
+  @mock.patch.object(Result, 'ProcessCpustatsResults')
+  @mock.patch.object(Result, 'ProcessTurbostatResults')
+  def test_process_results_with_turbostat_and_cpustats_logs(
+      self, mock_proc_turbo, mock_proc_cpustats):
+    self.result.GetKeyvals = self.FakeGetKeyvals
 
     self.result.retval = 0
     self.result.turbostat_log_file = '/tmp/turbostat.log'
+    self.result.cpustats_log_file = '/tmp/cpustats.log'
+    mock_proc_turbo.return_value = {
+        'cpufreq': {
+            'all': [1, 2, 3]
+        },
+        'cputemp': {
+            'all': [5.0, 6.0, 7.0]
+        }
+    }
     self.result.ProcessResults()
-    self.assertTrue(self.callProcessTutbostatResults)
-    self.assertEqual(len(self.result.keyvals), 6)
+    mock_proc_turbo.assert_has_calls([mock.call()])
+    mock_proc_cpustats.assert_not_called()
+    self.assertEqual(len(self.result.keyvals), 8)
     self.assertEqual(
         self.result.keyvals, {
             'Total': 10,
-            'cpufreq_avg': 2,
-            'cpufreq_max': 3,
-            'cpufreq_min': 1,
-            'cputemp': 6,
+            'cpufreq_all_avg': 2,
+            'cpufreq_all_max': 3,
+            'cpufreq_all_min': 1,
+            'cputemp_all_avg': 6.0,
+            'cputemp_all_min': 5.0,
+            'cputemp_all_max': 7.0,
             'retval': 0
         })
 
+  @mock.patch.object(Result, 'ProcessCpustatsResults')
+  @mock.patch.object(Result, 'ProcessTurbostatResults')
+  def test_process_results_without_cpu_data(self, mock_proc_turbo,
+                                            mock_proc_cpustats):
+    self.result.GetKeyvals = self.FakeGetKeyvals
+
+    self.result.retval = 0
+    self.result.turbostat_log_file = ''
+    self.result.cpustats_log_file = ''
+    self.result.ProcessResults()
+    mock_proc_turbo.assert_not_called()
+    mock_proc_cpustats.assert_not_called()
+    self.assertEqual(len(self.result.keyvals), 2)
+    self.assertEqual(self.result.keyvals, {'Total': 10, 'retval': 0})
+
   @mock.patch.object(misc, 'GetInsideChrootPath')
   @mock.patch.object(command_executer.CommandExecuter,
                      'ChrootRunCommandWOutput')
diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py
index 8f183cc9..15d495c7 100644
--- a/crosperf/results_organizer.py
+++ b/crosperf/results_organizer.py
@@ -187,9 +187,11 @@ def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False):
     if not show_all_results:
       summary_list = summary_file.get(benchmark.name)
       if summary_list:
-        summary_list += [
-            'retval', 'cpufreq_avg', 'cpufreq_min', 'cpufreq_max', 'cputemp'
-        ]
+        for key in benchmark_run.result.keyvals.keys():
+          if any(
+              key.startswith(added_key)
+              for added_key in ['retval', 'cpufreq', 'cputemp']):
+            summary_list.append(key)
       else:
         # Did not find test_name in json file; show everything.
         show_all_results = True