diff options
Diffstat (limited to 'crosperf/results_cache.py')
-rw-r--r-- | crosperf/results_cache.py | 758 |
1 files changed, 758 insertions, 0 deletions
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py new file mode 100644 index 00000000..29e118e8 --- /dev/null +++ b/crosperf/results_cache.py @@ -0,0 +1,758 @@ +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Module to deal with result cache.""" + +from __future__ import print_function + +import glob +import hashlib +import os +import pickle +import re +import tempfile +import json +import sys + +from cros_utils import command_executer +from cros_utils import misc + +from image_checksummer import ImageChecksummer + +import results_report +import test_flag + +SCRATCH_DIR = os.path.expanduser('~/cros_scratch') +RESULTS_FILE = 'results.txt' +MACHINE_FILE = 'machine.txt' +AUTOTEST_TARBALL = 'autotest.tbz2' +PERF_RESULTS_FILE = 'perf-results.txt' +CACHE_KEYS_FILE = 'cache_keys.txt' + + +class Result(object): + """Class for holding the results of a single test run. + + This class manages what exactly is stored inside the cache without knowing + what the key of the cache is. For runs with perf, it stores perf.data, + perf.report, etc. The key generation is handled by the ResultsCache class. + """ + + def __init__(self, logger, label, log_level, machine, cmd_exec=None): + self.chromeos_root = label.chromeos_root + self._logger = logger + self.ce = cmd_exec or command_executer.GetCommandExecuter( + self._logger, log_level=log_level) + self.temp_dir = None + self.label = label + self.results_dir = None + self.log_level = log_level + self.machine = machine + self.perf_data_files = [] + self.perf_report_files = [] + self.results_file = [] + self.chrome_version = '' + self.err = None + self.chroot_results_dir = '' + self.test_name = '' + self.keyvals = None + self.board = None + self.suite = None + self.retval = None + self.out = None + + def CopyFilesTo(self, dest_dir, files_to_copy): + file_index = 0 + for file_to_copy in files_to_copy: + if not os.path.isdir(dest_dir): + command = 'mkdir -p %s' % dest_dir + self.ce.RunCommand(command) + dest_file = os.path.join(dest_dir, + ('%s.%s' % (os.path.basename(file_to_copy), + file_index))) + ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) + if ret: + raise IOError('Could not copy results file: %s' % file_to_copy) + + def CopyResultsTo(self, dest_dir): + self.CopyFilesTo(dest_dir, self.perf_data_files) + self.CopyFilesTo(dest_dir, self.perf_report_files) + if len(self.perf_data_files) or len(self.perf_report_files): + self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) + + def GetNewKeyvals(self, keyvals_dict): + # Initialize 'units' dictionary. + units_dict = {} + for k in keyvals_dict: + units_dict[k] = '' + results_files = self.GetDataMeasurementsFiles() + for f in results_files: + # Make sure we can find the results file + if os.path.exists(f): + data_filename = f + else: + # Otherwise get the base filename and create the correct + # path for it. + _, f_base = misc.GetRoot(f) + data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', + self.temp_dir, f_base) + if data_filename.find('.json') > 0: + raw_dict = dict() + if os.path.exists(data_filename): + with open(data_filename, 'r') as data_file: + raw_dict = json.load(data_file) + + if 'charts' in raw_dict: + raw_dict = raw_dict['charts'] + for k1 in raw_dict: + field_dict = raw_dict[k1] + for k2 in field_dict: + result_dict = field_dict[k2] + key = k1 + '__' + k2 + if 'value' in result_dict: + keyvals_dict[key] = result_dict['value'] + elif 'values' in result_dict: + values = result_dict['values'] + if ('type' in result_dict and + result_dict['type'] == 'list_of_scalar_values' and values and + values != 'null'): + keyvals_dict[key] = sum(values) / float(len(values)) + else: + keyvals_dict[key] = values + units_dict[key] = result_dict['units'] + else: + if os.path.exists(data_filename): + with open(data_filename, 'r') as data_file: + lines = data_file.readlines() + for line in lines: + tmp_dict = json.loads(line) + graph_name = tmp_dict['graph'] + graph_str = (graph_name + '__') if graph_name else '' + key = graph_str + tmp_dict['description'] + keyvals_dict[key] = tmp_dict['value'] + units_dict[key] = tmp_dict['units'] + + return keyvals_dict, units_dict + + def AppendTelemetryUnits(self, keyvals_dict, units_dict): + """keyvals_dict is the dict of key-value used to generate Crosperf reports. + + units_dict is a dictionary of the units for the return values in + keyvals_dict. We need to associate the units with the return values, + for Telemetry tests, so that we can include the units in the reports. + This function takes each value in keyvals_dict, finds the corresponding + unit in the units_dict, and replaces the old value with a list of the + old value and the units. This later gets properly parsed in the + ResultOrganizer class, for generating the reports. + """ + + results_dict = {} + for k in keyvals_dict: + # We don't want these lines in our reports; they add no useful data. + if k == '' or k == 'telemetry_Crosperf': + continue + val = keyvals_dict[k] + units = units_dict[k] + new_val = [val, units] + results_dict[k] = new_val + return results_dict + + def GetKeyvals(self): + results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') + if not self.temp_dir: + self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) + command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) + self.ce.RunCommand(command, print_to_console=False) + + command = ('python generate_test_report --no-color --csv %s' % + (os.path.join('/tmp', os.path.basename(self.temp_dir)))) + _, out, _ = self.ce.ChrootRunCommandWOutput( + self.chromeos_root, command, print_to_console=False) + keyvals_dict = {} + tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, + self.temp_dir) + for line in out.splitlines(): + tokens = re.split('=|,', line) + key = tokens[-2] + if key.startswith(tmp_dir_in_chroot): + key = key[len(tmp_dir_in_chroot) + 1:] + value = tokens[-1] + keyvals_dict[key] = value + + # Check to see if there is a perf_measurements file and get the + # data from it if so. + keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) + if self.suite == 'telemetry_Crosperf': + # For telemtry_Crosperf results, append the units to the return + # results, for use in generating the reports. + keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) + return keyvals_dict + + def GetResultsDir(self): + mo = re.search(r'Results placed in (\S+)', self.out) + if mo: + result = mo.group(1) + return result + raise RuntimeError('Could not find results directory.') + + def FindFilesInResultsDir(self, find_args): + if not self.results_dir: + return None + + command = 'find %s %s' % (self.results_dir, find_args) + ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) + if ret: + raise RuntimeError('Could not run find command!') + return out + + def GetResultsFile(self): + return self.FindFilesInResultsDir('-name results-chart.json').splitlines() + + def GetPerfDataFiles(self): + return self.FindFilesInResultsDir('-name perf.data').splitlines() + + def GetPerfReportFiles(self): + return self.FindFilesInResultsDir('-name perf.data.report').splitlines() + + def GetDataMeasurementsFiles(self): + result = self.FindFilesInResultsDir('-name perf_measurements').splitlines() + if not result: + result = \ + self.FindFilesInResultsDir('-name results-chart.json').splitlines() + return result + + def GeneratePerfReportFiles(self): + perf_report_files = [] + for perf_data_file in self.perf_data_files: + # Generate a perf.report and store it side-by-side with the perf.data + # file. + chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, + perf_data_file) + perf_report_file = '%s.report' % perf_data_file + if os.path.exists(perf_report_file): + raise RuntimeError('Perf report file already exists: %s' % + perf_report_file) + chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root, + perf_report_file) + perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') + + perf_file = '/usr/sbin/perf' + if os.path.exists(perf_path): + perf_file = '/usr/bin/perf' + + command = ('%s report ' + '-n ' + '--symfs /build/%s ' + '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' + '--kallsyms /build/%s/boot/System.map-* ' + '-i %s --stdio ' + '> %s' % (perf_file, self.board, self.board, self.board, + chroot_perf_data_file, chroot_perf_report_file)) + self.ce.ChrootRunCommand(self.chromeos_root, command) + + # Add a keyval to the dictionary for the events captured. + perf_report_files.append( + misc.GetOutsideChrootPath(self.chromeos_root, + chroot_perf_report_file)) + return perf_report_files + + def GatherPerfResults(self): + report_id = 0 + for perf_report_file in self.perf_report_files: + with open(perf_report_file, 'r') as f: + report_contents = f.read() + for group in re.findall(r'Events: (\S+) (\S+)', report_contents): + num_events = group[0] + event_name = group[1] + key = 'perf_%s_%s' % (report_id, event_name) + value = str(misc.UnitToNumber(num_events)) + self.keyvals[key] = value + + def PopulateFromRun(self, out, err, retval, test, suite): + self.board = self.label.board + self.out = out + self.err = err + self.retval = retval + self.test_name = test + self.suite = suite + self.chroot_results_dir = self.GetResultsDir() + self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, + self.chroot_results_dir) + self.results_file = self.GetResultsFile() + self.perf_data_files = self.GetPerfDataFiles() + # Include all perf.report data in table. + self.perf_report_files = self.GeneratePerfReportFiles() + # TODO(asharif): Do something similar with perf stat. + + # Grab keyvals from the directory. + self.ProcessResults() + + def ProcessJsonResults(self): + # Open and parse the json results file generated by telemetry/test_that. + if not self.results_file: + raise IOError('No results file found.') + filename = self.results_file[0] + if not filename.endswith('.json'): + raise IOError('Attempt to call json on non-json file: %s' % filename) + + if not os.path.exists(filename): + return {} + + keyvals = {} + with open(filename, 'r') as f: + raw_dict = json.load(f) + if 'charts' in raw_dict: + raw_dict = raw_dict['charts'] + for k, field_dict in raw_dict.iteritems(): + for item in field_dict: + keyname = k + '__' + item + value_dict = field_dict[item] + if 'value' in value_dict: + result = value_dict['value'] + elif 'values' in value_dict: + values = value_dict['values'] + if not values: + continue + if ('type' in value_dict and + value_dict['type'] == 'list_of_scalar_values' and + values != 'null'): + result = sum(values) / float(len(values)) + else: + result = values + units = value_dict['units'] + new_value = [result, units] + keyvals[keyname] = new_value + return keyvals + + def ProcessResults(self, use_cache=False): + # Note that this function doesn't know anything about whether there is a + # cache hit or miss. It should process results agnostic of the cache hit + # state. + if self.results_file and self.results_file[0].find( + 'results-chart.json') != -1: + self.keyvals = self.ProcessJsonResults() + else: + if not use_cache: + print('\n ** WARNING **: Had to use deprecated output-method to ' + 'collect results.\n') + self.keyvals = self.GetKeyvals() + self.keyvals['retval'] = self.retval + # Generate report from all perf.data files. + # Now parse all perf report files and include them in keyvals. + self.GatherPerfResults() + + def GetChromeVersionFromCache(self, cache_dir): + # Read chrome_version from keys file, if present. + chrome_version = '' + keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) + if os.path.exists(keys_file): + with open(keys_file, 'r') as f: + lines = f.readlines() + for l in lines: + if l.startswith('Google Chrome '): + chrome_version = l + if chrome_version.endswith('\n'): + chrome_version = chrome_version[:-1] + break + return chrome_version + + def PopulateFromCacheDir(self, cache_dir, test, suite): + self.test_name = test + self.suite = suite + # Read in everything from the cache directory. + with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: + self.out = pickle.load(f) + self.err = pickle.load(f) + self.retval = pickle.load(f) + + # Untar the tarball to a temporary directory + self.temp_dir = tempfile.mkdtemp( + dir=os.path.join(self.chromeos_root, 'chroot', 'tmp')) + + command = ('cd %s && tar xf %s' % + (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) + ret = self.ce.RunCommand(command, print_to_console=False) + if ret: + raise RuntimeError('Could not untar cached tarball') + self.results_dir = self.temp_dir + self.results_file = self.GetDataMeasurementsFiles() + self.perf_data_files = self.GetPerfDataFiles() + self.perf_report_files = self.GetPerfReportFiles() + self.chrome_version = self.GetChromeVersionFromCache(cache_dir) + self.ProcessResults(use_cache=True) + + def CleanUp(self, rm_chroot_tmp): + if rm_chroot_tmp and self.results_dir: + dirname, basename = misc.GetRoot(self.results_dir) + if basename.find('test_that_results_') != -1: + command = 'rm -rf %s' % self.results_dir + else: + command = 'rm -rf %s' % dirname + self.ce.RunCommand(command) + if self.temp_dir: + command = 'rm -rf %s' % self.temp_dir + self.ce.RunCommand(command) + + def StoreToCacheDir(self, cache_dir, machine_manager, key_list): + # Create the dir if it doesn't exist. + temp_dir = tempfile.mkdtemp() + + # Store to the temp directory. + with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: + pickle.dump(self.out, f) + pickle.dump(self.err, f) + pickle.dump(self.retval, f) + + if not test_flag.GetTestMode(): + with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: + f.write('%s\n' % self.label.name) + f.write('%s\n' % self.label.chrome_version) + f.write('%s\n' % self.machine.checksum_string) + for k in key_list: + f.write(k) + f.write('\n') + + if self.results_dir: + tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) + command = ('cd %s && ' + 'tar ' + '--exclude=var/spool ' + '--exclude=var/log ' + '-cjf %s .' % (self.results_dir, tarball)) + ret = self.ce.RunCommand(command) + if ret: + raise RuntimeError("Couldn't store autotest output directory.") + # Store machine info. + # TODO(asharif): Make machine_manager a singleton, and don't pass it into + # this function. + with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: + f.write(machine_manager.machine_checksum_string[self.label.name]) + + if os.path.exists(cache_dir): + command = 'rm -rf {0}'.format(cache_dir) + self.ce.RunCommand(command) + + command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) + command += 'chmod g+x {0} && '.format(temp_dir) + command += 'mv {0} {1}'.format(temp_dir, cache_dir) + ret = self.ce.RunCommand(command) + if ret: + command = 'rm -rf {0}'.format(temp_dir) + self.ce.RunCommand(command) + raise RuntimeError('Could not move dir %s to dir %s' % + (temp_dir, cache_dir)) + + @classmethod + def CreateFromRun(cls, + logger, + log_level, + label, + machine, + out, + err, + retval, + test, + suite='telemetry_Crosperf'): + if suite == 'telemetry': + result = TelemetryResult(logger, label, log_level, machine) + else: + result = cls(logger, label, log_level, machine) + result.PopulateFromRun(out, err, retval, test, suite) + return result + + @classmethod + def CreateFromCacheHit(cls, + logger, + log_level, + label, + machine, + cache_dir, + test, + suite='telemetry_Crosperf'): + if suite == 'telemetry': + result = TelemetryResult(logger, label, log_level, machine) + else: + result = cls(logger, label, log_level, machine) + try: + result.PopulateFromCacheDir(cache_dir, test, suite) + + except RuntimeError as e: + logger.LogError('Exception while using cache: %s' % e) + return None + return result + + +class TelemetryResult(Result): + """Class to hold the results of a single Telemetry run.""" + + def __init__(self, logger, label, log_level, machine, cmd_exec=None): + super(TelemetryResult, self).__init__(logger, label, log_level, machine, + cmd_exec) + + def PopulateFromRun(self, out, err, retval, test, suite): + self.out = out + self.err = err + self.retval = retval + + self.ProcessResults() + + # pylint: disable=arguments-differ + def ProcessResults(self): + # The output is: + # url,average_commit_time (ms),... + # www.google.com,33.4,21.2,... + # We need to convert to this format: + # {"www.google.com:average_commit_time (ms)": "33.4", + # "www.google.com:...": "21.2"} + # Added note: Occasionally the output comes back + # with "JSON.stringify(window.automation.GetResults())" on + # the first line, and then the rest of the output as + # described above. + + lines = self.out.splitlines() + self.keyvals = {} + + if lines: + if lines[0].startswith('JSON.stringify'): + lines = lines[1:] + + if not lines: + return + labels = lines[0].split(',') + for line in lines[1:]: + fields = line.split(',') + if len(fields) != len(labels): + continue + for i in xrange(1, len(labels)): + key = '%s %s' % (fields[0], labels[i]) + value = fields[i] + self.keyvals[key] = value + self.keyvals['retval'] = self.retval + + def PopulateFromCacheDir(self, cache_dir, test, suite): + self.test_name = test + self.suite = suite + with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: + self.out = pickle.load(f) + self.err = pickle.load(f) + self.retval = pickle.load(f) + + self.chrome_version = \ + super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir) + self.ProcessResults() + + +class CacheConditions(object): + """Various Cache condition values, for export.""" + + # Cache hit only if the result file exists. + CACHE_FILE_EXISTS = 0 + + # Cache hit if the checksum of cpuinfo and totalmem of + # the cached result and the new run match. + MACHINES_MATCH = 1 + + # Cache hit if the image checksum of the cached result and the new run match. + CHECKSUMS_MATCH = 2 + + # Cache hit only if the cached result was successful + RUN_SUCCEEDED = 3 + + # Never a cache hit. + FALSE = 4 + + # Cache hit if the image path matches the cached image path. + IMAGE_PATH_MATCH = 5 + + # Cache hit if the uuid of hard disk mataches the cached one + + SAME_MACHINE_MATCH = 6 + + +class ResultsCache(object): + """Class to handle the cache for storing/retrieving test run results. + + This class manages the key of the cached runs without worrying about what + is exactly stored (value). The value generation is handled by the Results + class. + """ + CACHE_VERSION = 6 + + def __init__(self): + # Proper initialization happens in the Init function below. + self.chromeos_image = None + self.chromeos_root = None + self.test_name = None + self.iteration = None + self.test_args = None + self.profiler_args = None + self.board = None + self.cache_conditions = None + self.machine_manager = None + self.machine = None + self._logger = None + self.ce = None + self.label = None + self.share_cache = None + self.suite = None + self.log_level = None + self.show_all = None + self.run_local = None + + def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, + profiler_args, machine_manager, machine, board, cache_conditions, + logger_to_use, log_level, label, share_cache, suite, + show_all_results, run_local): + self.chromeos_image = chromeos_image + self.chromeos_root = chromeos_root + self.test_name = test_name + self.iteration = iteration + self.test_args = test_args + self.profiler_args = profiler_args + self.board = board + self.cache_conditions = cache_conditions + self.machine_manager = machine_manager + self.machine = machine + self._logger = logger_to_use + self.ce = command_executer.GetCommandExecuter( + self._logger, log_level=log_level) + self.label = label + self.share_cache = share_cache + self.suite = suite + self.log_level = log_level + self.show_all = show_all_results + self.run_local = run_local + + def GetCacheDirForRead(self): + matching_dirs = [] + for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): + matching_dirs += glob.glob(glob_path) + + if matching_dirs: + # Cache file found. + return matching_dirs[0] + return None + + def GetCacheDirForWrite(self, get_keylist=False): + cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] + if get_keylist: + args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, + self.run_local) + version, image = results_report.ParseChromeosImage( + self.label.chromeos_image) + keylist = [ + version, image, self.label.board, self.machine.name, self.test_name, + str(self.iteration), args_str + ] + return cache_path, keylist + return cache_path + + def FormCacheDir(self, list_of_strings): + cache_key = ' '.join(list_of_strings) + cache_dir = misc.GetFilenameFromString(cache_key) + if self.label.cache_dir: + cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) + cache_path = [os.path.join(cache_home, cache_dir)] + else: + cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] + + if len(self.share_cache): + for path in [x.strip() for x in self.share_cache.split(',')]: + if os.path.exists(path): + cache_path.append(os.path.join(path, cache_dir)) + else: + self._logger.LogFatal('Unable to find shared cache: %s' % path) + + return cache_path + + def GetCacheKeyList(self, read): + if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: + machine_checksum = '*' + else: + machine_checksum = self.machine_manager.machine_checksum[self.label.name] + if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: + checksum = '*' + elif self.label.image_type == 'trybot': + checksum = hashlib.md5(self.label.chromeos_image).hexdigest() + elif self.label.image_type == 'official': + checksum = '*' + else: + checksum = ImageChecksummer().Checksum(self.label, self.log_level) + + if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: + image_path_checksum = '*' + else: + image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() + + machine_id_checksum = '' + if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: + machine_id_checksum = '*' + else: + if self.machine and self.machine.name in self.label.remote: + machine_id_checksum = self.machine.machine_id_checksum + else: + for machine in self.machine_manager.GetMachines(self.label): + if machine.name == self.label.remote[0]: + machine_id_checksum = machine.machine_id_checksum + break + + temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, + self.run_local) + test_args_checksum = hashlib.md5(temp_test_args).hexdigest() + return (image_path_checksum, self.test_name, str(self.iteration), + test_args_checksum, checksum, machine_checksum, machine_id_checksum, + str(self.CACHE_VERSION)) + + def ReadResult(self): + if CacheConditions.FALSE in self.cache_conditions: + cache_dir = self.GetCacheDirForWrite() + command = 'rm -rf %s' % (cache_dir,) + self.ce.RunCommand(command) + return None + cache_dir = self.GetCacheDirForRead() + + if not cache_dir: + return None + + if not os.path.isdir(cache_dir): + return None + + if self.log_level == 'verbose': + self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) + result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, + self.machine, cache_dir, self.test_name, + self.suite) + if not result: + return None + + if (result.retval == 0 or + CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): + return result + + return None + + def StoreResult(self, result): + cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) + result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) + + +class MockResultsCache(ResultsCache): + """Class for mock testing, corresponding to ResultsCache class.""" + + def Init(self, *args): + pass + + def ReadResult(self): + return None + + def StoreResult(self, result): + pass + + +class MockResult(Result): + """Class for mock testing, corresponding to Result class.""" + + def PopulateFromRun(self, out, err, retval, test, suite): + self.out = out + self.err = err + self.retval = retval |