diff options
author | Ahmad Sharif <asharif@chromium.org> | 2012-12-20 12:09:49 -0800 |
---|---|---|
committer | Ahmad Sharif <asharif@chromium.org> | 2012-12-20 12:09:49 -0800 |
commit | 4467f004e7f0854963bec90daff1879fbd9d2fec (patch) | |
tree | aac36caa6279aa532e2d6234e50ee812f2db0c8d /crosperf/results_cache.py | |
parent | f395c26437cbdabc2960447fba89b226f4409e82 (diff) | |
download | toolchain-utils-4467f004e7f0854963bec90daff1879fbd9d2fec.tar.gz |
Synced repos to: 64740
Diffstat (limited to 'crosperf/results_cache.py')
-rw-r--r-- | crosperf/results_cache.py | 76 |
1 files changed, 47 insertions, 29 deletions
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index 1c33e720..c0600962 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -11,7 +11,6 @@ import re import tempfile from utils import command_executer -from utils import logger from utils import misc from image_checksummer import ImageChecksummer @@ -28,11 +27,13 @@ class Result(object): what the key of the cache is. For runs with perf, it stores perf.data, perf.report, etc. The key generation is handled by the ResultsCache class. """ - def __init__(self, chromeos_root, logger): + + def __init__(self, chromeos_root, logger, label_name): self._chromeos_root = chromeos_root self._logger = logger self._ce = command_executer.GetCommandExecuter(self._logger) self._temp_dir = None + self.label_name = label_name def _CopyFilesTo(self, dest_dir, files_to_copy): file_index = 0 @@ -63,7 +64,7 @@ class Result(object): command = ("python %s --no-color --csv %s" % (generate_test_report, self.results_dir)) - [ret, out, err] = self._ce.RunCommand(command, return_output=True) + [_, out, _] = self._ce.RunCommand(command, return_output=True) keyvals_dict = {} for line in out.splitlines(): tokens = re.split("=|,", line) @@ -76,7 +77,7 @@ class Result(object): return keyvals_dict def _GetResultsDir(self): - mo = re.search("Results placed in (\S+)", self.out) + mo = re.search(r"Results placed in (\S+)", self.out) if mo: result = mo.group(1) return result @@ -85,7 +86,7 @@ class Result(object): def _FindFilesInResultsDir(self, find_args): command = "find %s %s" % (self.results_dir, find_args) - ret, out, err = self._ce.RunCommand(command, return_output=True) + ret, out, _ = self._ce.RunCommand(command, return_output=True) if ret: raise Exception("Could not run find command!") return out @@ -108,23 +109,22 @@ class Result(object): raise Exception("Perf report file already exists: %s" % perf_report_file) chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root, - perf_report_file) + perf_report_file) command = ("/usr/sbin/perf report " "-n " "--symfs /build/%s " "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux " "--kallsyms /build/%s/boot/System.map-* " "-i %s --stdio " - "| head -n1000 " - "| tee %s" % + "> %s" % (self._board, self._board, self._board, chroot_perf_data_file, chroot_perf_report_file)) - ret, out, err = self._ce.ChrootRunCommand(self._chromeos_root, - command, - return_output=True) + self._ce.ChrootRunCommand(self._chromeos_root, + command) + # Add a keyval to the dictionary for the events captured. perf_report_files.append( misc.GetOutsideChrootPath(self._chromeos_root, @@ -136,7 +136,7 @@ class Result(object): for perf_report_file in self.perf_report_files: with open(perf_report_file, "r") as f: report_contents = f.read() - for group in re.findall("Events: (\S+) (\S+)", report_contents): + for group in re.findall(r"Events: (\S+) (\S+)", report_contents): num_events = group[0] event_name = group[1] key = "perf_%s_%s" % (report_id, event_name) @@ -188,14 +188,12 @@ class Result(object): self.perf_data_files = self._GetPerfDataFiles() self.perf_report_files = self._GetPerfReportFiles() self._ProcessResults() - self.CleanUp() def CleanUp(self): if self._temp_dir: command = "rm -rf %s" % self._temp_dir self._ce.RunCommand(command) - def StoreToCacheDir(self, cache_dir, machine_manager): # Create the dir if it doesn't exist. command = "mkdir -p %s" % cache_dir @@ -221,17 +219,18 @@ class Result(object): # TODO(asharif): Make machine_manager a singleton, and don't pass it into # this function. with open(os.path.join(cache_dir, MACHINE_FILE), "w") as f: - f.write(machine_manager.machine_checksum_string) + f.write(machine_manager.machine_checksum_string[self.label_name]) @classmethod - def CreateFromRun(cls, logger, chromeos_root, board, out, err, retval): - result = cls(chromeos_root, logger) + def CreateFromRun(cls, logger, chromeos_root, board, label_name, + out, err, retval): + result = cls(chromeos_root, logger, label_name) result._PopulateFromRun(board, out, err, retval) return result @classmethod - def CreateFromCacheHit(cls, chromeos_root, logger, cache_dir): - result = cls(chromeos_root, logger) + def CreateFromCacheHit(cls, chromeos_root, logger, cache_dir, label_name): + result = cls(chromeos_root, logger, label_name) try: result._PopulateFromCacheDir(cache_dir) except Exception as e: @@ -260,17 +259,21 @@ class CacheConditions(object): # Cache hit if the image path matches the cached image path. IMAGE_PATH_MATCH = 5 + # Cache hit if the uuid of hard disk mataches the cached one + + SAME_MACHINE_MATCH = 6 + class ResultsCache(object): """ This class manages the key of the cached runs without worrying about what is exactly stored (value). The value generation is handled by the Results class. """ - CACHE_VERSION = 5 + CACHE_VERSION = 6 def Init(self, chromeos_image, chromeos_root, autotest_name, iteration, autotest_args, machine_manager, board, cache_conditions, - logger_to_use): + logger_to_use, label): self.chromeos_image = chromeos_image self.chromeos_root = chromeos_root self.autotest_name = autotest_name @@ -281,6 +284,7 @@ class ResultsCache(object): self.machine_manager = machine_manager self._logger = logger_to_use self._ce = command_executer.GetCommandExecuter(self._logger) + self.label = label def _GetCacheDirForRead(self): glob_path = self._FormCacheDir(self._GetCacheKeyList(True)) @@ -288,9 +292,6 @@ class ResultsCache(object): if matching_dirs: # Cache file found. - if len(matching_dirs) > 1: - self._logger.LogError("Multiple compatible cache files: %s." % - " ".join(matching_dirs)) return matching_dirs[0] else: return None @@ -308,7 +309,7 @@ class ResultsCache(object): if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: machine_checksum = "*" else: - machine_checksum = self.machine_manager.machine_checksum + machine_checksum = self.machine_manager.machine_checksum[self.label.name] if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: checksum = "*" else: @@ -319,13 +320,22 @@ class ResultsCache(object): else: image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() + if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: + machine_id_checksum = "*" + else: + for machine in self.machine_manager.GetMachines(self.label): + if machine.name == self.label.remote[0]: + machine_id_checksum = machine.machine_id_checksum + break + autotest_args_checksum = hashlib.md5( - "".join(self.autotest_args)).hexdigest() + "".join(self.autotest_args)).hexdigest() return (image_path_checksum, self.autotest_name, str(self.iteration), autotest_args_checksum, checksum, machine_checksum, + machine_id_checksum, str(self.CACHE_VERSION)) def ReadResult(self): @@ -342,7 +352,7 @@ class ResultsCache(object): self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir) result = Result.CreateFromCacheHit(self.chromeos_root, - self._logger, cache_dir) + self._logger, cache_dir, self.label.name) if not result: return None @@ -358,12 +368,20 @@ class ResultsCache(object): result.StoreToCacheDir(cache_dir, self.machine_manager) -class MockResultsCache(object): +class MockResultsCache(ResultsCache): def Init(self, *args): pass def ReadResult(self): - return Result("Results placed in /tmp/test", "", 0) + return None def StoreResult(self, result): pass + + +class MockResult(Result): + def _PopulateFromRun(self, out, err, retval): + self.out = out + self.err = err + self.retval = retval + |