1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
#!/usr/bin/python
#
# Copyright 2011 Google Inc. All Rights Reserved.
import getpass
import glob
import hashlib
import os
import pickle
import re
from image_checksummer import ImageChecksummer
from perf_processor import PerfProcessor
from utils import command_executer
from utils import logger
SCRATCH_DIR = "/home/%s/cros_scratch" % getpass.getuser()
RESULTS_FILE = "results.txt"
AUTOTEST_TARBALL = "autotest.tbz2"
PERF_RESULTS_FILE = "perf-results.txt"
class Result(object):
def __init__(self, out, err, retval, keyvals):
self.out = out
self.err = err
self.retval = retval
self.keyvals = keyvals
class CacheConditions(object):
# Cache hit only if the result file exists.
CACHE_FILE_EXISTS = 0
# Cache hit if the ip address of the cached result and the new run match.
REMOTES_MATCH = 1
# Cache hit if the image checksum of the cached result and the new run match.
CHECKSUMS_MATCH = 2
# Cache hit only if the cached result was successful
RUN_SUCCEEDED = 3
# Never a cache hit.
FALSE = 4
# Cache hit if the image path matches the cached image path.
IMAGE_PATH_MATCH = 5
class ResultsCache(object):
CACHE_VERSION = 2
def Init(self, chromeos_image, chromeos_root, autotest_name, iteration,
autotest_args, remote, board, cache_conditions,
logger_to_use):
self.chromeos_image = chromeos_image
self.chromeos_root = chromeos_root
self.autotest_name = autotest_name
self.iteration = iteration
self.autotest_args = autotest_args,
self.remote = remote
self.board = board
self.cache_conditions = cache_conditions
self._logger = logger_to_use
self._ce = command_executer.GetCommandExecuter(self._logger)
def _GetCacheDirForRead(self):
glob_path = self._FormCacheDir(self._GetCacheKeyList(True))
matching_dirs = glob.glob(glob_path)
if matching_dirs:
# Cache file found.
if len(matching_dirs) > 1:
self._logger.LogError("Multiple compatible cache files: %s." %
" ".join(matching_dirs))
return matching_dirs[0]
else:
return None
def _GetCacheDirForWrite(self):
return self._FormCacheDir(self._GetCacheKeyList(False))
def _FormCacheDir(self, list_of_strings):
cache_key = " ".join(list_of_strings)
cache_dir = self._ConvertToFilename(cache_key)
cache_path = os.path.join(SCRATCH_DIR, cache_dir)
return cache_path
def _GetCacheKeyList(self, read):
if read and CacheConditions.REMOTES_MATCH not in self.cache_conditions:
remote = "*"
else:
remote = self.remote
if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
checksum = "*"
else:
checksum = ImageChecksummer().Checksum(self.chromeos_image)
if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
image_path_checksum = "*"
else:
image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
autotest_args_checksum = hashlib.md5(
"".join(self.autotest_args)).hexdigest()
return (image_path_checksum,
self.autotest_name, str(self.iteration),
autotest_args_checksum,
checksum,
remote,
str(self.CACHE_VERSION))
def ReadResult(self):
if CacheConditions.FALSE in self.cache_conditions:
return None
cache_dir = self._GetCacheDirForRead()
if not cache_dir:
return None
try:
cache_file = os.path.join(cache_dir, RESULTS_FILE)
self._logger.LogOutput("Trying to read from cache file: %s" % cache_file)
with open(cache_file, "rb") as f:
result = pickle.load(f)
if (result.retval == 0 or
CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
return result
except Exception, e:
if CacheConditions.CACHE_FILE_EXISTS not in self.cache_conditions:
# Cache file not found but just return a failure.
return Result("", "", 1, {})
raise e
def StoreResult(self, result):
cache_dir = self._GetCacheDirForWrite()
cache_file = os.path.join(cache_dir, RESULTS_FILE)
command = "mkdir -p %s" % cache_dir
ret = self._ce.RunCommand(command)
assert ret == 0, "Couldn't create cache dir"
with open(cache_file, "wb") as f:
pickle.dump(result, f)
def StoreAutotestOutput(self, results_dir):
host_results_dir = os.path.join(self.chromeos_root, "chroot",
results_dir[1:])
tarball = os.path.join(self._GetCacheDirForWrite(), AUTOTEST_TARBALL)
command = ("cd %s && tar cjf %s ." % (host_results_dir, tarball))
ret = self._ce.RunCommand(command)
if ret:
raise Exception("Couldn't store autotest output directory.")
def ReadAutotestOutput(self, destination):
cache_dir = self._GetCacheDirForRead()
tarball = os.path.join(cache_dir, AUTOTEST_TARBALL)
if not os.path.exists(tarball):
raise Exception("Cached autotest tarball does not exist at '%s'." %
tarball)
command = ("cd %s && tar xjf %s ." % (destination, tarball))
ret = self._ce.RunCommand(command)
if ret:
raise Exception("Couldn't read autotest output directory.")
def StorePerfResults(self, perf):
perf_path = os.path.join(self._GetCacheDirForWrite(), PERF_RESULTS_FILE)
with open(perf_path, "wb") as f:
pickle.dump(perf.report, f)
pickle.dump(perf.output, f)
def ReadPerfResults(self):
cache_dir = self._GetCacheDirForRead()
perf_path = os.path.join(cache_dir, PERF_RESULTS_FILE)
with open(perf_path, "rb") as f:
report = pickle.load(f)
output = pickle.load(f)
return PerfProcessor.PerfResults(report, output)
def _ConvertToFilename(self, text):
ret = text
ret = re.sub("/", "__", ret)
ret = re.sub(" ", "_", ret)
ret = re.sub("=", "", ret)
ret = re.sub("\"", "", ret)
return ret
class MockResultsCache(object):
def Init(self, *args):
pass
def ReadResult(self):
return Result("Results placed in /tmp/test", "", 0)
def StoreResult(self, result):
pass
def StoreAutotestOutput(self, results_dir):
pass
def ReadAutotestOutput(self, destination):
pass
def StorePerfResults(self, perf):
pass
def ReadPerfResults(self):
return PerfProcessor.PerfResults("", "")
|
