1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
|
# Copyright (c) 2014-2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download images from Cloud Storage."""
from __future__ import print_function
import ast
import os
import test_flag
from cros_utils import command_executer
class MissingImage(Exception):
"""Raised when the requested image does not exist in gs://"""
class MissingFile(Exception):
"""Raised when the requested file does not exist in gs://"""
class ImageDownloader(object):
"""Download images from Cloud Storage."""
def __init__(self, logger_to_use=None, log_level='verbose', cmd_exec=None):
self._logger = logger_to_use
self.log_level = log_level
self._ce = cmd_exec or command_executer.GetCommandExecuter(
self._logger, log_level=self.log_level)
def GetBuildID(self, chromeos_root, xbuddy_label):
# Get the translation of the xbuddy_label into the real Google Storage
# image name.
command = ('cd ~/trunk/src/third_party/toolchain-utils/crosperf; '
"python translate_xbuddy.py '%s'" % xbuddy_label)
_, build_id_tuple_str, _ = self._ce.ChrootRunCommandWOutput(chromeos_root,
command)
if not build_id_tuple_str:
raise MissingImage("Unable to find image for '%s'" % xbuddy_label)
build_id_tuple = ast.literal_eval(build_id_tuple_str)
build_id = build_id_tuple[0]
return build_id
def DownloadImage(self, chromeos_root, build_id, image_name):
if self.log_level == 'average':
self._logger.LogOutput('Preparing to download %s image to local '
'directory.' % build_id)
# Make sure the directory for downloading the image exists.
download_path = os.path.join(chromeos_root, 'chroot/tmp', build_id)
image_path = os.path.join(download_path, 'chromiumos_test_image.bin')
if not os.path.exists(download_path):
os.makedirs(download_path)
# Check to see if the image has already been downloaded. If not,
# download the image.
if not os.path.exists(image_path):
command = 'gsutil cp %s /tmp/%s' % (image_name, build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
status = self._ce.ChrootRunCommand(chromeos_root, command)
downloaded_image_name = os.path.join(download_path,
'chromiumos_test_image.tar.xz')
if status != 0 or not os.path.exists(downloaded_image_name):
raise MissingImage('Cannot download image: %s.' % downloaded_image_name)
return image_path
def UncompressImage(self, chromeos_root, build_id):
# Check to see if the file has already been uncompresssed, etc.
if os.path.exists(
os.path.join(chromeos_root, 'chroot/tmp', build_id,
'chromiumos_test_image.bin')):
return
# Uncompress and untar the downloaded image.
command = ('cd /tmp/%s ; tar -Jxf chromiumos_test_image.tar.xz ' % build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Uncompressing and un-tarring may take a couple of minutes...'
'please be patient.)')
try:
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
raise MissingImage('Cannot uncompress image: %s.' % build_id)
except Exception:
# Exception in uncompressing file, so cleanup
command = ('cd /tmp/%s ; rm -f chromiumos_test_image.bin; ' % build_id)
# Remove the partially extracted bin file to avoid issues with future runs
_ = self._ce.ChrootRunCommand(chromeos_root, command)
# Raise exception again
raise
# Remove uncompressed file
command = ('cd /tmp/%s ; rm -f chromiumos_test_image.tar.xz; ' % build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Removing file chromiumos_test_image.tar.xz.)')
# try removing file, its ok to have an error, print if encountered
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
print('(Warning: Could not remove file chromiumos_test_image.tar.xz .)')
def DownloadSingleAutotestFile(self, chromeos_root, build_id,
package_file_name):
# Verify if package files exist
status = 0
gs_package_name = ('gs://chromeos-image-archive/%s/%s' %
(build_id, package_file_name))
if not test_flag.GetTestMode():
cmd = 'gsutil ls %s' % gs_package_name
status = self._ce.ChrootRunCommand(chromeos_root, cmd)
if status != 0:
raise MissingFile('Cannot find autotest package file: %s.' %
package_file_name)
if self.log_level == 'average':
self._logger.LogOutput('Preparing to download %s package to local '
'directory.' % package_file_name)
# Make sure the directory for downloading the package exists.
download_path = os.path.join(chromeos_root, 'chroot/tmp', build_id)
package_path = os.path.join(download_path, package_file_name)
if not os.path.exists(download_path):
os.makedirs(download_path)
# Check to see if the package file has already been downloaded. If not,
# download it.
if not os.path.exists(package_path):
command = 'gsutil cp %s /tmp/%s' % (gs_package_name, build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
status = self._ce.ChrootRunCommand(chromeos_root, command)
if status != 0 or not os.path.exists(package_path):
raise MissingFile('Cannot download package: %s .' % package_path)
def UncompressSingleAutotestFile(self, chromeos_root, build_id,
package_file_name, uncompress_cmd):
# Uncompress file
command = ('cd /tmp/%s ; %s %s' %
(build_id, uncompress_cmd, package_file_name))
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Uncompressing autotest file %s .)' % package_file_name)
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
raise MissingFile('Cannot uncompress file: %s.' % package_file_name)
# Remove uncompressed downloaded file
command = ('cd /tmp/%s ; rm -f %s' % (build_id, package_file_name))
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Removing processed autotest file %s .)' % package_file_name)
# try removing file, its ok to have an error, print if encountered
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
print('(Warning: Could not remove file %s .)' % package_file_name)
def VerifyAutotestFilesExist(self, chromeos_root, build_id, package_file):
# Quickly verify if the files are there
status = 0
gs_package_name = ('gs://chromeos-image-archive/%s/%s' %
(build_id, package_file))
if not test_flag.GetTestMode():
cmd = 'gsutil ls %s' % gs_package_name
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % cmd)
status = self._ce.ChrootRunCommand(chromeos_root, cmd)
if status != 0:
print('(Warning: Could not find file %s )' % gs_package_name)
return 1
# Package exists on server
return 0
def DownloadAutotestFiles(self, chromeos_root, build_id):
# Download autest package files (3 files)
autotest_packages_name = ('autotest_packages.tar')
autotest_server_package_name = ('autotest_server_package.tar.bz2')
autotest_control_files_name = ('control_files.tar')
# Quickly verify if the files are there
# If not, just exit with warning
status = self.VerifyAutotestFilesExist(chromeos_root, build_id,
autotest_packages_name)
if status != 0:
default_autotest_dir = '~/trunk/src/third_party/autotest/files'
print('(Warning: Could not download autotest packages .)\n'
'(Warning: Defaulting autotest path to %s .' %
default_autotest_dir)
return default_autotest_dir
# Autotest directory relative path wrt chroot
autotest_rel_path = os.path.join('/tmp', build_id, 'autotest_files')
# Absolute Path to download files
autotest_path = os.path.join(chromeos_root, 'chroot/tmp', build_id,
'autotest_files')
if not os.path.exists(autotest_path):
self.DownloadSingleAutotestFile(chromeos_root, build_id,
autotest_packages_name)
self.DownloadSingleAutotestFile(chromeos_root, build_id,
autotest_server_package_name)
self.DownloadSingleAutotestFile(chromeos_root, build_id,
autotest_control_files_name)
self.UncompressSingleAutotestFile(chromeos_root, build_id,
autotest_packages_name, 'tar -xvf ')
self.UncompressSingleAutotestFile(chromeos_root, build_id,
autotest_server_package_name,
'tar -jxvf ')
self.UncompressSingleAutotestFile(chromeos_root, build_id,
autotest_control_files_name,
'tar -xvf ')
# Rename created autotest directory to autotest_files
command = ('cd /tmp/%s ; mv autotest autotest_files' % build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Moving downloaded autotest files to autotest_files)')
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
raise MissingFile('Could not create directory autotest_files')
return autotest_rel_path
def Run(self, chromeos_root, xbuddy_label, autotest_path):
build_id = self.GetBuildID(chromeos_root, xbuddy_label)
image_name = ('gs://chromeos-image-archive/%s/chromiumos_test_image.tar.xz'
% build_id)
# Verify that image exists for build_id, before attempting to
# download it.
status = 0
if not test_flag.GetTestMode():
cmd = 'gsutil ls %s' % image_name
status = self._ce.ChrootRunCommand(chromeos_root, cmd)
if status != 0:
raise MissingImage('Cannot find official image: %s.' % image_name)
image_path = self.DownloadImage(chromeos_root, build_id, image_name)
self.UncompressImage(chromeos_root, build_id)
if self.log_level != 'quiet':
self._logger.LogOutput('Using image from %s.' % image_path)
if autotest_path == '':
autotest_path = self.DownloadAutotestFiles(chromeos_root, build_id)
return image_path, autotest_path
|
