1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
|
# Copyright (c) 2014-2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download images from Cloud Storage."""
from __future__ import print_function
import ast
import os
import test_flag
from cros_utils import command_executer
class MissingImage(Exception):
"""Raised when the requested image does not exist in gs://"""
class MissingFile(Exception):
"""Raised when the requested file does not exist in gs://"""
class ImageDownloader(object):
"""Download images from Cloud Storage."""
def __init__(self, logger_to_use=None, log_level='verbose', cmd_exec=None):
self._logger = logger_to_use
self.log_level = log_level
self._ce = cmd_exec or command_executer.GetCommandExecuter(
self._logger, log_level=self.log_level)
def GetBuildID(self, chromeos_root, xbuddy_label):
# Get the translation of the xbuddy_label into the real Google Storage
# image name.
command = ('cd ~/trunk/src/third_party/toolchain-utils/crosperf; '
"python translate_xbuddy.py '%s'" % xbuddy_label)
_, build_id_tuple_str, _ = self._ce.ChrootRunCommandWOutput(chromeos_root,
command)
if not build_id_tuple_str:
raise MissingImage("Unable to find image for '%s'" % xbuddy_label)
build_id_tuple = ast.literal_eval(build_id_tuple_str)
build_id = build_id_tuple[0]
return build_id
def DownloadImage(self, chromeos_root, build_id, image_name):
if self.log_level == 'average':
self._logger.LogOutput('Preparing to download %s image to local '
'directory.' % build_id)
# Make sure the directory for downloading the image exists.
download_path = os.path.join(chromeos_root, 'chroot/tmp', build_id)
image_path = os.path.join(download_path, 'chromiumos_test_image.bin')
if not os.path.exists(download_path):
os.makedirs(download_path)
# Check to see if the image has already been downloaded. If not,
# download the image.
if not os.path.exists(image_path):
command = 'gsutil cp %s /tmp/%s' % (image_name, build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
status = self._ce.ChrootRunCommand(chromeos_root, command)
downloaded_image_name = os.path.join(download_path,
'chromiumos_test_image.tar.xz')
if status != 0 or not os.path.exists(downloaded_image_name):
raise MissingImage('Cannot download image: %s.' % downloaded_image_name)
return image_path
def UncompressImage(self, chromeos_root, build_id):
# Check to see if the file has already been uncompresssed, etc.
if os.path.exists(
os.path.join(chromeos_root, 'chroot/tmp', build_id,
'chromiumos_test_image.bin')):
return
# Uncompress and untar the downloaded image.
command = ('cd /tmp/%s ;unxz chromiumos_test_image.tar.xz; '
'tar -xvf chromiumos_test_image.tar' % build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Uncompressing and un-tarring may take a couple of minutes...'
'please be patient.)')
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
raise MissingImage('Cannot uncompress image: %s.' % build_id)
def DownloadSingleAutotestFile(self, chromeos_root, build_id,
package_file_name):
# Verify if package files exist
status = 0
gs_package_name = ('gs://chromeos-image-archive/%s/%s' %
(build_id, package_file_name))
if not test_flag.GetTestMode():
cmd = 'gsutil ls %s' % gs_package_name
status = self._ce.ChrootRunCommand(chromeos_root, cmd)
if status != 0:
raise MissingFile('Cannot find autotest package file: %s.' %
package_file_name)
if self.log_level == 'average':
self._logger.LogOutput('Preparing to download %s package to local '
'directory.' % package_file_name)
# Make sure the directory for downloading the package exists.
download_path = os.path.join(chromeos_root, 'chroot/tmp', build_id)
package_path = os.path.join(download_path, package_file_name)
if not os.path.exists(download_path):
os.makedirs(download_path)
# Check to see if the package file has already been downloaded. If not,
# download it.
if not os.path.exists(package_path):
command = 'gsutil cp %s /tmp/%s' % (gs_package_name, build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
status = self._ce.ChrootRunCommand(chromeos_root, command)
if status != 0 or not os.path.exists(package_path):
raise MissingFile('Cannot download package: %s .' % package_path)
def UncompressSingleAutotestFile(self, chromeos_root, build_id,
package_file_name, uncompress_cmd):
# Uncompress file
command = ('cd /tmp/%s ; %s %s' %
(build_id, uncompress_cmd, package_file_name))
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Uncompressing autotest file %s .)' % package_file_name)
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
raise MissingFile('Cannot uncompress file: %s.' % package_file_name)
# Remove uncompressed downloaded file
command = ('cd /tmp/%s ; rm -f %s' % (build_id, package_file_name))
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Removing processed autotest file %s .)' % package_file_name)
# try removing file, its ok to have an error, print if encountered
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
print('(Warning: Could not remove file %s .)' % package_file_name)
def DownloadAutotestFiles(self, chromeos_root, build_id):
# Download autest package files (3 files)
autotest_packages_name = ('autotest_packages.tar')
autotest_server_package_name = ('autotest_server_package.tar.bz2')
autotest_control_files_name = ('control_files.tar')
# Autotest directory relative path wrt chroot
autotest_rel_path = os.path.join('/tmp', build_id, 'autotest_files')
# Absolute Path to download files
autotest_path = os.path.join(chromeos_root, 'chroot/tmp', build_id,
'autotest_files')
if not os.path.exists(autotest_path):
self.DownloadSingleAutotestFile(chromeos_root, build_id,
autotest_packages_name)
self.DownloadSingleAutotestFile(chromeos_root, build_id,
autotest_server_package_name)
self.DownloadSingleAutotestFile(chromeos_root, build_id,
autotest_control_files_name)
self.UncompressSingleAutotestFile(chromeos_root, build_id,
autotest_packages_name, 'tar -xvf ')
self.UncompressSingleAutotestFile(chromeos_root, build_id,
autotest_server_package_name,
'tar -jxvf ')
self.UncompressSingleAutotestFile(chromeos_root, build_id,
autotest_control_files_name,
'tar -xvf ')
# Rename created autotest directory to autotest_files
command = ('cd /tmp/%s ; mv autotest autotest_files' % build_id)
if self.log_level != 'verbose':
self._logger.LogOutput('CMD: %s' % command)
print('(Moving downloaded autotest files to autotest_files)')
retval = self._ce.ChrootRunCommand(chromeos_root, command)
if retval != 0:
raise MissingFile('Could not create directory autotest_files')
return autotest_rel_path
def Run(self, chromeos_root, xbuddy_label, autotest_path):
build_id = self.GetBuildID(chromeos_root, xbuddy_label)
image_name = ('gs://chromeos-image-archive/%s/chromiumos_test_image.tar.xz'
% build_id)
# Verify that image exists for build_id, before attempting to
# download it.
status = 0
if not test_flag.GetTestMode():
cmd = 'gsutil ls %s' % image_name
status = self._ce.ChrootRunCommand(chromeos_root, cmd)
if status != 0:
raise MissingImage('Cannot find official image: %s.' % image_name)
image_path = self.DownloadImage(chromeos_root, build_id, image_name)
self.UncompressImage(chromeos_root, build_id)
if self.log_level != 'quiet':
self._logger.LogOutput('Using image from %s.' % image_path)
if autotest_path == '':
autotest_path = self.DownloadAutotestFiles(chromeos_root, build_id)
return image_path, autotest_path
|
