1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
#!/usr/bin/env python2
#
# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script for running llvm validation tests on ChromeOS.
This script launches a buildbot to build ChromeOS with the llvm on
a particular board; then it finds and downloads the trybot image and the
corresponding official image, and runs test for correctness.
It then generates a report, emails it to the c-compiler-chrome, as
well as copying the result into a directory.
"""
# Script to test different toolchains against ChromeOS benchmarks.
from __future__ import print_function
import argparse
import datetime
import os
import sys
import time
from cros_utils import command_executer
from cros_utils import logger
from cros_utils import buildbot_utils
CROSTC_ROOT = '/usr/local/google/crostc'
ROLE_ACCOUNT = 'mobiletc-prebuild'
TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
MAIL_PROGRAM = '~/var/bin/mail-sheriff'
VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
START_DATE = datetime.date(2016, 1, 1)
TEST_PER_DAY = 3
# Information about Rotating Boards
# Board Arch Reference Platform Kernel
# Board Version
# -------- ------ --------- ---------- -------
# caroline x86_64 glados skylake-y 3.18
# daisy armv7 daisy exynos-5250 3.8.11
# eve x86_64 poppy kabylake-y 4.4.79
# fizz x86_64 fizz kabylake-u/r 4.4
# gale armv7 3.18
# kahlee x86_64 kahlee stoney ridge 4.14
# kevin armv7 gru rockchip-3399 4.4.79
# lakitu x86_64 4.4.79
# link x86_64 ivybridge ivybridge 3.8.11
# lumpy x86_64 -- sandybridge 3.8.11
# nyan_big armv7 nyan tegra 3.10.18
# peach_pit armv7 peach exynos-5420 3.8.11
# peppy x86_64 slippy haswell 3.8.11
# pyro x86_64 reef apollo lake 4.4.79
# sentry x86_64 kunimitsu skylake-u 3.18
# swanky x86_64 rambi baytrail 4.4.79
# terra x86_64 strago braswell 3.18
# whirlwind armv7 3.14
# zoombini x86_64 zoombini cannonlake-y 4.14
TEST_BOARD = [
'caroline',
'daisy',
'eve',
'fizz',
'gale',
'kahlee',
'kevin',
'lakitu',
'link',
'lumpy',
'nyan_big',
'peach_pit',
'peppy',
'pyro',
'sentry',
'swanky',
'terra',
'whirlwind',
'zoombini',
]
class ToolchainVerifier(object):
"""Class for the toolchain verifier."""
def __init__(self, board, chromeos_root, weekday, patches, compiler):
self._board = board
self._chromeos_root = chromeos_root
self._base_dir = os.getcwd()
self._ce = command_executer.GetCommandExecuter()
self._l = logger.GetLogger()
self._compiler = compiler
self._build = '%s-%s-toolchain-tryjob' % (board, compiler)
self._patches = patches.split(',') if patches else []
self._patches_string = '_'.join(str(p) for p in self._patches)
if not weekday:
self._weekday = time.strftime('%a')
else:
self._weekday = weekday
self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
def _FinishSetup(self):
"""Make sure testing_rsa file is properly set up."""
# Fix protections on ssh key
command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
'/chrome-src-internal/src/third_party/chromite/ssh_keys'
'/testing_rsa')
ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
if ret_val != 0:
raise RuntimeError('chmod for testing_rsa failed')
def DoAll(self, crostc_dir):
"""Main function inside ToolchainComparator class.
Launch trybot, get image names, create crosperf experiment file, run
crosperf, and copy images into seven-day report directories.
"""
date_str = datetime.date.today()
description = 'master_%s_%s_%s' % (self._patches_string, self._build,
date_str)
if crostc_dir:
_ = buildbot_utils.GetTrybotImage(
self._chromeos_root,
self._build,
self._patches,
description,
tryjob_flags=['--hwtest'],
credentials_dir=crostc_dir,
async=True)
else:
_ = buildbot_utils.GetTrybotImage(
self._chromeos_root,
self._build,
self._patches,
description,
tryjob_flags=['--hwtest'],
async=True)
return 0
def Main(argv):
"""The main function."""
# Common initializations
command_executer.InitCommandExecuter()
parser = argparse.ArgumentParser()
parser.add_argument(
'--chromeos_root',
dest='chromeos_root',
help='The chromeos root from which to run tests.')
parser.add_argument(
'--weekday',
default='',
dest='weekday',
help='The day of the week for which to run tests.')
parser.add_argument(
'--board', default='', dest='board', help='The board to test.')
parser.add_argument(
'--patch',
dest='patches',
default='',
help='The patches to use for the testing, '
"seprate the patch numbers with ',' "
'for more than one patches.')
parser.add_argument(
'--compiler',
dest='compiler',
help='Which compiler (llvm, llvm-next or gcc) to use for '
'testing.')
parser.add_argument(
'--crostc_dir',
dest='crostc_dir',
help='Path to the directory containing the '
'chromeos-toolchain-credentials.json file; normally in the '
'crostc repo.')
options = parser.parse_args(argv[1:])
if not options.chromeos_root:
print('Please specify the ChromeOS root directory.')
return 1
if not options.compiler:
print('Please specify which compiler to test (gcc, llvm, or llvm-next).')
return 1
if options.board:
fv = ToolchainVerifier(options.board, options.chromeos_root,
options.weekday, options.patches, options.compiler)
return fv.Doall()
today = datetime.date.today()
delta = today - START_DATE
days = delta.days
start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
for i in range(TEST_PER_DAY):
try:
board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
options.patches, options.compiler)
fv.DoAll(options.crostc_dir)
except SystemExit:
logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
with open(logfile, 'w') as f:
f.write('Verifier got an exception, please check the log.\n')
if __name__ == '__main__':
retval = Main(sys.argv)
sys.exit(retval)
|
