1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script for running llvm validation tests on ChromeOS.
This script launches a buildbot to build ChromeOS with the llvm on
a particular board; then it finds and downloads the trybot image and the
corresponding official image, and runs test for correctness.
It then generates a report, emails it to the c-compiler-chrome, as
well as copying the result into a directory.
"""
# Script to test different toolchains against ChromeOS benchmarks.
from __future__ import print_function
import argparse
import datetime
import os
import sys
import time
from cros_utils import command_executer
from cros_utils import logger
from cros_utils import buildbot_utils
CROSTC_ROOT = '/usr/local/google/crostc'
ROLE_ACCOUNT = 'mobiletc-prebuild'
TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
MAIL_PROGRAM = '~/var/bin/mail-sheriff'
VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
START_DATE = datetime.date(2016, 1, 1)
TEST_PER_DAY = 4
DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/'
# Information about Rotating Boards
# Board Arch Reference Platform Kernel
# Board Version
# ------------ ------- ------------ ------------- -------
# atlas x86_64 poppy kabylake-y 4.4.*
# cave x86_64 glados skylake-y 3.18
# coral x86_64 reef apollo lake 4.4.*
# cyan x86_64 strago braswell 3.18
# elm aarch64 oak mediatek-8173 3.18 arm32 userspace
# eve x86_64 poppy kabylake-u/r 4.4.*
# gale armv7 3.18
# grunt x86_64 grunt stoney ridge 4.14.*
# fizz-moblab x86_64 4.4
# kevin aarch64 gru rockchip-3399 4.4.* arm32 userspace
# kevin64 aarch64 gru rockchip-3399 4.4.* arm64 userspace
# lakitu x86_64 4.4.*
# nyan_kitty armv7 nyan tegra 3.10.18
# octopus x86_64 octopus GLK 4.14.*
# sentry x86_64 kunimitsu skylake-u 3.18
# tidus x86_64 auron broadwell 3.14
# veyron_mighty armv7 veyron-pinky rockchip-3288 3.14
# whirlwind armv7 3.14
# winky x86_64 rambi baytrail 4.4.*
TEST_BOARD = [
'atlas',
'cave',
'coral',
'cyan',
'elm',
# 'eve', tested by amd64-llvm-next-toolchain builder.
'gale',
'grunt',
'fizz-moblab',
# 'kevin', tested by arm64-llvm-next-toolchain builder.
'kevin64',
'lakitu',
'nyan_kitty',
'octopus',
'sentry',
'tidus',
# 'veyron_mighty', tested by arm-llvm-next-toolchain builder.
'whirlwind',
'winky',
]
class ToolchainVerifier(object):
"""Class for the toolchain verifier."""
def __init__(self, board, chromeos_root, weekday, patches, compiler):
self._board = board
self._chromeos_root = chromeos_root
self._base_dir = os.getcwd()
self._ce = command_executer.GetCommandExecuter()
self._l = logger.GetLogger()
self._compiler = compiler
self._build = '%s-%s-toolchain-tryjob' % (board, compiler)
self._patches = patches.split(',') if patches else []
self._patches_string = '_'.join(str(p) for p in self._patches)
if not weekday:
self._weekday = time.strftime('%a')
else:
self._weekday = weekday
self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
def DoAll(self):
"""Main function inside ToolchainComparator class.
Launch trybot, get image names, create crosperf experiment file, run
crosperf, and copy images into seven-day report directories.
"""
buildbucket_id, _ = buildbot_utils.GetTrybotImage(
self._chromeos_root,
self._build,
self._patches,
tryjob_flags=['--hwtest'],
asynchronous=True)
return buildbucket_id
def WriteRotatingReportsData(results_dict, date):
"""Write data for waterfall report."""
fname = '%d-%02d-%02d.builds' % (date.year, date.month, date.day)
filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
with open(filename, 'w', encoding='utf-8') as out_file:
for board in results_dict.keys():
buildbucket_id = results_dict[board]
out_file.write('%s,%s\n' % (buildbucket_id, board))
def Main(argv):
"""The main function."""
# Common initializations
command_executer.InitCommandExecuter()
parser = argparse.ArgumentParser()
parser.add_argument(
'--chromeos_root',
dest='chromeos_root',
help='The chromeos root from which to run tests.')
parser.add_argument(
'--weekday',
default='',
dest='weekday',
help='The day of the week for which to run tests.')
parser.add_argument(
'--board', default='', dest='board', help='The board to test.')
parser.add_argument(
'--patch',
dest='patches',
default='',
help='The patches to use for the testing, '
"seprate the patch numbers with ',' "
'for more than one patches.')
parser.add_argument(
'--compiler',
dest='compiler',
help='Which compiler (llvm, llvm-next or gcc) to use for '
'testing.')
options = parser.parse_args(argv[1:])
if not options.chromeos_root:
print('Please specify the ChromeOS root directory.')
return 1
if not options.compiler:
print('Please specify which compiler to test (gcc, llvm, or llvm-next).')
return 1
if options.board:
fv = ToolchainVerifier(options.board, options.chromeos_root,
options.weekday, options.patches, options.compiler)
return fv.DoAll()
today = datetime.date.today()
delta = today - START_DATE
days = delta.days
start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
results_dict = dict()
for i in range(TEST_PER_DAY):
try:
board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
options.patches, options.compiler)
buildbucket_id = fv.DoAll()
if buildbucket_id:
results_dict[board] = buildbucket_id
except SystemExit:
logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
with open(logfile, 'w', encoding='utf-8') as f:
f.write('Verifier got an exception, please check the log.\n')
WriteRotatingReportsData(results_dict, today)
if __name__ == '__main__':
retval = Main(sys.argv)
sys.exit(retval)
|
