1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
This recipe can be used by components like v8 to verify blink tests with a
low false positive rate. Similar to a trybot, this recipe compares test
failures from a build with a current component revision with test failures
from a build with a pinned component revision.
Summary of the recipe flow:
1. Sync chromium to HEAD
2. Sync blink to HEAD
3. Sync component X to revision Y
4. Run blink tests
-> In case of failures:
5. Sync chromium to same revision as 1
6. Sync blink to same revision as 2
7. Sync component X to pinned revision from DEPS file
8. Run blink tests
-> If failures in 4 don't happen in 8, then revision Y reveals a problem not
present in the pinned revision
Revision Y will be the revision property as provided by buildbot or HEAD (i.e.
in a forced build with no revision provided).
"""
from recipe_engine.types import freeze
DEPS = [
'bot_update',
'chromium',
'chromium_tests',
'gclient',
'json',
'path',
'platform',
'properties',
'python',
'raw_io',
'step',
'test_utils',
]
def V8Builder(config, bits, platform):
chromium_configs = []
if config == 'Debug':
chromium_configs.append('v8_optimize_medium')
chromium_configs.append('v8_slow_dchecks')
return {
'gclient_apply_config': ['show_v8_revision'],
'chromium_apply_config': chromium_configs,
'chromium_config_kwargs': {
'BUILD_CONFIG': config,
'TARGET_BITS': bits,
},
'test_args': ['--no-pixel-tests'],
'additional_expectations': [
'v8', 'tools', 'blink_tests', 'TestExpectations',
],
'component': {'path': 'src/v8', 'revision': '%s'},
'testing': {'platform': platform},
}
BUILDERS = freeze({
'client.v8.fyi': {
'builders': {
'V8-Blink Win': V8Builder('Release', 32, 'win'),
'V8-Blink Mac': V8Builder('Release', 64, 'mac'),
'V8-Blink Linux 32': V8Builder('Release', 32, 'linux'),
'V8-Blink Linux 64': V8Builder('Release', 64, 'linux'),
'V8-Blink Linux 64 (dbg)': V8Builder('Debug', 64, 'linux'),
},
},
})
def RunSteps(api):
mastername = api.properties.get('mastername')
buildername = api.properties.get('buildername')
master_dict = BUILDERS.get(mastername, {})
bot_config = master_dict.get('builders', {}).get(buildername)
# Sync chromium to HEAD.
api.gclient.set_config('chromium')
api.gclient.c.revisions['src'] = 'HEAD'
api.chromium.set_config('blink',
**bot_config.get('chromium_config_kwargs', {}))
for c in bot_config.get('gclient_apply_config', []):
api.gclient.apply_config(c)
for c in bot_config.get('chromium_apply_config', []):
api.chromium.apply_config(c)
# Sync component to current component revision.
component_revision = api.properties.get('revision') or 'HEAD'
api.gclient.c.revisions[bot_config['component']['path']] = (
bot_config['component']['revision'] % component_revision)
# Ensure we remember the chromium revision.
api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision'
step_result = api.bot_update.ensure_checkout(force=True)
api.chromium.runhooks()
api.chromium.compile()
api.chromium.runtest('webkit_unit_tests', xvfb=True)
def component_pinned_fn(_failing_steps):
bot_update_json = step_result.json.output
api.gclient.c.revisions['src'] = str(
bot_update_json['properties']['got_cr_revision'])
# Reset component revision to the pinned revision from chromium's DEPS
# for comparison.
del api.gclient.c.revisions[bot_config['component']['path']]
# Update without changing got_revision. The first sync is the revision
# that is tested. The second is just for comparison. Setting got_revision
# again confuses the waterfall's console view.
api.bot_update.ensure_checkout(force=True, update_presentation=False)
api.chromium.runhooks()
api.chromium.compile()
extra_args = list(bot_config.get('test_args', []))
if bot_config.get('additional_expectations'):
extra_args.extend([
'--additional-expectations',
api.path['checkout'].join(*bot_config['additional_expectations']),
])
tests = [
api.chromium_tests.steps.BlinkTest(extra_args=extra_args),
]
api.test_utils.determine_new_failures(api, tests, component_pinned_fn)
def _sanitize_nonalpha(text):
return ''.join(c if c.isalnum() else '_' for c in text)
def GenTests(api):
canned_test = api.test_utils.canned_test_output
with_patch = 'webkit_tests (with patch)'
without_patch = 'webkit_tests (without patch)'
def properties(mastername, buildername):
return (
api.properties.generic(mastername=mastername,
buildername=buildername,
revision='20123')
)
for mastername, master_config in BUILDERS.iteritems():
for buildername, bot_config in master_config['builders'].iteritems():
test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername),
_sanitize_nonalpha(buildername))
tests = []
for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')):
test = (
properties(mastername, buildername) +
api.platform(
bot_config['testing']['platform'],
bot_config.get(
'chromium_config_kwargs', {}).get('TARGET_BITS', 64)) +
api.test(test_name + suffix) +
api.override_step_data(with_patch, canned_test(passing=pass_first))
)
if not pass_first:
test += api.override_step_data(
without_patch, canned_test(passing=False, minimal=True))
tests.append(test)
for test in tests:
yield test
# This tests that if the first fails, but the second pass succeeds
# that we fail the whole build.
yield (
api.test('minimal_pass_continues') +
properties('client.v8.fyi', 'V8-Blink Linux 64') +
api.override_step_data(with_patch, canned_test(passing=False)) +
api.override_step_data(without_patch,
canned_test(passing=True, minimal=True))
)
# This tests what happens if something goes horribly wrong in
# run-webkit-tests and we return an internal error; the step should
# be considered a hard failure and we shouldn't try to compare the
# lists of failing tests.
# 255 == test_run_results.UNEXPECTED_ERROR_EXIT_STATUS in run-webkit-tests.
yield (
api.test('webkit_tests_unexpected_error') +
properties('client.v8.fyi', 'V8-Blink Linux 64') +
api.override_step_data(with_patch, canned_test(passing=False, retcode=255))
)
# TODO(dpranke): crbug.com/357866 . This tests what happens if we exceed the
# number of failures specified with --exit-after-n-crashes-or-times or
# --exit-after-n-failures; the step should be considered a hard failure and
# we shouldn't try to compare the lists of failing tests.
# 130 == test_run_results.INTERRUPTED_EXIT_STATUS in run-webkit-tests.
yield (
api.test('webkit_tests_interrupted') +
properties('client.v8.fyi', 'V8-Blink Linux 64') +
api.override_step_data(with_patch, canned_test(passing=False, retcode=130))
)
# This tests what happens if we don't trip the thresholds listed
# above, but fail more tests than we can safely fit in a return code.
# (this should be a soft failure and we can still retry w/o the patch
# and compare the lists of failing tests).
yield (
api.test('too_many_failures_for_retcode') +
properties('client.v8.fyi', 'V8-Blink Linux 64') +
api.override_step_data(with_patch,
canned_test(passing=False,
num_additional_failures=125)) +
api.override_step_data(without_patch,
canned_test(passing=True, minimal=True))
)
|
