1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
|
#!/usr/bin/python2
# Script to compare testsuite failures against a list of known-to-fail
# tests.
# Contributed by Diego Novillo <dnovillo@google.com>
# Overhaul by Krystian Baclawski <kbaclawski@google.com>
#
# Copyright (C) 2011 Free Software Foundation, Inc.
#
# This file is part of GCC.
#
# GCC is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GCC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GCC; see the file COPYING. If not, write to
# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
"""This script provides a coarser XFAILing mechanism that requires no
detailed DejaGNU markings. This is useful in a variety of scenarios:
- Development branches with many known failures waiting to be fixed.
- Release branches with known failures that are not considered
important for the particular release criteria used in that branch.
The script must be executed from the toplevel build directory. When
executed it will:
1) Determine the target built: TARGET
2) Determine the source directory: SRCDIR
3) Look for a failure manifest file in
<SRCDIR>/contrib/testsuite-management/<TARGET>.xfail
4) Collect all the <tool>.sum files from the build tree.
5) Produce a report stating:
a) Failures expected in the manifest but not present in the build.
b) Failures in the build not expected in the manifest.
6) If all the build failures are expected in the manifest, it exits
with exit code 0. Otherwise, it exits with error code 1.
"""
import optparse
import logging
import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from dejagnu.manifest import Manifest
from dejagnu.summary import DejaGnuTestResult
from dejagnu.summary import DejaGnuTestRun
# Pattern for naming manifest files. The first argument should be
# the toplevel GCC source directory. The second argument is the
# target triple used during the build.
_MANIFEST_PATH_PATTERN = '%s/contrib/testsuite-management/%s.xfail'
def GetMakefileVars(makefile_path):
assert os.path.exists(makefile_path)
with open(makefile_path) as lines:
kvs = [line.split('=', 1) for line in lines if '=' in line]
return dict((k.strip(), v.strip()) for k, v in kvs)
def GetSumFiles(build_dir):
summaries = []
for root, _, filenames in os.walk(build_dir):
summaries.extend([os.path.join(root, filename)
for filename in filenames if filename.endswith('.sum')])
return map(os.path.normpath, summaries)
def ValidBuildDirectory(build_dir, target):
mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')]
extra_paths = [os.path.join(build_dir, target),
os.path.join(build_dir, 'build-%s' % target)]
return (all(map(os.path.exists, mandatory_paths)) and
any(map(os.path.exists, extra_paths)))
def GetManifestPath(build_dir):
makefile = GetMakefileVars(os.path.join(build_dir, 'Makefile'))
srcdir = makefile['srcdir']
target = makefile['target']
if not ValidBuildDirectory(build_dir, target):
target = makefile['target_alias']
if not ValidBuildDirectory(build_dir, target):
logging.error('%s is not a valid GCC top level build directory.', build_dir)
sys.exit(1)
logging.info('Discovered source directory: "%s"', srcdir)
logging.info('Discovered build target: "%s"', target)
return _MANIFEST_PATH_PATTERN % (srcdir, target)
def CompareResults(manifest, actual):
"""Compare sets of results and return two lists:
- List of results present in MANIFEST but missing from ACTUAL.
- List of results present in ACTUAL but missing from MANIFEST.
"""
# Report all the actual results not present in the manifest.
actual_vs_manifest = actual - manifest
# Filter out tests marked flaky.
manifest_without_flaky_tests = set(filter(lambda result: not result.flaky,
manifest))
# Simlarly for all the tests in the manifest.
manifest_vs_actual = manifest_without_flaky_tests - actual
return actual_vs_manifest, manifest_vs_actual
def LogResults(level, results):
log_fun = getattr(logging, level)
for num, result in enumerate(sorted(results), start=1):
log_fun(' %d) %s', num, result)
def CheckExpectedResults(manifest_path, build_dir):
logging.info('Reading manifest file: "%s"', manifest_path)
manifest = set(Manifest.FromFile(manifest_path))
logging.info('Getting actual results from build directory: "%s"',
os.path.realpath(build_dir))
summaries = GetSumFiles(build_dir)
actual = set()
for summary in summaries:
test_run = DejaGnuTestRun.FromFile(summary)
failures = set(Manifest.FromDejaGnuTestRun(test_run))
actual.update(failures)
if manifest:
logging.debug('Tests expected to fail:')
LogResults('debug', manifest)
if actual:
logging.debug('Actual test failures:')
LogResults('debug', actual)
actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual)
if actual_vs_manifest:
logging.info('Build results not in the manifest:')
LogResults('info', actual_vs_manifest)
if manifest_vs_actual:
logging.info('Manifest results not present in the build:')
LogResults('info', manifest_vs_actual)
logging.info('NOTE: This is not a failure! ',
'It just means that the manifest expected these tests to '
'fail, but they worked in this configuration.')
if actual_vs_manifest or manifest_vs_actual:
sys.exit(1)
logging.info('No unexpected failures.')
def ProduceManifest(manifest_path, build_dir, overwrite):
if os.path.exists(manifest_path) and not overwrite:
logging.error('Manifest file "%s" already exists.', manifest_path)
logging.error('Use --force to overwrite.')
sys.exit(1)
testruns = map(DejaGnuTestRun.FromFile, GetSumFiles(build_dir))
manifests = map(Manifest.FromDejaGnuTestRun, testruns)
with open(manifest_path, 'w') as manifest_file:
manifest_strings = [manifest.Generate() for manifest in manifests]
logging.info('Writing manifest to "%s".', manifest_path)
manifest_file.write('\n'.join(manifest_strings))
def Main(argv):
parser = optparse.OptionParser(usage=__doc__)
parser.add_option(
'-b',
'--build_dir',
dest='build_dir',
action='store',
metavar='PATH',
default=os.getcwd(),
help='Build directory to check. (default: current directory)')
parser.add_option('-m',
'--manifest',
dest='manifest',
action='store_true',
help='Produce the manifest for the current build.')
parser.add_option(
'-f',
'--force',
dest='force',
action='store_true',
help=('Overwrite an existing manifest file, if user requested creating '
'new one. (default: False)'))
parser.add_option('-v',
'--verbose',
dest='verbose',
action='store_true',
help='Increase verbosity.')
options, _ = parser.parse_args(argv[1:])
if options.verbose:
logging.root.setLevel(logging.DEBUG)
manifest_path = GetManifestPath(options.build_dir)
if options.manifest:
ProduceManifest(manifest_path, options.build_dir, options.force)
else:
CheckExpectedResults(manifest_path, options.build_dir)
if __name__ == '__main__':
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
Main(sys.argv)
|
