diff options
Diffstat (limited to 'deprecated/automation/clients/report/validate_failures.py')
| -rwxr-xr-x | deprecated/automation/clients/report/validate_failures.py | 239 |
1 files changed, 239 insertions, 0 deletions
diff --git a/deprecated/automation/clients/report/validate_failures.py b/deprecated/automation/clients/report/validate_failures.py new file mode 100755 index 00000000..d8776ba5 --- /dev/null +++ b/deprecated/automation/clients/report/validate_failures.py @@ -0,0 +1,239 @@ +#!/usr/bin/python2 + +# Script to compare testsuite failures against a list of known-to-fail +# tests. + +# Contributed by Diego Novillo <dnovillo@google.com> +# Overhaul by Krystian Baclawski <kbaclawski@google.com> +# +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +"""This script provides a coarser XFAILing mechanism that requires no +detailed DejaGNU markings. This is useful in a variety of scenarios: + +- Development branches with many known failures waiting to be fixed. +- Release branches with known failures that are not considered + important for the particular release criteria used in that branch. + +The script must be executed from the toplevel build directory. When +executed it will: + +1) Determine the target built: TARGET +2) Determine the source directory: SRCDIR +3) Look for a failure manifest file in + <SRCDIR>/contrib/testsuite-management/<TARGET>.xfail +4) Collect all the <tool>.sum files from the build tree. +5) Produce a report stating: + a) Failures expected in the manifest but not present in the build. + b) Failures in the build not expected in the manifest. +6) If all the build failures are expected in the manifest, it exits + with exit code 0. Otherwise, it exits with error code 1. +""" + +import optparse +import logging +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from dejagnu.manifest import Manifest +from dejagnu.summary import DejaGnuTestResult +from dejagnu.summary import DejaGnuTestRun + +# Pattern for naming manifest files. The first argument should be +# the toplevel GCC source directory. The second argument is the +# target triple used during the build. +_MANIFEST_PATH_PATTERN = '%s/contrib/testsuite-management/%s.xfail' + + +def GetMakefileVars(makefile_path): + assert os.path.exists(makefile_path) + + with open(makefile_path) as lines: + kvs = [line.split('=', 1) for line in lines if '=' in line] + + return dict((k.strip(), v.strip()) for k, v in kvs) + + +def GetSumFiles(build_dir): + summaries = [] + + for root, _, filenames in os.walk(build_dir): + summaries.extend([os.path.join(root, filename) + for filename in filenames if filename.endswith('.sum')]) + + return map(os.path.normpath, summaries) + + +def ValidBuildDirectory(build_dir, target): + mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')] + + extra_paths = [os.path.join(build_dir, target), + os.path.join(build_dir, 'build-%s' % target)] + + return (all(map(os.path.exists, mandatory_paths)) and + any(map(os.path.exists, extra_paths))) + + +def GetManifestPath(build_dir): + makefile = GetMakefileVars(os.path.join(build_dir, 'Makefile')) + srcdir = makefile['srcdir'] + target = makefile['target'] + + if not ValidBuildDirectory(build_dir, target): + target = makefile['target_alias'] + + if not ValidBuildDirectory(build_dir, target): + logging.error('%s is not a valid GCC top level build directory.', build_dir) + sys.exit(1) + + logging.info('Discovered source directory: "%s"', srcdir) + logging.info('Discovered build target: "%s"', target) + + return _MANIFEST_PATH_PATTERN % (srcdir, target) + + +def CompareResults(manifest, actual): + """Compare sets of results and return two lists: + - List of results present in MANIFEST but missing from ACTUAL. + - List of results present in ACTUAL but missing from MANIFEST. + """ + # Report all the actual results not present in the manifest. + actual_vs_manifest = actual - manifest + + # Filter out tests marked flaky. + manifest_without_flaky_tests = set(filter(lambda result: not result.flaky, + manifest)) + + # Simlarly for all the tests in the manifest. + manifest_vs_actual = manifest_without_flaky_tests - actual + + return actual_vs_manifest, manifest_vs_actual + + +def LogResults(level, results): + log_fun = getattr(logging, level) + + for num, result in enumerate(sorted(results), start=1): + log_fun(' %d) %s', num, result) + + +def CheckExpectedResults(manifest_path, build_dir): + logging.info('Reading manifest file: "%s"', manifest_path) + + manifest = set(Manifest.FromFile(manifest_path)) + + logging.info('Getting actual results from build directory: "%s"', + os.path.realpath(build_dir)) + + summaries = GetSumFiles(build_dir) + + actual = set() + + for summary in summaries: + test_run = DejaGnuTestRun.FromFile(summary) + failures = set(Manifest.FromDejaGnuTestRun(test_run)) + actual.update(failures) + + if manifest: + logging.debug('Tests expected to fail:') + LogResults('debug', manifest) + + if actual: + logging.debug('Actual test failures:') + LogResults('debug', actual) + + actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual) + + if actual_vs_manifest: + logging.info('Build results not in the manifest:') + LogResults('info', actual_vs_manifest) + + if manifest_vs_actual: + logging.info('Manifest results not present in the build:') + LogResults('info', manifest_vs_actual) + logging.info('NOTE: This is not a failure! ', + 'It just means that the manifest expected these tests to ' + 'fail, but they worked in this configuration.') + + if actual_vs_manifest or manifest_vs_actual: + sys.exit(1) + + logging.info('No unexpected failures.') + + +def ProduceManifest(manifest_path, build_dir, overwrite): + if os.path.exists(manifest_path) and not overwrite: + logging.error('Manifest file "%s" already exists.', manifest_path) + logging.error('Use --force to overwrite.') + sys.exit(1) + + testruns = map(DejaGnuTestRun.FromFile, GetSumFiles(build_dir)) + manifests = map(Manifest.FromDejaGnuTestRun, testruns) + + with open(manifest_path, 'w') as manifest_file: + manifest_strings = [manifest.Generate() for manifest in manifests] + logging.info('Writing manifest to "%s".', manifest_path) + manifest_file.write('\n'.join(manifest_strings)) + + +def Main(argv): + parser = optparse.OptionParser(usage=__doc__) + parser.add_option( + '-b', + '--build_dir', + dest='build_dir', + action='store', + metavar='PATH', + default=os.getcwd(), + help='Build directory to check. (default: current directory)') + parser.add_option('-m', + '--manifest', + dest='manifest', + action='store_true', + help='Produce the manifest for the current build.') + parser.add_option( + '-f', + '--force', + dest='force', + action='store_true', + help=('Overwrite an existing manifest file, if user requested creating ' + 'new one. (default: False)')) + parser.add_option('-v', + '--verbose', + dest='verbose', + action='store_true', + help='Increase verbosity.') + options, _ = parser.parse_args(argv[1:]) + + if options.verbose: + logging.root.setLevel(logging.DEBUG) + + manifest_path = GetManifestPath(options.build_dir) + + if options.manifest: + ProduceManifest(manifest_path, options.build_dir, options.force) + else: + CheckExpectedResults(manifest_path, options.build_dir) + + +if __name__ == '__main__': + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + Main(sys.argv) |