diff options
Diffstat (limited to 'deprecated/automation/clients/report')
8 files changed, 960 insertions, 0 deletions
diff --git a/deprecated/automation/clients/report/dejagnu.sh b/deprecated/automation/clients/report/dejagnu.sh new file mode 100755 index 00000000..fadd8a0c --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +export PYTHONPATH="$(pwd)" + +python dejagnu/main.py $@ diff --git a/deprecated/automation/clients/report/dejagnu/__init__.py b/deprecated/automation/clients/report/dejagnu/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/__init__.py @@ -0,0 +1 @@ + diff --git a/deprecated/automation/clients/report/dejagnu/main.py b/deprecated/automation/clients/report/dejagnu/main.py new file mode 100644 index 00000000..62f095e1 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/main.py @@ -0,0 +1,137 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +from contextlib import contextmanager +import glob +from itertools import chain +import logging +import optparse +import os.path +import sys + +from manifest import Manifest +import report +from summary import DejaGnuTestRun + + +def ExpandGlobExprList(paths): + """Returns an iterator that goes over expanded glob paths.""" + return chain.from_iterable(map(glob.glob, paths)) + + +@contextmanager +def OptionChecker(parser): + """Provides scoped environment for command line option checking.""" + try: + yield + except SystemExit as ex: + parser.print_help() + print '' + sys.exit('ERROR: %s' % str(ex)) + + +def ManifestCommand(argv): + parser = optparse.OptionParser( + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate manifest files. Manifest files store a list ' + 'of failed tests that should be ignored. Generated files are ' + 'stored in current directory under following name: ' + '${tool}-${board}.xfail (e.g. "gcc-unix.xfail").'), + usage='Usage: %prog manifest [file.sum] (file2.sum ...)') + + _, args = parser.parse_args(argv[2:]) + + with OptionChecker(parser): + if not args: + sys.exit('At least one *.sum file required.') + + for filename in chain.from_iterable(map(glob.glob, args)): + test_run = DejaGnuTestRun.FromFile(filename) + + manifest = Manifest.FromDejaGnuTestRun(test_run) + manifest_filename = '%s-%s.xfail' % (test_run.tool, test_run.board) + + with open(manifest_filename, 'w') as manifest_file: + manifest_file.write(manifest.Generate()) + + logging.info('Wrote manifest to "%s" file.', manifest_filename) + + +def ReportCommand(argv): + parser = optparse.OptionParser( + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate a single report file in selected format ' + '(currently only HTML).'), + usage=('Usage: %prog report (-m manifest.xfail) [-o report.html] ' + '[file.sum (file2.sum ...)')) + parser.add_option( + '-o', + dest='output', + type='string', + default=None, + help=('Suppress failures for test listed in provided manifest files. ' + '(use -m for each manifest file you want to read)')) + parser.add_option( + '-m', + dest='manifests', + type='string', + action='append', + default=None, + help=('Suppress failures for test listed in provided manifest files. ' + '(use -m for each manifest file you want to read)')) + + opts, args = parser.parse_args(argv[2:]) + + with OptionChecker(parser): + if not args: + sys.exit('At least one *.sum file required.') + + if not opts.output: + sys.exit('Please provide name for report file.') + + manifests = [] + + for filename in ExpandGlobExprList(opts.manifests or []): + logging.info('Using "%s" manifest.', filename) + manifests.append(Manifest.FromFile(filename)) + + test_runs = [DejaGnuTestRun.FromFile(filename) + for filename in chain.from_iterable(map(glob.glob, args))] + + html = report.Generate(test_runs, manifests) + + if html: + with open(opts.output, 'w') as html_file: + html_file.write(html) + logging.info('Wrote report to "%s" file.', opts.output) + else: + sys.exit(1) + + +def HelpCommand(argv): + sys.exit('\n'.join([ + 'Usage: %s command [options]' % os.path.basename(argv[ + 0]), '', 'Commands:', + ' manifest - manage files containing a list of suppressed test failures', + ' report - generate report file for selected test runs' + ])) + + +def Main(argv): + try: + cmd_name = argv[1] + except IndexError: + cmd_name = None + + cmd_map = {'manifest': ManifestCommand, 'report': ReportCommand} + cmd_map.get(cmd_name, HelpCommand)(argv) + + +if __name__ == '__main__': + FORMAT = '%(asctime)-15s %(levelname)s %(message)s' + logging.basicConfig(format=FORMAT, level=logging.INFO) + + Main(sys.argv) diff --git a/deprecated/automation/clients/report/dejagnu/manifest.py b/deprecated/automation/clients/report/dejagnu/manifest.py new file mode 100644 index 00000000..5831d1b0 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/manifest.py @@ -0,0 +1,103 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from collections import namedtuple +from cStringIO import StringIO +import logging + +from summary import DejaGnuTestResult + + +class Manifest(namedtuple('Manifest', 'tool board results')): + """Stores a list of unsuccessful tests. + + Any line that starts with '#@' marker carries auxiliary data in form of a + key-value pair, for example: + + #@ tool: * + #@ board: unix + + So far tool and board parameters are recognized. Their value can contain + arbitrary glob expression. Based on aforementioned parameters given manifest + will be applied for all test results, but only in selected test runs. Note + that all parameters are optional. Their default value is '*' (i.e. for all + tools/boards). + + The meaning of lines above is as follows: corresponding test results to follow + should only be suppressed if test run was performed on "unix" board. + + The summary line used to build the test result should have this format: + + attrlist | UNRESOLVED: gcc.dg/unroll_1.c (test for excess errors) + ^^^^^^^^ ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^ + optional result name variant + attributes + """ + SUPPRESSIBLE_RESULTS = ['FAIL', 'UNRESOLVED', 'XPASS', 'ERROR'] + + @classmethod + def FromDejaGnuTestRun(cls, test_run): + results = [result + for result in test_run.results + if result.result in cls.SUPPRESSIBLE_RESULTS] + + return cls(test_run.tool, test_run.board, results) + + @classmethod + def FromFile(cls, filename): + """Creates manifest instance from a file in format described above.""" + params = {} + results = [] + + with open(filename, 'r') as manifest_file: + for line in manifest_file: + if line.startswith('#@'): + # parse a line with a parameter + try: + key, value = line[2:].split(':', 1) + except ValueError: + logging.warning('Malformed parameter line: "%s".', line) + else: + params[key.strip()] = value.strip() + else: + # remove comment + try: + line, _ = line.split('#', 1) + except ValueError: + pass + + line = line.strip() + + if line: + # parse a line with a test result + result = DejaGnuTestResult.FromLine(line) + + if result: + results.append(result) + else: + logging.warning('Malformed test result line: "%s".', line) + + tool = params.get('tool', '*') + board = params.get('board', '*') + + return cls(tool, board, results) + + def Generate(self): + """Dumps manifest to string.""" + text = StringIO() + + for name in ['tool', 'board']: + text.write('#@ {0}: {1}\n'.format(name, getattr(self, name))) + + text.write('\n') + + for result in sorted(self.results, key=lambda r: r.result): + text.write('{0}\n'.format(result)) + + return text.getvalue() + + def __iter__(self): + return iter(self.results) diff --git a/deprecated/automation/clients/report/dejagnu/report.html b/deprecated/automation/clients/report/dejagnu/report.html new file mode 100644 index 00000000..39b39e09 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/report.html @@ -0,0 +1,94 @@ +<link type="text/css" rel="Stylesheet" +href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.16/themes/ui-lightness/jquery-ui.css"/> + +<script type="text/javascript" src="https://www.google.com/jsapi"></script> +<script type="text/javascript"> + google.load("visualization", "1.1", {packages: ["corechart", "table"]}); + google.load("jquery", "1.6.2"); + google.load("jqueryui", "1.8.16"); + + function drawChart(name, label, table) { + var data = google.visualization.arrayToDataTable(table); + var chart = new google.visualization.PieChart( + document.getElementById(name)); + + chart.draw(data, + {title: label, pieSliceText: "value", width: 800, height: 400}); + } + + function drawTable(name, table) { + var data = google.visualization.arrayToDataTable(table); + var table = new google.visualization.Table( + document.getElementById(name)); + + table.draw(data, { + showRowNumber: false, allowHtml: true, sortColumn: 0}); + } + + google.setOnLoadCallback(function () { + $( "#testruns" ).tabs(); + + {% for test_run in test_runs %} + $( "#testrun{{ test_run.id }}" ).tabs(); + + {% for result_type, group in test_run.groups.items %} + $( "#testrun{{ test_run.id }}-{{ result_type }}-tables" ).accordion({ + autoHeight: false, collapsible: true, active: false }); + + drawChart( + "testrun{{ test_run.id }}-{{ result_type }}-chart", + "DejaGNU test {{ result_type }} summary for {{ test_run.name }}", + [ + ["Result", "Count"], + {% for result, count in group.summary %} + ["{{ result }}", {{ count }}],{% endfor %} + ]); + + {% for description, test_list in group.tests %} + {% if test_list %} + drawTable( + "testrun{{ test_run.id }}-{{ result_type }}-table-{{ forloop.counter }}", + [ + ["Test", "Variant"], + {% for test, variant in test_list %} + ["{{ test }}", "{{ variant }}"],{% endfor %} + ]); + {% endif %} + {% endfor %} + {% endfor %} + {% endfor %} + }); +</script> + +<div id="testruns"> + <ul> + {% for test_run in test_runs %} + <li><a href="#testrun{{ test_run.id }}">{{ test_run.name }}</a></li> + {% endfor %} + </ul> + + {% for test_run in test_runs %} + <div id="testrun{{ test_run.id }}" style="padding: 0px"> + <ul> + {% for result_type, group in test_run.groups.items %} + <li> + <a href="#testrun{{ test_run.id }}-{{ forloop.counter }}">{{ result_type }}</a> + </li> + {% endfor %} + </ul> + {% for result_type, group in test_run.groups.items %} + <div id="testrun{{ test_run.id }}-{{ forloop.counter }}"> + <div id="testrun{{ test_run.id }}-{{ result_type }}-chart" style="text-align: center"></div> + <div id="testrun{{ test_run.id }}-{{ result_type }}-tables"> + {% for description, test_list in group.tests %} + {% if test_list %} + <h3><a href="#">{{ description }}</a></h3> + <div id="testrun{{ test_run.id }}-{{ result_type }}-table-{{ forloop.counter }}"></div> + {% endif %} + {% endfor %} + </div> + </div> + {% endfor %} + </div> +{% endfor %} +</div> diff --git a/deprecated/automation/clients/report/dejagnu/report.py b/deprecated/automation/clients/report/dejagnu/report.py new file mode 100644 index 00000000..191a5389 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/report.py @@ -0,0 +1,115 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +import logging +import os.path + +RESULT_DESCRIPTION = { + 'ERROR': 'DejaGNU errors', + 'FAIL': 'Failed tests', + 'NOTE': 'DejaGNU notices', + 'PASS': 'Passed tests', + 'UNRESOLVED': 'Unresolved tests', + 'UNSUPPORTED': 'Unsupported tests', + 'UNTESTED': 'Not executed tests', + 'WARNING': 'DejaGNU warnings', + 'XFAIL': 'Expected test failures', + 'XPASS': 'Unexpectedly passed tests' +} + +RESULT_GROUPS = { + 'Successes': ['PASS', 'XFAIL'], + 'Failures': ['FAIL', 'XPASS', 'UNRESOLVED'], + 'Suppressed': ['!FAIL', '!XPASS', '!UNRESOLVED', '!ERROR'], + 'Framework': ['UNTESTED', 'UNSUPPORTED', 'ERROR', 'WARNING', 'NOTE'] +} + +ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) + + +def _GetResultDescription(name): + if name.startswith('!'): + name = name[1:] + + try: + return RESULT_DESCRIPTION[name] + except KeyError: + raise ValueError('Unknown result: "%s"' % name) + + +def _PrepareSummary(res_types, summary): + + def GetResultCount(res_type): + return summary.get(res_type, 0) + + return [(_GetResultDescription(rt), GetResultCount(rt)) for rt in res_types] + + +def _PrepareTestList(res_types, tests): + + def GetTestsByResult(res_type): + return [(test.name, test.variant or '') + for test in sorted(tests) if test.result == res_type] + + return [(_GetResultDescription(rt), GetTestsByResult(rt)) + for rt in res_types if rt != 'PASS'] + + +def Generate(test_runs, manifests): + """Generate HTML report from provided test runs. + + Args: + test_runs: DejaGnuTestRun objects list. + manifests: Manifest object list that will drive test result suppression. + + Returns: + String to which the HTML report was rendered. + """ + tmpl_args = [] + + for test_run_id, test_run in enumerate(test_runs): + logging.info('Generating report for: %s.', test_run) + + test_run.CleanUpTestResults() + test_run.SuppressTestResults(manifests) + + # Generate summary and test list for each result group + groups = {} + + for res_group, res_types in RESULT_GROUPS.items(): + summary_all = _PrepareSummary(res_types, test_run.summary) + tests_all = _PrepareTestList(res_types, test_run.results) + + has_2nd = lambda tuple2: bool(tuple2[1]) + summary = filter(has_2nd, summary_all) + tests = filter(has_2nd, tests_all) + + if summary or tests: + groups[res_group] = {'summary': summary, 'tests': tests} + + tmpl_args.append({ + 'id': test_run_id, + 'name': '%s @%s' % (test_run.tool, test_run.board), + 'groups': groups + }) + + logging.info('Rendering report in HTML format.') + + try: + from django import template + from django.template import loader + from django.conf import settings + except ImportError: + logging.error('Django framework not installed!') + logging.error('Failed to generate report in HTML format!') + return '' + + settings.configure(DEBUG=True, + TEMPLATE_DEBUG=True, + TEMPLATE_DIRS=(ROOT_PATH,)) + + tmpl = loader.get_template('report.html') + ctx = template.Context({'test_runs': tmpl_args}) + + return tmpl.render(ctx) diff --git a/deprecated/automation/clients/report/dejagnu/summary.py b/deprecated/automation/clients/report/dejagnu/summary.py new file mode 100644 index 00000000..d573c691 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/summary.py @@ -0,0 +1,262 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +from collections import defaultdict +from collections import namedtuple +from datetime import datetime +from fnmatch import fnmatch +from itertools import groupby +import logging +import os.path +import re + + +class DejaGnuTestResult(namedtuple('Result', 'name variant result flaky')): + """Stores the result of a single test case.""" + + # avoid adding __dict__ to the class + __slots__ = () + + LINE_RE = re.compile(r'([A-Z]+):\s+([\w/+.-]+)(.*)') + + @classmethod + def FromLine(cls, line): + """Alternate constructor which takes a string and parses it.""" + try: + attrs, line = line.split('|', 1) + + if attrs.strip() != 'flaky': + return None + + line = line.strip() + flaky = True + except ValueError: + flaky = False + + fields = cls.LINE_RE.match(line.strip()) + + if fields: + result, path, variant = fields.groups() + + # some of the tests are generated in build dir and are issued from there, + # because every test run is performed in randomly named tmp directory we + # need to remove random part + try: + # assume that 2nd field is a test path + path_parts = path.split('/') + + index = path_parts.index('testsuite') + path = '/'.join(path_parts[index + 1:]) + except ValueError: + path = '/'.join(path_parts) + + # Remove junk from test description. + variant = variant.strip(', ') + + substitutions = [ + # remove include paths - they contain name of tmp directory + ('-I\S+', ''), + # compress white spaces + ('\s+', ' ') + ] + + for pattern, replacement in substitutions: + variant = re.sub(pattern, replacement, variant) + + # Some tests separate last component of path by space, so actual filename + # ends up in description instead of path part. Correct that. + try: + first, rest = variant.split(' ', 1) + except ValueError: + pass + else: + if first.endswith('.o'): + path = os.path.join(path, first) + variant = rest + + # DejaGNU framework errors don't contain path part at all, so description + # part has to be reconstructed. + if not any(os.path.basename(path).endswith('.%s' % suffix) + for suffix in ['h', 'c', 'C', 'S', 'H', 'cc', 'i', 'o']): + variant = '%s %s' % (path, variant) + path = '' + + # Some tests are picked up from current directory (presumably DejaGNU + # generates some test files). Remove the prefix for these files. + if path.startswith('./'): + path = path[2:] + + return cls(path, variant or '', result, flaky=flaky) + + def __str__(self): + """Returns string representation of a test result.""" + if self.flaky: + fmt = 'flaky | ' + else: + fmt = '' + fmt += '{2}: {0}' + if self.variant: + fmt += ' {1}' + return fmt.format(*self) + + +class DejaGnuTestRun(object): + """Container for test results that were a part of single test run. + + The class stores also metadata related to the test run. + + Attributes: + board: Name of DejaGNU board, which was used to run the tests. + date: The date when the test run was started. + target: Target triple. + host: Host triple. + tool: The tool that was tested (e.g. gcc, binutils, g++, etc.) + results: a list of DejaGnuTestResult objects. + """ + + __slots__ = ('board', 'date', 'target', 'host', 'tool', 'results') + + def __init__(self, **kwargs): + assert all(name in self.__slots__ for name in kwargs) + + self.results = set() + self.date = kwargs.get('date', datetime.now()) + + for name in ('board', 'target', 'tool', 'host'): + setattr(self, name, kwargs.get(name, 'unknown')) + + @classmethod + def FromFile(cls, filename): + """Alternate constructor - reads a DejaGNU output file.""" + test_run = cls() + test_run.FromDejaGnuOutput(filename) + test_run.CleanUpTestResults() + return test_run + + @property + def summary(self): + """Returns a summary as {ResultType -> Count} dictionary.""" + summary = defaultdict(int) + + for r in self.results: + summary[r.result] += 1 + + return summary + + def _ParseBoard(self, fields): + self.board = fields.group(1).strip() + + def _ParseDate(self, fields): + self.date = datetime.strptime(fields.group(2).strip(), '%a %b %d %X %Y') + + def _ParseTarget(self, fields): + self.target = fields.group(2).strip() + + def _ParseHost(self, fields): + self.host = fields.group(2).strip() + + def _ParseTool(self, fields): + self.tool = fields.group(1).strip() + + def FromDejaGnuOutput(self, filename): + """Read in and parse DejaGNU output file.""" + + logging.info('Reading "%s" DejaGNU output file.', filename) + + with open(filename, 'r') as report: + lines = [line.strip() for line in report.readlines() if line.strip()] + + parsers = ((re.compile(r'Running target (.*)'), self._ParseBoard), + (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate), + (re.compile(r'=== (.*) tests ==='), self._ParseTool), + (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget), + (re.compile(r'Host(\s+)is (.*)'), self._ParseHost)) + + for line in lines: + result = DejaGnuTestResult.FromLine(line) + + if result: + self.results.add(result) + else: + for regexp, parser in parsers: + fields = regexp.match(line) + if fields: + parser(fields) + break + + logging.debug('DejaGNU output file parsed successfully.') + logging.debug(self) + + def CleanUpTestResults(self): + """Remove certain test results considered to be spurious. + + 1) Large number of test reported as UNSUPPORTED are also marked as + UNRESOLVED. If that's the case remove latter result. + 2) If a test is performed on compiler output and for some reason compiler + fails, we don't want to report all failures that depend on the former. + """ + name_key = lambda v: v.name + results_by_name = sorted(self.results, key=name_key) + + for name, res_iter in groupby(results_by_name, key=name_key): + results = set(res_iter) + + # If DejaGnu was unable to compile a test it will create following result: + failed = DejaGnuTestResult(name, '(test for excess errors)', 'FAIL', + False) + + # If a test compilation failed, remove all results that are dependent. + if failed in results: + dependants = set(filter(lambda r: r.result != 'FAIL', results)) + + self.results -= dependants + + for res in dependants: + logging.info('Removed {%s} dependance.', res) + + # Remove all UNRESOLVED results that were also marked as UNSUPPORTED. + unresolved = [res._replace(result='UNRESOLVED') + for res in results if res.result == 'UNSUPPORTED'] + + for res in unresolved: + if res in self.results: + self.results.remove(res) + logging.info('Removed {%s} duplicate.', res) + + def _IsApplicable(self, manifest): + """Checks if test results need to be reconsidered based on the manifest.""" + check_list = [(self.tool, manifest.tool), (self.board, manifest.board)] + + return all(fnmatch(text, pattern) for text, pattern in check_list) + + def SuppressTestResults(self, manifests): + """Suppresses all test results listed in manifests.""" + + # Get a set of tests results that are going to be suppressed if they fail. + manifest_results = set() + + for manifest in filter(self._IsApplicable, manifests): + manifest_results |= set(manifest.results) + + suppressed_results = self.results & manifest_results + + for result in sorted(suppressed_results): + logging.debug('Result suppressed for {%s}.', result) + + new_result = '!' + result.result + + # Mark result suppression as applied. + manifest_results.remove(result) + + # Rewrite test result. + self.results.remove(result) + self.results.add(result._replace(result=new_result)) + + for result in sorted(manifest_results): + logging.warning('Result {%s} listed in manifest but not suppressed.', + result) + + def __str__(self): + return '{0}, {1} @{2} on {3}'.format(self.target, self.tool, self.board, + self.date) diff --git a/deprecated/automation/clients/report/validate_failures.py b/deprecated/automation/clients/report/validate_failures.py new file mode 100755 index 00000000..d8776ba5 --- /dev/null +++ b/deprecated/automation/clients/report/validate_failures.py @@ -0,0 +1,239 @@ +#!/usr/bin/python2 + +# Script to compare testsuite failures against a list of known-to-fail +# tests. + +# Contributed by Diego Novillo <dnovillo@google.com> +# Overhaul by Krystian Baclawski <kbaclawski@google.com> +# +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +"""This script provides a coarser XFAILing mechanism that requires no +detailed DejaGNU markings. This is useful in a variety of scenarios: + +- Development branches with many known failures waiting to be fixed. +- Release branches with known failures that are not considered + important for the particular release criteria used in that branch. + +The script must be executed from the toplevel build directory. When +executed it will: + +1) Determine the target built: TARGET +2) Determine the source directory: SRCDIR +3) Look for a failure manifest file in + <SRCDIR>/contrib/testsuite-management/<TARGET>.xfail +4) Collect all the <tool>.sum files from the build tree. +5) Produce a report stating: + a) Failures expected in the manifest but not present in the build. + b) Failures in the build not expected in the manifest. +6) If all the build failures are expected in the manifest, it exits + with exit code 0. Otherwise, it exits with error code 1. +""" + +import optparse +import logging +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from dejagnu.manifest import Manifest +from dejagnu.summary import DejaGnuTestResult +from dejagnu.summary import DejaGnuTestRun + +# Pattern for naming manifest files. The first argument should be +# the toplevel GCC source directory. The second argument is the +# target triple used during the build. +_MANIFEST_PATH_PATTERN = '%s/contrib/testsuite-management/%s.xfail' + + +def GetMakefileVars(makefile_path): + assert os.path.exists(makefile_path) + + with open(makefile_path) as lines: + kvs = [line.split('=', 1) for line in lines if '=' in line] + + return dict((k.strip(), v.strip()) for k, v in kvs) + + +def GetSumFiles(build_dir): + summaries = [] + + for root, _, filenames in os.walk(build_dir): + summaries.extend([os.path.join(root, filename) + for filename in filenames if filename.endswith('.sum')]) + + return map(os.path.normpath, summaries) + + +def ValidBuildDirectory(build_dir, target): + mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')] + + extra_paths = [os.path.join(build_dir, target), + os.path.join(build_dir, 'build-%s' % target)] + + return (all(map(os.path.exists, mandatory_paths)) and + any(map(os.path.exists, extra_paths))) + + +def GetManifestPath(build_dir): + makefile = GetMakefileVars(os.path.join(build_dir, 'Makefile')) + srcdir = makefile['srcdir'] + target = makefile['target'] + + if not ValidBuildDirectory(build_dir, target): + target = makefile['target_alias'] + + if not ValidBuildDirectory(build_dir, target): + logging.error('%s is not a valid GCC top level build directory.', build_dir) + sys.exit(1) + + logging.info('Discovered source directory: "%s"', srcdir) + logging.info('Discovered build target: "%s"', target) + + return _MANIFEST_PATH_PATTERN % (srcdir, target) + + +def CompareResults(manifest, actual): + """Compare sets of results and return two lists: + - List of results present in MANIFEST but missing from ACTUAL. + - List of results present in ACTUAL but missing from MANIFEST. + """ + # Report all the actual results not present in the manifest. + actual_vs_manifest = actual - manifest + + # Filter out tests marked flaky. + manifest_without_flaky_tests = set(filter(lambda result: not result.flaky, + manifest)) + + # Simlarly for all the tests in the manifest. + manifest_vs_actual = manifest_without_flaky_tests - actual + + return actual_vs_manifest, manifest_vs_actual + + +def LogResults(level, results): + log_fun = getattr(logging, level) + + for num, result in enumerate(sorted(results), start=1): + log_fun(' %d) %s', num, result) + + +def CheckExpectedResults(manifest_path, build_dir): + logging.info('Reading manifest file: "%s"', manifest_path) + + manifest = set(Manifest.FromFile(manifest_path)) + + logging.info('Getting actual results from build directory: "%s"', + os.path.realpath(build_dir)) + + summaries = GetSumFiles(build_dir) + + actual = set() + + for summary in summaries: + test_run = DejaGnuTestRun.FromFile(summary) + failures = set(Manifest.FromDejaGnuTestRun(test_run)) + actual.update(failures) + + if manifest: + logging.debug('Tests expected to fail:') + LogResults('debug', manifest) + + if actual: + logging.debug('Actual test failures:') + LogResults('debug', actual) + + actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual) + + if actual_vs_manifest: + logging.info('Build results not in the manifest:') + LogResults('info', actual_vs_manifest) + + if manifest_vs_actual: + logging.info('Manifest results not present in the build:') + LogResults('info', manifest_vs_actual) + logging.info('NOTE: This is not a failure! ', + 'It just means that the manifest expected these tests to ' + 'fail, but they worked in this configuration.') + + if actual_vs_manifest or manifest_vs_actual: + sys.exit(1) + + logging.info('No unexpected failures.') + + +def ProduceManifest(manifest_path, build_dir, overwrite): + if os.path.exists(manifest_path) and not overwrite: + logging.error('Manifest file "%s" already exists.', manifest_path) + logging.error('Use --force to overwrite.') + sys.exit(1) + + testruns = map(DejaGnuTestRun.FromFile, GetSumFiles(build_dir)) + manifests = map(Manifest.FromDejaGnuTestRun, testruns) + + with open(manifest_path, 'w') as manifest_file: + manifest_strings = [manifest.Generate() for manifest in manifests] + logging.info('Writing manifest to "%s".', manifest_path) + manifest_file.write('\n'.join(manifest_strings)) + + +def Main(argv): + parser = optparse.OptionParser(usage=__doc__) + parser.add_option( + '-b', + '--build_dir', + dest='build_dir', + action='store', + metavar='PATH', + default=os.getcwd(), + help='Build directory to check. (default: current directory)') + parser.add_option('-m', + '--manifest', + dest='manifest', + action='store_true', + help='Produce the manifest for the current build.') + parser.add_option( + '-f', + '--force', + dest='force', + action='store_true', + help=('Overwrite an existing manifest file, if user requested creating ' + 'new one. (default: False)')) + parser.add_option('-v', + '--verbose', + dest='verbose', + action='store_true', + help='Increase verbosity.') + options, _ = parser.parse_args(argv[1:]) + + if options.verbose: + logging.root.setLevel(logging.DEBUG) + + manifest_path = GetManifestPath(options.build_dir) + + if options.manifest: + ProduceManifest(manifest_path, options.build_dir, options.force) + else: + CheckExpectedResults(manifest_path, options.build_dir) + + +if __name__ == '__main__': + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + Main(sys.argv) |