diff options
Diffstat (limited to 'deprecated/automation/clients/report/dejagnu/summary.py')
-rw-r--r-- | deprecated/automation/clients/report/dejagnu/summary.py | 262 |
1 files changed, 262 insertions, 0 deletions
diff --git a/deprecated/automation/clients/report/dejagnu/summary.py b/deprecated/automation/clients/report/dejagnu/summary.py new file mode 100644 index 00000000..d573c691 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/summary.py @@ -0,0 +1,262 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +from collections import defaultdict +from collections import namedtuple +from datetime import datetime +from fnmatch import fnmatch +from itertools import groupby +import logging +import os.path +import re + + +class DejaGnuTestResult(namedtuple('Result', 'name variant result flaky')): + """Stores the result of a single test case.""" + + # avoid adding __dict__ to the class + __slots__ = () + + LINE_RE = re.compile(r'([A-Z]+):\s+([\w/+.-]+)(.*)') + + @classmethod + def FromLine(cls, line): + """Alternate constructor which takes a string and parses it.""" + try: + attrs, line = line.split('|', 1) + + if attrs.strip() != 'flaky': + return None + + line = line.strip() + flaky = True + except ValueError: + flaky = False + + fields = cls.LINE_RE.match(line.strip()) + + if fields: + result, path, variant = fields.groups() + + # some of the tests are generated in build dir and are issued from there, + # because every test run is performed in randomly named tmp directory we + # need to remove random part + try: + # assume that 2nd field is a test path + path_parts = path.split('/') + + index = path_parts.index('testsuite') + path = '/'.join(path_parts[index + 1:]) + except ValueError: + path = '/'.join(path_parts) + + # Remove junk from test description. + variant = variant.strip(', ') + + substitutions = [ + # remove include paths - they contain name of tmp directory + ('-I\S+', ''), + # compress white spaces + ('\s+', ' ') + ] + + for pattern, replacement in substitutions: + variant = re.sub(pattern, replacement, variant) + + # Some tests separate last component of path by space, so actual filename + # ends up in description instead of path part. Correct that. + try: + first, rest = variant.split(' ', 1) + except ValueError: + pass + else: + if first.endswith('.o'): + path = os.path.join(path, first) + variant = rest + + # DejaGNU framework errors don't contain path part at all, so description + # part has to be reconstructed. + if not any(os.path.basename(path).endswith('.%s' % suffix) + for suffix in ['h', 'c', 'C', 'S', 'H', 'cc', 'i', 'o']): + variant = '%s %s' % (path, variant) + path = '' + + # Some tests are picked up from current directory (presumably DejaGNU + # generates some test files). Remove the prefix for these files. + if path.startswith('./'): + path = path[2:] + + return cls(path, variant or '', result, flaky=flaky) + + def __str__(self): + """Returns string representation of a test result.""" + if self.flaky: + fmt = 'flaky | ' + else: + fmt = '' + fmt += '{2}: {0}' + if self.variant: + fmt += ' {1}' + return fmt.format(*self) + + +class DejaGnuTestRun(object): + """Container for test results that were a part of single test run. + + The class stores also metadata related to the test run. + + Attributes: + board: Name of DejaGNU board, which was used to run the tests. + date: The date when the test run was started. + target: Target triple. + host: Host triple. + tool: The tool that was tested (e.g. gcc, binutils, g++, etc.) + results: a list of DejaGnuTestResult objects. + """ + + __slots__ = ('board', 'date', 'target', 'host', 'tool', 'results') + + def __init__(self, **kwargs): + assert all(name in self.__slots__ for name in kwargs) + + self.results = set() + self.date = kwargs.get('date', datetime.now()) + + for name in ('board', 'target', 'tool', 'host'): + setattr(self, name, kwargs.get(name, 'unknown')) + + @classmethod + def FromFile(cls, filename): + """Alternate constructor - reads a DejaGNU output file.""" + test_run = cls() + test_run.FromDejaGnuOutput(filename) + test_run.CleanUpTestResults() + return test_run + + @property + def summary(self): + """Returns a summary as {ResultType -> Count} dictionary.""" + summary = defaultdict(int) + + for r in self.results: + summary[r.result] += 1 + + return summary + + def _ParseBoard(self, fields): + self.board = fields.group(1).strip() + + def _ParseDate(self, fields): + self.date = datetime.strptime(fields.group(2).strip(), '%a %b %d %X %Y') + + def _ParseTarget(self, fields): + self.target = fields.group(2).strip() + + def _ParseHost(self, fields): + self.host = fields.group(2).strip() + + def _ParseTool(self, fields): + self.tool = fields.group(1).strip() + + def FromDejaGnuOutput(self, filename): + """Read in and parse DejaGNU output file.""" + + logging.info('Reading "%s" DejaGNU output file.', filename) + + with open(filename, 'r') as report: + lines = [line.strip() for line in report.readlines() if line.strip()] + + parsers = ((re.compile(r'Running target (.*)'), self._ParseBoard), + (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate), + (re.compile(r'=== (.*) tests ==='), self._ParseTool), + (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget), + (re.compile(r'Host(\s+)is (.*)'), self._ParseHost)) + + for line in lines: + result = DejaGnuTestResult.FromLine(line) + + if result: + self.results.add(result) + else: + for regexp, parser in parsers: + fields = regexp.match(line) + if fields: + parser(fields) + break + + logging.debug('DejaGNU output file parsed successfully.') + logging.debug(self) + + def CleanUpTestResults(self): + """Remove certain test results considered to be spurious. + + 1) Large number of test reported as UNSUPPORTED are also marked as + UNRESOLVED. If that's the case remove latter result. + 2) If a test is performed on compiler output and for some reason compiler + fails, we don't want to report all failures that depend on the former. + """ + name_key = lambda v: v.name + results_by_name = sorted(self.results, key=name_key) + + for name, res_iter in groupby(results_by_name, key=name_key): + results = set(res_iter) + + # If DejaGnu was unable to compile a test it will create following result: + failed = DejaGnuTestResult(name, '(test for excess errors)', 'FAIL', + False) + + # If a test compilation failed, remove all results that are dependent. + if failed in results: + dependants = set(filter(lambda r: r.result != 'FAIL', results)) + + self.results -= dependants + + for res in dependants: + logging.info('Removed {%s} dependance.', res) + + # Remove all UNRESOLVED results that were also marked as UNSUPPORTED. + unresolved = [res._replace(result='UNRESOLVED') + for res in results if res.result == 'UNSUPPORTED'] + + for res in unresolved: + if res in self.results: + self.results.remove(res) + logging.info('Removed {%s} duplicate.', res) + + def _IsApplicable(self, manifest): + """Checks if test results need to be reconsidered based on the manifest.""" + check_list = [(self.tool, manifest.tool), (self.board, manifest.board)] + + return all(fnmatch(text, pattern) for text, pattern in check_list) + + def SuppressTestResults(self, manifests): + """Suppresses all test results listed in manifests.""" + + # Get a set of tests results that are going to be suppressed if they fail. + manifest_results = set() + + for manifest in filter(self._IsApplicable, manifests): + manifest_results |= set(manifest.results) + + suppressed_results = self.results & manifest_results + + for result in sorted(suppressed_results): + logging.debug('Result suppressed for {%s}.', result) + + new_result = '!' + result.result + + # Mark result suppression as applied. + manifest_results.remove(result) + + # Rewrite test result. + self.results.remove(result) + self.results.add(result._replace(result=new_result)) + + for result in sorted(manifest_results): + logging.warning('Result {%s} listed in manifest but not suppressed.', + result) + + def __str__(self): + return '{0}, {1} @{2} on {3}'.format(self.target, self.tool, self.board, + self.date) |