diff options
Diffstat (limited to 'python/helpers/coverage')
34 files changed, 2726 insertions, 1005 deletions
diff --git a/python/helpers/coverage/__init__.py b/python/helpers/coverage/__init__.py index d8dbc0f6dc88..193b7a107ebd 100644 --- a/python/helpers/coverage/__init__.py +++ b/python/helpers/coverage/__init__.py @@ -5,19 +5,13 @@ http://nedbatchelder.com/code/coverage """ -__version__ = "3.5" # see detailed history in CHANGES.txt - -__url__ = "http://nedbatchelder.com/code/coverage" -if max(__version__).isalpha(): - # For pre-releases, use a version-specific URL. - __url__ += "/" + __version__ +from coverage.version import __version__, __url__ from coverage.control import coverage, process_startup from coverage.data import CoverageData from coverage.cmdline import main, CoverageScript from coverage.misc import CoverageException - # Module-level functions. The original API to this module was based on # functions defined directly in the module, with a singleton of the coverage() # class. That design hampered programmability, so the current api uses @@ -36,12 +30,34 @@ def _singleton_method(name): called. """ + # Disable pylint msg W0612, because a bunch of variables look unused, but + # they're accessed via locals(). + # pylint: disable=W0612 + def wrapper(*args, **kwargs): """Singleton wrapper around a coverage method.""" global _the_coverage if not _the_coverage: _the_coverage = coverage(auto_data=True) return getattr(_the_coverage, name)(*args, **kwargs) + + import inspect + meth = getattr(coverage, name) + args, varargs, kw, defaults = inspect.getargspec(meth) + argspec = inspect.formatargspec(args[1:], varargs, kw, defaults) + docstring = meth.__doc__ + wrapper.__doc__ = ("""\ + A first-use-singleton wrapper around coverage.%(name)s. + + This wrapper is provided for backward compatibility with legacy code. + New code should use coverage.%(name)s directly. + + %(name)s%(argspec)s: + + %(docstring)s + """ % locals() + ) + return wrapper @@ -57,10 +73,26 @@ report = _singleton_method('report') annotate = _singleton_method('annotate') +# On Windows, we encode and decode deep enough that something goes wrong and +# the encodings.utf_8 module is loaded and then unloaded, I don't know why. +# Adding a reference here prevents it from being unloaded. Yuk. +import encodings.utf_8 + +# Because of the "from coverage.control import fooey" lines at the top of the +# file, there's an entry for coverage.coverage in sys.modules, mapped to None. +# This makes some inspection tools (like pydoc) unable to find the class +# coverage.coverage. So remove that entry. +import sys +try: + del sys.modules['coverage.coverage'] +except KeyError: + pass + + # COPYRIGHT AND LICENSE # # Copyright 2001 Gareth Rees. All rights reserved. -# Copyright 2004-2010 Ned Batchelder. All rights reserved. +# Copyright 2004-2013 Ned Batchelder. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/python/helpers/coverage/__main__.py b/python/helpers/coverage/__main__.py index af5fa9f6819f..55e0d259e04a 100644 --- a/python/helpers/coverage/__main__.py +++ b/python/helpers/coverage/__main__.py @@ -1,3 +1,4 @@ -"""Coverage.py's main entrypoint.""" +"""Coverage.py's main entry point.""" +import sys from coverage.cmdline import main -main() +sys.exit(main()) diff --git a/python/helpers/coverage/annotate.py b/python/helpers/coverage/annotate.py index a556d853cdbd..5c396784445c 100644 --- a/python/helpers/coverage/annotate.py +++ b/python/helpers/coverage/annotate.py @@ -2,6 +2,7 @@ import os, re +from coverage.backward import sorted # pylint: disable=W0622 from coverage.report import Reporter class AnnotateReporter(Reporter): @@ -26,20 +27,20 @@ class AnnotateReporter(Reporter): """ - def __init__(self, coverage, ignore_errors=False): - super(AnnotateReporter, self).__init__(coverage, ignore_errors) + def __init__(self, coverage, config): + super(AnnotateReporter, self).__init__(coverage, config) self.directory = None blank_re = re.compile(r"\s*(#|$)") else_re = re.compile(r"\s*else\s*:\s*(#|$)") - def report(self, morfs, config, directory=None): + def report(self, morfs, directory=None): """Run the report. See `coverage.report()` for arguments. """ - self.report_files(self.annotate_file, morfs, config, directory) + self.report_files(self.annotate_file, morfs, directory) def annotate_file(self, cu, analysis): """Annotate a single file. @@ -59,9 +60,9 @@ class AnnotateReporter(Reporter): dest_file = filename + ",cover" dest = open(dest_file, 'w') - statements = analysis.statements - missing = analysis.missing - excluded = analysis.excluded + statements = sorted(analysis.statements) + missing = sorted(analysis.missing) + excluded = sorted(analysis.excluded) lineno = 0 i = 0 diff --git a/python/helpers/coverage/backward.py b/python/helpers/coverage/backward.py index f0a34ac4ceb1..7d2685459782 100644 --- a/python/helpers/coverage/backward.py +++ b/python/helpers/coverage/backward.py @@ -6,7 +6,7 @@ # W0611: Unused import blah # W0622: Redefining built-in blah -import os, sys +import os, re, sys # Python 2.3 doesn't have `set` try: @@ -24,6 +24,31 @@ except NameError: lst.sort() return lst +# Python 2.3 doesn't have `reversed`. +try: + reversed = reversed +except NameError: + def reversed(iterable): + """A 2.3-compatible implementation of `reversed`.""" + lst = list(iterable) + return lst[::-1] + +# rpartition is new in 2.5 +try: + "".rpartition +except AttributeError: + def rpartition(s, sep): + """Implement s.rpartition(sep) for old Pythons.""" + i = s.rfind(sep) + if i == -1: + return ('', '', s) + else: + return (s[:i], sep, s[i+len(sep):]) +else: + def rpartition(s, sep): + """A common interface for new Pythons.""" + return s.rpartition(sep) + # Pythons 2 and 3 differ on where to get StringIO try: from cStringIO import StringIO @@ -49,6 +74,18 @@ try: except NameError: range = range +# A function to iterate listlessly over a dict's items. +try: + {}.iteritems +except AttributeError: + def iitems(d): + """Produce the items from dict `d`.""" + return d.items() +else: + def iitems(d): + """Produce the items from dict `d`.""" + return d.iteritems() + # Exec is a statement in Py2, a function in Py3 if sys.version_info >= (3, 0): def exec_code_object(code, global_map): @@ -66,21 +103,32 @@ else: ) ) -# ConfigParser was renamed to the more-standard configparser -try: - import configparser -except ImportError: - import ConfigParser as configparser - -# Python 3.2 provides `tokenize.open`, the best way to open source files. -try: +# Reading Python source and interpreting the coding comment is a big deal. +if sys.version_info >= (3, 0): + # Python 3.2 provides `tokenize.open`, the best way to open source files. import tokenize - open_source = tokenize.open # pylint: disable=E1101 -except AttributeError: + try: + open_source = tokenize.open # pylint: disable=E1101 + except AttributeError: + from io import TextIOWrapper + detect_encoding = tokenize.detect_encoding # pylint: disable=E1101 + # Copied from the 3.2 stdlib: + def open_source(fname): + """Open a file in read only mode using the encoding detected by + detect_encoding(). + """ + buffer = open(fname, 'rb') + encoding, _ = detect_encoding(buffer.readline) + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True) + text.mode = 'r' + return text +else: def open_source(fname): """Open a source file the best way.""" return open(fname, "rU") + # Python 3.x is picky about bytes and strings, so provide methods to # get them right, and make them no-ops in 2.x if sys.version_info >= (3, 0): @@ -92,6 +140,19 @@ if sys.version_info >= (3, 0): """Convert bytes `b` to a string.""" return b.decode('utf8') + def binary_bytes(byte_values): + """Produce a byte string with the ints from `byte_values`.""" + return bytes(byte_values) + + def byte_to_int(byte_value): + """Turn an element of a bytes object into an int.""" + return byte_value + + def bytes_to_ints(bytes_value): + """Turn a bytes object into a sequence of ints.""" + # In Py3, iterating bytes gives ints. + return bytes_value + else: def to_bytes(s): """Convert string `s` to bytes (no-op in 2.x).""" @@ -101,6 +162,19 @@ else: """Convert bytes `b` to a string (no-op in 2.x).""" return b + def binary_bytes(byte_values): + """Produce a byte string with the ints from `byte_values`.""" + return "".join([chr(b) for b in byte_values]) + + def byte_to_int(byte_value): + """Turn an element of a bytes object into an int.""" + return ord(byte_value) + + def bytes_to_ints(bytes_value): + """Turn a bytes object into a sequence of ints.""" + for byte in bytes_value: + yield ord(byte) + # Md5 is available in different places. try: import hashlib diff --git a/python/helpers/coverage/bytecode.py b/python/helpers/coverage/bytecode.py index ab522d6c1c6c..85360638528e 100644 --- a/python/helpers/coverage/bytecode.py +++ b/python/helpers/coverage/bytecode.py @@ -1,14 +1,25 @@ """Bytecode manipulation for coverage.py""" -import opcode, sys, types +import opcode, types + +from coverage.backward import byte_to_int class ByteCode(object): """A single bytecode.""" def __init__(self): + # The offset of this bytecode in the code object. self.offset = -1 + + # The opcode, defined in the `opcode` module. self.op = -1 + + # The argument, a small integer, whose meaning depends on the opcode. self.arg = -1 + + # The offset in the code object of the next bytecode. self.next_offset = -1 + + # The offset to jump to. self.jump_to = -1 @@ -18,44 +29,34 @@ class ByteCodes(object): Returns `ByteCode` objects. """ + # pylint: disable=R0924 def __init__(self, code): self.code = code - self.offset = 0 - if sys.version_info >= (3, 0): - def __getitem__(self, i): - return self.code[i] - else: - def __getitem__(self, i): - return ord(self.code[i]) + def __getitem__(self, i): + return byte_to_int(self.code[i]) def __iter__(self): - return self - - def __next__(self): - if self.offset >= len(self.code): - raise StopIteration + offset = 0 + while offset < len(self.code): + bc = ByteCode() + bc.op = self[offset] + bc.offset = offset - bc = ByteCode() - bc.op = self[self.offset] - bc.offset = self.offset + next_offset = offset+1 + if bc.op >= opcode.HAVE_ARGUMENT: + bc.arg = self[offset+1] + 256*self[offset+2] + next_offset += 2 - next_offset = self.offset+1 - if bc.op >= opcode.HAVE_ARGUMENT: - bc.arg = self[self.offset+1] + 256*self[self.offset+2] - next_offset += 2 + label = -1 + if bc.op in opcode.hasjrel: + label = next_offset + bc.arg + elif bc.op in opcode.hasjabs: + label = bc.arg + bc.jump_to = label - label = -1 - if bc.op in opcode.hasjrel: - label = next_offset + bc.arg - elif bc.op in opcode.hasjabs: - label = bc.arg - bc.jump_to = label - - bc.next_offset = self.offset = next_offset - return bc - - next = __next__ # Py2k uses an old-style non-dunder name. + bc.next_offset = offset = next_offset + yield bc class CodeObjects(object): @@ -64,18 +65,11 @@ class CodeObjects(object): self.stack = [code] def __iter__(self): - return self - - def __next__(self): - if self.stack: + while self.stack: # We're going to return the code object on the stack, but first # push its children for later returning. code = self.stack.pop() for c in code.co_consts: if isinstance(c, types.CodeType): self.stack.append(c) - return code - - raise StopIteration - - next = __next__ + yield code diff --git a/python/helpers/coverage/cmdline.py b/python/helpers/coverage/cmdline.py index 1ce5e0f54ad6..ea112a8b8f2d 100644 --- a/python/helpers/coverage/cmdline.py +++ b/python/helpers/coverage/cmdline.py @@ -1,10 +1,11 @@ """Command-line support for Coverage.""" -import optparse, re, sys, traceback +import optparse, os, sys, time, traceback from coverage.backward import sorted # pylint: disable=W0622 from coverage.execfile import run_python_file, run_python_module from coverage.misc import CoverageException, ExceptionDuringRun, NoSource +from coverage.debug import info_formatter class Opts(object): @@ -19,11 +20,18 @@ class Opts(object): '', '--branch', action='store_true', help="Measure branch coverage in addition to statement coverage." ) + debug = optparse.make_option( + '', '--debug', action='store', metavar="OPTS", + help="Debug options, separated by commas" + ) directory = optparse.make_option( - '-d', '--directory', action='store', - metavar="DIR", + '-d', '--directory', action='store', metavar="DIR", help="Write the output files to DIR." ) + fail_under = optparse.make_option( + '', '--fail-under', action='store', metavar="MIN", type="int", + help="Exit with a status of 2 if the total coverage is less than MIN." + ) help = optparse.make_option( '-h', '--help', action='store_true', help="Get help on this command." @@ -89,6 +97,10 @@ class Opts(object): help="Use a simpler but slower trace method. Try this if you get " "seemingly impossible results!" ) + title = optparse.make_option( + '', '--title', action='store', metavar="TITLE", + help="A text string to use as the title on the HTML." + ) version = optparse.make_option( '', '--version', action='store_true', help="Display version information and exit." @@ -110,7 +122,9 @@ class CoverageOptionParser(optparse.OptionParser, object): self.set_defaults( actions=[], branch=None, + debug=None, directory=None, + fail_under=None, help=None, ignore_errors=None, include=None, @@ -122,6 +136,7 @@ class CoverageOptionParser(optparse.OptionParser, object): show_missing=None, source=None, timid=None, + title=None, erase_first=None, version=None, ) @@ -273,9 +288,11 @@ CMDS = { 'html': CmdOptionParser("html", [ Opts.directory, + Opts.fail_under, Opts.ignore_errors, Opts.omit, Opts.include, + Opts.title, ] + GLOBAL_ARGS, usage = "[options] [modules]", description = "Create an HTML report of the coverage of the files. " @@ -285,6 +302,7 @@ CMDS = { 'report': CmdOptionParser("report", [ + Opts.fail_under, Opts.ignore_errors, Opts.omit, Opts.include, @@ -298,6 +316,7 @@ CMDS = { [ Opts.append, Opts.branch, + Opts.debug, Opts.pylib, Opts.parallel_mode, Opts.module, @@ -314,20 +333,20 @@ CMDS = { 'xml': CmdOptionParser("xml", [ + Opts.fail_under, Opts.ignore_errors, Opts.omit, Opts.include, Opts.output_xml, ] + GLOBAL_ARGS, cmd = "xml", - defaults = {'outfile': 'coverage.xml'}, usage = "[options] [modules]", description = "Generate an XML report of coverage results." ), } -OK, ERR = 0, 1 +OK, ERR, FAIL_UNDER = 0, 1, 2 class CoverageScript(object): @@ -346,27 +365,10 @@ class CoverageScript(object): self.run_python_file = _run_python_file or run_python_file self.run_python_module = _run_python_module or run_python_module self.help_fn = _help_fn or self.help + self.classic = False self.coverage = None - def help(self, error=None, topic=None, parser=None): - """Display an error message, or the named topic.""" - assert error or topic or parser - if error: - print(error) - print("Use 'coverage help' for help.") - elif parser: - print(parser.format_help().strip()) - else: - # Parse out the topic we want from HELP_TOPICS - topic_list = re.split("(?m)^=+ (\w+) =+$", HELP_TOPICS) - topics = dict(zip(topic_list[1::2], topic_list[2::2])) - help_msg = topics.get(topic, '').strip() - if help_msg: - print(help_msg % self.covpkg.__dict__) - else: - print("Don't know topic %r" % topic) - def command_line(self, argv): """The bulk of the command line interface to Coverage. @@ -376,15 +378,14 @@ class CoverageScript(object): """ # Collect the command-line options. - if not argv: self.help_fn(topic='minimum_help') return OK # The command syntax we parse depends on the first argument. Classic # syntax always starts with an option. - classic = argv[0].startswith('-') - if classic: + self.classic = argv[0].startswith('-') + if self.classic: parser = ClassicOptionParser() else: parser = CMDS.get(argv[0]) @@ -398,64 +399,19 @@ class CoverageScript(object): if not ok: return ERR - # Handle help. - if options.help: - if classic: - self.help_fn(topic='help') - else: - self.help_fn(parser=parser) - return OK - - if "help" in options.actions: - if args: - for a in args: - parser = CMDS.get(a) - if parser: - self.help_fn(parser=parser) - else: - self.help_fn(topic=a) - else: - self.help_fn(topic='help') - return OK - - # Handle version. - if options.version: - self.help_fn(topic='version') + # Handle help and version. + if self.do_help(options, args, parser): return OK # Check for conflicts and problems in the options. - for i in ['erase', 'execute']: - for j in ['annotate', 'html', 'report', 'combine']: - if (i in options.actions) and (j in options.actions): - self.help_fn("You can't specify the '%s' and '%s' " - "options at the same time." % (i, j)) - return ERR - - if not options.actions: - self.help_fn( - "You must specify at least one of -e, -x, -c, -r, -a, or -b." - ) - return ERR - args_allowed = ( - 'execute' in options.actions or - 'annotate' in options.actions or - 'html' in options.actions or - 'debug' in options.actions or - 'report' in options.actions or - 'xml' in options.actions - ) - if not args_allowed and args: - self.help_fn("Unexpected arguments: %s" % " ".join(args)) - return ERR - - if 'execute' in options.actions and not args: - self.help_fn("Nothing to do.") + if not self.args_ok(options, args): return ERR # Listify the list options. source = unshell_list(options.source) omit = unshell_list(options.omit) include = unshell_list(options.include) + debug = unshell_list(options.debug) # Do something. self.coverage = self.covpkg.coverage( @@ -467,41 +423,11 @@ class CoverageScript(object): source = source, omit = omit, include = include, + debug = debug, ) if 'debug' in options.actions: - if not args: - self.help_fn("What information would you like: data, sys?") - return ERR - for info in args: - if info == 'sys': - print("-- sys ----------------------------------------") - for label, info in self.coverage.sysinfo(): - if info == []: - info = "-none-" - if isinstance(info, list): - print("%15s:" % label) - for e in info: - print("%15s %s" % ("", e)) - else: - print("%15s: %s" % (label, info)) - elif info == 'data': - print("-- data ---------------------------------------") - self.coverage.load() - print("path: %s" % self.coverage.data.filename) - print("has_arcs: %r" % self.coverage.data.has_arcs()) - summary = self.coverage.data.summary(fullpath=True) - if summary: - filenames = sorted(summary.keys()) - print("\n%d files:" % len(filenames)) - for f in filenames: - print("%s: %d lines" % (f, summary[f])) - else: - print("No data collected") - else: - self.help_fn("Don't know what you mean by %r" % info) - return ERR - return OK + return self.do_debug(args) if 'erase' in options.actions or options.erase_first: self.coverage.erase() @@ -509,22 +435,7 @@ class CoverageScript(object): self.coverage.load() if 'execute' in options.actions: - # Run the script. - self.coverage.start() - code_ran = True - try: - try: - if options.module: - self.run_python_module(args[0], args) - else: - self.run_python_file(args[0], args) - except NoSource: - code_ran = False - raise - finally: - if code_ran: - self.coverage.stop() - self.coverage.save() + self.do_execute(options, args) if 'combine' in options.actions: self.coverage.combine() @@ -539,18 +450,167 @@ class CoverageScript(object): ) if 'report' in options.actions: - self.coverage.report( + total = self.coverage.report( show_missing=options.show_missing, **report_args) if 'annotate' in options.actions: self.coverage.annotate( directory=options.directory, **report_args) if 'html' in options.actions: - self.coverage.html_report( - directory=options.directory, **report_args) + total = self.coverage.html_report( + directory=options.directory, title=options.title, + **report_args) if 'xml' in options.actions: outfile = options.outfile - self.coverage.xml_report(outfile=outfile, **report_args) + total = self.coverage.xml_report(outfile=outfile, **report_args) + if options.fail_under is not None: + if total >= options.fail_under: + return OK + else: + return FAIL_UNDER + else: + return OK + + def help(self, error=None, topic=None, parser=None): + """Display an error message, or the named topic.""" + assert error or topic or parser + if error: + print(error) + print("Use 'coverage help' for help.") + elif parser: + print(parser.format_help().strip()) + else: + help_msg = HELP_TOPICS.get(topic, '').strip() + if help_msg: + print(help_msg % self.covpkg.__dict__) + else: + print("Don't know topic %r" % topic) + + def do_help(self, options, args, parser): + """Deal with help requests. + + Return True if it handled the request, False if not. + + """ + # Handle help. + if options.help: + if self.classic: + self.help_fn(topic='help') + else: + self.help_fn(parser=parser) + return True + + if "help" in options.actions: + if args: + for a in args: + parser = CMDS.get(a) + if parser: + self.help_fn(parser=parser) + else: + self.help_fn(topic=a) + else: + self.help_fn(topic='help') + return True + + # Handle version. + if options.version: + self.help_fn(topic='version') + return True + + return False + + def args_ok(self, options, args): + """Check for conflicts and problems in the options. + + Returns True if everything is ok, or False if not. + + """ + for i in ['erase', 'execute']: + for j in ['annotate', 'html', 'report', 'combine']: + if (i in options.actions) and (j in options.actions): + self.help_fn("You can't specify the '%s' and '%s' " + "options at the same time." % (i, j)) + return False + + if not options.actions: + self.help_fn( + "You must specify at least one of -e, -x, -c, -r, -a, or -b." + ) + return False + args_allowed = ( + 'execute' in options.actions or + 'annotate' in options.actions or + 'html' in options.actions or + 'debug' in options.actions or + 'report' in options.actions or + 'xml' in options.actions + ) + if not args_allowed and args: + self.help_fn("Unexpected arguments: %s" % " ".join(args)) + return False + + if 'execute' in options.actions and not args: + self.help_fn("Nothing to do.") + return False + + return True + + def do_execute(self, options, args): + """Implementation of 'coverage run'.""" + + # Set the first path element properly. + old_path0 = sys.path[0] + + # Run the script. + self.coverage.start() + code_ran = True + try: + try: + if options.module: + sys.path[0] = '' + self.run_python_module(args[0], args) + else: + filename = args[0] + sys.path[0] = os.path.abspath(os.path.dirname(filename)) + self.run_python_file(filename, args) + except NoSource: + code_ran = False + raise + finally: + self.coverage.stop() + if code_ran: + self.coverage.save() + + # Restore the old path + sys.path[0] = old_path0 + + def do_debug(self, args): + """Implementation of 'coverage debug'.""" + + if not args: + self.help_fn("What information would you like: data, sys?") + return ERR + for info in args: + if info == 'sys': + print("-- sys ----------------------------------------") + for line in info_formatter(self.coverage.sysinfo()): + print(" %s" % line) + elif info == 'data': + print("-- data ---------------------------------------") + self.coverage.load() + print("path: %s" % self.coverage.data.filename) + print("has_arcs: %r" % self.coverage.data.has_arcs()) + summary = self.coverage.data.summary(fullpath=True) + if summary: + filenames = sorted(summary.keys()) + print("\n%d files:" % len(filenames)) + for f in filenames: + print("%s: %d lines" % (f, summary[f])) + else: + print("No data collected") + else: + self.help_fn("Don't know what you mean by %r" % info) + return ERR return OK @@ -568,10 +628,10 @@ def unshell_list(s): return s.split(',') -HELP_TOPICS = r""" - -== classic ==================================================================== -Coverage.py version %(__version__)s +HELP_TOPICS = { +# ------------------------- +'classic': +r"""Coverage.py version %(__version__)s Measure, collect, and report on code coverage in Python programs. Usage: @@ -615,8 +675,9 @@ coverage -a [-d DIR] [-i] [-o DIR,...] [FILE1 FILE2 ...] Coverage data is saved in the file .coverage by default. Set the COVERAGE_FILE environment variable to save it somewhere else. - -== help ======================================================================= +""", +# ------------------------- +'help': """\ Coverage.py, version %(__version__)s Measure, collect, and report on code coverage in Python programs. @@ -635,26 +696,32 @@ Commands: Use "coverage help <command>" for detailed help on any command. Use "coverage help classic" for help on older command syntax. For more information, see %(__url__)s - -== minimum_help =============================================================== +""", +# ------------------------- +'minimum_help': """\ Code coverage for Python. Use 'coverage help' for help. - -== version ==================================================================== +""", +# ------------------------- +'version': """\ Coverage.py, version %(__version__)s. %(__url__)s - -""" +""", +} def main(argv=None): - """The main entrypoint to Coverage. + """The main entry point to Coverage. - This is installed as the script entrypoint. + This is installed as the script entry point. """ if argv is None: argv = sys.argv[1:] try: + start = time.clock() status = CoverageScript().command_line(argv) + end = time.clock() + if 0: + print("time: %.3fs" % (end - start)) except ExceptionDuringRun: # An exception was caught while running the product code. The # sys.exc_info() return tuple is packed into an ExceptionDuringRun diff --git a/python/helpers/coverage/codeunit.py b/python/helpers/coverage/codeunit.py index 55f44a240acf..ca1ae5c56d6b 100644 --- a/python/helpers/coverage/codeunit.py +++ b/python/helpers/coverage/codeunit.py @@ -52,8 +52,10 @@ class CodeUnit(object): else: f = morf # .pyc files should always refer to a .py instead. - if f.endswith('.pyc'): + if f.endswith('.pyc') or f.endswith('.pyo'): f = f[:-1] + elif f.endswith('$py.class'): # Jython + f = f[:-9] + ".py" self.filename = self.file_locator.canonical_filename(f) if hasattr(morf, '__name__'): @@ -77,12 +79,18 @@ class CodeUnit(object): # Annoying comparison operators. Py3k wants __lt__ etc, and Py2k needs all # of them defined. - def __lt__(self, other): return self.name < other.name - def __le__(self, other): return self.name <= other.name - def __eq__(self, other): return self.name == other.name - def __ne__(self, other): return self.name != other.name - def __gt__(self, other): return self.name > other.name - def __ge__(self, other): return self.name >= other.name + def __lt__(self, other): + return self.name < other.name + def __le__(self, other): + return self.name <= other.name + def __eq__(self, other): + return self.name == other.name + def __ne__(self, other): + return self.name != other.name + def __gt__(self, other): + return self.name > other.name + def __ge__(self, other): + return self.name >= other.name def flat_rootname(self): """A base for a flat filename to correspond to this code unit. @@ -113,5 +121,25 @@ class CodeUnit(object): # Couldn't find source. raise CoverageException( - "No source for code %r." % self.filename + "No source for code '%s'." % self.filename ) + + def should_be_python(self): + """Does it seem like this file should contain Python? + + This is used to decide if a file reported as part of the exection of + a program was really likely to have contained Python in the first + place. + + """ + # Get the file extension. + _, ext = os.path.splitext(self.filename) + + # Anything named *.py* should be Python. + if ext.startswith('.py'): + return True + # A file with no extension should be Python. + if not ext: + return True + # Everything else is probably not Python. + return False diff --git a/python/helpers/coverage/collector.py b/python/helpers/coverage/collector.py index 9c40d16c7b19..8ba7d87cd4e0 100644 --- a/python/helpers/coverage/collector.py +++ b/python/helpers/coverage/collector.py @@ -1,13 +1,24 @@ """Raw data collector for Coverage.""" -import sys, threading +import os, sys, threading try: # Use the C extension code when we can, for speed. - from coverage.tracer import Tracer + from coverage.tracer import CTracer # pylint: disable=F0401,E0611 except ImportError: # Couldn't import the C extension, maybe it isn't built. - Tracer = None + if os.getenv('COVERAGE_TEST_TRACER') == 'c': + # During testing, we use the COVERAGE_TEST_TRACER env var to indicate + # that we've fiddled with the environment to test this fallback code. + # If we thought we had a C tracer, but couldn't import it, then exit + # quickly and clearly instead of dribbling confusing errors. I'm using + # sys.exit here instead of an exception because an exception here + # causes all sorts of other noise in unittest. + sys.stderr.write( + "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n" + ) + sys.exit(1) + CTracer = None class PyTracer(object): @@ -40,12 +51,19 @@ class PyTracer(object): self.last_exc_back = None self.last_exc_firstlineno = 0 self.arcs = False + self.thread = None + self.stopped = False def _trace(self, frame, event, arg_unused): """The trace function passed to sys.settrace.""" - #print("trace event: %s %r @%d" % ( - # event, frame.f_code.co_filename, frame.f_lineno)) + if self.stopped: + return + + if 0: + sys.stderr.write("trace event: %s %r @%d\n" % ( + event, frame.f_code.co_filename, frame.f_lineno + )) if self.last_exc_back: if frame == self.last_exc_back: @@ -61,10 +79,11 @@ class PyTracer(object): # in this file. self.data_stack.append((self.cur_file_data, self.last_line)) filename = frame.f_code.co_filename - tracename = self.should_trace_cache.get(filename) - if tracename is None: + if filename not in self.should_trace_cache: tracename = self.should_trace(filename, frame) self.should_trace_cache[filename] = tracename + else: + tracename = self.should_trace_cache[filename] #print("called, stack is %d deep, tracename is %r" % ( # len(self.data_stack), tracename)) if tracename: @@ -105,15 +124,24 @@ class PyTracer(object): Return a Python function suitable for use with sys.settrace(). """ + self.thread = threading.currentThread() sys.settrace(self._trace) return self._trace def stop(self): """Stop this Tracer.""" + self.stopped = True + if self.thread != threading.currentThread(): + # Called on a different thread than started us: we can't unhook + # ourseves, but we've set the flag that we should stop, so we won't + # do any more tracing. + return + if hasattr(sys, "gettrace") and self.warn: if sys.gettrace() != self._trace: msg = "Trace function changed, measurement is likely wrong: %r" - self.warn(msg % sys.gettrace()) + self.warn(msg % (sys.gettrace(),)) + #print("Stopping tracer on %s" % threading.current_thread().ident) sys.settrace(None) def get_stats(self): @@ -146,7 +174,7 @@ class Collector(object): """Create a collector. `should_trace` is a function, taking a filename, and returning a - canonicalized filename, or False depending on whether the file should + canonicalized filename, or None depending on whether the file should be traced or not. If `timid` is true, then a slower simpler trace function will be @@ -173,7 +201,7 @@ class Collector(object): else: # Being fast: use the C Tracer if it is available, else the Python # trace function. - self._trace_class = Tracer or PyTracer + self._trace_class = CTracer or PyTracer def __repr__(self): return "<Collector at 0x%x>" % id(self) @@ -190,7 +218,7 @@ class Collector(object): # A cache of the results from should_trace, the decision about whether # to trace execution in a file. A dict of filename to (filename or - # False). + # None). self.should_trace_cache = {} # Our active Tracers. @@ -232,9 +260,29 @@ class Collector(object): if self._collectors: self._collectors[-1].pause() self._collectors.append(self) - #print >>sys.stderr, "Started: %r" % self._collectors + #print("Started: %r" % self._collectors, file=sys.stderr) + + # Check to see whether we had a fullcoverage tracer installed. + traces0 = [] + if hasattr(sys, "gettrace"): + fn0 = sys.gettrace() + if fn0: + tracer0 = getattr(fn0, '__self__', None) + if tracer0: + traces0 = getattr(tracer0, 'traces', []) + # Install the tracer on this thread. - self._start_tracer() + fn = self._start_tracer() + + for args in traces0: + (frame, event, arg), lineno = args + try: + fn(frame, event, arg, lineno=lineno) + except TypeError: + raise Exception( + "fullcoverage must be run with the C trace function." + ) + # Install our installation tracer in threading, to jump start other # threads. threading.settrace(self._installation_trace) diff --git a/python/helpers/coverage/config.py b/python/helpers/coverage/config.py index 6b441ddc45fc..87318ff12452 100644 --- a/python/helpers/coverage/config.py +++ b/python/helpers/coverage/config.py @@ -1,7 +1,76 @@ """Config file for coverage.py""" -import os -from coverage.backward import configparser # pylint: disable=W0622 +import os, re, sys +from coverage.backward import string_class, iitems + +# In py3, # ConfigParser was renamed to the more-standard configparser +try: + import configparser # pylint: disable=F0401 +except ImportError: + import ConfigParser as configparser + + +class HandyConfigParser(configparser.RawConfigParser): + """Our specialization of ConfigParser.""" + + def read(self, filename): + """Read a filename as UTF-8 configuration data.""" + kwargs = {} + if sys.version_info >= (3, 2): + kwargs['encoding'] = "utf-8" + return configparser.RawConfigParser.read(self, filename, **kwargs) + + def get(self, *args, **kwargs): + v = configparser.RawConfigParser.get(self, *args, **kwargs) + def dollar_replace(m): + """Called for each $replacement.""" + # Only one of the groups will have matched, just get its text. + word = [w for w in m.groups() if w is not None][0] + if word == "$": + return "$" + else: + return os.environ.get(word, '') + + dollar_pattern = r"""(?x) # Use extended regex syntax + \$(?: # A dollar sign, then + (?P<v1>\w+) | # a plain word, + {(?P<v2>\w+)} | # or a {-wrapped word, + (?P<char>[$]) # or a dollar sign. + ) + """ + v = re.sub(dollar_pattern, dollar_replace, v) + return v + + def getlist(self, section, option): + """Read a list of strings. + + The value of `section` and `option` is treated as a comma- and newline- + separated list of strings. Each value is stripped of whitespace. + + Returns the list of strings. + + """ + value_list = self.get(section, option) + values = [] + for value_line in value_list.split('\n'): + for value in value_line.split(','): + value = value.strip() + if value: + values.append(value) + return values + + def getlinelist(self, section, option): + """Read a list of full-line strings. + + The value of `section` and `option` is treated as a newline-separated + list of strings. Each value is stripped of whitespace. + + Returns the list of strings. + + """ + value_list = self.get(section, option) + return list(filter(None, value_list.split('\n'))) + # The default line exclusion regexes DEFAULT_EXCLUDE = [ @@ -29,9 +98,12 @@ class CoverageConfig(object): operation of coverage.py. """ - def __init__(self): """Initialize the configuration attributes to their defaults.""" + # Metadata about the config. + self.attempted_config_files = [] + self.config_files = [] + # Defaults for [run] self.branch = False self.cover_pylib = False @@ -39,6 +111,7 @@ class CoverageConfig(object): self.parallel = False self.timid = False self.source = None + self.debug = [] # Defaults for [report] self.exclude_list = DEFAULT_EXCLUDE[:] @@ -48,13 +121,19 @@ class CoverageConfig(object): self.partial_list = DEFAULT_PARTIAL[:] self.partial_always_list = DEFAULT_PARTIAL_ALWAYS[:] self.precision = 0 + self.show_missing = False # Defaults for [html] self.html_dir = "htmlcov" + self.extra_css = None + self.html_title = "Coverage report" # Defaults for [xml] self.xml_output = "coverage.xml" + # Defaults for [paths] + self.paths = {} + def from_environment(self, env_var): """Read configuration from the `env_var` environment variable.""" # Timidity: for nose users, read an environment variable. This is a @@ -64,93 +143,71 @@ class CoverageConfig(object): if env: self.timid = ('--timid' in env) + MUST_BE_LIST = ["omit", "include", "debug"] + def from_args(self, **kwargs): """Read config values from `kwargs`.""" - for k, v in kwargs.items(): + for k, v in iitems(kwargs): if v is not None: + if k in self.MUST_BE_LIST and isinstance(v, string_class): + v = [v] setattr(self, k, v) - def from_file(self, *files): - """Read configuration from .rc files. + def from_file(self, filename): + """Read configuration from a .rc file. - Each argument in `files` is a file name to read. + `filename` is a file name to read. """ - cp = configparser.RawConfigParser() - cp.read(files) + self.attempted_config_files.append(filename) + cp = HandyConfigParser() + files_read = cp.read(filename) + if files_read is not None: # return value changed in 2.4 + self.config_files.extend(files_read) + + for option_spec in self.CONFIG_FILE_OPTIONS: + self.set_attr_from_config_option(cp, *option_spec) + + # [paths] is special + if cp.has_section('paths'): + for option in cp.options('paths'): + self.paths[option] = cp.getlist('paths', option) + + CONFIG_FILE_OPTIONS = [ # [run] - if cp.has_option('run', 'branch'): - self.branch = cp.getboolean('run', 'branch') - if cp.has_option('run', 'cover_pylib'): - self.cover_pylib = cp.getboolean('run', 'cover_pylib') - if cp.has_option('run', 'data_file'): - self.data_file = cp.get('run', 'data_file') - if cp.has_option('run', 'include'): - self.include = self.get_list(cp, 'run', 'include') - if cp.has_option('run', 'omit'): - self.omit = self.get_list(cp, 'run', 'omit') - if cp.has_option('run', 'parallel'): - self.parallel = cp.getboolean('run', 'parallel') - if cp.has_option('run', 'source'): - self.source = self.get_list(cp, 'run', 'source') - if cp.has_option('run', 'timid'): - self.timid = cp.getboolean('run', 'timid') + ('branch', 'run:branch', 'boolean'), + ('cover_pylib', 'run:cover_pylib', 'boolean'), + ('data_file', 'run:data_file'), + ('debug', 'run:debug', 'list'), + ('include', 'run:include', 'list'), + ('omit', 'run:omit', 'list'), + ('parallel', 'run:parallel', 'boolean'), + ('source', 'run:source', 'list'), + ('timid', 'run:timid', 'boolean'), # [report] - if cp.has_option('report', 'exclude_lines'): - self.exclude_list = \ - self.get_line_list(cp, 'report', 'exclude_lines') - if cp.has_option('report', 'ignore_errors'): - self.ignore_errors = cp.getboolean('report', 'ignore_errors') - if cp.has_option('report', 'include'): - self.include = self.get_list(cp, 'report', 'include') - if cp.has_option('report', 'omit'): - self.omit = self.get_list(cp, 'report', 'omit') - if cp.has_option('report', 'partial_branches'): - self.partial_list = \ - self.get_line_list(cp, 'report', 'partial_branches') - if cp.has_option('report', 'partial_branches_always'): - self.partial_always_list = \ - self.get_line_list(cp, 'report', 'partial_branches_always') - if cp.has_option('report', 'precision'): - self.precision = cp.getint('report', 'precision') + ('exclude_list', 'report:exclude_lines', 'linelist'), + ('ignore_errors', 'report:ignore_errors', 'boolean'), + ('include', 'report:include', 'list'), + ('omit', 'report:omit', 'list'), + ('partial_list', 'report:partial_branches', 'linelist'), + ('partial_always_list', 'report:partial_branches_always', 'linelist'), + ('precision', 'report:precision', 'int'), + ('show_missing', 'report:show_missing', 'boolean'), # [html] - if cp.has_option('html', 'directory'): - self.html_dir = cp.get('html', 'directory') + ('html_dir', 'html:directory'), + ('extra_css', 'html:extra_css'), + ('html_title', 'html:title'), # [xml] - if cp.has_option('xml', 'output'): - self.xml_output = cp.get('xml', 'output') - - def get_list(self, cp, section, option): - """Read a list of strings from the ConfigParser `cp`. - - The value of `section` and `option` is treated as a comma- and newline- - separated list of strings. Each value is stripped of whitespace. - - Returns the list of strings. - - """ - value_list = cp.get(section, option) - values = [] - for value_line in value_list.split('\n'): - for value in value_line.split(','): - value = value.strip() - if value: - values.append(value) - return values - - def get_line_list(self, cp, section, option): - """Read a list of full-line strings from the ConfigParser `cp`. - - The value of `section` and `option` is treated as a newline-separated - list of strings. Each value is stripped of whitespace. - - Returns the list of strings. - - """ - value_list = cp.get(section, option) - return list(filter(None, value_list.split('\n'))) - + ('xml_output', 'xml:output'), + ] + + def set_attr_from_config_option(self, cp, attr, where, type_=''): + """Set an attribute on self if it exists in the ConfigParser.""" + section, option = where.split(":") + if cp.has_option(section, option): + method = getattr(cp, 'get'+type_) + setattr(self, attr, method(section, option)) diff --git a/python/helpers/coverage/control.py b/python/helpers/coverage/control.py index 5ca1ef9538cc..f75a3dda5b1b 100644 --- a/python/helpers/coverage/control.py +++ b/python/helpers/coverage/control.py @@ -3,21 +3,31 @@ import atexit, os, random, socket, sys from coverage.annotate import AnnotateReporter -from coverage.backward import string_class +from coverage.backward import string_class, iitems, sorted # pylint: disable=W0622 from coverage.codeunit import code_unit_factory, CodeUnit from coverage.collector import Collector from coverage.config import CoverageConfig from coverage.data import CoverageData +from coverage.debug import DebugControl from coverage.files import FileLocator, TreeMatcher, FnmatchMatcher -from coverage.files import find_python_files +from coverage.files import PathAliases, find_python_files, prep_patterns from coverage.html import HtmlReporter from coverage.misc import CoverageException, bool_or_none, join_regex +from coverage.misc import file_be_gone from coverage.results import Analysis, Numbers from coverage.summary import SummaryReporter from coverage.xmlreport import XmlReporter +# Pypy has some unusual stuff in the "stdlib". Consider those locations +# when deciding where the stdlib is. +try: + import _structseq # pylint: disable=F0401 +except ImportError: + _structseq = None + + class coverage(object): - """Programmatic access to Coverage. + """Programmatic access to coverage.py. To use:: @@ -25,14 +35,15 @@ class coverage(object): cov = coverage() cov.start() - #.. blah blah (run your code) blah blah .. + #.. call your code .. cov.stop() cov.html_report(directory='covhtml') """ def __init__(self, data_file=None, data_suffix=None, cover_pylib=None, auto_data=False, timid=None, branch=None, config_file=True, - source=None, omit=None, include=None): + source=None, omit=None, include=None, debug=None, + debug_file=None): """ `data_file` is the base name of the data file to use, defaulting to ".coverage". `data_suffix` is appended (with a dot) to `data_file` to @@ -67,6 +78,10 @@ class coverage(object): `include` will be measured, files that match `omit` will not. Each will also accept a single string argument. + `debug` is a list of strings indicating what debugging information is + desired. `debug_file` is the file to write debug messages to, + defaulting to stderr. + """ from coverage import __version__ @@ -96,18 +111,16 @@ class coverage(object): self.config.data_file = env_data_file # 4: from constructor arguments: - if isinstance(omit, string_class): - omit = [omit] - if isinstance(include, string_class): - include = [include] self.config.from_args( data_file=data_file, cover_pylib=cover_pylib, timid=timid, branch=branch, parallel=bool_or_none(data_suffix), - source=source, omit=omit, include=include + source=source, omit=omit, include=include, debug=debug, ) + # Create and configure the debugging controller. + self.debug = DebugControl(self.config.debug, debug_file or sys.stderr) + self.auto_data = auto_data - self.atexit_registered = False # _exclude_re is a dict mapping exclusion list names to compiled # regexes. @@ -125,8 +138,8 @@ class coverage(object): else: self.source_pkgs.append(src) - self.omit = self._prep_patterns(self.config.omit) - self.include = self._prep_patterns(self.config.include) + self.omit = prep_patterns(self.config.omit) + self.include = prep_patterns(self.config.include) self.collector = Collector( self._should_trace, timid=self.config.timid, @@ -151,7 +164,8 @@ class coverage(object): # started rather than wherever the process eventually chdir'd to. self.data = CoverageData( basename=self.config.data_file, - collector="coverage v%s" % __version__ + collector="coverage v%s" % __version__, + debug=self.debug, ) # The dirs for files considered "installed with the interpreter". @@ -162,9 +176,9 @@ class coverage(object): # environments (virtualenv, for example), these modules may be # spread across a few locations. Look at all the candidate modules # we've imported, and take all the different ones. - for m in (atexit, os, random, socket): - if hasattr(m, "__file__"): - m_dir = self._canonical_dir(m.__file__) + for m in (atexit, os, random, socket, _structseq): + if m is not None and hasattr(m, "__file__"): + m_dir = self._canonical_dir(m) if m_dir not in self.pylib_dirs: self.pylib_dirs.append(m_dir) @@ -172,63 +186,61 @@ class coverage(object): # where we are. self.cover_dir = self._canonical_dir(__file__) - # The matchers for _should_trace, created when tracing starts. + # The matchers for _should_trace. self.source_match = None self.pylib_match = self.cover_match = None self.include_match = self.omit_match = None - # Only _harvest_data once per measurement cycle. - self._harvested = False - # Set the reporting precision. Numbers.set_precision(self.config.precision) - # When tearing down the coverage object, modules can become None. - # Saving the modules as object attributes avoids problems, but it is - # quite ad-hoc which modules need to be saved and which references - # need to use the object attributes. - self.socket = socket - self.os = os - self.random = random + # Is it ok for no data to be collected? + self._warn_no_data = True + self._warn_unimported_source = True + + # State machine variables: + # Have we started collecting and not stopped it? + self._started = False + # Have we measured some data and not harvested it? + self._measured = False - def _canonical_dir(self, f): - """Return the canonical directory of the file `f`.""" - return os.path.split(self.file_locator.canonical_filename(f))[0] + atexit.register(self._atexit) + + def _canonical_dir(self, morf): + """Return the canonical directory of the module or file `morf`.""" + return os.path.split(CodeUnit(morf, self.file_locator).filename)[0] def _source_for_file(self, filename): """Return the source file for `filename`.""" if not filename.endswith(".py"): if filename[-4:-1] == ".py": filename = filename[:-1] + elif filename.endswith("$py.class"): # jython + filename = filename[:-9] + ".py" return filename - def _should_trace(self, filename, frame): - """Decide whether to trace execution in `filename` + def _should_trace_with_reason(self, filename, frame): + """Decide whether to trace execution in `filename`, with a reason. This function is called from the trace function. As each new file name is encountered, this function determines whether it is traced or not. - Returns a canonicalized filename if it should be traced, False if it - should not. + Returns a pair of values: the first indicates whether the file should + be traced: it's a canonicalized filename if it should be traced, None + if it should not. The second value is a string, the resason for the + decision. """ - if os is None: - return False + if not filename: + # Empty string is pretty useless + return None, "empty string isn't a filename" if filename.startswith('<'): # Lots of non-file execution is represented with artificial # filenames like "<string>", "<doctest readme.txt[0]>", or # "<exec_function>". Don't ever trace these executions, since we # can't do anything with the data later anyway. - return False - - if filename.endswith(".html"): - # Jinja and maybe other templating systems compile templates into - # Python code, but use the template filename as the filename in - # the compiled code. Of course, those filenames are useless later - # so don't bother collecting. TODO: How should we really separate - # out good file extensions from bad? - return False + return None, "not a real filename" self._check_for_packages() @@ -248,64 +260,53 @@ class coverage(object): canonical = self.file_locator.canonical_filename(filename) - # If the user specified source, then that's authoritative about what to - # measure. If they didn't, then we have to exclude the stdlib and - # coverage.py directories. + # If the user specified source or include, then that's authoritative + # about the outer bound of what to measure and we don't have to apply + # any canned exclusions. If they didn't, then we have to exclude the + # stdlib and coverage.py directories. if self.source_match: if not self.source_match.match(canonical): - return False + return None, "falls outside the --source trees" + elif self.include_match: + if not self.include_match.match(canonical): + return None, "falls outside the --include trees" else: # If we aren't supposed to trace installed code, then check if this # is near the Python standard library and skip it if so. if self.pylib_match and self.pylib_match.match(canonical): - return False + return None, "is in the stdlib" # We exclude the coverage code itself, since a little of it will be # measured otherwise. if self.cover_match and self.cover_match.match(canonical): - return False + return None, "is part of coverage.py" - # Check the file against the include and omit patterns. - if self.include_match and not self.include_match.match(canonical): - return False + # Check the file against the omit pattern. if self.omit_match and self.omit_match.match(canonical): - return False + return None, "is inside an --omit pattern" - return canonical + return canonical, "because we love you" + + def _should_trace(self, filename, frame): + """Decide whether to trace execution in `filename`. - # To log what should_trace returns, change this to "if 1:" - if 0: - _real_should_trace = _should_trace - def _should_trace(self, filename, frame): # pylint: disable=E0102 - """A logging decorator around the real _should_trace function.""" - ret = self._real_should_trace(filename, frame) - print("should_trace: %r -> %r" % (filename, ret)) - return ret + Calls `_should_trace_with_reason`, and returns just the decision. + + """ + canonical, reason = self._should_trace_with_reason(filename, frame) + if self.debug.should('trace'): + if not canonical: + msg = "Not tracing %r: %s" % (filename, reason) + else: + msg = "Tracing %r" % (filename,) + self.debug.write(msg) + return canonical def _warn(self, msg): """Use `msg` as a warning.""" self._warnings.append(msg) sys.stderr.write("Coverage.py warning: %s\n" % msg) - def _prep_patterns(self, patterns): - """Prepare the file patterns for use in a `FnmatchMatcher`. - - If a pattern starts with a wildcard, it is used as a pattern - as-is. If it does not start with a wildcard, then it is made - absolute with the current directory. - - If `patterns` is None, an empty list is returned. - - """ - patterns = patterns or [] - prepped = [] - for p in patterns or []: - if p.startswith("*") or p.startswith("?"): - prepped.append(p) - else: - prepped.append(self.file_locator.abs_file(p)) - return prepped - def _check_for_packages(self): """Update the source_match matcher with latest imported packages.""" # Our self.source_pkgs attribute is a list of package names we want to @@ -325,17 +326,23 @@ class coverage(object): try: pkg_file = mod.__file__ except AttributeError: - self._warn("Module %s has no Python source." % pkg) + pkg_file = None else: d, f = os.path.split(pkg_file) - if f.startswith('__init__.'): + if f.startswith('__init__'): # This is actually a package, return the directory. pkg_file = d else: pkg_file = self._source_for_file(pkg_file) pkg_file = self.file_locator.canonical_filename(pkg_file) + if not os.path.exists(pkg_file): + pkg_file = None + + if pkg_file: self.source.append(pkg_file) self.source_match.add(pkg_file) + else: + self._warn("Module %s has no Python source." % pkg) for pkg in found: self.source_pkgs.remove(pkg) @@ -354,17 +361,21 @@ class coverage(object): self.data.read() def start(self): - """Start measuring code coverage.""" + """Start measuring code coverage. + + Coverage measurement actually occurs in functions called after `start` + is invoked. Statements in the same scope as `start` won't be measured. + + Once you invoke `start`, you must also call `stop` eventually, or your + process might not shut down cleanly. + + """ if self.run_suffix: # Calling start() means we're running code, so use the run_suffix # as the data_suffix when we eventually save the data. self.data_suffix = self.run_suffix if self.auto_data: self.load() - # Save coverage data when Python exits. - if not self.atexit_registered: - atexit.register(self.save) - self.atexit_registered = True # Create the matchers we need for _should_trace if self.source or self.source_pkgs: @@ -379,13 +390,31 @@ class coverage(object): if self.omit: self.omit_match = FnmatchMatcher(self.omit) - self._harvested = False + # The user may want to debug things, show info if desired. + if self.debug.should('config'): + self.debug.write("Configuration values:") + config_info = sorted(self.config.__dict__.items()) + self.debug.write_formatted_info(config_info) + + if self.debug.should('sys'): + self.debug.write("Debugging info:") + self.debug.write_formatted_info(self.sysinfo()) + self.collector.start() + self._started = True + self._measured = True def stop(self): """Stop measuring code coverage.""" + self._started = False self.collector.stop() - self._harvest_data() + + def _atexit(self): + """Clean up on process shutdown.""" + if self._started: + self.stop() + if self.auto_data: + self.save() def erase(self): """Erase previously-collected coverage data. @@ -449,9 +478,15 @@ class coverage(object): # plenty of distinguishing information. We do this here in # `save()` at the last minute so that the pid will be correct even # if the process forks. - data_suffix = "%s.%s.%06d" % ( - self.socket.gethostname(), self.os.getpid(), - self.random.randint(0, 99999) + extra = "" + if _TEST_NAME_FILE: + f = open(_TEST_NAME_FILE) + test_name = f.read() + f.close() + extra = "." + test_name + data_suffix = "%s%s.%s.%06d" % ( + socket.gethostname(), extra, os.getpid(), + random.randint(0, 999999) ) self._harvest_data() @@ -465,7 +500,14 @@ class coverage(object): current measurements. """ - self.data.combine_parallel_data() + aliases = None + if self.config.paths: + aliases = PathAliases(self.file_locator) + for paths in self.config.paths.values(): + result = paths[0] + for pattern in paths[1:]: + aliases.add(pattern, result) + self.data.combine_parallel_data(aliases=aliases) def _harvest_data(self): """Get the collected data and reset the collector. @@ -473,27 +515,37 @@ class coverage(object): Also warn about various problems collecting data. """ - if not self._harvested: - self.data.add_line_data(self.collector.get_line_data()) - self.data.add_arc_data(self.collector.get_arc_data()) - self.collector.reset() + if not self._measured: + return - # If there are still entries in the source_pkgs list, then we never - # encountered those packages. + self.data.add_line_data(self.collector.get_line_data()) + self.data.add_arc_data(self.collector.get_arc_data()) + self.collector.reset() + + # If there are still entries in the source_pkgs list, then we never + # encountered those packages. + if self._warn_unimported_source: for pkg in self.source_pkgs: self._warn("Module %s was never imported." % pkg) - # Find out if we got any data. - summary = self.data.summary() - if not summary: - self._warn("No data was collected.") + # Find out if we got any data. + summary = self.data.summary() + if not summary and self._warn_no_data: + self._warn("No data was collected.") + + # Find files that were never executed at all. + for src in self.source: + for py_file in find_python_files(src): + py_file = self.file_locator.canonical_filename(py_file) + + if self.omit_match and self.omit_match.match(py_file): + # Turns out this file was omitted, so don't pull it back + # in as unexecuted. + continue - # Find files that were never executed at all. - for src in self.source: - for py_file in find_python_files(src): - self.data.touch_file(py_file) + self.data.touch_file(py_file) - self._harvested = True + self._measured = False # Backward compatibility with version 1. def analysis(self, morf): @@ -520,8 +572,11 @@ class coverage(object): """ analysis = self._analyze(morf) return ( - analysis.filename, analysis.statements, analysis.excluded, - analysis.missing, analysis.missing_formatted() + analysis.filename, + sorted(analysis.statements), + sorted(analysis.excluded), + sorted(analysis.missing), + analysis.missing_formatted(), ) def _analyze(self, it): @@ -530,6 +585,7 @@ class coverage(object): Returns an `Analysis` object. """ + self._harvest_data() if not isinstance(it, CodeUnit): it = code_unit_factory(it, self.file_locator)[0] @@ -548,14 +604,16 @@ class coverage(object): match those patterns will be included in the report. Modules matching `omit` will not be included in the report. + Returns a float, the total percentage covered. + """ + self._harvest_data() self.config.from_args( - ignore_errors=ignore_errors, omit=omit, include=include - ) - reporter = SummaryReporter( - self, show_missing, self.config.ignore_errors + ignore_errors=ignore_errors, omit=omit, include=include, + show_missing=show_missing, ) - reporter.report(morfs, outfile=file, config=self.config) + reporter = SummaryReporter(self, self.config) + return reporter.report(morfs, outfile=file) def annotate(self, morfs=None, directory=None, ignore_errors=None, omit=None, include=None): @@ -569,25 +627,39 @@ class coverage(object): See `coverage.report()` for other arguments. """ + self._harvest_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include ) - reporter = AnnotateReporter(self, self.config.ignore_errors) - reporter.report(morfs, config=self.config, directory=directory) + reporter = AnnotateReporter(self, self.config) + reporter.report(morfs, directory=directory) def html_report(self, morfs=None, directory=None, ignore_errors=None, - omit=None, include=None): + omit=None, include=None, extra_css=None, title=None): """Generate an HTML report. + The HTML is written to `directory`. The file "index.html" is the + overview starting point, with links to more detailed pages for + individual modules. + + `extra_css` is a path to a file of other CSS to apply on the page. + It will be copied into the HTML directory. + + `title` is a text string (not HTML) to use as the title of the HTML + report. + See `coverage.report()` for other arguments. + Returns a float, the total percentage covered. + """ + self._harvest_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include, - html_dir=directory, + html_dir=directory, extra_css=extra_css, html_title=title, ) - reporter = HtmlReporter(self, self.config.ignore_errors) - reporter.report(morfs, config=self.config) + reporter = HtmlReporter(self, self.config) + return reporter.report(morfs) def xml_report(self, morfs=None, outfile=None, ignore_errors=None, omit=None, include=None): @@ -600,12 +672,16 @@ class coverage(object): See `coverage.report()` for other arguments. + Returns a float, the total percentage covered. + """ + self._harvest_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include, xml_output=outfile, ) file_to_close = None + delete_file = False if self.config.xml_output: if self.config.xml_output == '-': outfile = sys.stdout @@ -613,11 +689,17 @@ class coverage(object): outfile = open(self.config.xml_output, "w") file_to_close = outfile try: - reporter = XmlReporter(self, self.config.ignore_errors) - reporter.report(morfs, outfile=outfile, config=self.config) + try: + reporter = XmlReporter(self, self.config) + return reporter.report(morfs, outfile=outfile) + except CoverageException: + delete_file = True + raise finally: if file_to_close: file_to_close.close() + if delete_file: + file_be_gone(self.config.xml_output) def sysinfo(self): """Return a list of (key, value) pairs showing internal information.""" @@ -625,22 +707,43 @@ class coverage(object): import coverage as covmod import platform, re + try: + implementation = platform.python_implementation() + except AttributeError: + implementation = "unknown" + info = [ ('version', covmod.__version__), ('coverage', covmod.__file__), ('cover_dir', self.cover_dir), ('pylib_dirs', self.pylib_dirs), ('tracer', self.collector.tracer_name()), + ('config_files', self.config.attempted_config_files), + ('configs_read', self.config.config_files), ('data_path', self.data.filename), ('python', sys.version.replace('\n', '')), ('platform', platform.platform()), + ('implementation', implementation), + ('executable', sys.executable), ('cwd', os.getcwd()), ('path', sys.path), - ('environment', [ - ("%s = %s" % (k, v)) for k, v in os.environ.items() - if re.search("^COV|^PY", k) - ]), + ('environment', sorted([ + ("%s = %s" % (k, v)) for k, v in iitems(os.environ) + if re.search(r"^COV|^PY", k) + ])), + ('command_line', " ".join(getattr(sys, 'argv', ['???']))), ] + if self.source_match: + info.append(('source_match', self.source_match.info())) + if self.include_match: + info.append(('include_match', self.include_match.info())) + if self.omit_match: + info.append(('omit_match', self.omit_match.info())) + if self.cover_match: + info.append(('cover_match', self.cover_match.info())) + if self.pylib_match: + info.append(('pylib_match', self.pylib_match.info())) + return info @@ -667,7 +770,10 @@ def process_startup(): cps = os.environ.get("COVERAGE_PROCESS_START") if cps: cov = coverage(config_file=cps, auto_data=True) - if os.environ.get("COVERAGE_COVERAGE"): - # Measuring coverage within coverage.py takes yet more trickery. - cov.cover_dir = "Please measure coverage.py!" cov.start() + cov._warn_no_data = False + cov._warn_unimported_source = False + + +# A hack for debugging testing in subprocesses. +_TEST_NAME_FILE = "" #"/tmp/covtest.txt" diff --git a/python/helpers/coverage/data.py b/python/helpers/coverage/data.py index 3263cb38883b..fb88c5b1e638 100644 --- a/python/helpers/coverage/data.py +++ b/python/helpers/coverage/data.py @@ -2,7 +2,9 @@ import os -from coverage.backward import pickle, sorted # pylint: disable=W0622 +from coverage.backward import iitems, pickle, sorted # pylint: disable=W0622 +from coverage.files import PathAliases +from coverage.misc import file_be_gone class CoverageData(object): @@ -21,15 +23,18 @@ class CoverageData(object): """ - def __init__(self, basename=None, collector=None): + def __init__(self, basename=None, collector=None, debug=None): """Create a CoverageData. `basename` is the name of the file to use for storing data. `collector` is a string describing the coverage measurement software. + `debug` is a `DebugControl` object for writing debug messages. + """ self.collector = collector or 'unknown' + self.debug = debug self.use_file = True @@ -59,10 +64,6 @@ class CoverageData(object): # self.arcs = {} - self.os = os - self.sorted = sorted - self.pickle = pickle - def usefile(self, use_file=True): """Set whether or not to use a disk file for data.""" self.use_file = use_file @@ -92,21 +93,21 @@ class CoverageData(object): def erase(self): """Erase the data, both in this object, and from its file storage.""" if self.use_file: - if self.filename and os.path.exists(self.filename): - os.remove(self.filename) + if self.filename: + file_be_gone(self.filename) self.lines = {} self.arcs = {} def line_data(self): """Return the map from filenames to lists of line numbers executed.""" return dict( - [(f, self.sorted(lmap.keys())) for f, lmap in self.lines.items()] + [(f, sorted(lmap.keys())) for f, lmap in iitems(self.lines)] ) def arc_data(self): """Return the map from filenames to lists of line number pairs.""" return dict( - [(f, self.sorted(amap.keys())) for f, amap in self.arcs.items()] + [(f, sorted(amap.keys())) for f, amap in iitems(self.arcs)] ) def write_file(self, filename): @@ -123,10 +124,13 @@ class CoverageData(object): if self.collector: data['collector'] = self.collector + if self.debug and self.debug.should('dataio'): + self.debug.write("Writing data to %r" % (filename,)) + # Write the pickle to the file. fdata = open(filename, 'wb') try: - self.pickle.dump(data, fdata, 2) + pickle.dump(data, fdata, 2) finally: fdata.close() @@ -136,6 +140,8 @@ class CoverageData(object): def raw_data(self, filename): """Return the raw pickled data from `filename`.""" + if self.debug and self.debug.should('dataio'): + self.debug.write("Reading data from %r" % (filename,)) fdata = open(filename, 'rb') try: data = pickle.load(fdata) @@ -158,33 +164,39 @@ class CoverageData(object): # Unpack the 'lines' item. lines = dict([ (f, dict.fromkeys(linenos, None)) - for f, linenos in data.get('lines', {}).items() + for f, linenos in iitems(data.get('lines', {})) ]) # Unpack the 'arcs' item. arcs = dict([ (f, dict.fromkeys(arcpairs, None)) - for f, arcpairs in data.get('arcs', {}).items() + for f, arcpairs in iitems(data.get('arcs', {})) ]) except Exception: pass return lines, arcs - def combine_parallel_data(self): + def combine_parallel_data(self, aliases=None): """Combine a number of data files together. Treat `self.filename` as a file prefix, and combine the data from all of the data files starting with that prefix plus a dot. + If `aliases` is provided, it's a `PathAliases` object that is used to + re-map paths to match the local machine's. + """ + aliases = aliases or PathAliases() data_dir, local = os.path.split(self.filename) localdot = local + '.' for f in os.listdir(data_dir or '.'): if f.startswith(localdot): full_path = os.path.join(data_dir, f) new_lines, new_arcs = self._read_file(full_path) - for filename, file_data in new_lines.items(): + for filename, file_data in iitems(new_lines): + filename = aliases.map(filename) self.lines.setdefault(filename, {}).update(file_data) - for filename, file_data in new_arcs.items(): + for filename, file_data in iitems(new_arcs): + filename = aliases.map(filename) self.arcs.setdefault(filename, {}).update(file_data) if f != local: os.remove(full_path) @@ -195,7 +207,7 @@ class CoverageData(object): `line_data` is { filename: { lineno: None, ... }, ...} """ - for filename, linenos in line_data.items(): + for filename, linenos in iitems(line_data): self.lines.setdefault(filename, {}).update(linenos) def add_arc_data(self, arc_data): @@ -204,7 +216,7 @@ class CoverageData(object): `arc_data` is { filename: { (l1,l2): None, ... }, ...} """ - for filename, arcs in arc_data.items(): + for filename, arcs in iitems(arc_data): self.arcs.setdefault(filename, {}).update(arcs) def touch_file(self, filename): @@ -245,8 +257,8 @@ class CoverageData(object): if fullpath: filename_fn = lambda f: f else: - filename_fn = self.os.path.basename - for filename, lines in self.lines.items(): + filename_fn = os.path.basename + for filename, lines in iitems(self.lines): summ[filename_fn(filename)] = len(lines) return summ diff --git a/python/helpers/coverage/debug.py b/python/helpers/coverage/debug.py new file mode 100644 index 000000000000..104f3b1d0a43 --- /dev/null +++ b/python/helpers/coverage/debug.py @@ -0,0 +1,54 @@ +"""Control of and utilities for debugging.""" + +import os + + +# When debugging, it can be helpful to force some options, especially when +# debugging the configuration mechanisms you usually use to control debugging! +# This is a list of forced debugging options. +FORCED_DEBUG = [] + + +class DebugControl(object): + """Control and output for debugging.""" + + def __init__(self, options, output): + """Configure the options and output file for debugging.""" + self.options = options + self.output = output + + def should(self, option): + """Decide whether to output debug information in category `option`.""" + return (option in self.options or option in FORCED_DEBUG) + + def write(self, msg): + """Write a line of debug output.""" + if self.should('pid'): + msg = "pid %5d: %s" % (os.getpid(), msg) + self.output.write(msg+"\n") + self.output.flush() + + def write_formatted_info(self, info): + """Write a sequence of (label,data) pairs nicely.""" + for line in info_formatter(info): + self.write(" %s" % line) + + +def info_formatter(info): + """Produce a sequence of formatted lines from info. + + `info` is a sequence of pairs (label, data). The produced lines are + nicely formatted, ready to print. + + """ + label_len = max([len(l) for l, _d in info]) + for label, data in info: + if data == []: + data = "-none-" + if isinstance(data, (list, tuple)): + prefix = "%*s:" % (label_len, label) + for e in data: + yield "%*s %s" % (label_len+1, prefix, e) + prefix = "" + else: + yield "%*s: %s" % (label_len, label, data) diff --git a/python/helpers/coverage/execfile.py b/python/helpers/coverage/execfile.py index 71227b715a3e..f6ebdf79bb9e 100644 --- a/python/helpers/coverage/execfile.py +++ b/python/helpers/coverage/execfile.py @@ -1,9 +1,9 @@ """Execute files of Python code.""" -import imp, os, sys +import imp, marshal, os, sys from coverage.backward import exec_code_object, open_source -from coverage.misc import NoSource, ExceptionDuringRun +from coverage.misc import ExceptionDuringRun, NoCode, NoSource try: @@ -65,6 +65,8 @@ def run_python_module(modulename, args): openfile.close() # Finally, hand the file off to run_python_file for execution. + pathname = os.path.abspath(pathname) + args[0] = pathname run_python_file(pathname, args, package=packagename) @@ -82,34 +84,22 @@ def run_python_file(filename, args, package=None): main_mod = imp.new_module('__main__') sys.modules['__main__'] = main_mod main_mod.__file__ = filename - main_mod.__package__ = package + if package: + main_mod.__package__ = package main_mod.__builtins__ = BUILTINS - # Set sys.argv and the first path element properly. + # Set sys.argv properly. old_argv = sys.argv - old_path0 = sys.path[0] sys.argv = args - sys.path[0] = os.path.abspath(os.path.dirname(filename)) try: - # Open the source file. - try: - source_file = open_source(filename) - except IOError: - raise NoSource("No file to run: %r" % filename) + # Make a code object somehow. + if filename.endswith(".pyc") or filename.endswith(".pyo"): + code = make_code_from_pyc(filename) + else: + code = make_code_from_py(filename) - try: - source = source_file.read() - finally: - source_file.close() - - # We have the source. `compile` still needs the last line to be clean, - # so make sure it is, then compile a code object from it. - if source[-1] != '\n': - source += '\n' - code = compile(source, filename, "exec") - - # Execute the source file. + # Execute the code object. try: exec_code_object(code, main_mod.__dict__) except SystemExit: @@ -130,4 +120,52 @@ def run_python_file(filename, args, package=None): # Restore the old argv and path sys.argv = old_argv - sys.path[0] = old_path0 + +def make_code_from_py(filename): + """Get source from `filename` and make a code object of it.""" + # Open the source file. + try: + source_file = open_source(filename) + except IOError: + raise NoSource("No file to run: %r" % filename) + + try: + source = source_file.read() + finally: + source_file.close() + + # We have the source. `compile` still needs the last line to be clean, + # so make sure it is, then compile a code object from it. + if not source or source[-1] != '\n': + source += '\n' + code = compile(source, filename, "exec") + + return code + + +def make_code_from_pyc(filename): + """Get a code object from a .pyc file.""" + try: + fpyc = open(filename, "rb") + except IOError: + raise NoCode("No file to run: %r" % filename) + + try: + # First four bytes are a version-specific magic number. It has to + # match or we won't run the file. + magic = fpyc.read(4) + if magic != imp.get_magic(): + raise NoCode("Bad magic number in .pyc file") + + # Skip the junk in the header that we don't need. + fpyc.read(4) # Skip the moddate. + if sys.version_info >= (3, 3): + # 3.3 added another long to the header (size), skip it. + fpyc.read(4) + + # The rest of the file is the code object we want. + code = marshal.load(fpyc) + finally: + fpyc.close() + + return code diff --git a/python/helpers/coverage/files.py b/python/helpers/coverage/files.py index a68a0a7fef88..464535a81653 100644 --- a/python/helpers/coverage/files.py +++ b/python/helpers/coverage/files.py @@ -1,23 +1,21 @@ """File wrangling.""" from coverage.backward import to_string -import fnmatch, os, sys +from coverage.misc import CoverageException +import fnmatch, os, os.path, re, sys +import ntpath, posixpath class FileLocator(object): """Understand how filenames work.""" def __init__(self): # The absolute path to our current directory. - self.relative_dir = self.abs_file(os.curdir) + os.sep + self.relative_dir = os.path.normcase(abs_file(os.curdir) + os.sep) # Cache of results of calling the canonical_filename() method, to # avoid duplicating work. self.canonical_filename_cache = {} - def abs_file(self, filename): - """Return the absolute normalized form of `filename`.""" - return os.path.normcase(os.path.abspath(os.path.realpath(filename))) - def relative_filename(self, filename): """Return the relative form of `filename`. @@ -25,8 +23,9 @@ class FileLocator(object): `FileLocator` was constructed. """ - if filename.startswith(self.relative_dir): - filename = filename.replace(self.relative_dir, "") + fnorm = os.path.normcase(filename) + if fnorm.startswith(self.relative_dir): + filename = filename[len(self.relative_dir):] return filename def canonical_filename(self, filename): @@ -36,19 +35,15 @@ class FileLocator(object): """ if filename not in self.canonical_filename_cache: - f = filename - if os.path.isabs(f) and not os.path.exists(f): - if self.get_zip_data(f) is None: - f = os.path.basename(f) - if not os.path.isabs(f): + if not os.path.isabs(filename): for path in [os.curdir] + sys.path: if path is None: continue - g = os.path.join(path, f) - if os.path.exists(g): - f = g + f = os.path.join(path, filename) + if os.path.exists(f): + filename = f break - cf = self.abs_file(f) + cf = abs_file(filename) self.canonical_filename_cache[filename] = cf return self.canonical_filename_cache[filename] @@ -77,6 +72,78 @@ class FileLocator(object): return None +if sys.platform == 'win32': + + def actual_path(path): + """Get the actual path of `path`, including the correct case.""" + if path in actual_path.cache: + return actual_path.cache[path] + + head, tail = os.path.split(path) + if not tail: + actpath = head + elif not head: + actpath = tail + else: + head = actual_path(head) + if head in actual_path.list_cache: + files = actual_path.list_cache[head] + else: + try: + files = os.listdir(head) + except OSError: + files = [] + actual_path.list_cache[head] = files + normtail = os.path.normcase(tail) + for f in files: + if os.path.normcase(f) == normtail: + tail = f + break + actpath = os.path.join(head, tail) + actual_path.cache[path] = actpath + return actpath + + actual_path.cache = {} + actual_path.list_cache = {} + +else: + def actual_path(filename): + """The actual path for non-Windows platforms.""" + return filename + + +def abs_file(filename): + """Return the absolute normalized form of `filename`.""" + path = os.path.expandvars(os.path.expanduser(filename)) + path = os.path.abspath(os.path.realpath(path)) + path = actual_path(path) + return path + + +def isabs_anywhere(filename): + """Is `filename` an absolute path on any OS?""" + return ntpath.isabs(filename) or posixpath.isabs(filename) + + +def prep_patterns(patterns): + """Prepare the file patterns for use in a `FnmatchMatcher`. + + If a pattern starts with a wildcard, it is used as a pattern + as-is. If it does not start with a wildcard, then it is made + absolute with the current directory. + + If `patterns` is None, an empty list is returned. + + """ + prepped = [] + for p in patterns or []: + if p.startswith("*") or p.startswith("?"): + prepped.append(p) + else: + prepped.append(abs_file(p)) + return prepped + + class TreeMatcher(object): """A matcher for files in a tree.""" def __init__(self, directories): @@ -85,6 +152,10 @@ class TreeMatcher(object): def __repr__(self): return "<TreeMatcher %r>" % self.dirs + def info(self): + """A list of strings for displaying when dumping state.""" + return self.dirs + def add(self, directory): """Add another directory to the list we match for.""" self.dirs.append(directory) @@ -110,6 +181,10 @@ class FnmatchMatcher(object): def __repr__(self): return "<FnmatchMatcher %r>" % self.pats + def info(self): + """A list of strings for displaying when dumping state.""" + return self.pats + def match(self, fpath): """Does `fpath` match one of our filename patterns?""" for pat in self.pats: @@ -118,14 +193,117 @@ class FnmatchMatcher(object): return False +def sep(s): + """Find the path separator used in this string, or os.sep if none.""" + sep_match = re.search(r"[\\/]", s) + if sep_match: + the_sep = sep_match.group(0) + else: + the_sep = os.sep + return the_sep + + +class PathAliases(object): + """A collection of aliases for paths. + + When combining data files from remote machines, often the paths to source + code are different, for example, due to OS differences, or because of + serialized checkouts on continuous integration machines. + + A `PathAliases` object tracks a list of pattern/result pairs, and can + map a path through those aliases to produce a unified path. + + `locator` is a FileLocator that is used to canonicalize the results. + + """ + def __init__(self, locator=None): + self.aliases = [] + self.locator = locator + + def add(self, pattern, result): + """Add the `pattern`/`result` pair to the list of aliases. + + `pattern` is an `fnmatch`-style pattern. `result` is a simple + string. When mapping paths, if a path starts with a match against + `pattern`, then that match is replaced with `result`. This models + isomorphic source trees being rooted at different places on two + different machines. + + `pattern` can't end with a wildcard component, since that would + match an entire tree, and not just its root. + + """ + # The pattern can't end with a wildcard component. + pattern = pattern.rstrip(r"\/") + if pattern.endswith("*"): + raise CoverageException("Pattern must not end with wildcards.") + pattern_sep = sep(pattern) + + # The pattern is meant to match a filepath. Let's make it absolute + # unless it already is, or is meant to match any prefix. + if not pattern.startswith('*') and not isabs_anywhere(pattern): + pattern = abs_file(pattern) + pattern += pattern_sep + + # Make a regex from the pattern. fnmatch always adds a \Z or $ to + # match the whole string, which we don't want. + regex_pat = fnmatch.translate(pattern).replace(r'\Z(', '(') + if regex_pat.endswith("$"): + regex_pat = regex_pat[:-1] + # We want */a/b.py to match on Windows too, so change slash to match + # either separator. + regex_pat = regex_pat.replace(r"\/", r"[\\/]") + # We want case-insensitive matching, so add that flag. + regex = re.compile(r"(?i)" + regex_pat) + + # Normalize the result: it must end with a path separator. + result_sep = sep(result) + result = result.rstrip(r"\/") + result_sep + self.aliases.append((regex, result, pattern_sep, result_sep)) + + def map(self, path): + """Map `path` through the aliases. + + `path` is checked against all of the patterns. The first pattern to + match is used to replace the root of the path with the result root. + Only one pattern is ever used. If no patterns match, `path` is + returned unchanged. + + The separator style in the result is made to match that of the result + in the alias. + + """ + for regex, result, pattern_sep, result_sep in self.aliases: + m = regex.match(path) + if m: + new = path.replace(m.group(0), result) + if pattern_sep != result_sep: + new = new.replace(pattern_sep, result_sep) + if self.locator: + new = self.locator.canonical_filename(new) + return new + return path + + def find_python_files(dirname): - """Yield all of the importable Python files in `dirname`, recursively.""" - for dirpath, dirnames, filenames in os.walk(dirname, topdown=True): - if '__init__.py' not in filenames: + """Yield all of the importable Python files in `dirname`, recursively. + + To be importable, the files have to be in a directory with a __init__.py, + except for `dirname` itself, which isn't required to have one. The + assumption is that `dirname` was specified directly, so the user knows + best, but subdirectories are checked for a __init__.py to be sure we only + find the importable files. + + """ + for i, (dirpath, dirnames, filenames) in enumerate(os.walk(dirname)): + if i > 0 and '__init__.py' not in filenames: # If a directory doesn't have __init__.py, then it isn't # importable and neither are its files del dirnames[:] continue for filename in filenames: - if fnmatch.fnmatch(filename, "*.py"): + # We're only interested in files that look like reasonable Python + # files: Must end with .py or .pyw, and must not have certain funny + # characters that probably mean they are editor junk. + if re.match(r"^[^.#~!$@%^&*()+=,]+\.pyw?$", filename): yield os.path.join(dirpath, filename) diff --git a/python/helpers/coverage/fullcoverage/encodings.py b/python/helpers/coverage/fullcoverage/encodings.py new file mode 100644 index 000000000000..6a258d6710c4 --- /dev/null +++ b/python/helpers/coverage/fullcoverage/encodings.py @@ -0,0 +1,57 @@ +"""Imposter encodings module that installs a coverage-style tracer. + +This is NOT the encodings module; it is an imposter that sets up tracing +instrumentation and then replaces itself with the real encodings module. + +If the directory that holds this file is placed first in the PYTHONPATH when +using "coverage" to run Python's tests, then this file will become the very +first module imported by the internals of Python 3. It installs a +coverage-compatible trace function that can watch Standard Library modules +execute from the very earliest stages of Python's own boot process. This fixes +a problem with coverage - that it starts too late to trace the coverage of many +of the most fundamental modules in the Standard Library. + +""" + +import sys + +class FullCoverageTracer(object): + def __init__(self): + # `traces` is a list of trace events. Frames are tricky: the same + # frame object is used for a whole scope, with new line numbers + # written into it. So in one scope, all the frame objects are the + # same object, and will eventually all will point to the last line + # executed. So we keep the line numbers alongside the frames. + # The list looks like: + # + # traces = [ + # ((frame, event, arg), lineno), ... + # ] + # + self.traces = [] + + def fullcoverage_trace(self, *args): + frame, event, arg = args + self.traces.append((args, frame.f_lineno)) + return self.fullcoverage_trace + +sys.settrace(FullCoverageTracer().fullcoverage_trace) + +# In coverage/files.py is actual_filename(), which uses glob.glob. I don't +# understand why, but that use of glob borks everything if fullcoverage is in +# effect. So here we make an ugly hail-mary pass to switch off glob.glob over +# there. This means when using fullcoverage, Windows path names will not be +# their actual case. + +#sys.fullcoverage = True + +# Finally, remove our own directory from sys.path; remove ourselves from +# sys.modules; and re-import "encodings", which will be the real package +# this time. Note that the delete from sys.modules dictionary has to +# happen last, since all of the symbols in this module will become None +# at that exact moment, including "sys". + +parentdir = max(filter(__file__.startswith, sys.path), key=len) +sys.path.remove(parentdir) +del sys.modules['encodings'] +import encodings diff --git a/python/helpers/coverage/html.py b/python/helpers/coverage/html.py index fffd9b45b41a..5242236c1ed9 100644 --- a/python/helpers/coverage/html.py +++ b/python/helpers/coverage/html.py @@ -1,21 +1,45 @@ """HTML reporting for Coverage.""" -import os, re, shutil +import os, re, shutil, sys import coverage from coverage.backward import pickle from coverage.misc import CoverageException, Hasher -from coverage.phystokens import source_token_lines +from coverage.phystokens import source_token_lines, source_encoding from coverage.report import Reporter +from coverage.results import Numbers from coverage.templite import Templite -# Disable pylint msg W0612, because a bunch of variables look unused, but -# they're accessed in a Templite context via locals(). -# pylint: disable=W0612 -def data_filename(fname): - """Return the path to a data file of ours.""" - return os.path.join(os.path.split(__file__)[0], fname) +# Static files are looked for in a list of places. +STATIC_PATH = [ + # The place Debian puts system Javascript libraries. + "/usr/share/javascript", + + # Our htmlfiles directory. + os.path.join(os.path.dirname(__file__), "htmlfiles"), +] + +def data_filename(fname, pkgdir=""): + """Return the path to a data file of ours. + + The file is searched for on `STATIC_PATH`, and the first place it's found, + is returned. + + Each directory in `STATIC_PATH` is searched as-is, and also, if `pkgdir` + is provided, at that subdirectory. + + """ + for static_dir in STATIC_PATH: + static_filename = os.path.join(static_dir, fname) + if os.path.exists(static_filename): + return static_filename + if pkgdir: + static_filename = os.path.join(static_dir, pkgdir, fname) + if os.path.exists(static_filename): + return static_filename + raise CoverageException("Couldn't find static file %r" % fname) + def data(fname): """Return the contents of a data file of ours.""" @@ -31,26 +55,27 @@ class HtmlReporter(Reporter): # These files will be copied from the htmlfiles dir to the output dir. STATIC_FILES = [ - "style.css", - "jquery-1.4.3.min.js", - "jquery.hotkeys.js", - "jquery.isonscreen.js", - "jquery.tablesorter.min.js", - "coverage_html.js", - "keybd_closed.png", - "keybd_open.png", + ("style.css", ""), + ("jquery.min.js", "jquery"), + ("jquery.hotkeys.js", "jquery-hotkeys"), + ("jquery.isonscreen.js", "jquery-isonscreen"), + ("jquery.tablesorter.min.js", "jquery-tablesorter"), + ("coverage_html.js", ""), + ("keybd_closed.png", ""), + ("keybd_open.png", ""), ] - def __init__(self, cov, ignore_errors=False): - super(HtmlReporter, self).__init__(cov, ignore_errors) + def __init__(self, cov, config): + super(HtmlReporter, self).__init__(cov, config) self.directory = None self.template_globals = { 'escape': escape, + 'title': self.config.html_title, '__url__': coverage.__url__, '__version__': coverage.__version__, } self.source_tmpl = Templite( - data("htmlfiles/pyfile.html"), self.template_globals + data("pyfile.html"), self.template_globals ) self.coverage = cov @@ -58,29 +83,34 @@ class HtmlReporter(Reporter): self.files = [] self.arcs = self.coverage.data.has_arcs() self.status = HtmlStatus() + self.extra_css = None + self.totals = Numbers() - def report(self, morfs, config=None): + def report(self, morfs): """Generate an HTML report for `morfs`. - `morfs` is a list of modules or filenames. `config` is a - CoverageConfig instance. + `morfs` is a list of modules or filenames. """ - assert config.html_dir, "must provide a directory for html reporting" + assert self.config.html_dir, "must give a directory for html reporting" # Read the status data. - self.status.read(config.html_dir) + self.status.read(self.config.html_dir) # Check that this run used the same settings as the last run. m = Hasher() - m.update(config) + m.update(self.config) these_settings = m.digest() if self.status.settings_hash() != these_settings: self.status.reset() self.status.set_settings_hash(these_settings) + # The user may have extra CSS they want copied. + if self.config.extra_css: + self.extra_css = os.path.basename(self.config.extra_css) + # Process all the files. - self.report_files(self.html_file, morfs, config, config.html_dir) + self.report_files(self.html_file, morfs, self.config.html_dir) if not self.files: raise CoverageException("No data to report.") @@ -88,13 +118,34 @@ class HtmlReporter(Reporter): # Write the index file. self.index_file() - # Create the once-per-directory files. - for static in self.STATIC_FILES: + self.make_local_static_report_files() + + return self.totals.pc_covered + + def make_local_static_report_files(self): + """Make local instances of static files for HTML report.""" + # The files we provide must always be copied. + for static, pkgdir in self.STATIC_FILES: shutil.copyfile( - data_filename("htmlfiles/" + static), + data_filename(static, pkgdir), os.path.join(self.directory, static) ) + # The user may have extra CSS they want copied. + if self.extra_css: + shutil.copyfile( + self.config.extra_css, + os.path.join(self.directory, self.extra_css) + ) + + def write_html(self, fname, html): + """Write `html` to `fname`, properly encoded.""" + fout = open(fname, "wb") + try: + fout.write(html.encode('ascii', 'xmlcharrefreplace')) + finally: + fout.close() + def file_hash(self, source, cu): """Compute a hash that changes if the file needs to be re-reported.""" m = Hasher() @@ -121,11 +172,20 @@ class HtmlReporter(Reporter): self.status.set_file_hash(flat_rootname, this_hash) + # If need be, determine the encoding of the source file. We use it + # later to properly write the HTML. + if sys.version_info < (3, 0): + encoding = source_encoding(source) + # Some UTF8 files have the dreaded UTF8 BOM. If so, junk it. + if encoding.startswith("utf-8") and source[:3] == "\xef\xbb\xbf": + source = source[3:] + encoding = "utf-8" + + # Get the numbers for this file. nums = analysis.numbers - missing_branch_arcs = analysis.missing_branch_arcs() - n_par = 0 # accumulated below. - arcs = self.arcs + if self.arcs: + missing_branch_arcs = analysis.missing_branch_arcs() # These classes determine which lines are highlighted by default. c_run = "run hide_run" @@ -149,7 +209,6 @@ class HtmlReporter(Reporter): line_class.append(c_mis) elif self.arcs and lineno in missing_branch_arcs: line_class.append(c_par) - n_par += 1 annlines = [] for b in missing_branch_arcs[lineno]: if b < 0: @@ -184,19 +243,22 @@ class HtmlReporter(Reporter): }) # Write the HTML page for this file. + html = spaceless(self.source_tmpl.render({ + 'c_exc': c_exc, 'c_mis': c_mis, 'c_par': c_par, 'c_run': c_run, + 'arcs': self.arcs, 'extra_css': self.extra_css, + 'cu': cu, 'nums': nums, 'lines': lines, + })) + + if sys.version_info < (3, 0): + html = html.decode(encoding) + html_filename = flat_rootname + ".html" html_path = os.path.join(self.directory, html_filename) - html = spaceless(self.source_tmpl.render(locals())) - fhtml = open(html_path, 'w') - try: - fhtml.write(html) - finally: - fhtml.close() + self.write_html(html_path, html) # Save this file's information for the index file. index_info = { 'nums': nums, - 'par': n_par, 'html_filename': html_filename, 'name': cu.name, } @@ -206,19 +268,24 @@ class HtmlReporter(Reporter): def index_file(self): """Write the index.html file for this report.""" index_tmpl = Templite( - data("htmlfiles/index.html"), self.template_globals + data("index.html"), self.template_globals ) - files = self.files - arcs = self.arcs - - totals = sum([f['nums'] for f in files]) + self.totals = sum([f['nums'] for f in self.files]) - fhtml = open(os.path.join(self.directory, "index.html"), "w") - try: - fhtml.write(index_tmpl.render(locals())) - finally: - fhtml.close() + html = index_tmpl.render({ + 'arcs': self.arcs, + 'extra_css': self.extra_css, + 'files': self.files, + 'totals': self.totals, + }) + + if sys.version_info < (3, 0): + html = html.decode("utf-8") + self.write_html( + os.path.join(self.directory, "index.html"), + html + ) # Write the latest hashes for next time. self.status.write(self.directory) @@ -243,8 +310,12 @@ class HtmlStatus(object): usable = False try: status_file = os.path.join(directory, self.STATUS_FILE) - status = pickle.load(open(status_file, "rb")) - except IOError: + fstatus = open(status_file, "rb") + try: + status = pickle.load(fstatus) + finally: + fstatus.close() + except (IOError, ValueError): usable = False else: usable = True @@ -321,5 +392,5 @@ def spaceless(html): Get rid of some. """ - html = re.sub(">\s+<p ", ">\n<p ", html) + html = re.sub(r">\s+<p ", ">\n<p ", html) return html diff --git a/python/helpers/coverage/htmlfiles/coverage_html.js b/python/helpers/coverage/htmlfiles/coverage_html.js index da3e22c812c1..b24006d25e00 100644 --- a/python/helpers/coverage/htmlfiles/coverage_html.js +++ b/python/helpers/coverage/htmlfiles/coverage_html.js @@ -122,6 +122,11 @@ coverage.pyfile_ready = function ($) { .bind('keydown', '1', coverage.to_first_chunk) ; + $(".button_toggle_run").click(function (evt) {coverage.toggle_lines(evt.target, "run");}); + $(".button_toggle_exc").click(function (evt) {coverage.toggle_lines(evt.target, "exc");}); + $(".button_toggle_mis").click(function (evt) {coverage.toggle_lines(evt.target, "mis");}); + $(".button_toggle_par").click(function (evt) {coverage.toggle_lines(evt.target, "par");}); + coverage.assign_shortkeys(); coverage.wire_up_help_panel(); }; @@ -369,4 +374,3 @@ coverage.scroll_window = function (to_pos) { coverage.finish_scrolling = function () { $("html,body").stop(true, true); }; - diff --git a/python/helpers/coverage/htmlfiles/index.html b/python/helpers/coverage/htmlfiles/index.html index 04b314a3427c..c831823dd239 100644 --- a/python/helpers/coverage/htmlfiles/index.html +++ b/python/helpers/coverage/htmlfiles/index.html @@ -2,9 +2,12 @@ <html> <head> <meta http-equiv='Content-Type' content='text/html; charset=utf-8'> - <title>Coverage report</title> + <title>{{ title|escape }}</title> <link rel='stylesheet' href='style.css' type='text/css'> - <script type='text/javascript' src='jquery-1.4.3.min.js'></script> + {% if extra_css %} + <link rel='stylesheet' href='{{ extra_css }}' type='text/css'> + {% endif %} + <script type='text/javascript' src='jquery.min.js'></script> <script type='text/javascript' src='jquery.tablesorter.min.js'></script> <script type='text/javascript' src='jquery.hotkeys.js'></script> <script type='text/javascript' src='coverage_html.js'></script> @@ -16,7 +19,7 @@ <div id='header'> <div class='content'> - <h1>Coverage report: + <h1>{{ title|escape }}: <span class='pc_cov'>{{totals.pc_covered_str}}%</span> </h1> <img id='keyboard_icon' src='keybd_closed.png'> @@ -66,7 +69,7 @@ <td>{{totals.n_excluded}}</td> {% if arcs %} <td>{{totals.n_branches}}</td> - <td>{{totals.n_missing_branches}}</td> + <td>{{totals.n_partial_branches}}</td> {% endif %} <td class='right'>{{totals.pc_covered_str}}%</td> </tr> @@ -80,7 +83,7 @@ <td>{{file.nums.n_excluded}}</td> {% if arcs %} <td>{{file.nums.n_branches}}</td> - <td>{{file.nums.n_missing_branches}}</td> + <td>{{file.nums.n_partial_branches}}</td> {% endif %} <td class='right'>{{file.nums.pc_covered_str}}%</td> </tr> diff --git a/python/helpers/coverage/htmlfiles/jquery-1.4.3.min.js b/python/helpers/coverage/htmlfiles/jquery.min.js index c941a5f7a9f3..c941a5f7a9f3 100644 --- a/python/helpers/coverage/htmlfiles/jquery-1.4.3.min.js +++ b/python/helpers/coverage/htmlfiles/jquery.min.js diff --git a/python/helpers/coverage/htmlfiles/keybd_closed.png b/python/helpers/coverage/htmlfiles/keybd_closed.png Binary files differindex 6843abf0998b..f2b0418d2a33 100644..100755 --- a/python/helpers/coverage/htmlfiles/keybd_closed.png +++ b/python/helpers/coverage/htmlfiles/keybd_closed.png diff --git a/python/helpers/coverage/htmlfiles/keybd_open.png b/python/helpers/coverage/htmlfiles/keybd_open.png Binary files differindex 5a681ea70582..a77961db5424 100644..100755 --- a/python/helpers/coverage/htmlfiles/keybd_open.png +++ b/python/helpers/coverage/htmlfiles/keybd_open.png diff --git a/python/helpers/coverage/htmlfiles/pyfile.html b/python/helpers/coverage/htmlfiles/pyfile.html index ee0a3b1bbc8b..88c158dd41de 100644 --- a/python/helpers/coverage/htmlfiles/pyfile.html +++ b/python/helpers/coverage/htmlfiles/pyfile.html @@ -7,7 +7,10 @@ <meta http-equiv='X-UA-Compatible' content='IE=emulateIE7' /> <title>Coverage for {{cu.name|escape}}: {{nums.pc_covered_str}}%</title> <link rel='stylesheet' href='style.css' type='text/css'> - <script type='text/javascript' src='jquery-1.4.3.min.js'></script> + {% if extra_css %} + <link rel='stylesheet' href='{{ extra_css }}' type='text/css'> + {% endif %} + <script type='text/javascript' src='jquery.min.js'></script> <script type='text/javascript' src='jquery.hotkeys.js'></script> <script type='text/javascript' src='jquery.isonscreen.js'></script> <script type='text/javascript' src='coverage_html.js'></script> @@ -24,12 +27,12 @@ </h1> <img id='keyboard_icon' src='keybd_closed.png'> <h2 class='stats'> - {{nums.n_statements}} statements - <span class='{{c_run}} shortkey_r' onclick='coverage.toggle_lines(this, "run")'>{{nums.n_executed}} run</span> - <span class='{{c_mis}} shortkey_m' onclick='coverage.toggle_lines(this, "mis")'>{{nums.n_missing}} missing</span> - <span class='{{c_exc}} shortkey_x' onclick='coverage.toggle_lines(this, "exc")'>{{nums.n_excluded}} excluded</span> + {{nums.n_statements}} statements + <span class='{{c_run}} shortkey_r button_toggle_run'>{{nums.n_executed}} run</span> + <span class='{{c_mis}} shortkey_m button_toggle_mis'>{{nums.n_missing}} missing</span> + <span class='{{c_exc}} shortkey_x button_toggle_exc'>{{nums.n_excluded}} excluded</span> {% if arcs %} - <span class='{{c_par}} shortkey_p' onclick='coverage.toggle_lines(this, "par")'>{{n_par}} partial</span> + <span class='{{c_par}} shortkey_p button_toggle_par'>{{nums.n_partial_branches}} partial</span> {% endif %} </h2> </div> diff --git a/python/helpers/coverage/htmlfiles/style.css b/python/helpers/coverage/htmlfiles/style.css index c40357b8b482..811c64019c06 100644 --- a/python/helpers/coverage/htmlfiles/style.css +++ b/python/helpers/coverage/htmlfiles/style.css @@ -24,8 +24,8 @@ html>body { /* Set base font size to 12/16 */ p { - font-size: .75em; /* 12/16 */ - line-height: 1.3333em; /* 16/12 */ + font-size: .75em; /* 12/16 */ + line-height: 1.33333333em; /* 16/12 */ } table { @@ -102,6 +102,31 @@ h2.stats { border-color: #999 #ccc #ccc #999; } +.stats span.run { + background: #ddffdd; +} +.stats span.exc { + background: #eeeeee; +} +.stats span.mis { + background: #ffdddd; +} +.stats span.hide_run { + background: #eeffee; +} +.stats span.hide_exc { + background: #f5f5f5; +} +.stats span.hide_mis { + background: #ffeeee; +} +.stats span.par { + background: #ffffaa; +} +.stats span.hide_par { + background: #ffffcc; +} + /* Help panel */ #keyboard_icon { float: right; diff --git a/python/helpers/coverage/misc.py b/python/helpers/coverage/misc.py index fd9be8572a5b..0378173fcc3d 100644 --- a/python/helpers/coverage/misc.py +++ b/python/helpers/coverage/misc.py @@ -1,6 +1,10 @@ """Miscellaneous stuff for Coverage.""" +import errno import inspect +import os +import sys + from coverage.backward import md5, sorted # pylint: disable=W0622 from coverage.backward import string_class, to_bytes @@ -34,6 +38,8 @@ def format_lines(statements, lines): i = 0 j = 0 start = None + statements = sorted(statements) + lines = sorted(lines) while i < len(statements) and j < len(lines): if statements[i] == lines[j]: if start == None: @@ -50,6 +56,12 @@ def format_lines(statements, lines): return ret +def short_stack(): + """Return a string summarizing the call stack.""" + stack = inspect.stack()[:0:-1] + return "\n".join(["%30s : %s @%d" % (t[3],t[1],t[2]) for t in stack]) + + def expensive(fn): """A decorator to cache the result of an expensive operation. @@ -76,13 +88,23 @@ def bool_or_none(b): def join_regex(regexes): """Combine a list of regexes into one that matches any of them.""" if len(regexes) > 1: - return "(" + ")|(".join(regexes) + ")" + return "|".join(["(%s)" % r for r in regexes]) elif regexes: return regexes[0] else: return "" +def file_be_gone(path): + """Remove a file, and don't get annoyed if it doesn't exist.""" + try: + os.remove(path) + except OSError: + _, e, _ = sys.exc_info() + if e.errno != errno.ENOENT: + raise + + class Hasher(object): """Hashes Python data into md5.""" def __init__(self): @@ -93,8 +115,10 @@ class Hasher(object): self.md5.update(to_bytes(str(type(v)))) if isinstance(v, string_class): self.md5.update(to_bytes(v)) + elif v is None: + pass elif isinstance(v, (int, float)): - self.update(str(v)) + self.md5.update(to_bytes(str(v))) elif isinstance(v, (tuple, list)): for e in v: self.update(e) @@ -126,6 +150,10 @@ class NoSource(CoverageException): """We couldn't find the source for a module.""" pass +class NoCode(NoSource): + """We couldn't find any code at all.""" + pass + class NotPython(CoverageException): """A source file turned out not to be parsable Python.""" pass diff --git a/python/helpers/coverage/parser.py b/python/helpers/coverage/parser.py index cbbb5a6a2ec1..7a145a2a5346 100644 --- a/python/helpers/coverage/parser.py +++ b/python/helpers/coverage/parser.py @@ -1,9 +1,11 @@ """Code parsing for Coverage.""" -import glob, opcode, os, re, sys, token, tokenize +import dis, re, sys, token, tokenize from coverage.backward import set, sorted, StringIO # pylint: disable=W0622 -from coverage.backward import open_source +from coverage.backward import open_source, range # pylint: disable=W0622 +from coverage.backward import reversed # pylint: disable=W0622 +from coverage.backward import bytes_to_ints from coverage.bytecode import ByteCodes, CodeObjects from coverage.misc import nice_pair, expensive, join_regex from coverage.misc import CoverageException, NoSource, NotPython @@ -32,9 +34,13 @@ class CodeParser(object): except IOError: _, err, _ = sys.exc_info() raise NoSource( - "No source for code: %r: %s" % (self.filename, err) + "No source for code: '%s': %s" % (self.filename, err) ) + # Scrap the BOM if it exists. + if self.text and ord(self.text[0]) == 0xfeff: + self.text = self.text[1:] + self.exclude = exclude self.show_tokens = False @@ -102,9 +108,9 @@ class CodeParser(object): first_line = None empty = True - tokgen = tokenize.generate_tokens(StringIO(self.text).readline) + tokgen = generate_tokens(self.text) for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen: - if self.show_tokens: # pragma: no cover + if self.show_tokens: # pragma: not covered print("%10s %5s %-20r %r" % ( tokenize.tok_name.get(toktype, toktype), nice_pair((slineno, elineno)), ttext, ltext @@ -130,8 +136,7 @@ class CodeParser(object): # (a trick from trace.py in the stdlib.) This works for # 99.9999% of cases. For the rest (!) see: # http://stackoverflow.com/questions/1769332/x/1769794#1769794 - for i in range(slineno, elineno+1): - self.docstrings.add(i) + self.docstrings.update(range(slineno, elineno+1)) elif toktype == token.NEWLINE: if first_line is not None and elineno != first_line: # We're at the end of a line, and we've ended on a @@ -170,16 +175,18 @@ class CodeParser(object): first_line = line return first_line - def first_lines(self, lines, ignore=None): + def first_lines(self, lines, *ignores): """Map the line numbers in `lines` to the correct first line of the statement. - Skip any line mentioned in `ignore`. + Skip any line mentioned in any of the sequences in `ignores`. - Returns a sorted list of the first lines. + Returns a set of the first lines. """ - ignore = ignore or [] + ignore = set() + for ign in ignores: + ignore.update(ign) lset = set() for l in lines: if l in ignore: @@ -187,23 +194,34 @@ class CodeParser(object): new_l = self.first_line(l) if new_l not in ignore: lset.add(new_l) - return sorted(lset) + return lset def parse_source(self): """Parse source text to find executable lines, excluded lines, etc. - Return values are 1) a sorted list of executable line numbers, and - 2) a sorted list of excluded line numbers. + Return values are 1) a set of executable line numbers, and 2) a set of + excluded line numbers. Reported line numbers are normalized to the first line of multi-line statements. """ - self._raw_parse() + try: + self._raw_parse() + except (tokenize.TokenError, IndentationError): + _, tokerr, _ = sys.exc_info() + msg, lineno = tokerr.args + raise NotPython( + "Couldn't parse '%s' as Python source: '%s' at %s" % + (self.filename, msg, lineno) + ) excluded_lines = self.first_lines(self.excluded) - ignore = excluded_lines + list(self.docstrings) - lines = self.first_lines(self.statement_starts, ignore) + lines = self.first_lines( + self.statement_starts, + excluded_lines, + self.docstrings + ) return lines, excluded_lines @@ -258,8 +276,8 @@ class CodeParser(object): ## Opcodes that guide the ByteParser. def _opcode(name): - """Return the opcode by name from the opcode module.""" - return opcode.opmap[name] + """Return the opcode by name from the dis module.""" + return dis.opmap[name] def _opcode_set(*names): """Return a set of opcodes by the names in `names`.""" @@ -297,7 +315,7 @@ OPS_EXCEPT_BLOCKS = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY') OPS_POP_BLOCK = _opcode_set('POP_BLOCK') # Opcodes that have a jump destination, but aren't really a jump. -OPS_NO_JUMP = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY') +OPS_NO_JUMP = OPS_PUSH_BLOCK # Individual opcodes we need below. OP_BREAK_LOOP = _opcode('BREAK_LOOP') @@ -314,6 +332,7 @@ class ByteParser(object): def __init__(self, code=None, text=None, filename=None): if code: self.code = code + self.text = text else: if not text: assert filename, "If no code or text, need a filename" @@ -322,6 +341,7 @@ class ByteParser(object): text = sourcef.read() finally: sourcef.close() + self.text = text try: # Python 2.3 and 2.4 don't like partial last lines, so be sure @@ -350,69 +370,54 @@ class ByteParser(object): The iteration includes `self` as its first value. """ - return map(lambda c: ByteParser(code=c), CodeObjects(self.code)) - - # Getting numbers from the lnotab value changed in Py3.0. - if sys.version_info >= (3, 0): - def _lnotab_increments(self, lnotab): - """Return a list of ints from the lnotab bytes in 3.x""" - return list(lnotab) - else: - def _lnotab_increments(self, lnotab): - """Return a list of ints from the lnotab string in 2.x""" - return [ord(c) for c in lnotab] + children = CodeObjects(self.code) + return [ByteParser(code=c, text=self.text) for c in children] def _bytes_lines(self): """Map byte offsets to line numbers in `code`. Uses co_lnotab described in Python/compile.c to map byte offsets to - line numbers. Returns a list: [(b0, l0), (b1, l1), ...] + line numbers. Produces a sequence: (b0, l0), (b1, l1), ... + + Only byte offsets that correspond to line numbers are included in the + results. """ # Adapted from dis.py in the standard library. - byte_increments = self._lnotab_increments(self.code.co_lnotab[0::2]) - line_increments = self._lnotab_increments(self.code.co_lnotab[1::2]) + byte_increments = bytes_to_ints(self.code.co_lnotab[0::2]) + line_increments = bytes_to_ints(self.code.co_lnotab[1::2]) - bytes_lines = [] last_line_num = None line_num = self.code.co_firstlineno byte_num = 0 for byte_incr, line_incr in zip(byte_increments, line_increments): if byte_incr: if line_num != last_line_num: - bytes_lines.append((byte_num, line_num)) + yield (byte_num, line_num) last_line_num = line_num byte_num += byte_incr line_num += line_incr if line_num != last_line_num: - bytes_lines.append((byte_num, line_num)) - return bytes_lines + yield (byte_num, line_num) def _find_statements(self): """Find the statements in `self.code`. - Return a set of line numbers that start statements. Recurses into all - code objects reachable from `self.code`. + Produce a sequence of line numbers that start statements. Recurses + into all code objects reachable from `self.code`. """ - stmts = set() for bp in self.child_parsers(): # Get all of the lineno information from this code. for _, l in bp._bytes_lines(): - stmts.add(l) - return stmts - - def _disassemble(self): # pragma: no cover - """Disassemble code, for ad-hoc experimenting.""" - - import dis - - for bp in self.child_parsers(): - print("\n%s: " % bp.code) - dis.dis(bp.code) - print("Bytes lines: %r" % bp._bytes_lines()) + yield l - print("") + def _block_stack_repr(self, block_stack): + """Get a string version of `block_stack`, for debugging.""" + blocks = ", ".join( + ["(%s, %r)" % (dis.opname[b[0]], b[1]) for b in block_stack] + ) + return "[" + blocks + "]" def _split_into_chunks(self): """Split the code object into a list of `Chunk` objects. @@ -423,10 +428,11 @@ class ByteParser(object): Returns a list of `Chunk` objects. """ - # The list of chunks so far, and the one we're working on. chunks = [] chunk = None + + # A dict mapping byte offsets of line starts to the line numbers. bytes_lines_map = dict(self._bytes_lines()) # The block stack: loops and try blocks get pushed here for the @@ -441,24 +447,38 @@ class ByteParser(object): # We have to handle the last two bytecodes specially. ult = penult = None - for bc in ByteCodes(self.code.co_code): + # Get a set of all of the jump-to points. + jump_to = set() + bytecodes = list(ByteCodes(self.code.co_code)) + for bc in bytecodes: + if bc.jump_to >= 0: + jump_to.add(bc.jump_to) + + chunk_lineno = 0 + + # Walk the byte codes building chunks. + for bc in bytecodes: # Maybe have to start a new chunk + start_new_chunk = False + first_chunk = False if bc.offset in bytes_lines_map: # Start a new chunk for each source line number. - if chunk: - chunk.exits.add(bc.offset) - chunk = Chunk(bc.offset, bytes_lines_map[bc.offset]) - chunks.append(chunk) + start_new_chunk = True + chunk_lineno = bytes_lines_map[bc.offset] + first_chunk = True + elif bc.offset in jump_to: + # To make chunks have a single entrance, we have to make a new + # chunk when we get to a place some bytecode jumps to. + start_new_chunk = True elif bc.op in OPS_CHUNK_BEGIN: # Jumps deserve their own unnumbered chunk. This fixes # problems with jumps to jumps getting confused. + start_new_chunk = True + + if not chunk or start_new_chunk: if chunk: chunk.exits.add(bc.offset) - chunk = Chunk(bc.offset) - chunks.append(chunk) - - if not chunk: - chunk = Chunk(bc.offset) + chunk = Chunk(bc.offset, chunk_lineno, first_chunk) chunks.append(chunk) # Look at the opcode @@ -487,15 +507,11 @@ class ByteParser(object): chunk.exits.add(block_stack[-1][1]) chunk = None if bc.op == OP_END_FINALLY: - if block_stack: - # A break that goes through a finally will jump to whatever - # block is on top of the stack. - chunk.exits.add(block_stack[-1][1]) # For the finally clause we need to find the closest exception # block, and use its jump target as an exit. - for iblock in range(len(block_stack)-1, -1, -1): - if block_stack[iblock][0] in OPS_EXCEPT_BLOCKS: - chunk.exits.add(block_stack[iblock][1]) + for block in reversed(block_stack): + if block[0] in OPS_EXCEPT_BLOCKS: + chunk.exits.add(block[1]) break if bc.op == OP_COMPARE_OP and bc.arg == COMPARE_EXCEPTION: # This is an except clause. We want to overlook the next @@ -521,23 +537,33 @@ class ByteParser(object): last_chunk = chunks[-1] last_chunk.exits.remove(ex) last_chunk.exits.add(penult.offset) - chunk = Chunk(penult.offset) + chunk = Chunk( + penult.offset, last_chunk.line, False + ) chunk.exits.add(ex) chunks.append(chunk) # Give all the chunks a length. - chunks[-1].length = bc.next_offset - chunks[-1].byte + chunks[-1].length = bc.next_offset - chunks[-1].byte # pylint: disable=W0631,C0301 for i in range(len(chunks)-1): chunks[i].length = chunks[i+1].byte - chunks[i].byte + #self.validate_chunks(chunks) return chunks + def validate_chunks(self, chunks): + """Validate the rule that chunks have a single entrance.""" + # starts is the entrances to the chunks + starts = set([ch.byte for ch in chunks]) + for ch in chunks: + assert all([(ex in starts or ex < 0) for ex in ch.exits]) + def _arcs(self): """Find the executable arcs in the code. - Returns a set of pairs, (from,to). From and to are integer line - numbers. If from is < 0, then the arc is an entrance into the code - object. If to is < 0, the arc is an exit from the code object. + Yields pairs: (from,to). From and to are integer line numbers. If + from is < 0, then the arc is an entrance into the code object. If to + is < 0, the arc is an exit from the code object. """ chunks = self._split_into_chunks() @@ -545,65 +571,43 @@ class ByteParser(object): # A map from byte offsets to chunks jumped into. byte_chunks = dict([(c.byte, c) for c in chunks]) - # Build a map from byte offsets to actual lines reached. - byte_lines = {} - bytes_to_add = set([c.byte for c in chunks]) + # There's always an entrance at the first chunk. + yield (-1, byte_chunks[0].line) - while bytes_to_add: - byte_to_add = bytes_to_add.pop() - if byte_to_add in byte_lines or byte_to_add < 0: + # Traverse from the first chunk in each line, and yield arcs where + # the trace function will be invoked. + for chunk in chunks: + if not chunk.first: continue - # Which lines does this chunk lead to? - bytes_considered = set() - bytes_to_consider = [byte_to_add] - lines = set() - - while bytes_to_consider: - byte = bytes_to_consider.pop() - bytes_considered.add(byte) - - # Find chunk for byte - try: - ch = byte_chunks[byte] - except KeyError: - for ch in chunks: - if ch.byte <= byte < ch.byte+ch.length: - break - else: - # No chunk for this byte! - raise Exception("Couldn't find chunk @ %d" % byte) - byte_chunks[byte] = ch - - if ch.line: - lines.add(ch.line) - else: - for ex in ch.exits: - if ex < 0: - lines.add(ex) - elif ex not in bytes_considered: - bytes_to_consider.append(ex) - - bytes_to_add.update(ch.exits) - - byte_lines[byte_to_add] = lines - - # Figure out for each chunk where the exits go. - arcs = set() - for chunk in chunks: - if chunk.line: - for ex in chunk.exits: + chunks_considered = set() + chunks_to_consider = [chunk] + while chunks_to_consider: + # Get the chunk we're considering, and make sure we don't + # consider it again + this_chunk = chunks_to_consider.pop() + chunks_considered.add(this_chunk) + + # For each exit, add the line number if the trace function + # would be triggered, or add the chunk to those being + # considered if not. + for ex in this_chunk.exits: if ex < 0: - exit_lines = [ex] + yield (chunk.line, ex) else: - exit_lines = byte_lines[ex] - for exit_line in exit_lines: - if chunk.line != exit_line: - arcs.add((chunk.line, exit_line)) - for line in byte_lines[0]: - arcs.add((-1, line)) - - return arcs + next_chunk = byte_chunks[ex] + if next_chunk in chunks_considered: + continue + + # The trace function is invoked if visiting the first + # bytecode in a line, or if the transition is a + # backward jump. + backward_jump = next_chunk.byte < this_chunk.byte + if next_chunk.first or backward_jump: + if next_chunk.line != chunk.line: + yield (chunk.line, next_chunk.line) + else: + chunks_to_consider.append(next_chunk) def _all_chunks(self): """Returns a list of `Chunk` objects for this code and its children. @@ -631,11 +635,11 @@ class ByteParser(object): class Chunk(object): - """A sequence of bytecodes with a single entrance. + """A sequence of byte codes with a single entrance. To analyze byte code, we have to divide it into chunks, sequences of byte - codes such that each basic block has only one entrance, the first - instruction in the block. + codes such that each chunk has only one entrance, the first instruction in + the block. This is almost the CS concept of `basic block`_, except that we're willing to have many exits from a chunk, and "basic block" is a more cumbersome @@ -643,158 +647,54 @@ class Chunk(object): .. _basic block: http://en.wikipedia.org/wiki/Basic_block + `line` is the source line number containing this chunk. + + `first` is true if this is the first chunk in the source line. + An exit < 0 means the chunk can leave the code (return). The exit is the negative of the starting line number of the code block. """ - def __init__(self, byte, line=0): + def __init__(self, byte, line, first): self.byte = byte self.line = line + self.first = first self.length = 0 self.exits = set() def __repr__(self): - return "<%d+%d @%d %r>" % ( - self.byte, self.length, self.line, list(self.exits) + if self.first: + bang = "!" + else: + bang = "" + return "<%d+%d @%d%s %r>" % ( + self.byte, self.length, self.line, bang, list(self.exits) ) -class AdHocMain(object): # pragma: no cover - """An ad-hoc main for code parsing experiments.""" +class CachedTokenizer(object): + """A one-element cache around tokenize.generate_tokens. - def main(self, args): - """A main function for trying the code from the command line.""" + When reporting, coverage.py tokenizes files twice, once to find the + structure of the file, and once to syntax-color it. Tokenizing is + expensive, and easily cached. - from optparse import OptionParser + This is a one-element cache so that our twice-in-a-row tokenizing doesn't + actually tokenize twice. - parser = OptionParser() - parser.add_option( - "-c", action="store_true", dest="chunks", - help="Show basic block chunks" - ) - parser.add_option( - "-d", action="store_true", dest="dis", - help="Disassemble" - ) - parser.add_option( - "-R", action="store_true", dest="recursive", - help="Recurse to find source files" - ) - parser.add_option( - "-s", action="store_true", dest="source", - help="Show analyzed source" - ) - parser.add_option( - "-t", action="store_true", dest="tokens", - help="Show tokens" + """ + def __init__(self): + self.last_text = None + self.last_tokens = None + + def generate_tokens(self, text): + """A stand-in for `tokenize.generate_tokens`.""" + if text != self.last_text: + self.last_text = text + self.last_tokens = list( + tokenize.generate_tokens(StringIO(text).readline) ) + return self.last_tokens - options, args = parser.parse_args() - if options.recursive: - if args: - root = args[0] - else: - root = "." - for root, _, _ in os.walk(root): - for f in glob.glob(root + "/*.py"): - self.adhoc_one_file(options, f) - else: - self.adhoc_one_file(options, args[0]) - - def adhoc_one_file(self, options, filename): - """Process just one file.""" - - if options.dis or options.chunks: - try: - bp = ByteParser(filename=filename) - except CoverageException: - _, err, _ = sys.exc_info() - print("%s" % (err,)) - return - - if options.dis: - print("Main code:") - bp._disassemble() - - if options.chunks: - chunks = bp._all_chunks() - if options.recursive: - print("%6d: %s" % (len(chunks), filename)) - else: - print("Chunks: %r" % chunks) - arcs = bp._all_arcs() - print("Arcs: %r" % sorted(arcs)) - - if options.source or options.tokens: - cp = CodeParser(filename=filename, exclude=r"no\s*cover") - cp.show_tokens = options.tokens - cp._raw_parse() - - if options.source: - if options.chunks: - arc_width, arc_chars = self.arc_ascii_art(arcs) - else: - arc_width, arc_chars = 0, {} - - exit_counts = cp.exit_counts() - - for i, ltext in enumerate(cp.lines): - lineno = i+1 - m0 = m1 = m2 = m3 = a = ' ' - if lineno in cp.statement_starts: - m0 = '-' - exits = exit_counts.get(lineno, 0) - if exits > 1: - m1 = str(exits) - if lineno in cp.docstrings: - m2 = '"' - if lineno in cp.classdefs: - m2 = 'C' - if lineno in cp.excluded: - m3 = 'x' - a = arc_chars.get(lineno, '').ljust(arc_width) - print("%4d %s%s%s%s%s %s" % - (lineno, m0, m1, m2, m3, a, ltext) - ) - - def arc_ascii_art(self, arcs): - """Draw arcs as ascii art. - - Returns a width of characters needed to draw all the arcs, and a - dictionary mapping line numbers to ascii strings to draw for that line. - - """ - arc_chars = {} - for lfrom, lto in sorted(arcs): - if lfrom < 0: - arc_chars[lto] = arc_chars.get(lto, '') + 'v' - elif lto < 0: - arc_chars[lfrom] = arc_chars.get(lfrom, '') + '^' - else: - if lfrom == lto - 1: - # Don't show obvious arcs. - continue - if lfrom < lto: - l1, l2 = lfrom, lto - else: - l1, l2 = lto, lfrom - w = max([len(arc_chars.get(l, '')) for l in range(l1, l2+1)]) - for l in range(l1, l2+1): - if l == lfrom: - ch = '<' - elif l == lto: - ch = '>' - else: - ch = '|' - arc_chars[l] = arc_chars.get(l, '').ljust(w) + ch - arc_width = 0 - - if arc_chars: - arc_width = max([len(a) for a in arc_chars.values()]) - else: - arc_width = 0 - - return arc_width, arc_chars - -if __name__ == '__main__': - AdHocMain().main(sys.argv[1:]) +# Create our generate_tokens cache as a callable replacement function. +generate_tokens = CachedTokenizer().generate_tokens diff --git a/python/helpers/coverage/phystokens.py b/python/helpers/coverage/phystokens.py index fc4f2c9057b1..99b1d5ba0c79 100644 --- a/python/helpers/coverage/phystokens.py +++ b/python/helpers/coverage/phystokens.py @@ -1,7 +1,9 @@ """Better tokenizing for coverage.py.""" -import keyword, re, token, tokenize -from coverage.backward import StringIO # pylint: disable=W0622 +import codecs, keyword, re, sys, token, tokenize +from coverage.backward import set # pylint: disable=W0622 +from coverage.parser import generate_tokens + def phys_tokens(toks): """Return all physical tokens, even line continuations. @@ -18,7 +20,7 @@ def phys_tokens(toks): last_ttype = None for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks: if last_lineno != elineno: - if last_line and last_line[-2:] == "\\\n": + if last_line and last_line.endswith("\\\n"): # We are at the beginning of a new line, and the last line # ended with a backslash. We probably have to inject a # backslash token into the stream. Unfortunately, there's more @@ -74,11 +76,11 @@ def source_token_lines(source): is indistinguishable from a final line with a newline. """ - ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL] + ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL]) line = [] col = 0 source = source.expandtabs(8).replace('\r\n', '\n') - tokgen = tokenize.generate_tokens(StringIO(source).readline) + tokgen = generate_tokens(source) for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen): mark_start = True for part in re.split('(\n)', ttext): @@ -106,3 +108,103 @@ def source_token_lines(source): if line: yield line + +def source_encoding(source): + """Determine the encoding for `source` (a string), according to PEP 263. + + Returns a string, the name of the encoding. + + """ + # Note: this function should never be called on Python 3, since py3 has + # built-in tools to do this. + assert sys.version_info < (3, 0) + + # This is mostly code adapted from Py3.2's tokenize module. + + cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)") + + # Do this so the detect_encode code we copied will work. + readline = iter(source.splitlines(True)).next + + def _get_normal_name(orig_enc): + """Imitates get_normal_name in tokenizer.c.""" + # Only care about the first 12 characters. + enc = orig_enc[:12].lower().replace("_", "-") + if re.match(r"^utf-8($|-)", enc): + return "utf-8" + if re.match(r"^(latin-1|iso-8859-1|iso-latin-1)($|-)", enc): + return "iso-8859-1" + return orig_enc + + # From detect_encode(): + # It detects the encoding from the presence of a utf-8 bom or an encoding + # cookie as specified in pep-0263. If both a bom and a cookie are present, + # but disagree, a SyntaxError will be raised. If the encoding cookie is an + # invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, + # 'utf-8-sig' is returned. + + # If no encoding is specified, then the default will be returned. The + # default varied with version. + + if sys.version_info <= (2, 4): + default = 'iso-8859-1' + else: + default = 'ascii' + + bom_found = False + encoding = None + + def read_or_stop(): + """Get the next source line, or ''.""" + try: + return readline() + except StopIteration: + return '' + + def find_cookie(line): + """Find an encoding cookie in `line`.""" + try: + line_string = line.decode('ascii') + except UnicodeDecodeError: + return None + + matches = cookie_re.findall(line_string) + if not matches: + return None + encoding = _get_normal_name(matches[0]) + try: + codec = codecs.lookup(encoding) + except LookupError: + # This behaviour mimics the Python interpreter + raise SyntaxError("unknown encoding: " + encoding) + + if bom_found: + # codecs in 2.3 were raw tuples of functions, assume the best. + codec_name = getattr(codec, 'name', encoding) + if codec_name != 'utf-8': + # This behaviour mimics the Python interpreter + raise SyntaxError('encoding problem: utf-8') + encoding += '-sig' + return encoding + + first = read_or_stop() + if first.startswith(codecs.BOM_UTF8): + bom_found = True + first = first[3:] + default = 'utf-8-sig' + if not first: + return default + + encoding = find_cookie(first) + if encoding: + return encoding + + second = read_or_stop() + if not second: + return default + + encoding = find_cookie(second) + if encoding: + return encoding + + return default diff --git a/python/helpers/coverage/report.py b/python/helpers/coverage/report.py index 6c5510ad4677..34f44422298e 100644 --- a/python/helpers/coverage/report.py +++ b/python/helpers/coverage/report.py @@ -2,20 +2,21 @@ import fnmatch, os from coverage.codeunit import code_unit_factory +from coverage.files import prep_patterns from coverage.misc import CoverageException, NoSource, NotPython class Reporter(object): """A base class for all reporters.""" - def __init__(self, coverage, ignore_errors=False): + def __init__(self, coverage, config): """Create a reporter. - `coverage` is the coverage instance. `ignore_errors` controls how - skittish the reporter will be during file processing. + `coverage` is the coverage instance. `config` is an instance of + CoverageConfig, for controlling all sorts of behavior. """ self.coverage = coverage - self.ignore_errors = ignore_errors + self.config = config # The code units to report on. Set by find_code_units. self.code_units = [] @@ -24,19 +25,18 @@ class Reporter(object): # classes. self.directory = None - def find_code_units(self, morfs, config): + def find_code_units(self, morfs): """Find the code units we'll report on. - `morfs` is a list of modules or filenames. `config` is a - CoverageConfig instance. + `morfs` is a list of modules or filenames. """ morfs = morfs or self.coverage.data.measured_files() file_locator = self.coverage.file_locator self.code_units = code_unit_factory(morfs, file_locator) - if config.include: - patterns = [file_locator.abs_file(p) for p in config.include] + if self.config.include: + patterns = prep_patterns(self.config.include) filtered = [] for cu in self.code_units: for pattern in patterns: @@ -45,8 +45,8 @@ class Reporter(object): break self.code_units = filtered - if config.omit: - patterns = [file_locator.abs_file(p) for p in config.omit] + if self.config.omit: + patterns = prep_patterns(self.config.omit) filtered = [] for cu in self.code_units: for pattern in patterns: @@ -58,7 +58,7 @@ class Reporter(object): self.code_units.sort() - def report_files(self, report_fn, morfs, config, directory=None): + def report_files(self, report_fn, morfs, directory=None): """Run a reporting function on a number of morfs. `report_fn` is called for each relative morf in `morfs`. It is called @@ -69,10 +69,8 @@ class Reporter(object): where `code_unit` is the `CodeUnit` for the morf, and `analysis` is the `Analysis` for the morf. - `config` is a CoverageConfig instance. - """ - self.find_code_units(morfs, config) + self.find_code_units(morfs) if not self.code_units: raise CoverageException("No data to report.") @@ -84,6 +82,11 @@ class Reporter(object): for cu in self.code_units: try: report_fn(cu, self.coverage._analyze(cu)) - except (NoSource, NotPython): - if not self.ignore_errors: + except NoSource: + if not self.config.ignore_errors: + raise + except NotPython: + # Only report errors for .py files, and only if we didn't + # explicitly suppress those errors. + if cu.should_be_python() and not self.config.ignore_errors: raise diff --git a/python/helpers/coverage/results.py b/python/helpers/coverage/results.py index adfb8f42de5a..db6df0d30b7e 100644 --- a/python/helpers/coverage/results.py +++ b/python/helpers/coverage/results.py @@ -2,7 +2,7 @@ import os -from coverage.backward import set, sorted # pylint: disable=W0622 +from coverage.backward import iitems, set, sorted # pylint: disable=W0622 from coverage.misc import format_lines, join_regex, NoSource from coverage.parser import CodeParser @@ -15,16 +15,10 @@ class Analysis(object): self.code_unit = code_unit self.filename = self.code_unit.filename - ext = os.path.splitext(self.filename)[1] - source = None - if ext == '.py': - if not os.path.exists(self.filename): - source = self.coverage.file_locator.get_zip_data(self.filename) - if not source: - raise NoSource("No source for code: %r" % self.filename) + actual_filename, source = self.find_source(self.filename) self.parser = CodeParser( - text=source, filename=self.filename, + text=source, filename=actual_filename, exclude=self.coverage._exclude_regex('exclude') ) self.statements, self.excluded = self.parser.parse_source() @@ -32,7 +26,7 @@ class Analysis(object): # Identify missing statements. executed = self.coverage.data.executed_lines(self.filename) exec1 = self.parser.first_lines(executed) - self.missing = sorted(set(self.statements) - set(exec1)) + self.missing = self.statements - exec1 if self.coverage.data.has_arcs(): self.no_branch = self.parser.lines_matching( @@ -41,9 +35,12 @@ class Analysis(object): ) n_branches = self.total_branches() mba = self.missing_branch_arcs() - n_missing_branches = sum([len(v) for v in mba.values()]) + n_partial_branches = sum( + [len(v) for k,v in iitems(mba) if k not in self.missing] + ) + n_missing_branches = sum([len(v) for k,v in iitems(mba)]) else: - n_branches = n_missing_branches = 0 + n_branches = n_partial_branches = n_missing_branches = 0 self.no_branch = set() self.numbers = Numbers( @@ -52,9 +49,48 @@ class Analysis(object): n_excluded=len(self.excluded), n_missing=len(self.missing), n_branches=n_branches, + n_partial_branches=n_partial_branches, n_missing_branches=n_missing_branches, ) + def find_source(self, filename): + """Find the source for `filename`. + + Returns two values: the actual filename, and the source. + + The source returned depends on which of these cases holds: + + * The filename seems to be a non-source file: returns None + + * The filename is a source file, and actually exists: returns None. + + * The filename is a source file, and is in a zip file or egg: + returns the source. + + * The filename is a source file, but couldn't be found: raises + `NoSource`. + + """ + source = None + + base, ext = os.path.splitext(filename) + TRY_EXTS = { + '.py': ['.py', '.pyw'], + '.pyw': ['.pyw'], + } + try_exts = TRY_EXTS.get(ext) + if not try_exts: + return filename, None + + for try_ext in try_exts: + try_filename = base + try_ext + if os.path.exists(try_filename): + return try_filename, None + source = self.coverage.file_locator.get_zip_data(try_filename) + if source: + return try_filename, source + raise NoSource("No source for code: '%s'" % filename) + def missing_formatted(self): """The missing line numbers, formatted nicely. @@ -107,7 +143,7 @@ class Analysis(object): def branch_lines(self): """Returns a list of line numbers that have more than one exit.""" exit_counts = self.parser.exit_counts() - return [l1 for l1,count in exit_counts.items() if count > 1] + return [l1 for l1,count in iitems(exit_counts) if count > 1] def total_branches(self): """How many total branches are there?""" @@ -164,13 +200,14 @@ class Numbers(object): _near100 = 99.0 def __init__(self, n_files=0, n_statements=0, n_excluded=0, n_missing=0, - n_branches=0, n_missing_branches=0 + n_branches=0, n_partial_branches=0, n_missing_branches=0 ): self.n_files = n_files self.n_statements = n_statements self.n_excluded = n_excluded self.n_missing = n_missing self.n_branches = n_branches + self.n_partial_branches = n_partial_branches self.n_missing_branches = n_missing_branches def set_precision(cls, precision): @@ -234,8 +271,12 @@ class Numbers(object): nums.n_excluded = self.n_excluded + other.n_excluded nums.n_missing = self.n_missing + other.n_missing nums.n_branches = self.n_branches + other.n_branches - nums.n_missing_branches = (self.n_missing_branches + - other.n_missing_branches) + nums.n_partial_branches = ( + self.n_partial_branches + other.n_partial_branches + ) + nums.n_missing_branches = ( + self.n_missing_branches + other.n_missing_branches + ) return nums def __radd__(self, other): diff --git a/python/helpers/coverage/summary.py b/python/helpers/coverage/summary.py index 599ae78221ff..c99c53034aac 100644 --- a/python/helpers/coverage/summary.py +++ b/python/helpers/coverage/summary.py @@ -4,24 +4,23 @@ import sys from coverage.report import Reporter from coverage.results import Numbers +from coverage.misc import NotPython class SummaryReporter(Reporter): """A reporter for writing the summary report.""" - def __init__(self, coverage, show_missing=True, ignore_errors=False): - super(SummaryReporter, self).__init__(coverage, ignore_errors) - self.show_missing = show_missing + def __init__(self, coverage, config): + super(SummaryReporter, self).__init__(coverage, config) self.branches = coverage.data.has_arcs() - def report(self, morfs, outfile=None, config=None): + def report(self, morfs, outfile=None): """Writes a report summarizing coverage statistics per module. - `outfile` is a file object to write the summary to. `config` is a - CoverageConfig instance. + `outfile` is a file object to write the summary to. """ - self.find_code_units(morfs, config) + self.find_code_units(morfs) # Prepare the formatting strings max_name = max([len(cu.name) for cu in self.code_units] + [5]) @@ -30,12 +29,12 @@ class SummaryReporter(Reporter): header = (fmt_name % "Name") + " Stmts Miss" fmt_coverage = fmt_name + "%6d %6d" if self.branches: - header += " Branch BrPart" + header += " Branch BrMiss" fmt_coverage += " %6d %6d" width100 = Numbers.pc_str_width() header += "%*s" % (width100+4, "Cover") fmt_coverage += "%%%ds%%%%" % (width100+3,) - if self.show_missing: + if self.config.show_missing: header += " Missing" fmt_coverage += " %s" rule = "-" * len(header) + "\n" @@ -59,15 +58,19 @@ class SummaryReporter(Reporter): if self.branches: args += (nums.n_branches, nums.n_missing_branches) args += (nums.pc_covered_str,) - if self.show_missing: + if self.config.show_missing: args += (analysis.missing_formatted(),) outfile.write(fmt_coverage % args) total += nums - except KeyboardInterrupt: # pragma: no cover + except KeyboardInterrupt: # pragma: not covered raise except: - if not self.ignore_errors: + report_it = not self.config.ignore_errors + if report_it: typ, msg = sys.exc_info()[:2] + if typ is NotPython and not cu.should_be_python(): + report_it = False + if report_it: outfile.write(fmt_err % (cu.name, typ.__name__, msg)) if total.n_files > 1: @@ -76,6 +79,8 @@ class SummaryReporter(Reporter): if self.branches: args += (total.n_branches, total.n_missing_branches) args += (total.pc_covered_str,) - if self.show_missing: + if self.config.show_missing: args += ("",) outfile.write(fmt_coverage % args) + + return total.pc_covered diff --git a/python/helpers/coverage/templite.py b/python/helpers/coverage/templite.py index c39e061efd92..e5c0bafefb77 100644 --- a/python/helpers/coverage/templite.py +++ b/python/helpers/coverage/templite.py @@ -2,7 +2,53 @@ # Coincidentally named the same as http://code.activestate.com/recipes/496702/ -import re, sys +import re + +from coverage.backward import set # pylint: disable=W0622 + + +class CodeBuilder(object): + """Build source code conveniently.""" + + def __init__(self, indent=0): + self.code = [] + self.indent_amount = indent + + def add_line(self, line): + """Add a line of source to the code. + + Don't include indentations or newlines. + + """ + self.code.append(" " * self.indent_amount) + self.code.append(line) + self.code.append("\n") + + def add_section(self): + """Add a section, a sub-CodeBuilder.""" + sect = CodeBuilder(self.indent_amount) + self.code.append(sect) + return sect + + def indent(self): + """Increase the current indent for following lines.""" + self.indent_amount += 4 + + def dedent(self): + """Decrease the current indent for following lines.""" + self.indent_amount -= 4 + + def __str__(self): + return "".join([str(c) for c in self.code]) + + def get_function(self, fn_name): + """Compile the code, and return the function `fn_name`.""" + assert self.indent_amount == 0 + g = {} + code_text = str(self) + exec(code_text, g) + return g[fn_name] + class Templite(object): """A simple template renderer, for a nano-subset of Django syntax. @@ -39,53 +85,104 @@ class Templite(object): for context in contexts: self.context.update(context) + # We construct a function in source form, then compile it and hold onto + # it, and execute it to render the template. + code = CodeBuilder() + + code.add_line("def render(ctx, dot):") + code.indent() + vars_code = code.add_section() + self.all_vars = set() + self.loop_vars = set() + code.add_line("result = []") + code.add_line("a = result.append") + code.add_line("e = result.extend") + code.add_line("s = str") + + buffered = [] + def flush_output(): + """Force `buffered` to the code builder.""" + if len(buffered) == 1: + code.add_line("a(%s)" % buffered[0]) + elif len(buffered) > 1: + code.add_line("e([%s])" % ",".join(buffered)) + del buffered[:] + # Split the text to form a list of tokens. toks = re.split(r"(?s)({{.*?}}|{%.*?%}|{#.*?#})", text) - # Parse the tokens into a nested list of operations. Each item in the - # list is a tuple with an opcode, and arguments. They'll be - # interpreted by TempliteEngine. - # - # When parsing an action tag with nested content (if, for), the current - # ops list is pushed onto ops_stack, and the parsing continues in a new - # ops list that is part of the arguments to the if or for op. - ops = [] ops_stack = [] for tok in toks: if tok.startswith('{{'): - # Expression: ('exp', expr) - ops.append(('exp', tok[2:-2].strip())) + # An expression to evaluate. + buffered.append("s(%s)" % self.expr_code(tok[2:-2].strip())) elif tok.startswith('{#'): # Comment: ignore it and move on. continue elif tok.startswith('{%'): # Action tag: split into words and parse further. + flush_output() words = tok[2:-2].strip().split() if words[0] == 'if': - # If: ('if', (expr, body_ops)) - if_ops = [] + # An if statement: evaluate the expression to determine if. assert len(words) == 2 - ops.append(('if', (words[1], if_ops))) - ops_stack.append(ops) - ops = if_ops + ops_stack.append('if') + code.add_line("if %s:" % self.expr_code(words[1])) + code.indent() elif words[0] == 'for': - # For: ('for', (varname, listexpr, body_ops)) + # A loop: iterate over expression result. assert len(words) == 4 and words[2] == 'in' - for_ops = [] - ops.append(('for', (words[1], words[3], for_ops))) - ops_stack.append(ops) - ops = for_ops + ops_stack.append('for') + self.loop_vars.add(words[1]) + code.add_line( + "for c_%s in %s:" % ( + words[1], + self.expr_code(words[3]) + ) + ) + code.indent() elif words[0].startswith('end'): # Endsomething. Pop the ops stack - ops = ops_stack.pop() - assert ops[-1][0] == words[0][3:] + end_what = words[0][3:] + if ops_stack[-1] != end_what: + raise SyntaxError("Mismatched end tag: %r" % end_what) + ops_stack.pop() + code.dedent() else: - raise SyntaxError("Don't understand tag %r" % words) + raise SyntaxError("Don't understand tag: %r" % words[0]) else: - ops.append(('lit', tok)) + # Literal content. If it isn't empty, output it. + if tok: + buffered.append("%r" % tok) + flush_output() - assert not ops_stack, "Unmatched action tag: %r" % ops_stack[-1][0] - self.ops = ops + for var_name in self.all_vars - self.loop_vars: + vars_code.add_line("c_%s = ctx[%r]" % (var_name, var_name)) + + if ops_stack: + raise SyntaxError("Unmatched action tag: %r" % ops_stack[-1]) + + code.add_line("return ''.join(result)") + code.dedent() + self.render_function = code.get_function('render') + + def expr_code(self, expr): + """Generate a Python expression for `expr`.""" + if "|" in expr: + pipes = expr.split("|") + code = self.expr_code(pipes[0]) + for func in pipes[1:]: + self.all_vars.add(func) + code = "c_%s(%s)" % (func, code) + elif "." in expr: + dots = expr.split(".") + code = self.expr_code(dots[0]) + args = [repr(d) for d in dots[1:]] + code = "dot(%s, %s)" % (code, ", ".join(args)) + else: + self.all_vars.add(expr) + code = "c_%s" % expr + return code def render(self, context=None): """Render this template by applying it to `context`. @@ -97,70 +194,15 @@ class Templite(object): ctx = dict(self.context) if context: ctx.update(context) - - # Run it through an engine, and return the result. - engine = _TempliteEngine(ctx) - engine.execute(self.ops) - return "".join(engine.result) - - -class _TempliteEngine(object): - """Executes Templite objects to produce strings.""" - def __init__(self, context): - self.context = context - self.result = [] - - def execute(self, ops): - """Execute `ops` in the engine. - - Called recursively for the bodies of if's and loops. - - """ - for op, args in ops: - if op == 'lit': - self.result.append(args) - elif op == 'exp': - try: - self.result.append(str(self.evaluate(args))) - except: - exc_class, exc, _ = sys.exc_info() - new_exc = exc_class("Couldn't evaluate {{ %s }}: %s" - % (args, exc)) - raise new_exc - elif op == 'if': - expr, body = args - if self.evaluate(expr): - self.execute(body) - elif op == 'for': - var, lis, body = args - vals = self.evaluate(lis) - for val in vals: - self.context[var] = val - self.execute(body) - else: - raise AssertionError("TempliteEngine doesn't grok op %r" % op) - - def evaluate(self, expr): - """Evaluate an expression. - - `expr` can have pipes and dots to indicate data access and filtering. - - """ - if "|" in expr: - pipes = expr.split("|") - value = self.evaluate(pipes[0]) - for func in pipes[1:]: - value = self.evaluate(func)(value) - elif "." in expr: - dots = expr.split('.') - value = self.evaluate(dots[0]) - for dot in dots[1:]: - try: - value = getattr(value, dot) - except AttributeError: - value = value[dot] - if hasattr(value, '__call__'): - value = value() - else: - value = self.context[expr] + return self.render_function(ctx, self.do_dots) + + def do_dots(self, value, *dots): + """Evaluate dotted expressions at runtime.""" + for dot in dots: + try: + value = getattr(value, dot) + except AttributeError: + value = value[dot] + if hasattr(value, '__call__'): + value = value() return value diff --git a/python/helpers/coverage/tracer.c b/python/helpers/coverage/tracer.c new file mode 100644 index 000000000000..97dd113b8b3a --- /dev/null +++ b/python/helpers/coverage/tracer.c @@ -0,0 +1,730 @@ +/* C-based Tracer for Coverage. */ + +#include "Python.h" +#include "compile.h" /* in 2.3, this wasn't part of Python.h */ +#include "eval.h" /* or this. */ +#include "structmember.h" +#include "frameobject.h" + +/* Compile-time debugging helpers */ +#undef WHAT_LOG /* Define to log the WHAT params in the trace function. */ +#undef TRACE_LOG /* Define to log our bookkeeping. */ +#undef COLLECT_STATS /* Collect counters: stats are printed when tracer is stopped. */ + +#if COLLECT_STATS +#define STATS(x) x +#else +#define STATS(x) +#endif + +/* Py 2.x and 3.x compatibility */ + +#ifndef Py_TYPE +#define Py_TYPE(o) (((PyObject*)(o))->ob_type) +#endif + +#if PY_MAJOR_VERSION >= 3 + +#define MyText_Type PyUnicode_Type +#define MyText_Check(o) PyUnicode_Check(o) +#define MyText_AS_BYTES(o) PyUnicode_AsASCIIString(o) +#define MyText_AS_STRING(o) PyBytes_AS_STRING(o) +#define MyInt_FromLong(l) PyLong_FromLong(l) + +#define MyType_HEAD_INIT PyVarObject_HEAD_INIT(NULL, 0) + +#else + +#define MyText_Type PyString_Type +#define MyText_Check(o) PyString_Check(o) +#define MyText_AS_BYTES(o) (Py_INCREF(o), o) +#define MyText_AS_STRING(o) PyString_AS_STRING(o) +#define MyInt_FromLong(l) PyInt_FromLong(l) + +#define MyType_HEAD_INIT PyObject_HEAD_INIT(NULL) 0, + +#endif /* Py3k */ + +/* The values returned to indicate ok or error. */ +#define RET_OK 0 +#define RET_ERROR -1 + +/* An entry on the data stack. For each call frame, we need to record the + dictionary to capture data, and the last line number executed in that + frame. +*/ +typedef struct { + PyObject * file_data; /* PyMem_Malloc'ed, a borrowed ref. */ + int last_line; +} DataStackEntry; + +/* The CTracer type. */ + +typedef struct { + PyObject_HEAD + + /* Python objects manipulated directly by the Collector class. */ + PyObject * should_trace; + PyObject * warn; + PyObject * data; + PyObject * should_trace_cache; + PyObject * arcs; + + /* Has the tracer been started? */ + int started; + /* Are we tracing arcs, or just lines? */ + int tracing_arcs; + + /* + The data stack is a stack of dictionaries. Each dictionary collects + data for a single source file. The data stack parallels the call stack: + each call pushes the new frame's file data onto the data stack, and each + return pops file data off. + + The file data is a dictionary whose form depends on the tracing options. + If tracing arcs, the keys are line number pairs. If not tracing arcs, + the keys are line numbers. In both cases, the value is irrelevant + (None). + */ + /* The index of the last-used entry in data_stack. */ + int depth; + /* The file data at each level, or NULL if not recording. */ + DataStackEntry * data_stack; + int data_stack_alloc; /* number of entries allocated at data_stack. */ + + /* The current file_data dictionary. Borrowed. */ + PyObject * cur_file_data; + + /* The line number of the last line recorded, for tracing arcs. + -1 means there was no previous line, as when entering a code object. + */ + int last_line; + + /* The parent frame for the last exception event, to fix missing returns. */ + PyFrameObject * last_exc_back; + int last_exc_firstlineno; + +#if COLLECT_STATS + struct { + unsigned int calls; + unsigned int lines; + unsigned int returns; + unsigned int exceptions; + unsigned int others; + unsigned int new_files; + unsigned int missed_returns; + unsigned int stack_reallocs; + unsigned int errors; + } stats; +#endif /* COLLECT_STATS */ +} CTracer; + +#define STACK_DELTA 100 + +static int +CTracer_init(CTracer *self, PyObject *args_unused, PyObject *kwds_unused) +{ +#if COLLECT_STATS + self->stats.calls = 0; + self->stats.lines = 0; + self->stats.returns = 0; + self->stats.exceptions = 0; + self->stats.others = 0; + self->stats.new_files = 0; + self->stats.missed_returns = 0; + self->stats.stack_reallocs = 0; + self->stats.errors = 0; +#endif /* COLLECT_STATS */ + + self->should_trace = NULL; + self->warn = NULL; + self->data = NULL; + self->should_trace_cache = NULL; + self->arcs = NULL; + + self->started = 0; + self->tracing_arcs = 0; + + self->depth = -1; + self->data_stack = PyMem_Malloc(STACK_DELTA*sizeof(DataStackEntry)); + if (self->data_stack == NULL) { + STATS( self->stats.errors++; ) + PyErr_NoMemory(); + return RET_ERROR; + } + self->data_stack_alloc = STACK_DELTA; + + self->cur_file_data = NULL; + self->last_line = -1; + + self->last_exc_back = NULL; + + return RET_OK; +} + +static void +CTracer_dealloc(CTracer *self) +{ + if (self->started) { + PyEval_SetTrace(NULL, NULL); + } + + Py_XDECREF(self->should_trace); + Py_XDECREF(self->warn); + Py_XDECREF(self->data); + Py_XDECREF(self->should_trace_cache); + + PyMem_Free(self->data_stack); + + Py_TYPE(self)->tp_free((PyObject*)self); +} + +#if TRACE_LOG +static const char * +indent(int n) +{ + static const char * spaces = + " " + " " + " " + " " + ; + return spaces + strlen(spaces) - n*2; +} + +static int logging = 0; +/* Set these constants to be a file substring and line number to start logging. */ +static const char * start_file = "tests/views"; +static int start_line = 27; + +static void +showlog(int depth, int lineno, PyObject * filename, const char * msg) +{ + if (logging) { + printf("%s%3d ", indent(depth), depth); + if (lineno) { + printf("%4d", lineno); + } + else { + printf(" "); + } + if (filename) { + PyObject *ascii = MyText_AS_BYTES(filename); + printf(" %s", MyText_AS_STRING(ascii)); + Py_DECREF(ascii); + } + if (msg) { + printf(" %s", msg); + } + printf("\n"); + } +} + +#define SHOWLOG(a,b,c,d) showlog(a,b,c,d) +#else +#define SHOWLOG(a,b,c,d) +#endif /* TRACE_LOG */ + +#if WHAT_LOG +static const char * what_sym[] = {"CALL", "EXC ", "LINE", "RET "}; +#endif + +/* Record a pair of integers in self->cur_file_data. */ +static int +CTracer_record_pair(CTracer *self, int l1, int l2) +{ + int ret = RET_OK; + + PyObject * t = Py_BuildValue("(ii)", l1, l2); + if (t != NULL) { + if (PyDict_SetItem(self->cur_file_data, t, Py_None) < 0) { + STATS( self->stats.errors++; ) + ret = RET_ERROR; + } + Py_DECREF(t); + } + else { + STATS( self->stats.errors++; ) + ret = RET_ERROR; + } + return ret; +} + +/* + * The Trace Function + */ +static int +CTracer_trace(CTracer *self, PyFrameObject *frame, int what, PyObject *arg_unused) +{ + int ret = RET_OK; + PyObject * filename = NULL; + PyObject * tracename = NULL; + #if WHAT_LOG || TRACE_LOG + PyObject * ascii = NULL; + #endif + + #if WHAT_LOG + if (what <= sizeof(what_sym)/sizeof(const char *)) { + ascii = MyText_AS_BYTES(frame->f_code->co_filename); + printf("trace: %s @ %s %d\n", what_sym[what], MyText_AS_STRING(ascii), frame->f_lineno); + Py_DECREF(ascii); + } + #endif + + #if TRACE_LOG + ascii = MyText_AS_BYTES(frame->f_code->co_filename); + if (strstr(MyText_AS_STRING(ascii), start_file) && frame->f_lineno == start_line) { + logging = 1; + } + Py_DECREF(ascii); + #endif + + /* See below for details on missing-return detection. */ + if (self->last_exc_back) { + if (frame == self->last_exc_back) { + /* Looks like someone forgot to send a return event. We'll clear + the exception state and do the RETURN code here. Notice that the + frame we have in hand here is not the correct frame for the RETURN, + that frame is gone. Our handling for RETURN doesn't need the + actual frame, but we do log it, so that will look a little off if + you're looking at the detailed log. + + If someday we need to examine the frame when doing RETURN, then + we'll need to keep more of the missed frame's state. + */ + STATS( self->stats.missed_returns++; ) + if (self->depth >= 0) { + if (self->tracing_arcs && self->cur_file_data) { + if (CTracer_record_pair(self, self->last_line, -self->last_exc_firstlineno) < 0) { + return RET_ERROR; + } + } + SHOWLOG(self->depth, frame->f_lineno, frame->f_code->co_filename, "missedreturn"); + self->cur_file_data = self->data_stack[self->depth].file_data; + self->last_line = self->data_stack[self->depth].last_line; + self->depth--; + } + } + self->last_exc_back = NULL; + } + + + switch (what) { + case PyTrace_CALL: /* 0 */ + STATS( self->stats.calls++; ) + /* Grow the stack. */ + self->depth++; + if (self->depth >= self->data_stack_alloc) { + STATS( self->stats.stack_reallocs++; ) + /* We've outgrown our data_stack array: make it bigger. */ + int bigger = self->data_stack_alloc + STACK_DELTA; + DataStackEntry * bigger_data_stack = PyMem_Realloc(self->data_stack, bigger * sizeof(DataStackEntry)); + if (bigger_data_stack == NULL) { + STATS( self->stats.errors++; ) + PyErr_NoMemory(); + self->depth--; + return RET_ERROR; + } + self->data_stack = bigger_data_stack; + self->data_stack_alloc = bigger; + } + + /* Push the current state on the stack. */ + self->data_stack[self->depth].file_data = self->cur_file_data; + self->data_stack[self->depth].last_line = self->last_line; + + /* Check if we should trace this line. */ + filename = frame->f_code->co_filename; + tracename = PyDict_GetItem(self->should_trace_cache, filename); + if (tracename == NULL) { + STATS( self->stats.new_files++; ) + /* We've never considered this file before. */ + /* Ask should_trace about it. */ + PyObject * args = Py_BuildValue("(OO)", filename, frame); + tracename = PyObject_Call(self->should_trace, args, NULL); + Py_DECREF(args); + if (tracename == NULL) { + /* An error occurred inside should_trace. */ + STATS( self->stats.errors++; ) + return RET_ERROR; + } + if (PyDict_SetItem(self->should_trace_cache, filename, tracename) < 0) { + STATS( self->stats.errors++; ) + return RET_ERROR; + } + } + else { + Py_INCREF(tracename); + } + + /* If tracename is a string, then we're supposed to trace. */ + if (MyText_Check(tracename)) { + PyObject * file_data = PyDict_GetItem(self->data, tracename); + if (file_data == NULL) { + file_data = PyDict_New(); + if (file_data == NULL) { + STATS( self->stats.errors++; ) + return RET_ERROR; + } + ret = PyDict_SetItem(self->data, tracename, file_data); + Py_DECREF(file_data); + if (ret < 0) { + STATS( self->stats.errors++; ) + return RET_ERROR; + } + } + self->cur_file_data = file_data; + /* Make the frame right in case settrace(gettrace()) happens. */ + Py_INCREF(self); + frame->f_trace = (PyObject*)self; + SHOWLOG(self->depth, frame->f_lineno, filename, "traced"); + } + else { + self->cur_file_data = NULL; + SHOWLOG(self->depth, frame->f_lineno, filename, "skipped"); + } + + Py_DECREF(tracename); + + self->last_line = -1; + break; + + case PyTrace_RETURN: /* 3 */ + STATS( self->stats.returns++; ) + /* A near-copy of this code is above in the missing-return handler. */ + if (self->depth >= 0) { + if (self->tracing_arcs && self->cur_file_data) { + int first = frame->f_code->co_firstlineno; + if (CTracer_record_pair(self, self->last_line, -first) < 0) { + return RET_ERROR; + } + } + + SHOWLOG(self->depth, frame->f_lineno, frame->f_code->co_filename, "return"); + self->cur_file_data = self->data_stack[self->depth].file_data; + self->last_line = self->data_stack[self->depth].last_line; + self->depth--; + } + break; + + case PyTrace_LINE: /* 2 */ + STATS( self->stats.lines++; ) + if (self->depth >= 0) { + SHOWLOG(self->depth, frame->f_lineno, frame->f_code->co_filename, "line"); + if (self->cur_file_data) { + /* We're tracing in this frame: record something. */ + if (self->tracing_arcs) { + /* Tracing arcs: key is (last_line,this_line). */ + if (CTracer_record_pair(self, self->last_line, frame->f_lineno) < 0) { + return RET_ERROR; + } + } + else { + /* Tracing lines: key is simply this_line. */ + PyObject * this_line = MyInt_FromLong(frame->f_lineno); + if (this_line == NULL) { + STATS( self->stats.errors++; ) + return RET_ERROR; + } + ret = PyDict_SetItem(self->cur_file_data, this_line, Py_None); + Py_DECREF(this_line); + if (ret < 0) { + STATS( self->stats.errors++; ) + return RET_ERROR; + } + } + } + self->last_line = frame->f_lineno; + } + break; + + case PyTrace_EXCEPTION: + /* Some code (Python 2.3, and pyexpat anywhere) fires an exception event + without a return event. To detect that, we'll keep a copy of the + parent frame for an exception event. If the next event is in that + frame, then we must have returned without a return event. We can + synthesize the missing event then. + + Python itself fixed this problem in 2.4. Pyexpat still has the bug. + I've reported the problem with pyexpat as http://bugs.python.org/issue6359 . + If it gets fixed, this code should still work properly. Maybe some day + the bug will be fixed everywhere coverage.py is supported, and we can + remove this missing-return detection. + + More about this fix: http://nedbatchelder.com/blog/200907/a_nasty_little_bug.html + */ + STATS( self->stats.exceptions++; ) + self->last_exc_back = frame->f_back; + self->last_exc_firstlineno = frame->f_code->co_firstlineno; + break; + + default: + STATS( self->stats.others++; ) + break; + } + + return RET_OK; +} + +/* + * Python has two ways to set the trace function: sys.settrace(fn), which + * takes a Python callable, and PyEval_SetTrace(func, obj), which takes + * a C function and a Python object. The way these work together is that + * sys.settrace(pyfn) calls PyEval_SetTrace(builtin_func, pyfn), using the + * Python callable as the object in PyEval_SetTrace. So sys.gettrace() + * simply returns the Python object used as the second argument to + * PyEval_SetTrace. So sys.gettrace() will return our self parameter, which + * means it must be callable to be used in sys.settrace(). + * + * So we make our self callable, equivalent to invoking our trace function. + * + * To help with the process of replaying stored frames, this function has an + * optional keyword argument: + * + * def CTracer_call(frame, event, arg, lineno=0) + * + * If provided, the lineno argument is used as the line number, and the + * frame's f_lineno member is ignored. + */ +static PyObject * +CTracer_call(CTracer *self, PyObject *args, PyObject *kwds) +{ + PyFrameObject *frame; + PyObject *what_str; + PyObject *arg; + int lineno = 0; + int what; + int orig_lineno; + PyObject *ret = NULL; + + static char *what_names[] = { + "call", "exception", "line", "return", + "c_call", "c_exception", "c_return", + NULL + }; + + #if WHAT_LOG + printf("pytrace\n"); + #endif + + static char *kwlist[] = {"frame", "event", "arg", "lineno", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!O|i:Tracer_call", kwlist, + &PyFrame_Type, &frame, &MyText_Type, &what_str, &arg, &lineno)) { + goto done; + } + + /* In Python, the what argument is a string, we need to find an int + for the C function. */ + for (what = 0; what_names[what]; what++) { + PyObject *ascii = MyText_AS_BYTES(what_str); + int should_break = !strcmp(MyText_AS_STRING(ascii), what_names[what]); + Py_DECREF(ascii); + if (should_break) { + break; + } + } + + /* Save off the frame's lineno, and use the forced one, if provided. */ + orig_lineno = frame->f_lineno; + if (lineno > 0) { + frame->f_lineno = lineno; + } + + /* Invoke the C function, and return ourselves. */ + if (CTracer_trace(self, frame, what, arg) == RET_OK) { + Py_INCREF(self); + ret = (PyObject *)self; + } + + /* Clean up. */ + frame->f_lineno = orig_lineno; + +done: + return ret; +} + +static PyObject * +CTracer_start(CTracer *self, PyObject *args_unused) +{ + PyEval_SetTrace((Py_tracefunc)CTracer_trace, (PyObject*)self); + self->started = 1; + self->tracing_arcs = self->arcs && PyObject_IsTrue(self->arcs); + self->last_line = -1; + + /* start() returns a trace function usable with sys.settrace() */ + Py_INCREF(self); + return (PyObject *)self; +} + +static PyObject * +CTracer_stop(CTracer *self, PyObject *args_unused) +{ + if (self->started) { + PyEval_SetTrace(NULL, NULL); + self->started = 0; + } + + return Py_BuildValue(""); +} + +static PyObject * +CTracer_get_stats(CTracer *self) +{ +#if COLLECT_STATS + return Py_BuildValue( + "{sI,sI,sI,sI,sI,sI,sI,sI,si,sI}", + "calls", self->stats.calls, + "lines", self->stats.lines, + "returns", self->stats.returns, + "exceptions", self->stats.exceptions, + "others", self->stats.others, + "new_files", self->stats.new_files, + "missed_returns", self->stats.missed_returns, + "stack_reallocs", self->stats.stack_reallocs, + "stack_alloc", self->data_stack_alloc, + "errors", self->stats.errors + ); +#else + return Py_BuildValue(""); +#endif /* COLLECT_STATS */ +} + +static PyMemberDef +CTracer_members[] = { + { "should_trace", T_OBJECT, offsetof(CTracer, should_trace), 0, + PyDoc_STR("Function indicating whether to trace a file.") }, + + { "warn", T_OBJECT, offsetof(CTracer, warn), 0, + PyDoc_STR("Function for issuing warnings.") }, + + { "data", T_OBJECT, offsetof(CTracer, data), 0, + PyDoc_STR("The raw dictionary of trace data.") }, + + { "should_trace_cache", T_OBJECT, offsetof(CTracer, should_trace_cache), 0, + PyDoc_STR("Dictionary caching should_trace results.") }, + + { "arcs", T_OBJECT, offsetof(CTracer, arcs), 0, + PyDoc_STR("Should we trace arcs, or just lines?") }, + + { NULL } +}; + +static PyMethodDef +CTracer_methods[] = { + { "start", (PyCFunction) CTracer_start, METH_VARARGS, + PyDoc_STR("Start the tracer") }, + + { "stop", (PyCFunction) CTracer_stop, METH_VARARGS, + PyDoc_STR("Stop the tracer") }, + + { "get_stats", (PyCFunction) CTracer_get_stats, METH_VARARGS, + PyDoc_STR("Get statistics about the tracing") }, + + { NULL } +}; + +static PyTypeObject +CTracerType = { + MyType_HEAD_INIT + "coverage.CTracer", /*tp_name*/ + sizeof(CTracer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)CTracer_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + (ternaryfunc)CTracer_call, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "CTracer objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + CTracer_methods, /* tp_methods */ + CTracer_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)CTracer_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + +/* Module definition */ + +#define MODULE_DOC PyDoc_STR("Fast coverage tracer.") + +#if PY_MAJOR_VERSION >= 3 + +static PyModuleDef +moduledef = { + PyModuleDef_HEAD_INIT, + "coverage.tracer", + MODULE_DOC, + -1, + NULL, /* methods */ + NULL, + NULL, /* traverse */ + NULL, /* clear */ + NULL +}; + + +PyObject * +PyInit_tracer(void) +{ + PyObject * mod = PyModule_Create(&moduledef); + if (mod == NULL) { + return NULL; + } + + CTracerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&CTracerType) < 0) { + Py_DECREF(mod); + return NULL; + } + + Py_INCREF(&CTracerType); + PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType); + + return mod; +} + +#else + +void +inittracer(void) +{ + PyObject * mod; + + mod = Py_InitModule3("coverage.tracer", NULL, MODULE_DOC); + if (mod == NULL) { + return; + } + + CTracerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&CTracerType) < 0) { + return; + } + + Py_INCREF(&CTracerType); + PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType); +} + +#endif /* Py3k */ diff --git a/python/helpers/coverage/tracer.pyd b/python/helpers/coverage/tracer.pyd Binary files differdeleted file mode 100644 index a13aa032abe1..000000000000 --- a/python/helpers/coverage/tracer.pyd +++ /dev/null diff --git a/python/helpers/coverage/version.py b/python/helpers/coverage/version.py new file mode 100644 index 000000000000..a43bde8023e3 --- /dev/null +++ b/python/helpers/coverage/version.py @@ -0,0 +1,9 @@ +"""The version and URL for coverage.py""" +# This file is exec'ed in setup.py, don't import anything! + +__version__ = "3.7.1" # see detailed history in CHANGES.txt + +__url__ = "http://nedbatchelder.com/code/coverage" +if max(__version__).isalpha(): + # For pre-releases, use a version-specific URL. + __url__ += "/" + __version__ diff --git a/python/helpers/coverage/xmlreport.py b/python/helpers/coverage/xmlreport.py index 5f6cc87e2fef..26ac02ad13d0 100644 --- a/python/helpers/coverage/xmlreport.py +++ b/python/helpers/coverage/xmlreport.py @@ -4,7 +4,7 @@ import os, sys, time import xml.dom.minidom from coverage import __url__, __version__ -from coverage.backward import sorted # pylint: disable=W0622 +from coverage.backward import sorted, rpartition # pylint: disable=W0622 from coverage.report import Reporter def rate(hit, num): @@ -15,20 +15,19 @@ def rate(hit, num): class XmlReporter(Reporter): """A reporter for writing Cobertura-style XML coverage results.""" - def __init__(self, coverage, ignore_errors=False): - super(XmlReporter, self).__init__(coverage, ignore_errors) + def __init__(self, coverage, config): + super(XmlReporter, self).__init__(coverage, config) self.packages = None self.xml_out = None self.arcs = coverage.data.has_arcs() - def report(self, morfs, outfile=None, config=None): + def report(self, morfs, outfile=None): """Generate a Cobertura-compatible XML report for `morfs`. `morfs` is a list of modules or filenames. - `outfile` is a file object to write the XML to. `config` is a - CoverageConfig instance. + `outfile` is a file object to write the XML to. """ # Initial setup. @@ -54,7 +53,7 @@ class XmlReporter(Reporter): # Call xml_file for each file in the data. self.packages = {} - self.report_files(self.xml_file, morfs, config) + self.report_files(self.xml_file, morfs) lnum_tot, lhits_tot = 0, 0 bnum_tot, bhits_tot = 0, 0 @@ -85,14 +84,23 @@ class XmlReporter(Reporter): # Use the DOM to write the output file. outfile.write(self.xml_out.toprettyxml()) + # Return the total percentage. + denom = lnum_tot + bnum_tot + if denom == 0: + pct = 0.0 + else: + pct = 100.0 * (lhits_tot + bhits_tot) / denom + return pct + def xml_file(self, cu, analysis): """Add to the XML report for a single file.""" # Create the 'lines' and 'package' XML elements, which # are populated later. Note that a package == a directory. - dirname, fname = os.path.split(cu.name) - dirname = dirname or '.' - package = self.packages.setdefault(dirname, [ {}, 0, 0, 0, 0 ]) + package_name = rpartition(cu.name, ".")[0] + className = cu.name + + package = self.packages.setdefault(package_name, [{}, 0, 0, 0, 0]) xclass = self.xml_out.createElement("class") @@ -100,22 +108,22 @@ class XmlReporter(Reporter): xlines = self.xml_out.createElement("lines") xclass.appendChild(xlines) - className = fname.replace('.', '_') + xclass.setAttribute("name", className) - ext = os.path.splitext(cu.filename)[1] - xclass.setAttribute("filename", cu.name + ext) + filename = cu.file_locator.relative_filename(cu.filename) + xclass.setAttribute("filename", filename.replace("\\", "/")) xclass.setAttribute("complexity", "0") branch_stats = analysis.branch_stats() # For each statement, create an XML 'line' element. - for line in analysis.statements: + for line in sorted(analysis.statements): xline = self.xml_out.createElement("line") xline.setAttribute("number", str(line)) # Q: can we get info about the number of times a statement is # executed? If so, that should be recorded here. - xline.setAttribute("hits", str(int(not line in analysis.missing))) + xline.setAttribute("hits", str(int(line not in analysis.missing))) if self.arcs: if line in branch_stats: |
