path: root/python/helpers/pycharm_generator_utils/module_redeclarator.py

import keyword

from pycharm_generator_utils.util_methods import *
from pycharm_generator_utils.constants import *


class emptylistdict(dict):
    """defaultdict not available before 2.5; simplest reimplementation using [] as default"""

    def __getitem__(self, item):
        if item in self:
            return dict.__getitem__(self, item)
        else:
            it = []
            self.__setitem__(item, it)
            return it

class Buf(object):
    """Buffers data in a list, can write to a file. Indentation is provided externally."""

    def __init__(self, indenter):
        self.data = []
        self.indenter = indenter

    def put(self, data):
        if data:
            self.data.append(ensureUnicode(data))

    def out(self, indent, *what):
        """Output the arguments, indenting as needed, and adding an eol"""
        self.put(self.indenter.indent(indent))
        for item in what:
            self.put(item)
        self.put("\n")

    def flush_bytes(self, outfile):
        for data in self.data:
            outfile.write(data.encode(OUT_ENCODING, "replace"))

    def flush_str(self, outfile):
        for data in self.data:
            outfile.write(data)

    if version[0] < 3:
        flush = flush_bytes
    else:
        flush = flush_str

    def isEmpty(self):
        return len(self.data) == 0


class ClassBuf(Buf):
    def __init__(self, name, indenter):
        super(ClassBuf, self).__init__(indenter)
        self.name = name

#noinspection PyUnresolvedReferences,PyBroadException
class ModuleRedeclarator(object):
    def __init__(self, module, outfile, mod_filename, indent_size=4, doing_builtins=False):
        """
        Create new instance.
        @param module module to restore.
        @param outfile output file, must be open and writable.
        @param mod_filename filename of binary module (the .dll or .so)
        @param indent_size amount of space characters per indent
        """
        self.module = module
        self.outfile = outfile # where we finally write
        self.mod_filename = mod_filename
        # we write things into buffers out-of-order
        self.header_buf = Buf(self)
        self.imports_buf = Buf(self)
        self.functions_buf = Buf(self)
        self.classes_buf = Buf(self)
        self.classes_buffs = list()
        self.footer_buf = Buf(self)
        self.indent_size = indent_size
        self._indent_step = " " * self.indent_size
        self.split_modules = False
        #
        self.imported_modules = {"": the_builtins} # explicit module imports: {"name": module}
        self.hidden_imports = {} # {'real_mod_name': 'alias'}; we alias names with "__" since we don't want them exported
        # ^ used for things that we don't re-export but need to import, e.g. certain base classes in gnome.
        self._defined = {} # stores True for every name defined so far, to break circular refs in values
        self.doing_builtins = doing_builtins
        self.ret_type_cache = {}
        self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names

    def _initializeQApp4(self):
        try:  # QtGui should be imported _before_ QtCore package.
            # This is done for the QWidget references from QtCore (such as QSignalMapper). Known bug in PyQt 4.7+
            # Causes "TypeError: C++ type 'QWidget*' is not supported as a native Qt signal type"
            import PyQt4.QtGui
        except ImportError:
            pass

        # manually instantiate and keep reference to singleton QCoreApplication (we don't want it to be deleted during the introspection)
        # use QCoreApplication instead of QApplication to avoid blinking app in Dock on Mac OS
        try:
            from PyQt4.QtCore import QCoreApplication
            self.app = QCoreApplication([])
            return
        except ImportError:
            pass

    def _initializeQApp5(self):
        try:
            from PyQt5.QtCore import QCoreApplication
            self.app = QCoreApplication([])
            return
        except ImportError:
            pass

    def indent(self, level):
        """Return indentation whitespace for given level."""
        return self._indent_step * level

    def flush(self):
        init = None
        try:
            if self.split_modules:
                mod_path = module_to_package_name(self.outfile)

                fname = build_output_name(mod_path, "__init__")
                init = fopen(fname, "w")
                for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf):
                    buf.flush(init)

                data = ""
                for buf in self.classes_buffs:
                    fname = build_output_name(mod_path, buf.name)
                    dummy = fopen(fname, "w")
                    self.header_buf.flush(dummy)
                    self.imports_buf.flush(dummy)
                    buf.flush(dummy)
                    data += self.create_local_import(buf.name)
                    dummy.close()

                init.write(data)
                self.footer_buf.flush(init)
            else:
                init = fopen(self.outfile, "w")
                for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf):
                    buf.flush(init)

                for buf in self.classes_buffs:
                    buf.flush(init)

                self.footer_buf.flush(init)

        finally:
            if init is not None and not init.closed:
                init.close()

    # Some builtin classes effectively change __init__ signature without overriding it.
    # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS
    def fake_builtin_init(self):
        pass # just a callable, sig doesn't matter

    fake_builtin_init.__doc__ = object.__init__.__doc__ # this forces class's doc to be used instead

    def create_local_import(self, name):
        if len(name.split(".")) > 1: return ""
        data = "from "
        if version[0] >= 3:
            data += "."
        data += name + " import " + name + "\n"
        return data

    def find_imported_name(self, item):
        """
        Finds out how the item is represented in imported modules.
        @param item what to check
        @return qualified name (like "sys.stdin") or None
        """
        # TODO: return a pair, not a glued string
        if not isinstance(item, SIMPLEST_TYPES):
            for mname in self.imported_modules:
                m = self.imported_modules[mname]
                for inner_name in m.__dict__:
                    suspect = getattr(m, inner_name)
                    if suspect is item:
                        if mname:
                            mname += "."
                        elif self.module is the_builtins: # don't short-circuit builtins
                            return None
                        return mname + inner_name
        return None

    _initializers = (
        (dict, "{}"),
        (tuple, "()"),
        (list, "[]"),
    )

    def invent_initializer(self, a_type):
        """
        Returns an innocuous initializer expression for a_type, or "None"
        """
        for initializer_type, r in self._initializers:
            if initializer_type == a_type:
                return r
                # NOTE: here we could handle things like defaultdict, sets, etc if we wanted
        return "None"


    def fmt_value(self, out, p_value, indent, prefix="", postfix="", as_name=None, seen_values=None):
        """
        Formats and outputs value (it occupies an entire line or several lines).
        @param out function that does output (a Buf.out)
        @param p_value the value.
        @param indent indent level.
        @param prefix text to print before the value
        @param postfix text to print after the value
        @param as_name hints which name are we trying to print; helps with circular refs.
        @param seen_values a list of keys we've seen if we're processing a dict
        """
        SELF_VALUE = "<value is a self-reference, replaced by this string>"
        ERR_VALUE = "<failed to retrieve the value>"
        if isinstance(p_value, SIMPLEST_TYPES):
            out(indent, prefix, reliable_repr(p_value), postfix)
        else:
            if sys.platform == "cli":
                imported_name = None
            else:
                imported_name = self.find_imported_name(p_value)
            if imported_name:
                out(indent, prefix, imported_name, postfix)
                # TODO: kind of self.used_imports[imported_name].append(p_value) but split imported_name
                # else we could potentially return smth we did not otherwise import. but not likely.
            else:
                if isinstance(p_value, (list, tuple)):
                    if not seen_values:
                        seen_values = [p_value]
                    if len(p_value) == 0:
                        out(indent, prefix, repr(p_value), postfix)
                    else:
                        if isinstance(p_value, list):
                            lpar, rpar = "[", "]"
                        else:
                            lpar, rpar = "(", ")"
                        out(indent, prefix, lpar)
                        for value in p_value:
                            if value in seen_values:
                                value = SELF_VALUE
                            elif not isinstance(value, SIMPLEST_TYPES):
                                seen_values.append(value)
                            self.fmt_value(out, value, indent + 1, postfix=",", seen_values=seen_values)
                        out(indent, rpar, postfix)
                elif isinstance(p_value, dict):
                    if len(p_value) == 0:
                        out(indent, prefix, repr(p_value), postfix)
                    else:
                        if not seen_values:
                            seen_values = [p_value]
                        out(indent, prefix, "{")
                        keys = list(p_value.keys())
                        try:
                            keys.sort()
                        except TypeError:
                            pass # unsortable keys happen, e,g, in py3k _ctypes
                        for k in keys:
                            value = p_value[k]

                            try:
                                is_seen = value in seen_values
                            except:
                                is_seen = False
                                value = ERR_VALUE

                            if is_seen:
                                value = SELF_VALUE
                            elif not isinstance(value, SIMPLEST_TYPES):
                                seen_values.append(value)
                            if isinstance(k, SIMPLEST_TYPES):
                                self.fmt_value(out, value, indent + 1, prefix=repr(k) + ": ", postfix=",",
                                               seen_values=seen_values)
                            else:
                                # both key and value need fancy formatting
                                self.fmt_value(out, k, indent + 1, postfix=": ", seen_values=seen_values)
                                self.fmt_value(out, value, indent + 2, seen_values=seen_values)
                                out(indent + 1, ",")
                        out(indent, "}", postfix)
                else: # something else, maybe representable
                    # look up this value in the module.
                    if sys.platform == "cli":
                        out(indent, prefix, "None", postfix)
                        return
                    found_name = ""
                    for inner_name in self.module.__dict__:
                        if self.module.__dict__[inner_name] is p_value:
                            found_name = inner_name
                            break
                    if self._defined.get(found_name, False):
                        out(indent, prefix, found_name, postfix)
                    else:
                        # a forward / circular declaration happens
                        notice = ""
                        try:
                            representation = repr(p_value)
                        except Exception:
                            import traceback
                            traceback.print_exc(file=sys.stderr)
                            return
                        real_value = cleanup(representation)
                        if found_name:
                            if found_name == as_name:
                                notice = " # (!) real value is %r" % real_value
                                real_value = "None"
                            else:
                                notice = " # (!) forward: %s, real value is %r" % (found_name, real_value)
                        if SANE_REPR_RE.match(real_value):
                            out(indent, prefix, real_value, postfix, notice)
                        else:
                            if not found_name:
                                notice = " # (!) real value is %r" % real_value
                            out(indent, prefix, "None", postfix, notice)

    def get_ret_type(self, attr):
        """
        Returns a return type string as given by T_RETURN in tokens, or None
        """
        if attr:
            ret_type = RET_TYPE.get(attr, None)
            if ret_type:
                return ret_type
            thing = getattr(self.module, attr, None)
            if thing:
                if not isinstance(thing, type) and is_callable(thing): # a function
                    return None # TODO: maybe divinate a return type; see pygame.mixer.Channel
                return attr
                # adds no noticeable slowdown, I did measure. dch.
            for im_name, im_module in self.imported_modules.items():
                cache_key = (im_name, attr)
                cached = self.ret_type_cache.get(cache_key, None)
                if cached:
                    return cached
                ret_type = getattr(im_module, attr, None)
                if ret_type:
                    if isinstance(ret_type, type):
                        # detect a constructor
                        constr_args = detect_constructor(ret_type)
                        if constr_args is None:
                            constr_args = "*(), **{}" # a silly catch-all constructor
                        reference = "%s(%s)" % (attr, constr_args)
                    elif is_callable(ret_type): # a function, classes are ruled out above
                        return None
                    else:
                        reference = attr
                    if im_name:
                        result = "%s.%s" % (im_name, reference)
                    else: # built-in
                        result = reference
                    self.ret_type_cache[cache_key] = result
                    return result
                    # TODO: handle things like "[a, b,..] and (foo,..)"
        return None


    SIG_DOC_NOTE = "restored from __doc__"
    SIG_DOC_UNRELIABLY = "NOTE: unreliably restored from __doc__ "

    def restore_by_docstring(self, signature_string, class_name, deco=None, ret_hint=None):
        """
        @param signature_string: parameter list extracted from the doc string.
        @param class_name: name of the containing class, or None
        @param deco: decorator to use
        @param ret_hint: return type hint, if available
        @return (reconstructed_spec, return_type, note) or (None, _, _) if failed.
        """
        action("restoring func %r of class %r", signature_string, class_name)
        # parse
        parsing_failed = False
        ret_type = None
        try:
            # strict parsing
            tokens = paramSeqAndRest.parseString(signature_string, True)
            ret_name = None
            if tokens:
                ret_t = tokens[-1]
                if ret_t[0] is T_RETURN:
                    ret_name = ret_t[1]
            ret_type = self.get_ret_type(ret_name) or self.get_ret_type(ret_hint)
        except ParseException:
            # it did not parse completely; scavenge what we can
            parsing_failed = True
            tokens = []
            try:
                # most unrestrictive parsing
                tokens = paramSeq.parseString(signature_string, False)
            except ParseException:
                pass
                #
        seq = transform_seq(tokens)

        # add safe defaults for unparsed
        if parsing_failed:
            doc_node = self.SIG_DOC_UNRELIABLY
            starred = None
            double_starred = None
            for one in seq:
                if type(one) is str:
                    if one.startswith("**"):
                        double_starred = one
                    elif one.startswith("*"):
                        starred = one
            if not starred:
                seq.append("*args")
            if not double_starred:
                seq.append("**kwargs")
        else:
            doc_node = self.SIG_DOC_NOTE

        # add 'self' if needed YYY
        if class_name and (not seq or seq[0] != 'self'):
            first_param = propose_first_param(deco)
            if first_param:
                seq.insert(0, first_param)
        seq = make_names_unique(seq)
        return (seq, ret_type, doc_node)

    def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False):
        """
        @param func_doc: __doc__ of the function.
        @param func_id: name to look for as identifier of the function in docstring
        @param func_name: name of the function.
        @param class_name: name of the containing class, or None
        @param deco: decorator to use
        @return (reconstructed_spec, return_literal, note) or (None, _, _) if failed.
        """
        if sip_generated:
            overloads = []
            for part in func_doc.split('\n'):
                signature = func_id + '('
                i = part.find(signature)
                if i >= 0:
                    overloads.append(part[i + len(signature):])
            if len(overloads) > 1:
                docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads]
                ret_types = []
                for result in docstring_results:
                    rt = result[1]
                    if rt and rt not in ret_types:
                        ret_types.append(rt)
                if ret_types:
                    ret_literal = " or ".join(ret_types)
                else:
                    ret_literal = None
                param_lists = [result[0] for result in docstring_results]
                spec = build_signature(func_name, restore_parameters_for_overloads(param_lists))
                return (spec, ret_literal, "restored from __doc__ with multiple overloads")

        # find the first thing to look like a definition
        prefix_re = re.compile("\s*(?:(\w+)[ \\t]+)?" + func_id + "\s*\(") # "foo(..." or "int foo(..."
        match = prefix_re.search(func_doc) # Note: this and previous line may consume up to 35% of time
        # parse the part that looks right
        if match:
            ret_hint = match.group(1)
            params, ret_literal, doc_note = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint)
            spec = func_name + flatten(params)
            return (spec, ret_literal, doc_note)
        else:
            return (None, None, None)


    def is_predefined_builtin(self, module_name, class_name, func_name):
        return self.doing_builtins and module_name == BUILTIN_MOD_NAME and (
            class_name, func_name) in PREDEFINED_BUILTIN_SIGS


    def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None):
        """
        Restore function argument list as best we can.
        @param out output function of a Buf
        @param p_func function or method object
        @param p_name function name as known to owner
        @param indent indentation level
        @param p_class the class that contains this function as a method
        @param p_modname module name
        @param seen {id(func): name} map of functions already seen in the same namespace;
               id() because *some* functions are unhashable (eg _elementtree.Comment in py2.7)
        """
        action("redoing func %r of class %r", p_name, p_class)
        if seen is not None:
            other_func = seen.get(id(p_func), None)
            if other_func and getattr(other_func, "__doc__", None) is getattr(p_func, "__doc__", None):
                # _bisect.bisect == _bisect.bisect_right in py31, but docs differ
                out(indent, p_name, " = ", seen[id(p_func)])
                out(indent, "")
                return
            else:
                seen[id(p_func)] = p_name
                # real work
        if classname is None:
            classname = p_class and p_class.__name__ or None
        if p_class and hasattr(p_class, '__mro__'):
            sip_generated = [base_t for base_t in p_class.__mro__ if 'sip.simplewrapper' in str(base_t)]
        else:
            sip_generated = False
        deco = None
        deco_comment = ""
        mod_class_method_tuple = (p_modname, classname, p_name)
        ret_literal = None
        is_init = False
        # any decorators?
        action("redoing decos of func %r of class %r", p_name, p_class)
        if self.doing_builtins and p_modname == BUILTIN_MOD_NAME:
            deco = KNOWN_DECORATORS.get((classname, p_name), None)
            if deco:
                deco_comment = " # known case"
        elif p_class and p_name in p_class.__dict__:
            # detect native methods declared with METH_CLASS flag
            descriptor = p_class.__dict__[p_name]
            if p_name != "__new__" and type(descriptor).__name__.startswith('classmethod'):
                # 'classmethod_descriptor' in Python 2.x and 3.x, 'classmethod' in Jython
                deco = "classmethod"
            elif type(p_func).__name__.startswith('staticmethod'):
                deco = "staticmethod"
        if p_name == "__new__":
            deco = "staticmethod"
            deco_comment = " # known case of __new__"

        action("redoing innards of func %r of class %r", p_name, p_class)
        if deco and HAS_DECORATORS:
            out(indent, "@", deco, deco_comment)
        if inspect and inspect.isfunction(p_func):
            out(indent, "def ", p_name, restore_by_inspect(p_func), ": # reliably restored by inspect", )
            out_doc_attr(out, p_func, indent + 1, p_class)
        elif self.is_predefined_builtin(*mod_class_method_tuple):
            spec, sig_note = restore_predefined_builtin(classname, p_name)
            out(indent, "def ", spec, ": # ", sig_note)
            out_doc_attr(out, p_func, indent + 1, p_class)
        elif sys.platform == 'cli' and is_clr_type(p_class):
            is_static, spec, sig_note = restore_clr(p_name, p_class)
            if is_static:
                out(indent, "@staticmethod")
            if not spec: return
            if sig_note:
                out(indent, "def ", spec, ": #", sig_note)
            else:
                out(indent, "def ", spec, ":")
            if not p_name in ['__gt__', '__ge__', '__lt__', '__le__', '__ne__', '__reduce_ex__', '__str__']:
                out_doc_attr(out, p_func, indent + 1, p_class)
        elif mod_class_method_tuple in PREDEFINED_MOD_CLASS_SIGS:
            sig, ret_literal = PREDEFINED_MOD_CLASS_SIGS[mod_class_method_tuple]
            if classname:
                ofwhat = "%s.%s.%s" % mod_class_method_tuple
            else:
                ofwhat = "%s.%s" % (p_modname, p_name)
            out(indent, "def ", p_name, sig, ": # known case of ", ofwhat)
            out_doc_attr(out, p_func, indent + 1, p_class)
        else:
            # __doc__ is our best source of arglist
            sig_note = "real signature unknown"
            spec = ""
            is_init = (p_name == "__init__" and p_class is not None)
            funcdoc = None
            if is_init and hasattr(p_class, "__doc__"):
                if hasattr(p_func, "__doc__"):
                    funcdoc = p_func.__doc__
                if funcdoc == object.__init__.__doc__:
                    funcdoc = p_class.__doc__
            elif hasattr(p_func, "__doc__"):
                funcdoc = p_func.__doc__
            sig_restored = False
            action("parsing doc of func %r of class %r", p_name, p_class)
            if isinstance(funcdoc, STR_TYPES):
                (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, p_name, p_name, classname, deco,
                                                                      sip_generated)
                if spec is None and p_name == '__init__' and classname:
                    (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, classname, p_name, classname, deco,
                                                                          sip_generated)
                sig_restored = spec is not None
                if more_notes:
                    if sig_note:
                        sig_note += "; "
                    sig_note += more_notes
            if not sig_restored:
                # use an allow-all declaration
                decl = []
                if p_class:
                    first_param = propose_first_param(deco)
                    if first_param:
                        decl.append(first_param)
                decl.append("*args")
                decl.append("**kwargs")
                spec = p_name + "(" + ", ".join(decl) + ")"
            out(indent, "def ", spec, ": # ", sig_note)
            # to reduce size of stubs, don't output same docstring twice for class and its __init__ method
            if not is_init or funcdoc != p_class.__doc__:
                out_docstring(out, funcdoc, indent + 1)
                # body
        if ret_literal and not is_init:
            out(indent + 1, "return ", ret_literal)
        else:
            out(indent + 1, "pass")
        if deco and not HAS_DECORATORS:
            out(indent, p_name, " = ", deco, "(", p_name, ")", deco_comment)
        out(0, "") # empty line after each item


    def redo_class(self, out, p_class, p_name, indent, p_modname=None, seen=None, inspect_dir=False):
        """
        Restores a class definition.
        @param out output function of a relevant buf
        @param p_class the class object
        @param p_name class name as known to owner
        @param indent indentation level
        @param p_modname name of module
        @param seen {class: name} map of classes already seen in the same namespace
        """
        action("redoing class %r of module %r", p_name, p_modname)
        if seen is not None:
            if p_class in seen:
                out(indent, p_name, " = ", seen[p_class])
                out(indent, "")
                return
            else:
                seen[p_class] = p_name
        bases = get_bases(p_class)
        base_def = ""
        skipped_bases = []
        if bases:
            skip_qualifiers = [p_modname, BUILTIN_MOD_NAME, 'exceptions']
            skip_qualifiers.extend(KNOWN_FAKE_REEXPORTERS.get(p_modname, ()))
            bases_list = [] # what we'll render in the class decl
            for base in bases:
                if [1 for (cls, mdl) in KNOWN_FAKE_BASES if cls == base and mdl != self.module]:
                    # our base is a wrapper and our module is not its defining module
                    skipped_bases.append(str(base))
                    continue
                    # somehow import every base class
                base_name = base.__name__
                qual_module_name = qualifier_of(base, skip_qualifiers)
                got_existing_import = False
                if qual_module_name:
                    if qual_module_name in self.used_imports:
                        import_list = self.used_imports[qual_module_name]
                        if base in import_list:
                            bases_list.append(base_name) # unqualified: already set to import
                            got_existing_import = True
                    if not got_existing_import:
                        mangled_qualifier = "__" + qual_module_name.replace('.', '_') # foo.bar -> __foo_bar
                        bases_list.append(mangled_qualifier + "." + base_name)
                        self.hidden_imports[qual_module_name] = mangled_qualifier
                else:
                    bases_list.append(base_name)
            base_def = "(" + ", ".join(bases_list) + ")"

            for base in bases_list:
                local_import = self.create_local_import(base)
                if local_import:
                    out(indent, local_import)
        out(indent, "class ", p_name, base_def, ":",
            skipped_bases and " # skipped bases: " + ", ".join(skipped_bases) or "")
        out_doc_attr(out, p_class, indent + 1)
        # inner parts
        methods = {}
        properties = {}
        others = {}
        we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object"
        field_source = {}
        try:
            if hasattr(p_class, "__dict__") and not inspect_dir:
                field_source = p_class.__dict__
                field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here
            else:
                field_keys = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare
        except:
            field_keys = ()
        for item_name in field_keys:
            if item_name in ("__doc__", "__module__"):
                if we_are_the_base_class:
                    item = "" # must be declared in base types
                else:
                    continue # in all other cases must be skipped
            elif keyword.iskeyword(item_name):  # for example, PyQt4 contains definitions of methods named 'exec'
                continue
            else:
                try:
                    item = getattr(p_class, item_name) # let getters do the magic
                except AttributeError:
                    item = field_source[item_name] # have it raw
                except Exception:
                    continue
            if is_callable(item) and not isinstance(item, type):
                methods[item_name] = item
            elif is_property(item):
                properties[item_name] = item
            else:
                others[item_name] = item
                #
        if we_are_the_base_class:
            others["__dict__"] = {} # force-feed it, for __dict__ does not contain a reference to itself :)
            # add fake __init__s to have the right sig
        if p_class in FAKE_BUILTIN_INITS:
            methods["__init__"] = self.fake_builtin_init
            note("Faking init of %s", p_name)
        elif '__init__' not in methods:
            init_method = getattr(p_class, '__init__', None)
            if init_method:
                methods['__init__'] = init_method

        #
        seen_funcs = {}
        for item_name in sorted_no_case(methods.keys()):
            item = methods[item_name]
            try:
                self.redo_function(out, item, item_name, indent + 1, p_class, p_modname, classname=p_name, seen=seen_funcs)
            except:
                handle_error_func(item_name, out)
                #
        known_props = KNOWN_PROPS.get(p_modname, {})
        a_setter = "lambda self, v: None"
        a_deleter = "lambda self: None"
        for item_name in sorted_no_case(properties.keys()):
            item = properties[item_name]
            prop_docstring = getattr(item, '__doc__', None)
            prop_key = (p_name, item_name)
            if prop_key in known_props:
                prop_descr = known_props.get(prop_key, None)
                if prop_descr is None:
                    continue # explicitly omitted
                acc_line, getter_and_type = prop_descr
                if getter_and_type:
                    getter, prop_type = getter_and_type
                else:
                    getter, prop_type = None, None
                out(indent + 1, item_name,
                    " = property(", format_accessors(acc_line, getter, a_setter, a_deleter), ")"
                )
                if prop_type:
                    if prop_docstring:
                        out(indent + 1, '"""', prop_docstring)
                        out(0, "")
                        out(indent + 1, ':type: ', prop_type)
                        out(indent + 1, '"""')
                    else:
                        out(indent + 1, '""":type: ', prop_type, '"""')
                    out(0, "")
            else:
                out(indent + 1, item_name, " = property(lambda self: object(), lambda self, v: None, lambda self: None)  # default")
                if prop_docstring:
                    out(indent + 1, '"""', prop_docstring, '"""')
                out(0, "")
        if properties:
            out(0, "") # empty line after the block
            #
        for item_name in sorted_no_case(others.keys()):
            item = others[item_name]
            self.fmt_value(out, item, indent + 1, prefix=item_name + " = ")
        if p_name == "object":
            out(indent + 1, "__module__ = ''")
        if others:
            out(0, "") # empty line after the block
            #
        if not methods and not properties and not others:
            out(indent + 1, "pass")



    def redo_simple_header(self, p_name):
        """Puts boilerplate code on the top"""
        out = self.header_buf.out # 1st class methods rule :)
        out(0, "# encoding: %s" % OUT_ENCODING) # line 1
        # NOTE: maybe encoding should be selectable
        if hasattr(self.module, "__name__"):
            self_name = self.module.__name__
            if self_name != p_name:
                mod_name = " calls itself " + self_name
            else:
                mod_name = ""
        else:
            mod_name = " does not know its name"
        out(0, "# module ", p_name, mod_name) # line 2

        BUILT_IN_HEADER = "(built-in)"
        if self.mod_filename:
            filename = self.mod_filename
        elif p_name in sys.builtin_module_names:
            filename = BUILT_IN_HEADER
        else:
            filename = getattr(self.module, "__file__", BUILT_IN_HEADER)

        out(0, "# from %s" % filename)  # line 3
        out(0, "# by generator %s" % VERSION) # line 4
        if p_name == BUILTIN_MOD_NAME and version[0] == 2 and version[1] >= 6:
            out(0, "from __future__ import print_function")
        out_doc_attr(out, self.module, 0)


    def redo_imports(self):
        module_type = type(sys)
        for item_name in self.module.__dict__.keys():
            try:
                item = self.module.__dict__[item_name]
            except:
                continue
            if type(item) is module_type: # not isinstance, py2.7 + PyQt4.QtCore on windows have a bug here
                self.imported_modules[item_name] = item
                self.add_import_header_if_needed()
                ref_notice = getattr(item, "__file__", str(item))
                if hasattr(item, "__name__"):
                    self.imports_buf.out(0, "import ", item.__name__, " as ", item_name, " # ", ref_notice)
                else:
                    self.imports_buf.out(0, item_name, " = None # ??? name unknown; ", ref_notice)

    def add_import_header_if_needed(self):
        if self.imports_buf.isEmpty():
            self.imports_buf.out(0, "")
            self.imports_buf.out(0, "# imports")


    def redo(self, p_name, inspect_dir):
        """
        Restores module declarations.
        Intended for built-in modules and thus does not handle import statements.
        @param p_name name of module
        """
        action("redoing header of module %r %r", p_name, str(self.module))

        if "pyqt4" in p_name.lower():   # qt4 specific patch
            self._initializeQApp4()
        elif "pyqt5" in p_name.lower():   # qt5 specific patch
            self._initializeQApp5()

        self.redo_simple_header(p_name)

        # find whatever other self.imported_modules the module knows; effectively these are imports
        action("redoing imports of module %r %r", p_name, str(self.module))
        try:
            self.redo_imports()
        except:
            pass

        action("redoing innards of module %r %r", p_name, str(self.module))

        module_type = type(sys)
        # group what we have into buckets
        vars_simple = {}
        vars_complex = {}
        funcs = {}
        classes = {}
        module_dict = self.module.__dict__
        if inspect_dir:
            module_dict = dir(self.module)
        for item_name in module_dict:
            note("looking at %s", item_name)
            if item_name in (
                "__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"):
                continue # handled otherwise
            try:
                item = getattr(self.module, item_name) # let getters do the magic
            except AttributeError:
                if not item_name in self.module.__dict__: continue
                item = self.module.__dict__[item_name] # have it raw
                # check if it has percolated from an imported module
            except NotImplementedError:
                if not item_name in self.module.__dict__: continue
                item = self.module.__dict__[item_name] # have it raw

            # unless we're adamantly positive that the name was imported, we assume it is defined here
            mod_name = None # module from which p_name might have been imported
            # IronPython has non-trivial reexports in System module, but not in others:
            skip_modname = sys.platform == "cli" and p_name != "System"
            surely_not_imported_mods = KNOWN_FAKE_REEXPORTERS.get(p_name, ())
            ## can't figure weirdness in some modules, assume no reexports:
            #skip_modname =  skip_modname or p_name in self.KNOWN_FAKE_REEXPORTERS
            if not skip_modname:
                try:
                    mod_name = getattr(item, '__module__', None)
                except:
                    pass
                    # we assume that module foo.bar never imports foo; foo may import foo.bar. (see pygame and pygame.rect)
            maybe_import_mod_name = mod_name or ""
            import_is_from_top = len(p_name) > len(maybe_import_mod_name) and p_name.startswith(maybe_import_mod_name)
            note("mod_name = %s, prospective = %s,  from top = %s", mod_name, maybe_import_mod_name, import_is_from_top)
            want_to_import = False
            if (mod_name
                and mod_name != BUILTIN_MOD_NAME
                and mod_name != p_name
                and mod_name not in surely_not_imported_mods
                and not import_is_from_top
            ):
                # import looks valid, but maybe it's a .py file? we're certain not to import from .py
                # e.g. this rules out _collections import collections and builtins import site.
                try:
                    imported = __import__(mod_name) # ok to repeat, Python caches for us
                    if imported:
                        qualifiers = mod_name.split(".")[1:]
                        for qual in qualifiers:
                            imported = getattr(imported, qual, None)
                            if not imported:
                                break
                        imported_path = (getattr(imported, '__file__', False) or "").lower()
                        want_to_import = not (imported_path.endswith('.py') or imported_path.endswith('.pyc'))
                        note("path of %r is %r, want? %s", mod_name, imported_path, want_to_import)
                except ImportError:
                    want_to_import = False
                    # NOTE: if we fail to import, we define 'imported' names here lest we lose them at all
                if want_to_import:
                    import_list = self.used_imports[mod_name]
                    if item_name not in import_list:
                        import_list.append(item_name)
            if not want_to_import:
                if isinstance(item, type) or type(item).__name__ == 'classobj':
                    classes[item_name] = item
                elif is_callable(item): # some classes are callable, check them before functions
                    funcs[item_name] = item
                elif isinstance(item, module_type):
                    continue # self.imported_modules handled above already
                else:
                    if isinstance(item, SIMPLEST_TYPES):
                        vars_simple[item_name] = item
                    else:
                        vars_complex[item_name] = item

        # sort and output every bucket
        action("outputting innards of module %r %r", p_name, str(self.module))
        #
        omitted_names = OMIT_NAME_IN_MODULE.get(p_name, [])
        if vars_simple:
            out = self.functions_buf.out
            prefix = "" # try to group variables by common prefix
            PREFIX_LEN = 2 # default prefix length if we can't guess better
            out(0, "# Variables with simple values")
            for item_name in sorted_no_case(vars_simple.keys()):
                if item_name in omitted_names:
                    out(0, "# definition of " + item_name + " omitted")
                    continue
                item = vars_simple[item_name]
                # track the prefix
                if len(item_name) >= PREFIX_LEN:
                    prefix_pos = string.rfind(item_name, "_") # most prefixes end in an underscore
                    if prefix_pos < 1:
                        prefix_pos = PREFIX_LEN
                    beg = item_name[0:prefix_pos]
                    if prefix != beg:
                        out(0, "") # space out from other prefix
                        prefix = beg
                else:
                    prefix = ""
                    # output
                replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None)
                if replacement is not None:
                    out(0, item_name, " = ", replacement, " # real value of type ", str(type(item)), " replaced")
                elif is_skipped_in_module(p_name, item_name):
                    t_item = type(item)
                    out(0, item_name, " = ", self.invent_initializer(t_item), " # real value of type ", str(t_item),
                        " skipped")
                else:
                    self.fmt_value(out, item, 0, prefix=item_name + " = ")
                self._defined[item_name] = True
            out(0, "") # empty line after vars
            #
        if funcs:
            out = self.functions_buf.out
            out(0, "# functions")
            out(0, "")
            seen_funcs = {}
            for item_name in sorted_no_case(funcs.keys()):
                if item_name in omitted_names:
                    out(0, "# definition of ", item_name, " omitted")
                    continue
                item = funcs[item_name]
                try:
                    self.redo_function(out, item, item_name, 0, p_modname=p_name, seen=seen_funcs)
                except:
                    handle_error_func(item_name, out)
        else:
            self.functions_buf.out(0, "# no functions")
            #
        if classes:
            self.classes_buf.out(0, "# classes")
            self.classes_buf.out(0, "")
            seen_classes = {}
            # sort classes so that inheritance order is preserved
            cls_list = [] # items are (class_name, mro_tuple)
            for cls_name in sorted_no_case(classes.keys()):
                cls = classes[cls_name]
                ins_index = len(cls_list)
                for i in range(ins_index):
                    maybe_child_bases = cls_list[i][1]
                    if cls in maybe_child_bases:
                        ins_index = i # we could not go farther than current ins_index
                        break         # ...and need not go fartehr than first known child
                cls_list.insert(ins_index, (cls_name, get_mro(cls)))
            self.split_modules = self.mod_filename and len(cls_list) >= 30
            for item_name in [cls_item[0] for cls_item in cls_list]:
                buf = ClassBuf(item_name, self)
                self.classes_buffs.append(buf)
                out = buf.out
                if item_name in omitted_names:
                    out(0, "# definition of ", item_name, " omitted")
                    continue
                item = classes[item_name]
                self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir)
                self._defined[item_name] = True
                out(0, "") # empty line after each item

            if self.doing_builtins and p_name == BUILTIN_MOD_NAME and version[0] < 3:
                # classobj still supported
                txt = classobj_txt
                self.classes_buf.out(0, txt)

            if self.doing_builtins and p_name == BUILTIN_MOD_NAME:
                txt = create_generator()
                self.classes_buf.out(0, txt)

                # Fake <type 'namedtuple'>
                if version[0] >= 3 or (version[0] == 2 and version[1] >= 6):
                    namedtuple_text = create_named_tuple()
                    self.classes_buf.out(0, namedtuple_text)

        else:
            self.classes_buf.out(0, "# no classes")
            #
        if vars_complex:
            out = self.footer_buf.out
            out(0, "# variables with complex values")
            out(0, "")
            for item_name in sorted_no_case(vars_complex.keys()):
                if item_name in omitted_names:
                    out(0, "# definition of " + item_name + " omitted")
                    continue
                item = vars_complex[item_name]
                if str(type(item)) == "<type 'namespace#'>":
                    continue            # this is an IronPython submodule, we mustn't generate a reference for it in the base module
                replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None)
                if replacement is not None:
                    out(0, item_name + " = " + replacement + " # real value of type " + str(type(item)) + " replaced")
                elif is_skipped_in_module(p_name, item_name):
                    t_item = type(item)
                    out(0, item_name + " = " + self.invent_initializer(t_item) + " # real value of type " + str(
                        t_item) + " skipped")
                else:
                    self.fmt_value(out, item, 0, prefix=item_name + " = ", as_name=item_name)
                self._defined[item_name] = True
                out(0, "") # empty line after each item
        values_to_add = ADD_VALUE_IN_MODULE.get(p_name, None)
        if values_to_add:
            self.footer_buf.out(0, "# intermittent names")
            for value in values_to_add:
                self.footer_buf.out(0, value)
                # imports: last, because previous parts could alter used_imports or hidden_imports
        self.output_import_froms()
        if self.imports_buf.isEmpty():
            self.imports_buf.out(0, "# no imports")
        self.imports_buf.out(0, "") # empty line after imports

    def output_import_froms(self):
        """Mention all imported names known within the module, wrapping as per PEP."""
        out = self.imports_buf.out
        if self.used_imports:
            self.add_import_header_if_needed()
            for mod_name in sorted_no_case(self.used_imports.keys()):
                import_names = self.used_imports[mod_name]
                if import_names:
                    self._defined[mod_name] = True
                    right_pos = 0 # tracks width of list to fold it at right margin
                    import_heading = "from % s import (" % mod_name
                    right_pos += len(import_heading)
                    names_pack = [import_heading]
                    indent_level = 0
                    import_names = list(import_names)
                    import_names.sort()
                    for n in import_names:
                        self._defined[n] = True
                        len_n = len(n)
                        if right_pos + len_n >= 78:
                            out(indent_level, *names_pack)
                            names_pack = [n, ", "]
                            if indent_level == 0:
                                indent_level = 1 # all but first line is indented
                            right_pos = self.indent_size + len_n + 2
                        else:
                            names_pack.append(n)
                            names_pack.append(", ")
                            right_pos += (len_n + 2)
                            # last line is...
                    if indent_level == 0: # one line
                        names_pack[0] = names_pack[0][:-1] # cut off lpar
                        names_pack[-1] = "" # cut last comma
                    else: # last line of multiline
                        names_pack[-1] = ")" # last comma -> rpar
                    out(indent_level, *names_pack)

                    out(0, "") # empty line after group

        if self.hidden_imports:
            self.add_import_header_if_needed()
            for mod_name in sorted_no_case(self.hidden_imports.keys()):
                out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name])
            out(0, "") # empty line after group


def module_to_package_name(module_name):
    return re.sub(r"(.*)\.py$", r"\1", module_name)