diff options
Diffstat (limited to 'lib/python2.7/imputil.py')
| -rw-r--r-- | lib/python2.7/imputil.py | 725 |
1 files changed, 725 insertions, 0 deletions
diff --git a/lib/python2.7/imputil.py b/lib/python2.7/imputil.py new file mode 100644 index 0000000..a5fa6ea --- /dev/null +++ b/lib/python2.7/imputil.py @@ -0,0 +1,725 @@ +""" +Import utilities + +Exported classes: + ImportManager Manage the import process + + Importer Base class for replacing standard import functions + BuiltinImporter Emulate the import mechanism for builtin and frozen modules + + DynLoadSuffixImporter +""" +from warnings import warnpy3k +warnpy3k("the imputil module has been removed in Python 3.0", stacklevel=2) +del warnpy3k + +# note: avoid importing non-builtin modules +import imp ### not available in Jython? +import sys +import __builtin__ + +# for the DirectoryImporter +import struct +import marshal + +__all__ = ["ImportManager","Importer","BuiltinImporter"] + +_StringType = type('') +_ModuleType = type(sys) ### doesn't work in Jython... + +class ImportManager: + "Manage the import process." + + def install(self, namespace=vars(__builtin__)): + "Install this ImportManager into the specified namespace." + + if isinstance(namespace, _ModuleType): + namespace = vars(namespace) + + # Note: we have no notion of "chaining" + + # Record the previous import hook, then install our own. + self.previous_importer = namespace['__import__'] + self.namespace = namespace + namespace['__import__'] = self._import_hook + + ### fix this + #namespace['reload'] = self._reload_hook + + def uninstall(self): + "Restore the previous import mechanism." + self.namespace['__import__'] = self.previous_importer + + def add_suffix(self, suffix, importFunc): + assert hasattr(importFunc, '__call__') + self.fs_imp.add_suffix(suffix, importFunc) + + ###################################################################### + # + # PRIVATE METHODS + # + + clsFilesystemImporter = None + + def __init__(self, fs_imp=None): + # we're definitely going to be importing something in the future, + # so let's just load the OS-related facilities. + if not _os_stat: + _os_bootstrap() + + # This is the Importer that we use for grabbing stuff from the + # filesystem. It defines one more method (import_from_dir) for our use. + if fs_imp is None: + cls = self.clsFilesystemImporter or _FilesystemImporter + fs_imp = cls() + self.fs_imp = fs_imp + + # Initialize the set of suffixes that we recognize and import. + # The default will import dynamic-load modules first, followed by + # .py files (or a .py file's cached bytecode) + for desc in imp.get_suffixes(): + if desc[2] == imp.C_EXTENSION: + self.add_suffix(desc[0], + DynLoadSuffixImporter(desc).import_file) + self.add_suffix('.py', py_suffix_importer) + + def _import_hook(self, fqname, globals=None, locals=None, fromlist=None): + """Python calls this hook to locate and import a module.""" + + parts = fqname.split('.') + + # determine the context of this import + parent = self._determine_import_context(globals) + + # if there is a parent, then its importer should manage this import + if parent: + module = parent.__importer__._do_import(parent, parts, fromlist) + if module: + return module + + # has the top module already been imported? + try: + top_module = sys.modules[parts[0]] + except KeyError: + + # look for the topmost module + top_module = self._import_top_module(parts[0]) + if not top_module: + # the topmost module wasn't found at all. + raise ImportError, 'No module named ' + fqname + + # fast-path simple imports + if len(parts) == 1: + if not fromlist: + return top_module + + if not top_module.__dict__.get('__ispkg__'): + # __ispkg__ isn't defined (the module was not imported by us), + # or it is zero. + # + # In the former case, there is no way that we could import + # sub-modules that occur in the fromlist (but we can't raise an + # error because it may just be names) because we don't know how + # to deal with packages that were imported by other systems. + # + # In the latter case (__ispkg__ == 0), there can't be any sub- + # modules present, so we can just return. + # + # In both cases, since len(parts) == 1, the top_module is also + # the "bottom" which is the defined return when a fromlist + # exists. + return top_module + + importer = top_module.__dict__.get('__importer__') + if importer: + return importer._finish_import(top_module, parts[1:], fromlist) + + # Grrr, some people "import os.path" or do "from os.path import ..." + if len(parts) == 2 and hasattr(top_module, parts[1]): + if fromlist: + return getattr(top_module, parts[1]) + else: + return top_module + + # If the importer does not exist, then we have to bail. A missing + # importer means that something else imported the module, and we have + # no knowledge of how to get sub-modules out of the thing. + raise ImportError, 'No module named ' + fqname + + def _determine_import_context(self, globals): + """Returns the context in which a module should be imported. + + The context could be a loaded (package) module and the imported module + will be looked for within that package. The context could also be None, + meaning there is no context -- the module should be looked for as a + "top-level" module. + """ + + if not globals or not globals.get('__importer__'): + # globals does not refer to one of our modules or packages. That + # implies there is no relative import context (as far as we are + # concerned), and it should just pick it off the standard path. + return None + + # The globals refer to a module or package of ours. It will define + # the context of the new import. Get the module/package fqname. + parent_fqname = globals['__name__'] + + # if a package is performing the import, then return itself (imports + # refer to pkg contents) + if globals['__ispkg__']: + parent = sys.modules[parent_fqname] + assert globals is parent.__dict__ + return parent + + i = parent_fqname.rfind('.') + + # a module outside of a package has no particular import context + if i == -1: + return None + + # if a module in a package is performing the import, then return the + # package (imports refer to siblings) + parent_fqname = parent_fqname[:i] + parent = sys.modules[parent_fqname] + assert parent.__name__ == parent_fqname + return parent + + def _import_top_module(self, name): + # scan sys.path looking for a location in the filesystem that contains + # the module, or an Importer object that can import the module. + for item in sys.path: + if isinstance(item, _StringType): + module = self.fs_imp.import_from_dir(item, name) + else: + module = item.import_top(name) + if module: + return module + return None + + def _reload_hook(self, module): + "Python calls this hook to reload a module." + + # reloading of a module may or may not be possible (depending on the + # importer), but at least we can validate that it's ours to reload + importer = module.__dict__.get('__importer__') + if not importer: + ### oops. now what... + pass + + # okay. it is using the imputil system, and we must delegate it, but + # we don't know what to do (yet) + ### we should blast the module dict and do another get_code(). need to + ### flesh this out and add proper docco... + raise SystemError, "reload not yet implemented" + + +class Importer: + "Base class for replacing standard import functions." + + def import_top(self, name): + "Import a top-level module." + return self._import_one(None, name, name) + + ###################################################################### + # + # PRIVATE METHODS + # + def _finish_import(self, top, parts, fromlist): + # if "a.b.c" was provided, then load the ".b.c" portion down from + # below the top-level module. + bottom = self._load_tail(top, parts) + + # if the form is "import a.b.c", then return "a" + if not fromlist: + # no fromlist: return the top of the import tree + return top + + # the top module was imported by self. + # + # this means that the bottom module was also imported by self (just + # now, or in the past and we fetched it from sys.modules). + # + # since we imported/handled the bottom module, this means that we can + # also handle its fromlist (and reliably use __ispkg__). + + # if the bottom node is a package, then (potentially) import some + # modules. + # + # note: if it is not a package, then "fromlist" refers to names in + # the bottom module rather than modules. + # note: for a mix of names and modules in the fromlist, we will + # import all modules and insert those into the namespace of + # the package module. Python will pick up all fromlist names + # from the bottom (package) module; some will be modules that + # we imported and stored in the namespace, others are expected + # to be present already. + if bottom.__ispkg__: + self._import_fromlist(bottom, fromlist) + + # if the form is "from a.b import c, d" then return "b" + return bottom + + def _import_one(self, parent, modname, fqname): + "Import a single module." + + # has the module already been imported? + try: + return sys.modules[fqname] + except KeyError: + pass + + # load the module's code, or fetch the module itself + result = self.get_code(parent, modname, fqname) + if result is None: + return None + + module = self._process_result(result, fqname) + + # insert the module into its parent + if parent: + setattr(parent, modname, module) + return module + + def _process_result(self, result, fqname): + ispkg, code, values = result + # did get_code() return an actual module? (rather than a code object) + is_module = isinstance(code, _ModuleType) + + # use the returned module, or create a new one to exec code into + if is_module: + module = code + else: + module = imp.new_module(fqname) + + ### record packages a bit differently?? + module.__importer__ = self + module.__ispkg__ = ispkg + + # insert additional values into the module (before executing the code) + module.__dict__.update(values) + + # the module is almost ready... make it visible + sys.modules[fqname] = module + + # execute the code within the module's namespace + if not is_module: + try: + exec code in module.__dict__ + except: + if fqname in sys.modules: + del sys.modules[fqname] + raise + + # fetch from sys.modules instead of returning module directly. + # also make module's __name__ agree with fqname, in case + # the "exec code in module.__dict__" played games on us. + module = sys.modules[fqname] + module.__name__ = fqname + return module + + def _load_tail(self, m, parts): + """Import the rest of the modules, down from the top-level module. + + Returns the last module in the dotted list of modules. + """ + for part in parts: + fqname = "%s.%s" % (m.__name__, part) + m = self._import_one(m, part, fqname) + if not m: + raise ImportError, "No module named " + fqname + return m + + def _import_fromlist(self, package, fromlist): + 'Import any sub-modules in the "from" list.' + + # if '*' is present in the fromlist, then look for the '__all__' + # variable to find additional items (modules) to import. + if '*' in fromlist: + fromlist = list(fromlist) + \ + list(package.__dict__.get('__all__', [])) + + for sub in fromlist: + # if the name is already present, then don't try to import it (it + # might not be a module!). + if sub != '*' and not hasattr(package, sub): + subname = "%s.%s" % (package.__name__, sub) + submod = self._import_one(package, sub, subname) + if not submod: + raise ImportError, "cannot import name " + subname + + def _do_import(self, parent, parts, fromlist): + """Attempt to import the module relative to parent. + + This method is used when the import context specifies that <self> + imported the parent module. + """ + top_name = parts[0] + top_fqname = parent.__name__ + '.' + top_name + top_module = self._import_one(parent, top_name, top_fqname) + if not top_module: + # this importer and parent could not find the module (relatively) + return None + + return self._finish_import(top_module, parts[1:], fromlist) + + ###################################################################### + # + # METHODS TO OVERRIDE + # + def get_code(self, parent, modname, fqname): + """Find and retrieve the code for the given module. + + parent specifies a parent module to define a context for importing. It + may be None, indicating no particular context for the search. + + modname specifies a single module (not dotted) within the parent. + + fqname specifies the fully-qualified module name. This is a + (potentially) dotted name from the "root" of the module namespace + down to the modname. + If there is no parent, then modname==fqname. + + This method should return None, or a 3-tuple. + + * If the module was not found, then None should be returned. + + * The first item of the 2- or 3-tuple should be the integer 0 or 1, + specifying whether the module that was found is a package or not. + + * The second item is the code object for the module (it will be + executed within the new module's namespace). This item can also + be a fully-loaded module object (e.g. loaded from a shared lib). + + * The third item is a dictionary of name/value pairs that will be + inserted into new module before the code object is executed. This + is provided in case the module's code expects certain values (such + as where the module was found). When the second item is a module + object, then these names/values will be inserted *after* the module + has been loaded/initialized. + """ + raise RuntimeError, "get_code not implemented" + + +###################################################################### +# +# Some handy stuff for the Importers +# + +# byte-compiled file suffix character +_suffix_char = __debug__ and 'c' or 'o' + +# byte-compiled file suffix +_suffix = '.py' + _suffix_char + +def _compile(pathname, timestamp): + """Compile (and cache) a Python source file. + + The file specified by <pathname> is compiled to a code object and + returned. + + Presuming the appropriate privileges exist, the bytecodes will be + saved back to the filesystem for future imports. The source file's + modification timestamp must be provided as a Long value. + """ + codestring = open(pathname, 'rU').read() + if codestring and codestring[-1] != '\n': + codestring = codestring + '\n' + code = __builtin__.compile(codestring, pathname, 'exec') + + # try to cache the compiled code + try: + f = open(pathname + _suffix_char, 'wb') + except IOError: + pass + else: + f.write('\0\0\0\0') + f.write(struct.pack('<I', timestamp)) + marshal.dump(code, f) + f.flush() + f.seek(0, 0) + f.write(imp.get_magic()) + f.close() + + return code + +_os_stat = _os_path_join = None +def _os_bootstrap(): + "Set up 'os' module replacement functions for use during import bootstrap." + + names = sys.builtin_module_names + + join = None + if 'posix' in names: + sep = '/' + from posix import stat + elif 'nt' in names: + sep = '\\' + from nt import stat + elif 'dos' in names: + sep = '\\' + from dos import stat + elif 'os2' in names: + sep = '\\' + from os2 import stat + else: + raise ImportError, 'no os specific module found' + + if join is None: + def join(a, b, sep=sep): + if a == '': + return b + lastchar = a[-1:] + if lastchar == '/' or lastchar == sep: + return a + b + return a + sep + b + + global _os_stat + _os_stat = stat + + global _os_path_join + _os_path_join = join + +def _os_path_isdir(pathname): + "Local replacement for os.path.isdir()." + try: + s = _os_stat(pathname) + except OSError: + return None + return (s.st_mode & 0170000) == 0040000 + +def _timestamp(pathname): + "Return the file modification time as a Long." + try: + s = _os_stat(pathname) + except OSError: + return None + return long(s.st_mtime) + + +###################################################################### +# +# Emulate the import mechanism for builtin and frozen modules +# +class BuiltinImporter(Importer): + def get_code(self, parent, modname, fqname): + if parent: + # these modules definitely do not occur within a package context + return None + + # look for the module + if imp.is_builtin(modname): + type = imp.C_BUILTIN + elif imp.is_frozen(modname): + type = imp.PY_FROZEN + else: + # not found + return None + + # got it. now load and return it. + module = imp.load_module(modname, None, modname, ('', '', type)) + return 0, module, { } + + +###################################################################### +# +# Internal importer used for importing from the filesystem +# +class _FilesystemImporter(Importer): + def __init__(self): + self.suffixes = [ ] + + def add_suffix(self, suffix, importFunc): + assert hasattr(importFunc, '__call__') + self.suffixes.append((suffix, importFunc)) + + def import_from_dir(self, dir, fqname): + result = self._import_pathname(_os_path_join(dir, fqname), fqname) + if result: + return self._process_result(result, fqname) + return None + + def get_code(self, parent, modname, fqname): + # This importer is never used with an empty parent. Its existence is + # private to the ImportManager. The ImportManager uses the + # import_from_dir() method to import top-level modules/packages. + # This method is only used when we look for a module within a package. + assert parent + + for submodule_path in parent.__path__: + code = self._import_pathname(_os_path_join(submodule_path, modname), fqname) + if code is not None: + return code + return self._import_pathname(_os_path_join(parent.__pkgdir__, modname), + fqname) + + def _import_pathname(self, pathname, fqname): + if _os_path_isdir(pathname): + result = self._import_pathname(_os_path_join(pathname, '__init__'), + fqname) + if result: + values = result[2] + values['__pkgdir__'] = pathname + values['__path__'] = [ pathname ] + return 1, result[1], values + return None + + for suffix, importFunc in self.suffixes: + filename = pathname + suffix + try: + finfo = _os_stat(filename) + except OSError: + pass + else: + return importFunc(filename, finfo, fqname) + return None + +###################################################################### +# +# SUFFIX-BASED IMPORTERS +# + +def py_suffix_importer(filename, finfo, fqname): + file = filename[:-3] + _suffix + t_py = long(finfo[8]) + t_pyc = _timestamp(file) + + code = None + if t_pyc is not None and t_pyc >= t_py: + f = open(file, 'rb') + if f.read(4) == imp.get_magic(): + t = struct.unpack('<I', f.read(4))[0] + if t == t_py: + code = marshal.load(f) + f.close() + if code is None: + file = filename + code = _compile(file, t_py) + + return 0, code, { '__file__' : file } + +class DynLoadSuffixImporter: + def __init__(self, desc): + self.desc = desc + + def import_file(self, filename, finfo, fqname): + fp = open(filename, self.desc[1]) + module = imp.load_module(fqname, fp, filename, self.desc) + module.__file__ = filename + return 0, module, { } + + +###################################################################### + +def _print_importers(): + items = sys.modules.items() + items.sort() + for name, module in items: + if module: + print name, module.__dict__.get('__importer__', '-- no importer') + else: + print name, '-- non-existent module' + +def _test_revamp(): + ImportManager().install() + sys.path.insert(0, BuiltinImporter()) + +###################################################################### + +# +# TODO +# +# from Finn Bock: +# type(sys) is not a module in Jython. what to use instead? +# imp.C_EXTENSION is not in Jython. same for get_suffixes and new_module +# +# given foo.py of: +# import sys +# sys.modules['foo'] = sys +# +# ---- standard import mechanism +# >>> import foo +# >>> foo +# <module 'sys' (built-in)> +# +# ---- revamped import mechanism +# >>> import imputil +# >>> imputil._test_revamp() +# >>> import foo +# >>> foo +# <module 'foo' from 'foo.py'> +# +# +# from MAL: +# should BuiltinImporter exist in sys.path or hard-wired in ImportManager? +# need __path__ processing +# performance +# move chaining to a subclass [gjs: it's been nuked] +# deinstall should be possible +# query mechanism needed: is a specific Importer installed? +# py/pyc/pyo piping hooks to filter/process these files +# wish list: +# distutils importer hooked to list of standard Internet repositories +# module->file location mapper to speed FS-based imports +# relative imports +# keep chaining so that it can play nice with other import hooks +# +# from Gordon: +# push MAL's mapper into sys.path[0] as a cache (hard-coded for apps) +# +# from Guido: +# need to change sys.* references for rexec environs +# need hook for MAL's walk-me-up import strategy, or Tim's absolute strategy +# watch out for sys.modules[...] is None +# flag to force absolute imports? (speeds _determine_import_context and +# checking for a relative module) +# insert names of archives into sys.path (see quote below) +# note: reload does NOT blast module dict +# shift import mechanisms and policies around; provide for hooks, overrides +# (see quote below) +# add get_source stuff +# get_topcode and get_subcode +# CRLF handling in _compile +# race condition in _compile +# refactoring of os.py to deal with _os_bootstrap problem +# any special handling to do for importing a module with a SyntaxError? +# (e.g. clean up the traceback) +# implement "domain" for path-type functionality using pkg namespace +# (rather than FS-names like __path__) +# don't use the word "private"... maybe "internal" +# +# +# Guido's comments on sys.path caching: +# +# We could cache this in a dictionary: the ImportManager can have a +# cache dict mapping pathnames to importer objects, and a separate +# method for coming up with an importer given a pathname that's not yet +# in the cache. The method should do a stat and/or look at the +# extension to decide which importer class to use; you can register new +# importer classes by registering a suffix or a Boolean function, plus a +# class. If you register a new importer class, the cache is zapped. +# The cache is independent from sys.path (but maintained per +# ImportManager instance) so that rearrangements of sys.path do the +# right thing. If a path is dropped from sys.path the corresponding +# cache entry is simply no longer used. +# +# My/Guido's comments on factoring ImportManager and Importer: +# +# > However, we still have a tension occurring here: +# > +# > 1) implementing policy in ImportManager assists in single-point policy +# > changes for app/rexec situations +# > 2) implementing policy in Importer assists in package-private policy +# > changes for normal, operating conditions +# > +# > I'll see if I can sort out a way to do this. Maybe the Importer class will +# > implement the methods (which can be overridden to change policy) by +# > delegating to ImportManager. +# +# Maybe also think about what kind of policies an Importer would be +# likely to want to change. I have a feeling that a lot of the code +# there is actually not so much policy but a *necessity* to get things +# working given the calling conventions for the __import__ hook: whether +# to return the head or tail of a dotted name, or when to do the "finish +# fromlist" stuff. +# |