"""Utilities to support packages."""
from collections import namedtuple
from functools import singledispatch as simplegeneric
import importlib.machinery
from types import ModuleType
'get_importer', 'iter_importers', 'get_loader', 'find_loader',
'walk_packages', 'iter_modules', 'get_data',
'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
ModuleInfo = namedtuple('ModuleInfo', 'module_finder name ispkg')
ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.'
def _get_spec(finder, name):
"""Return the finder-specific module spec."""
# Works with legacy finders.
find_spec = finder.find_spec
loader = finder.find_module(name)
return importlib.util.spec_from_loader(name, loader)
# This helper is needed in order for the PEP 302 emulation to
# correctly handle compiled files
if magic != importlib.util.MAGIC_NUMBER:
stream.read(8) # Skip timestamp and size
return marshal.load(stream)
def walk_packages(path=None, prefix='', onerror=None):
"""Yields ModuleInfo for all modules recursively
on path, or, if path is None, all accessible modules.
'path' should be either None or a list of paths to look for
'prefix' is a string to output on the front of every module name
Note that this function must import all *packages* (NOT all
modules!) on the given path, in order to access the __path__
attribute to find submodules.
'onerror' is a function which gets called with one argument (the
name of the package which was being imported) if any exception
occurs while trying to import a package. If no onerror function is
supplied, ImportErrors are caught and ignored, while all other
exceptions are propagated, terminating the search.
# list all modules python can access
# list all submodules of ctypes
walk_packages(ctypes.__path__, ctypes.__name__+'.')
for info in iter_modules(path, prefix):
path = getattr(sys.modules[info.name], '__path__', None) or []
# don't traverse path items we've seen before
path = [p for p in path if not seen(p)]
yield from walk_packages(path, info.name+'.', onerror)
def iter_modules(path=None, prefix=''):
"""Yields ModuleInfo for all submodules on path,
or, if path is None, all top-level modules on sys.path.
'path' should be either None or a list of paths to look for
'prefix' is a string to output on the front of every module name
importers = iter_importers()
importers = map(get_importer, path)
for name, ispkg in iter_importer_modules(i, prefix):
yield ModuleInfo(i, name, ispkg)
def iter_importer_modules(importer, prefix=''):
if not hasattr(importer, 'iter_modules'):
return importer.iter_modules(prefix)
# Implement a file walker for the normal importlib path hook
def _iter_file_finder_modules(importer, prefix=''):
if importer.path is None or not os.path.isdir(importer.path):
filenames = os.listdir(importer.path)
# ignore unreadable directories like import does
filenames.sort() # handle packages before same-named modules
modname = inspect.getmodulename(fn)
if modname=='__init__' or modname in yielded:
path = os.path.join(importer.path, fn)
if not modname and os.path.isdir(path) and '.' not in fn:
dircontents = os.listdir(path)
# ignore unreadable directories like import does
subname = inspect.getmodulename(fn)
if modname and '.' not in modname:
yield prefix + modname, ispkg
iter_importer_modules.register(
importlib.machinery.FileFinder, _iter_file_finder_modules)
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
imp = importlib.import_module('imp')
"""PEP 302 Finder that wraps Python's "classic" import algorithm
ImpImporter(dirname) produces a PEP 302 finder that searches that
directory. ImpImporter(None) produces a PEP 302 finder that searches
the current sys.path, plus any modules that are frozen or built-in.
Note that ImpImporter does not currently support being used by placement
def __init__(self, path=None):
warnings.warn("This emulation is deprecated, use 'importlib' instead",
def find_module(self, fullname, path=None):
# Note: we ignore 'path' argument since it is only used via meta_path
subname = fullname.split(".")[-1]
if subname != fullname and self.path is None:
path = [os.path.realpath(self.path)]
file, filename, etc = imp.find_module(subname, path)
return ImpLoader(fullname, file, filename, etc)
def iter_modules(self, prefix=''):
if self.path is None or not os.path.isdir(self.path):
filenames = os.listdir(self.path)
# ignore unreadable directories like import does
filenames.sort() # handle packages before same-named modules
modname = inspect.getmodulename(fn)
if modname=='__init__' or modname in yielded:
path = os.path.join(self.path, fn)
if not modname and os.path.isdir(path) and '.' not in fn:
dircontents = os.listdir(path)
# ignore unreadable directories like import does
subname = inspect.getmodulename(fn)
if modname and '.' not in modname:
yield prefix + modname, ispkg
"""PEP 302 Loader that wraps Python's "classic" import algorithm
def __init__(self, fullname, file, filename, etc):
warnings.warn("This emulation is deprecated, use 'importlib' instead",
def load_module(self, fullname):
mod = imp.load_module(fullname, self.file, self.filename, self.etc)
# Note: we don't set __loader__ because we want the module to look
# normal; i.e. this is just a wrapper for standard import machinery
def get_data(self, pathname):
with open(pathname, "rb") as file:
if self.file and self.file.closed:
if mod_type==imp.PY_SOURCE:
self.file = open(self.filename, 'r')
elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
self.file = open(self.filename, 'rb')
def _fix_name(self, fullname):
elif fullname != self.fullname:
raise ImportError("Loader for module %s cannot handle "
"module %s" % (self.fullname, fullname))
def is_package(self, fullname):
fullname = self._fix_name(fullname)
return self.etc[2]==imp.PKG_DIRECTORY
def get_code(self, fullname=None):
fullname = self._fix_name(fullname)
if mod_type==imp.PY_SOURCE:
source = self.get_source(fullname)
self.code = compile(source, self.filename, 'exec')
elif mod_type==imp.PY_COMPILED:
self.code = read_code(self.file)
elif mod_type==imp.PKG_DIRECTORY:
self.code = self._get_delegate().get_code()
def get_source(self, fullname=None):
fullname = self._fix_name(fullname)
if mod_type==imp.PY_SOURCE:
self.source = self.file.read()
elif mod_type==imp.PY_COMPILED:
if os.path.exists(self.filename[:-1]):
with open(self.filename[:-1], 'r') as f:
elif mod_type==imp.PKG_DIRECTORY:
self.source = self._get_delegate().get_source()
finder = ImpImporter(self.filename)
spec = _get_spec(finder, '__init__')
def get_filename(self, fullname=None):
fullname = self._fix_name(fullname)
if mod_type==imp.PKG_DIRECTORY:
return self._get_delegate().get_filename()
elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
from zipimport import zipimporter
def iter_zipimport_modules(importer, prefix=''):
dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
_prefix = importer.prefix
if not fn.startswith(_prefix):
fn = fn[plen:].split(os.sep)
if len(fn)==2 and fn[1].startswith('__init__.py'):
yield prefix + fn[0], True
modname = inspect.getmodulename(fn[0])
if modname and '.' not in modname and modname not in yielded:
yield prefix + modname, False
iter_importer_modules.register(zipimporter, iter_zipimport_modules)
def get_importer(path_item):
"""Retrieve a finder for the given path item
The returned finder is cached in sys.path_importer_cache
if it was newly created by a path hook.
The cache (or part of it) can be cleared manually if a
rescan of sys.path_hooks is necessary.
importer = sys.path_importer_cache[path_item]
for path_hook in sys.path_hooks:
importer = path_hook(path_item)
sys.path_importer_cache.setdefault(path_item, importer)
def iter_importers(fullname=""):
"""Yield finders for the given module name
If fullname contains a '.', the finders will be for the package
containing fullname, otherwise they will be all registered top level
finders (i.e. those on both sys.meta_path and sys.path_hooks).
If the named module is in a package, that package is imported as a side
effect of invoking this function.
If no module name is specified, all top level finders are produced.
if fullname.startswith('.'):
msg = "Relative module name {!r} not supported".format(fullname)
# Get the containing package's __path__
pkg_name = fullname.rpartition(".")[0]
pkg = importlib.import_module(pkg_name)
path = getattr(pkg, '__path__', None)
def get_loader(module_or_name):
"""Get a "loader" object for module_or_name
Returns None if the module cannot be found or imported.
If the named module is not already imported, its containing package
(if any) is imported, in order to establish the package __path__.
if module_or_name in sys.modules:
module_or_name = sys.modules[module_or_name]
if module_or_name is None:
if isinstance(module_or_name, ModuleType):
loader = getattr(module, '__loader__', None)
if getattr(module, '__spec__', None) is None:
fullname = module.__name__
fullname = module_or_name
return find_loader(fullname)
def find_loader(fullname):
"""Find a "loader" object for fullname
This is a backwards compatibility wrapper around
importlib.util.find_spec that converts most failures to ImportError
and only returns the loader rather than the full spec
if fullname.startswith('.'):
msg = "Relative module name {!r} not supported".format(fullname)
spec = importlib.util.find_spec(fullname)
except (ImportError, AttributeError, TypeError, ValueError) as ex:
# This hack fixes an impedance mismatch between pkgutil and
# importlib, where the latter raises other errors for cases where
# pkgutil previously raised ImportError
msg = "Error while finding loader for {!r} ({}: {})"
raise ImportError(msg.format(fullname, type(ex), ex)) from ex
return spec.loader if spec is not None else None
def extend_path(path, name):
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
