jpayne@68: """Utility code for constructing importers, etc."""
jpayne@68: from . import abc
jpayne@68: from ._bootstrap import module_from_spec
jpayne@68: from ._bootstrap import _resolve_name
jpayne@68: from ._bootstrap import spec_from_loader
jpayne@68: from ._bootstrap import _find_spec
jpayne@68: from ._bootstrap_external import MAGIC_NUMBER
jpayne@68: from ._bootstrap_external import _RAW_MAGIC_NUMBER
jpayne@68: from ._bootstrap_external import cache_from_source
jpayne@68: from ._bootstrap_external import decode_source
jpayne@68: from ._bootstrap_external import source_from_cache
jpayne@68: from ._bootstrap_external import spec_from_file_location
jpayne@68: 
jpayne@68: from contextlib import contextmanager
jpayne@68: import _imp
jpayne@68: import functools
jpayne@68: import sys
jpayne@68: import types
jpayne@68: import warnings
jpayne@68: 
jpayne@68: 
jpayne@68: def source_hash(source_bytes):
jpayne@68:     "Return the hash of *source_bytes* as used in hash-based pyc files."
jpayne@68:     return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes)
jpayne@68: 
jpayne@68: 
jpayne@68: def resolve_name(name, package):
jpayne@68:     """Resolve a relative module name to an absolute one."""
jpayne@68:     if not name.startswith('.'):
jpayne@68:         return name
jpayne@68:     elif not package:
jpayne@68:         raise ValueError(f'no package specified for {repr(name)} '
jpayne@68:                          '(required for relative module names)')
jpayne@68:     level = 0
jpayne@68:     for character in name:
jpayne@68:         if character != '.':
jpayne@68:             break
jpayne@68:         level += 1
jpayne@68:     return _resolve_name(name[level:], package, level)
jpayne@68: 
jpayne@68: 
jpayne@68: def _find_spec_from_path(name, path=None):
jpayne@68:     """Return the spec for the specified module.
jpayne@68: 
jpayne@68:     First, sys.modules is checked to see if the module was already imported. If
jpayne@68:     so, then sys.modules[name].__spec__ is returned. If that happens to be
jpayne@68:     set to None, then ValueError is raised. If the module is not in
jpayne@68:     sys.modules, then sys.meta_path is searched for a suitable spec with the
jpayne@68:     value of 'path' given to the finders. None is returned if no spec could
jpayne@68:     be found.
jpayne@68: 
jpayne@68:     Dotted names do not have their parent packages implicitly imported. You will
jpayne@68:     most likely need to explicitly import all parent packages in the proper
jpayne@68:     order for a submodule to get the correct spec.
jpayne@68: 
jpayne@68:     """
jpayne@68:     if name not in sys.modules:
jpayne@68:         return _find_spec(name, path)
jpayne@68:     else:
jpayne@68:         module = sys.modules[name]
jpayne@68:         if module is None:
jpayne@68:             return None
jpayne@68:         try:
jpayne@68:             spec = module.__spec__
jpayne@68:         except AttributeError:
jpayne@68:             raise ValueError('{}.__spec__ is not set'.format(name)) from None
jpayne@68:         else:
jpayne@68:             if spec is None:
jpayne@68:                 raise ValueError('{}.__spec__ is None'.format(name))
jpayne@68:             return spec
jpayne@68: 
jpayne@68: 
jpayne@68: def find_spec(name, package=None):
jpayne@68:     """Return the spec for the specified module.
jpayne@68: 
jpayne@68:     First, sys.modules is checked to see if the module was already imported. If
jpayne@68:     so, then sys.modules[name].__spec__ is returned. If that happens to be
jpayne@68:     set to None, then ValueError is raised. If the module is not in
jpayne@68:     sys.modules, then sys.meta_path is searched for a suitable spec with the
jpayne@68:     value of 'path' given to the finders. None is returned if no spec could
jpayne@68:     be found.
jpayne@68: 
jpayne@68:     If the name is for submodule (contains a dot), the parent module is
jpayne@68:     automatically imported.
jpayne@68: 
jpayne@68:     The name and package arguments work the same as importlib.import_module().
jpayne@68:     In other words, relative module names (with leading dots) work.
jpayne@68: 
jpayne@68:     """
jpayne@68:     fullname = resolve_name(name, package) if name.startswith('.') else name
jpayne@68:     if fullname not in sys.modules:
jpayne@68:         parent_name = fullname.rpartition('.')[0]
jpayne@68:         if parent_name:
jpayne@68:             parent = __import__(parent_name, fromlist=['__path__'])
jpayne@68:             try:
jpayne@68:                 parent_path = parent.__path__
jpayne@68:             except AttributeError as e:
jpayne@68:                 raise ModuleNotFoundError(
jpayne@68:                     f"__path__ attribute not found on {parent_name!r} "
jpayne@68:                     f"while trying to find {fullname!r}", name=fullname) from e
jpayne@68:         else:
jpayne@68:             parent_path = None
jpayne@68:         return _find_spec(fullname, parent_path)
jpayne@68:     else:
jpayne@68:         module = sys.modules[fullname]
jpayne@68:         if module is None:
jpayne@68:             return None
jpayne@68:         try:
jpayne@68:             spec = module.__spec__
jpayne@68:         except AttributeError:
jpayne@68:             raise ValueError('{}.__spec__ is not set'.format(name)) from None
jpayne@68:         else:
jpayne@68:             if spec is None:
jpayne@68:                 raise ValueError('{}.__spec__ is None'.format(name))
jpayne@68:             return spec
jpayne@68: 
jpayne@68: 
jpayne@68: @contextmanager
jpayne@68: def _module_to_load(name):
jpayne@68:     is_reload = name in sys.modules
jpayne@68: 
jpayne@68:     module = sys.modules.get(name)
jpayne@68:     if not is_reload:
jpayne@68:         # This must be done before open() is called as the 'io' module
jpayne@68:         # implicitly imports 'locale' and would otherwise trigger an
jpayne@68:         # infinite loop.
jpayne@68:         module = type(sys)(name)
jpayne@68:         # This must be done before putting the module in sys.modules
jpayne@68:         # (otherwise an optimization shortcut in import.c becomes wrong)
jpayne@68:         module.__initializing__ = True
jpayne@68:         sys.modules[name] = module
jpayne@68:     try:
jpayne@68:         yield module
jpayne@68:     except Exception:
jpayne@68:         if not is_reload:
jpayne@68:             try:
jpayne@68:                 del sys.modules[name]
jpayne@68:             except KeyError:
jpayne@68:                 pass
jpayne@68:     finally:
jpayne@68:         module.__initializing__ = False
jpayne@68: 
jpayne@68: 
jpayne@68: def set_package(fxn):
jpayne@68:     """Set __package__ on the returned module.
jpayne@68: 
jpayne@68:     This function is deprecated.
jpayne@68: 
jpayne@68:     """
jpayne@68:     @functools.wraps(fxn)
jpayne@68:     def set_package_wrapper(*args, **kwargs):
jpayne@68:         warnings.warn('The import system now takes care of this automatically.',
jpayne@68:                       DeprecationWarning, stacklevel=2)
jpayne@68:         module = fxn(*args, **kwargs)
jpayne@68:         if getattr(module, '__package__', None) is None:
jpayne@68:             module.__package__ = module.__name__
jpayne@68:             if not hasattr(module, '__path__'):
jpayne@68:                 module.__package__ = module.__package__.rpartition('.')[0]
jpayne@68:         return module
jpayne@68:     return set_package_wrapper
jpayne@68: 
jpayne@68: 
jpayne@68: def set_loader(fxn):
jpayne@68:     """Set __loader__ on the returned module.
jpayne@68: 
jpayne@68:     This function is deprecated.
jpayne@68: 
jpayne@68:     """
jpayne@68:     @functools.wraps(fxn)
jpayne@68:     def set_loader_wrapper(self, *args, **kwargs):
jpayne@68:         warnings.warn('The import system now takes care of this automatically.',
jpayne@68:                       DeprecationWarning, stacklevel=2)
jpayne@68:         module = fxn(self, *args, **kwargs)
jpayne@68:         if getattr(module, '__loader__', None) is None:
jpayne@68:             module.__loader__ = self
jpayne@68:         return module
jpayne@68:     return set_loader_wrapper
jpayne@68: 
jpayne@68: 
jpayne@68: def module_for_loader(fxn):
jpayne@68:     """Decorator to handle selecting the proper module for loaders.
jpayne@68: 
jpayne@68:     The decorated function is passed the module to use instead of the module
jpayne@68:     name. The module passed in to the function is either from sys.modules if
jpayne@68:     it already exists or is a new module. If the module is new, then __name__
jpayne@68:     is set the first argument to the method, __loader__ is set to self, and
jpayne@68:     __package__ is set accordingly (if self.is_package() is defined) will be set
jpayne@68:     before it is passed to the decorated function (if self.is_package() does
jpayne@68:     not work for the module it will be set post-load).
jpayne@68: 
jpayne@68:     If an exception is raised and the decorator created the module it is
jpayne@68:     subsequently removed from sys.modules.
jpayne@68: 
jpayne@68:     The decorator assumes that the decorated function takes the module name as
jpayne@68:     the second argument.
jpayne@68: 
jpayne@68:     """
jpayne@68:     warnings.warn('The import system now takes care of this automatically.',
jpayne@68:                   DeprecationWarning, stacklevel=2)
jpayne@68:     @functools.wraps(fxn)
jpayne@68:     def module_for_loader_wrapper(self, fullname, *args, **kwargs):
jpayne@68:         with _module_to_load(fullname) as module:
jpayne@68:             module.__loader__ = self
jpayne@68:             try:
jpayne@68:                 is_package = self.is_package(fullname)
jpayne@68:             except (ImportError, AttributeError):
jpayne@68:                 pass
jpayne@68:             else:
jpayne@68:                 if is_package:
jpayne@68:                     module.__package__ = fullname
jpayne@68:                 else:
jpayne@68:                     module.__package__ = fullname.rpartition('.')[0]
jpayne@68:             # If __package__ was not set above, __import__() will do it later.
jpayne@68:             return fxn(self, module, *args, **kwargs)
jpayne@68: 
jpayne@68:     return module_for_loader_wrapper
jpayne@68: 
jpayne@68: 
jpayne@68: class _LazyModule(types.ModuleType):
jpayne@68: 
jpayne@68:     """A subclass of the module type which triggers loading upon attribute access."""
jpayne@68: 
jpayne@68:     def __getattribute__(self, attr):
jpayne@68:         """Trigger the load of the module and return the attribute."""
jpayne@68:         # All module metadata must be garnered from __spec__ in order to avoid
jpayne@68:         # using mutated values.
jpayne@68:         # Stop triggering this method.
jpayne@68:         self.__class__ = types.ModuleType
jpayne@68:         # Get the original name to make sure no object substitution occurred
jpayne@68:         # in sys.modules.
jpayne@68:         original_name = self.__spec__.name
jpayne@68:         # Figure out exactly what attributes were mutated between the creation
jpayne@68:         # of the module and now.
jpayne@68:         attrs_then = self.__spec__.loader_state['__dict__']
jpayne@68:         original_type = self.__spec__.loader_state['__class__']
jpayne@68:         attrs_now = self.__dict__
jpayne@68:         attrs_updated = {}
jpayne@68:         for key, value in attrs_now.items():
jpayne@68:             # Code that set the attribute may have kept a reference to the
jpayne@68:             # assigned object, making identity more important than equality.
jpayne@68:             if key not in attrs_then:
jpayne@68:                 attrs_updated[key] = value
jpayne@68:             elif id(attrs_now[key]) != id(attrs_then[key]):
jpayne@68:                 attrs_updated[key] = value
jpayne@68:         self.__spec__.loader.exec_module(self)
jpayne@68:         # If exec_module() was used directly there is no guarantee the module
jpayne@68:         # object was put into sys.modules.
jpayne@68:         if original_name in sys.modules:
jpayne@68:             if id(self) != id(sys.modules[original_name]):
jpayne@68:                 raise ValueError(f"module object for {original_name!r} "
jpayne@68:                                   "substituted in sys.modules during a lazy "
jpayne@68:                                   "load")
jpayne@68:         # Update after loading since that's what would happen in an eager
jpayne@68:         # loading situation.
jpayne@68:         self.__dict__.update(attrs_updated)
jpayne@68:         return getattr(self, attr)
jpayne@68: 
jpayne@68:     def __delattr__(self, attr):
jpayne@68:         """Trigger the load and then perform the deletion."""
jpayne@68:         # To trigger the load and raise an exception if the attribute
jpayne@68:         # doesn't exist.
jpayne@68:         self.__getattribute__(attr)
jpayne@68:         delattr(self, attr)
jpayne@68: 
jpayne@68: 
jpayne@68: class LazyLoader(abc.Loader):
jpayne@68: 
jpayne@68:     """A loader that creates a module which defers loading until attribute access."""
jpayne@68: 
jpayne@68:     @staticmethod
jpayne@68:     def __check_eager_loader(loader):
jpayne@68:         if not hasattr(loader, 'exec_module'):
jpayne@68:             raise TypeError('loader must define exec_module()')
jpayne@68: 
jpayne@68:     @classmethod
jpayne@68:     def factory(cls, loader):
jpayne@68:         """Construct a callable which returns the eager loader made lazy."""
jpayne@68:         cls.__check_eager_loader(loader)
jpayne@68:         return lambda *args, **kwargs: cls(loader(*args, **kwargs))
jpayne@68: 
jpayne@68:     def __init__(self, loader):
jpayne@68:         self.__check_eager_loader(loader)
jpayne@68:         self.loader = loader
jpayne@68: 
jpayne@68:     def create_module(self, spec):
jpayne@68:         return self.loader.create_module(spec)
jpayne@68: 
jpayne@68:     def exec_module(self, module):
jpayne@68:         """Make the module load lazily."""
jpayne@68:         module.__spec__.loader = self.loader
jpayne@68:         module.__loader__ = self.loader
jpayne@68:         # Don't need to worry about deep-copying as trying to set an attribute
jpayne@68:         # on an object would have triggered the load,
jpayne@68:         # e.g. ``module.__spec__.loader = None`` would trigger a load from
jpayne@68:         # trying to access module.__spec__.
jpayne@68:         loader_state = {}
jpayne@68:         loader_state['__dict__'] = module.__dict__.copy()
jpayne@68:         loader_state['__class__'] = module.__class__
jpayne@68:         module.__spec__.loader_state = loader_state
jpayne@68:         module.__class__ = _LazyModule