jpayne@68: # Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
jpayne@68: #
jpayne@68: # Permission to use, copy, modify, and distribute this software and its
jpayne@68: # documentation for any purpose and without fee is hereby granted,
jpayne@68: # provided that the above copyright notice appear in all copies and that
jpayne@68: # both that copyright notice and this permission notice appear in
jpayne@68: # supporting documentation, and that the name of Vinay Sajip
jpayne@68: # not be used in advertising or publicity pertaining to distribution
jpayne@68: # of the software without specific, written prior permission.
jpayne@68: # VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
jpayne@68: # ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
jpayne@68: # VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
jpayne@68: # ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
jpayne@68: # IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
jpayne@68: # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
jpayne@68: 
jpayne@68: """
jpayne@68: Configuration functions for the logging package for Python. The core package
jpayne@68: is based on PEP 282 and comments thereto in comp.lang.python, and influenced
jpayne@68: by Apache's log4j system.
jpayne@68: 
jpayne@68: Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
jpayne@68: 
jpayne@68: To use, simply 'import logging' and log away!
jpayne@68: """
jpayne@68: 
jpayne@68: import errno
jpayne@68: import io
jpayne@68: import logging
jpayne@68: import logging.handlers
jpayne@68: import re
jpayne@68: import struct
jpayne@68: import sys
jpayne@68: import threading
jpayne@68: import traceback
jpayne@68: 
jpayne@68: from socketserver import ThreadingTCPServer, StreamRequestHandler
jpayne@68: 
jpayne@68: 
jpayne@68: DEFAULT_LOGGING_CONFIG_PORT = 9030
jpayne@68: 
jpayne@68: RESET_ERROR = errno.ECONNRESET
jpayne@68: 
jpayne@68: #
jpayne@68: #   The following code implements a socket listener for on-the-fly
jpayne@68: #   reconfiguration of logging.
jpayne@68: #
jpayne@68: #   _listener holds the server object doing the listening
jpayne@68: _listener = None
jpayne@68: 
jpayne@68: def fileConfig(fname, defaults=None, disable_existing_loggers=True):
jpayne@68:     """
jpayne@68:     Read the logging configuration from a ConfigParser-format file.
jpayne@68: 
jpayne@68:     This can be called several times from an application, allowing an end user
jpayne@68:     the ability to select from various pre-canned configurations (if the
jpayne@68:     developer provides a mechanism to present the choices and load the chosen
jpayne@68:     configuration).
jpayne@68:     """
jpayne@68:     import configparser
jpayne@68: 
jpayne@68:     if isinstance(fname, configparser.RawConfigParser):
jpayne@68:         cp = fname
jpayne@68:     else:
jpayne@68:         cp = configparser.ConfigParser(defaults)
jpayne@68:         if hasattr(fname, 'readline'):
jpayne@68:             cp.read_file(fname)
jpayne@68:         else:
jpayne@68:             cp.read(fname)
jpayne@68: 
jpayne@68:     formatters = _create_formatters(cp)
jpayne@68: 
jpayne@68:     # critical section
jpayne@68:     logging._acquireLock()
jpayne@68:     try:
jpayne@68:         _clearExistingHandlers()
jpayne@68: 
jpayne@68:         # Handlers add themselves to logging._handlers
jpayne@68:         handlers = _install_handlers(cp, formatters)
jpayne@68:         _install_loggers(cp, handlers, disable_existing_loggers)
jpayne@68:     finally:
jpayne@68:         logging._releaseLock()
jpayne@68: 
jpayne@68: 
jpayne@68: def _resolve(name):
jpayne@68:     """Resolve a dotted name to a global object."""
jpayne@68:     name = name.split('.')
jpayne@68:     used = name.pop(0)
jpayne@68:     found = __import__(used)
jpayne@68:     for n in name:
jpayne@68:         used = used + '.' + n
jpayne@68:         try:
jpayne@68:             found = getattr(found, n)
jpayne@68:         except AttributeError:
jpayne@68:             __import__(used)
jpayne@68:             found = getattr(found, n)
jpayne@68:     return found
jpayne@68: 
jpayne@68: def _strip_spaces(alist):
jpayne@68:     return map(str.strip, alist)
jpayne@68: 
jpayne@68: def _create_formatters(cp):
jpayne@68:     """Create and return formatters"""
jpayne@68:     flist = cp["formatters"]["keys"]
jpayne@68:     if not len(flist):
jpayne@68:         return {}
jpayne@68:     flist = flist.split(",")
jpayne@68:     flist = _strip_spaces(flist)
jpayne@68:     formatters = {}
jpayne@68:     for form in flist:
jpayne@68:         sectname = "formatter_%s" % form
jpayne@68:         fs = cp.get(sectname, "format", raw=True, fallback=None)
jpayne@68:         dfs = cp.get(sectname, "datefmt", raw=True, fallback=None)
jpayne@68:         stl = cp.get(sectname, "style", raw=True, fallback='%')
jpayne@68:         c = logging.Formatter
jpayne@68:         class_name = cp[sectname].get("class")
jpayne@68:         if class_name:
jpayne@68:             c = _resolve(class_name)
jpayne@68:         f = c(fs, dfs, stl)
jpayne@68:         formatters[form] = f
jpayne@68:     return formatters
jpayne@68: 
jpayne@68: 
jpayne@68: def _install_handlers(cp, formatters):
jpayne@68:     """Install and return handlers"""
jpayne@68:     hlist = cp["handlers"]["keys"]
jpayne@68:     if not len(hlist):
jpayne@68:         return {}
jpayne@68:     hlist = hlist.split(",")
jpayne@68:     hlist = _strip_spaces(hlist)
jpayne@68:     handlers = {}
jpayne@68:     fixups = [] #for inter-handler references
jpayne@68:     for hand in hlist:
jpayne@68:         section = cp["handler_%s" % hand]
jpayne@68:         klass = section["class"]
jpayne@68:         fmt = section.get("formatter", "")
jpayne@68:         try:
jpayne@68:             klass = eval(klass, vars(logging))
jpayne@68:         except (AttributeError, NameError):
jpayne@68:             klass = _resolve(klass)
jpayne@68:         args = section.get("args", '()')
jpayne@68:         args = eval(args, vars(logging))
jpayne@68:         kwargs = section.get("kwargs", '{}')
jpayne@68:         kwargs = eval(kwargs, vars(logging))
jpayne@68:         h = klass(*args, **kwargs)
jpayne@68:         if "level" in section:
jpayne@68:             level = section["level"]
jpayne@68:             h.setLevel(level)
jpayne@68:         if len(fmt):
jpayne@68:             h.setFormatter(formatters[fmt])
jpayne@68:         if issubclass(klass, logging.handlers.MemoryHandler):
jpayne@68:             target = section.get("target", "")
jpayne@68:             if len(target): #the target handler may not be loaded yet, so keep for later...
jpayne@68:                 fixups.append((h, target))
jpayne@68:         handlers[hand] = h
jpayne@68:     #now all handlers are loaded, fixup inter-handler references...
jpayne@68:     for h, t in fixups:
jpayne@68:         h.setTarget(handlers[t])
jpayne@68:     return handlers
jpayne@68: 
jpayne@68: def _handle_existing_loggers(existing, child_loggers, disable_existing):
jpayne@68:     """
jpayne@68:     When (re)configuring logging, handle loggers which were in the previous
jpayne@68:     configuration but are not in the new configuration. There's no point
jpayne@68:     deleting them as other threads may continue to hold references to them;
jpayne@68:     and by disabling them, you stop them doing any logging.
jpayne@68: 
jpayne@68:     However, don't disable children of named loggers, as that's probably not
jpayne@68:     what was intended by the user. Also, allow existing loggers to NOT be
jpayne@68:     disabled if disable_existing is false.
jpayne@68:     """
jpayne@68:     root = logging.root
jpayne@68:     for log in existing:
jpayne@68:         logger = root.manager.loggerDict[log]
jpayne@68:         if log in child_loggers:
jpayne@68:             if not isinstance(logger, logging.PlaceHolder):
jpayne@68:                 logger.setLevel(logging.NOTSET)
jpayne@68:                 logger.handlers = []
jpayne@68:                 logger.propagate = True
jpayne@68:         else:
jpayne@68:             logger.disabled = disable_existing
jpayne@68: 
jpayne@68: def _install_loggers(cp, handlers, disable_existing):
jpayne@68:     """Create and install loggers"""
jpayne@68: 
jpayne@68:     # configure the root first
jpayne@68:     llist = cp["loggers"]["keys"]
jpayne@68:     llist = llist.split(",")
jpayne@68:     llist = list(_strip_spaces(llist))
jpayne@68:     llist.remove("root")
jpayne@68:     section = cp["logger_root"]
jpayne@68:     root = logging.root
jpayne@68:     log = root
jpayne@68:     if "level" in section:
jpayne@68:         level = section["level"]
jpayne@68:         log.setLevel(level)
jpayne@68:     for h in root.handlers[:]:
jpayne@68:         root.removeHandler(h)
jpayne@68:     hlist = section["handlers"]
jpayne@68:     if len(hlist):
jpayne@68:         hlist = hlist.split(",")
jpayne@68:         hlist = _strip_spaces(hlist)
jpayne@68:         for hand in hlist:
jpayne@68:             log.addHandler(handlers[hand])
jpayne@68: 
jpayne@68:     #and now the others...
jpayne@68:     #we don't want to lose the existing loggers,
jpayne@68:     #since other threads may have pointers to them.
jpayne@68:     #existing is set to contain all existing loggers,
jpayne@68:     #and as we go through the new configuration we
jpayne@68:     #remove any which are configured. At the end,
jpayne@68:     #what's left in existing is the set of loggers
jpayne@68:     #which were in the previous configuration but
jpayne@68:     #which are not in the new configuration.
jpayne@68:     existing = list(root.manager.loggerDict.keys())
jpayne@68:     #The list needs to be sorted so that we can
jpayne@68:     #avoid disabling child loggers of explicitly
jpayne@68:     #named loggers. With a sorted list it is easier
jpayne@68:     #to find the child loggers.
jpayne@68:     existing.sort()
jpayne@68:     #We'll keep the list of existing loggers
jpayne@68:     #which are children of named loggers here...
jpayne@68:     child_loggers = []
jpayne@68:     #now set up the new ones...
jpayne@68:     for log in llist:
jpayne@68:         section = cp["logger_%s" % log]
jpayne@68:         qn = section["qualname"]
jpayne@68:         propagate = section.getint("propagate", fallback=1)
jpayne@68:         logger = logging.getLogger(qn)
jpayne@68:         if qn in existing:
jpayne@68:             i = existing.index(qn) + 1 # start with the entry after qn
jpayne@68:             prefixed = qn + "."
jpayne@68:             pflen = len(prefixed)
jpayne@68:             num_existing = len(existing)
jpayne@68:             while i < num_existing:
jpayne@68:                 if existing[i][:pflen] == prefixed:
jpayne@68:                     child_loggers.append(existing[i])
jpayne@68:                 i += 1
jpayne@68:             existing.remove(qn)
jpayne@68:         if "level" in section:
jpayne@68:             level = section["level"]
jpayne@68:             logger.setLevel(level)
jpayne@68:         for h in logger.handlers[:]:
jpayne@68:             logger.removeHandler(h)
jpayne@68:         logger.propagate = propagate
jpayne@68:         logger.disabled = 0
jpayne@68:         hlist = section["handlers"]
jpayne@68:         if len(hlist):
jpayne@68:             hlist = hlist.split(",")
jpayne@68:             hlist = _strip_spaces(hlist)
jpayne@68:             for hand in hlist:
jpayne@68:                 logger.addHandler(handlers[hand])
jpayne@68: 
jpayne@68:     #Disable any old loggers. There's no point deleting
jpayne@68:     #them as other threads may continue to hold references
jpayne@68:     #and by disabling them, you stop them doing any logging.
jpayne@68:     #However, don't disable children of named loggers, as that's
jpayne@68:     #probably not what was intended by the user.
jpayne@68:     #for log in existing:
jpayne@68:     #    logger = root.manager.loggerDict[log]
jpayne@68:     #    if log in child_loggers:
jpayne@68:     #        logger.level = logging.NOTSET
jpayne@68:     #        logger.handlers = []
jpayne@68:     #        logger.propagate = 1
jpayne@68:     #    elif disable_existing_loggers:
jpayne@68:     #        logger.disabled = 1
jpayne@68:     _handle_existing_loggers(existing, child_loggers, disable_existing)
jpayne@68: 
jpayne@68: 
jpayne@68: def _clearExistingHandlers():
jpayne@68:     """Clear and close existing handlers"""
jpayne@68:     logging._handlers.clear()
jpayne@68:     logging.shutdown(logging._handlerList[:])
jpayne@68:     del logging._handlerList[:]
jpayne@68: 
jpayne@68: 
jpayne@68: IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)
jpayne@68: 
jpayne@68: 
jpayne@68: def valid_ident(s):
jpayne@68:     m = IDENTIFIER.match(s)
jpayne@68:     if not m:
jpayne@68:         raise ValueError('Not a valid Python identifier: %r' % s)
jpayne@68:     return True
jpayne@68: 
jpayne@68: 
jpayne@68: class ConvertingMixin(object):
jpayne@68:     """For ConvertingXXX's, this mixin class provides common functions"""
jpayne@68: 
jpayne@68:     def convert_with_key(self, key, value, replace=True):
jpayne@68:         result = self.configurator.convert(value)
jpayne@68:         #If the converted value is different, save for next time
jpayne@68:         if value is not result:
jpayne@68:             if replace:
jpayne@68:                 self[key] = result
jpayne@68:             if type(result) in (ConvertingDict, ConvertingList,
jpayne@68:                                ConvertingTuple):
jpayne@68:                 result.parent = self
jpayne@68:                 result.key = key
jpayne@68:         return result
jpayne@68: 
jpayne@68:     def convert(self, value):
jpayne@68:         result = self.configurator.convert(value)
jpayne@68:         if value is not result:
jpayne@68:             if type(result) in (ConvertingDict, ConvertingList,
jpayne@68:                                ConvertingTuple):
jpayne@68:                 result.parent = self
jpayne@68:         return result
jpayne@68: 
jpayne@68: 
jpayne@68: # The ConvertingXXX classes are wrappers around standard Python containers,
jpayne@68: # and they serve to convert any suitable values in the container. The
jpayne@68: # conversion converts base dicts, lists and tuples to their wrapped
jpayne@68: # equivalents, whereas strings which match a conversion format are converted
jpayne@68: # appropriately.
jpayne@68: #
jpayne@68: # Each wrapper should have a configurator attribute holding the actual
jpayne@68: # configurator to use for conversion.
jpayne@68: 
jpayne@68: class ConvertingDict(dict, ConvertingMixin):
jpayne@68:     """A converting dictionary wrapper."""
jpayne@68: 
jpayne@68:     def __getitem__(self, key):
jpayne@68:         value = dict.__getitem__(self, key)
jpayne@68:         return self.convert_with_key(key, value)
jpayne@68: 
jpayne@68:     def get(self, key, default=None):
jpayne@68:         value = dict.get(self, key, default)
jpayne@68:         return self.convert_with_key(key, value)
jpayne@68: 
jpayne@68:     def pop(self, key, default=None):
jpayne@68:         value = dict.pop(self, key, default)
jpayne@68:         return self.convert_with_key(key, value, replace=False)
jpayne@68: 
jpayne@68: class ConvertingList(list, ConvertingMixin):
jpayne@68:     """A converting list wrapper."""
jpayne@68:     def __getitem__(self, key):
jpayne@68:         value = list.__getitem__(self, key)
jpayne@68:         return self.convert_with_key(key, value)
jpayne@68: 
jpayne@68:     def pop(self, idx=-1):
jpayne@68:         value = list.pop(self, idx)
jpayne@68:         return self.convert(value)
jpayne@68: 
jpayne@68: class ConvertingTuple(tuple, ConvertingMixin):
jpayne@68:     """A converting tuple wrapper."""
jpayne@68:     def __getitem__(self, key):
jpayne@68:         value = tuple.__getitem__(self, key)
jpayne@68:         # Can't replace a tuple entry.
jpayne@68:         return self.convert_with_key(key, value, replace=False)
jpayne@68: 
jpayne@68: class BaseConfigurator(object):
jpayne@68:     """
jpayne@68:     The configurator base class which defines some useful defaults.
jpayne@68:     """
jpayne@68: 
jpayne@68:     CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$')
jpayne@68: 
jpayne@68:     WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
jpayne@68:     DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
jpayne@68:     INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
jpayne@68:     DIGIT_PATTERN = re.compile(r'^\d+$')
jpayne@68: 
jpayne@68:     value_converters = {
jpayne@68:         'ext' : 'ext_convert',
jpayne@68:         'cfg' : 'cfg_convert',
jpayne@68:     }
jpayne@68: 
jpayne@68:     # We might want to use a different one, e.g. importlib
jpayne@68:     importer = staticmethod(__import__)
jpayne@68: 
jpayne@68:     def __init__(self, config):
jpayne@68:         self.config = ConvertingDict(config)
jpayne@68:         self.config.configurator = self
jpayne@68: 
jpayne@68:     def resolve(self, s):
jpayne@68:         """
jpayne@68:         Resolve strings to objects using standard import and attribute
jpayne@68:         syntax.
jpayne@68:         """
jpayne@68:         name = s.split('.')
jpayne@68:         used = name.pop(0)
jpayne@68:         try:
jpayne@68:             found = self.importer(used)
jpayne@68:             for frag in name:
jpayne@68:                 used += '.' + frag
jpayne@68:                 try:
jpayne@68:                     found = getattr(found, frag)
jpayne@68:                 except AttributeError:
jpayne@68:                     self.importer(used)
jpayne@68:                     found = getattr(found, frag)
jpayne@68:             return found
jpayne@68:         except ImportError:
jpayne@68:             e, tb = sys.exc_info()[1:]
jpayne@68:             v = ValueError('Cannot resolve %r: %s' % (s, e))
jpayne@68:             v.__cause__, v.__traceback__ = e, tb
jpayne@68:             raise v
jpayne@68: 
jpayne@68:     def ext_convert(self, value):
jpayne@68:         """Default converter for the ext:// protocol."""
jpayne@68:         return self.resolve(value)
jpayne@68: 
jpayne@68:     def cfg_convert(self, value):
jpayne@68:         """Default converter for the cfg:// protocol."""
jpayne@68:         rest = value
jpayne@68:         m = self.WORD_PATTERN.match(rest)
jpayne@68:         if m is None:
jpayne@68:             raise ValueError("Unable to convert %r" % value)
jpayne@68:         else:
jpayne@68:             rest = rest[m.end():]
jpayne@68:             d = self.config[m.groups()[0]]
jpayne@68:             #print d, rest
jpayne@68:             while rest:
jpayne@68:                 m = self.DOT_PATTERN.match(rest)
jpayne@68:                 if m:
jpayne@68:                     d = d[m.groups()[0]]
jpayne@68:                 else:
jpayne@68:                     m = self.INDEX_PATTERN.match(rest)
jpayne@68:                     if m:
jpayne@68:                         idx = m.groups()[0]
jpayne@68:                         if not self.DIGIT_PATTERN.match(idx):
jpayne@68:                             d = d[idx]
jpayne@68:                         else:
jpayne@68:                             try:
jpayne@68:                                 n = int(idx) # try as number first (most likely)
jpayne@68:                                 d = d[n]
jpayne@68:                             except TypeError:
jpayne@68:                                 d = d[idx]
jpayne@68:                 if m:
jpayne@68:                     rest = rest[m.end():]
jpayne@68:                 else:
jpayne@68:                     raise ValueError('Unable to convert '
jpayne@68:                                      '%r at %r' % (value, rest))
jpayne@68:         #rest should be empty
jpayne@68:         return d
jpayne@68: 
jpayne@68:     def convert(self, value):
jpayne@68:         """
jpayne@68:         Convert values to an appropriate type. dicts, lists and tuples are
jpayne@68:         replaced by their converting alternatives. Strings are checked to
jpayne@68:         see if they have a conversion format and are converted if they do.
jpayne@68:         """
jpayne@68:         if not isinstance(value, ConvertingDict) and isinstance(value, dict):
jpayne@68:             value = ConvertingDict(value)
jpayne@68:             value.configurator = self
jpayne@68:         elif not isinstance(value, ConvertingList) and isinstance(value, list):
jpayne@68:             value = ConvertingList(value)
jpayne@68:             value.configurator = self
jpayne@68:         elif not isinstance(value, ConvertingTuple) and\
jpayne@68:                  isinstance(value, tuple):
jpayne@68:             value = ConvertingTuple(value)
jpayne@68:             value.configurator = self
jpayne@68:         elif isinstance(value, str): # str for py3k
jpayne@68:             m = self.CONVERT_PATTERN.match(value)
jpayne@68:             if m:
jpayne@68:                 d = m.groupdict()
jpayne@68:                 prefix = d['prefix']
jpayne@68:                 converter = self.value_converters.get(prefix, None)
jpayne@68:                 if converter:
jpayne@68:                     suffix = d['suffix']
jpayne@68:                     converter = getattr(self, converter)
jpayne@68:                     value = converter(suffix)
jpayne@68:         return value
jpayne@68: 
jpayne@68:     def configure_custom(self, config):
jpayne@68:         """Configure an object with a user-supplied factory."""
jpayne@68:         c = config.pop('()')
jpayne@68:         if not callable(c):
jpayne@68:             c = self.resolve(c)
jpayne@68:         props = config.pop('.', None)
jpayne@68:         # Check for valid identifiers
jpayne@68:         kwargs = {k: config[k] for k in config if valid_ident(k)}
jpayne@68:         result = c(**kwargs)
jpayne@68:         if props:
jpayne@68:             for name, value in props.items():
jpayne@68:                 setattr(result, name, value)
jpayne@68:         return result
jpayne@68: 
jpayne@68:     def as_tuple(self, value):
jpayne@68:         """Utility function which converts lists to tuples."""
jpayne@68:         if isinstance(value, list):
jpayne@68:             value = tuple(value)
jpayne@68:         return value
jpayne@68: 
jpayne@68: class DictConfigurator(BaseConfigurator):
jpayne@68:     """
jpayne@68:     Configure logging using a dictionary-like object to describe the
jpayne@68:     configuration.
jpayne@68:     """
jpayne@68: 
jpayne@68:     def configure(self):
jpayne@68:         """Do the configuration."""
jpayne@68: 
jpayne@68:         config = self.config
jpayne@68:         if 'version' not in config:
jpayne@68:             raise ValueError("dictionary doesn't specify a version")
jpayne@68:         if config['version'] != 1:
jpayne@68:             raise ValueError("Unsupported version: %s" % config['version'])
jpayne@68:         incremental = config.pop('incremental', False)
jpayne@68:         EMPTY_DICT = {}
jpayne@68:         logging._acquireLock()
jpayne@68:         try:
jpayne@68:             if incremental:
jpayne@68:                 handlers = config.get('handlers', EMPTY_DICT)
jpayne@68:                 for name in handlers:
jpayne@68:                     if name not in logging._handlers:
jpayne@68:                         raise ValueError('No handler found with '
jpayne@68:                                          'name %r'  % name)
jpayne@68:                     else:
jpayne@68:                         try:
jpayne@68:                             handler = logging._handlers[name]
jpayne@68:                             handler_config = handlers[name]
jpayne@68:                             level = handler_config.get('level', None)
jpayne@68:                             if level:
jpayne@68:                                 handler.setLevel(logging._checkLevel(level))
jpayne@68:                         except Exception as e:
jpayne@68:                             raise ValueError('Unable to configure handler '
jpayne@68:                                              '%r' % name) from e
jpayne@68:                 loggers = config.get('loggers', EMPTY_DICT)
jpayne@68:                 for name in loggers:
jpayne@68:                     try:
jpayne@68:                         self.configure_logger(name, loggers[name], True)
jpayne@68:                     except Exception as e:
jpayne@68:                         raise ValueError('Unable to configure logger '
jpayne@68:                                          '%r' % name) from e
jpayne@68:                 root = config.get('root', None)
jpayne@68:                 if root:
jpayne@68:                     try:
jpayne@68:                         self.configure_root(root, True)
jpayne@68:                     except Exception as e:
jpayne@68:                         raise ValueError('Unable to configure root '
jpayne@68:                                          'logger') from e
jpayne@68:             else:
jpayne@68:                 disable_existing = config.pop('disable_existing_loggers', True)
jpayne@68: 
jpayne@68:                 _clearExistingHandlers()
jpayne@68: 
jpayne@68:                 # Do formatters first - they don't refer to anything else
jpayne@68:                 formatters = config.get('formatters', EMPTY_DICT)
jpayne@68:                 for name in formatters:
jpayne@68:                     try:
jpayne@68:                         formatters[name] = self.configure_formatter(
jpayne@68:                                                             formatters[name])
jpayne@68:                     except Exception as e:
jpayne@68:                         raise ValueError('Unable to configure '
jpayne@68:                                          'formatter %r' % name) from e
jpayne@68:                 # Next, do filters - they don't refer to anything else, either
jpayne@68:                 filters = config.get('filters', EMPTY_DICT)
jpayne@68:                 for name in filters:
jpayne@68:                     try:
jpayne@68:                         filters[name] = self.configure_filter(filters[name])
jpayne@68:                     except Exception as e:
jpayne@68:                         raise ValueError('Unable to configure '
jpayne@68:                                          'filter %r' % name) from e
jpayne@68: 
jpayne@68:                 # Next, do handlers - they refer to formatters and filters
jpayne@68:                 # As handlers can refer to other handlers, sort the keys
jpayne@68:                 # to allow a deterministic order of configuration
jpayne@68:                 handlers = config.get('handlers', EMPTY_DICT)
jpayne@68:                 deferred = []
jpayne@68:                 for name in sorted(handlers):
jpayne@68:                     try:
jpayne@68:                         handler = self.configure_handler(handlers[name])
jpayne@68:                         handler.name = name
jpayne@68:                         handlers[name] = handler
jpayne@68:                     except Exception as e:
jpayne@68:                         if 'target not configured yet' in str(e.__cause__):
jpayne@68:                             deferred.append(name)
jpayne@68:                         else:
jpayne@68:                             raise ValueError('Unable to configure handler '
jpayne@68:                                              '%r' % name) from e
jpayne@68: 
jpayne@68:                 # Now do any that were deferred
jpayne@68:                 for name in deferred:
jpayne@68:                     try:
jpayne@68:                         handler = self.configure_handler(handlers[name])
jpayne@68:                         handler.name = name
jpayne@68:                         handlers[name] = handler
jpayne@68:                     except Exception as e:
jpayne@68:                         raise ValueError('Unable to configure handler '
jpayne@68:                                          '%r' % name) from e
jpayne@68: 
jpayne@68:                 # Next, do loggers - they refer to handlers and filters
jpayne@68: 
jpayne@68:                 #we don't want to lose the existing loggers,
jpayne@68:                 #since other threads may have pointers to them.
jpayne@68:                 #existing is set to contain all existing loggers,
jpayne@68:                 #and as we go through the new configuration we
jpayne@68:                 #remove any which are configured. At the end,
jpayne@68:                 #what's left in existing is the set of loggers
jpayne@68:                 #which were in the previous configuration but
jpayne@68:                 #which are not in the new configuration.
jpayne@68:                 root = logging.root
jpayne@68:                 existing = list(root.manager.loggerDict.keys())
jpayne@68:                 #The list needs to be sorted so that we can
jpayne@68:                 #avoid disabling child loggers of explicitly
jpayne@68:                 #named loggers. With a sorted list it is easier
jpayne@68:                 #to find the child loggers.
jpayne@68:                 existing.sort()
jpayne@68:                 #We'll keep the list of existing loggers
jpayne@68:                 #which are children of named loggers here...
jpayne@68:                 child_loggers = []
jpayne@68:                 #now set up the new ones...
jpayne@68:                 loggers = config.get('loggers', EMPTY_DICT)
jpayne@68:                 for name in loggers:
jpayne@68:                     if name in existing:
jpayne@68:                         i = existing.index(name) + 1 # look after name
jpayne@68:                         prefixed = name + "."
jpayne@68:                         pflen = len(prefixed)
jpayne@68:                         num_existing = len(existing)
jpayne@68:                         while i < num_existing:
jpayne@68:                             if existing[i][:pflen] == prefixed:
jpayne@68:                                 child_loggers.append(existing[i])
jpayne@68:                             i += 1
jpayne@68:                         existing.remove(name)
jpayne@68:                     try:
jpayne@68:                         self.configure_logger(name, loggers[name])
jpayne@68:                     except Exception as e:
jpayne@68:                         raise ValueError('Unable to configure logger '
jpayne@68:                                          '%r' % name) from e
jpayne@68: 
jpayne@68:                 #Disable any old loggers. There's no point deleting
jpayne@68:                 #them as other threads may continue to hold references
jpayne@68:                 #and by disabling them, you stop them doing any logging.
jpayne@68:                 #However, don't disable children of named loggers, as that's
jpayne@68:                 #probably not what was intended by the user.
jpayne@68:                 #for log in existing:
jpayne@68:                 #    logger = root.manager.loggerDict[log]
jpayne@68:                 #    if log in child_loggers:
jpayne@68:                 #        logger.level = logging.NOTSET
jpayne@68:                 #        logger.handlers = []
jpayne@68:                 #        logger.propagate = True
jpayne@68:                 #    elif disable_existing:
jpayne@68:                 #        logger.disabled = True
jpayne@68:                 _handle_existing_loggers(existing, child_loggers,
jpayne@68:                                          disable_existing)
jpayne@68: 
jpayne@68:                 # And finally, do the root logger
jpayne@68:                 root = config.get('root', None)
jpayne@68:                 if root:
jpayne@68:                     try:
jpayne@68:                         self.configure_root(root)
jpayne@68:                     except Exception as e:
jpayne@68:                         raise ValueError('Unable to configure root '
jpayne@68:                                          'logger') from e
jpayne@68:         finally:
jpayne@68:             logging._releaseLock()
jpayne@68: 
jpayne@68:     def configure_formatter(self, config):
jpayne@68:         """Configure a formatter from a dictionary."""
jpayne@68:         if '()' in config:
jpayne@68:             factory = config['()'] # for use in exception handler
jpayne@68:             try:
jpayne@68:                 result = self.configure_custom(config)
jpayne@68:             except TypeError as te:
jpayne@68:                 if "'format'" not in str(te):
jpayne@68:                     raise
jpayne@68:                 #Name of parameter changed from fmt to format.
jpayne@68:                 #Retry with old name.
jpayne@68:                 #This is so that code can be used with older Python versions
jpayne@68:                 #(e.g. by Django)
jpayne@68:                 config['fmt'] = config.pop('format')
jpayne@68:                 config['()'] = factory
jpayne@68:                 result = self.configure_custom(config)
jpayne@68:         else:
jpayne@68:             fmt = config.get('format', None)
jpayne@68:             dfmt = config.get('datefmt', None)
jpayne@68:             style = config.get('style', '%')
jpayne@68:             cname = config.get('class', None)
jpayne@68: 
jpayne@68:             if not cname:
jpayne@68:                 c = logging.Formatter
jpayne@68:             else:
jpayne@68:                 c = _resolve(cname)
jpayne@68: 
jpayne@68:             # A TypeError would be raised if "validate" key is passed in with a formatter callable
jpayne@68:             # that does not accept "validate" as a parameter
jpayne@68:             if 'validate' in config:  # if user hasn't mentioned it, the default will be fine
jpayne@68:                 result = c(fmt, dfmt, style, config['validate'])
jpayne@68:             else:
jpayne@68:                 result = c(fmt, dfmt, style)
jpayne@68: 
jpayne@68:         return result
jpayne@68: 
jpayne@68:     def configure_filter(self, config):
jpayne@68:         """Configure a filter from a dictionary."""
jpayne@68:         if '()' in config:
jpayne@68:             result = self.configure_custom(config)
jpayne@68:         else:
jpayne@68:             name = config.get('name', '')
jpayne@68:             result = logging.Filter(name)
jpayne@68:         return result
jpayne@68: 
jpayne@68:     def add_filters(self, filterer, filters):
jpayne@68:         """Add filters to a filterer from a list of names."""
jpayne@68:         for f in filters:
jpayne@68:             try:
jpayne@68:                 filterer.addFilter(self.config['filters'][f])
jpayne@68:             except Exception as e:
jpayne@68:                 raise ValueError('Unable to add filter %r' % f) from e
jpayne@68: 
jpayne@68:     def configure_handler(self, config):
jpayne@68:         """Configure a handler from a dictionary."""
jpayne@68:         config_copy = dict(config)  # for restoring in case of error
jpayne@68:         formatter = config.pop('formatter', None)
jpayne@68:         if formatter:
jpayne@68:             try:
jpayne@68:                 formatter = self.config['formatters'][formatter]
jpayne@68:             except Exception as e:
jpayne@68:                 raise ValueError('Unable to set formatter '
jpayne@68:                                  '%r' % formatter) from e
jpayne@68:         level = config.pop('level', None)
jpayne@68:         filters = config.pop('filters', None)
jpayne@68:         if '()' in config:
jpayne@68:             c = config.pop('()')
jpayne@68:             if not callable(c):
jpayne@68:                 c = self.resolve(c)
jpayne@68:             factory = c
jpayne@68:         else:
jpayne@68:             cname = config.pop('class')
jpayne@68:             klass = self.resolve(cname)
jpayne@68:             #Special case for handler which refers to another handler
jpayne@68:             if issubclass(klass, logging.handlers.MemoryHandler) and\
jpayne@68:                 'target' in config:
jpayne@68:                 try:
jpayne@68:                     th = self.config['handlers'][config['target']]
jpayne@68:                     if not isinstance(th, logging.Handler):
jpayne@68:                         config.update(config_copy)  # restore for deferred cfg
jpayne@68:                         raise TypeError('target not configured yet')
jpayne@68:                     config['target'] = th
jpayne@68:                 except Exception as e:
jpayne@68:                     raise ValueError('Unable to set target handler '
jpayne@68:                                      '%r' % config['target']) from e
jpayne@68:             elif issubclass(klass, logging.handlers.SMTPHandler) and\
jpayne@68:                 'mailhost' in config:
jpayne@68:                 config['mailhost'] = self.as_tuple(config['mailhost'])
jpayne@68:             elif issubclass(klass, logging.handlers.SysLogHandler) and\
jpayne@68:                 'address' in config:
jpayne@68:                 config['address'] = self.as_tuple(config['address'])
jpayne@68:             factory = klass
jpayne@68:         props = config.pop('.', None)
jpayne@68:         kwargs = {k: config[k] for k in config if valid_ident(k)}
jpayne@68:         try:
jpayne@68:             result = factory(**kwargs)
jpayne@68:         except TypeError as te:
jpayne@68:             if "'stream'" not in str(te):
jpayne@68:                 raise
jpayne@68:             #The argument name changed from strm to stream
jpayne@68:             #Retry with old name.
jpayne@68:             #This is so that code can be used with older Python versions
jpayne@68:             #(e.g. by Django)
jpayne@68:             kwargs['strm'] = kwargs.pop('stream')
jpayne@68:             result = factory(**kwargs)
jpayne@68:         if formatter:
jpayne@68:             result.setFormatter(formatter)
jpayne@68:         if level is not None:
jpayne@68:             result.setLevel(logging._checkLevel(level))
jpayne@68:         if filters:
jpayne@68:             self.add_filters(result, filters)
jpayne@68:         if props:
jpayne@68:             for name, value in props.items():
jpayne@68:                 setattr(result, name, value)
jpayne@68:         return result
jpayne@68: 
jpayne@68:     def add_handlers(self, logger, handlers):
jpayne@68:         """Add handlers to a logger from a list of names."""
jpayne@68:         for h in handlers:
jpayne@68:             try:
jpayne@68:                 logger.addHandler(self.config['handlers'][h])
jpayne@68:             except Exception as e:
jpayne@68:                 raise ValueError('Unable to add handler %r' % h) from e
jpayne@68: 
jpayne@68:     def common_logger_config(self, logger, config, incremental=False):
jpayne@68:         """
jpayne@68:         Perform configuration which is common to root and non-root loggers.
jpayne@68:         """
jpayne@68:         level = config.get('level', None)
jpayne@68:         if level is not None:
jpayne@68:             logger.setLevel(logging._checkLevel(level))
jpayne@68:         if not incremental:
jpayne@68:             #Remove any existing handlers
jpayne@68:             for h in logger.handlers[:]:
jpayne@68:                 logger.removeHandler(h)
jpayne@68:             handlers = config.get('handlers', None)
jpayne@68:             if handlers:
jpayne@68:                 self.add_handlers(logger, handlers)
jpayne@68:             filters = config.get('filters', None)
jpayne@68:             if filters:
jpayne@68:                 self.add_filters(logger, filters)
jpayne@68: 
jpayne@68:     def configure_logger(self, name, config, incremental=False):
jpayne@68:         """Configure a non-root logger from a dictionary."""
jpayne@68:         logger = logging.getLogger(name)
jpayne@68:         self.common_logger_config(logger, config, incremental)
jpayne@68:         propagate = config.get('propagate', None)
jpayne@68:         if propagate is not None:
jpayne@68:             logger.propagate = propagate
jpayne@68: 
jpayne@68:     def configure_root(self, config, incremental=False):
jpayne@68:         """Configure a root logger from a dictionary."""
jpayne@68:         root = logging.getLogger()
jpayne@68:         self.common_logger_config(root, config, incremental)
jpayne@68: 
jpayne@68: dictConfigClass = DictConfigurator
jpayne@68: 
jpayne@68: def dictConfig(config):
jpayne@68:     """Configure logging using a dictionary."""
jpayne@68:     dictConfigClass(config).configure()
jpayne@68: 
jpayne@68: 
jpayne@68: def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
jpayne@68:     """
jpayne@68:     Start up a socket server on the specified port, and listen for new
jpayne@68:     configurations.
jpayne@68: 
jpayne@68:     These will be sent as a file suitable for processing by fileConfig().
jpayne@68:     Returns a Thread object on which you can call start() to start the server,
jpayne@68:     and which you can join() when appropriate. To stop the server, call
jpayne@68:     stopListening().
jpayne@68: 
jpayne@68:     Use the ``verify`` argument to verify any bytes received across the wire
jpayne@68:     from a client. If specified, it should be a callable which receives a
jpayne@68:     single argument - the bytes of configuration data received across the
jpayne@68:     network - and it should return either ``None``, to indicate that the
jpayne@68:     passed in bytes could not be verified and should be discarded, or a
jpayne@68:     byte string which is then passed to the configuration machinery as
jpayne@68:     normal. Note that you can return transformed bytes, e.g. by decrypting
jpayne@68:     the bytes passed in.
jpayne@68:     """
jpayne@68: 
jpayne@68:     class ConfigStreamHandler(StreamRequestHandler):
jpayne@68:         """
jpayne@68:         Handler for a logging configuration request.
jpayne@68: 
jpayne@68:         It expects a completely new logging configuration and uses fileConfig
jpayne@68:         to install it.
jpayne@68:         """
jpayne@68:         def handle(self):
jpayne@68:             """
jpayne@68:             Handle a request.
jpayne@68: 
jpayne@68:             Each request is expected to be a 4-byte length, packed using
jpayne@68:             struct.pack(">L", n), followed by the config file.
jpayne@68:             Uses fileConfig() to do the grunt work.
jpayne@68:             """
jpayne@68:             try:
jpayne@68:                 conn = self.connection
jpayne@68:                 chunk = conn.recv(4)
jpayne@68:                 if len(chunk) == 4:
jpayne@68:                     slen = struct.unpack(">L", chunk)[0]
jpayne@68:                     chunk = self.connection.recv(slen)
jpayne@68:                     while len(chunk) < slen:
jpayne@68:                         chunk = chunk + conn.recv(slen - len(chunk))
jpayne@68:                     if self.server.verify is not None:
jpayne@68:                         chunk = self.server.verify(chunk)
jpayne@68:                     if chunk is not None:   # verified, can process
jpayne@68:                         chunk = chunk.decode("utf-8")
jpayne@68:                         try:
jpayne@68:                             import json
jpayne@68:                             d =json.loads(chunk)
jpayne@68:                             assert isinstance(d, dict)
jpayne@68:                             dictConfig(d)
jpayne@68:                         except Exception:
jpayne@68:                             #Apply new configuration.
jpayne@68: 
jpayne@68:                             file = io.StringIO(chunk)
jpayne@68:                             try:
jpayne@68:                                 fileConfig(file)
jpayne@68:                             except Exception:
jpayne@68:                                 traceback.print_exc()
jpayne@68:                     if self.server.ready:
jpayne@68:                         self.server.ready.set()
jpayne@68:             except OSError as e:
jpayne@68:                 if e.errno != RESET_ERROR:
jpayne@68:                     raise
jpayne@68: 
jpayne@68:     class ConfigSocketReceiver(ThreadingTCPServer):
jpayne@68:         """
jpayne@68:         A simple TCP socket-based logging config receiver.
jpayne@68:         """
jpayne@68: 
jpayne@68:         allow_reuse_address = 1
jpayne@68: 
jpayne@68:         def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
jpayne@68:                      handler=None, ready=None, verify=None):
jpayne@68:             ThreadingTCPServer.__init__(self, (host, port), handler)
jpayne@68:             logging._acquireLock()
jpayne@68:             self.abort = 0
jpayne@68:             logging._releaseLock()
jpayne@68:             self.timeout = 1
jpayne@68:             self.ready = ready
jpayne@68:             self.verify = verify
jpayne@68: 
jpayne@68:         def serve_until_stopped(self):
jpayne@68:             import select
jpayne@68:             abort = 0
jpayne@68:             while not abort:
jpayne@68:                 rd, wr, ex = select.select([self.socket.fileno()],
jpayne@68:                                            [], [],
jpayne@68:                                            self.timeout)
jpayne@68:                 if rd:
jpayne@68:                     self.handle_request()
jpayne@68:                 logging._acquireLock()
jpayne@68:                 abort = self.abort
jpayne@68:                 logging._releaseLock()
jpayne@68:             self.server_close()
jpayne@68: 
jpayne@68:     class Server(threading.Thread):
jpayne@68: 
jpayne@68:         def __init__(self, rcvr, hdlr, port, verify):
jpayne@68:             super(Server, self).__init__()
jpayne@68:             self.rcvr = rcvr
jpayne@68:             self.hdlr = hdlr
jpayne@68:             self.port = port
jpayne@68:             self.verify = verify
jpayne@68:             self.ready = threading.Event()
jpayne@68: 
jpayne@68:         def run(self):
jpayne@68:             server = self.rcvr(port=self.port, handler=self.hdlr,
jpayne@68:                                ready=self.ready,
jpayne@68:                                verify=self.verify)
jpayne@68:             if self.port == 0:
jpayne@68:                 self.port = server.server_address[1]
jpayne@68:             self.ready.set()
jpayne@68:             global _listener
jpayne@68:             logging._acquireLock()
jpayne@68:             _listener = server
jpayne@68:             logging._releaseLock()
jpayne@68:             server.serve_until_stopped()
jpayne@68: 
jpayne@68:     return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify)
jpayne@68: 
jpayne@68: def stopListening():
jpayne@68:     """
jpayne@68:     Stop the listening server which was created with a call to listen().
jpayne@68:     """
jpayne@68:     global _listener
jpayne@68:     logging._acquireLock()
jpayne@68:     try:
jpayne@68:         if _listener:
jpayne@68:             _listener.abort = 1
jpayne@68:             _listener = None
jpayne@68:     finally:
jpayne@68:         logging._releaseLock()