diff options
Diffstat (limited to 'Lib/logging')
| -rw-r--r-- | Lib/logging/__init__.py | 613 | ||||
| -rw-r--r-- | Lib/logging/config.py | 704 | ||||
| -rw-r--r-- | Lib/logging/handlers.py | 287 |
3 files changed, 1293 insertions, 311 deletions
diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 685efeb..e4b34a1 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2001-2009 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2010 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -23,7 +23,8 @@ Copyright (C) 2001-2010 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ -import sys, os, time, io, traceback, warnings +import sys, os, time, io, traceback, warnings, weakref +from string import Template __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', 'FATAL', 'FileHandler', 'Filter', 'Formatter', 'Handler', 'INFO', @@ -31,7 +32,8 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig', 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', - 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning'] + 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning', + 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort'] try: import codecs @@ -46,15 +48,13 @@ except ImportError: __author__ = "Vinay Sajip <vinay_sajip@red-dove.com>" __status__ = "production" -__version__ = "0.5.0.7" -__date__ = "20 January 2009" +__version__ = "0.5.1.2" +__date__ = "07 February 2010" #--------------------------------------------------------------------------- # Miscellaneous module data #--------------------------------------------------------------------------- -_unicode = 'unicode' in dir(__builtins__) - # # _srcfile is used when walking the stack to check when we've got the first # caller stack frame. @@ -176,19 +176,34 @@ def addLevelName(level, levelName): finally: _releaseLock() +def _checkLevel(level): + if isinstance(level, int): + rv = level + elif str(level) == level: + if level not in _levelNames: + raise ValueError("Unknown level: %r" % level) + rv = _levelNames[level] + else: + raise TypeError("Level not an integer or a valid string: %r" % level) + return rv + #--------------------------------------------------------------------------- # Thread-related stuff #--------------------------------------------------------------------------- # #_lock is used to serialize access to shared data structures in this module. -#This needs to be an RLock because fileConfig() creates Handlers and so -#might arbitrary user threads. Since Handler.__init__() updates the shared -#dictionary _handlers, it needs to acquire the lock. But if configuring, +#This needs to be an RLock because fileConfig() creates and configures +#Handlers, and so might arbitrary user threads. Since Handler code updates the +#shared dictionary _handlers, it needs to acquire the lock. But if configuring, #the lock would already have been acquired - so we need an RLock. #The same argument applies to Loggers and Manager.loggerDict. # -_lock = None +if thread: + _lock = threading.RLock() +else: + _lock = None + def _acquireLock(): """ @@ -196,9 +211,6 @@ def _acquireLock(): This should be released with _releaseLock(). """ - global _lock - if (not _lock) and thread: - _lock = threading.RLock() if _lock: _lock.acquire() @@ -213,7 +225,7 @@ def _releaseLock(): # The logging record #--------------------------------------------------------------------------- -class LogRecord: +class LogRecord(object): """ A LogRecord instance represents an event being logged. @@ -226,7 +238,7 @@ class LogRecord: information to be logged. """ def __init__(self, name, level, pathname, lineno, - msg, args, exc_info, func=None): + msg, args, exc_info, func=None, sinfo=None, **kwargs): """ Initialize a logging record with interesting information. """ @@ -260,6 +272,7 @@ class LogRecord: self.module = "Unknown module" self.exc_info = exc_info self.exc_text = None # used to cache the traceback text + self.stack_info = sinfo self.lineno = lineno self.funcName = func self.created = ct @@ -274,11 +287,17 @@ class LogRecord: if not logMultiprocessing: self.processName = None else: - try: - from multiprocessing import current_process - self.processName = current_process().name - except ImportError: - self.processName = None + self.processName = 'MainProcess' + mp = sys.modules.get('multiprocessing') + if mp is not None: + # Errors may occur if multiprocessing has not finished loading + # yet - e.g. if a custom import hook causes third-party code + # to run when multiprocessing calls import. See issue 8200 + # for an example + try: + self.processName = mp.current_process().name + except StandardError: + pass if logProcesses and hasattr(os, 'getpid'): self.process = os.getpid() else: @@ -295,19 +314,33 @@ class LogRecord: Return the message for this LogRecord after merging any user-supplied arguments with the message. """ - if not _unicode: #if no unicode support... - msg = str(self.msg) - else: - msg = self.msg - if not isinstance(msg, str): - try: - msg = str(self.msg) - except UnicodeError: - msg = self.msg #Defer encoding till later + msg = str(self.msg) if self.args: msg = msg % self.args return msg +# +# Determine which class to use when instantiating log records. +# +_logRecordFactory = LogRecord + +def setLogRecordFactory(factory): + """ + Set the factory to be used when instantiating a log record. + + :param factory: A callable which will be called to instantiate + a log record. + """ + global _logRecordFactory + _logRecordFactory = factory + +def getLogRecordFactory(): + """ + Return the factory to be used when instantiating a log record. + """ + + return _logRecordFactory + def makeLogRecord(dict): """ Make a LogRecord whose attributes are defined by the specified dictionary, @@ -315,7 +348,7 @@ def makeLogRecord(dict): a socket connection (which is sent as a dictionary) into a LogRecord instance. """ - rv = LogRecord(None, None, "", 0, "", (), None, None) + rv = _logRecordFactory(None, None, "", 0, "", (), None, None) rv.__dict__.update(dict) return rv @@ -323,7 +356,53 @@ def makeLogRecord(dict): # Formatter classes and functions #--------------------------------------------------------------------------- -class Formatter: +class PercentStyle(object): + + default_format = '%(message)s' + asctime_format = '%(asctime)s' + asctime_search = '%(asctime)' + + def __init__(self, fmt): + self._fmt = fmt or self.default_format + + def usesTime(self): + return self._fmt.find(self.asctime_search) >= 0 + + def format(self, record): + return self._fmt % record.__dict__ + +class StrFormatStyle(PercentStyle): + default_format = '{message}' + asctime_format = '{asctime}' + asctime_search = '{asctime' + + def format(self, record): + return self._fmt.format(**record.__dict__) + + +class StringTemplateStyle(PercentStyle): + default_format = '${message}' + asctime_format = '${asctime}' + asctime_search = '${asctime}' + + def __init__(self, fmt): + self._fmt = fmt or self.default_format + self._tpl = Template(self._fmt) + + def usesTime(self): + fmt = self._fmt + return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0 + + def format(self, record): + return self._tpl.substitute(**record.__dict__) + +_STYLES = { + '%': PercentStyle, + '{': StrFormatStyle, + '$': StringTemplateStyle +} + +class Formatter(object): """ Formatter instances are used to convert a LogRecord to text. @@ -331,7 +410,7 @@ class Formatter: responsible for converting a LogRecord to (usually) a string which can be interpreted by either a human or an external system. The base Formatter allows a formatting string to be specified. If none is supplied, the - default value of "%s(message)\\n" is used. + default value of "%s(message)" is used. The Formatter can be initialized with a format string which makes use of knowledge of the LogRecord attributes - e.g. the default value mentioned @@ -367,18 +446,26 @@ class Formatter: converter = time.localtime - def __init__(self, fmt=None, datefmt=None): + def __init__(self, fmt=None, datefmt=None, style='%'): """ Initialize the formatter with specified format strings. Initialize the formatter either with the specified format string, or a default as described above. Allow for specialized date formatting with the optional datefmt argument (if omitted, you get the ISO8601 format). + + Use a style parameter of '%', '{' or '$' to specify that you want to + use one of %-formatting, :meth:`str.format` (``{}``) formatting or + :class:`string.Template` formatting in your format string. + + .. versionchanged: 3.2 + Added the ``style`` parameter. """ - if fmt: - self._fmt = fmt - else: - self._fmt = "%(message)s" + if style not in _STYLES: + raise ValueError('Style must be one of: %s' % ','.join( + _STYLES.keys())) + self._style = _STYLES[style](fmt) + self._fmt = self._style._fmt self.datefmt = datefmt def formatTime(self, record, datefmt=None): @@ -404,7 +491,7 @@ class Formatter: s = time.strftime(datefmt, ct) else: t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - s = "%s,%03d" % (t, record.msecs) + s = "%s,%03d" % (t, record.msecs) # the use of % here is internal return s def formatException(self, ei): @@ -415,13 +502,39 @@ class Formatter: traceback.print_exception() """ sio = io.StringIO() - traceback.print_exception(ei[0], ei[1], ei[2], None, sio) + tb = ei[2] + # See issues #9427, #1553375. Commented out for now. + #if getattr(self, 'fullstack', False): + # traceback.print_stack(tb.tb_frame.f_back, file=sio) + traceback.print_exception(ei[0], ei[1], tb, None, sio) s = sio.getvalue() sio.close() if s[-1:] == "\n": s = s[:-1] return s + def usesTime(self): + """ + Check if the format uses the creation time of the record. + """ + return self._style.usesTime() + + def formatMessage(self, record): + return self._style.format(record) + + def formatStack(self, stack_info): + """ + This method is provided as an extension point for specialized + formatting of stack information. + + The input data is a string as returned from a call to + :func:`traceback.print_stack`, but with the last trailing newline + removed. + + The base implementation just returns the value passed in. + """ + return stack_info + def format(self, record): """ Format the specified record as text. @@ -430,15 +543,15 @@ class Formatter: string formatting operation which yields the returned string. Before formatting the dictionary, a couple of preparatory steps are carried out. The message attribute of the record is computed - using LogRecord.getMessage(). If the formatting string contains - "%(asctime)", formatTime() is called to format the event time. - If there is exception information, it is formatted using - formatException() and appended to the message. + using LogRecord.getMessage(). If the formatting string uses the + time (as determined by a call to usesTime(), formatTime() is + called to format the event time. If there is exception information, + it is formatted using formatException() and appended to the message. """ record.message = record.getMessage() - if self._fmt.find("%(asctime)") >= 0: + if self.usesTime(): record.asctime = self.formatTime(record, self.datefmt) - s = self._fmt % record.__dict__ + s = self.formatMessage(record) if record.exc_info: # Cache the traceback text to avoid converting it multiple times # (it's constant anyway) @@ -448,6 +561,10 @@ class Formatter: if s[-1:] != "\n": s = s + "\n" s = s + record.exc_text + if record.stack_info: + if s[-1:] != "\n": + s = s + "\n" + s = s + self.formatStack(record.stack_info) return s # @@ -455,7 +572,7 @@ class Formatter: # _defaultFormatter = Formatter() -class BufferingFormatter: +class BufferingFormatter(object): """ A formatter suitable for formatting a number of records. """ @@ -497,7 +614,7 @@ class BufferingFormatter: # Filter classes and functions #--------------------------------------------------------------------------- -class Filter: +class Filter(object): """ Filter instances are used to perform arbitrary filtering of LogRecords. @@ -534,7 +651,7 @@ class Filter: return 0 return (record.name[self.nlen] == ".") -class Filterer: +class Filterer(object): """ A base class for loggers and handlers which allows them to share common code. @@ -566,10 +683,18 @@ class Filterer: The default is to allow the record to be logged; any filter can veto this and the record is then dropped. Returns a zero value if a record is to be dropped, else non-zero. + + .. versionchanged: 3.2 + + Allow filters to be just callables. """ rv = 1 for f in self.filters: - if not f.filter(record): + if hasattr(f, 'filter'): + result = f.filter(record) + else: + result = f(record) # assume callable - will raise if not + if not result: rv = 0 break return rv @@ -578,9 +703,34 @@ class Filterer: # Handler classes and functions #--------------------------------------------------------------------------- -_handlers = {} #repository of handlers (for flushing when shutdown called) +_handlers = weakref.WeakValueDictionary() #map of handler names to handlers _handlerList = [] # added to allow handlers to be removed in reverse of order initialized +def _removeHandlerRef(wr): + """ + Remove a handler reference from the internal cleanup list. + """ + # This function can be called during module teardown, when globals are + # set to None. If _acquireLock is None, assume this is the case and do + # nothing. + if _acquireLock is not None: + _acquireLock() + try: + if wr in _handlerList: + _handlerList.remove(wr) + finally: + _releaseLock() + +def _addHandlerRef(handler): + """ + Add a handler to the internal cleanup list using a weak reference. + """ + _acquireLock() + try: + _handlerList.append(weakref.ref(handler, _removeHandlerRef)) + finally: + _releaseLock() + class Handler(Filterer): """ Handler instances dispatch logging events to specific destinations. @@ -596,16 +746,28 @@ class Handler(Filterer): and the filter list to empty. """ Filterer.__init__(self) - self.level = level + self._name = None + self.level = _checkLevel(level) self.formatter = None - #get the module data lock, as we're updating a shared structure. + # Add the handler to the global _handlerList (for cleanup on shutdown) + _addHandlerRef(self) + self.createLock() + + def get_name(self): + return self._name + + def set_name(self, name): _acquireLock() - try: #unlikely to raise an exception, but you never know... - _handlers[self] = 1 - _handlerList.insert(0, self) + try: + if self._name in _handlers: + del _handlers[self._name] + self._name = name + if name: + _handlers[name] = self finally: _releaseLock() - self.createLock() + + name = property(get_name, set_name) def createLock(self): """ @@ -634,7 +796,7 @@ class Handler(Filterer): """ Set the logging level of this handler. """ - self.level = level + self.level = _checkLevel(level) def format(self, record): """ @@ -696,16 +858,16 @@ class Handler(Filterer): """ Tidy up any resources used by the handler. - This version does removes the handler from an internal list - of handlers which is closed when shutdown() is called. Subclasses + This version removes the handler from an internal map of handlers, + _handlers, which is used for handler lookup by name. Subclasses should ensure that this gets called from overridden close() methods. """ #get the module data lock, as we're updating a shared structure. _acquireLock() try: #unlikely to raise an exception, but you never know... - del _handlers[self] - _handlerList.remove(self) + if self._name and self._name in _handlers: + del _handlers[self._name] finally: _releaseLock() @@ -724,7 +886,10 @@ class Handler(Filterer): if raiseExceptions: ei = sys.exc_info() try: - traceback.print_exception(ei[0], ei[1], ei[2], None, sys.stderr) + traceback.print_exception(ei[0], ei[1], ei[2], + None, sys.stderr) + sys.stderr.write('Logged from file %s, line %s\n' % ( + record.filename, record.lineno)) except IOError: pass # see issue 5971 finally: @@ -737,6 +902,8 @@ class StreamHandler(Handler): sys.stdout or sys.stderr may be used. """ + terminator = '\n' + def __init__(self, stream=None): """ Initialize the handler. @@ -769,28 +936,8 @@ class StreamHandler(Handler): try: msg = self.format(record) stream = self.stream - fs = "%s\n" - if not _unicode: #if no unicode support... - stream.write(fs % msg) - else: - try: - if (isinstance(msg, unicode) and - getattr(stream, 'encoding', None)): - fs = fs.decode(stream.encoding) - try: - stream.write(fs % msg) - except UnicodeEncodeError: - #Printing to terminals sometimes fails. For example, - #with an encoding of 'cp1251', the above write will - #work if written to a stream opened or wrapped by - #the codecs module, but fail when writing to a - #terminal even when the codepage is set to cp1251. - #An extra encoding step seems to be needed. - stream.write((fs % msg).encode(stream.encoding)) - else: - stream.write(fs % msg) - except UnicodeError: - stream.write(fs % msg.encode("UTF-8")) + stream.write(msg) + stream.write(self.terminator) self.flush() except (KeyboardInterrupt, SystemExit): raise @@ -853,11 +1000,31 @@ class FileHandler(StreamHandler): self.stream = self._open() StreamHandler.emit(self, record) +class _StderrHandler(StreamHandler): + """ + This class is like a StreamHandler using sys.stderr, but always uses + whatever sys.stderr is currently set to rather than the value of + sys.stderr at handler construction time. + """ + def __init__(self, level=NOTSET): + """ + Initialize the handler. + """ + Handler.__init__(self, level) + + @property + def stream(self): + return sys.stderr + + +_defaultLastResort = _StderrHandler(WARNING) +lastResort = _defaultLastResort + #--------------------------------------------------------------------------- # Manager classes and functions #--------------------------------------------------------------------------- -class PlaceHolder: +class PlaceHolder(object): """ PlaceHolder instances are used in the Manager logger hierarchy to take the place of nodes for which no loggers have been defined. This class is @@ -867,16 +1034,13 @@ class PlaceHolder: """ Initialize with the specified logger being a child of this placeholder. """ - #self.loggers = [alogger] self.loggerMap = { alogger : None } def append(self, alogger): """ Add the specified logger as a child of this placeholder. """ - #if alogger not in self.loggers: if alogger not in self.loggerMap: - #self.loggers.append(alogger) self.loggerMap[alogger] = None # @@ -904,7 +1068,7 @@ def getLoggerClass(): return _loggerClass -class Manager: +class Manager(object): """ There is [under normal circumstances] just one Manager instance, which holds the hierarchy of loggers. @@ -915,8 +1079,10 @@ class Manager: """ self.root = rootnode self.disable = 0 - self.emittedNoHandlerWarning = 0 + self.emittedNoHandlerWarning = False self.loggerDict = {} + self.loggerClass = None + self.logRecordFactory = None def getLogger(self, name): """ @@ -936,13 +1102,13 @@ class Manager: rv = self.loggerDict[name] if isinstance(rv, PlaceHolder): ph = rv - rv = _loggerClass(name) + rv = (self.loggerClass or _loggerClass)(name) rv.manager = self self.loggerDict[name] = rv self._fixupChildren(ph, rv) self._fixupParents(rv) else: - rv = _loggerClass(name) + rv = (self.loggerClass or _loggerClass)(name) rv.manager = self self.loggerDict[name] = rv self._fixupParents(rv) @@ -950,6 +1116,23 @@ class Manager: _releaseLock() return rv + def setLoggerClass(self, klass): + """ + Set the class to be used when instantiating a logger with this Manager. + """ + if klass != Logger: + if not issubclass(klass, Logger): + raise TypeError("logger not derived from logging.Logger: " + + klass.__name__) + self.loggerClass = klass + + def setLogRecordFactory(self, factory): + """ + Set the factory to be used when instantiating a log record with this + Manager. + """ + self.logRecordFactory = factory + def _fixupParents(self, alogger): """ Ensure that there are either loggers or placeholders all the way @@ -1012,7 +1195,7 @@ class Logger(Filterer): """ Filterer.__init__(self) self.name = name - self.level = level + self.level = _checkLevel(level) self.parent = None self.propagate = 1 self.handlers = [] @@ -1022,7 +1205,7 @@ class Logger(Filterer): """ Set the logging level of this logger. """ - self.level = level + self.level = _checkLevel(level) def debug(self, msg, *args, **kwargs): """ @@ -1074,11 +1257,12 @@ class Logger(Filterer): if self.isEnabledFor(ERROR): self._log(ERROR, msg, args, **kwargs) - def exception(self, msg, *args): + def exception(self, msg, *args, **kwargs): """ Convenience method for logging an ERROR with exception information. """ - self.error(msg, exc_info=1, *args) + kwargs['exc_info'] = True + self.error(msg, *args, **kwargs) def critical(self, msg, *args, **kwargs): """ @@ -1111,7 +1295,7 @@ class Logger(Filterer): if self.isEnabledFor(level): self._log(level, msg, args, **kwargs) - def findCaller(self): + def findCaller(self, stack_info=False): """ Find the stack frame of the caller so that we can note the source file name, line number and function name. @@ -1121,23 +1305,34 @@ class Logger(Filterer): #IronPython isn't run with -X:Frames. if f is not None: f = f.f_back - rv = "(unknown file)", 0, "(unknown function)" + rv = "(unknown file)", 0, "(unknown function)", None while hasattr(f, "f_code"): co = f.f_code filename = os.path.normcase(co.co_filename) if filename == _srcfile: f = f.f_back continue - rv = (filename, f.f_lineno, co.co_name) + sinfo = None + if stack_info: + sio = io.StringIO() + sio.write('Stack (most recent call last):\n') + traceback.print_stack(f, file=sio) + sinfo = sio.getvalue() + if sinfo[-1] == '\n': + sinfo = sinfo[:-1] + sio.close() + rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) break return rv - def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None): + def makeRecord(self, name, level, fn, lno, msg, args, exc_info, + func=None, extra=None, sinfo=None): """ A factory method which can be overridden in subclasses to create specialized LogRecords. """ - rv = LogRecord(name, level, fn, lno, msg, args, exc_info, func) + rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func, + sinfo) if extra is not None: for key in extra: if (key in ["message", "asctime"]) or (key in rv.__dict__): @@ -1145,17 +1340,18 @@ class Logger(Filterer): rv.__dict__[key] = extra[key] return rv - def _log(self, level, msg, args, exc_info=None, extra=None): + def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): """ Low-level logging routine which creates a LogRecord and then calls all the handlers of this logger to handle the record. """ + sinfo = None if _srcfile: #IronPython doesn't track Python frames, so findCaller throws an #exception on some versions of IronPython. We trap it here so that #IronPython can use logging. try: - fn, lno, func = self.findCaller() + fn, lno, func, sinfo = self.findCaller(stack_info) except ValueError: fn, lno, func = "(unknown file)", 0, "(unknown function)" else: @@ -1163,7 +1359,8 @@ class Logger(Filterer): if exc_info: if not isinstance(exc_info, tuple): exc_info = sys.exc_info() - record = self.makeRecord(self.name, level, fn, lno, msg, args, exc_info, func, extra) + record = self.makeRecord(self.name, level, fn, lno, msg, args, + exc_info, func, extra, sinfo) self.handle(record) def handle(self, record): @@ -1198,6 +1395,28 @@ class Logger(Filterer): finally: _releaseLock() + def hasHandlers(self): + """ + See if this logger has any handlers configured. + + Loop through all handlers for this logger and its parents in the + logger hierarchy. Return True if a handler was found, else False. + Stop searching up the hierarchy whenever a logger with the "propagate" + attribute set to zero is found - that will be the last logger which + is checked for the existence of handlers. + """ + c = self + rv = False + while c: + if c.handlers: + rv = True + break + if not c.propagate: + break + else: + c = c.parent + return rv + def callHandlers(self, record): """ Pass a record to all relevant handlers. @@ -1219,10 +1438,13 @@ class Logger(Filterer): c = None #break out else: c = c.parent - if (found == 0) and raiseExceptions and not self.manager.emittedNoHandlerWarning: - sys.stderr.write("No handlers could be found for logger" - " \"%s\"\n" % self.name) - self.manager.emittedNoHandlerWarning = 1 + if (found == 0): + if lastResort: + lastResort.handle(record) + elif raiseExceptions and not self.manager.emittedNoHandlerWarning: + sys.stderr.write("No handlers could be found for logger" + " \"%s\"\n" % self.name) + self.manager.emittedNoHandlerWarning = True def getEffectiveLevel(self): """ @@ -1246,6 +1468,25 @@ class Logger(Filterer): return 0 return level >= self.getEffectiveLevel() + def getChild(self, suffix): + """ + Get a logger which is a descendant to this one. + + This is a convenience method, such that + + logging.getLogger('abc').getChild('def.ghi') + + is the same as + + logging.getLogger('abc.def.ghi') + + It's useful, for example, when the parent logger is named using + __name__ rather than a literal string. + """ + if self.root is not self: + suffix = '.'.join((self.name, suffix)) + return self.manager.getLogger(suffix) + class RootLogger(Logger): """ A root logger is not that different to any other logger, except that @@ -1260,7 +1501,7 @@ class RootLogger(Logger): _loggerClass = Logger -class LoggerAdapter: +class LoggerAdapter(object): """ An adapter for loggers which makes it easier to specify contextual information in logging output. @@ -1293,62 +1534,82 @@ class LoggerAdapter: kwargs["extra"] = self.extra return msg, kwargs + # + # Boilerplate convenience methods + # def debug(self, msg, *args, **kwargs): """ - Delegate a debug call to the underlying logger, after adding - contextual information from this adapter instance. + Delegate a debug call to the underlying logger. """ - msg, kwargs = self.process(msg, kwargs) - self.logger.debug(msg, *args, **kwargs) + self.log(DEBUG, msg, *args, **kwargs) def info(self, msg, *args, **kwargs): """ - Delegate an info call to the underlying logger, after adding - contextual information from this adapter instance. + Delegate an info call to the underlying logger. """ - msg, kwargs = self.process(msg, kwargs) - self.logger.info(msg, *args, **kwargs) + self.log(INFO, msg, *args, **kwargs) def warning(self, msg, *args, **kwargs): """ - Delegate a warning call to the underlying logger, after adding - contextual information from this adapter instance. + Delegate a warning call to the underlying logger. """ - msg, kwargs = self.process(msg, kwargs) - self.logger.warning(msg, *args, **kwargs) + self.log(WARNING, msg, *args, **kwargs) + + warn = warning def error(self, msg, *args, **kwargs): """ - Delegate an error call to the underlying logger, after adding - contextual information from this adapter instance. + Delegate an error call to the underlying logger. """ - msg, kwargs = self.process(msg, kwargs) - self.logger.error(msg, *args, **kwargs) + self.log(ERROR, msg, *args, **kwargs) def exception(self, msg, *args, **kwargs): """ - Delegate an exception call to the underlying logger, after adding - contextual information from this adapter instance. + Delegate an exception call to the underlying logger. """ - msg, kwargs = self.process(msg, kwargs) kwargs["exc_info"] = 1 - self.logger.error(msg, *args, **kwargs) + self.log(ERROR, msg, *args, **kwargs) def critical(self, msg, *args, **kwargs): """ - Delegate a critical call to the underlying logger, after adding - contextual information from this adapter instance. + Delegate a critical call to the underlying logger. """ - msg, kwargs = self.process(msg, kwargs) - self.logger.critical(msg, *args, **kwargs) + self.log(CRITICAL, msg, *args, **kwargs) def log(self, level, msg, *args, **kwargs): """ Delegate a log call to the underlying logger, after adding contextual information from this adapter instance. """ - msg, kwargs = self.process(msg, kwargs) - self.logger.log(level, msg, *args, **kwargs) + if self.isEnabledFor(level): + msg, kwargs = self.process(msg, kwargs) + self.logger._log(level, msg, args, **kwargs) + + def isEnabledFor(self, level): + """ + Is this logger enabled for level 'level'? + """ + if self.logger.manager.disable >= level: + return False + return level >= self.getEffectiveLevel() + + def setLevel(self, level): + """ + Set the specified level on the underlying logger. + """ + self.logger.setLevel(level) + + def getEffectiveLevel(self): + """ + Get the effective level for the underlying logger. + """ + return self.logger.getEffectiveLevel() + + def hasHandlers(self): + """ + See if the underlying logger has any handlers. + """ + return self.logger.hasHandlers() root = RootLogger(WARNING) Logger.root = root @@ -1381,6 +1642,10 @@ def basicConfig(**kwargs): (if filemode is unspecified, it defaults to 'a'). format Use the specified format string for the handler. datefmt Use the specified date/time format. + style If a format string is specified, use this to specify the + type of format string (possible values '%', '{', '$', for + %-formatting, :meth:`str.format` and :class:`string.Template` + - defaults to '%'). level Set the root logger level to the specified level. stream Use the specified stream to initialize the StreamHandler. Note that this argument is incompatible with 'filename' - if both @@ -1391,6 +1656,9 @@ def basicConfig(**kwargs): remembered that StreamHandler does not close its stream (since it may be using sys.stdout or sys.stderr), whereas FileHandler closes its stream when the handler is closed. + + .. versionchanged: 3.2 + Added the ``style`` parameter. """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads @@ -1406,7 +1674,8 @@ def basicConfig(**kwargs): hdlr = StreamHandler(stream) fs = kwargs.get("format", BASIC_FORMAT) dfs = kwargs.get("datefmt", None) - fmt = Formatter(fs, dfs) + style = kwargs.get("style", '%') + fmt = Formatter(fs, dfs, style) hdlr.setFormatter(fmt) root.addHandler(hdlr) level = kwargs.get("level") @@ -1433,7 +1702,9 @@ def getLogger(name=None): def critical(msg, *args, **kwargs): """ - Log a message with severity 'CRITICAL' on the root logger. + Log a message with severity 'CRITICAL' on the root logger. If the logger + has no handlers, call basicConfig() to add a console handler with a + pre-defined format. """ if len(root.handlers) == 0: basicConfig() @@ -1443,22 +1714,28 @@ fatal = critical def error(msg, *args, **kwargs): """ - Log a message with severity 'ERROR' on the root logger. + Log a message with severity 'ERROR' on the root logger. If the logger has + no handlers, call basicConfig() to add a console handler with a pre-defined + format. """ if len(root.handlers) == 0: basicConfig() root.error(msg, *args, **kwargs) -def exception(msg, *args): +def exception(msg, *args, **kwargs): """ - Log a message with severity 'ERROR' on the root logger, - with exception information. + Log a message with severity 'ERROR' on the root logger, with exception + information. If the logger has no handlers, basicConfig() is called to add + a console handler with a pre-defined format. """ - error(msg, exc_info=1, *args) + kwargs['exc_info'] = True + error(msg, *args, **kwargs) def warning(msg, *args, **kwargs): """ - Log a message with severity 'WARNING' on the root logger. + Log a message with severity 'WARNING' on the root logger. If the logger has + no handlers, call basicConfig() to add a console handler with a pre-defined + format. """ if len(root.handlers) == 0: basicConfig() @@ -1468,7 +1745,9 @@ warn = warning def info(msg, *args, **kwargs): """ - Log a message with severity 'INFO' on the root logger. + Log a message with severity 'INFO' on the root logger. If the logger has + no handlers, call basicConfig() to add a console handler with a pre-defined + format. """ if len(root.handlers) == 0: basicConfig() @@ -1476,7 +1755,9 @@ def info(msg, *args, **kwargs): def debug(msg, *args, **kwargs): """ - Log a message with severity 'DEBUG' on the root logger. + Log a message with severity 'DEBUG' on the root logger. If the logger has + no handlers, call basicConfig() to add a console handler with a pre-defined + format. """ if len(root.handlers) == 0: basicConfig() @@ -1484,7 +1765,9 @@ def debug(msg, *args, **kwargs): def log(level, msg, *args, **kwargs): """ - Log 'msg % args' with the integer severity 'level' on the root logger. + Log 'msg % args' with the integer severity 'level' on the root logger. If + the logger has no handlers, call basicConfig() to add a console handler + with a pre-defined format. """ if len(root.handlers) == 0: basicConfig() @@ -1492,7 +1775,7 @@ def log(level, msg, *args, **kwargs): def disable(level): """ - Disable all logging calls less severe than 'level'. + Disable all logging calls of severity 'level' and below. """ root.manager.disable = level @@ -1503,32 +1786,32 @@ def shutdown(handlerList=_handlerList): Should be called at application exit. """ - for h in handlerList[:]: + for wr in reversed(handlerList[:]): #errors might occur, for example, if files are locked #we just ignore them if raiseExceptions is not set try: - h.acquire() - h.flush() - h.close() + h = wr() + if h: + try: + h.acquire() + h.flush() + h.close() + except (IOError, ValueError): + # Ignore errors which might be caused + # because handlers have been closed but + # references to them are still around at + # application exit. + pass + finally: + h.release() except: if raiseExceptions: raise #else, swallow - finally: - h.release() #Let's try and shutdown automatically on application exit... -try: - import atexit - atexit.register(shutdown) -except ImportError: # for Python versions < 2.0 - def exithook(status, old_exit=sys.exit): - try: - shutdown() - finally: - old_exit(status) - - sys.exit = exithook +import atexit +atexit.register(shutdown) # Null handler diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 25f34ec..c7359ca 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2007 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2010 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -24,7 +24,8 @@ Copyright (C) 2001-2010 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ -import sys, logging, logging.handlers, socket, struct, os, traceback +import sys, logging, logging.handlers, socket, struct, os, traceback, re +import types, io try: import _thread as thread @@ -57,16 +58,12 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True): the ability to select from various pre-canned configurations (if the developer provides a mechanism to present the choices and load the chosen configuration). - In versions of ConfigParser which have the readfp method [typically - shipped in 2.x versions of Python], you can pass in a file-like object - rather than a filename, in which case the file-like object will be read - using readfp. """ import configparser cp = configparser.ConfigParser(defaults) - if hasattr(cp, 'readfp') and hasattr(fname, 'readline'): - cp.readfp(fname) + if hasattr(fname, 'readline'): + cp.read_file(fname) else: cp.read(fname) @@ -101,9 +98,12 @@ def _resolve(name): def _strip_spaces(alist): return map(lambda x: x.strip(), alist) +def _encoded(s): + return s if isinstance(s, str) else s.encode('utf-8') + def _create_formatters(cp): """Create and return formatters""" - flist = cp.get("formatters", "keys") + flist = cp["formatters"]["keys"] if not len(flist): return {} flist = flist.split(",") @@ -111,20 +111,12 @@ def _create_formatters(cp): formatters = {} for form in flist: sectname = "formatter_%s" % form - opts = cp.options(sectname) - if "format" in opts: - fs = cp.get(sectname, "format", 1) - else: - fs = None - if "datefmt" in opts: - dfs = cp.get(sectname, "datefmt", 1) - else: - dfs = None + fs = cp.get(sectname, "format", raw=True, fallback=None) + dfs = cp.get(sectname, "datefmt", raw=True, fallback=None) c = logging.Formatter - if "class" in opts: - class_name = cp.get(sectname, "class") - if class_name: - c = _resolve(class_name) + class_name = cp[sectname].get("class") + if class_name: + c = _resolve(class_name) f = c(fs, dfs) formatters[form] = f return formatters @@ -132,7 +124,7 @@ def _create_formatters(cp): def _install_handlers(cp, formatters): """Install and return handlers""" - hlist = cp.get("handlers", "keys") + hlist = cp["handlers"]["keys"] if not len(hlist): return {} hlist = hlist.split(",") @@ -140,30 +132,23 @@ def _install_handlers(cp, formatters): handlers = {} fixups = [] #for inter-handler references for hand in hlist: - sectname = "handler_%s" % hand - klass = cp.get(sectname, "class") - opts = cp.options(sectname) - if "formatter" in opts: - fmt = cp.get(sectname, "formatter") - else: - fmt = "" + section = cp["handler_%s" % hand] + klass = section["class"] + fmt = section.get("formatter", "") try: klass = eval(klass, vars(logging)) except (AttributeError, NameError): klass = _resolve(klass) - args = cp.get(sectname, "args") + args = section["args"] args = eval(args, vars(logging)) h = klass(*args) - if "level" in opts: - level = cp.get(sectname, "level") + if "level" in section: + level = section["level"] h.setLevel(logging._levelNames[level]) if len(fmt): h.setFormatter(formatters[fmt]) if issubclass(klass, logging.handlers.MemoryHandler): - if "target" in opts: - target = cp.get(sectname,"target") - else: - target = "" + target = section.get("target", "") if len(target): #the target handler may not be loaded yet, so keep for later... fixups.append((h, target)) handlers[hand] = h @@ -172,25 +157,44 @@ def _install_handlers(cp, formatters): h.setTarget(handlers[t]) return handlers +def _handle_existing_loggers(existing, child_loggers, disable_existing): + """ + When (re)configuring logging, handle loggers which were in the previous + configuration but are not in the new configuration. There's no point + deleting them as other threads may continue to hold references to them; + and by disabling them, you stop them doing any logging. + + However, don't disable children of named loggers, as that's probably not + what was intended by the user. Also, allow existing loggers to NOT be + disabled if disable_existing is false. + """ + root = logging.root + for log in existing: + logger = root.manager.loggerDict[log] + if log in child_loggers: + logger.level = logging.NOTSET + logger.handlers = [] + logger.propagate = True + elif disable_existing: + logger.disabled = True -def _install_loggers(cp, handlers, disable_existing_loggers): +def _install_loggers(cp, handlers, disable_existing): """Create and install loggers""" # configure the root first - llist = cp.get("loggers", "keys") + llist = cp["loggers"]["keys"] llist = llist.split(",") llist = list(map(lambda x: x.strip(), llist)) llist.remove("root") - sectname = "logger_root" + section = cp["logger_root"] root = logging.root log = root - opts = cp.options(sectname) - if "level" in opts: - level = cp.get(sectname, "level") + if "level" in section: + level = section["level"] log.setLevel(logging._levelNames[level]) for h in root.handlers[:]: root.removeHandler(h) - hlist = cp.get(sectname, "handlers") + hlist = section["handlers"] if len(hlist): hlist = hlist.split(",") hlist = _strip_spaces(hlist) @@ -211,19 +215,15 @@ def _install_loggers(cp, handlers, disable_existing_loggers): #avoid disabling child loggers of explicitly #named loggers. With a sorted list it is easier #to find the child loggers. - existing.sort() + existing.sort(key=_encoded) #We'll keep the list of existing loggers #which are children of named loggers here... child_loggers = [] #now set up the new ones... for log in llist: - sectname = "logger_%s" % log - qn = cp.get(sectname, "qualname") - opts = cp.options(sectname) - if "propagate" in opts: - propagate = cp.getint(sectname, "propagate") - else: - propagate = 1 + section = cp["logger_%s" % log] + qn = section["qualname"] + propagate = section.getint("propagate", fallback=1) logger = logging.getLogger(qn) if qn in existing: i = existing.index(qn) + 1 # start with the entry after qn @@ -235,14 +235,14 @@ def _install_loggers(cp, handlers, disable_existing_loggers): child_loggers.append(existing[i]) i += 1 existing.remove(qn) - if "level" in opts: - level = cp.get(sectname, "level") + if "level" in section: + level = section["level"] logger.setLevel(logging._levelNames[level]) for h in logger.handlers[:]: logger.removeHandler(h) logger.propagate = propagate logger.disabled = 0 - hlist = cp.get(sectname, "handlers") + hlist = section["handlers"] if len(hlist): hlist = hlist.split(",") hlist = _strip_spaces(hlist) @@ -254,14 +254,526 @@ def _install_loggers(cp, handlers, disable_existing_loggers): #and by disabling them, you stop them doing any logging. #However, don't disable children of named loggers, as that's #probably not what was intended by the user. - for log in existing: - logger = root.manager.loggerDict[log] - if log in child_loggers: - logger.level = logging.NOTSET - logger.handlers = [] - logger.propagate = 1 - elif disable_existing_loggers: - logger.disabled = 1 + #for log in existing: + # logger = root.manager.loggerDict[log] + # if log in child_loggers: + # logger.level = logging.NOTSET + # logger.handlers = [] + # logger.propagate = 1 + # elif disable_existing_loggers: + # logger.disabled = 1 + _handle_existing_loggers(existing, child_loggers, disable_existing) + +IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I) + + +def valid_ident(s): + m = IDENTIFIER.match(s) + if not m: + raise ValueError('Not a valid Python identifier: %r' % s) + return True + + +# The ConvertingXXX classes are wrappers around standard Python containers, +# and they serve to convert any suitable values in the container. The +# conversion converts base dicts, lists and tuples to their wrapped +# equivalents, whereas strings which match a conversion format are converted +# appropriately. +# +# Each wrapper should have a configurator attribute holding the actual +# configurator to use for conversion. + +class ConvertingDict(dict): + """A converting dictionary wrapper.""" + + def __getitem__(self, key): + value = dict.__getitem__(self, key) + result = self.configurator.convert(value) + #If the converted value is different, save for next time + if value is not result: + self[key] = result + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result + + def get(self, key, default=None): + value = dict.get(self, key, default) + result = self.configurator.convert(value) + #If the converted value is different, save for next time + if value is not result: + self[key] = result + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result + + def pop(self, key, default=None): + value = dict.pop(self, key, default) + result = self.configurator.convert(value) + if value is not result: + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result + +class ConvertingList(list): + """A converting list wrapper.""" + def __getitem__(self, key): + value = list.__getitem__(self, key) + result = self.configurator.convert(value) + #If the converted value is different, save for next time + if value is not result: + self[key] = result + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result + + def pop(self, idx=-1): + value = list.pop(self, idx) + result = self.configurator.convert(value) + if value is not result: + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + return result + +class ConvertingTuple(tuple): + """A converting tuple wrapper.""" + def __getitem__(self, key): + value = tuple.__getitem__(self, key) + result = self.configurator.convert(value) + if value is not result: + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result + +class BaseConfigurator(object): + """ + The configurator base class which defines some useful defaults. + """ + + CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$') + + WORD_PATTERN = re.compile(r'^\s*(\w+)\s*') + DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*') + INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*') + DIGIT_PATTERN = re.compile(r'^\d+$') + + value_converters = { + 'ext' : 'ext_convert', + 'cfg' : 'cfg_convert', + } + + # We might want to use a different one, e.g. importlib + importer = staticmethod(__import__) + + def __init__(self, config): + self.config = ConvertingDict(config) + self.config.configurator = self + + def resolve(self, s): + """ + Resolve strings to objects using standard import and attribute + syntax. + """ + name = s.split('.') + used = name.pop(0) + try: + found = self.importer(used) + for frag in name: + used += '.' + frag + try: + found = getattr(found, frag) + except AttributeError: + self.importer(used) + found = getattr(found, frag) + return found + except ImportError: + e, tb = sys.exc_info()[1:] + v = ValueError('Cannot resolve %r: %s' % (s, e)) + v.__cause__, v.__traceback__ = e, tb + raise v + + def ext_convert(self, value): + """Default converter for the ext:// protocol.""" + return self.resolve(value) + + def cfg_convert(self, value): + """Default converter for the cfg:// protocol.""" + rest = value + m = self.WORD_PATTERN.match(rest) + if m is None: + raise ValueError("Unable to convert %r" % value) + else: + rest = rest[m.end():] + d = self.config[m.groups()[0]] + #print d, rest + while rest: + m = self.DOT_PATTERN.match(rest) + if m: + d = d[m.groups()[0]] + else: + m = self.INDEX_PATTERN.match(rest) + if m: + idx = m.groups()[0] + if not self.DIGIT_PATTERN.match(idx): + d = d[idx] + else: + try: + n = int(idx) # try as number first (most likely) + d = d[n] + except TypeError: + d = d[idx] + if m: + rest = rest[m.end():] + else: + raise ValueError('Unable to convert ' + '%r at %r' % (value, rest)) + #rest should be empty + return d + + def convert(self, value): + """ + Convert values to an appropriate type. dicts, lists and tuples are + replaced by their converting alternatives. Strings are checked to + see if they have a conversion format and are converted if they do. + """ + if not isinstance(value, ConvertingDict) and isinstance(value, dict): + value = ConvertingDict(value) + value.configurator = self + elif not isinstance(value, ConvertingList) and isinstance(value, list): + value = ConvertingList(value) + value.configurator = self + elif not isinstance(value, ConvertingTuple) and\ + isinstance(value, tuple): + value = ConvertingTuple(value) + value.configurator = self + elif isinstance(value, str): # str for py3k + m = self.CONVERT_PATTERN.match(value) + if m: + d = m.groupdict() + prefix = d['prefix'] + converter = self.value_converters.get(prefix, None) + if converter: + suffix = d['suffix'] + converter = getattr(self, converter) + value = converter(suffix) + return value + + def configure_custom(self, config): + """Configure an object with a user-supplied factory.""" + c = config.pop('()') + if not hasattr(c, '__call__'): + c = self.resolve(c) + props = config.pop('.', None) + # Check for valid identifiers + kwargs = dict([(k, config[k]) for k in config if valid_ident(k)]) + result = c(**kwargs) + if props: + for name, value in props.items(): + setattr(result, name, value) + return result + + def as_tuple(self, value): + """Utility function which converts lists to tuples.""" + if isinstance(value, list): + value = tuple(value) + return value + +class DictConfigurator(BaseConfigurator): + """ + Configure logging using a dictionary-like object to describe the + configuration. + """ + + def configure(self): + """Do the configuration.""" + + config = self.config + if 'version' not in config: + raise ValueError("dictionary doesn't specify a version") + if config['version'] != 1: + raise ValueError("Unsupported version: %s" % config['version']) + incremental = config.pop('incremental', False) + EMPTY_DICT = {} + logging._acquireLock() + try: + if incremental: + handlers = config.get('handlers', EMPTY_DICT) + for name in handlers: + if name not in logging._handlers: + raise ValueError('No handler found with ' + 'name %r' % name) + else: + try: + handler = logging._handlers[name] + handler_config = handlers[name] + level = handler_config.get('level', None) + if level: + handler.setLevel(logging._checkLevel(level)) + except Exception as e: + raise ValueError('Unable to configure handler ' + '%r: %s' % (name, e)) + loggers = config.get('loggers', EMPTY_DICT) + for name in loggers: + try: + self.configure_logger(name, loggers[name], True) + except Exception as e: + raise ValueError('Unable to configure logger ' + '%r: %s' % (name, e)) + root = config.get('root', None) + if root: + try: + self.configure_root(root, True) + except Exception as e: + raise ValueError('Unable to configure root ' + 'logger: %s' % e) + else: + disable_existing = config.pop('disable_existing_loggers', True) + + logging._handlers.clear() + del logging._handlerList[:] + + # Do formatters first - they don't refer to anything else + formatters = config.get('formatters', EMPTY_DICT) + for name in formatters: + try: + formatters[name] = self.configure_formatter( + formatters[name]) + except Exception as e: + raise ValueError('Unable to configure ' + 'formatter %r: %s' % (name, e)) + # Next, do filters - they don't refer to anything else, either + filters = config.get('filters', EMPTY_DICT) + for name in filters: + try: + filters[name] = self.configure_filter(filters[name]) + except Exception as e: + raise ValueError('Unable to configure ' + 'filter %r: %s' % (name, e)) + + # Next, do handlers - they refer to formatters and filters + # As handlers can refer to other handlers, sort the keys + # to allow a deterministic order of configuration + handlers = config.get('handlers', EMPTY_DICT) + for name in sorted(handlers): + try: + handler = self.configure_handler(handlers[name]) + handler.name = name + handlers[name] = handler + except Exception as e: + raise ValueError('Unable to configure handler ' + '%r: %s' % (name, e)) + # Next, do loggers - they refer to handlers and filters + + #we don't want to lose the existing loggers, + #since other threads may have pointers to them. + #existing is set to contain all existing loggers, + #and as we go through the new configuration we + #remove any which are configured. At the end, + #what's left in existing is the set of loggers + #which were in the previous configuration but + #which are not in the new configuration. + root = logging.root + existing = list(root.manager.loggerDict.keys()) + #The list needs to be sorted so that we can + #avoid disabling child loggers of explicitly + #named loggers. With a sorted list it is easier + #to find the child loggers. + existing.sort(key=_encoded) + #We'll keep the list of existing loggers + #which are children of named loggers here... + child_loggers = [] + #now set up the new ones... + loggers = config.get('loggers', EMPTY_DICT) + for name in loggers: + if name in existing: + i = existing.index(name) + 1 # look after name + prefixed = name + "." + pflen = len(prefixed) + num_existing = len(existing) + while i < num_existing: + if existing[i][:pflen] == prefixed: + child_loggers.append(existing[i]) + i += 1 + existing.remove(name) + try: + self.configure_logger(name, loggers[name]) + except Exception as e: + raise ValueError('Unable to configure logger ' + '%r: %s' % (name, e)) + + #Disable any old loggers. There's no point deleting + #them as other threads may continue to hold references + #and by disabling them, you stop them doing any logging. + #However, don't disable children of named loggers, as that's + #probably not what was intended by the user. + #for log in existing: + # logger = root.manager.loggerDict[log] + # if log in child_loggers: + # logger.level = logging.NOTSET + # logger.handlers = [] + # logger.propagate = True + # elif disable_existing: + # logger.disabled = True + _handle_existing_loggers(existing, child_loggers, + disable_existing) + + # And finally, do the root logger + root = config.get('root', None) + if root: + try: + self.configure_root(root) + except Exception as e: + raise ValueError('Unable to configure root ' + 'logger: %s' % e) + finally: + logging._releaseLock() + + def configure_formatter(self, config): + """Configure a formatter from a dictionary.""" + if '()' in config: + factory = config['()'] # for use in exception handler + try: + result = self.configure_custom(config) + except TypeError as te: + if "'format'" not in str(te): + raise + #Name of parameter changed from fmt to format. + #Retry with old name. + #This is so that code can be used with older Python versions + #(e.g. by Django) + config['fmt'] = config.pop('format') + config['()'] = factory + result = self.configure_custom(config) + else: + fmt = config.get('format', None) + dfmt = config.get('datefmt', None) + result = logging.Formatter(fmt, dfmt) + return result + + def configure_filter(self, config): + """Configure a filter from a dictionary.""" + if '()' in config: + result = self.configure_custom(config) + else: + name = config.get('name', '') + result = logging.Filter(name) + return result + + def add_filters(self, filterer, filters): + """Add filters to a filterer from a list of names.""" + for f in filters: + try: + filterer.addFilter(self.config['filters'][f]) + except Exception as e: + raise ValueError('Unable to add filter %r: %s' % (f, e)) + + def configure_handler(self, config): + """Configure a handler from a dictionary.""" + formatter = config.pop('formatter', None) + if formatter: + try: + formatter = self.config['formatters'][formatter] + except Exception as e: + raise ValueError('Unable to set formatter ' + '%r: %s' % (formatter, e)) + level = config.pop('level', None) + filters = config.pop('filters', None) + if '()' in config: + c = config.pop('()') + if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType: + c = self.resolve(c) + factory = c + else: + klass = self.resolve(config.pop('class')) + #Special case for handler which refers to another handler + if issubclass(klass, logging.handlers.MemoryHandler) and\ + 'target' in config: + try: + config['target'] = self.config['handlers'][config['target']] + except Exception as e: + raise ValueError('Unable to set target handler ' + '%r: %s' % (config['target'], e)) + elif issubclass(klass, logging.handlers.SMTPHandler) and\ + 'mailhost' in config: + config['mailhost'] = self.as_tuple(config['mailhost']) + elif issubclass(klass, logging.handlers.SysLogHandler) and\ + 'address' in config: + config['address'] = self.as_tuple(config['address']) + factory = klass + kwargs = dict([(k, config[k]) for k in config if valid_ident(k)]) + try: + result = factory(**kwargs) + except TypeError as te: + if "'stream'" not in str(te): + raise + #The argument name changed from strm to stream + #Retry with old name. + #This is so that code can be used with older Python versions + #(e.g. by Django) + kwargs['strm'] = kwargs.pop('stream') + result = factory(**kwargs) + if formatter: + result.setFormatter(formatter) + if level is not None: + result.setLevel(logging._checkLevel(level)) + if filters: + self.add_filters(result, filters) + return result + + def add_handlers(self, logger, handlers): + """Add handlers to a logger from a list of names.""" + for h in handlers: + try: + logger.addHandler(self.config['handlers'][h]) + except Exception as e: + raise ValueError('Unable to add handler %r: %s' % (h, e)) + + def common_logger_config(self, logger, config, incremental=False): + """ + Perform configuration which is common to root and non-root loggers. + """ + level = config.get('level', None) + if level is not None: + logger.setLevel(logging._checkLevel(level)) + if not incremental: + #Remove any existing handlers + for h in logger.handlers[:]: + logger.removeHandler(h) + handlers = config.get('handlers', None) + if handlers: + self.add_handlers(logger, handlers) + filters = config.get('filters', None) + if filters: + self.add_filters(logger, filters) + + def configure_logger(self, name, config, incremental=False): + """Configure a non-root logger from a dictionary.""" + logger = logging.getLogger(name) + self.common_logger_config(logger, config, incremental) + propagate = config.get('propagate', None) + if propagate is not None: + logger.propagate = propagate + + def configure_root(self, config, incremental=False): + """Configure a root logger from a dictionary.""" + root = logging.getLogger() + self.common_logger_config(root, config, incremental) + +dictConfigClass = DictConfigurator + +def dictConfig(config): + """Configure logging using a dictionary.""" + dictConfigClass(config).configure() def listen(port=DEFAULT_LOGGING_CONFIG_PORT): @@ -301,24 +813,26 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT): chunk = self.connection.recv(slen) while len(chunk) < slen: chunk = chunk + conn.recv(slen - len(chunk)) - #Apply new configuration. We'd like to be able to - #create a StringIO and pass that in, but unfortunately - #1.5.2 ConfigParser does not support reading file - #objects, only actual files. So we create a temporary - #file and remove it later. - file = tempfile.mktemp(".ini") - f = open(file, "w") - f.write(chunk) - f.close() + chunk = chunk.decode("utf-8") try: - fileConfig(file) - except (KeyboardInterrupt, SystemExit): - raise + import json + d =json.loads(chunk) + assert isinstance(d, dict) + dictConfig(d) except: - traceback.print_exc() - os.remove(file) + #Apply new configuration. + + file = io.StringIO(chunk) + try: + fileConfig(file) + except (KeyboardInterrupt, SystemExit): + raise + except: + traceback.print_exc() + if self.server.ready: + self.server.ready.set() except socket.error as e: - if not isinstancetype(e.args, tuple): + if not isinstance(e.args, tuple): raise else: errcode = e.args[0] @@ -333,12 +847,13 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT): allow_reuse_address = 1 def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, - handler=None): + handler=None, ready=None): ThreadingTCPServer.__init__(self, (host, port), handler) logging._acquireLock() self.abort = 0 logging._releaseLock() self.timeout = 1 + self.ready = ready def serve_until_stopped(self): import select @@ -354,17 +869,28 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT): logging._releaseLock() self.socket.close() - def serve(rcvr, hdlr, port): - server = rcvr(port=port, handler=hdlr) - global _listener - logging._acquireLock() - _listener = server - logging._releaseLock() - server.serve_until_stopped() + class Server(threading.Thread): + + def __init__(self, rcvr, hdlr, port): + super(Server, self).__init__() + self.rcvr = rcvr + self.hdlr = hdlr + self.port = port + self.ready = threading.Event() + + def run(self): + server = self.rcvr(port=self.port, handler=self.hdlr, + ready=self.ready) + if self.port == 0: + self.port = server.server_address[1] + self.ready.set() + global _listener + logging._acquireLock() + _listener = server + logging._releaseLock() + server.serve_until_stopped() - return threading.Thread(target=serve, - args=(ConfigSocketReceiver, - ConfigStreamHandler, port)) + return Server(ConfigSocketReceiver, ConfigStreamHandler, port) def stopListening(): """ diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index e9dac43..96384bd 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1,4 +1,4 @@ -# Copyright 2001-2007 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2010 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -19,7 +19,7 @@ Additional handlers for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python, and influenced by Apache's log4j system. -Copyright (C) 2001-2009 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2010 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ @@ -41,6 +41,7 @@ DEFAULT_UDP_LOGGING_PORT = 9021 DEFAULT_HTTP_LOGGING_PORT = 9022 DEFAULT_SOAP_LOGGING_PORT = 9023 SYSLOG_UDP_PORT = 514 +SYSLOG_TCP_PORT = 514 _MIDNIGHT = 24 * 60 * 60 # number of seconds in a day @@ -102,8 +103,13 @@ class RotatingFileHandler(BaseRotatingHandler): If maxBytes is zero, rollover never occurs. """ + # If rotation/rollover is wanted, it doesn't make sense to use another + # mode. If for example 'w' were specified, then if there were multiple + # runs of the calling application, the logs from previous runs would be + # lost if the 'w' is respected, because the log file would be truncated + # on each run. if maxBytes > 0: - mode = 'a' # doesn't make sense otherwise! + mode = 'a' BaseRotatingHandler.__init__(self, filename, mode, encoding, delay) self.maxBytes = maxBytes self.backupCount = backupCount @@ -120,7 +126,6 @@ class RotatingFileHandler(BaseRotatingHandler): sfn = "%s.%d" % (self.baseFilename, i) dfn = "%s.%d" % (self.baseFilename, i + 1) if os.path.exists(sfn): - #print "%s -> %s" % (sfn, dfn) if os.path.exists(dfn): os.remove(dfn) os.rename(sfn, dfn) @@ -128,7 +133,6 @@ class RotatingFileHandler(BaseRotatingHandler): if os.path.exists(dfn): os.remove(dfn) os.rename(self.baseFilename, dfn) - #print "%s -> %s" % (self.baseFilename, dfn) self.mode = 'w' self.stream = self._open() @@ -156,7 +160,7 @@ class TimedRotatingFileHandler(BaseRotatingHandler): If backupCount is > 0, when rollover is done, no more than backupCount files are kept - the oldest ones are deleted. """ - def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=0, utc=False): + def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False): BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) self.when = when.upper() self.backupCount = backupCount @@ -278,7 +282,6 @@ class TimedRotatingFileHandler(BaseRotatingHandler): t = int(time.time()) if t >= self.rolloverAt: return 1 - #print "No need to rollover: %d, %d" % (t, self.rolloverAt) return 0 def getFilesToDelete(self): @@ -326,14 +329,8 @@ class TimedRotatingFileHandler(BaseRotatingHandler): os.remove(dfn) os.rename(self.baseFilename, dfn) if self.backupCount > 0: - # find the oldest log file and delete it - #s = glob.glob(self.baseFilename + ".20*") - #if len(s) > self.backupCount: - # s.sort() - # os.remove(s[0]) for s in self.getFilesToDelete(): os.remove(s) - #print "%s -> %s" % (self.baseFilename, dfn) self.mode = 'w' self.stream = self._open() currentTime = int(time.time()) @@ -634,7 +631,8 @@ class SysLogHandler(logging.Handler): LOG_NEWS = 7 # network news subsystem LOG_UUCP = 8 # UUCP subsystem LOG_CRON = 9 # clock daemon - LOG_AUTHPRIV = 10 # security/authorization messages (private) + LOG_AUTHPRIV = 10 # security/authorization messages (private) + LOG_FTP = 11 # FTP daemon # other codes through 15 reserved for system use LOG_LOCAL0 = 16 # reserved for local use @@ -666,6 +664,7 @@ class SysLogHandler(logging.Handler): "authpriv": LOG_AUTHPRIV, "cron": LOG_CRON, "daemon": LOG_DAEMON, + "ftp": LOG_FTP, "kern": LOG_KERN, "lpr": LOG_LPR, "mail": LOG_MAIL, @@ -696,7 +695,8 @@ class SysLogHandler(logging.Handler): "CRITICAL" : "critical" } - def __init__(self, address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER): + def __init__(self, address=('localhost', SYSLOG_UDP_PORT), + facility=LOG_USER, socktype=socket.SOCK_DGRAM): """ Initialize a handler. @@ -708,13 +708,16 @@ class SysLogHandler(logging.Handler): self.address = address self.facility = facility + self.socktype = socktype + if isinstance(address, str): self.unixsocket = 1 self._connect_unixsocket(address) else: self.unixsocket = 0 - self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - + self.socket = socket.socket(socket.AF_INET, socktype) + if socktype == socket.SOCK_STREAM: + self.socket.connect(address) self.formatter = None def _connect_unixsocket(self, address): @@ -727,12 +730,6 @@ class SysLogHandler(logging.Handler): self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.socket.connect(address) - # curious: when talking to the unix-domain '/dev/log' socket, a - # zero-terminator seems to be required. this string is placed - # into a class variable so that it can be overridden if - # necessary. - log_format_string = '<%d>%s\000' - def encodePriority(self, facility, priority): """ Encode the facility and priority. You can pass in strings or @@ -771,18 +768,19 @@ class SysLogHandler(logging.Handler): The record is formatted, and then sent to the syslog server. If exception information is present, it is NOT sent to the server. """ - msg = self.format(record) + msg = self.format(record) + '\000' """ We need to convert record level to lowercase, maybe this will change in the future. """ - msg = self.log_format_string % ( - self.encodePriority(self.facility, - self.mapPriority(record.levelname)), - msg) + prio = '<%d>' % self.encodePriority(self.facility, + self.mapPriority(record.levelname)) + prio = prio.encode('utf-8') + # Message is a string. Convert to bytes as required by RFC 5424 msg = msg.encode('utf-8') if codecs: msg = codecs.BOM_UTF8 + msg + msg = prio + msg try: if self.unixsocket: try: @@ -790,8 +788,10 @@ class SysLogHandler(logging.Handler): except socket.error: self._connect_unixsocket(self.address) self.socket.send(msg) - else: + elif self.socktype == socket.SOCK_DGRAM: self.socket.sendto(msg, self.address) + else: + self.socket.sendall(msg) except (KeyboardInterrupt, SystemExit): raise except: @@ -801,7 +801,8 @@ class SMTPHandler(logging.Handler): """ A handler class which sends an SMTP email for each logging event. """ - def __init__(self, mailhost, fromaddr, toaddrs, subject, credentials=None): + def __init__(self, mailhost, fromaddr, toaddrs, subject, + credentials=None, secure=None): """ Initialize the handler. @@ -809,7 +810,12 @@ class SMTPHandler(logging.Handler): line of the email. To specify a non-standard SMTP port, use the (host, port) tuple format for the mailhost argument. To specify authentication credentials, supply a (username, password) tuple - for the credentials argument. + for the credentials argument. To specify the use of a secure + protocol (TLS), pass in a tuple for the secure argument. This will + only be used when authentication credentials are supplied. The tuple + will be either an empty tuple, or a single-value tuple with the name + of a keyfile, or a 2-value tuple with the names of the keyfile and + certificate file. (This tuple is passed to the `starttls` method). """ logging.Handler.__init__(self) if isinstance(mailhost, tuple): @@ -825,6 +831,7 @@ class SMTPHandler(logging.Handler): toaddrs = [toaddrs] self.toaddrs = toaddrs self.subject = subject + self.secure = secure def getSubject(self, record): """ @@ -835,24 +842,6 @@ class SMTPHandler(logging.Handler): """ return self.subject - weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] - - monthname = [None, - 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', - 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] - - def date_time(self): - """ - Return the current date and time formatted for a MIME header. - Needed for Python 1.5.2 (no email package available) - """ - year, month, day, hh, mm, ss, wd, y, z = time.gmtime(time.time()) - s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( - self.weekdayname[wd], - day, self.monthname[month], year, - hh, mm, ss) - return s - def emit(self, record): """ Emit a record. @@ -861,10 +850,7 @@ class SMTPHandler(logging.Handler): """ try: import smtplib - try: - from email.utils import formatdate - except ImportError: - formatdate = self.date_time + from email.utils import formatdate port = self.mailport if not port: port = smtplib.SMTP_PORT @@ -876,6 +862,10 @@ class SMTPHandler(logging.Handler): self.getSubject(record), formatdate(), msg) if self.username: + if self.secure is not None: + smtp.ehlo() + smtp.starttls(*self.secure) + smtp.ehlo() smtp.login(self.username, self.password) smtp.sendmail(self.fromaddr, self.toaddrs, msg) smtp.quit() @@ -989,7 +979,7 @@ class HTTPHandler(logging.Handler): A class which sends records to a Web server, using either GET or POST semantics. """ - def __init__(self, host, url, method="GET"): + def __init__(self, host, url, method="GET", secure=False, credentials=None): """ Initialize the instance with the host, the request URL, and the method ("GET" or "POST") @@ -1001,12 +991,14 @@ class HTTPHandler(logging.Handler): self.host = host self.url = url self.method = method + self.secure = secure + self.credentials = credentials def mapLogRecord(self, record): """ Default implementation of mapping the log record into a dict that is sent as the CGI data. Overwrite in your class. - Contributed by Franz Glasner. + Contributed by Franz Glasner. """ return record.__dict__ @@ -1019,7 +1011,10 @@ class HTTPHandler(logging.Handler): try: import http.client, urllib.parse host = self.host - h = http.client.HTTP(host) + if self.secure: + h = http.client.HTTPSConnection(host) + else: + h = http.client.HTTPConnection(host) url = self.url data = urllib.parse.urlencode(self.mapLogRecord(record)) if self.method == "GET": @@ -1039,8 +1034,13 @@ class HTTPHandler(logging.Handler): h.putheader("Content-type", "application/x-www-form-urlencoded") h.putheader("Content-length", str(len(data))) + if self.credentials: + import base64 + s = ('u%s:%s' % self.credentials).encode('utf-8') + s = 'Basic ' + base64.b64encode(s).strip() + h.putheader('Authorization', s) h.endheaders(data if self.method == "POST" else None) - h.getreply() #can't do anything with the result + h.getresponse() #can't do anything with the result except (KeyboardInterrupt, SystemExit): raise except: @@ -1133,6 +1133,8 @@ class MemoryHandler(BufferingHandler): For a MemoryHandler, flushing means just sending the buffered records to the target, if there is one. Override if you want different behaviour. + + The record buffer is also cleared by this operation. """ if self.target: for record in self.buffer: @@ -1146,3 +1148,174 @@ class MemoryHandler(BufferingHandler): self.flush() self.target = None BufferingHandler.close(self) + + +class QueueHandler(logging.Handler): + """ + This handler sends events to a queue. Typically, it would be used together + with a multiprocessing Queue to centralise logging to file in one process + (in a multi-process application), so as to avoid file write contention + between processes. + + This code is new in Python 3.2, but this class can be copy pasted into + user code for use with earlier Python versions. + """ + + def __init__(self, queue): + """ + Initialise an instance, using the passed queue. + """ + logging.Handler.__init__(self) + self.queue = queue + + def enqueue(self, record): + """ + Enqueue a record. + + The base implementation uses put_nowait. You may want to override + this method if you want to use blocking, timeouts or custom queue + implementations. + """ + self.queue.put_nowait(record) + + def prepare(self, record): + """ + Prepares a record for queuing. The object returned by this method is + enqueued. + + The base implementation formats the record to merge the message + and arguments, and removes unpickleable items from the record + in-place. + + You might want to override this method if you want to convert + the record to a dict or JSON string, or send a modified copy + of the record while leaving the original intact. + """ + # The format operation gets traceback text into record.exc_text + # (if there's exception data), and also puts the message into + # record.message. We can then use this to replace the original + # msg + args, as these might be unpickleable. We also zap the + # exc_info attribute, as it's no longer needed and, if not None, + # will typically not be pickleable. + self.format(record) + record.msg = record.message + record.args = None + record.exc_info = None + return record + + def emit(self, record): + """ + Emit a record. + + Writes the LogRecord to the queue, preparing it for pickling first. + """ + try: + self.enqueue(self.prepare(record)) + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + +class QueueListener(object): + """ + This class implements an internal threaded listener which watches for + LogRecords being added to a queue, removes them and passes them to a + list of handlers for processing. + """ + _sentinel = None + + def __init__(self, queue, *handlers): + """ + Initialise an instance with the specified queue and + handlers. + """ + self.queue = queue + self.handlers = handlers + self._stop = threading.Event() + self._thread = None + + def dequeue(self, block): + """ + Dequeue a record and return it, optionally blocking. + + The base implementation uses get. You may want to override this method + if you want to use timeouts or work with custom queue implementations. + """ + return self.queue.get(block) + + def start(self): + """ + Start the listener. + + This starts up a background thread to monitor the queue for + LogRecords to process. + """ + self._thread = t = threading.Thread(target=self._monitor) + t.setDaemon(True) + t.start() + + def prepare(self , record): + """ + Prepare a record for handling. + + This method just returns the passed-in record. You may want to + override this method if you need to do any custom marshalling or + manipulation of the record before passing it to the handlers. + """ + return record + + def handle(self, record): + """ + Handle a record. + + This just loops through the handlers offering them the record + to handle. + """ + record = self.prepare(record) + for handler in self.handlers: + handler.handle(record) + + def _monitor(self): + """ + Monitor the queue for records, and ask the handler + to deal with them. + + This method runs on a separate, internal thread. + The thread will terminate if it sees a sentinel object in the queue. + """ + q = self.queue + has_task_done = hasattr(q, 'task_done') + while not self._stop.isSet(): + try: + record = self.dequeue(True) + if record is self._sentinel: + break + self.handle(record) + if has_task_done: + q.task_done() + except queue.Empty: + pass + # There might still be records in the queue. + while True: + try: + record = self.dequeue(False) + if record is self._sentinel: + break + self.handle(record) + if has_task_done: + q.task_done() + except queue.Empty: + break + + def stop(self): + """ + Stop the listener. + + This asks the thread to terminate, and then waits for it to do so. + Note that if you don't call this before your application exits, there + may be some records still left on the queue, which won't be processed. + """ + self._stop.set() + self.queue.put_nowait(self._sentinel) + self._thread.join() + self._thread = None |