diff options
author | Vinay Sajip <vinay_sajip@yahoo.co.uk> | 2010-12-26 18:47:51 (GMT) |
---|---|---|
committer | Vinay Sajip <vinay_sajip@yahoo.co.uk> | 2010-12-26 18:47:51 (GMT) |
commit | e6f1e435d4e01d2fdb349610565e99b792ace1a8 (patch) | |
tree | 295c9b8db54d518c29fa4c569d000906fa162f1a /Doc | |
parent | 0d4bcf4c71cdb887cc6be4cbe8c72168a2952a4f (diff) | |
download | cpython-e6f1e435d4e01d2fdb349610565e99b792ace1a8.zip cpython-e6f1e435d4e01d2fdb349610565e99b792ace1a8.tar.gz cpython-e6f1e435d4e01d2fdb349610565e99b792ace1a8.tar.bz2 |
Improved logging cookbook for logging with multiprocessing.
Diffstat (limited to 'Doc')
-rw-r--r-- | Doc/howto/logging-cookbook.rst | 100 |
1 files changed, 100 insertions, 0 deletions
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 7bf7273..61492e4 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -807,6 +807,106 @@ the basis for code meeting your own specific requirements:: if __name__ == '__main__': main() +A variant of the above script keeps the logging in the main process, in a +separate thread:: + + import logging + import logging.config + import logging.handlers + from multiprocessing import Process, Queue + import random + import threading + import time + + def logger_thread(q): + while True: + record = q.get() + if record is None: + break + logger = logging.getLogger(record.name) + logger.handle(record) + + + def worker_process(q): + qh = logging.handlers.QueueHandler(q) + root = logging.getLogger() + root.setLevel(logging.DEBUG) + root.addHandler(qh) + levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, + logging.CRITICAL] + loggers = ['foo', 'foo.bar', 'foo.bar.baz', + 'spam', 'spam.ham', 'spam.ham.eggs'] + for i in range(100): + lvl = random.choice(levels) + logger = logging.getLogger(random.choice(loggers)) + logger.log(lvl, 'Message no. %d', i) + + if __name__ == '__main__': + q = Queue() + d = { + 'version': 1, + 'formatters': { + 'detailed': { + 'class': 'logging.Formatter', + 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'level': 'INFO', + }, + 'file': { + 'class': 'logging.FileHandler', + 'filename': 'mplog.log', + 'mode': 'w', + 'formatter': 'detailed', + }, + 'foofile': { + 'class': 'logging.FileHandler', + 'filename': 'mplog-foo.log', + 'mode': 'w', + 'formatter': 'detailed', + }, + 'errors': { + 'class': 'logging.FileHandler', + 'filename': 'mplog-errors.log', + 'mode': 'w', + 'level': 'ERROR', + 'formatter': 'detailed', + }, + }, + 'loggers': { + 'foo': { + 'handlers' : ['foofile'] + } + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['console', 'file', 'errors'] + }, + } + workers = [] + for i in range(5): + wp = Process(target=worker_process, name='worker %d' % (i + 1), args=(q,)) + workers.append(wp) + wp.start() + logging.config.dictConfig(d) + lp = threading.Thread(target=logger_thread, args=(q,)) + lp.start() + # At this point, the main process could do some useful work of its own + # Once it's done that, it can wait for the workers to terminate... + for wp in workers: + wp.join() + # And now tell the logging thread to finish up, too + q.put(None) + lp.join() + +This variant shows how you can e.g. apply configuration for particular loggers +- e.g. the ``foo`` logger has a special handler which stores all events in the +``foo`` subsystem in a file ``mplog-foo.log``. This will be used by the logging +machinery in the main process (even though the logging events are generated in +the worker processes) to direct the messages to the appropriate destinations. Using file rotation ------------------- |