summaryrefslogtreecommitdiffstats
path: root/Doc
diff options
context:
space:
mode:
authorVinay Sajip <vinay_sajip@yahoo.co.uk>2010-12-26 18:47:51 (GMT)
committerVinay Sajip <vinay_sajip@yahoo.co.uk>2010-12-26 18:47:51 (GMT)
commite6f1e435d4e01d2fdb349610565e99b792ace1a8 (patch)
tree295c9b8db54d518c29fa4c569d000906fa162f1a /Doc
parent0d4bcf4c71cdb887cc6be4cbe8c72168a2952a4f (diff)
downloadcpython-e6f1e435d4e01d2fdb349610565e99b792ace1a8.zip
cpython-e6f1e435d4e01d2fdb349610565e99b792ace1a8.tar.gz
cpython-e6f1e435d4e01d2fdb349610565e99b792ace1a8.tar.bz2
Improved logging cookbook for logging with multiprocessing.
Diffstat (limited to 'Doc')
-rw-r--r--Doc/howto/logging-cookbook.rst100
1 files changed, 100 insertions, 0 deletions
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index 7bf7273..61492e4 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -807,6 +807,106 @@ the basis for code meeting your own specific requirements::
if __name__ == '__main__':
main()
+A variant of the above script keeps the logging in the main process, in a
+separate thread::
+
+ import logging
+ import logging.config
+ import logging.handlers
+ from multiprocessing import Process, Queue
+ import random
+ import threading
+ import time
+
+ def logger_thread(q):
+ while True:
+ record = q.get()
+ if record is None:
+ break
+ logger = logging.getLogger(record.name)
+ logger.handle(record)
+
+
+ def worker_process(q):
+ qh = logging.handlers.QueueHandler(q)
+ root = logging.getLogger()
+ root.setLevel(logging.DEBUG)
+ root.addHandler(qh)
+ levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR,
+ logging.CRITICAL]
+ loggers = ['foo', 'foo.bar', 'foo.bar.baz',
+ 'spam', 'spam.ham', 'spam.ham.eggs']
+ for i in range(100):
+ lvl = random.choice(levels)
+ logger = logging.getLogger(random.choice(loggers))
+ logger.log(lvl, 'Message no. %d', i)
+
+ if __name__ == '__main__':
+ q = Queue()
+ d = {
+ 'version': 1,
+ 'formatters': {
+ 'detailed': {
+ 'class': 'logging.Formatter',
+ 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s'
+ }
+ },
+ 'handlers': {
+ 'console': {
+ 'class': 'logging.StreamHandler',
+ 'level': 'INFO',
+ },
+ 'file': {
+ 'class': 'logging.FileHandler',
+ 'filename': 'mplog.log',
+ 'mode': 'w',
+ 'formatter': 'detailed',
+ },
+ 'foofile': {
+ 'class': 'logging.FileHandler',
+ 'filename': 'mplog-foo.log',
+ 'mode': 'w',
+ 'formatter': 'detailed',
+ },
+ 'errors': {
+ 'class': 'logging.FileHandler',
+ 'filename': 'mplog-errors.log',
+ 'mode': 'w',
+ 'level': 'ERROR',
+ 'formatter': 'detailed',
+ },
+ },
+ 'loggers': {
+ 'foo': {
+ 'handlers' : ['foofile']
+ }
+ },
+ 'root': {
+ 'level': 'DEBUG',
+ 'handlers': ['console', 'file', 'errors']
+ },
+ }
+ workers = []
+ for i in range(5):
+ wp = Process(target=worker_process, name='worker %d' % (i + 1), args=(q,))
+ workers.append(wp)
+ wp.start()
+ logging.config.dictConfig(d)
+ lp = threading.Thread(target=logger_thread, args=(q,))
+ lp.start()
+ # At this point, the main process could do some useful work of its own
+ # Once it's done that, it can wait for the workers to terminate...
+ for wp in workers:
+ wp.join()
+ # And now tell the logging thread to finish up, too
+ q.put(None)
+ lp.join()
+
+This variant shows how you can e.g. apply configuration for particular loggers
+- e.g. the ``foo`` logger has a special handler which stores all events in the
+``foo`` subsystem in a file ``mplog-foo.log``. This will be used by the logging
+machinery in the main process (even though the logging events are generated in
+the worker processes) to direct the messages to the appropriate destinations.
Using file rotation
-------------------