Logging with Multiprocessing in python

Question

I am currently struggling to get a simple multiprocessing log working:

I am using the MultiProcessingLog from this answer: https://stackoverflow.com/a/894284/293195

I have a simple ConverterImage which should be able to spit out the log, which works, but the exceptions and tracelog never appear in the log? Does somebody know whats the problem here?

import os, traceback,logging
import multiprocessing, threading, logging, sys, traceback
from multiprocessing import Pool, Manager
from logging import FileHandler

class MultiProcessingLog(logging.Handler):
    def __init__(self, name, mode):
        logging.Handler.__init__(self)

        self._handler = FileHandler(name, mode)
        self.queue = multiprocessing.Queue(-1)

        t = threading.Thread(target=self.receive)
        t.daemon = True
        t.start()

    def setFormatter(self, fmt):
        logging.Handler.setFormatter(self, fmt)
        self._handler.setFormatter(fmt)

    def receive(self):
        while True:
            try:
                record = self.queue.get()
                self._handler.emit(record)
            except (KeyboardInterrupt, SystemExit):
                raise
            except EOFError:
                break
            except:
                traceback.print_exc(file=sys.stderr)

    def send(self, s):
        self.queue.put_nowait(s)

    def _format_record(self, record):
        # ensure that exc_info and args
        # have been stringified.  Removes any chance of
        # unpickleable things inside and possibly reduces
        # message size sent over the pipe
        if record.args:
            record.msg = record.msg % record.args
            record.args = None
        if record.exc_info:
            dummy = self.format(record)
            record.exc_info = None

        return record

    def emit(self, record):
        try:
            s = self._format_record(record)
            self.send(s)
        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            self.handleError(record)

    def close(self):
        self._handler.close()
        logging.Handler.close(self)


class ConvertImage:
    def __init__(self, logger=None):
        self.logger = logger

    def __call__(self,f):
        self.process(f)

    def process(self,f):
        try:
            logging.info("Process %i" % os.getpid() )
            raise NameError("Stupid error")

        except Exception as e:
            logging.info("Exception: " + e.message)
            exc_buffer = io.StringIO()
            traceback.print_exc(file=exc_buffer)
            logging.info(exc_buffer.getvalue())
            raise e
        except:
            logging.info("Exception!")
            exc_buffer = io.StringIO()
            traceback.print_exc(file=exc_buffer)
            logging.info(exc_buffer.getvalue())
            raise


mpl = MultiProcessingLog("ImageProcessing.log", mode='w+')
mpl.setFormatter( logging.Formatter('%(asctime)s - %(lineno)d - %(levelname)-8s - %(message)s') )
logger = logging.getLogger()
logger.addHandler(mpl)
logger.setLevel(logging.DEBUG)


pool = Pool();
converter = ConvertImage()
# map converter.process function over all files
result = pool.map_async(converter, ["A","B","C"]);

pool.close();
pool.join()
logging.shutdown()

score 0 · Answer 1 · answered Jun 29 '15 at 13:10

0

In ConvertImage, you use logging module functions instead of self.logger methods. Can you try the code bellow?

def process(self,f):
    try:
        self.logger.info("Process %i" % os.getpid() )
        raise NameError("Stupid error")

    except Exception as e:
        self.logger.info("Exception: " + e.message)
        exc_buffer = io.StringIO()
        traceback.print_exc(file=exc_buffer)
        self.logger.info(exc_buffer.getvalue())
        raise e
    except:
        self.logger.info("Exception!")
        exc_buffer = io.StringIO()
        traceback.print_exc(file=exc_buffer)
        self.logger.info(exc_buffer.getvalue())
        raise

answered Jun 29 '15 at 13:10

JoshRomRock

492
4
10

That is an additional problem, when I try to use the internal logger instance in ConverterImage then the code does not output anything, I think it crashes behind... – Gabriel Jun 29 '15 at 14:56
It works already, actually, as in the example: http://stackoverflow.com/a/17582422/293195 – Gabriel Jun 29 '15 at 14:58

Logging with Multiprocessing in python

1 Answers1