Logging to multiple files in Python 3

Question

I have defined the logging configuration for my Python app:

from logging import config
from os.path import expanduser


def load_logconfig(app):

    home = expanduser('~')
    default_file_prefix = 'my_app'
    default_log_level = 'DEBUG' if 'debug_mode' in app.registry.settings else 'WARN'
    extra_loggers = ['requests', 'selenium', 'easyprocess', 'transaction', 'pyvirtualdisplay']
    excludes = ['defaults', 'temp', 'test']

    default_logging = {
        'version': 1,
        'formatters': {
            'short': {
                'format': '%(asctime)s\t%(message)s',
            },
            'long': {
                'format': '%(asctime)s %(levelname)-5.5s [%(threadName)s]\t%(message)s'
            },
            'generic': {
                'format': '%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s]\t%(message)s'
            }
        },
        'handlers': {
            'console': {
                'level': default_log_level,
                'class': 'logging.StreamHandler',
                'stream': 'ext://sys.stdout',
                'formatter': 'long'
            },
            'debuglog': {
                'class': 'logging.handlers.RotatingFileHandler',
                'level': default_log_level,
                'filename': '{}/logs/my_app/debug.log'.format(home),
                'formatter': 'generic',
                'maxBytes': 102400000,
                'backupCount': 20,
                'encoding': 'utf8'
            },
            'warnlog': {
                'class': 'logging.handlers.RotatingFileHandler',
                'level': 'ERROR',
                'filename': '{}/logs/my_app/error.log'.format(home),
                'formatter': 'generic',
                'maxBytes': 102400000,
                'backupCount': 20,
                'encoding': 'utf8'
            }
        },
        'loggers': {
            'sqlalchemy': {'level': 'INFO', 'qualname': 'sqlalchemy.engine', 'propagate': False},
        },
        'root': {
            'level': default_log_level,
            'handlers': ['console', 'debuglog', 'warnlog'],
            'propagate': False
        }
    }

    for logger in extra_loggers:
        default_logging['loggers'][logger] = {'level': 'WARN', 'qualname': logger, 'propagate': False}

    for endpoint in app.registry.cornice_services:
        endpoint_name = endpoint.split('/')[1]
        handler_name = '{}_{}'.format(default_file_prefix, endpoint_name)
        if handler_name not in default_logging['handlers'] and len(endpoint_name) > 0 and endpoint_name not in excludes:
            default_logging['handlers'][handler_name] = {
                'level': default_log_level,
                'class': 'logging.handlers.RotatingFileHandler',
                'filename': '{}/logs/my_app/{}_{}.log'.format(home, default_file_prefix, endpoint_name),
                'formatter': 'generic',
                'maxBytes': 102400000,
                'backupCount': 20,
                'encoding': 'utf8'
            }
            default_logging['root']['handlers'].append(handler_name)

    config.dictConfig(default_logging)

and I have a helper function which I am using for logging:

import logging


def write_log(request, loglevel, message, mongo_log=False, send_email=False, **kwargs):
    requested_name = request.path.split('/')[1]
    log = logging.getLogger('my_app_{}'.format(requested_name))
    level = logging.getLevelName(loglevel.upper())
    msg = '[{}] [{}] {}'.format(request.remote_addr, request.method, message)

    log.log(level, msg)

    # do some other stuff...

    return True

And its usage is fairly simple:

write_log(request, 'debug', 'some informative message', send_email=True)

What I would like to achieve here is to write log messages in different files, based on what I call here endpoint - e.g. if endpoint users is requested, messages should be written in my_app_users.log file. Messages from companies endpoint should be written in my_app_companies.log file, etc.
With my current setup, log messages are written in all the files but I just cannot figure out why. I assume it might be that I misunderstood usage of logging.getLogger() but I'm not sure how to properly address the handler which I want to use each time.

errata · Accepted Answer · 2016-04-05 16:40:30Z

2

Ah well, after one more reading about logging, I realized that I didn't add things I needed into default_logging['loggers']. Once I defined them there, everything worked as expected.

answered Apr 5, 2016 at 16:40

errata

6,07110 gold badges59 silver badges103 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

Logging to multiple files in Python 3

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related