Python Logging: handlers, formatters, rotation and structured logging

Python Logging: handlers, formatters, rotation and structured logging

Python's logging module (standard library) is the correct tool for recording application events. Avoid print() for debugging in production: logging lets you control verbosity, write to multiple destinations, and disable messages without removing code.

1. Basic configuration

import logging

logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)

logging.debug("Detailed debug information")
logging.info("Normal application flow")
logging.warning("Unexpected but non-critical event")
logging.error("Error that prevents an operation")
logging.critical("Severe error — app may not function")

Levels and when to use them

2. Named loggers (hierarchy)

import logging

# Each module should have its own logger
logger = logging.getLogger(__name__)

def process_file(path):
    logger.info("Processing: %s", path)
    try:
        logger.debug("File processed successfully: %s", path)
    except FileNotFoundError:
        logger.error("File not found: %s", path)
    except Exception as e:
        logger.exception("Unexpected error processing %s", path)
        # logger.exception automatically includes the traceback

3. Handlers: multiple destinations

import logging

logger = logging.getLogger("my_app")
logger.setLevel(logging.DEBUG)

# Handler 1: console (WARNING and above only)
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
console.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))

# Handler 2: file (all levels)
file_h = logging.FileHandler("app.log", encoding="utf-8")
file_h.setLevel(logging.DEBUG)
file_h.setFormatter(logging.Formatter(
    "%(asctime)s [%(levelname)-8s] %(name)s:%(lineno)d — %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S"
))

logger.addHandler(console)
logger.addHandler(file_h)

logger.debug("File only")
logger.warning("Console and file")

4. Log rotation

import logging
from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler

logger = logging.getLogger("rotation")
logger.setLevel(logging.DEBUG)

# By size: max 5 MB, keep 3 backups
size_handler = RotatingFileHandler(
    "app.log",
    maxBytes=5 * 1024 * 1024,
    backupCount=3,
    encoding="utf-8",
)
# Creates: app.log, app.log.1, app.log.2, app.log.3

# By time: new file at midnight
time_handler = TimedRotatingFileHandler(
    "app_daily.log",
    when="midnight",
    interval=1,
    backupCount=7,
    encoding="utf-8",
)

fmt = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
size_handler.setFormatter(fmt)
time_handler.setFormatter(fmt)

logger.addHandler(size_handler)
logger.addHandler(time_handler)

5. Dictionary configuration (recommended)

import logging
import logging.config

LOGGING_CONFIG = {
    "version": 1,
    "disable_existing_loggers": False,
    "formatters": {
        "detailed": {
            "format": "%(asctime)s [%(levelname)-8s] %(name)s:%(lineno)d — %(message)s",
            "datefmt": "%Y-%m-%d %H:%M:%S",
        },
        "simple": {"format": "[%(levelname)s] %(message)s"},
    },
    "handlers": {
        "console": {
            "class": "logging.StreamHandler",
            "level": "WARNING",
            "formatter": "simple",
            "stream": "ext://sys.stdout",
        },
        "file": {
            "class": "logging.handlers.RotatingFileHandler",
            "level": "DEBUG",
            "formatter": "detailed",
            "filename": "app.log",
            "maxBytes": 10485760,
            "backupCount": 5,
            "encoding": "utf-8",
        },
    },
    "loggers": {
        "my_app": {
            "level": "DEBUG",
            "handlers": ["console", "file"],
            "propagate": False,
        },
    },
    "root": {"level": "WARNING", "handlers": ["console"]},
}

logging.config.dictConfig(LOGGING_CONFIG)
logger = logging.getLogger("my_app")
logger.info("Logging configuration applied")

6. JSON structured logging

pip install python-json-logger

import logging
from pythonjsonlogger import jsonlogger

logger  = logging.getLogger("my_service")
handler = logging.StreamHandler()

formatter = jsonlogger.JsonFormatter(
    "%(asctime)s %(name)s %(levelname)s %(message)s",
    datefmt="%Y-%m-%dT%H:%M:%S",
)
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)

logger.info("User authenticated", extra={
    "user_id": 42,
    "ip":      "192.168.1.1",
    "action":  "login",
})
# Output: {"asctime": "2025-06-15T14:30:00", "name": "my_service",
#           "levelname": "INFO", "message": "User authenticated",
#           "user_id": 42, "ip": "192.168.1.1", "action": "login"}

7. Context with LoggerAdapter

import logging

class ContextLogger(logging.LoggerAdapter):
    def process(self, msg, kwargs):
        return f"[req={self.extra.get('request_id','?')}] {msg}", kwargs

base = logging.getLogger("my_app")

def handle_request(request_id):
    log = ContextLogger(base, {"request_id": request_id})
    log.info("Request received")
    log.warning("Token about to expire")
    log.info("Request completed")

handle_request("abc-123")

8. Capture exceptions correctly

import logging
logger = logging.getLogger(__name__)

# BAD: loses the traceback
try:
    result = 1 / 0
except ZeroDivisionError:
    logger.error("Division by zero")   # No traceback

# GOOD: logger.exception includes full traceback
try:
    result = 1 / 0
except ZeroDivisionError:
    logger.exception("Division by zero")

# Also valid with exc_info=True
try:
    result = int("not-a-number")
except ValueError as e:
    logger.error("Conversion error: %s", e, exc_info=True)

9. Silence noisy third-party libraries

import logging

logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("httpx").setLevel(logging.WARNING)

# Or: set everything to WARNING, then your app to DEBUG
logging.basicConfig(level=logging.WARNING)
logging.getLogger("my_app").setLevel(logging.DEBUG)

10. Best practices

• Use logging.getLogger(__name__) in each module — never use the root logger directly.
• Use %s not f-strings: logger.info("Value: %s", v) — avoids string formatting if message won't be emitted.
• Use logger.exception() inside except blocks to include the traceback.
• Don't use print() in production code — use logging with the appropriate level.
• Configure from the entry point (main.py, app.py), not inside internal modules.
• Rotate log files to prevent disk fill-up.
• JSON in production, readable text in development.

Handler summary

Related conversions

Frequent conversions across the catalogue:

• JPG to PNG
• MP4 to MP3
• PDF to DOCX
• PNG to JPG
• MP3 to WAV
• DOCX to PDF