Compressing Files with Python: zip, tar, gzip and lzma

Compressing Files with Python: zip, tar, gzip and lzma

Python's standard library covers all major compression formats with no external dependencies. This guide shows how to create, read, and extract compressed archives efficiently and securely.

zipfile — create and read ZIP

import zipfile
import pathlib

# Create a ZIP with multiple files
with zipfile.ZipFile('archive.zip', 'w',
                     compression=zipfile.ZIP_DEFLATED, compresslevel=9) as zf:
    zf.write('report.pdf',  arcname='docs/report.pdf')
    zf.write('photo.jpg',   arcname='images/photo.jpg')
    zf.writestr('README.txt', 'Auto-generated ZIP archive\n')

    # Add an entire folder recursively
    folder = pathlib.Path('data/')
    for f in folder.rglob('*'):
        if f.is_file():
            zf.write(f, arcname=str(f))

# List contents
with zipfile.ZipFile('archive.zip', 'r') as zf:
    for info in zf.infolist():
        print(f"  {info.filename} — {info.file_size/1024:.1f} KB "
              f"→ {info.compress_size/1024:.1f} KB compressed")

# Extract all
with zipfile.ZipFile('archive.zip', 'r') as zf:
    zf.extractall('destination/')

# Extract a single file
with zipfile.ZipFile('archive.zip', 'r') as zf:
    zf.extract('docs/report.pdf', path='destination/')

# Read content without extracting to disk
with zipfile.ZipFile('archive.zip', 'r') as zf:
    with zf.open('README.txt') as f:
        content = f.read().decode('utf-8')
print(content)

Password-protected ZIP

import zipfile

with zipfile.ZipFile('protected.zip', 'w', compression=zipfile.ZIP_DEFLATED) as zf:
    zf.setpassword(b'my-secret-password')
    zf.write('secret.txt')

with zipfile.ZipFile('protected.zip', 'r') as zf:
    zf.extractall('dest/', pwd=b'my-secret-password')

# Verify integrity without extracting
with zipfile.ZipFile('archive.zip', 'r') as zf:
    bad_file = zf.testzip()   # None if OK, name of first bad file otherwise
    if bad_file:
        print(f"Corrupted file: {bad_file}")
    else:
        print("ZIP is intact")

Preventing zip bombs and path traversal

import zipfile
import pathlib

MAX_UNCOMPRESSED = 100 * 1024 * 1024  # 100 MB
MAX_RATIO        = 50                  # compressed-to-uncompressed ratio limit
MAX_FILES        = 1000

def safe_extract(zip_path: str, dest: str) -> list[str]:
    """Extract a ZIP with security validations."""
    dest_path = pathlib.Path(dest).resolve()
    extracted = []

    with zipfile.ZipFile(zip_path, 'r') as zf:
        members = zf.infolist()

        if len(members) > MAX_FILES:
            raise ValueError(f"Too many files in ZIP: {len(members)}")

        total = sum(m.file_size for m in members)
        if total > MAX_UNCOMPRESSED:
            raise ValueError(f"Uncompressed size too large: {total/1024**2:.1f} MB")

        for m in members:
            # Prevent path traversal
            target = (dest_path / m.filename).resolve()
            if not str(target).startswith(str(dest_path)):
                raise ValueError(f"Path traversal detected: {m.filename}")

            # Prevent zip bomb
            if m.file_size > 0 and m.compress_size > 0:
                ratio = m.file_size / m.compress_size
                if ratio > MAX_RATIO:
                    raise ValueError(f"Possible zip bomb: ratio={ratio:.1f} in {m.filename}")

            zf.extract(m, dest_path)
            extracted.append(str(target))

    return extracted

files = safe_extract('downloaded.zip', 'safe_extract/')

tarfile — TAR, TAR.GZ, TAR.BZ2, TAR.XZ

import tarfile
import pathlib

# Create TAR.GZ (common on Linux)
with tarfile.open('backup.tar.gz', 'w:gz') as tar:
    tar.add('documents/', arcname='documents')
    tar.add('config.json', arcname='config.json')

# Create TAR.XZ (maximum compression with lzma)
with tarfile.open('backup.tar.xz', 'w:xz') as tar:
    folder = pathlib.Path('project/')
    tar.add(folder, arcname=folder.name)

# List contents
with tarfile.open('backup.tar.gz', 'r:gz') as tar:
    for member in tar.getmembers():
        print(f"  {member.name} — {member.size/1024:.1f} KB")

# Extract all
with tarfile.open('backup.tar.gz', 'r:gz') as tar:
    tar.extractall('restored/')

# Safe extraction (Python 3.12+)
with tarfile.open('backup.tar.gz', 'r:gz') as tar:
    tar.extractall('restored/', filter='data')
    # 'data' ignores dangerous metadata (setuid bits, device files, etc.)

TAR streaming — process without extracting to disk

import tarfile

def list_tar(path: str) -> list[dict]:
    """Read TAR metadata without loading the full archive into memory."""
    result = []
    with tarfile.open(path, 'r:*') as tar:   # r:* auto-detects compression
        for m in tar:
            result.append({'name': m.name, 'size': m.size,
                           'is_dir': m.isdir(), 'mtime': m.mtime})
    return result

def read_file_from_tar(tar_path: str, member_name: str) -> bytes:
    """Read a single file's content without extracting the whole archive."""
    with tarfile.open(tar_path, 'r:*') as tar:
        member = tar.getmember(member_name)
        f = tar.extractfile(member)
        return f.read() if f else b''

content = read_file_from_tar('backup.tar.gz', 'config.json')
print(content.decode())

gzip and lzma — single-file compression

import gzip
import lzma
import pathlib

original = pathlib.Path('data.csv')

# gzip
gz_path = pathlib.Path('data.csv.gz')
with original.open('rb') as f_in, gzip.open(gz_path, 'wb', compresslevel=9) as f_out:
    f_out.write(f_in.read())

# Read gzip without extracting to disk
with gzip.open('data.csv.gz', 'rt', encoding='utf-8') as f:
    for line in f:
        pass   # streaming line-by-line processing

# lzma — best compression ratio (slower)
xz_path = pathlib.Path('data.csv.xz')
with original.open('rb') as f_in, lzma.open(xz_path, 'wb', preset=9) as f_out:
    f_out.write(f_in.read())

# Compare sizes
for path in [original, gz_path, xz_path]:
    print(f"{path.name}: {path.stat().st_size/1024:.1f} KB")

shutil — high-level archive operations

import shutil

# Create archive in one call
# Formats: 'zip', 'tar', 'gztar', 'bztar', 'xztar'
shutil.make_archive(
    'project_backup',  # base name (no extension)
    'gztar',           # format → creates project_backup.tar.gz
    root_dir='.',
    base_dir='project',
)

# Extract any supported format
shutil.unpack_archive('project_backup.tar.gz', extract_dir='restored/')

print(shutil.get_archive_formats())
# gztar, bztar, xztar, tar, zip

Practical example: incremental backup with compression

import zipfile
import pathlib
import hashlib
import json
from datetime import datetime

MANIFEST = 'backup_manifest.json'

def file_hash(path: pathlib.Path) -> str:
    h = hashlib.sha256()
    with path.open('rb') as f:
        while chunk := f.read(65536):
            h.update(chunk)
    return h.hexdigest()

def incremental_backup(source: str, dest: str) -> pathlib.Path | None:
    """Create a ZIP containing only new or changed files since the last backup."""
    src_path  = pathlib.Path(source)
    dest_path = pathlib.Path(dest)
    dest_path.mkdir(parents=True, exist_ok=True)

    manifest_path  = dest_path / MANIFEST
    old_manifest   = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
    new_manifest   = {}
    to_include     = []

    for f in src_path.rglob('*'):
        if not f.is_file():
            continue
        key  = str(f.relative_to(src_path))
        sha  = file_hash(f)
        new_manifest[key] = sha
        if old_manifest.get(key) != sha:
            to_include.append(f)

    if not to_include:
        print("No changes since last backup")
        return None

    ts       = datetime.now().strftime('%Y%m%d_%H%M%S')
    zip_path = dest_path / f"backup_incr_{ts}.zip"

    with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED,
                         compresslevel=6) as zf:
        for f in to_include:
            arcname = str(f.relative_to(src_path))
            zf.write(f, arcname=arcname)
            print(f"  + {arcname}")

    manifest_path.write_text(json.dumps(new_manifest, indent=2))
    print(f"\nBackup: {len(to_include)} files → {zip_path.name} "
          f"({zip_path.stat().st_size/1024:.1f} KB)")
    return zip_path

incremental_backup('project/', 'backups/')

Best practices

• zipfile.ZIP_DEFLATED with compresslevel=6-9 for good compression; ZIP_LZMA for maximum within ZIP (not all clients support it).
• tarfile with :xz for long-term archives where size matters more than speed; :gz for daily use.
• Always validate extracted paths — (dest / member.filename).resolve() must start with dest to prevent path traversal.
• tar.extractall(filter='data') in Python 3.12+ — safe extraction that ignores dangerous metadata (setuid bits, device files).
• extractfile() for streaming TAR reads — read individual file content without extracting the entire archive to disk.
• shutil.make_archive/unpack_archive are sufficient for most simple cases; use zipfile/tarfile directly when you need granular control.

Related conversions

Archive format conversions used most often:

• RAR to ZIP
• 7Z to ZIP
• TAR to ZIP
• GZ to ZIP
• ZIP to RAR
• ISO to ZIP