Web Scraping with Python: BeautifulSoup and requests Step by Step

Web Scraping with Python: BeautifulSoup and requests Step by Step

Web scraping is the technique of automatically extracting data from websites. The simplest Python combination is requests (downloads HTML) + BeautifulSoup4 (parses HTML).

1. Installation and first request

pip install requests beautifulsoup4 lxml

import requests
from bs4 import BeautifulSoup

URL = "https://books.toscrape.com/"

headers = {"User-Agent": "Mozilla/5.0 (compatible; MyScraper/1.0)"}

r = requests.get(URL, headers=headers, timeout=10)
r.raise_for_status()   # Raises exception on HTTP error

soup = BeautifulSoup(r.text, "lxml")
print(soup.title.text)   # Books to Scrape. We love being scraped!

2. Selectors: find, find_all and CSS selectors

# find: first matching element
first_h1 = soup.find("h1")
print(first_h1.text)

# find_all: all matching elements
links = soup.find_all("a", href=True)
for link in links[:5]:
    print(link["href"], link.text.strip())

# Filter by CSS class
articles = soup.find_all("article", class_="product_pod")
print(f"Books on homepage: {len(articles)}")

# CSS selectors with select()
titles = soup.select("article.product_pod h3 a")
for t in titles[:5]:
    print(t["title"])

prices = soup.select("p.price_color")
for p in prices[:5]:
    print(p.text.strip())   # £51.77

3. Extract structured data from a page

import requests
from bs4 import BeautifulSoup

def extract_books(url):
    r    = requests.get(url, timeout=10)
    soup = BeautifulSoup(r.text, "lxml")

    books = []
    for art in soup.select("article.product_pod"):
        title  = art.select_one("h3 a")["title"]
        price  = art.select_one("p.price_color").text.strip()
        rating = art.select_one("p.star-rating")["class"][1]
        stock  = art.select_one("p.availability").text.strip()
        href   = art.select_one("h3 a")["href"]
        books.append({
            "title":  title,
            "price":  price,
            "rating": rating,
            "stock":  stock,
            "url":    f"https://books.toscrape.com/{href}",
        })
    return books

books = extract_books("https://books.toscrape.com/")
for book in books[:3]:
    print(book)

4. Pagination: multiple pages

import requests
from bs4 import BeautifulSoup
import time

BASE = "https://books.toscrape.com/catalogue/"
all_books = []
page = 1

while True:
    url = f"{BASE}page-{page}.html"
    r   = requests.get(url, timeout=10)

    if r.status_code == 404:
        print(f"Finished at page {page - 1}")
        break

    soup = BeautifulSoup(r.text, "lxml")
    for art in soup.select("article.product_pod"):
        all_books.append({
            "title": art.select_one("h3 a")["title"],
            "price": art.select_one("p.price_color").text.strip(),
        })

    print(f"Page {page}: {len(all_books)} books total")
    page += 1
    time.sleep(1)   # Be polite to the server

print(f"Total books: {len(all_books)}")

5. Extract HTML tables

import requests
from bs4 import BeautifulSoup

url  = "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html"
r    = requests.get(url, timeout=10)
soup = BeautifulSoup(r.text, "lxml")

table = soup.select_one("table.table-striped")
data  = {}
for row in table.find_all("tr"):
    key   = row.find("th").text.strip()
    value = row.find("td").text.strip()
    data[key] = value

print(data)
# {'UPC': 'a897fe39b1053632', 'Product Type': 'Books', ...}

6. Download images

import requests
from bs4 import BeautifulSoup
from pathlib import Path
import time

def download_image(url, dest: Path):
    r = requests.get(url, timeout=15, stream=True)
    r.raise_for_status()
    with open(dest, "wb") as f:
        for chunk in r.iter_content(8192):
            f.write(chunk)
    print(f"Saved: {dest.name}")

BASE = "https://books.toscrape.com"
r    = requests.get(BASE, timeout=10)
soup = BeautifulSoup(r.text, "lxml")

folder = Path("book_images")
folder.mkdir(exist_ok=True)

for i, img in enumerate(soup.select("img"), 1):
    src  = img["src"].replace("../", "")
    url  = f"{BASE}/{src}"
    name = Path(src).name
    download_image(url, folder / name)
    time.sleep(0.5)
    if i >= 5:
        break

7. Sessions and cookies

import requests
from bs4 import BeautifulSoup

session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (compatible; MyScraper/1.0)",
    "Accept-Language": "en-US,en;q=0.9",
})

LOGIN_URL = "https://example.com/login"
r    = session.get(LOGIN_URL)
soup = BeautifulSoup(r.text, "lxml")
csrf = soup.find("input", {"name": "_token"})["value"]

session.post(LOGIN_URL, data={
    "_token":   csrf,
    "email":    "user@example.com",
    "password": "secret",
})

r2 = session.get("https://example.com/dashboard")
print(r2.status_code)

8. JavaScript-rendered pages with Playwright

pip install playwright
playwright install chromium

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page    = browser.new_page()
    page.goto("https://quotes.toscrape.com/js/")
    page.wait_for_selector(".quote")

    for quote in page.query_selector_all(".quote")[:3]:
        text   = quote.query_selector(".text").inner_text()
        author = quote.query_selector(".author").inner_text()
        print(f'"{text}" — {author}')

    browser.close()

9. Save data to CSV

import csv
import requests
from bs4 import BeautifulSoup

def scrape_books_to_csv(pages=3):
    books = []
    BASE  = "https://books.toscrape.com/catalogue/page-{}.html"

    for page in range(1, pages + 1):
        r    = requests.get(BASE.format(page), timeout=10)
        soup = BeautifulSoup(r.text, "lxml")
        for art in soup.select("article.product_pod"):
            books.append({
                "title":  art.select_one("h3 a")["title"],
                "price":  art.select_one("p.price_color").text.strip(),
                "rating": art.select_one("p.star-rating")["class"][1],
            })

    with open("books.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["title", "price", "rating"])
        writer.writeheader()
        writer.writerows(books)

    print(f"Saved {len(books)} books to books.csv")

scrape_books_to_csv()

10. Best practices and ethics

• Check robots.txt: https://site.com/robots.txt shows which paths are allowed.
• Respect rate limits: add time.sleep(1) between requests.
• User-Agent: identify your scraper honestly.
• Cache locally: save downloaded HTML during development.
• Don't scrape personal data or sites that forbid it in their Terms of Service.

import requests, time

def safe_get(url, retries=3):
    for attempt in range(retries):
        try:
            r = requests.get(url, timeout=10)
            r.raise_for_status()
            return r
        except requests.exceptions.HTTPError as e:
            print(f"HTTP {e.response.status_code}: {url}")
            break
        except requests.exceptions.ConnectionError:
            print(f"Attempt {attempt+1}/{retries}: no connection")
            time.sleep(2 ** attempt)
    return None

Tool summary

Related conversions

Frequent conversions across the catalogue:

• JPG to PNG
• MP4 to MP3
• PDF to DOCX
• PNG to JPG
• MP3 to WAV
• DOCX to PDF