Automating the Browser with Python and Playwright
Playwright is the most modern browser automation tool for Python, offering native async API, multi-browser support (Chromium, Firefox, WebKit), network interception, and automatic waiting.
Installation
pip install playwright
playwright install chromium
Navigate and Screenshot
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto("https://example.com")
page.wait_for_load_state("networkidle")
page.screenshot(path="screenshot.png", full_page=True)
print(f"Title: {page.title()}")
browser.close()
Fill Forms
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto("https://httpbin.org/forms/post")
page.fill('input[name="custname"]', "Alice Johnson")
page.fill('input[name="custemail"]', "alice@example.com")
page.select_option('select[name="size"]', 'medium')
page.click('button[type="submit"]')
page.wait_for_load_state("networkidle")
browser.close()
Wait for Dynamic Elements
from playwright.sync_api import sync_playwright, TimeoutError
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto("https://news.ycombinator.com")
page.wait_for_selector(".storylink", timeout=10000)
titles = page.query_selector_all(".storylink")
for t in titles[:5]:
print(f" - {t.inner_text()}")
browser.close()
JavaScript Evaluation
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto("https://github.com/trending")
page.wait_for_load_state("networkidle")
repos = page.evaluate("""
() => Array.from(document.querySelectorAll('article')).slice(0,5).map(a => ({
name: a.querySelector('h2')?.innerText?.trim(),
}))
""")
for r in repos:
print(r['name'])
browser.close()
Intercept API Responses
from playwright.sync_api import sync_playwright
def intercept_api(url, pattern):
captured = []
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
def on_resp(resp):
if pattern in resp.url:
try: captured.append(resp.json())
except Exception: pass
page.on("response", on_resp)
page.goto(url, wait_until="networkidle")
browser.close()
return captured
Async Parallel Scraping
import asyncio
from playwright.async_api import async_playwright
async def scrape_urls(urls):
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
async def fetch(url):
page = await browser.new_page()
try:
await page.goto(url, timeout=30000)
return {"url": url, "title": await page.title()}
except Exception as e:
return {"url": url, "error": str(e)}
finally:
await page.close()
results = await asyncio.gather(*[fetch(u) for u in urls])
await browser.close()
return results
results = asyncio.run(scrape_urls(["https://example.com", "https://python.org"]))
for r in results:
print(f"{r.get('title','Error')}: {r['url']}")
Generate PDF from URL
from playwright.sync_api import sync_playwright
def url_to_pdf(url, output_pdf):
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto(url, wait_until="networkidle")
page.pdf(path=output_pdf, format="A4", print_background=True)
browser.close()
url_to_pdf("https://python.org", "python_docs.pdf")
Session Persistence
from playwright.sync_api import sync_playwright
import json
def login_save(login_url, user, pwd, cookies_file):
with sync_playwright() as p:
browser = p.chromium.launch(headless=False)
page = browser.new_page()
page.goto(login_url)
page.fill('input[name="username"]', user)
page.fill('input[name="password"]', pwd)
page.click('button[type="submit"]')
page.wait_for_load_state("networkidle")
with open(cookies_file, 'w') as f:
json.dump(page.context.storage_state(), f)
browser.close()
def use_session(url, cookies_file):
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
ctx = browser.new_context(storage_state=cookies_file)
page = ctx.new_page()
page.goto(url)
print(f"Authenticated: {page.title()}")
browser.close()
Additional Resource
For converting screenshots and documents between PNG, PDF, WebP and other formats without any coding, use KaijuConverter — free and no registration required.
Related conversions
Frequent conversions across the catalogue: