Python Scraper: Requests + BeautifulSoup to CSV

import csv
import time

import requests
from bs4 import BeautifulSoup

session = requests.Session()
session.headers["User-Agent"] = "Mozilla/5.0 (compatible; research-script/1.0)"


def scrape(urls, out_file):
    with open(out_file, "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(["title", "rating", "url"])
        for url in urls:
            response = session.get(url, timeout=20)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, "html.parser")
            for item in soup.select(".item"):            # adjust selectors to the page
                title = item.select_one(".title")
                rating = item.select_one(".rating")
                writer.writerow([
                    title.get_text(strip=True) if title else "",
                    rating.get_text(strip=True) if rating else "",
                    url,
                ])
            time.sleep(2)                                  # be polite: one page every 2 seconds


if __name__ == "__main__":
    scrape(["https://example.com/page/1", "https://example.com/page/2"], "results.csv")

Check the site’s terms and robots.txt before scraping, and prefer an official API when one exists. Selectors change when a site is redesigned, so keep them in one place.

Keep reading