Custom Scrapy Downloader Middleware | Snippets

Scrapy middlewares sit between the engine and the downloader, letting you modify every request and response globally. This middleware rotates User-Agent strings from a pool and implements exponential backoff on failures — two essentials for production scrapers that need to stay under the radar.

import random
import time
from scrapy import signals
from scrapy.http import Request, Response
from scrapy.exceptions import IgnoreRequest

class SmartDownloaderMiddleware:
    """Rotate User-Agents and retry with exponential backoff."""

    USER_AGENTS = [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 Safari/17.5",
        "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Edge/131.0",
    ]
    MAX_RETRIES = 3
    BACKOFF_BASE = 2  # seconds

    @classmethod
    def from_crawler(cls, crawler):
        middleware = cls()
        crawler.signals.connect(middleware.spider_opened, signal=signals.spider_opened)
        return middleware

    def spider_opened(self, spider):
        spider.logger.info("SmartDownloaderMiddleware enabled")

    def process_request(self, request: Request, spider) -> None:
        # Rotate User-Agent on every request
        request.headers["User-Agent"] = random.choice(self.USER_AGENTS)

    def process_response(self, request: Request, response: Response, spider) -> Response:
        # Retry on server errors (5xx) with backoff
        if 500 <= response.status < 600:
            retries = request.meta.get("retry_count", 0)
            if retries < self.MAX_RETRIES:
                delay = self.BACKOFF_BASE ** retries
                spider.logger.warning(
                    f"Got {response.status} for {request.url}, "
                    f"retrying in {delay}s (attempt {retries + 1}/{self.MAX_RETRIES})"
                )
                retry_req = request.copy()
                retry_req.meta["retry_count"] = retries + 1
                retry_req.dont_filter = True
                retry_req.meta["download_delay"] = delay
                return retry_req  # type: ignore[return-value]
        return response

    def process_exception(self, request: Request, exception, spider) -> Request | None:
        # Retry on connection errors
        retries = request.meta.get("retry_count", 0)
        if retries < self.MAX_RETRIES:
            spider.logger.warning(f"Connection error for {request.url}: {exception}")
            retry_req = request.copy()
            retry_req.meta["retry_count"] = retries + 1
            retry_req.dont_filter = True
            return retry_req
        return None

Enable it in your settings.py:

DOWNLOADER_MIDDLEWARES = {
    "myproject.middlewares.SmartDownloaderMiddleware": 543,
}

When to use this

Use a custom downloader middleware when you need consistent request-level behavior across all spiders in a Scrapy project — User-Agent rotation, custom retry logic, proxy injection, or request signing. It keeps spider code clean and makes scraping infrastructure reusable.