config.py

"""Configuration settings for Marktplaats scraper."""

import os

# ScrapingAnt API settings
SCRAPINGANT_API_KEY = os.environ.get("SCRAPINGANT_API_KEY", "")
SCRAPINGANT_BASE_URL = "https://api.scrapingant.com/v2/general"

# Marktplaats base URL
MARKTPLAATS_BASE_URL = "https://www.marktplaats.nl"

# Request configuration
REQUEST_CONFIG = {
    "timeout": 120000,  # 2 minutes in milliseconds
    "retry_count": 3,
    "retry_delay": 5,  # seconds
}

# Available categories for scraping
# Format: "slug": "Display Name"
CATEGORIES = {
    # Electronics
    "computers-en-software": "Computers en Software",
    "computers-en-software/windows-laptops": "Windows Laptops",
    "computers-en-software/apple-macbook": "Apple MacBook",
    "computers-en-software/monitoren": "Monitoren",
    "computers-en-software/tablets": "Tablets",
    # Audio & TV
    "audio-tv-en-foto": "Audio, TV en Foto",
    "audio-tv-en-foto/televisies": "Televisies",
    "audio-tv-en-foto/luidsprekers": "Luidsprekers",
    "audio-tv-en-foto/koptelefoons": "Koptelefoons",
    # Phones
    "telecommunicatie/mobiele-telefoons": "Mobiele Telefoons",
    "telecommunicatie/mobiele-telefoons-apple-iphone": "Apple iPhone",
    "telecommunicatie/mobiele-telefoons-samsung": "Samsung Phones",
    # Gaming
    "spelcomputers-en-games": "Spelcomputers en Games",
    "spelcomputers-en-games/playstation": "PlayStation",
    "spelcomputers-en-games/xbox": "Xbox",
    "spelcomputers-en-games/nintendo": "Nintendo",
    # Home & Garden
    "huis-en-inrichting": "Huis en Inrichting",
    "huis-en-inrichting/meubels": "Meubels",
    "tuin-en-terras": "Tuin en Terras",
    # Vehicles
    "fietsen-en-brommers": "Fietsen en Brommers",
    "fietsen-en-brommers/fietsen-heren": "Heren Fietsen",
    "fietsen-en-brommers/fietsen-dames": "Dames Fietsen",
    # Fashion
    "kleding-dames": "Kleding Dames",
    "kleding-heren": "Kleding Heren",
    "sieraden-tassen-en-uiterlijk": "Sieraden, Tassen en Uiterlijk",
}

# Default categories to scrape if none specified
DEFAULT_CATEGORIES = [
    "computers-en-software/windows-laptops",
    "telecommunicatie/mobiele-telefoons-apple-iphone",
]

# CSS Selectors
SELECTORS = {
    "listing_container": ".hz-Listing",
    "listing_link": 'a[href*="/v/"]',
    "title": "h3",
    "wait_for_content": ".hz-Listing",
}