main.py

#!/usr/bin/env python3
"""CLI entry point for Marktplaats scraper."""

import argparse
import os
import sys
from pathlib import Path

from config import CATEGORIES, DEFAULT_CATEGORIES
from scraper import MarktplaatsScraper


def list_categories():
    """Print available categories."""
    print("\nAvailable categories:")
    print("-" * 60)

    # Group categories by type
    groups = {
        "Electronics": [],
        "Audio & TV": [],
        "Phones": [],
        "Gaming": [],
        "Home & Garden": [],
        "Vehicles": [],
        "Fashion": [],
        "Other": [],
    }

    for slug, name in CATEGORIES.items():
        if "computer" in slug or "laptop" in slug or "monitor" in slug or "tablet" in slug:
            groups["Electronics"].append((slug, name))
        elif "audio" in slug or "tv" in slug or "foto" in slug:
            groups["Audio & TV"].append((slug, name))
        elif "telefoon" in slug or "telecom" in slug:
            groups["Phones"].append((slug, name))
        elif "spel" in slug or "game" in slug or "playstation" in slug or "xbox" in slug or "nintendo" in slug:
            groups["Gaming"].append((slug, name))
        elif "huis" in slug or "tuin" in slug or "meubel" in slug:
            groups["Home & Garden"].append((slug, name))
        elif "fiets" in slug or "brommer" in slug:
            groups["Vehicles"].append((slug, name))
        elif "kleding" in slug or "sieraden" in slug:
            groups["Fashion"].append((slug, name))
        else:
            groups["Other"].append((slug, name))

    for group_name, items in groups.items():
        if items:
            print(f"\n{group_name}:")
            for slug, name in items:
                print(f"  {name}: {slug}")

    print()


def main():
    """Main entry point."""
    parser = argparse.ArgumentParser(
        description="Scrape Marktplaats listings using ScrapingAnt API",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python main.py -v                           # Scrape default categories
  python main.py --categories "computers-en-software/windows-laptops" -v
  python main.py --all-categories -v          # Scrape all available categories
  python main.py --pages 3 -v                 # Scrape 3 pages per category
  python main.py --list-categories            # List available categories
        """,
    )

    parser.add_argument(
        "-v", "--verbose", action="store_true", help="Enable verbose output"
    )

    parser.add_argument(
        "--categories",
        type=str,
        nargs="+",
        help="Category slugs to scrape (e.g., computers-en-software/windows-laptops)",
    )

    parser.add_argument(
        "--all-categories",
        action="store_true",
        help="Scrape all available categories",
    )

    parser.add_argument(
        "--pages",
        type=int,
        default=2,
        help="Number of pages to scrape per category (default: 2)",
    )

    parser.add_argument(
        "--list-categories",
        action="store_true",
        help="List all available categories and exit",
    )

    parser.add_argument(
        "--output-csv",
        type=str,
        default="output/marktplaats_listings.csv",
        help="Path for CSV output file (default: output/marktplaats_listings.csv)",
    )

    parser.add_argument(
        "--output-json",
        type=str,
        default="output/marktplaats_listings.json",
        help="Path for JSON output file (default: output/marktplaats_listings.json)",
    )

    parser.add_argument(
        "--api-key",
        type=str,
        help="ScrapingAnt API key (or set SCRAPINGANT_API_KEY env var)",
    )

    args = parser.parse_args()

    # Handle --list-categories
    if args.list_categories:
        list_categories()
        return 0

    # Determine categories to scrape
    if args.all_categories:
        categories = list(CATEGORIES.keys())
    elif args.categories:
        categories = args.categories
    else:
        categories = DEFAULT_CATEGORIES

    # Validate categories
    invalid = [c for c in categories if c not in CATEGORIES]
    if invalid:
        print(f"Warning: Unknown categories will be used as-is: {invalid}")

    # Check for API key
    api_key = args.api_key or os.environ.get("SCRAPINGANT_API_KEY")
    if not api_key:
        print("Error: ScrapingAnt API key is required.")
        print("Set SCRAPINGANT_API_KEY environment variable or use --api-key option.")
        print("Get your API key at: https://scrapingant.com/")
        return 1

    # Create output directory if needed
    for output_path in [args.output_csv, args.output_json]:
        output_dir = Path(output_path).parent
        output_dir.mkdir(parents=True, exist_ok=True)

    # Initialize scraper
    try:
        scraper = MarktplaatsScraper(api_key=api_key, verbose=args.verbose)
    except ValueError as e:
        print(f"Error: {e}")
        return 1

    # Scrape categories
    print(f"Scraping {len(categories)} category(ies)...")
    print(f"Pages per category: {args.pages}")
    print()

    total = scraper.scrape_categories(categories, max_pages=args.pages)

    if total == 0:
        print("No listings found.")
        return 0

    # Export results
    csv_path = scraper.export_csv(args.output_csv)
    json_path = scraper.export_json(args.output_json)

    print()
    print(f"Scraped {total} total listings")
    print(f"CSV saved to: {csv_path}")
    print(f"JSON saved to: {json_path}")

    return 0


if __name__ == "__main__":
    sys.exit(main())