collect_data.py

"""
yeye — Step 1: OpenStreetMap Data Collection for Togo
Pulls emergency-relevant infrastructure using the Overpass API.
"""

import requests
import json
import time
import pandas as pd
import os

OVERPASS_URL = "https://overpass-api.de/api/interpreter"
OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "raw")

# Togo bounding box: south, west, north, east
TOGO_BBOX = "6.1,-0.15,11.14,1.8"

QUERIES = {
    "hospitals": f"""
        [out:json][timeout:60];
        (
          node["amenity"="hospital"]({TOGO_BBOX});
          way["amenity"="hospital"]({TOGO_BBOX});
          node["amenity"="clinic"]({TOGO_BBOX});
          node["healthcare"="hospital"]({TOGO_BBOX});
          node["healthcare"="clinic"]({TOGO_BBOX});
        );
        out center tags;
    """,
    "fire_stations": f"""
        [out:json][timeout:60];
        (
          node["amenity"="fire_station"]({TOGO_BBOX});
          way["amenity"="fire_station"]({TOGO_BBOX});
        );
        out center tags;
    """,
    "police": f"""
        [out:json][timeout:60];
        (
          node["amenity"="police"]({TOGO_BBOX});
          way["amenity"="police"]({TOGO_BBOX});
        );
        out center tags;
    """,
    "fuel_stations": f"""
        [out:json][timeout:60];
        (
          node["amenity"="fuel"]({TOGO_BBOX});
          way["amenity"="fuel"]({TOGO_BBOX});
        );
        out center tags;
    """,
    "pharmacies": f"""
        [out:json][timeout:60];
        (
          node["amenity"="pharmacy"]({TOGO_BBOX});
          way["amenity"="pharmacy"]({TOGO_BBOX});
        );
        out center tags;
    """,
    "roads_main": f"""
        [out:json][timeout:90];
        (
          way["highway"~"motorway|trunk|primary|secondary"]({TOGO_BBOX});
        );
        out geom tags;
    """
}

def fetch_osm(name, query, retries=3):
    print(f"  Fetching {name}...")
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    for attempt in range(retries):
        try:
            resp = requests.post(OVERPASS_URL, data={"data": query}, timeout=120)
            if resp.status_code in (429, 504):
                wait = 30 * (attempt + 1)
                print(f"  ! {name}: HTTP {resp.status_code}, retrying in {wait}s...")
                time.sleep(wait)
                continue
            resp.raise_for_status()
            data = resp.json()
            path = os.path.join(OUTPUT_DIR, f"osm_{name}.json")
            with open(path, "w") as f:
                json.dump(data, f)
            count = len(data.get("elements", []))
            print(f"  ✓ {name}: {count} elements saved")
            return count
        except Exception as e:
            print(f"  ✗ {name} failed: {e}")
            if attempt < retries - 1:
                time.sleep(15)
    return 0

def parse_to_csv(name):
    """Convert OSM JSON to a clean CSV with lat/lon and key tags."""
    path = os.path.join(OUTPUT_DIR, f"osm_{name}.json")
    if not os.path.exists(path):
        return
    with open(path) as f:
        data = json.load(f)

    rows = []
    for el in data.get("elements", []):
        # Nodes have lat/lon directly; ways have a center
        lat = el.get("lat") or (el.get("center") or {}).get("lat")
        lon = el.get("lon") or (el.get("center") or {}).get("lon")
        if not lat or not lon:
            continue
        tags = el.get("tags", {})
        rows.append({
            "osm_id": el.get("id"),
            "type": el.get("type"),
            "lat": lat,
            "lon": lon,
            "name": tags.get("name", tags.get("name:fr", "")),
            "name_en": tags.get("name:en", ""),
            "amenity": tags.get("amenity", ""),
            "healthcare": tags.get("healthcare", ""),
            "operator": tags.get("operator", ""),
            "beds": tags.get("beds", ""),
            "emergency": tags.get("emergency", ""),
            "phone": tags.get("phone", tags.get("contact:phone", "")),
            "opening_hours": tags.get("opening_hours", ""),
            "highway": tags.get("highway", ""),
            "surface": tags.get("surface", ""),
            "category": name
        })

    if rows:
        df = pd.DataFrame(rows)
        csv_path = os.path.join(OUTPUT_DIR, f"osm_{name}.csv")
        df.to_csv(csv_path, index=False)
        print(f"  ✓ {name}: {len(df)} records → CSV")
        return df
    return None

if __name__ == "__main__":
    print("\n=== yeye: OSM Data Collection ===\n")
    results = {}
    for name, query in QUERIES.items():
        count = fetch_osm(name, query)
        results[name] = count
        time.sleep(5)  # Be polite to Overpass API

    print("\n--- Parsing to CSV ---")
    dfs = {}
    for name in QUERIES:
        if name != "roads_main":
            df = parse_to_csv(name)
            if df is not None:
                dfs[name] = df

    # Merge all point features into one master file
    if dfs:
        master = pd.concat(dfs.values(), ignore_index=True)
        master_path = os.path.join(OUTPUT_DIR, "togo_infrastructure_master.csv")
        master.to_csv(master_path, index=False)
        print(f"\n✓ Master file: {len(master)} total infrastructure points")
        print(f"  Saved to: {master_path}")

    print("\n=== Summary ===")
    for name, count in results.items():
        print(f"  {name}: {count} elements")