fetch_html.py

"""
Fetch raw HTML from any URL - Chocodata Web Scraper API

Runnable example. It calls the LIVE API and prints the real response.

    pip install requests
    export CHOCODATA_API_KEY="your_key"      # free: 1,000 requests, one-time
    python web_scraper_api_codes/fetch_html.py
    python web_scraper_api_codes/fetch_html.py https://www.ebay.com/sch/i.html?_nkw=laptop

Docs: https://chocodata.com/docs
"""
import os
import sys

import requests

API = "https://api.chocodata.com/api/v1/universal"
KEY = os.environ.get("CHOCODATA_API_KEY")

if not KEY:
    sys.exit("Set CHOCODATA_API_KEY first. Free key (1,000 requests, one-time): https://chocodata.com")


def _check(r) -> None:
    """Map the API's documented errors onto actionable messages instead of a traceback."""
    if r.status_code == 400:
        issues = r.json().get("issues", [])
        detail = "; ".join(f"{'.'.join(str(p) for p in i.get('path', []))}: {i.get('message')}" for i in issues)
        sys.exit(f"400 invalid_params: {detail or r.text[:160]}. Fix the query string.")
    if r.status_code == 401:
        sys.exit("401 INVALID_API_KEY: key missing or not recognised. Get one: https://chocodata.com")
    if r.status_code == 402:
        sys.exit("402 INSUFFICIENT_CREDITS: balance exhausted. Top up or upgrade: https://chocodata.com/pricing")
    if r.status_code == 404:
        sys.exit("404 item_not_found: the target returned 404 for this URL. "
                 "Check it exists. Retrying will not help, and you were not charged.")
    if r.status_code == 429:
        sys.exit("429 RATE_LIMITED: over 120 requests/60s or your plan's concurrency. Back off and retry.")
    if r.status_code == 502:
        sys.exit("502 target_unreachable: the target refused every attempt. Retryable, and you were not charged.")
    r.raise_for_status()


def fetch(url: str, country: str = None) -> dict:
    """Fetch any URL and return the raw response. parse=auto detects JSON vs HTML."""
    params = {"api_key": KEY, "url": url}
    if country:
        params["country"] = country
    r = requests.get(API, params=params, timeout=90)
    _check(r)
    return r.json()


if __name__ == "__main__":
    target = sys.argv[1] if len(sys.argv) > 1 else "https://example.com"
    data = fetch(target)
    body = data.get("html") or data.get("text") or ""

    print(f"url          : {data['url']}")
    print(f"content_type : {data['content_type']}")
    print(f"chars        : {len(body):,}")
    print()
    print(body[:600])
    print()
    print(f"OK: {len(body):,} chars of {data['content_type']} from {data['url']}")