#!/usr/bin/env python3
"""
Web scraper via headless Chrome print-to-pdf + PyMuPDF text extraction.
No requests/beautifulsoup4 needed -- only uses packages already in the Docker image.
Outputs JSON to stdout for agent consumption.
"""

import argparse
import ipaddress
import json
import os
import shutil
import subprocess
import sys
import tempfile
from typing import Dict, Optional
from urllib.parse import urlparse

# ---------------------------------------------------------------------------
# SSRF guard
# ---------------------------------------------------------------------------

BLOCKED_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "[::1]", "metadata.google.internal"}


def is_private_ip(hostname):
    # type: (str) -> bool
    try:
        addr = ipaddress.ip_address(hostname)
        return addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def validate_url(url):
    # type: (str) -> Optional[str]
    parsed = urlparse(url)
    if parsed.scheme not in ("http", "https"):
        return "Unsupported scheme: %s. Only http/https allowed." % parsed.scheme
    hostname = parsed.hostname or ""
    if hostname in BLOCKED_HOSTS:
        return "Blocked host: %s" % hostname
    if is_private_ip(hostname):
        return "Private/reserved IP blocked: %s" % hostname
    return None


# ---------------------------------------------------------------------------
# Chrome binary detection
# ---------------------------------------------------------------------------

CHROME_CANDIDATES = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]


def find_chrome():
    # type: () -> Optional[str]
    for name in CHROME_CANDIDATES:
        path = shutil.which(name)
        if path:
            return path
    return None


# ---------------------------------------------------------------------------
# Static scrape via curl (no extra deps)
# ---------------------------------------------------------------------------

def scrape_static(url, timeout):
    # type: (str, int) -> Dict
    """Lightweight fetch using curl + basic text extraction."""
    curl = shutil.which("curl")
    if not curl:
        return {"status": "error", "mode": "static", "url": url,
                "error": "curl not found on PATH"}

    try:
        result = subprocess.run(
            [curl, "-sL", "--max-time", str(timeout),
             "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/137.0",
             "-H", "Accept: text/html,*/*;q=0.8",
             url],
            capture_output=True, text=True, timeout=timeout + 5,
        )
    except subprocess.TimeoutExpired:
        return {"status": "error", "mode": "static", "url": url,
                "error": "curl timed out after %ds" % timeout}

    if result.returncode != 0:
        return {"status": "error", "mode": "static", "url": url,
                "error": "curl failed (code %d): %s" % (result.returncode, result.stderr[:300])}

    html = result.stdout
    text = _extract_text_from_html(html)
    title = _extract_title_from_html(html)

    return {
        "status": "ok",
        "mode": "static",
        "url": url,
        "title": title,
        "text": text,
        "length": len(text),
    }


def _extract_title_from_html(html):
    # type: (str) -> str
    """Extract <title> content without bs4."""
    import re
    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
    return m.group(1).strip() if m else ""


def _extract_text_from_html(html):
    # type: (str) -> str
    """Rough HTML-to-text without bs4: strip tags, collapse whitespace."""
    import re
    text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r"<[^>]+>", "\n", text)
    text = re.sub(r"&nbsp;", " ", text)
    text = re.sub(r"&amp;", "&", text)
    text = re.sub(r"&lt;", "<", text)
    text = re.sub(r"&gt;", ">", text)
    text = re.sub(r"&#\d+;", "", text)
    text = re.sub(r"&\w+;", "", text)
    lines = [line.strip() for line in text.split("\n") if line.strip()]
    return "\n".join(lines)


# ---------------------------------------------------------------------------
# Dynamic scrape: Chrome --print-to-pdf + PyMuPDF
# ---------------------------------------------------------------------------

def scrape_dynamic(url, timeout):
    # type: (str, int) -> Dict
    chrome = find_chrome()
    if not chrome:
        return {
            "status": "error", "mode": "dynamic", "url": url,
            "error": "No Chrome/Chromium found. Searched: %s. Use --mode static instead."
                     % ", ".join(CHROME_CANDIDATES),
        }

    tmpdir = tempfile.mkdtemp(prefix="web-scraper-")
    pdf_path = os.path.join(tmpdir, "page.pdf")

    try:
        js_budget_ms = max(5000, timeout * 1000)
        cmd = [
            chrome,
            "--headless=new",
            "--print-to-pdf=" + pdf_path,
            "--no-sandbox",
            "--disable-gpu",
            "--disable-software-rasterizer",
            "--disable-dev-shm-usage",
            "--no-pdf-header-footer",
            "--virtual-time-budget=%d" % js_budget_ms,
            url,
        ]

        try:
            subprocess.run(
                cmd, capture_output=True, text=True,
                timeout=timeout + 30,
            )
        except subprocess.TimeoutExpired:
            return {"status": "error", "mode": "dynamic", "url": url,
                    "error": "Chrome timed out after %ds" % timeout}

        if not os.path.exists(pdf_path) or os.path.getsize(pdf_path) == 0:
            return {"status": "error", "mode": "dynamic", "url": url,
                    "error": "Chrome did not produce a PDF file"}

        text = _extract_text_from_pdf(pdf_path)
        title = _guess_title(text)

        return {
            "status": "ok",
            "mode": "dynamic",
            "url": url,
            "title": title,
            "text": text,
            "length": len(text),
        }
    finally:
        try:
            shutil.rmtree(tmpdir, ignore_errors=True)
        except Exception:
            pass


def _extract_text_from_pdf(pdf_path):
    # type: (str) -> str
    """Extract text from PDF using PyMuPDF (already in requirements.txt)."""
    try:
        import fitz  # PyMuPDF
    except ImportError:
        return _extract_text_from_pdf_fallback(pdf_path)

    doc = fitz.open(pdf_path)
    pages = []
    for page in doc:
        pages.append(page.get_text())
    doc.close()
    text = "\n".join(pages).strip()
    lines = [line.strip() for line in text.split("\n") if line.strip()]
    return "\n".join(lines)


def _extract_text_from_pdf_fallback(pdf_path):
    # type: (str) -> str
    """Fallback: try pdfminer if PyMuPDF is not available."""
    try:
        from pdfminer.high_level import extract_text
        return extract_text(pdf_path).strip()
    except ImportError:
        return "[error] Neither PyMuPDF nor pdfminer available. Install with: pip3 install --break-system-packages PyMuPDF"


def _guess_title(text):
    # type: (str) -> str
    """Use the first non-empty line as title."""
    for line in text.split("\n"):
        stripped = line.strip()
        if stripped and len(stripped) < 200:
            return stripped
    return ""


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main():
    parser = argparse.ArgumentParser(description="Web scraper for OpenClaw agents")
    parser.add_argument("url", help="URL to scrape (http/https only)")
    parser.add_argument("--mode", choices=["static", "dynamic"], default="static",
                        help="static: curl + text extract; dynamic: Chrome PDF + PyMuPDF (default: static)")
    parser.add_argument("--timeout", type=int, default=15,
                        help="Request timeout in seconds (default: 15)")
    parser.add_argument("--max-chars", type=int, default=50000,
                        help="Max output characters (default: 50000)")
    args = parser.parse_args()

    err = validate_url(args.url)
    if err:
        print(json.dumps({"status": "error", "url": args.url, "error": err}, ensure_ascii=False))
        sys.exit(1)

    try:
        if args.mode == "static":
            result = scrape_static(args.url, args.timeout)
        else:
            result = scrape_dynamic(args.url, args.timeout)
    except Exception as e:
        result = {"status": "error", "url": args.url, "mode": args.mode,
                  "error": "%s: %s" % (type(e).__name__, e)}

    if result.get("status") == "ok" and len(result.get("text", "")) > args.max_chars:
        result["text"] = result["text"][:args.max_chars] + "\n\n... [truncated]"
        result["truncated"] = True
        result["length"] = len(result["text"])

    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()
