#!/usr/bin/env python3
"""
Extract `documentHtml` from captured document content JSON bodies in `xhr_captured/`.
Writes one .html file per document with a readable filename and prints a short report.

Usage:
  python scrapers/unpack_document_content.py --input-dir xhr_captured --out-dir xhr_captured
"""
import argparse
import glob
import json
import os
from pathlib import Path
import html


def slugify(s):
    keep = "abcdefghijklmnopqrstuvwxyz0123456789-_"
    s = s.lower()
    out = []
    for ch in s:
        if ch in keep:
            out.append(ch)
        else:
            out.append("-")
    res = "".join(out)
    # collapse -
    while "--" in res:
        res = res.replace("--","-")
    return res.strip("-")[:200]


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input-dir", default="xhr_captured")
    parser.add_argument("--out-dir", default="xhr_captured")
    args = parser.parse_args()

    pattern = os.path.join(args.input_dir, "*document_content_*.json")
    files = sorted(glob.glob(pattern))
    if not files:
        print("No document_content JSON files found in", args.input_dir)
        return

    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    created = []
    for p in files:
        try:
            with open(p, "r", encoding="utf-8") as f:
                meta = json.load(f)
        except Exception as e:
            print("Failed to read JSON:", p, e)
            continue
        doc_html = None
        body_file = meta.get("response_body_file")

        # If there is an explicit body file, try to open it. Some captures wrote the
        # body into a separate file; others accidentally used the same filename as
        # the metadata file. Try both.
        if body_file:
            bf = Path(body_file)
            if not bf.exists():
                bf = Path(args.input_dir) / Path(body_file).name
            if bf.exists() and bf.is_file():
                try:
                    with open(bf, "r", encoding="utf-8") as f:
                        body = json.load(f)
                    # body is often a dict containing documentHtml
                    doc_html = body.get("documentHtml")
                except Exception:
                    # not JSON or unreadable
                    doc_html = None

        # Fallback: sometimes the metadata includes an escaped JSON snippet in
        # `response_excerpt` which contains documentHtml. Try to parse it.
        if not doc_html:
            candidate = meta.get("response_excerpt")
            if candidate and "documentHtml" in candidate:
                try:
                    excerpt_json = json.loads(candidate)
                    doc_html = excerpt_json.get("documentHtml")
                except Exception:
                    doc_html = None

        if not doc_html:
            # nothing to extract from this file
            continue

        if not doc_html:
            continue

        # documentHtml may contain escaped characters; ensure it's properly unescaped
        # It's typically a string with HTML already; we'll write it verbatim.
        # Build output filename from meta url or source filename
        url = meta.get("url") or p
        # try to extract last path part or uuid
        name_hint = url.rstrip("/").split('/')[-1]
        ts = meta.get("timestamp") or "unknown"
        out_name = f"extracted_document_{name_hint}_{ts}.html"
        out_path = out_dir / out_name
        # If the doc_html looks like JSON-escaped, unescape HTML entities
        try:
            content = doc_html
            # if it looks like it's escaped with backslashes, unescape using unicode-escape then html.unescape
            # but avoid corrupting valid content
            if "\\n" in content or "\\u" in content:
                try:
                    content = content.encode('utf-8').decode('unicode_escape')
                except Exception:
                    pass
            content = html.unescape(content)
        except Exception:
            pass

        try:
            with open(out_path, "w", encoding="utf-8") as f:
                f.write(content)
            created.append(str(out_path))
        except Exception as e:
            print("Failed to write:", out_path, e)

    print(f"Extracted {len(created)} document HTML files")
    for c in created[:20]:
        print(" -", c)

if __name__ == '__main__':
    main()