Add scripts for capturing network responses and extracting document content

- Implement capture_with_cdp.py to capture all network response bodies using Chrome DevTools Protocol.
- Create extract_sections.py to extract sections from STATdx snapshot HTML files into JSON Lines.
- Add unpack_document_content.py to extract `documentHtml` from captured JSON bodies and save as HTML files.
- Update .gitignore to include playwright_profile.

scrapers/unpack_document_content.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""
+Extract `documentHtml` from captured document content JSON bodies in `xhr_captured/`.
+Writes one .html file per document with a readable filename and prints a short report.
+
+Usage:
+  python scrapers/unpack_document_content.py --input-dir xhr_captured --out-dir xhr_captured
+"""
+import argparse
+import glob
+import json
+import os
+from pathlib import Path
+import html
+
+
+def slugify(s):
+    keep = "abcdefghijklmnopqrstuvwxyz0123456789-_"
+    s = s.lower()
+    out = []
+    for ch in s:
+        if ch in keep:
+            out.append(ch)
+        else:
+            out.append("-")
+    res = "".join(out)
+    # collapse -
+    while "--" in res:
+        res = res.replace("--","-")
+    return res.strip("-")[:200]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input-dir", default="xhr_captured")
+    parser.add_argument("--out-dir", default="xhr_captured")
+    args = parser.parse_args()
+
+    pattern = os.path.join(args.input_dir, "*document_content_*.json")
+    files = sorted(glob.glob(pattern))
+    if not files:
+        print("No document_content JSON files found in", args.input_dir)
+        return
+
+    out_dir = Path(args.out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    created = []
+    for p in files:
+        try:
+            with open(p, "r", encoding="utf-8") as f:
+                meta = json.load(f)
+        except Exception as e:
+            print("Failed to read JSON:", p, e)
+            continue
+        doc_html = None
+        body_file = meta.get("response_body_file")
+
+        # If there is an explicit body file, try to open it. Some captures wrote the
+        # body into a separate file; others accidentally used the same filename as
+        # the metadata file. Try both.
+        if body_file:
+            bf = Path(body_file)
+            if not bf.exists():
+                bf = Path(args.input_dir) / Path(body_file).name
+            if bf.exists() and bf.is_file():
+                try:
+                    with open(bf, "r", encoding="utf-8") as f:
+                        body = json.load(f)
+                    # body is often a dict containing documentHtml
+                    doc_html = body.get("documentHtml")
+                except Exception:
+                    # not JSON or unreadable
+                    doc_html = None
+
+        # Fallback: sometimes the metadata includes an escaped JSON snippet in
+        # `response_excerpt` which contains documentHtml. Try to parse it.
+        if not doc_html:
+            candidate = meta.get("response_excerpt")
+            if candidate and "documentHtml" in candidate:
+                try:
+                    excerpt_json = json.loads(candidate)
+                    doc_html = excerpt_json.get("documentHtml")
+                except Exception:
+                    doc_html = None
+
+        if not doc_html:
+            # nothing to extract from this file
+            continue
+
+        if not doc_html:
+            continue
+
+        # documentHtml may contain escaped characters; ensure it's properly unescaped
+        # It's typically a string with HTML already; we'll write it verbatim.
+        # Build output filename from meta url or source filename
+        url = meta.get("url") or p
+        # try to extract last path part or uuid
+        name_hint = url.rstrip("/").split('/')[-1]
+        ts = meta.get("timestamp") or "unknown"
+        out_name = f"extracted_document_{name_hint}_{ts}.html"
+        out_path = out_dir / out_name
+        # If the doc_html looks like JSON-escaped, unescape HTML entities
+        try:
+            content = doc_html
+            # if it looks like it's escaped with backslashes, unescape using unicode-escape then html.unescape
+            # but avoid corrupting valid content
+            if "\\n" in content or "\\u" in content:
+                try:
+                    content = content.encode('utf-8').decode('unicode_escape')
+                except Exception:
+                    pass
+            content = html.unescape(content)
+        except Exception:
+            pass
+
+        try:
+            with open(out_path, "w", encoding="utf-8") as f:
+                f.write(content)
+            created.append(str(out_path))
+        except Exception as e:
+            print("Failed to write:", out_path, e)
+
+    print(f"Extracted {len(created)} document HTML files")
+    for c in created[:20]:
+        print(" -", c)
+
+if __name__ == '__main__':
+    main()