Ross
e3d081cbff
- Implement capture_with_cdp.py to capture all network response bodies using Chrome DevTools Protocol. - Create extract_sections.py to extract sections from STATdx snapshot HTML files into JSON Lines. - Add unpack_document_content.py to extract `documentHtml` from captured JSON bodies and save as HTML files. - Update .gitignore to include playwright_profile.
130 lines
4.3 KiB
Python
130 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Extract `documentHtml` from captured document content JSON bodies in `xhr_captured/`.
|
|
Writes one .html file per document with a readable filename and prints a short report.
|
|
|
|
Usage:
|
|
python scrapers/unpack_document_content.py --input-dir xhr_captured --out-dir xhr_captured
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import html
|
|
|
|
|
|
def slugify(s):
|
|
keep = "abcdefghijklmnopqrstuvwxyz0123456789-_"
|
|
s = s.lower()
|
|
out = []
|
|
for ch in s:
|
|
if ch in keep:
|
|
out.append(ch)
|
|
else:
|
|
out.append("-")
|
|
res = "".join(out)
|
|
# collapse -
|
|
while "--" in res:
|
|
res = res.replace("--","-")
|
|
return res.strip("-")[:200]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--input-dir", default="xhr_captured")
|
|
parser.add_argument("--out-dir", default="xhr_captured")
|
|
args = parser.parse_args()
|
|
|
|
pattern = os.path.join(args.input_dir, "*document_content_*.json")
|
|
files = sorted(glob.glob(pattern))
|
|
if not files:
|
|
print("No document_content JSON files found in", args.input_dir)
|
|
return
|
|
|
|
out_dir = Path(args.out_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
created = []
|
|
for p in files:
|
|
try:
|
|
with open(p, "r", encoding="utf-8") as f:
|
|
meta = json.load(f)
|
|
except Exception as e:
|
|
print("Failed to read JSON:", p, e)
|
|
continue
|
|
doc_html = None
|
|
body_file = meta.get("response_body_file")
|
|
|
|
# If there is an explicit body file, try to open it. Some captures wrote the
|
|
# body into a separate file; others accidentally used the same filename as
|
|
# the metadata file. Try both.
|
|
if body_file:
|
|
bf = Path(body_file)
|
|
if not bf.exists():
|
|
bf = Path(args.input_dir) / Path(body_file).name
|
|
if bf.exists() and bf.is_file():
|
|
try:
|
|
with open(bf, "r", encoding="utf-8") as f:
|
|
body = json.load(f)
|
|
# body is often a dict containing documentHtml
|
|
doc_html = body.get("documentHtml")
|
|
except Exception:
|
|
# not JSON or unreadable
|
|
doc_html = None
|
|
|
|
# Fallback: sometimes the metadata includes an escaped JSON snippet in
|
|
# `response_excerpt` which contains documentHtml. Try to parse it.
|
|
if not doc_html:
|
|
candidate = meta.get("response_excerpt")
|
|
if candidate and "documentHtml" in candidate:
|
|
try:
|
|
excerpt_json = json.loads(candidate)
|
|
doc_html = excerpt_json.get("documentHtml")
|
|
except Exception:
|
|
doc_html = None
|
|
|
|
if not doc_html:
|
|
# nothing to extract from this file
|
|
continue
|
|
|
|
if not doc_html:
|
|
continue
|
|
|
|
# documentHtml may contain escaped characters; ensure it's properly unescaped
|
|
# It's typically a string with HTML already; we'll write it verbatim.
|
|
# Build output filename from meta url or source filename
|
|
url = meta.get("url") or p
|
|
# try to extract last path part or uuid
|
|
name_hint = url.rstrip("/").split('/')[-1]
|
|
ts = meta.get("timestamp") or "unknown"
|
|
out_name = f"extracted_document_{name_hint}_{ts}.html"
|
|
out_path = out_dir / out_name
|
|
# If the doc_html looks like JSON-escaped, unescape HTML entities
|
|
try:
|
|
content = doc_html
|
|
# if it looks like it's escaped with backslashes, unescape using unicode-escape then html.unescape
|
|
# but avoid corrupting valid content
|
|
if "\\n" in content or "\\u" in content:
|
|
try:
|
|
content = content.encode('utf-8').decode('unicode_escape')
|
|
except Exception:
|
|
pass
|
|
content = html.unescape(content)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
f.write(content)
|
|
created.append(str(out_path))
|
|
except Exception as e:
|
|
print("Failed to write:", out_path, e)
|
|
|
|
print(f"Extracted {len(created)} document HTML files")
|
|
for c in created[:20]:
|
|
print(" -", c)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|