ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
Ross
2025-10-18 15:20:26 +01:00
parent 96d0de8e1a
commit 5983ca3252
125 changed files with 6098 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/document_to_markdown.py
+270 -72
View File
@@ -11,6 +11,7 @@ converts that HTML to Markdown using BeautifulSoup-based rules.
Output: a .md file next to the JSON (or in --output-dir) with the same base name.
"""
from __future__ import annotations
import argparse
@@ -33,12 +34,22 @@ import sys
IMAGE_GROUPS = {}
CAPTURE_INPUT_DIR = None
DOCUMENT_SUMMARYS = {}
DDX = {}
TABLES = {}
ANATOMY = {}
CASES = {}
def text_of(node) -> str:
"""Return the text content of a node, stripping extra whitespace."""
if node is None:
return ""
s = node.get_text(separator=" ", strip=True) if hasattr(node, "get_text") else str(node).strip()
s = (
node.get_text(separator=" ", strip=True)
if hasattr(node, "get_text")
else str(node).strip()
)
# collapse multiple spaces
return re.sub(r"\s+", " ", s)
@@ -111,9 +122,9 @@ def node_to_md(node, indent=0) -> str:
if name == "blockquote":
content = text_of(node)
lines = content.splitlines()
return "\n" + "\n".join(
"> " + line for line in lines if line.strip()
) + "\n\n"
return (
"\n" + "\n".join("> " + line for line in lines if line.strip()) + "\n\n"
)
if name == "table":
# simple table conversion: try header row then body rows
@@ -166,7 +177,11 @@ def find_html_in_json(obj) -> str | None:
return None
# common keys
for key in ("documentHtml", "html", "content", "document_html", "bodyHtml"):
if key in obj and isinstance(obj[key], str) and ("<" in obj[key] and ">" in obj[key]):
if (
key in obj
and isinstance(obj[key], str)
and ("<" in obj[key] and ">" in obj[key])
):
return obj[key]
# search nested
@@ -193,7 +208,7 @@ def slugify(s: str, max_len: int = 80) -> str:
return s
#def find_article_name_in_json(obj) -> str | None:
# def find_article_name_in_json(obj) -> str | None:
# """Look for an article name in known keys or nested values.
#
# Common keys include: aname, articleName, name, title, documentTitle
@@ -316,8 +331,6 @@ def recursive_search_for_value(obj, value: str) -> bool:
return False
def find_breadcrumbs_for_docid(docid: str, search_dir: str) -> list:
"""Return a list of breadcrumb strings associated with docid by scanning
captured JSONs. Preserves discovery order and HTML-unescapes values.
@@ -367,6 +380,7 @@ def find_breadcrumbs_for_docid(docid: str, search_dir: str) -> list:
base = bc_lists[0].copy()
# try to find a leaf name for this doc within the JSON
leaf = None
def recurse_find_leaf(obj):
nonlocal leaf
if leaf:
@@ -378,7 +392,11 @@ def find_breadcrumbs_for_docid(docid: str, search_dir: str) -> list:
return
# some files use enhancedDocumentName/documentName
for k in ("enhancedDocumentName", "documentName", "name", "title"):
if k in obj and isinstance(obj[k], str) and obj.get("documentId") == docid:
if (
k in obj
and isinstance(obj[k], str)
and obj.get("documentId") == docid
):
leaf = html.unescape(obj.get(k))
return
for v in obj.values():
@@ -415,11 +433,16 @@ def find_breadcrumbs_for_docid(docid: str, search_dir: str) -> list:
bc_lists = []
if isinstance(data, dict):
bc_lists = []
# try to locate breadcrumbs within same file by re-loading
def find_bcs(obj):
if isinstance(obj, dict):
if obj.get("breadcrumbs") and isinstance(obj.get("breadcrumbs"), list):
return collect_names_from_breadcrumbs(obj.get("breadcrumbs"))
if obj.get("breadcrumbs") and isinstance(
obj.get("breadcrumbs"), list
):
return collect_names_from_breadcrumbs(
obj.get("breadcrumbs")
)
for v in obj.values():
res = find_bcs(v)
if res:
@@ -476,7 +499,11 @@ def find_title_in_capture_index(docid: str, base_dir: str) -> str | None:
results = sr.get("results")
if isinstance(results, list):
for r in results:
if isinstance(r, dict) and r.get("id") == docid and r.get("title"):
if (
isinstance(r, dict)
and r.get("id") == docid
and r.get("title")
):
return r.get("title")
except Exception:
return None
@@ -503,12 +530,12 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
docid = base.split("_document_content_")[1].split("_")[0]
logger.debug(f"Extracted docid: {docid}")
#logger.debug(f"DOCUMENT_SUMMARYS keys: {DOCUMENT_SUMMARYS.get(docid)}")
# logger.debug(f"DOCUMENT_SUMMARYS keys: {DOCUMENT_SUMMARYS.get(docid)}")
article_name = DOCUMENT_SUMMARYS.get(docid).get("title")
# if still not found, try to extract from inline HTML
#if not article_name and isinstance(data_peek, dict):
# if not article_name and isinstance(data_peek, dict):
# html_candidate = find_html_in_json(data_peek)
# if html_candidate:
# title_from_html = extract_title_from_html(html_candidate)
@@ -567,7 +594,6 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
if docid:
front_lines.append(f"docid: {json.dumps(docid)}")
summary_data = DOCUMENT_SUMMARYS.get(docid, {})
# Try to extract summary fields from the capture JSON (data_peek) or the loaded data
@@ -580,46 +606,85 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
return v
# Authors (list of {key,value})
authors = safe_get('authors')
authors = safe_get("authors")
if isinstance(authors, list) and authors:
front_lines.append('authors:')
front_lines.append("authors:")
for a in authors:
try:
key = a.get('key')
val = a.get('value')
key = a.get("key")
val = a.get("value")
front_lines.append(f" - key: {json.dumps(key)}")
front_lines.append(f" value: {json.dumps(val)}")
except Exception:
continue
# Breadcrumbs: include name/slug/treeNodeId chain if available
bcs = safe_get('breadcrumbs')
bcs = safe_get("breadcrumbs")
if isinstance(bcs, list) and bcs:
front_lines.append('breadcrumbs:')
front_lines.append("breadcrumbs:")
for bc in bcs:
if not isinstance(bc, dict):
continue
name = bc.get('name')
slug = bc.get('slug')
tid = bc.get('treeNodeId') or bc.get('treeId')
front_lines.append(' -')
name = bc.get("name")
slug = bc.get("slug")
tid = bc.get("treeNodeId") or bc.get("treeId")
front_lines.append(" -")
front_lines.append(f" name: {json.dumps(name)}")
front_lines.append(f" slug: {json.dumps(slug)}")
front_lines.append(f" treeNodeId: {json.dumps(tid)}")
# Misc summary fields
for key in ('category','cmeTopicId','documentVersionId','imageCount','lastUpdated','pageDescription','pageKeywords','pageTitle','enhancedTitle','type'):
for key in (
"category",
"cmeTopicId",
"documentVersionId",
"imageCount",
"lastUpdated",
"pageDescription",
"pageKeywords",
"pageTitle",
"enhancedTitle",
"type",
):
val = safe_get(key)
if val is not None:
# booleans and numbers should be represented without JSON quoting
if isinstance(val, (bool,int)):
if isinstance(val, (bool, int)):
front_lines.append(f"{key}: {json.dumps(val)}")
else:
front_lines.append(f"{key}: {json.dumps(str(val))}")
# References: look for referencesHtml in the capture JSON and append
try:
refs_html = None
# prefer references from the peeked summary data, fall back to full data_peek
refs_html = recursive_find_key_any(summary_data, "referencesHtml")
if refs_html is None:
refs_html = recursive_find_key_any(data_peek, "referencesHtml")
if refs_html and isinstance(refs_html, str) and refs_html.strip():
try:
refs_md = html_to_markdown(refs_html)
# Append indicator to frontmatter
front_lines.append(f"references: {json.dumps(True)}")
# Append the references content to the markdown body (we'll combine front + body below)
md = (
md.rstrip()
+ "\n\n"
+ "## References\n\n"
+ refs_md
+ "\n"
)
except Exception:
logger.debug(
"Failed to convert referencesHtml for %s", out_path
)
except Exception:
pass
# collect breadcrumbs and render as YAML list if present
crumbs = []
if docid:
crumbs = find_breadcrumbs_for_docid(docid, os.path.dirname(path) or ".")
crumbs = find_breadcrumbs_for_docid(
docid, os.path.dirname(path) or "."
)
if crumbs:
front_lines.append("breadcrumbs:")
for c in crumbs:
@@ -648,7 +713,7 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
if isinstance(data_peek, dict) and "imageGroups" in data_peek:
for img_group in data_peek.get("imageGroups", []):
#logger.debug(f"Image group: {pformat(img_group)}")
# logger.debug(f"Image group: {pformat(img_group)}")
group_id = img_group.get("imageGroupId")
try:
@@ -657,8 +722,8 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
except KeyError:
logger.warning(f"Image group {group_id} not found in IMAGE_GROUPS.")
#logger.debug(f"Extracted image data to add: {pformat(image_data_to_add)}")
# logger.debug(f"Extracted image data to add: {pformat(image_data_to_add)}")
# If we have image groups to add, copy matching files and append a formatted
# Images section to the markdown. We search the capture input directory
@@ -679,26 +744,37 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
# render each image in the group
for img in images:
try:
iid = img.get('imageId') or img.get('id')
caption = img.get('caption') or ""
thumb = img.get('thumbnailUrl') or ""
image_title = img.get('imageTitle') or img.get('title') or img.get('enhancedTitle') or ""
iid = img.get("imageId") or img.get("id")
caption = img.get("caption") or ""
thumb = img.get("thumbnailUrl") or ""
image_title = (
img.get("imageTitle")
or img.get("title")
or img.get("enhancedTitle")
or ""
)
# find matching files under the capture input dir recursively
matches = []
if iid:
matches = glob.glob(os.path.join(search_root, "**", f"*{iid}*"), recursive=True)
matches = glob.glob(
os.path.join(search_root, "**", f"*{iid}*"),
recursive=True,
)
# also consider thumbnail URL id in case different naming
if not matches and isinstance(thumb, str) and thumb:
m = re.search(r"/thumbnail/([0-9a-fA-F-\-]+)", thumb)
if m:
tid = m.group(1)
matches = glob.glob(os.path.join(search_root, "**", f"*{tid}*"), recursive=True)
matches = glob.glob(
os.path.join(search_root, "**", f"*{tid}*"),
recursive=True,
)
rel_paths = []
for match in sorted(set(matches)):
# skip JSON metadata files; prefer binary images
b = os.path.basename(match)
if b.endswith('.json') or b.endswith('.meta.json'):
if b.endswith(".json") or b.endswith(".meta.json"):
# still copy metadata alongside image when present
# we'll copy meta files later if we also find an image
continue
@@ -706,13 +782,18 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
dst = os.path.join(images_dir, os.path.basename(match))
if os.path.abspath(match) != os.path.abspath(dst):
shutil.copy2(match, dst)
rel = os.path.join('images', os.path.basename(match))
rel = os.path.join("images", os.path.basename(match))
rel_paths.append(rel)
appended_images.append(rel)
# associate imageId -> rel + caption + title for later replacement
if iid:
# prefer explicit caption if available, and capture title
cval = caption or img.get('caption') or image_title or ''
cval = (
caption
or img.get("caption")
or image_title
or ""
)
image_map[iid] = (rel, cval, image_title)
except Exception:
continue
@@ -731,7 +812,7 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
# try to render a thumbnail path (as-is) if no local file
if thumb:
# normalize thumb path to a relative filename when possible
thumb_name = os.path.basename(thumb.split('?')[0])
thumb_name = os.path.basename(thumb.split("?")[0])
images_md_lines.append(f"![{caption}]({thumb_name})")
if image_title:
images_md_lines.append(f"**{image_title}**")
@@ -754,21 +835,49 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
bn = os.path.basename(rel)
# 1) replace markdown image links where the URL contains the imageId
if title:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(iid) + r'[^)]*\)', f'![{cap}]({rel})\n\n**{title}**\n\n*{cap}*', md)
md = re.sub(
r"!\[[^\]]*\]\([^)]*" + re.escape(iid) + r"[^)]*\)",
f"![{cap}]({rel})\n\n**{title}**\n\n*{cap}*",
md,
)
else:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(iid) + r'[^)]*\)', f'![{cap}]({rel})\n\n*{cap}*', md)
md = re.sub(
r"!\[[^\]]*\]\([^)]*" + re.escape(iid) + r"[^)]*\)",
f"![{cap}]({rel})\n\n*{cap}*",
md,
)
# 2) replace markdown image links where URL contains the basename
if title:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(bn) + r'[^)]*\)', f'![{cap}]({rel})\n\n**{title}**\n\n*{cap}*', md)
md = re.sub(
r"!\[[^\]]*\]\([^)]*" + re.escape(bn) + r"[^)]*\)",
f"![{cap}]({rel})\n\n**{title}**\n\n*{cap}*",
md,
)
else:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(bn) + r'[^)]*\)', f'![{cap}]({rel})\n\n*{cap}*', md)
md = re.sub(
r"!\[[^\]]*\]\([^)]*" + re.escape(bn) + r"[^)]*\)",
f"![{cap}]({rel})\n\n*{cap}*",
md,
)
# 3) replace HTML <img ... src="...basename..."> with markdown image
if title:
md = re.sub(r'<img[^>]+src=["\"][^"\']*' + re.escape(bn) + r'[^"\']*["\"][^>]*>', f'![{cap}]({rel})\n\n**{title}**\n\n*{cap}*', md)
md = re.sub(
r'<img[^>]+src=["\"][^"\']*'
+ re.escape(bn)
+ r'[^"\']*["\"][^>]*>',
f"![{cap}]({rel})\n\n**{title}**\n\n*{cap}*",
md,
)
else:
md = re.sub(r'<img[^>]+src=["\"][^"\']*' + re.escape(bn) + r'[^"\']*["\"][^>]*>', f'![{cap}]({rel})\n\n*{cap}*', md)
md = re.sub(
r'<img[^>]+src=["\"][^"\']*'
+ re.escape(bn)
+ r'[^"\']*["\"][^>]*>',
f"![{cap}]({rel})\n\n*{cap}*",
md,
)
# 4) replace full URLs ending with basename
md = re.sub(r'https?://[^)\s]*' + re.escape(bn), rel, md)
md = re.sub(r"https?://[^)\s]*" + re.escape(bn), rel, md)
# 5) replace URL-encoded variants
try:
q = urllib.parse.quote(bn)
@@ -785,13 +894,13 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
images_index_path = os.path.join(out_dir, "_images_index.json")
images_index = {}
if os.path.exists(images_index_path):
with open(images_index_path, 'r', encoding='utf-8') as ixf:
with open(images_index_path, "r", encoding="utf-8") as ixf:
try:
images_index = json.load(ixf)
except Exception:
images_index = {}
images_index[out_path] = appended_images
with open(images_index_path, 'w', encoding='utf-8') as ixf:
with open(images_index_path, "w", encoding="utf-8") as ixf:
json.dump(images_index, ixf, indent=2, ensure_ascii=False)
except Exception:
pass
@@ -802,10 +911,22 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
# Before writing, compute a normalized content hash to detect duplicates
def normalize_for_hash(s: str) -> str:
# remove ISO8601-like timestamps and common date patterns, collapse whitespace
s2 = re.sub(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:?\d{2})?", "", s)
s2 = re.sub(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:?\d{2})?", "", s
)
s2 = re.sub(r"\b\d{1,2}/\d{1,2}/\d{2,4}\b", "", s2)
# remove lines that are just timestamps or contain 'Last updated'
s2 = "\n".join([ln for ln in s2.splitlines() if not re.match(r"^\s*(Last updated|LastVisited|LastVisitedAsDate|Updated).*$", ln, re.I)])
s2 = "\n".join(
[
ln
for ln in s2.splitlines()
if not re.match(
r"^\s*(Last updated|LastVisited|LastVisitedAsDate|Updated).*$",
ln,
re.I,
)
]
)
s2 = re.sub(r"\s+", " ", s2).strip()
return s2
@@ -844,6 +965,7 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
try:
log_path = os.path.join(target_dir, "_saved_files.log")
from datetime import timezone
with open(log_path, "a", encoding="utf-8") as lf:
lf.write(f"{datetime.now(timezone.utc).isoformat()}\t{out_path}\n")
except Exception:
@@ -857,12 +979,32 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
def main(argv: Iterable[str] | None = None) -> int:
p = argparse.ArgumentParser(description="Convert captured JSON HTML to Markdown")
p.add_argument("--input-dir", default="xhr_captured_async", help="Directory with captured .json files")
p.add_argument("--output-dir", default="docs_md", help="Where to write .md files (defaults to input-dir) ")
p.add_argument("--pattern", default="*.json", help="Glob pattern to find files in input dir")
p.add_argument("--overwrite", action="store_true", help="Overwrite existing .md files")
p.add_argument("--verbose", "-v", action="store_true", help="Print processing details")
p.add_argument("--clear", "-c", action="store_true", default=False, help="Clear output directory before processing (default: false)")
p.add_argument(
"--input-dir",
default="xhr_captured_async",
help="Directory with captured .json files",
)
p.add_argument(
"--output-dir",
default="docs_md",
help="Where to write .md files (defaults to input-dir) ",
)
p.add_argument(
"--pattern", default="*.json", help="Glob pattern to find files in input dir"
)
p.add_argument(
"--overwrite", action="store_true", help="Overwrite existing .md files"
)
p.add_argument(
"--verbose", "-v", action="store_true", help="Print processing details"
)
p.add_argument(
"--clear",
"-c",
action="store_true",
default=False,
help="Clear output directory before processing (default: false)",
)
args = p.parse_args(list(argv) if argv is not None else None)
input_dir = args.input_dir
@@ -871,7 +1013,7 @@ def main(argv: Iterable[str] | None = None) -> int:
if args.clear and os.path.exists(output_dir):
# remove all .md files in output_dir
shutil.rmtree(output_dir, ignore_errors=True)
files = sorted(glob.glob(os.path.join(input_dir, args.pattern)))
if not files:
print(f"No files found in {input_dir} matching {args.pattern}")
@@ -881,14 +1023,17 @@ def main(argv: Iterable[str] | None = None) -> int:
for path in files:
base = os.path.basename(path)
# only process files that match the desired prefix
if not base.startswith("app.statdx.com_document_") or base.endswith("meta.json") or ("_media_" not in base):
if (
not base.startswith("app.statdx.com_document_")
or base.endswith("meta.json")
or ("_media_" not in base)
):
continue
with open(path, "r", encoding="utf-8") as f:
image_group_data = json.load(f)
for group in image_group_data:
group_id = group.get("groupId")
group_name = group.get("name")
images = group.get("images", [])
@@ -896,31 +1041,84 @@ def main(argv: Iterable[str] | None = None) -> int:
"name": group_name,
"images": images,
}
#logger.debug(f"Image group {n}: {pformat(group)}")
# logger.debug(f"Image group {n}: {pformat(group)}")
# Then cache document summary data
for path in files:
base = os.path.basename(path)
# only process files that match the desired prefix
if "document_summary" not in base or "meta" in base:
if "meta" in base:
continue
with open(path, "r", encoding="utf-8") as f:
summary_data = json.load(f)
doc_id = base.split("_document_summary_")[1].split("_")[0]
# only process files that match the desired prefix
if "document_summary" in base:
with open(path, "r", encoding="utf-8") as f:
summary_data = json.load(f)
DOCUMENT_SUMMARYS[doc_id] = summary_data #logger.debug(f"Document summary {n}: {pformat(doc)}")
doc_id = base.split("_document_summary_")[1].split("_")[0]
DOCUMENT_SUMMARYS[doc_id] = (
summary_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
)
elif "ddx" in base:
with open(path, "r", encoding="utf-8") as f:
ddx_data = json.load(f)
doc_id = base.split("_ddx_")[1].split("_")[0]
DDX[doc_id] = (
ddx_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
)
elif "tables" in base:
with open(path, "r", encoding="utf-8") as f:
tables_data = json.load(f)
doc_id = base.split("_tables_")[1].split("_")[0]
TABLES[doc_id] = (
tables_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
)
elif "anatomy" in base:
with open(path, "r", encoding="utf-8") as f:
anatomy_data = json.load(f)
doc_id = base.split("_anatomy_")[1].split("_")[0]
ANATOMY[doc_id] = (
anatomy_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
)
elif "cases" in base:
with open(path, "r", encoding="utf-8") as f:
cases_data = json.load(f)
doc_id = base.split("_cases_")[1].split("_")[0]
CASES[doc_id] = (
cases_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
)
logger.debug(f"Cached {len(IMAGE_GROUPS)} image groups from media files.")
logger.debug(f"Cached {len(DOCUMENT_SUMMARYS)} document summaries from summary files.")
logger.debug(
f"Cached {len(DOCUMENT_SUMMARYS)} document summaries from summary files."
)
ok = 0
for path in files:
base = os.path.basename(path)
# only process files that match the desired prefix
if not base.startswith("app.statdx.com_document_") or base.endswith("meta.json") or ("_media_" in base) or ("breadcrumbs" in base) or ("_summary_" in base) or ("_tables_" in base):
#if args.verbose:
if (
not base.startswith("app.statdx.com_document_")
or base.endswith("meta.json")
or ("_media_" in base)
or ("breadcrumbs" in base)
or ("_summary_" in base)
or ("_tables_" in base)
or ("_anatomy_" in base)
or ("_cases_" in base)
or ("_ddx_" in base)
):
# if args.verbose:
# print(f"SKIP (not matching prefix): {path}")
continue