ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance document processing in scrapers

Browse Source 
- Updated pediatric multiple sclerosis article reference ID.
- Improved document_to_markdown.py to handle differential diagnoses, tables, anatomy, and cases more effectively.
- Added logging for cached entries of DDX, Tables, Anatomy, and Cases.
- Adjusted file pattern matching to process all JSON files by default.
- Fixed document ID extraction logic for various data types.
This commit is contained in:
Ross
2025-10-18 15:41:33 +01:00
parent 5983ca3252
commit 66512aa439
71 changed files with 1176 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/document_to_markdown.py
+146 -7
View File
@@ -15,6 +15,7 @@ Output: a .md file next to the JSON (or in --output-dir) with the same base name
from __future__ import annotations
import argparse
import fnmatch
import glob
import json
import os
@@ -679,6 +680,135 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
)
except Exception:
pass
# DDX / Tables / Anatomy / Cases: detect and append to frontmatter and body
try:
# DDX: only consider cached DDX[docid]; if not present, ddx is treated as not existing
ddx_entry = None
ddx_html = None
ddx_list = None
if isinstance(DDX, dict) and docid and docid in DDX:
ddx_entry = DDX.get(docid)
logger.debug(f"Found DDX entry for docid {docid}: {ddx_entry}")
# ddx_entry may be HTML or structured list/dict
if isinstance(ddx_entry, dict):
ddx_html = ddx_entry.get("ddxHtml")
ddx_list = ddx_entry.get("ddx") or ddx_entry.get("differentialDiagnoses") or ddx_entry.get("differentials")
else:
# could be list or simple string
ddx_list = ddx_entry
else:
logger.debug(f"No cached DDX entry for docid {docid}")
logger.debug(f"DDX keys available: {list(DDX.keys())}")
logger.debug(f"DDX entry for docid {docid}: {ddx_entry}")
if ddx_html and isinstance(ddx_html, str) and ddx_html.strip():
try:
ddx_md = html_to_markdown(ddx_html)
md = md.rstrip() + "\n\n" + "## Differential diagnosis\n\n" + ddx_md + "\n"
front_lines.append(f"ddx: {json.dumps(True)}")
except Exception:
logger.debug("Failed to convert ddxHtml for %s", out_path)
elif ddx_list:
# render list representation
try:
front_lines.append(f"ddx: {json.dumps(True)}")
md = md.rstrip() + "\n\n" + "## Differential diagnosis\n\n"
if isinstance(ddx_list, list):
for item in ddx_list:
md += "- " + str(item).strip() + "\n"
else:
md += str(ddx_list).strip() + "\n"
md += "\n"
except Exception:
pass
except Exception:
pass
try:
# Tables: only consider cached TABLES[docid]
tables_entry = None
tables_html = None
tables_list = None
if isinstance(TABLES, dict) and docid and docid in TABLES:
tables_entry = TABLES.get(docid)
if isinstance(tables_entry, dict):
tables_html = tables_entry.get("tableHtml")
tables_list = tables_entry.get("tables")
else:
tables_list = tables_entry
if tables_html and isinstance(tables_html, str) and tables_html.strip():
try:
tbl_md = html_to_markdown(tables_html)
md = md.rstrip() + "\n\n" + "## Tables\n\n" + tbl_md + "\n"
front_lines.append(f"tables: {json.dumps(True)}")
except Exception:
logger.debug("Failed to convert tableHtml for %s", out_path)
elif tables_list:
try:
front_lines.append(f"tables: {json.dumps(len(tables_list) if isinstance(tables_list, list) else True)}")
md = md.rstrip() + "\n\n" + "## Tables\n\n"
if isinstance(tables_list, list):
for t in tables_list:
if isinstance(t, str) and "<" in t:
md += html_to_markdown(t) + "\n"
else:
md += str(t) + "\n\n"
else:
md += str(tables_list) + "\n"
except Exception:
pass
except Exception:
pass
try:
# Anatomy: only consider cached ANATOMY[docid]
anatomy_entry = None
anatomy_data = None
if isinstance(ANATOMY, dict) and docid and docid in ANATOMY:
anatomy_entry = ANATOMY.get(docid)
anatomy_data = anatomy_entry
if anatomy_data:
try:
if isinstance(anatomy_data, list):
front_lines.append('anatomy:')
for a in anatomy_data:
front_lines.append(f" - {json.dumps(str(a))}")
md = md.rstrip() + "\n\n" + "## Anatomy\n\n"
for a in anatomy_data:
md += "- " + str(a).strip() + "\n"
md += "\n"
else:
front_lines.append(f"anatomy: {json.dumps(str(anatomy_data))}")
md = md.rstrip() + "\n\n" + "## Anatomy\n\n" + str(anatomy_data).strip() + "\n"
except Exception:
pass
except Exception:
pass
try:
# Cases / caseStudies: only consider cached CASES[docid]
cases_entry = None
cases_data = None
if isinstance(CASES, dict) and docid and docid in CASES:
cases_entry = CASES.get(docid)
cases_data = cases_entry
if cases_data:
try:
front_lines.append(f"cases: {json.dumps(len(cases_data) if isinstance(cases_data, list) else True)}")
md = md.rstrip() + "\n\n" + "## Cases\n\n"
if isinstance(cases_data, list):
for c in cases_data:
if isinstance(c, str) and "<" in c:
md += html_to_markdown(c) + "\n"
else:
md += "- " + str(c).strip() + "\n"
md += "\n"
else:
md += str(cases_data).strip() + "\n"
except Exception:
pass
except Exception:
pass
# collect breadcrumbs and render as YAML list if present
crumbs = []
if docid:
@@ -990,7 +1120,7 @@ def main(argv: Iterable[str] | None = None) -> int:
help="Where to write .md files (defaults to input-dir) ",
)
p.add_argument(
"--pattern", default="*.json", help="Glob pattern to find files in input dir"
"--pattern", default="*", help="Glob pattern to limit files processed"
)
p.add_argument(
"--overwrite", action="store_true", help="Overwrite existing .md files"
@@ -1014,9 +1144,9 @@ def main(argv: Iterable[str] | None = None) -> int:
# remove all .md files in output_dir
shutil.rmtree(output_dir, ignore_errors=True)
files = sorted(glob.glob(os.path.join(input_dir, args.pattern)))
files = sorted(glob.glob(os.path.join(input_dir, "*.json")))
if not files:
print(f"No files found in {input_dir} matching {args.pattern}")
print(f"No files found in {input_dir} matching *.json")
return 1
# Start by caching image group metadata
@@ -1065,7 +1195,7 @@ def main(argv: Iterable[str] | None = None) -> int:
with open(path, "r", encoding="utf-8") as f:
ddx_data = json.load(f)
doc_id = base.split("_ddx_")[1].split("_")[0]
doc_id = base.split("_document_")[1].split("_")[0]
DDX[doc_id] = (
ddx_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
@@ -1074,7 +1204,7 @@ def main(argv: Iterable[str] | None = None) -> int:
with open(path, "r", encoding="utf-8") as f:
tables_data = json.load(f)
doc_id = base.split("_tables_")[1].split("_")[0]
doc_id = base.split("_document_")[1].split("_")[0]
TABLES[doc_id] = (
tables_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
@@ -1083,7 +1213,7 @@ def main(argv: Iterable[str] | None = None) -> int:
with open(path, "r", encoding="utf-8") as f:
anatomy_data = json.load(f)
doc_id = base.split("_anatomy_")[1].split("_")[0]
doc_id = base.split("_document_")[1].split("_")[0]
ANATOMY[doc_id] = (
anatomy_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
@@ -1092,7 +1222,7 @@ def main(argv: Iterable[str] | None = None) -> int:
with open(path, "r", encoding="utf-8") as f:
cases_data = json.load(f)
doc_id = base.split("_cases_")[1].split("_")[0]
doc_id = base.split("_document_")[1].split("_")[0]
CASES[doc_id] = (
cases_data # logger.debug(f"Document summary {n}: {pformat(doc)}")
@@ -1102,10 +1232,19 @@ def main(argv: Iterable[str] | None = None) -> int:
logger.debug(
f"Cached {len(DOCUMENT_SUMMARYS)} document summaries from summary files."
)
logger.debug(f"Cached {len(DDX)} DDX entries from ddx files.")
logger.debug(f"Cached {len(TABLES)} Tables entries from tables files.")
logger.debug(f"Cached {len(ANATOMY)} Anatomy entries from anatomy files.")
logger.debug(f"Cached {len(CASES)} Cases entries from cases files.")
ok = 0
for path in files:
base = os.path.basename(path)
if args.pattern and not fnmatch.fnmatch(base, args.pattern):
# if args.verbose:
# print(f"SKIP (pattern mismatch): {path}")
continue
# only process files that match the desired prefix
if (
not base.startswith("app.statdx.com_document_")