ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
Ross
2025-10-19 17:54:25 +01:00
parent 66512aa439
commit f1bfdd8e7a
381 changed files with 16749 additions and 4079 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/document_to_markdown.py
+8 -16
View File
@@ -684,14 +684,11 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
try:
# DDX: only consider cached DDX[docid]; if not present, ddx is treated as not existing
ddx_entry = None
ddx_html = None
ddx_list = None
if isinstance(DDX, dict) and docid and docid in DDX:
ddx_entry = DDX.get(docid)
logger.debug(f"Found DDX entry for docid {docid}: {ddx_entry}")
# ddx_entry may be HTML or structured list/dict
if isinstance(ddx_entry, dict):
ddx_html = ddx_entry.get("ddxHtml")
ddx_list = ddx_entry.get("ddx") or ddx_entry.get("differentialDiagnoses") or ddx_entry.get("differentials")
else:
# could be list or simple string
@@ -700,29 +697,23 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
logger.debug(f"No cached DDX entry for docid {docid}")
logger.debug(f"DDX keys available: {list(DDX.keys())}")
logger.debug(f"DDX entry for docid {docid}: {ddx_entry}")
if ddx_html and isinstance(ddx_html, str) and ddx_html.strip():
try:
ddx_md = html_to_markdown(ddx_html)
md = md.rstrip() + "\n\n" + "## Differential diagnosis\n\n" + ddx_md + "\n"
front_lines.append(f"ddx: {json.dumps(True)}")
except Exception:
logger.debug("Failed to convert ddxHtml for %s", out_path)
elif ddx_list:
if ddx_list:
# render list representation
try:
front_lines.append(f"ddx: {json.dumps(True)}")
md = md.rstrip() + "\n\n" + "## Differential diagnosis\n\n"
if isinstance(ddx_list, list):
for item in ddx_list:
md += "- " + str(item).strip() + "\n"
md += "### " + item.get("title") + "\n"
md += item.get("documentType") + ":" + item.get("documentId") + "\n\n"
else:
md += str(ddx_list).strip() + "\n"
md += "\n"
except Exception:
logger.debug(f"Failed to process DDX list for {out_path}")
pass
except Exception:
pass
logger.debug(f"Failed to process DDX for {out_path}")
try:
# Tables: only consider cached TABLES[docid]
@@ -774,8 +765,9 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
for a in anatomy_data:
front_lines.append(f" - {json.dumps(str(a))}")
md = md.rstrip() + "\n\n" + "## Anatomy\n\n"
for a in anatomy_data:
md += "- " + str(a).strip() + "\n"
for item in anatomy_data:
md += "### " + item.get("title") + "\n"
md += item.get("category", "").strip() + "/" + item.get("documentType").strip() + ":" + item.get("documentId") + "\n\n"
md += "\n"
else:
front_lines.append(f"anatomy: {json.dumps(str(anatomy_data))}")