.
This commit is contained in:
Ross
@@ -31,6 +31,7 @@ import urllib.parse
|
||||
from pprint import pformat
|
||||
from loguru import logger
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
IMAGE_GROUPS = {}
|
||||
CAPTURE_INPUT_DIR = None
|
||||
@@ -41,7 +42,6 @@ ANATOMY = {}
|
||||
CASES = {}
|
||||
|
||||
|
||||
|
||||
def text_of(node) -> str:
|
||||
"""Return the text content of a node, stripping extra whitespace."""
|
||||
if node is None:
|
||||
@@ -689,7 +689,11 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
ddx_entry = DDX.get(docid)
|
||||
# ddx_entry may be HTML or structured list/dict
|
||||
if isinstance(ddx_entry, dict):
|
||||
ddx_list = ddx_entry.get("ddx") or ddx_entry.get("differentialDiagnoses") or ddx_entry.get("differentials")
|
||||
ddx_list = (
|
||||
ddx_entry.get("ddx")
|
||||
or ddx_entry.get("differentialDiagnoses")
|
||||
or ddx_entry.get("differentials")
|
||||
)
|
||||
else:
|
||||
# could be list or simple string
|
||||
ddx_list = ddx_entry
|
||||
@@ -705,7 +709,12 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
if isinstance(ddx_list, list):
|
||||
for item in ddx_list:
|
||||
md += "### " + item.get("title") + "\n"
|
||||
md += item.get("documentType") + ":" + item.get("documentId") + "\n\n"
|
||||
md += (
|
||||
item.get("documentType")
|
||||
+ ":"
|
||||
+ item.get("documentId")
|
||||
+ "\n\n"
|
||||
)
|
||||
else:
|
||||
md += str(ddx_list).strip() + "\n"
|
||||
md += "\n"
|
||||
@@ -727,7 +736,11 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
tables_list = tables_entry.get("tables")
|
||||
else:
|
||||
tables_list = tables_entry
|
||||
if tables_html and isinstance(tables_html, str) and tables_html.strip():
|
||||
if (
|
||||
tables_html
|
||||
and isinstance(tables_html, str)
|
||||
and tables_html.strip()
|
||||
):
|
||||
try:
|
||||
tbl_md = html_to_markdown(tables_html)
|
||||
md = md.rstrip() + "\n\n" + "## Tables\n\n" + tbl_md + "\n"
|
||||
@@ -736,7 +749,9 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
logger.debug("Failed to convert tableHtml for %s", out_path)
|
||||
elif tables_list:
|
||||
try:
|
||||
front_lines.append(f"tables: {json.dumps(len(tables_list) if isinstance(tables_list, list) else True)}")
|
||||
front_lines.append(
|
||||
f"tables: {json.dumps(len(tables_list) if isinstance(tables_list, list) else True)}"
|
||||
)
|
||||
md = md.rstrip() + "\n\n" + "## Tables\n\n"
|
||||
if isinstance(tables_list, list):
|
||||
for t in tables_list:
|
||||
@@ -761,17 +776,32 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
if anatomy_data:
|
||||
try:
|
||||
if isinstance(anatomy_data, list):
|
||||
front_lines.append('anatomy:')
|
||||
front_lines.append("anatomy:")
|
||||
for a in anatomy_data:
|
||||
front_lines.append(f" - {json.dumps(str(a))}")
|
||||
md = md.rstrip() + "\n\n" + "## Anatomy\n\n"
|
||||
for item in anatomy_data:
|
||||
md += "### " + item.get("title") + "\n"
|
||||
md += item.get("category", "").strip() + "/" + item.get("documentType").strip() + ":" + item.get("documentId") + "\n\n"
|
||||
md += (
|
||||
item.get("category", "").strip()
|
||||
+ "/"
|
||||
+ item.get("documentType").strip()
|
||||
+ ":"
|
||||
+ item.get("documentId")
|
||||
+ "\n\n"
|
||||
)
|
||||
md += "\n"
|
||||
else:
|
||||
front_lines.append(f"anatomy: {json.dumps(str(anatomy_data))}")
|
||||
md = md.rstrip() + "\n\n" + "## Anatomy\n\n" + str(anatomy_data).strip() + "\n"
|
||||
front_lines.append(
|
||||
f"anatomy: {json.dumps(str(anatomy_data))}"
|
||||
)
|
||||
md = (
|
||||
md.rstrip()
|
||||
+ "\n\n"
|
||||
+ "## Anatomy\n\n"
|
||||
+ str(anatomy_data).strip()
|
||||
+ "\n"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
@@ -786,7 +816,9 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
cases_data = cases_entry
|
||||
if cases_data:
|
||||
try:
|
||||
front_lines.append(f"cases: {json.dumps(len(cases_data) if isinstance(cases_data, list) else True)}")
|
||||
front_lines.append(
|
||||
f"cases: {json.dumps(len(cases_data) if isinstance(cases_data, list) else True)}"
|
||||
)
|
||||
md = md.rstrip() + "\n\n" + "## Cases\n\n"
|
||||
if isinstance(cases_data, list):
|
||||
for c in cases_data:
|
||||
@@ -1072,6 +1104,13 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
# content already saved elsewhere — skip writing
|
||||
return False, f"duplicate: {existing}"
|
||||
|
||||
# Modify arrow urls to point to local folder (remove preceding /)
|
||||
md = re.sub(
|
||||
r"\/img/arrows/",
|
||||
r"img/arrows/",
|
||||
md,
|
||||
)
|
||||
|
||||
try:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
@@ -1127,6 +1166,12 @@ def main(argv: Iterable[str] | None = None) -> int:
|
||||
default=False,
|
||||
help="Clear output directory before processing (default: false)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--copy-annotation-images",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help="Copy annotation images to output directory",
|
||||
)
|
||||
args = p.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
input_dir = args.input_dir
|
||||
@@ -1141,6 +1186,18 @@ def main(argv: Iterable[str] | None = None) -> int:
|
||||
print(f"No files found in {input_dir} matching *.json")
|
||||
return 1
|
||||
|
||||
if args.copy_annotation_images:
|
||||
images = glob.glob(os.path.join(input_dir, "images/**", "*_img_arrows*"), recursive=True)
|
||||
|
||||
logger.debug(f"Found {len(images)} annotation image files to copy.")
|
||||
logger.debug(images)
|
||||
out_images_dir = Path(output_dir) / "articles" / "img" / "arrows"
|
||||
out_images_dir.mkdir(parents=True, exist_ok=True)
|
||||
for img_path in images:
|
||||
out_name = Path(img_path).name.split("_", 1)[1].split(".png", 1)[0] + ".png"
|
||||
out_name = out_name.rsplit("_", 1)[1]
|
||||
shutil.copy2(img_path, out_images_dir / out_name)
|
||||
|
||||
# Start by caching image group metadata
|
||||
for path in files:
|
||||
base = os.path.basename(path)
|
||||
@@ -1172,7 +1229,6 @@ def main(argv: Iterable[str] | None = None) -> int:
|
||||
if "meta" in base:
|
||||
continue
|
||||
|
||||
|
||||
# only process files that match the desired prefix
|
||||
if "document_summary" in base:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
|
||||
Reference in New Issue
Block a user