Add comprehensive articles on Vascular Dementia and Wallerian Degeneration
- Created a detailed article for Vascular Dementia covering key facts, terminology, imaging findings, differential diagnoses, pathology, clinical issues, and diagnostic checklist. - Developed an extensive article on Wallerian Degeneration including key facts, terminology, imaging features, differential diagnoses, pathology, clinical issues, and diagnostic checklist.
This commit is contained in:
Ross
@@ -163,11 +163,19 @@ async def run(args):
|
||||
fname = f"{safe}_{h}_{ts}.json"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, pretty)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
excerpt = pretty[:800]
|
||||
except Exception:
|
||||
fname = f"{safe}_{h}_{ts}.txt"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, txt)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
excerpt = txt[:800]
|
||||
else:
|
||||
try:
|
||||
@@ -186,12 +194,20 @@ async def run(args):
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
subdir = output / 'assets'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
# final fallback: try binary
|
||||
try:
|
||||
@@ -203,12 +219,20 @@ async def run(args):
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
subdir = output / 'assets'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
# skip if still failing
|
||||
return
|
||||
|
||||
@@ -19,9 +19,12 @@ import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
import shutil
|
||||
from typing import Iterable
|
||||
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
import html
|
||||
import hashlib
|
||||
|
||||
|
||||
def text_of(node) -> str:
|
||||
@@ -279,31 +282,190 @@ def find_title_for_docid(docid: str, search_dir: str) -> str | None:
|
||||
"""
|
||||
if not docid:
|
||||
return None
|
||||
pattern = os.path.join(search_dir, f"*{docid}*.json")
|
||||
for path in sorted(glob.glob(pattern)):
|
||||
|
||||
# Phase 1: scan all JSON files for breadcrumb/list-style captures and prefer
|
||||
# their enhancedDocumentName/name when present. Breadcrumb captures often
|
||||
# don't include the docid in their filename, so scan every JSON once.
|
||||
all_paths = sorted(glob.glob(os.path.join(search_dir, "*.json")))
|
||||
for path in all_paths:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except Exception:
|
||||
continue
|
||||
# direct title
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
try:
|
||||
if isinstance(item, dict) and item.get("documentId") == docid:
|
||||
ed = item.get("enhancedDocumentName") or item.get("name")
|
||||
if isinstance(ed, str) and ed.strip():
|
||||
return html.unescape(ed.strip())
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Phase 2: look only at files whose filename contains the docid for other
|
||||
# title signals (top-level names, recursive title keys, searchResults).
|
||||
pattern = os.path.join(search_dir, f"*{docid}*.json")
|
||||
paths = sorted(glob.glob(pattern))
|
||||
for path in paths:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# top-level keys commonly used
|
||||
if isinstance(data, dict):
|
||||
for key in ("enhancedDocumentName", "documentName", "name"):
|
||||
if key in data:
|
||||
v = data.get(key)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return html.unescape(v.strip())
|
||||
|
||||
# fallback: direct title anywhere in the JSON
|
||||
title = recursive_search_for_key(data, "title")
|
||||
if title:
|
||||
# some files include empty titles; skip blanks
|
||||
if isinstance(title, str) and title.strip():
|
||||
return title.strip()
|
||||
if title and isinstance(title, str) and title.strip():
|
||||
return html.unescape(title.strip())
|
||||
|
||||
# sometimes summary objects contain results with titles
|
||||
# try 'searchResults' -> 'results' list
|
||||
sr = data.get("searchResults") if isinstance(data, dict) else None
|
||||
if isinstance(sr, dict):
|
||||
results = sr.get("results")
|
||||
if isinstance(results, list):
|
||||
for r in results:
|
||||
if isinstance(r, dict) and r.get("id") == docid and r.get("title"):
|
||||
return r.get("title")
|
||||
return html.unescape(r.get("title"))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_breadcrumbs_for_docid(docid: str, search_dir: str) -> list:
|
||||
"""Return a list of breadcrumb strings associated with docid by scanning
|
||||
captured JSONs. Preserves discovery order and HTML-unescapes values.
|
||||
"""
|
||||
if not docid:
|
||||
return []
|
||||
|
||||
def collect_names_from_breadcrumbs(bc_list):
|
||||
names = []
|
||||
if isinstance(bc_list, list):
|
||||
for item in bc_list:
|
||||
if isinstance(item, dict) and item.get("name"):
|
||||
names.append(html.unescape(item.get("name")))
|
||||
return names
|
||||
|
||||
# First, prefer files whose filename contains the docid. These often
|
||||
# include a structured `breadcrumbs` list (ordered parents) and may
|
||||
# include a leaf node entry for the document.
|
||||
candidates = []
|
||||
doc_paths = sorted(glob.glob(os.path.join(search_dir, f"*{docid}*.json")))
|
||||
for path in doc_paths:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# look for a breadcrumbs list anywhere in this JSON
|
||||
# and prefer the first full list we find
|
||||
def recurse_find_breadcrumb_lists(obj):
|
||||
out = []
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k == "breadcrumbs" and isinstance(v, list):
|
||||
names = collect_names_from_breadcrumbs(v)
|
||||
if names:
|
||||
out.append(names)
|
||||
out.extend(recurse_find_breadcrumb_lists(v))
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
out.extend(recurse_find_breadcrumb_lists(item))
|
||||
return out
|
||||
|
||||
bc_lists = recurse_find_breadcrumb_lists(data)
|
||||
if bc_lists:
|
||||
# take the first breadcrumbs list as base path
|
||||
base = bc_lists[0].copy()
|
||||
# try to find a leaf name for this doc within the JSON
|
||||
leaf = None
|
||||
def recurse_find_leaf(obj):
|
||||
nonlocal leaf
|
||||
if leaf:
|
||||
return
|
||||
if isinstance(obj, dict):
|
||||
# node that references the document directly
|
||||
if obj.get("documentId") == docid and obj.get("name"):
|
||||
leaf = html.unescape(obj.get("name"))
|
||||
return
|
||||
# some files use enhancedDocumentName/documentName
|
||||
for k in ("enhancedDocumentName", "documentName", "name", "title"):
|
||||
if k in obj and isinstance(obj[k], str) and obj.get("documentId") == docid:
|
||||
leaf = html.unescape(obj.get(k))
|
||||
return
|
||||
for v in obj.values():
|
||||
recurse_find_leaf(v)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
recurse_find_leaf(item)
|
||||
|
||||
recurse_find_leaf(data)
|
||||
if leaf and (not base or base[-1] != leaf):
|
||||
base.append(leaf)
|
||||
if base and base not in candidates:
|
||||
candidates.append(base)
|
||||
|
||||
# If we found candidate breadcrumb paths from docid-matching files, return the first
|
||||
if candidates:
|
||||
return candidates[0]
|
||||
|
||||
# Otherwise, scan all JSONs for entries that explicitly list the docid and
|
||||
# attempt to reconstruct a breadcrumb path by combining any nearby breadcrumbs
|
||||
all_paths = sorted(glob.glob(os.path.join(search_dir, "*.json")))
|
||||
for path in all_paths:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# list-of-dicts captures where an item references the documentId
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
if isinstance(item, dict) and item.get("documentId") == docid:
|
||||
# if this file also contains a breadcrumbs list elsewhere, use it
|
||||
bc_lists = []
|
||||
if isinstance(data, dict):
|
||||
bc_lists = []
|
||||
# try to locate breadcrumbs within same file by re-loading
|
||||
def find_bcs(obj):
|
||||
if isinstance(obj, dict):
|
||||
if obj.get("breadcrumbs") and isinstance(obj.get("breadcrumbs"), list):
|
||||
return collect_names_from_breadcrumbs(obj.get("breadcrumbs"))
|
||||
for v in obj.values():
|
||||
res = find_bcs(v)
|
||||
if res:
|
||||
return res
|
||||
elif isinstance(obj, list):
|
||||
for it in obj:
|
||||
res = find_bcs(it)
|
||||
if res:
|
||||
return res
|
||||
return None
|
||||
|
||||
bcs = find_bcs(data)
|
||||
path_names = bcs or []
|
||||
# append the item's own enhancedDocumentName/name if present
|
||||
leaf = item.get("enhancedDocumentName") or item.get("name")
|
||||
if isinstance(leaf, str) and leaf.strip():
|
||||
leaf = html.unescape(leaf.strip())
|
||||
if not path_names or path_names[-1] != leaf:
|
||||
path_names = path_names + [leaf]
|
||||
if path_names:
|
||||
return path_names
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def find_title_in_capture_index(docid: str, base_dir: str) -> str | None:
|
||||
"""Look for a title for docid inside a capture_index.jsonl file in base_dir.
|
||||
|
||||
@@ -354,6 +516,9 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
except Exception:
|
||||
data_peek = None
|
||||
|
||||
# deterministic doc id (if present in filename)
|
||||
docid = None
|
||||
|
||||
article_name = None
|
||||
if isinstance(data_peek, dict):
|
||||
article_name = find_article_name_in_json(data_peek)
|
||||
@@ -363,13 +528,16 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
# try to parse a uuid-like docid from filename
|
||||
m = re.search(r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})", base)
|
||||
if m:
|
||||
docid = m.group(1)
|
||||
# first consult capture_index.jsonl if present
|
||||
title_from_index = find_title_in_capture_index(docid, os.path.dirname(path) or ".")
|
||||
if not title_from_index:
|
||||
title_from_index = find_title_for_docid(docid, os.path.dirname(path) or ".")
|
||||
if title_from_index:
|
||||
article_name = title_from_index
|
||||
docid = m.group(1)
|
||||
# Prefer nearby breadcrumb/document captures (which include
|
||||
# enhancedDocumentName/name) over titles that appear in
|
||||
# capture_index.jsonl. This makes section-level names
|
||||
# like "Brain Tumor in Newborn/Infant" the canonical title.
|
||||
title_from_nearby = find_title_for_docid(docid, os.path.dirname(path) or ".")
|
||||
if not title_from_nearby:
|
||||
title_from_nearby = find_title_in_capture_index(docid, os.path.dirname(path) or ".")
|
||||
if title_from_nearby:
|
||||
article_name = title_from_nearby
|
||||
|
||||
# if still not found, try to extract from inline HTML
|
||||
if not article_name and isinstance(data_peek, dict):
|
||||
@@ -381,11 +549,15 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
|
||||
if article_name:
|
||||
slug = slugify(article_name)
|
||||
out_base = f"{slug}_{name}" if slug else name
|
||||
# prefer deterministic filename based on docid when available to avoid duplicates
|
||||
if docid:
|
||||
out_base = f"{slug}_{docid}" if slug else docid
|
||||
else:
|
||||
out_base = f"{slug}_{name}" if slug else name
|
||||
# place titled articles into an articles/ subfolder
|
||||
target_dir = os.path.join(out_dir, "articles")
|
||||
|
||||
if article_name.startswith("https"):
|
||||
if isinstance(article_name, str) and article_name.startswith("https"):
|
||||
target_dir = os.path.join(out_dir, "external")
|
||||
else:
|
||||
# per request: only save files that have titles
|
||||
@@ -416,15 +588,79 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
|
||||
normalized_first = re.sub(r"[#\s]+", "", first_line).strip().lower()
|
||||
normalized_title = re.sub(r"[^a-z0-9]+", "", article_name.lower())
|
||||
if not normalized_first or normalized_title not in normalized_first:
|
||||
md = "# " + article_name.strip() + "\n\n" + md
|
||||
# Build YAML frontmatter with title and breadcrumbs (and docid)
|
||||
front_lines = ["---"]
|
||||
# use JSON quoting to safely escape strings in YAML
|
||||
front_lines.append(f"title: {json.dumps(article_name.strip())}")
|
||||
if docid:
|
||||
front_lines.append(f"docid: {json.dumps(docid)}")
|
||||
# collect breadcrumbs and render as YAML list if present
|
||||
crumbs = []
|
||||
if docid:
|
||||
crumbs = find_breadcrumbs_for_docid(docid, os.path.dirname(path) or ".")
|
||||
if crumbs:
|
||||
front_lines.append("breadcrumbs:")
|
||||
for c in crumbs:
|
||||
front_lines.append(f" - {json.dumps(c)}")
|
||||
front_lines.append("---\n")
|
||||
front = "\n".join(front_lines)
|
||||
|
||||
# Avoid duplicating an H1 that matches the title in the body
|
||||
md_lines = md.lstrip().splitlines()
|
||||
if md_lines:
|
||||
first = md_lines[0].strip()
|
||||
normalized_first = re.sub(r"[#\s]+", "", first).strip().lower()
|
||||
normalized_title = re.sub(r"[^a-z0-9]+", "", article_name.lower())
|
||||
if first.startswith("#") and normalized_title in normalized_first:
|
||||
# drop the first line (existing H1)
|
||||
md_lines = md_lines[1:]
|
||||
|
||||
md = front + "\n".join(md_lines).lstrip()
|
||||
except Exception:
|
||||
# be conservative: if anything goes wrong, keep the original md
|
||||
pass
|
||||
|
||||
# Before writing, compute a normalized content hash to detect duplicates
|
||||
def normalize_for_hash(s: str) -> str:
|
||||
# remove ISO8601-like timestamps and common date patterns, collapse whitespace
|
||||
s2 = re.sub(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:?\d{2})?", "", s)
|
||||
s2 = re.sub(r"\b\d{1,2}/\d{1,2}/\d{2,4}\b", "", s2)
|
||||
# remove lines that are just timestamps or contain 'Last updated'
|
||||
s2 = "\n".join([ln for ln in s2.splitlines() if not re.match(r"^\s*(Last updated|LastVisited|LastVisitedAsDate|Updated).*$", ln, re.I)])
|
||||
s2 = re.sub(r"\s+", " ", s2).strip()
|
||||
return s2
|
||||
|
||||
def content_hash(s: str) -> str:
|
||||
nh = normalize_for_hash(s)
|
||||
return hashlib.sha256(nh.encode("utf-8")).hexdigest()
|
||||
|
||||
# index stored at top-level out_dir to dedupe across articles/external
|
||||
index_path = os.path.join(out_dir, "_content_index.json")
|
||||
index = {}
|
||||
try:
|
||||
if os.path.exists(index_path):
|
||||
with open(index_path, "r", encoding="utf-8") as ix:
|
||||
index = json.load(ix)
|
||||
except Exception:
|
||||
index = {}
|
||||
|
||||
ch = content_hash(md)
|
||||
existing = index.get(ch)
|
||||
if existing and os.path.exists(existing) and not overwrite:
|
||||
# content already saved elsewhere — skip writing
|
||||
return False, f"duplicate: {existing}"
|
||||
|
||||
try:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
f.write(md)
|
||||
# update index
|
||||
try:
|
||||
index[ch] = out_path
|
||||
with open(index_path, "w", encoding="utf-8") as ix:
|
||||
json.dump(index, ix, indent=2, ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
# log saved file path with timestamp
|
||||
try:
|
||||
log_path = os.path.join(target_dir, "_saved_files.log")
|
||||
@@ -455,12 +691,8 @@ def main(argv: Iterable[str] | None = None) -> int:
|
||||
|
||||
if args.clear and os.path.exists(output_dir):
|
||||
# remove all .md files in output_dir
|
||||
for f in glob.glob(os.path.join(output_dir, "*.md")):
|
||||
try:
|
||||
os.remove(f)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
shutil.rmtree(output_dir, ignore_errors=True)
|
||||
|
||||
files = sorted(glob.glob(os.path.join(input_dir, args.pattern)))
|
||||
if not files:
|
||||
print(f"No files found in {input_dir} matching {args.pattern}")
|
||||
|
||||
Reference in New Issue
Block a user