ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
Ross
2025-10-18 13:27:30 +01:00
parent fbf4ab926b
commit 96d0de8e1a
429 changed files with 4948 additions and 10804 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/document_to_markdown.py
+91 -11
View File
@@ -282,6 +282,23 @@ def recursive_search_for_key(obj, key: str):
return None
def recursive_find_key_any(obj, key: str):
"""Recursively search for key in nested JSON-like object and return its value (any type)."""
if isinstance(obj, dict):
if key in obj:
return obj[key]
for v in obj.values():
res = recursive_find_key_any(v, key)
if res is not None:
return res
elif isinstance(obj, list):
for item in obj:
res = recursive_find_key_any(item, key)
if res is not None:
return res
return None
def recursive_search_for_value(obj, value: str) -> bool:
"""Return True if `value` appears as a substring in any string value inside obj."""
if value is None:
@@ -549,6 +566,56 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
front_lines.append(f"title: {json.dumps(article_name.strip())}")
if docid:
front_lines.append(f"docid: {json.dumps(docid)}")
summary_data = DOCUMENT_SUMMARYS.get(docid, {})
# Try to extract summary fields from the capture JSON (data_peek) or the loaded data
def safe_get(k):
v = None
try:
v = recursive_find_key_any(summary_data, k)
except Exception:
v = None
return v
# Authors (list of {key,value})
authors = safe_get('authors')
if isinstance(authors, list) and authors:
front_lines.append('authors:')
for a in authors:
try:
key = a.get('key')
val = a.get('value')
front_lines.append(f" - key: {json.dumps(key)}")
front_lines.append(f" value: {json.dumps(val)}")
except Exception:
continue
# Breadcrumbs: include name/slug/treeNodeId chain if available
bcs = safe_get('breadcrumbs')
if isinstance(bcs, list) and bcs:
front_lines.append('breadcrumbs:')
for bc in bcs:
if not isinstance(bc, dict):
continue
name = bc.get('name')
slug = bc.get('slug')
tid = bc.get('treeNodeId') or bc.get('treeId')
front_lines.append(' -')
front_lines.append(f" name: {json.dumps(name)}")
front_lines.append(f" slug: {json.dumps(slug)}")
front_lines.append(f" treeNodeId: {json.dumps(tid)}")
# Misc summary fields
for key in ('category','cmeTopicId','documentVersionId','imageCount','lastUpdated','pageDescription','pageKeywords','pageTitle','enhancedTitle','type'):
val = safe_get(key)
if val is not None:
# booleans and numbers should be represented without JSON quoting
if isinstance(val, (bool,int)):
front_lines.append(f"{key}: {json.dumps(val)}")
else:
front_lines.append(f"{key}: {json.dumps(str(val))}")
# collect breadcrumbs and render as YAML list if present
crumbs = []
if docid:
@@ -613,8 +680,9 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
for img in images:
try:
iid = img.get('imageId') or img.get('id')
caption = img.get('caption') or img.get('imageTitle') or img.get('title') or ""
caption = img.get('caption') or ""
thumb = img.get('thumbnailUrl') or ""
image_title = img.get('imageTitle') or img.get('title') or img.get('enhancedTitle') or ""
# find matching files under the capture input dir recursively
matches = []
if iid:
@@ -641,11 +709,11 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
rel = os.path.join('images', os.path.basename(match))
rel_paths.append(rel)
appended_images.append(rel)
# associate imageId -> rel + caption for later replacement
# associate imageId -> rel + caption + title for later replacement
if iid:
# prefer explicit caption if available
cval = caption or img.get('caption') or img.get('imageTitle') or img.get('title') or ''
image_map[iid] = (rel, cval)
# prefer explicit caption if available, and capture title
cval = caption or img.get('caption') or image_title or ''
image_map[iid] = (rel, cval, image_title)
except Exception:
continue
@@ -653,9 +721,10 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
if rel_paths:
for rp in rel_paths:
cap = caption or os.path.basename(rp)
# render image and then visible caption on the next line
# render image and then visible title (if any) and caption on subsequent lines
images_md_lines.append(f"![{cap}]({rp})")
# ensure caption is visible (italicized)
if image_title:
images_md_lines.append(f"**{image_title}**")
images_md_lines.append(f"*{cap}*")
images_md_lines.append("")
else:
@@ -664,6 +733,8 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
# normalize thumb path to a relative filename when possible
thumb_name = os.path.basename(thumb.split('?')[0])
images_md_lines.append(f"![{caption}]({thumb_name})")
if image_title:
images_md_lines.append(f"**{image_title}**")
images_md_lines.append(f"*{caption}*")
images_md_lines.append("")
else:
@@ -678,15 +749,24 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
# Before attaching, replace inline references to these images so
# their alt text shows the caption (not original alt).
try:
for iid, (rel, cap) in image_map.items():
for iid, (rel, cap, title) in image_map.items():
# replace markdown image links that include the imageId or filename
bn = os.path.basename(rel)
# 1) replace markdown image links where the URL contains the imageId
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(iid) + r'[^)]*\)', f'![{cap}]({rel})\n\n*{cap}*', md)
if title:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(iid) + r'[^)]*\)', f'![{cap}]({rel})\n\n**{title}**\n\n*{cap}*', md)
else:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(iid) + r'[^)]*\)', f'![{cap}]({rel})\n\n*{cap}*', md)
# 2) replace markdown image links where URL contains the basename
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(bn) + r'[^)]*\)', f'![{cap}]({rel})\n\n*{cap}*', md)
if title:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(bn) + r'[^)]*\)', f'![{cap}]({rel})\n\n**{title}**\n\n*{cap}*', md)
else:
md = re.sub(r'!\[[^\]]*\]\([^)]*' + re.escape(bn) + r'[^)]*\)', f'![{cap}]({rel})\n\n*{cap}*', md)
# 3) replace HTML <img ... src="...basename..."> with markdown image
md = re.sub(r'<img[^>]+src=["\"][^"\']*' + re.escape(bn) + r'[^"\']*["\"][^>]*>', f'![{cap}]({rel})\n\n*{cap}*', md)
if title:
md = re.sub(r'<img[^>]+src=["\"][^"\']*' + re.escape(bn) + r'[^"\']*["\"][^>]*>', f'![{cap}]({rel})\n\n**{title}**\n\n*{cap}*', md)
else:
md = re.sub(r'<img[^>]+src=["\"][^"\']*' + re.escape(bn) + r'[^"\']*["\"][^>]*>', f'![{cap}]({rel})\n\n*{cap}*', md)
# 4) replace full URLs ending with basename
md = re.sub(r'https?://[^)\s]*' + re.escape(bn), rel, md)
# 5) replace URL-encoded variants