feat: Implement image candidate discovery and association in document processing

scrapers/document_to_markdown.py

@@ -25,6 +25,7 @@ from typing import Iterable
 from bs4 import BeautifulSoup, NavigableString, Tag
 import html
 import hashlib
+import urllib.parse
 
 
 def text_of(node) -> str:
@@ -275,6 +276,23 @@ def recursive_search_for_key(obj, key: str):
     return None
 
 
+def recursive_search_for_value(obj, value: str) -> bool:
+    """Return True if `value` appears as a substring in any string value inside obj."""
+    if value is None:
+        return False
+    if isinstance(obj, str):
+        return value in obj
+    if isinstance(obj, dict):
+        for v in obj.values():
+            if recursive_search_for_value(v, value):
+                return True
+    elif isinstance(obj, list):
+        for it in obj:
+            if recursive_search_for_value(it, value):
+                return True
+    return False
+
+
 def find_title_for_docid(docid: str, search_dir: str) -> str | None:
     """Look for files in search_dir that contain the docid and try to extract a title.
 
@@ -620,6 +638,269 @@ def process_file(path: str, out_dir: str, overwrite: bool = False) -> tuple[bool
         # be conservative: if anything goes wrong, keep the original md
         pass
 
+    # (channel/frontmatter injection removed - handled elsewhere)
+
+    # --- find and save related image/response files ---
+    try:
+        input_dir = os.path.dirname(path) or "."
+
+        def find_image_candidates(search_dir: str, docid: str | None, article_name: str | None, image_group_ids: list | None = None) -> list:
+            """Find candidate media/image files in search_dir related to the document.
+
+            Preference order:
+            - files whose filename contains the docid
+            - files that contain any image_group_id
+            - files whose content contains the docid or article_name
+            - filenames that include 'image_' or 'thumbnail' heuristically
+            """
+            candidates = []
+            patterns = ["*image*.*", "*media*.*", "*resource*.*", "*thumbnail*.*"]
+            seen = set()
+            img_ids = set(image_group_ids or [])
+
+            for pat in patterns:
+                for p in glob.glob(os.path.join(search_dir, pat)):
+                    if p in seen:
+                        continue
+                    seen.add(p)
+                    bn = os.path.basename(p)
+
+                    # prefer files that reference docid in filename
+                    if docid and docid in bn:
+                        candidates.append(p)
+                        continue
+
+                    # if filename contains 'media' and we have image group ids, inspect JSON quickly
+                    if img_ids and 'media' in bn.lower() and bn.lower().endswith('.json'):
+                        try:
+                            with open(p, 'r', encoding='utf-8') as pf:
+                                j = json.load(pf)
+                        except Exception:
+                            j = None
+                        if j is not None:
+                            # search for imageGroupId tokens or referenced imageGroupId in the JSON
+                            jtxt = json.dumps(j)
+                            if any(igid in jtxt for igid in img_ids):
+                                candidates.append(p)
+                                continue
+
+                    # fallback to searching the file content for docid or article_name
+                    try:
+                        with open(p, 'r', encoding='utf-8', errors='ignore') as pf:
+                            txt = pf.read()
+                    except Exception:
+                        txt = ""
+                    if docid and docid in txt:
+                        candidates.append(p)
+                        continue
+                    if article_name and article_name in txt:
+                        candidates.append(p)
+                        continue
+
+                    # heuristic filename checks
+                    if 'image_' in bn.lower() or 'thumbnail' in bn.lower() or bn.lower().startswith('img_'):
+                        candidates.append(p)
+            return sorted(set(candidates))
+
+        # gather imageGroup ids from the capture JSON (if present)
+        image_group_ids = []
+        try:
+            # common structure: data.get('imageGroups') -> list of dicts with imageGroupId
+            igs = None
+            if isinstance(data, dict):
+                igs = data.get('imageGroups') or data.get('image_groups')
+            if isinstance(igs, list):
+                for it in igs:
+                    if isinstance(it, dict) and it.get('imageGroupId'):
+                        image_group_ids.append(it.get('imageGroupId'))
+        except Exception:
+            image_group_ids = []
+
+        img_candidates = find_image_candidates(input_dir, docid, article_name, image_group_ids)
+        copied_map = {}
+        if img_candidates:
+            images_dir = os.path.join(target_dir, "images")
+            os.makedirs(images_dir, exist_ok=True)
+
+            # Expand media JSON candidates: parse them for referenced image filenames
+            expanded = list(img_candidates)
+            # mapping of imageId -> caption discovered in media JSONs
+            media_captions = {}
+            for candidate in list(img_candidates):
+                bn = os.path.basename(candidate).lower()
+                if bn.endswith('.json') and ('media' in bn or 'image' in bn or 'resource' in bn):
+                    try:
+                        with open(candidate, 'r', encoding='utf-8') as mf:
+                            mj = json.load(mf)
+                    except Exception:
+                        mj = None
+                    if mj is None:
+                        continue
+
+                    # media JSONs may be lists of group objects or a dict; normalize to list
+                    entries = mj if isinstance(mj, list) else ([mj] if isinstance(mj, dict) else [])
+                    found_image_ids = set()
+                    for entry in entries:
+                        if not isinstance(entry, dict):
+                            continue
+                        gid = entry.get('groupId') or entry.get('imageGroupId') or entry.get('groupID')
+                        # if group id matches any of the capture's image group ids, collect imageIds
+                        if (not image_group_ids) or (gid and gid in image_group_ids):
+                            images = entry.get('images') or entry.get('imageList') or []
+                            if isinstance(images, list):
+                                for img_obj in images:
+                                    if isinstance(img_obj, dict):
+                                        iid = img_obj.get('imageId') or img_obj.get('id')
+                                        if iid:
+                                            found_image_ids.add(iid)
+                                        # also consider thumbnailUrl fields
+                                        thumb = img_obj.get('thumbnailUrl') or img_obj.get('url') or img_obj.get('src')
+                                        if isinstance(thumb, str):
+                                            m = re.search(r"/thumbnail/([0-9a-fA-F-]+)", thumb)
+                                            if m:
+                                                found_image_ids.add(m.group(1))
+
+                    # for every found image id, record caption and glob files in input_dir that contain it
+                    for iid in found_image_ids:
+                        # capture caption from media JSON (if available)
+                        # find the image object in entries to extract caption
+                        for entry in entries:
+                            if not isinstance(entry, dict):
+                                continue
+                            images = entry.get('images') or entry.get('imageList') or []
+                            if isinstance(images, list):
+                                for img_obj in images:
+                                    if isinstance(img_obj, dict):
+                                        iid2 = img_obj.get('imageId') or img_obj.get('id')
+                                        if iid2 == iid:
+                                            cap = img_obj.get('caption') or img_obj.get('title') or img_obj.get('imageTitle')
+                                            if isinstance(cap, str) and cap.strip():
+                                                media_captions[iid] = cap.strip()
+                        for match in glob.glob(os.path.join(input_dir, f"*{iid}*")):
+                            if match not in expanded:
+                                expanded.append(match)
+
+            # now copy files for all expanded candidates, grouping by stem
+            for src in expanded:
+                base = os.path.basename(src)
+                stem = base.split(".")[0]
+                for match in glob.glob(os.path.join(input_dir, stem + "*")):
+                    try:
+                        dst = os.path.join(images_dir, os.path.basename(match))
+                        if os.path.abspath(match) == os.path.abspath(dst):
+                            continue
+                        shutil.copy2(match, dst)
+                        copied_map[os.path.basename(match)] = os.path.join("images", os.path.basename(match))
+                        # if match references a known media imageId, record caption association so we can render it in the article
+                        for iid, cap in media_captions.items():
+                            if iid in os.path.basename(match):
+                                # store caption mapping keyed by the relative path
+                                try:
+                                    # use a separate mapping to avoid clobbering copied_map
+                                    if '_captions_map' not in locals():
+                                        _captions_map = {}
+                                except Exception:
+                                    _captions_map = {}
+                                _captions_map[copied_map[os.path.basename(match)]] = cap
+                    except Exception:
+                        continue
+
+        # rewrite markdown image links that reference captured filenames or urls
+        image_list = []
+        if copied_map:
+            # for each original filename, replace occurrences in md
+            for orig, rel in copied_map.items():
+                # plain filename
+                md = md.replace(orig, rel)
+                # URL-encoded variants (common in src attributes)
+                try:
+                    md = md.replace(urllib.parse.quote(orig), rel)
+                except Exception:
+                    pass
+                # if the original was a full URL, replace filename part
+                md = re.sub(r"https?://[^)\s]*" + re.escape(orig), rel, md)
+                # track image files (only common image extensions)
+                lower = orig.lower()
+                if any(lower.endswith(ext) for ext in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg')):
+                    image_list.append(rel)
+                else:
+                    # also consider the rel if it points to an actual image file in copied_map
+                    if any(rel.lower().endswith(ext) for ext in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg')):
+                        image_list.append(rel)
+
+        # If we copied images, append an Images section to the markdown and log the association
+        if image_list:
+            # try to extract captions from any copied .meta.json files next to the images
+            captions = {}
+            for img_rel in list(image_list):
+                img_path = os.path.join(target_dir, img_rel)
+                meta_path = None
+                # look for <name>.meta.json or <name>.json in the images dir
+                base = os.path.basename(img_path)
+                stem = os.path.splitext(base)[0]
+                for cand in (stem + '.meta.json', stem + '.json'):
+                    candp = os.path.join(os.path.dirname(img_path), cand)
+                    if os.path.exists(candp):
+                        meta_path = candp
+                        break
+                if meta_path:
+                    try:
+                        with open(meta_path, 'r', encoding='utf-8') as mf:
+                            mj = json.load(mf)
+                            # common keys for captions/titles
+                            for k in ('caption', 'alt', 'title', 'name'):
+                                if k in mj and isinstance(mj[k], str) and mj[k].strip():
+                                    captions[img_rel] = mj[k].strip()
+                                    break
+                    except Exception:
+                        pass
+                # if a caption was discovered earlier during media JSON parsing, use it (keyed by relative path)
+                try:
+                    if '_captions_map' in locals() and img_rel in _captions_map and img_rel not in captions:
+                        captions[img_rel] = _captions_map[img_rel]
+                except Exception:
+                    pass
+
+            # build Images markdown block
+            imgs_md_lines = ["\n\n## Images\n"]
+            for img in image_list:
+                cap = captions.get(img) or os.path.basename(img)
+                imgs_md_lines.append(f"![{cap}]({img})")
+                imgs_md_lines.append('')
+
+            md = md.rstrip() + "\n" + "\n".join(imgs_md_lines) + "\n"
+
+            # update a top-level images index so we can see associations across run
+            try:
+                images_index_path = os.path.join(out_dir, "_images_index.json")
+                images_index = {}
+                if os.path.exists(images_index_path):
+                    with open(images_index_path, 'r', encoding='utf-8') as ixf:
+                        try:
+                            images_index = json.load(ixf)
+                        except Exception:
+                            images_index = {}
+                # store relative image paths for this article
+                rels = image_list
+                images_index[out_path] = rels
+                with open(images_index_path, 'w', encoding='utf-8') as ixf:
+                    json.dump(images_index, ixf, indent=2, ensure_ascii=False)
+            except Exception:
+                pass
+
+            # append per-target saved images log
+            try:
+                log_path = os.path.join(target_dir, "_saved_images.log")
+                from datetime import timezone
+                with open(log_path, 'a', encoding='utf-8') as lf:
+                    lf.write(f"{datetime.now(timezone.utc).isoformat()}\t{out_path}\t{','.join(image_list)}\n")
+            except Exception:
+                pass
+
+    except Exception:
+        # non-fatal
+        pass
+
     # Before writing, compute a normalized content hash to detect duplicates
     def normalize_for_hash(s: str) -> str:
         # remove ISO8601-like timestamps and common date patterns, collapse whitespace