enhavce search
This commit is contained in:
Ross
+189
-2
@@ -20,7 +20,8 @@ import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
from typing import List, Dict, Any
|
||||
import re
|
||||
from typing import List, Dict, Any, Tuple, Optional
|
||||
|
||||
import yaml
|
||||
import fnmatch
|
||||
@@ -249,7 +250,135 @@ def match_author(authors: List[Dict[str, Any]], q: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def run_search(root: str, qkey: str, qval: str, mode: str = 'exact', targets: List[str] = None, or_queries: List[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
|
||||
_docid_map = {}
|
||||
|
||||
def get_doc_by_id(root: str, identifier: str) -> Tuple[Optional[Dict[str, Any]], Optional[str], Optional[str]]:
|
||||
global _docid_map
|
||||
if not _docid_map:
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
for fn in filenames:
|
||||
if fn.endswith('.md'):
|
||||
path = os.path.join(dirpath, fn)
|
||||
fm, content = read_md_file(path)
|
||||
basename = os.path.splitext(fn)[0]
|
||||
if fm:
|
||||
docid = fm.get('docid')
|
||||
doc_info = (fm, content, path)
|
||||
if docid:
|
||||
_docid_map[docid] = doc_info
|
||||
_docid_map[basename] = doc_info
|
||||
return _docid_map.get(identifier, (None, None, None))
|
||||
|
||||
|
||||
def extract_section_content(content: str, section_type: str) -> str:
|
||||
lines = content.split('\n')
|
||||
section_content = []
|
||||
in_section = False
|
||||
section_level = 0
|
||||
|
||||
if section_type == 'anatomy':
|
||||
pattern = re.compile(r'^#+\s+.*anatomy.*', re.IGNORECASE)
|
||||
elif section_type == 'differential':
|
||||
pattern = re.compile(r'^#+\s+.*differential.*', re.IGNORECASE)
|
||||
else:
|
||||
return ""
|
||||
|
||||
for line in lines:
|
||||
if pattern.match(line):
|
||||
in_section = True
|
||||
section_level = len(line) - len(line.lstrip('#'))
|
||||
continue
|
||||
|
||||
if in_section:
|
||||
if line.startswith('#'):
|
||||
current_level = len(line) - len(line.lstrip('#'))
|
||||
if current_level <= section_level:
|
||||
in_section = False
|
||||
continue
|
||||
section_content.append(line)
|
||||
|
||||
return '\n'.join(section_content)
|
||||
|
||||
|
||||
def find_links_in_text(text: str) -> List[Dict[str, str]]:
|
||||
links = []
|
||||
if not text:
|
||||
return links
|
||||
|
||||
# 1. Standard markdown links [Label](/document/slug/uuid)
|
||||
markdown_pattern = re.compile(r'\[([^\]]+)\]\((/document/[^/]+/([a-f0-9\-]{36}))\)')
|
||||
for m in markdown_pattern.finditer(text):
|
||||
links.append({
|
||||
'title': m.group(1),
|
||||
'docid': m.group(3)
|
||||
})
|
||||
|
||||
# 2. Raw ANATOMY links (e.g. Brain/ANATOMY:uuid or ANATOMY:uuid)
|
||||
anatomy_pattern = re.compile(r'(?:[a-zA-Z0-9\-]+/)?ANATOMY:([a-f0-9\-]{36})', re.IGNORECASE)
|
||||
for m in anatomy_pattern.finditer(text):
|
||||
docid = m.group(1)
|
||||
if not any(l['docid'] == docid for l in links):
|
||||
links.append({
|
||||
'title': 'Anatomy Document',
|
||||
'docid': docid
|
||||
})
|
||||
|
||||
# 3. Raw DDX links (e.g. DDX:uuid)
|
||||
ddx_pattern = re.compile(r'DDX:([a-f0-9\-]{36})', re.IGNORECASE)
|
||||
for m in ddx_pattern.finditer(text):
|
||||
docid = m.group(1)
|
||||
if not any(l['docid'] == docid for l in links):
|
||||
links.append({
|
||||
'title': 'Differential Diagnosis',
|
||||
'docid': docid
|
||||
})
|
||||
|
||||
return links
|
||||
|
||||
|
||||
def check_linked_sections(content: str, root: str = 'docs_md/articles') -> Dict[str, Any]:
|
||||
anatomy_text = extract_section_content(content, 'anatomy')
|
||||
anatomy_exists = bool(re.search(r'(?i)^#+\s+.*anatomy.*', content, re.MULTILINE))
|
||||
|
||||
diff_text = extract_section_content(content, 'differential')
|
||||
diff_exists = bool(re.search(r'(?i)^#+\s+.*differential.*', content, re.MULTILINE))
|
||||
|
||||
raw_anatomy_links = find_links_in_text(anatomy_text)
|
||||
raw_diff_links = find_links_in_text(diff_text)
|
||||
|
||||
anatomy_links = []
|
||||
for link in raw_anatomy_links:
|
||||
docid = link['docid']
|
||||
fm, _, _ = get_doc_by_id(root, docid)
|
||||
title = fm.get('title') or fm.get('pageTitle') if fm else link['title']
|
||||
anatomy_links.append({
|
||||
'title': title,
|
||||
'docid': docid
|
||||
})
|
||||
|
||||
diff_links = []
|
||||
for link in raw_diff_links:
|
||||
docid = link['docid']
|
||||
fm, _, _ = get_doc_by_id(root, docid)
|
||||
title = fm.get('title') or fm.get('pageTitle') if fm else link['title']
|
||||
diff_links.append({
|
||||
'title': title,
|
||||
'docid': docid
|
||||
})
|
||||
|
||||
return {
|
||||
'anatomy': {
|
||||
'exists': anatomy_exists,
|
||||
'links': anatomy_links
|
||||
},
|
||||
'differential': {
|
||||
'exists': diff_exists,
|
||||
'links': diff_links
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def run_search(root: str, qkey: str, qval: str, mode: str = 'exact', targets: List[str] = None, or_queries: List[Dict[str, Any]] = None, expand_links: bool = False) -> List[Dict[str, Any]]:
|
||||
out = []
|
||||
|
||||
# Normalize queries list
|
||||
@@ -456,7 +585,65 @@ def run_search(root: str, qkey: str, qval: str, mode: str = 'exact', targets: Li
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': sorted(list(set(reasons))),
|
||||
'snippet': snippet,
|
||||
'linked_info': check_linked_sections(content, root)
|
||||
})
|
||||
|
||||
if expand_links:
|
||||
seen_docids = {r.get('docid') for r in out if r.get('docid')}
|
||||
seen_paths = {r['path'] for r in out}
|
||||
|
||||
expanded_results = []
|
||||
for r in out:
|
||||
info = r.get('linked_info')
|
||||
if not info:
|
||||
continue
|
||||
|
||||
parent_title = r['title']
|
||||
|
||||
# Anatomy links
|
||||
for link in info['anatomy']['links']:
|
||||
docid = link['docid']
|
||||
if docid and docid not in seen_docids:
|
||||
fm, content, path = get_doc_by_id(root, docid)
|
||||
if fm:
|
||||
linked_fm_title = fm.get('title') or fm.get('pageTitle') or link['title']
|
||||
expanded_results.append({
|
||||
'path': path,
|
||||
'title': linked_fm_title,
|
||||
'docid': docid,
|
||||
'breadcrumbs': fm.get('breadcrumbs'),
|
||||
'authors': fm.get('authors'),
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': [f'Linked (Anatomy of {parent_title})'],
|
||||
'snippet': content[:300] + '...' if content else '',
|
||||
'linked_info': check_linked_sections(content, root)
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
seen_paths.add(path)
|
||||
|
||||
# Differential links
|
||||
for link in info['differential']['links']:
|
||||
docid = link['docid']
|
||||
if docid and docid not in seen_docids:
|
||||
fm, content, path = get_doc_by_id(root, docid)
|
||||
if fm:
|
||||
linked_fm_title = fm.get('title') or fm.get('pageTitle') or link['title']
|
||||
expanded_results.append({
|
||||
'path': path,
|
||||
'title': linked_fm_title,
|
||||
'docid': docid,
|
||||
'breadcrumbs': fm.get('breadcrumbs'),
|
||||
'authors': fm.get('authors'),
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': [f'Linked (Diff Diag of {parent_title})'],
|
||||
'snippet': content[:300] + '...' if content else '',
|
||||
'linked_info': check_linked_sections(content, root)
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
seen_paths.add(path)
|
||||
|
||||
out.extend(expanded_results)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user