update search
This commit is contained in:
Ross
+307
-224
@@ -46,6 +46,164 @@ def read_frontmatter(path: str) -> Dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
def read_md_file(path: str) -> tuple[Dict[str, Any], str]:
|
||||
"""Read frontmatter and content from a markdown file."""
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
except Exception:
|
||||
return {}, ""
|
||||
|
||||
fm = {}
|
||||
content = text
|
||||
if text.startswith('---'):
|
||||
parts = text.split('---', 2)
|
||||
if len(parts) >= 3:
|
||||
fm_text = parts[1]
|
||||
content = parts[2]
|
||||
try:
|
||||
data = yaml.safe_load(fm_text)
|
||||
if isinstance(data, dict):
|
||||
fm = data
|
||||
except Exception:
|
||||
pass
|
||||
return fm, content
|
||||
|
||||
|
||||
import re
|
||||
|
||||
def tokenize(text: str) -> List[str]:
|
||||
"""Tokenize text into lowercase alphanumeric words."""
|
||||
if not text:
|
||||
return []
|
||||
return re.findall(r'\b[a-zA-Z0-9]+\b', text.lower())
|
||||
|
||||
|
||||
def stem_word(word: str) -> str:
|
||||
"""A simple suffix-stripping stemmer for common English and medical terms."""
|
||||
word = word.lower().strip()
|
||||
if len(word) <= 3:
|
||||
return word
|
||||
|
||||
# Standard plural stripping
|
||||
if word.endswith('s') and not word.endswith('ss'):
|
||||
if word.endswith('ies'):
|
||||
word = word[:-3] + 'y'
|
||||
elif word.endswith('es'):
|
||||
if any(word.endswith(x) for x in ['ches', 'shes', 'xes', 'zes']):
|
||||
word = word[:-2]
|
||||
else:
|
||||
word = word[:-1]
|
||||
else:
|
||||
word = word[:-1]
|
||||
|
||||
# Common suffixes
|
||||
suffixes = [
|
||||
('ectomy', 6), ('otomy', 5), ('pathy', 5), ('itis', 4), ('osis', 4),
|
||||
('tion', 4), ('sion', 4), ('ness', 4), ('ment', 4), ('able', 4),
|
||||
('ible', 4), ('ical', 4), ('ing', 3), ('ive', 3), ('est', 3),
|
||||
('ism', 3), ('ity', 3), ('oma', 3), ('ed', 2), ('ly', 2),
|
||||
('al', 2), ('ic', 2), ('er', 2), ('or', 2), ('ia', 2), ('ar', 2), ('y', 1)
|
||||
]
|
||||
|
||||
for suffix, length in suffixes:
|
||||
if word.endswith(suffix) and len(word) - length >= 3:
|
||||
return word[:-length]
|
||||
return word
|
||||
|
||||
|
||||
def stem_text(text: str) -> Set[str]:
|
||||
"""Tokenize and stem all words in a text block."""
|
||||
return {stem_word(w) for w in tokenize(text)}
|
||||
|
||||
|
||||
def match_stemming(query_text: str, target_text: str) -> bool:
|
||||
if not query_text or not target_text:
|
||||
return False
|
||||
q_stems = {stem_word(w) for w in tokenize(query_text)}
|
||||
t_stems = stem_text(target_text)
|
||||
return q_stems.issubset(t_stems)
|
||||
|
||||
|
||||
def edit_distance(s1: str, s2: str) -> int:
|
||||
# Levenshtein distance
|
||||
if len(s1) < len(s2):
|
||||
s1, s2 = s2, s1
|
||||
if len(s2) == 0:
|
||||
return len(s1)
|
||||
|
||||
previous_row = list(range(len(s2) + 1))
|
||||
for i, c1 in enumerate(s1):
|
||||
current_row = [i + 1]
|
||||
for j, c2 in enumerate(s2):
|
||||
insertions = previous_row[j + 1] + 1
|
||||
deletions = current_row[j] + 1
|
||||
substitutions = previous_row[j] + (c1 != c2)
|
||||
current_row.append(min(insertions, deletions, substitutions))
|
||||
previous_row = current_row
|
||||
return previous_row[-1]
|
||||
|
||||
|
||||
def is_fuzzy_match_word(q_word: str, t_word: str) -> bool:
|
||||
if q_word == t_word or q_word in t_word or t_word in q_word:
|
||||
return True
|
||||
len_q, len_t = len(q_word), len(t_word)
|
||||
if abs(len_q - len_t) > 2:
|
||||
return False
|
||||
max_dist = 0
|
||||
if len_q >= 6:
|
||||
max_dist = 2
|
||||
elif len_q >= 4:
|
||||
max_dist = 1
|
||||
return edit_distance(q_word, t_word) <= max_dist
|
||||
|
||||
|
||||
def match_fuzzy(query_text: str, target_text: str) -> bool:
|
||||
if not query_text or not target_text:
|
||||
return False
|
||||
q_words = tokenize(query_text)
|
||||
t_words = set(tokenize(target_text))
|
||||
|
||||
for qw in q_words:
|
||||
found_match = False
|
||||
for tw in t_words:
|
||||
if is_fuzzy_match_word(qw, tw):
|
||||
found_match = True
|
||||
break
|
||||
if not found_match:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_content_snippet(content: str, query: str) -> str:
|
||||
"""Extract a small snippet from content around the first match of query terms."""
|
||||
if not content or not query:
|
||||
return ""
|
||||
q_words = tokenize(query)
|
||||
if not q_words:
|
||||
return content[:150] + "..." if len(content) > 150 else content
|
||||
|
||||
content_lower = content.lower()
|
||||
first_pos = -1
|
||||
for qw in q_words:
|
||||
pos = content_lower.find(qw)
|
||||
if pos != -1:
|
||||
if first_pos == -1 or pos < first_pos:
|
||||
first_pos = pos
|
||||
|
||||
if first_pos == -1:
|
||||
return content[:150] + "..." if len(content) > 150 else content
|
||||
|
||||
start = max(0, first_pos - 60)
|
||||
end = min(len(content), first_pos + 90)
|
||||
snippet = content[start:end]
|
||||
prefix = "..." if start > 0 else ""
|
||||
suffix = "..." if end < len(content) else ""
|
||||
snippet_clean = re.sub(r'\s+', ' ', snippet).strip()
|
||||
return f"{prefix}{snippet_clean}{suffix}"
|
||||
|
||||
|
||||
|
||||
def match_breadcrumbs(bcs: List[str], query: List[str]) -> bool:
|
||||
"""Return True if breadcrumbs list contains the query sequence (in order).
|
||||
|
||||
@@ -91,242 +249,163 @@ def match_author(authors: List[Dict[str, Any]], q: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def run_search(root: str, qkey: str, qval: str) -> List[Dict[str, Any]]:
|
||||
def run_search(root: str, qkey: str, qval: str, mode: str = 'exact', targets: List[str] = None) -> List[Dict[str, Any]]:
|
||||
out = []
|
||||
|
||||
# Helper to test if a string/list/dict matches under the chosen mode
|
||||
def value_matches(query: str, target_val: Any) -> bool:
|
||||
if target_val is None:
|
||||
return False
|
||||
|
||||
if isinstance(target_val, list):
|
||||
for item in target_val:
|
||||
if isinstance(item, dict):
|
||||
for k, v in item.items():
|
||||
if value_matches(query, v):
|
||||
return True
|
||||
else:
|
||||
if value_matches(query, item):
|
||||
return True
|
||||
return False
|
||||
elif isinstance(target_val, dict):
|
||||
for k, v in target_val.items():
|
||||
if value_matches(query, v):
|
||||
return True
|
||||
return False
|
||||
|
||||
s_val = str(target_val).strip()
|
||||
if not s_val:
|
||||
return False
|
||||
|
||||
if mode == 'stemming':
|
||||
return match_stemming(query, s_val)
|
||||
elif mode == 'fuzzy':
|
||||
return match_fuzzy(query, s_val)
|
||||
else:
|
||||
# Exact substring or wildcard match
|
||||
q_lower = query.lower().strip()
|
||||
s_lower = s_val.lower()
|
||||
if '*' in q_lower:
|
||||
return fnmatch.fnmatchcase(s_lower, q_lower)
|
||||
else:
|
||||
return q_lower in s_lower
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
for fn in filenames:
|
||||
if not fn.endswith('.md'):
|
||||
continue
|
||||
path = os.path.join(dirpath, fn)
|
||||
fm = read_frontmatter(path)
|
||||
if not fm:
|
||||
fm, content = read_md_file(path)
|
||||
if not fm and not content:
|
||||
continue
|
||||
|
||||
matched = False
|
||||
# Helper to test non-breadcrumb simple fields using the same semantics
|
||||
def match_field_value(v, qval_local: str) -> bool:
|
||||
if v is None:
|
||||
return False
|
||||
# If v is a list, check each element for a match
|
||||
if isinstance(v, list):
|
||||
for elem in v:
|
||||
s = str(elem)
|
||||
if '*' in qval_local:
|
||||
if fnmatch.fnmatchcase(s.lower(), qval_local.lower()):
|
||||
return True
|
||||
else:
|
||||
if qval_local.strip().lower() in s.lower():
|
||||
return True
|
||||
return False
|
||||
reasons = []
|
||||
snippet = ""
|
||||
|
||||
# Normalise query key. If none or global/all/empty, search all
|
||||
k_lower = (qkey or '').strip().lower()
|
||||
if not k_lower or k_lower in ('global', 'all', 'any'):
|
||||
# Determine which targets to test
|
||||
if targets:
|
||||
target_set = {t.lower() for t in targets}
|
||||
else:
|
||||
s = str(v)
|
||||
if '*' in qval_local:
|
||||
return fnmatch.fnmatchcase(s.lower(), qval_local.lower())
|
||||
else:
|
||||
return qval_local.strip().lower() in s.lower()
|
||||
|
||||
# Helper to evaluate a single clause (key, value) against this file's frontmatter
|
||||
def match_clause(cl_key: str, cl_val: str) -> bool:
|
||||
cl_key = (cl_key or '').strip()
|
||||
cl_val = (cl_val or '').strip()
|
||||
if not cl_key:
|
||||
cl_key = 'global'
|
||||
if cl_key == 'breadcrumbs':
|
||||
query_parts = [p.strip() for p in cl_val.split('>') if p.strip()]
|
||||
bcs = fm.get('breadcrumbs') or []
|
||||
if bcs and isinstance(bcs, list) and bcs and isinstance(bcs[0], dict):
|
||||
bcs_list = [d.get('name') or d.get('slug') or '' for d in bcs]
|
||||
else:
|
||||
bcs_list = bcs
|
||||
return match_breadcrumbs(bcs_list, query_parts)
|
||||
elif cl_key == 'authors':
|
||||
authors = fm.get('authors') or []
|
||||
return match_author(authors, cl_val)
|
||||
elif cl_key == 'pageKeywords':
|
||||
pk = fm.get('pageKeywords') or ''
|
||||
if isinstance(pk, list):
|
||||
tokens = [str(x) for x in pk]
|
||||
else:
|
||||
tokens = [t.strip() for t in str(pk).split(',') if t.strip()]
|
||||
if '*' in cl_val:
|
||||
for t in tokens:
|
||||
if fnmatch.fnmatchcase(t.lower(), cl_val.lower()):
|
||||
return True
|
||||
return False
|
||||
else:
|
||||
for t in tokens:
|
||||
if cl_val.strip().lower() in t.lower():
|
||||
return True
|
||||
return False
|
||||
elif cl_key == 'global':
|
||||
q = cl_val or ''
|
||||
# breadcrumbs
|
||||
target_set = {'title', 'docid', 'breadcrumbs', 'authors', 'keywords', 'category', 'type', 'content'}
|
||||
|
||||
# Single search box mode / global search:
|
||||
# 1. Title
|
||||
if 'title' in target_set:
|
||||
for f in ['title', 'enhancedTitle', 'pageTitle']:
|
||||
if value_matches(qval, fm.get(f)):
|
||||
matched = True
|
||||
reasons.append('Title')
|
||||
break
|
||||
# 2. DocID
|
||||
if 'docid' in target_set:
|
||||
if value_matches(qval, fm.get('docid')):
|
||||
matched = True
|
||||
reasons.append('DocID')
|
||||
# 3. Breadcrumbs
|
||||
if 'breadcrumbs' in target_set:
|
||||
bcs = fm.get('breadcrumbs') or []
|
||||
if isinstance(bcs, list) and bcs and isinstance(bcs[0], dict):
|
||||
bcs_list = [d.get('name') or d.get('slug') or '' for d in bcs]
|
||||
else:
|
||||
bcs_list = bcs
|
||||
if '>' in q:
|
||||
query_parts = [p.strip() for p in q.split('>') if p.strip()]
|
||||
if '>' in qval:
|
||||
query_parts = [p.strip() for p in qval.split('>') if p.strip()]
|
||||
if match_breadcrumbs(bcs_list, query_parts):
|
||||
return True
|
||||
else:
|
||||
for b in bcs_list:
|
||||
try:
|
||||
if q.strip().lower() in str(b).lower():
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
# authors
|
||||
if match_author(fm.get('authors') or [], q):
|
||||
return True
|
||||
# pageKeywords
|
||||
pk = fm.get('pageKeywords') or ''
|
||||
if isinstance(pk, list):
|
||||
tokens = [str(x) for x in pk]
|
||||
else:
|
||||
tokens = [t.strip() for t in str(pk).split(',') if t.strip()]
|
||||
if '*' in q:
|
||||
for t in tokens:
|
||||
if fnmatch.fnmatchcase(t.lower(), q.lower()):
|
||||
return True
|
||||
else:
|
||||
for t in tokens:
|
||||
if q.strip().lower() in t.lower():
|
||||
return True
|
||||
# other fields
|
||||
for field in ['category', 'title', 'enhancedTitle', 'type', 'docid']:
|
||||
if match_field_value(fm.get(field), q):
|
||||
return True
|
||||
return False
|
||||
else:
|
||||
return match_field_value(fm.get(cl_key), cl_val)
|
||||
|
||||
if qkey == 'breadcrumbs':
|
||||
# qval format: A>B>C
|
||||
query_parts = [p.strip() for p in qval.split('>') if p.strip()]
|
||||
bcs = fm.get('breadcrumbs') or []
|
||||
# support both list-of-strings and list-of-dicts
|
||||
if bcs and isinstance(bcs, list) and bcs and isinstance(bcs[0], dict):
|
||||
bcs_list = [d.get('name') or d.get('slug') or '' for d in bcs]
|
||||
else:
|
||||
bcs_list = bcs
|
||||
matched = match_breadcrumbs(bcs_list, query_parts)
|
||||
elif qkey == 'or':
|
||||
# qval is a set of clauses separated by '||', '|', or ';' where each clause is key:val
|
||||
# Example: "title:stroke|authors:Smith|global:keyword"
|
||||
sep_norm = (qval or '').replace('||', '|').replace(';', '|')
|
||||
clauses = [c.strip() for c in sep_norm.split('|') if c.strip()]
|
||||
for cl in clauses:
|
||||
if ':' in cl:
|
||||
ck, cv = cl.split(':', 1)
|
||||
else:
|
||||
ck, cv = 'global', cl
|
||||
try:
|
||||
if match_clause(ck, cv):
|
||||
matched = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
elif qkey == 'global':
|
||||
q = qval or ''
|
||||
# breadcrumbs: if query contains '>' treat as sequence, else substring match
|
||||
bcs = fm.get('breadcrumbs') or []
|
||||
if isinstance(bcs, list) and bcs and isinstance(bcs[0], dict):
|
||||
bcs_list = [d.get('name') or d.get('slug') or '' for d in bcs]
|
||||
else:
|
||||
bcs_list = bcs
|
||||
if '>' in q:
|
||||
query_parts = [p.strip() for p in q.split('>') if p.strip()]
|
||||
if match_breadcrumbs(bcs_list, query_parts):
|
||||
reasons.append('Breadcrumbs')
|
||||
else:
|
||||
if value_matches(qval, bcs_list):
|
||||
matched = True
|
||||
reasons.append('Breadcrumbs')
|
||||
# 4. Authors
|
||||
if 'authors' in target_set:
|
||||
if value_matches(qval, fm.get('authors')):
|
||||
matched = True
|
||||
else:
|
||||
# substring match on breadcrumb names
|
||||
for b in bcs_list:
|
||||
try:
|
||||
if q.strip().lower() in str(b).lower():
|
||||
matched = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# authors
|
||||
if not matched:
|
||||
authors = fm.get('authors') or []
|
||||
if match_author(authors, q):
|
||||
reasons.append('Authors')
|
||||
# 5. PageKeywords
|
||||
if 'keywords' in target_set or 'pagekeywords' in target_set:
|
||||
if value_matches(qval, fm.get('pageKeywords')):
|
||||
matched = True
|
||||
|
||||
# pageKeywords
|
||||
if not matched:
|
||||
pk = fm.get('pageKeywords') or ''
|
||||
tokens: List[str]
|
||||
if isinstance(pk, list):
|
||||
tokens = [str(x) for x in pk]
|
||||
else:
|
||||
tokens = [t.strip() for t in str(pk).split(',') if t.strip()]
|
||||
if '*' in q:
|
||||
for t in tokens:
|
||||
if fnmatch.fnmatchcase(t.lower(), q.lower()):
|
||||
matched = True
|
||||
break
|
||||
else:
|
||||
for t in tokens:
|
||||
if q.strip().lower() in t.lower():
|
||||
matched = True
|
||||
break
|
||||
|
||||
# other simple fields
|
||||
if not matched:
|
||||
for field in ['category', 'title', 'enhancedTitle', 'type', 'docid']:
|
||||
if match_field_value(fm.get(field), q):
|
||||
matched = True
|
||||
break
|
||||
elif qkey == 'authors':
|
||||
authors = fm.get('authors') or []
|
||||
matched = match_author(authors, qval)
|
||||
elif qkey == 'pageKeywords':
|
||||
pk = fm.get('pageKeywords') or ''
|
||||
# pageKeywords often is a comma-separated string; split into tokens
|
||||
tokens: List[str]
|
||||
if isinstance(pk, list):
|
||||
tokens = [str(x) for x in pk]
|
||||
else:
|
||||
tokens = [t.strip() for t in str(pk).split(',') if t.strip()]
|
||||
matched = False
|
||||
if '*' in qval:
|
||||
for t in tokens:
|
||||
if fnmatch.fnmatchcase(t.lower(), qval.lower()):
|
||||
matched = True
|
||||
break
|
||||
else:
|
||||
for t in tokens:
|
||||
if qval.strip().lower() in t.lower():
|
||||
matched = True
|
||||
break
|
||||
reasons.append('Keywords')
|
||||
# 6. Category
|
||||
if 'category' in target_set:
|
||||
if value_matches(qval, fm.get('category')):
|
||||
matched = True
|
||||
reasons.append('Category')
|
||||
# 7. Type
|
||||
if 'type' in target_set:
|
||||
if value_matches(qval, fm.get('type')):
|
||||
matched = True
|
||||
reasons.append('Type')
|
||||
# 8. Content
|
||||
if 'content' in target_set:
|
||||
if value_matches(qval, content):
|
||||
matched = True
|
||||
reasons.append('Content')
|
||||
snippet = get_content_snippet(content, qval)
|
||||
else:
|
||||
v = fm.get(qkey)
|
||||
if v is None:
|
||||
matched = False
|
||||
else:
|
||||
# If v is a list, check each element for a match
|
||||
if isinstance(v, list):
|
||||
matched = False
|
||||
for elem in v:
|
||||
s = str(elem)
|
||||
if '*' in qval:
|
||||
if fnmatch.fnmatchcase(s.lower(), qval.lower()):
|
||||
matched = True
|
||||
break
|
||||
else:
|
||||
if qval.strip().lower() in s.lower():
|
||||
matched = True
|
||||
break
|
||||
# Specific key search
|
||||
if k_lower == 'breadcrumbs':
|
||||
bcs = fm.get('breadcrumbs') or []
|
||||
if bcs and isinstance(bcs, list) and bcs and isinstance(bcs[0], dict):
|
||||
bcs_list = [d.get('name') or d.get('slug') or '' for d in bcs]
|
||||
else:
|
||||
s = str(v)
|
||||
if '*' in qval:
|
||||
matched = fnmatch.fnmatchcase(s.lower(), qval.lower())
|
||||
else:
|
||||
matched = qval.strip().lower() in s.lower()
|
||||
|
||||
bcs_list = bcs
|
||||
if '>' in qval:
|
||||
query_parts = [p.strip() for p in qval.split('>') if p.strip()]
|
||||
matched = match_breadcrumbs(bcs_list, query_parts)
|
||||
else:
|
||||
matched = value_matches(qval, bcs_list)
|
||||
if matched:
|
||||
reasons.append('Breadcrumbs')
|
||||
elif k_lower == 'authors':
|
||||
matched = value_matches(qval, fm.get('authors'))
|
||||
if matched:
|
||||
reasons.append('Authors')
|
||||
elif k_lower == 'pagekeywords':
|
||||
matched = value_matches(qval, fm.get('pageKeywords'))
|
||||
if matched:
|
||||
reasons.append('Keywords')
|
||||
elif k_lower == 'content':
|
||||
matched = value_matches(qval, content)
|
||||
if matched:
|
||||
reasons.append('Content')
|
||||
snippet = get_content_snippet(content, qval)
|
||||
elif k_lower in ('title', 'enhancedtitle', 'pagetitle'):
|
||||
for f in ['title', 'enhancedTitle', 'pageTitle']:
|
||||
if value_matches(qval, fm.get(f)):
|
||||
matched = True
|
||||
reasons.append('Title')
|
||||
break
|
||||
else:
|
||||
matched = value_matches(qval, fm.get(qkey))
|
||||
if matched:
|
||||
reasons.append(qkey.capitalize())
|
||||
|
||||
if matched:
|
||||
out.append({
|
||||
'path': path,
|
||||
@@ -335,6 +414,8 @@ def run_search(root: str, qkey: str, qval: str) -> List[Dict[str, Any]]:
|
||||
'breadcrumbs': fm.get('breadcrumbs'),
|
||||
'authors': fm.get('authors'),
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': sorted(list(set(reasons))),
|
||||
'snippet': snippet,
|
||||
})
|
||||
return out
|
||||
|
||||
@@ -354,21 +435,23 @@ def write_output(results: List[Dict[str, Any]], out_path: str, fmt: str = 'json'
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--root', default='docs_md/articles', help='Root dir with extracted markdown')
|
||||
p.add_argument('--query', required=True, help='Query in the form key:value where key is breadcrumbs/authors/pageKeywords/...')
|
||||
p.add_argument('--query', required=True, help='Query. Can be key:value or a raw search string')
|
||||
p.add_argument('--mode', choices=['exact', 'fuzzy', 'stemming'], default='exact', help='Search matching mode')
|
||||
p.add_argument('--out', default='results.json', help='Output file')
|
||||
p.add_argument('--format', choices=['json','csv'], default='json')
|
||||
p.add_argument('--copy-to', help='Optional: copy matched markdown files to this dir')
|
||||
p.add_argument("--copy-to-clear", default=True, action="store_true", help="Overwrite files when copying to --copy-to")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
if ':' not in args.query:
|
||||
print('Query must be key:value')
|
||||
return 2
|
||||
key, val = args.query.split(':', 1)
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if ':' in args.query:
|
||||
key, val = args.query.split(':', 1)
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
else:
|
||||
key = 'global'
|
||||
val = args.query.strip()
|
||||
|
||||
results = run_search(args.root, key, val)
|
||||
results = run_search(args.root, key, val, args.mode)
|
||||
write_output(results, args.out, args.format)
|
||||
print(f'Found {len(results)} matches. Wrote {args.out}')
|
||||
|
||||
|
||||
Reference in New Issue
Block a user