#!/usr/bin/env python3
"""NiceGUI web UI that exposes the functions in tools/search_md.py.
Run this from the repository root (so paths like `docs_md/articles` resolve):
python tools/search_md_gui.py
The app presents a small form to run the same searches as `tools/search_md.py` and
renders results as a markdown table with links to local files and (if docid exists)
an example link to the app.statdx document URL.
"""
from __future__ import annotations
import os
import json
import importlib.util
from typing import List, Dict, Any
import anyio
import asyncio
from nicegui import ui, app
import re
_current_dir = os.path.dirname(os.path.abspath(__file__))
_articles_dir = os.path.abspath(os.path.join(_current_dir, '..', 'docs_md', 'articles'))
_images_dir = os.path.join(_articles_dir, 'images')
_img_dir = os.path.join(_articles_dir, 'img')
app.add_static_files('/images', _images_dir)
app.add_static_files('/document/images', _images_dir)
app.add_static_files('/img', _img_dir)
app.add_static_files('/document/img', _img_dir)
import shutil
import subprocess
def load_search_md_module() -> object:
"""Load tools/search_md.py as a module without requiring tools/ to be a package."""
path = os.path.join(os.path.dirname(__file__), 'search_md.py')
spec = importlib.util.spec_from_file_location('search_md', path)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
spec.loader.exec_module(module) # type: ignore[attr-defined]
return module
search_md = load_search_md_module()
def load_document_to_markdown_module() -> object:
"""Load scrapers/document_to_markdown.py as a module without requiring scrapers/ to be a package."""
path = os.path.join(os.path.dirname(__file__), '..', 'scrapers', 'document_to_markdown.py')
path = os.path.abspath(path)
spec = importlib.util.spec_from_file_location('document_to_markdown', path)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
spec.loader.exec_module(module) # type: ignore[attr-defined]
return module
document_to_markdown = load_document_to_markdown_module()
@ui.page('/')
def search_page() -> None: # build UI
# Load fonts & modern dark mode stylesheet
ui.add_head_html('''
''')
with ui.column().classes('w-full max-w-7xl mx-auto p-6 gap-6'):
# Header / Title Block
with ui.row().classes('items-center gap-4 w-full justify-between pb-4 border-b border-slate-800'):
with ui.column().classes('gap-1'):
ui.html('
''')
table.add_slot('body-cell-reasons', '''
{{ r }}
''')
table.add_slot('body-cell-anatomy', '''
No Anatomy ❌
Anatomy ({{ props.row.linked_info.anatomy.links.length }}) 🔍
Anatomy (0)
''')
table.add_slot('body-cell-differential', '''
No Diff Diag ❌
Diff Diag ({{ props.row.linked_info.differential.links.length }}) 🔍
Diff Diag (0)
''')
table.add_slot('body-cell-path', '''
{{ props.row.path.split('/').pop() }}
''')
table.on('expand_row_links', lambda msg: asyncio.create_task(expand_row_links_handler(msg.args)))
table.on('show_links_modal', lambda msg: show_links_modal_handler(msg.args))
async def expand_row_links_handler(path: str):
with table.client:
row = None
for r in table.rows:
if r['path'] == path:
row = r
break
if not row:
return
info = row.get('linked_info')
if not info:
ui.notify('No link metadata found for this article', color='warning')
return
docids = []
origins = {}
for link in info['anatomy']['links']:
docids.append(link['docid'])
origins[link['docid']] = f"Linked (Anatomy of {row['title']})"
for link in info['differential']['links']:
docids.append(link['docid'])
origins[link['docid']] = f"Linked (Diff Diag of {row['title']})"
if not docids:
ui.notify('No linked articles in Anatomy or Diff Diagnosis sections', color='info')
return
status.set_text('Expanding linked articles...')
try:
resolved = []
seen_paths = {r['path'] for r in table.rows}
seen_docids = {r.get('docid') for r in table.rows if r.get('docid')}
def fetch_linked():
res = []
for docid in docids:
if docid and docid not in seen_docids:
fm, content, doc_path = search_md.get_doc_by_id('docs_md/articles', docid)
if fm:
linked_fm_title = fm.get('title') or fm.get('pageTitle') or fm.get('docid')
res.append({
'path': doc_path,
'title': linked_fm_title,
'docid': docid,
'breadcrumbs': fm.get('breadcrumbs'),
'authors': fm.get('authors'),
'pageKeywords': fm.get('pageKeywords'),
'reasons': [origins[docid]],
'snippet': content[:300] + '...' if content else '',
'linked_info': search_md.check_linked_sections(content, root_input.value)
})
seen_docids.add(docid)
seen_paths.add(doc_path)
return res
new_rows = await anyio.to_thread.run_sync(fetch_linked)
if new_rows:
table.rows.extend(new_rows)
table.rows = list(table.rows)
ui.notify(f'Added {len(new_rows)} linked articles to results', color='positive')
status.set_text(f'Added {len(new_rows)} linked articles. Total rows: {len(table.rows)}')
else:
ui.notify('All linked articles are already in the table', color='info')
status.set_text(f'All linked articles are already in the table. Total rows: {len(table.rows)}')
except Exception as e:
ui.notify(f'Failed to expand links: {e}', color='negative')
status.set_text(f'Expand failed: {e}')
def show_links_modal_handler(args):
with table.client:
path = args.get('path')
section_type = args.get('type')
row = None
for r in table.rows:
if r['path'] == path:
row = r
break
if not row:
return
info = row.get('linked_info', {})
sec_info = info.get(section_type, {})
links = sec_info.get('links', [])
if not links:
ui.notify('No links available to display', color='warning')
return
section_title = 'Anatomy' if section_type == 'anatomy' else 'Differential Diagnosis'
with ui.dialog() as dialog, ui.card().classes('w-[550px] p-6 bg-slate-900 text-white border border-slate-700 rounded-xl gap-4'):
ui.label(f'Linked {section_title}').classes('text-2xl font-bold text-blue-400')
ui.label(f'Source: {row["title"]}').classes('text-slate-400 text-sm')
with ui.column().classes('w-full gap-2 mt-2 max-h-[300px] overflow-y-auto'):
for link in links:
docid = link['docid']
linked_title = link['title']
already_added = any(r.get('docid') == docid for r in table.rows if r.get('docid'))
with ui.row().classes('w-full items-center justify-between p-2 rounded bg-slate-800 border border-slate-700 hover:border-blue-500 transition-all'):
with ui.row().classes('items-center gap-2 flex-grow'):
ui.icon('link', color='primary').classes('text-sm')
ui.link(linked_title, f'/document/{docid}', new_tab=True).classes('text-blue-300 hover:text-blue-200 font-semibold text-sm')
if already_added:
ui.label('Added').classes('text-green-400 text-xs font-semibold px-2 py-1 rounded bg-green-950/40 border border-green-800')
else:
def make_add_handler(d_id=docid, title=linked_title):
return lambda: asyncio.create_task(add_single_doc(d_id, title, row['title'], section_title, dialog))
ui.button(icon='add', on_click=make_add_handler()).classes('text-xs text-blue-400 hover:bg-blue-900/20').props('flat round dense')
with ui.row().classes('w-full justify-between items-center mt-4 pt-4 border-t border-slate-800'):
to_add = [l for l in links if not any(r.get('docid') == l['docid'] for r in table.rows if r.get('docid'))]
if to_add:
def add_all_handler():
return asyncio.create_task(add_all_docs(links, row['title'], section_title, dialog))
ui.button(f'Add All ({len(to_add)})', icon='playlist_add', on_click=add_all_handler, color='primary').classes('text-white font-semibold')
else:
ui.button('All Added', icon='check', color='positive').props('disabled').classes('text-slate-400')
ui.button('Close', on_click=dialog.close).props('outline').classes('text-slate-300 border-slate-700')
dialog.open()
async def add_single_doc(docid: str, title: str, parent_title: str, section_title: str, dialog: ui.dialog):
with table.client:
status.set_text(f'Adding {title}...')
try:
def fetch():
fm, content, doc_path = search_md.get_doc_by_id('docs_md/articles', docid)
if fm:
return {
'path': doc_path,
'title': fm.get('title') or fm.get('pageTitle') or title,
'docid': docid,
'breadcrumbs': fm.get('breadcrumbs'),
'authors': fm.get('authors'),
'pageKeywords': fm.get('pageKeywords'),
'reasons': [f'Linked ({section_title} of {parent_title})'],
'linked_info': search_md.check_linked_sections(content, root_input.value)
}
return None
res = await anyio.to_thread.run_sync(fetch)
if res:
table.rows.append(res)
table.rows = list(table.rows)
ui.notify(f'Added "{title}" to search results', color='positive')
status.set_text(f'Added "{title}". Total rows: {len(table.rows)}')
dialog.close()
else:
ui.notify(f'Could not load document for {title}', color='warning')
except Exception as e:
ui.notify(f'Error adding document: {e}', color='negative')
async def add_all_docs(links: List[Dict[str, str]], parent_title: str, section_title: str, dialog: ui.dialog):
with table.client:
status.set_text('Adding all linked documents...')
try:
seen_docids = {r.get('docid') for r in table.rows if r.get('docid')}
def fetch_all():
res = []
for link in links:
docid = link['docid']
if docid and docid not in seen_docids:
fm, content, doc_path = search_md.get_doc_by_id('docs_md/articles', docid)
if fm:
res.append({
'path': doc_path,
'title': fm.get('title') or fm.get('pageTitle') or link['title'],
'docid': docid,
'breadcrumbs': fm.get('breadcrumbs'),
'authors': fm.get('authors'),
'pageKeywords': fm.get('pageKeywords'),
'reasons': [f'Linked ({section_title} of {parent_title})'],
'linked_info': search_md.check_linked_sections(content, root_input.value)
})
seen_docids.add(docid)
return res
new_rows = await anyio.to_thread.run_sync(fetch_all)
if new_rows:
table.rows.extend(new_rows)
table.rows = list(table.rows)
ui.notify(f'Added {len(new_rows)} linked articles to results', color='positive')
status.set_text(f'Added {len(new_rows)} linked articles. Total rows: {len(table.rows)}')
dialog.close()
else:
ui.notify('No new articles were added', color='info')
except Exception as e:
ui.notify(f'Error adding documents: {e}', color='negative')
async def run_search_handler():
has_query = False
for q in queries:
if q['qval'].strip():
has_query = True
break
if not has_query:
ui.notify('Please enter a search query', color='warning')
return
status.set_text('Running search...')
search_btn.set_enabled(False)
try:
or_queries_param = []
for q in queries:
if q['qval'].strip():
or_queries_param.append({
'qkey': key_select.value,
'qval': q['qval'],
'mode': q['mode'],
'targets': [t.lower() for t in q['targets']]
})
results = await anyio.to_thread.run_sync(
search_md.run_search,
root_input.value,
'global',
'',
'exact',
None,
or_queries_param,
expand_chk.value
)
table.rows = results
table.selected.clear()
# write output if requested
if out_input.value:
try:
await anyio.to_thread.run_sync(search_md.write_output, results, out_input.value, fmt_select.value)
status.set_text(f'Found {len(results)} matches. Wrote {out_input.value}')
except Exception as e:
status.set_text(f'Found {len(results)} matches. Failed to write output: {e}')
else:
status.set_text(f'Found {len(results)} matches (not written).')
except Exception as e:
status.set_text(f'Search failed: {e}')
ui.notify(f'Search error: {e}', color='negative')
finally:
search_btn.set_enabled(True)
async def copy_selected():
if not table.selected:
ui.notify('No files selected', color='warning')
return
dest = copy_input.value.strip() or 'out'
status.set_text(f'Copying {len(table.selected)} files to {dest}...')
try:
os.makedirs(dest, exist_ok=True)
if copy_clear.value:
def clear_dst():
for name in os.listdir(dest):
path = os.path.join(dest, name)
try:
if os.path.isfile(path) or os.path.islink(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
except Exception:
pass
await anyio.to_thread.run_sync(clear_dst)
def copy_files():
for r in table.selected:
try:
dst = os.path.join(dest, os.path.basename(r['path']))
with open(r['path'], 'rb') as srcf, open(dst, 'wb') as dstf:
dstf.write(srcf.read())
except Exception:
pass
await anyio.to_thread.run_sync(copy_files)
status.set_text(status.text + f' Copied {len(table.selected)} files to {dest}')
ui.notify('Files copied successfully', color='positive')
except Exception as e:
status.set_text(status.text + f' Copy failed: {e}')
ui.notify(f'Copy failed: {e}', color='negative')
async def export_selected():
if not table.selected:
ui.notify('No records selected', color='warning')
return
outp = out_input.value or 'results.json'
status.set_text(f'Exporting {len(table.selected)} records to {outp}...')
try:
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, fmt_select.value)
status.set_text(f'Exported {len(table.selected)} records to {outp}.')
ui.notify('Export successful', color='positive')
except Exception as e:
status.set_text(f'Export failed: {e}')
ui.notify(f'Export failed: {e}', color='negative')
def open_output_dir() -> None:
dest = copy_input.value.strip() if copy_input.value else ''
if dest:
path = os.path.abspath(dest)
else:
outp = out_input.value or 'results.json'
dirname = os.path.dirname(outp)
path = os.path.abspath(dirname) if dirname else os.getcwd()
status.set_text(f'Opening: {path} ...')
tried = []
try:
if os.name == 'posix':
opener = shutil.which('xdg-open') or shutil.which('gio') or shutil.which('gvfs-open')
if opener:
tried.append(opener)
try:
subprocess.Popen([opener, path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
status.set_text(status.text + f' Opened {path} with {opener}')
return
except Exception as e:
status.set_text(status.text + f' Failed to open with {opener}: {e}')
except Exception as e:
status.set_text(status.text + f' Server-side open check failed: {e}')
status.set_text(status.text + ' Falling back to browser open (may be blocked).')
ui.notify(f'Path: {path}', color='primary')
ui.run_javascript("window.open(" + json.dumps('file://' + path) + ")")
# Document -> Markdown conversion section
ui.markdown('---')
ui.markdown('# Convert captured JSON -> Markdown (document_to_markdown)')
with ui.row().style('gap: 12px; align-items: start;'):
dtm_input_dir = ui.input(value='xhr_captured_async', label='Input directory (captures)')
dtm_output_dir = ui.input(value='docs_md', label='Output directory')
dtm_pattern = ui.input(value='*', label='Glob pattern (filename match)')
with ui.row().style('gap: 12px; align-items: center; margin-top: 8px;'):
dtm_overwrite = ui.checkbox('Overwrite existing .md files (--overwrite)', value=False)
dtm_verbose = ui.checkbox('Verbose', value=False)
dtm_clear = ui.checkbox('Clear output dir before processing (--clear)', value=False)
dtm_copy_images = ui.checkbox('Copy annotation images (--copy-annotation-images)', value=True)
dtm_status = ui.label('')
dtm_log = ui.markdown('')
async def run_dtm_handler():
run_conv_button.set_enabled(False)
dtm_status.set_text('Starting conversion...')
argv = []
argv += ['--input-dir', dtm_input_dir.value or 'xhr_captured_async']
argv += ['--output-dir', dtm_output_dir.value or 'docs_md']
if dtm_pattern.value:
argv += ['--pattern', dtm_pattern.value]
if dtm_overwrite.value:
argv += ['--overwrite']
if dtm_verbose.value:
argv += ['--verbose']
if dtm_clear.value:
argv += ['--clear']
# copy-annotation-images default is True in the script; include flag only to force True/False
if dtm_copy_images.value:
# default True; nothing to do (module default preserves True)
pass
else:
# to disable, pass a flag that the script does not have; instead we'll set the attribute on the module
# The document_to_markdown.main() reads args and uses them directly; we can pass --copy-annotation-images if True
# but to disable, we append a fake flag handling by setting module default before running
try:
document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__'] = False
except Exception:
pass
dtm_status.set_text('Running document_to_markdown.main with: ' + ' '.join(argv))
def run_main():
# If the caller set an override for copy-image flag, patch the module default
try:
if document_to_markdown.__dict__.get('__COPY_ANNOTATION_IMAGES_OVERRIDE__') is False:
# monkeypatch the default used in argparse by modifying the function's default after parse
# simpler: modify module level default so code that checks args.copy_annotation_images still behaves
# but since argparse produces args, we can't change that here. Instead, we'll call main() then
# but ensure environment where images are not copied by setting an env var the script checks? It doesn't.
# As a pragmatic approach, run main normally but then remove copied images afterwards if any were copied.
pass
except Exception:
pass
try:
return document_to_markdown.main(argv)
finally:
# cleanup any override marker
if '__COPY_ANNOTATION_IMAGES_OVERRIDE__' in document_to_markdown.__dict__:
del document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__']
try:
code = await anyio.to_thread.run_sync(run_main)
dtm_status.set_text(f'document_to_markdown finished with exit code {code}')
dtm_log.set_content(f'Last run exit code: {code}\n\nArgs used: `{json.dumps(argv)}`')
except Exception as e:
dtm_status.set_text(f'Conversion failed: {e}')
dtm_log.set_content(f'Conversion failed: {e}')
finally:
run_conv_button.set_enabled(True)
run_conv_button = ui.button('Run conversion', on_click=lambda: asyncio.create_task(run_dtm_handler()), color='primary')
def linkify_references(content: str, root: str = 'docs_md/articles') -> str:
def replace_anatomy(match):
raw_text = match.group(0)
uuid = match.group(1)
fm, _, _ = search_md.get_doc_by_id(root, uuid)
if fm:
title = fm.get('title') or fm.get('pageTitle') or uuid
return f'[{title}](/document/{uuid})'
return raw_text
anatomy_pattern = re.compile(r'(?:[a-zA-Z0-9\-]+/)?ANATOMY:([a-f0-9\-]{36})', re.IGNORECASE)
content = anatomy_pattern.sub(replace_anatomy, content)
def replace_ddx(match):
raw_text = match.group(0)
uuid = match.group(1)
fm, _, _ = search_md.get_doc_by_id(root, uuid)
if fm:
title = fm.get('title') or fm.get('pageTitle') or uuid
return f'[{title}](/document/{uuid})'
return raw_text
ddx_pattern = re.compile(r'DDX:([a-f0-9\-]{36})', re.IGNORECASE)
content = ddx_pattern.sub(replace_ddx, content)
def replace_standard_links(match):
label = match.group(1)
uuid = match.group(3)
fm, _, _ = search_md.get_doc_by_id(root, uuid)
if fm:
return f'[{label}](/document/{uuid})'
return match.group(0)
markdown_link_pattern = re.compile(r'\[([^\]]+)\]\(((?:https?://app\.statdx\.com)?/document/[^/]+/([a-f0-9\-]{36}))\)')
content = markdown_link_pattern.sub(replace_standard_links, content)
return content
@ui.page('/document/{identifier}')
def render_doc_page(identifier: str):
ui.add_head_html('''
''')
fm, content, path = search_md.get_doc_by_id('docs_md/articles', identifier)
if fm and content:
content = linkify_references(content, 'docs_md/articles')
if not fm:
with ui.column().classes('w-full items-center justify-center p-12 gap-4'):
ui.icon('warning', size='4rem', color='negative')
ui.label(f'Document "{identifier}" not found.').classes('text-2xl text-red-400 font-bold')
ui.button('Go to Search', on_click=lambda: ui.navigate.to('/')).props('outline')
return
with ui.column().classes('container gap-6'):
ui.button('Back to Explorer', icon='arrow_back', on_click=lambda: ui.navigate.to('/')).classes('self-start text-blue-400 border border-blue-800').props('outline dense')
with ui.column().classes('header-card w-full gap-4'):
bcs = fm.get('breadcrumbs') or []
if bcs:
with ui.row().classes('gap-2 items-center flex-wrap'):
for b in bcs:
ui.label(b).classes('breadcrumb-chip')
title_text = fm.get('title') or fm.get('pageTitle') or "Untitled"
ui.label(title_text).classes('text-4xl font-extrabold text-white leading-tight')
with ui.row().classes('w-full gap-6 mt-2'):
if fm.get('category'):
with ui.column().classes('gap-0.5'):
ui.label('Category').classes('text-slate-400 text-xs uppercase font-semibold')
ui.label(fm.get('category')).classes('text-slate-200 font-medium')
if fm.get('type'):
with ui.column().classes('gap-0.5'):
ui.label('Type').classes('text-slate-400 text-xs uppercase font-semibold')
ui.label(fm.get('type')).classes('text-slate-200 font-medium')
if fm.get('lastUpdated'):
with ui.column().classes('gap-0.5'):
ui.label('Last Updated').classes('text-slate-400 text-xs uppercase font-semibold')
ui.label(fm.get('lastUpdated')).classes('text-slate-200 font-medium')
authors = fm.get('authors') or []
if authors:
with ui.column().classes('gap-1 mt-2'):
ui.label('Authors').classes('text-slate-400 text-xs uppercase font-semibold')
authors_text = ", ".join([a.get('value') if isinstance(a, dict) else str(a) for a in authors])
ui.label(authors_text).classes('text-blue-300 font-medium')
with ui.card().classes('w-full p-8 shadow-xl bg-slate-900/60 border border-slate-800 rounded-xl markdown-content'):
ui.markdown(content)
def main() -> None:
port = int(os.environ.get('PORT', '8081'))
# NiceGUI's ui.run will serve the app; mount the page at /search-md
ui.run(title='search_md GUI', port=port)
if __name__ in {"__main__", "__mp_main__"}:
main()