1364 lines
65 KiB
Python
1364 lines
65 KiB
Python
#!/usr/bin/env python3
|
|
"""NiceGUI web UI that exposes the functions in tools/search_md.py.
|
|
|
|
Run this from the repository root (so paths like `docs_md/articles` resolve):
|
|
|
|
python tools/search_md_gui.py
|
|
|
|
The app presents a small form to run the same searches as `tools/search_md.py` and
|
|
renders results as a markdown table with links to local files and (if docid exists)
|
|
an example link to the app.statdx document URL.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import json
|
|
import importlib.util
|
|
from typing import List, Dict, Any
|
|
|
|
import anyio
|
|
import asyncio
|
|
from nicegui import ui, app
|
|
import re
|
|
|
|
_current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
_articles_dir = os.path.abspath(os.path.join(_current_dir, '..', 'docs_md', 'articles'))
|
|
_images_dir = os.path.join(_articles_dir, 'images')
|
|
_img_dir = os.path.join(_articles_dir, 'img')
|
|
|
|
app.add_static_files('/images', _images_dir)
|
|
app.add_static_files('/document/images', _images_dir)
|
|
app.add_static_files('/img', _img_dir)
|
|
app.add_static_files('/document/img', _img_dir)
|
|
import shutil
|
|
import subprocess
|
|
|
|
|
|
def load_search_md_module() -> object:
|
|
"""Load tools/search_md.py as a module without requiring tools/ to be a package."""
|
|
path = os.path.join(os.path.dirname(__file__), 'search_md.py')
|
|
spec = importlib.util.spec_from_file_location('search_md', path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
assert spec and spec.loader
|
|
spec.loader.exec_module(module) # type: ignore[attr-defined]
|
|
return module
|
|
|
|
|
|
search_md = load_search_md_module()
|
|
|
|
|
|
def load_document_to_markdown_module() -> object:
|
|
"""Load scrapers/document_to_markdown.py as a module without requiring scrapers/ to be a package."""
|
|
path = os.path.join(os.path.dirname(__file__), '..', 'scrapers', 'document_to_markdown.py')
|
|
path = os.path.abspath(path)
|
|
spec = importlib.util.spec_from_file_location('document_to_markdown', path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
assert spec and spec.loader
|
|
spec.loader.exec_module(module) # type: ignore[attr-defined]
|
|
return module
|
|
|
|
|
|
document_to_markdown = load_document_to_markdown_module()
|
|
|
|
|
|
@ui.page('/')
|
|
def search_page() -> None: # build UI
|
|
# Load fonts & modern dark mode stylesheet
|
|
ui.add_head_html('''
|
|
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&display=swap" rel="stylesheet">
|
|
<style>
|
|
body {
|
|
font-family: 'Outfit', sans-serif !important;
|
|
background-color: #0f172a !important;
|
|
color: #f8fafc !important;
|
|
}
|
|
.q-page-container {
|
|
background-color: #0f172a !important;
|
|
}
|
|
.q-card {
|
|
background-color: #1e293b !important;
|
|
color: #f8fafc !important;
|
|
border: 1px solid #334155 !important;
|
|
border-radius: 12px !important;
|
|
}
|
|
.q-table__container {
|
|
background-color: #1e293b !important;
|
|
color: #f8fafc !important;
|
|
border: 1px solid #334155 !important;
|
|
border-radius: 12px !important;
|
|
}
|
|
.q-table {
|
|
background-color: #1e293b !important;
|
|
color: #f8fafc !important;
|
|
}
|
|
.q-table th {
|
|
font-weight: 600 !important;
|
|
color: #94a3b8 !important;
|
|
background-color: #0f172a !important;
|
|
border-bottom: 2px solid #334155 !important;
|
|
}
|
|
.q-table tbody td {
|
|
color: #e2e8f0 !important;
|
|
border-bottom: 1px solid #1e293b !important;
|
|
}
|
|
.q-table__bottom {
|
|
background-color: #0f172a !important;
|
|
color: #94a3b8 !important;
|
|
border-top: 1px solid #334155 !important;
|
|
}
|
|
.q-field__native, .q-field__prefix, .q-field__suffix, .q-field__input {
|
|
color: #f8fafc !important;
|
|
}
|
|
.q-field__label {
|
|
color: #94a3b8 !important;
|
|
}
|
|
.q-field__marginal {
|
|
color: #94a3b8 !important;
|
|
}
|
|
/* Dropdown popups legibility overrides */
|
|
.q-menu {
|
|
background-color: #1e293b !important;
|
|
border: 1px solid #334155 !important;
|
|
color: #f8fafc !important;
|
|
box-shadow: 0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1) !important;
|
|
}
|
|
.q-item {
|
|
color: #e2e8f0 !important;
|
|
}
|
|
.q-item:hover, .q-item--active, .q-item--focused {
|
|
background-color: #334155 !important;
|
|
color: #f8fafc !important;
|
|
}
|
|
.q-item__label {
|
|
color: inherit !important;
|
|
}
|
|
.q-checkbox__bg {
|
|
border: 2px solid #64748b !important;
|
|
}
|
|
.q-item--active .q-checkbox__bg {
|
|
border: 2px solid #3b82f6 !important;
|
|
background: #3b82f6 !important;
|
|
}
|
|
.gradient-header {
|
|
background: linear-gradient(135deg, #3b82f6 0%, #a855f7 100%);
|
|
-webkit-background-clip: text;
|
|
-webkit-text-fill-color: transparent;
|
|
}
|
|
.badge-reason {
|
|
padding: 2px 8px;
|
|
border-radius: 4px;
|
|
font-size: 11px;
|
|
font-weight: 500;
|
|
display: inline-block;
|
|
margin-right: 4px;
|
|
}
|
|
.badge-title {
|
|
background-color: rgba(59, 130, 246, 0.2);
|
|
color: #93c5fd;
|
|
border: 1px solid rgba(59, 130, 246, 0.4);
|
|
}
|
|
.badge-content {
|
|
background-color: rgba(168, 85, 247, 0.2);
|
|
color: #e9d5ff;
|
|
border: 1px solid rgba(168, 85, 247, 0.4);
|
|
}
|
|
.badge-other {
|
|
background-color: rgba(148, 163, 184, 0.2);
|
|
color: #cbd5e1;
|
|
border: 1px solid rgba(148, 163, 184, 0.4);
|
|
}
|
|
.badge-status-exists {
|
|
background-color: rgba(16, 185, 129, 0.15);
|
|
color: #34d399;
|
|
border: 1px solid rgba(16, 185, 129, 0.35);
|
|
padding: 2px 8px;
|
|
border-radius: 4px;
|
|
font-size: 11px;
|
|
font-weight: 500;
|
|
display: inline-block;
|
|
}
|
|
.badge-status-exists.cursor-pointer {
|
|
cursor: pointer;
|
|
}
|
|
.badge-status-exists.cursor-pointer:hover {
|
|
background-color: rgba(16, 185, 129, 0.3) !important;
|
|
border-color: rgba(16, 185, 129, 0.6) !important;
|
|
text-decoration: underline;
|
|
}
|
|
.badge-status-missing {
|
|
background-color: rgba(239, 68, 68, 0.15);
|
|
color: #f87171;
|
|
border: 1px solid rgba(239, 68, 68, 0.35);
|
|
padding: 2px 8px;
|
|
border-radius: 4px;
|
|
font-size: 11px;
|
|
font-weight: 500;
|
|
display: inline-block;
|
|
}
|
|
.badge-status-zero {
|
|
background-color: rgba(59, 130, 246, 0.15);
|
|
color: #93c5fd;
|
|
border: 1px solid rgba(59, 130, 246, 0.35);
|
|
padding: 2px 8px;
|
|
border-radius: 4px;
|
|
font-size: 11px;
|
|
font-weight: 500;
|
|
display: inline-block;
|
|
}
|
|
@keyframes pulse {
|
|
0%, 100% { opacity: 1; }
|
|
50% { opacity: 0.7; }
|
|
}
|
|
.animate-pulse {
|
|
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
|
|
}
|
|
.ring-2 {
|
|
box-shadow: 0 0 0 2px var(--ring-color, #fbbf24), 0 0 12px 2px rgba(251, 191, 36, 0.3);
|
|
}
|
|
.ring-amber-400 { --ring-color: #fbbf24; }
|
|
.ring-offset-2 { outline-offset: 2px; }
|
|
</style>
|
|
''')
|
|
|
|
with ui.column().classes('w-full max-w-7xl mx-auto p-6 gap-6'):
|
|
# Header / Title Block
|
|
with ui.row().classes('items-center gap-4 w-full justify-between pb-4 border-b border-slate-800'):
|
|
with ui.column().classes('gap-1'):
|
|
ui.html('<h1 class="text-4xl font-extrabold gradient-header m-0">STATdx Article Explorer</h1>')
|
|
ui.label('Search extracted markdown files with fuzzy matching, stemming, and interactive selection.').classes('text-slate-400 text-base')
|
|
|
|
with ui.row().classes('gap-2'):
|
|
ui.button('Open Output Dir', icon='folder', on_click=lambda: open_output_dir(), color='secondary').props('outline')
|
|
|
|
# Main Search card
|
|
queries = [
|
|
{
|
|
'qval': '',
|
|
'mode': 'exact',
|
|
'targets': ['Title', 'Content', 'Breadcrumbs', 'Keywords', 'Category', 'Type']
|
|
}
|
|
]
|
|
|
|
with ui.card().classes('w-full p-6 shadow-xl gap-4'):
|
|
query_blocks_container = ui.column().classes('w-full gap-4')
|
|
|
|
def render_query_blocks():
|
|
query_blocks_container.clear()
|
|
with query_blocks_container:
|
|
for idx, q in enumerate(queries):
|
|
with ui.row().classes('w-full items-center gap-4 border border-slate-800 p-4 rounded-lg bg-slate-900/40 relative'):
|
|
label_text = "Query 1 (Primary)" if idx == 0 else f"OR Query {idx + 1}"
|
|
ui.label(label_text).classes('text-blue-400 font-semibold w-36')
|
|
|
|
# Bind value and update on change
|
|
qval_input = ui.input(
|
|
label='Search Query',
|
|
placeholder='Type search terms...',
|
|
value=q['qval'],
|
|
on_change=lambda e, q=q: q.update({'qval': e.value})
|
|
).classes('flex-grow').props('outlined dense').on('keydown.enter', lambda: asyncio.create_task(run_search_handler()))
|
|
|
|
mode_select = ui.select(
|
|
options={'exact': 'Exact Match', 'fuzzy': 'Fuzzy Match', 'stemming': 'Stemming (Related Words)'},
|
|
value=q['mode'],
|
|
label='Match Mode',
|
|
on_change=lambda e, q=q: q.update({'mode': e.value})
|
|
).classes('w-48').props('outlined dense')
|
|
|
|
targets_select = ui.select(
|
|
options={
|
|
'Title': 'Title',
|
|
'Content': 'Content',
|
|
'Breadcrumbs': 'Breadcrumbs',
|
|
'Authors': 'Authors',
|
|
'Keywords': 'Keywords',
|
|
'Category': 'Category',
|
|
'Type': 'Type'
|
|
},
|
|
value=q['targets'],
|
|
label='Search Targets',
|
|
multiple=True,
|
|
on_change=lambda e, q=q: q.update({'targets': e.value})
|
|
).classes('w-72').props('outlined dense use-chips')
|
|
|
|
if idx > 0:
|
|
ui.button(
|
|
icon='delete',
|
|
on_click=lambda _, idx=idx: remove_query_block(idx)
|
|
).classes('text-red-400 hover:text-red-500').props('flat round dense')
|
|
|
|
def add_query_block():
|
|
queries.append({
|
|
'qval': '',
|
|
'mode': 'exact',
|
|
'targets': ['Title', 'Content', 'Breadcrumbs', 'Keywords', 'Category', 'Type']
|
|
})
|
|
render_query_blocks()
|
|
|
|
def remove_query_block(idx: int):
|
|
if 0 < idx < len(queries):
|
|
queries.pop(idx)
|
|
render_query_blocks()
|
|
|
|
# Render initial query blocks
|
|
render_query_blocks()
|
|
|
|
# Persistent Search History Container
|
|
with ui.row().classes('w-full items-center gap-2 mt-2'):
|
|
ui.label('Recent Searches:').classes('text-slate-400 text-xs font-semibold')
|
|
history_container = ui.row().classes('gap-1 items-center flex-wrap')
|
|
clear_history_container = ui.row().classes('items-center')
|
|
|
|
def render_history_chips():
|
|
history_container.clear()
|
|
clear_history_container.clear()
|
|
history = load_history()
|
|
if not history:
|
|
with history_container:
|
|
ui.label('No history yet').classes('text-slate-500 text-xs italic')
|
|
return
|
|
with history_container:
|
|
for idx, item in enumerate(history[:10]):
|
|
def make_click(saved=item['queries']):
|
|
def handler():
|
|
nonlocal queries
|
|
queries.clear()
|
|
for sq in saved:
|
|
queries.append(json.loads(json.dumps(sq)))
|
|
render_query_blocks()
|
|
asyncio.create_task(run_search_handler())
|
|
return handler
|
|
def make_delete(label=item['label']):
|
|
def handler(e):
|
|
# value becomes False when chip is removed via X
|
|
if not e.value:
|
|
h = load_history()
|
|
h = [x for x in h if x['label'] != label]
|
|
save_history(h)
|
|
render_history_chips()
|
|
ui.notify('Removed from history', type='info')
|
|
return handler
|
|
ui.chip(
|
|
item['label'],
|
|
removable=True,
|
|
on_click=make_click(),
|
|
on_value_change=make_delete(),
|
|
).props('dense color="slate-800" text-color="blue-300" icon="history"').classes('text-xs hover:bg-slate-700 cursor-pointer')
|
|
with clear_history_container:
|
|
if history:
|
|
def clear_all_history():
|
|
save_history([])
|
|
render_history_chips()
|
|
ui.notify('Search history cleared', type='info')
|
|
ui.button(icon='delete_sweep', on_click=clear_all_history).props('flat round dense size="xs"').classes('text-slate-500 hover:text-red-400').tooltip('Clear all history')
|
|
|
|
render_history_chips()
|
|
|
|
with ui.row().classes('w-full justify-between items-center gap-4 mt-2'):
|
|
with ui.row().classes('items-center gap-4'):
|
|
ui.button('Add OR Query', icon='add', on_click=add_query_block).classes('text-blue-400 border border-blue-800').props('outline dense')
|
|
expand_chk = ui.checkbox('Expand with Linked Articles', value=False).classes('text-slate-300')
|
|
search_btn = ui.button('Search', icon='search', on_click=lambda: asyncio.create_task(run_search_handler())).classes('bg-blue-600 hover:bg-blue-700 text-white font-semibold py-3 px-8 rounded-lg shadow-md transition-all duration-300')
|
|
|
|
# Advanced Criteria Expansion Box
|
|
with ui.expansion('Advanced Configurations', icon='settings').classes('w-full text-slate-300 border border-slate-800 rounded-lg p-2'):
|
|
with ui.row().classes('w-full gap-4 items-start p-4'):
|
|
root_input = ui.input(value='docs_md/articles', label='Root directory').classes('w-64').props('outlined dense')
|
|
key_select = ui.select(['global', 'breadcrumbs', 'authors', 'pageKeywords', 'category', 'title', 'enhancedTitle', 'type', 'content'], value='global', label='Query key').classes('w-64').props('outlined dense')
|
|
fmt_select = ui.select(['json', 'csv', 'combined_json', 'combined_md'], value='json', label='Output format').classes('w-40').props('outlined dense')
|
|
out_input = ui.input(value='results.json', label='Output file (server-side)').classes('w-64').props('outlined dense')
|
|
copy_input = ui.input(value='out', label='Copy matched files to (optional)').classes('w-64').props('outlined dense')
|
|
copy_clear = ui.checkbox('Clear destination before copy', value=False).classes('self-center text-slate-300')
|
|
|
|
status = ui.label('').classes('text-blue-400 font-semibold text-lg px-2')
|
|
|
|
# Action bar for selected rows
|
|
# Hidden by default, displayed via bind_visibility_from
|
|
action_bar = ui.row().classes('w-full justify-between items-center gap-4 bg-slate-800 p-4 rounded-lg border border-slate-700')
|
|
action_bar_text = ui.label('').classes('text-white font-semibold')
|
|
with action_bar:
|
|
with ui.row().classes('gap-2'):
|
|
ui.button('Copy Selected Files', icon='content_copy', on_click=lambda: asyncio.create_task(copy_selected())).classes('bg-indigo-600 hover:bg-indigo-700 text-white font-medium px-4')
|
|
ui.button('Export Combined JSON', icon='library_books', on_click=lambda: asyncio.create_task(export_combined_selected())).classes('bg-teal-600 hover:bg-teal-700 text-white font-medium px-4')
|
|
ui.button('Export Combined MD', icon='article', on_click=lambda: asyncio.create_task(export_combined_md())).classes('bg-cyan-600 hover:bg-cyan-700 text-white font-medium px-4')
|
|
ui.button('Export Selected', icon='file_download', on_click=lambda: asyncio.create_task(export_selected())).classes('bg-emerald-600 hover:bg-emerald-700 text-white font-medium px-4')
|
|
ui.button('Deselect All', icon='close', on_click=lambda: table.selected.clear()).classes('bg-slate-700 hover:bg-slate-600 text-white font-medium px-4')
|
|
|
|
# Results table filter input
|
|
filter_card = ui.row().classes('w-full items-center gap-4')
|
|
with filter_card:
|
|
table_filter = ui.input(
|
|
placeholder='Filter results table below (live)...'
|
|
).classes('w-full').props('outlined dense clearable')
|
|
table_filter.add_slot('append', '<q-icon name="filter_list" />')
|
|
|
|
# Table definition
|
|
columns = [
|
|
{'name': 'title', 'label': 'Title', 'field': 'title', 'required': True, 'align': 'left', 'sortable': True},
|
|
{'name': 'reasons', 'label': 'Matched In', 'field': 'reasons', 'align': 'left', 'sortable': True},
|
|
{'name': 'anatomy', 'label': 'Anatomy', 'field': 'anatomy', 'align': 'center'},
|
|
{'name': 'differential', 'label': 'Diff Diagnosis', 'field': 'differential', 'align': 'center'},
|
|
{'name': 'docid', 'label': 'DocID', 'field': 'docid', 'align': 'left'},
|
|
{'name': 'path', 'label': 'Path', 'field': 'path', 'align': 'left', 'sortable': True},
|
|
]
|
|
|
|
table = ui.table(
|
|
columns=columns,
|
|
rows=[],
|
|
row_key='path',
|
|
selection='multiple',
|
|
).classes('w-full shadow-lg')
|
|
table.bind_filter_from(table_filter, 'value')
|
|
|
|
# Bind action bar visibility and selection text
|
|
action_bar.bind_visibility_from(table, 'selected', lambda sel: len(sel) > 0)
|
|
action_bar_text.bind_text_from(table, 'selected', lambda sel: f'{len(sel)} files selected')
|
|
|
|
# Add slot overrides to format table cells nicely
|
|
table.add_slot('body-cell-title', '''
|
|
<q-td :props="props">
|
|
<div class="row items-center no-wrap q-gutter-xs">
|
|
<template v-if="props.row.is_cached !== false">
|
|
<a :href="'/document/' + (props.row.docid || props.row.path.split('/').pop().replace('.md', ''))" target="_blank" class="text-blue-400 hover:text-blue-300 font-semibold" style="text-decoration: none;">
|
|
{{ props.row.title }}
|
|
</a>
|
|
<q-btn v-if="props.row.linked_info && ((props.row.linked_info.anatomy && props.row.linked_info.anatomy.links && props.row.linked_info.anatomy.links.length > 0) || (props.row.linked_info.differential && props.row.linked_info.differential.links && props.row.linked_info.differential.links.length > 0))"
|
|
flat round dense size="xs" icon="alt_route" color="primary"
|
|
@click.stop="$parent.$emit('expand_row_links', props.row.path)">
|
|
<q-tooltip>Expand Linked Articles</q-tooltip>
|
|
</q-btn>
|
|
</template>
|
|
<template v-else>
|
|
<span class="text-slate-400 font-semibold line-through q-mr-xs">{{ props.row.title }}</span>
|
|
<span class="badge-status-missing" style="background-color: rgba(239, 68, 68, 0.2); border-color: rgba(239, 68, 68, 0.4); color: #f87171;">Not Cached</span>
|
|
<q-btn flat round dense size="xs" icon="public" color="warning" class="q-ml-xs"
|
|
@click.stop="$parent.$emit('capture_article', { docid: props.row.docid, title: props.row.title })">
|
|
<q-tooltip>Open in Capture Browser</q-tooltip>
|
|
</q-btn>
|
|
</template>
|
|
</div>
|
|
</q-td>
|
|
''')
|
|
table.add_slot('body-cell-reasons', '''
|
|
<q-td :props="props">
|
|
<span v-for="r in props.row.reasons" :key="r"
|
|
class="badge-reason"
|
|
:class="r === 'Title' ? 'badge-title' : r === 'Content' ? 'badge-content' : 'badge-other'">
|
|
{{ r }}
|
|
</span>
|
|
</q-td>
|
|
''')
|
|
table.add_slot('body-cell-anatomy', '''
|
|
<q-td :props="props" class="text-center">
|
|
<span v-if="!props.row.linked_info || !props.row.linked_info.anatomy.exists" class="badge-status-missing">
|
|
No Anatomy ❌
|
|
</span>
|
|
<span v-else-if="props.row.linked_info.anatomy.links.length > 0"
|
|
class="badge-status-exists cursor-pointer"
|
|
@click.stop="$parent.$emit('show_links_modal', { path: props.row.path, type: 'anatomy' })">
|
|
Anatomy ({{ props.row.linked_info.anatomy.links.length }}) 🔍
|
|
</span>
|
|
<span v-else class="badge-status-zero">
|
|
Anatomy (0)
|
|
</span>
|
|
</q-td>
|
|
''')
|
|
table.add_slot('body-cell-differential', '''
|
|
<q-td :props="props" class="text-center">
|
|
<span v-if="!props.row.linked_info || !props.row.linked_info.differential.exists" class="badge-status-missing">
|
|
No Diff Diag ❌
|
|
</span>
|
|
<span v-else-if="props.row.linked_info.differential.links.length > 0"
|
|
class="badge-status-exists cursor-pointer"
|
|
@click.stop="$parent.$emit('show_links_modal', { path: props.row.path, type: 'differential' })">
|
|
Diff Diag ({{ props.row.linked_info.differential.links.length }}) 🔍
|
|
</span>
|
|
<span v-else class="badge-status-zero">
|
|
Diff Diag (0)
|
|
</span>
|
|
</q-td>
|
|
''')
|
|
table.add_slot('body-cell-path', '''
|
|
<q-td :props="props">
|
|
<span class="text-slate-400 text-xs">
|
|
{{ props.row.is_cached !== false ? props.row.path.split('/').pop() : 'N/A (Missing)' }}
|
|
</span>
|
|
</q-td>
|
|
''')
|
|
table.on('expand_row_links', lambda msg: asyncio.create_task(expand_row_links_handler(msg.args)))
|
|
table.on('show_links_modal', lambda msg: show_links_modal_handler(msg.args))
|
|
table.on('capture_article', lambda msg: asyncio.create_task(trigger_capture_handler(msg.args)))
|
|
|
|
async def wait_and_process_capture(docid: str, title: str):
|
|
import glob
|
|
processed = False
|
|
for i in range(30):
|
|
await asyncio.sleep(1.5)
|
|
|
|
# Check if JSON files matching the docid exist in input dir
|
|
json_files = glob.glob(f"xhr_captured_async/*{docid}*.json")
|
|
if json_files and not processed:
|
|
with table.client:
|
|
ui.notify(f"Captured data found for: {title}. Processing to Markdown...", type='info')
|
|
try:
|
|
await anyio.to_thread.run_sync(
|
|
document_to_markdown.main,
|
|
['--input-dir', 'xhr_captured_async', '--output-dir', 'docs_md']
|
|
)
|
|
processed = True
|
|
except Exception as e:
|
|
print(f"Error running document_to_markdown: {e}")
|
|
|
|
# Check if it has been successfully converted/cached
|
|
def check_cache():
|
|
search_md.clear_doc_cache()
|
|
return search_md.get_doc_by_id('docs_md/articles', docid)
|
|
fm, content, doc_path = await anyio.to_thread.run_sync(check_cache)
|
|
|
|
if fm:
|
|
# Successfully processed! Reload row data in table!
|
|
with table.client:
|
|
row_index = -1
|
|
for idx, r in enumerate(table.rows):
|
|
if r.get('docid') == docid:
|
|
row_index = idx
|
|
break
|
|
if row_index != -1:
|
|
linked_fm_title = fm.get('title') or fm.get('pageTitle') or title
|
|
updated_row = {
|
|
'path': doc_path,
|
|
'title': linked_fm_title,
|
|
'docid': docid,
|
|
'breadcrumbs': fm.get('breadcrumbs'),
|
|
'authors': fm.get('authors'),
|
|
'pageKeywords': fm.get('pageKeywords'),
|
|
'reasons': table.rows[row_index].get('reasons', []),
|
|
'snippet': content[:300] + '...' if content else '',
|
|
'linked_info': search_md.check_linked_sections(content, root_input.value),
|
|
'is_cached': True
|
|
}
|
|
table.rows[row_index] = updated_row
|
|
table.rows = list(table.rows)
|
|
ui.notify(f"Successfully cached and loaded: {linked_fm_title}!", type='positive')
|
|
else:
|
|
ui.notify(f"Cached document: {title}", type='positive')
|
|
return
|
|
|
|
# Registry for cross-section callbacks (populated later by conversion section)
|
|
_page_callbacks = {}
|
|
|
|
async def trigger_capture_handler(args: dict):
|
|
docid = args.get('docid')
|
|
title = args.get('title')
|
|
target_url = make_online_url(docid, title)
|
|
|
|
def check_and_send():
|
|
import urllib.request
|
|
import urllib.parse
|
|
try:
|
|
url = f"http://127.0.0.1:8089/open?url=" + urllib.parse.quote(target_url)
|
|
with urllib.request.urlopen(url, timeout=2.0) as resp:
|
|
return resp.status == 200
|
|
except Exception:
|
|
return False
|
|
|
|
def start_capture_poll_task(d_id, t_title):
|
|
asyncio.create_task(wait_and_process_capture(d_id, t_title))
|
|
|
|
with table.client:
|
|
ui.notify(f"Checking capture browser status for: {title}...", type='info')
|
|
ok = await asyncio.to_thread(check_and_send)
|
|
if ok:
|
|
ui.notify(f"Requested capture browser to open: {title}", type='positive')
|
|
start_capture_poll_task(docid, title)
|
|
# Highlight the conversion button to remind user to run it
|
|
cb = _page_callbacks.get('highlight_conversion')
|
|
if cb:
|
|
cb()
|
|
return
|
|
|
|
ui.notify("Capture browser not responding. Cleaning up existing capture processes...", type='warning')
|
|
try:
|
|
# Terminate any existing capture script processes
|
|
subprocess.run(['pkill', '-f', 'capture_passive_playwright_async.py'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
await asyncio.sleep(1.0) # brief pause for os to clean up
|
|
|
|
ui.notify("Launching fresh capture browser...", type='info')
|
|
subprocess.Popen([
|
|
'uv', 'run', 'python', 'scrapers/capture_passive_playwright_async.py',
|
|
'--output-dir', 'xhr_captured_async',
|
|
'--continuous'
|
|
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
|
|
# Wait 5 seconds for the browser context to open and port to listen
|
|
await asyncio.sleep(5.0)
|
|
|
|
ok_again = await asyncio.to_thread(check_and_send)
|
|
if ok_again:
|
|
ui.notify(f"Capture browser launched successfully. Opened: {title}", type='positive')
|
|
start_capture_poll_task(docid, title)
|
|
# Highlight the conversion button to remind user to run it
|
|
cb = _page_callbacks.get('highlight_conversion')
|
|
if cb:
|
|
cb()
|
|
else:
|
|
ui.notify("Capture browser process launched, but command port timed out. Try clicking again in a few seconds.", type='warning')
|
|
except Exception as e:
|
|
ui.notify(f"Failed to start capture browser: {e}", type='negative')
|
|
|
|
async def expand_row_links_handler(path: str):
|
|
with table.client:
|
|
row = None
|
|
for r in table.rows:
|
|
if r['path'] == path:
|
|
row = r
|
|
break
|
|
if not row:
|
|
return
|
|
|
|
info = row.get('linked_info')
|
|
if not info:
|
|
ui.notify('No link metadata found for this article', color='warning')
|
|
return
|
|
|
|
docids = []
|
|
origins = {}
|
|
for link in info['anatomy']['links']:
|
|
docids.append(link['docid'])
|
|
origins[link['docid']] = f"Linked (Anatomy of {row['title']})"
|
|
for link in info['differential']['links']:
|
|
docids.append(link['docid'])
|
|
origins[link['docid']] = f"Linked (Diff Diag of {row['title']})"
|
|
|
|
if not docids:
|
|
ui.notify('No linked articles in Anatomy or Diff Diagnosis sections', color='info')
|
|
return
|
|
|
|
status.set_text('Expanding linked articles...')
|
|
|
|
try:
|
|
resolved = []
|
|
seen_paths = {r['path'] for r in table.rows}
|
|
seen_docids = {r.get('docid') for r in table.rows if r.get('docid')}
|
|
|
|
def fetch_linked():
|
|
res = []
|
|
for docid in docids:
|
|
if docid and docid not in seen_docids:
|
|
fm, content, doc_path = search_md.get_doc_by_id('docs_md/articles', docid)
|
|
if fm:
|
|
linked_fm_title = fm.get('title') or fm.get('pageTitle') or fm.get('docid')
|
|
res.append({
|
|
'path': doc_path,
|
|
'title': linked_fm_title,
|
|
'docid': docid,
|
|
'breadcrumbs': fm.get('breadcrumbs'),
|
|
'authors': fm.get('authors'),
|
|
'pageKeywords': fm.get('pageKeywords'),
|
|
'reasons': [origins[docid]],
|
|
'snippet': content[:300] + '...' if content else '',
|
|
'linked_info': search_md.check_linked_sections(content, root_input.value),
|
|
'is_cached': True
|
|
})
|
|
seen_docids.add(docid)
|
|
seen_paths.add(doc_path)
|
|
else:
|
|
fallback_title = "Linked Document"
|
|
for l in info['anatomy']['links'] + info['differential']['links']:
|
|
if l['docid'] == docid:
|
|
fallback_title = l['title']
|
|
break
|
|
fake_path = f"missing_{docid}"
|
|
res.append({
|
|
'path': fake_path,
|
|
'title': fallback_title,
|
|
'docid': docid,
|
|
'breadcrumbs': [],
|
|
'authors': [],
|
|
'pageKeywords': [],
|
|
'reasons': [origins[docid]],
|
|
'snippet': '',
|
|
'linked_info': None,
|
|
'is_cached': False
|
|
})
|
|
seen_docids.add(docid)
|
|
seen_paths.add(fake_path)
|
|
return res
|
|
|
|
new_rows = await anyio.to_thread.run_sync(fetch_linked)
|
|
if new_rows:
|
|
table.rows.extend(new_rows)
|
|
table.rows = list(table.rows)
|
|
ui.notify(f'Added {len(new_rows)} linked articles to results', color='positive')
|
|
status.set_text(f'Added {len(new_rows)} linked articles. Total rows: {len(table.rows)}')
|
|
else:
|
|
ui.notify('All linked articles are already in the table', color='info')
|
|
status.set_text(f'All linked articles are already in the table. Total rows: {len(table.rows)}')
|
|
|
|
except Exception as e:
|
|
ui.notify(f'Failed to expand links: {e}', color='negative')
|
|
status.set_text(f'Expand failed: {e}')
|
|
|
|
def show_links_modal_handler(args):
|
|
with table.client:
|
|
path = args.get('path')
|
|
section_type = args.get('type')
|
|
|
|
row = None
|
|
for r in table.rows:
|
|
if r['path'] == path:
|
|
row = r
|
|
break
|
|
if not row:
|
|
return
|
|
|
|
info = row.get('linked_info', {})
|
|
sec_info = info.get(section_type, {})
|
|
links = sec_info.get('links', [])
|
|
if not links:
|
|
ui.notify('No links available to display', color='warning')
|
|
return
|
|
|
|
section_title = 'Anatomy' if section_type == 'anatomy' else 'Differential Diagnosis'
|
|
|
|
with ui.dialog() as dialog, ui.card().classes('w-[550px] p-6 bg-slate-900 text-white border border-slate-700 rounded-xl gap-4'):
|
|
ui.label(f'Linked {section_title}').classes('text-2xl font-bold text-blue-400')
|
|
ui.label(f'Source: {row["title"]}').classes('text-slate-400 text-sm')
|
|
|
|
with ui.column().classes('w-full gap-2 mt-2 max-h-[300px] overflow-y-auto'):
|
|
for link in links:
|
|
docid = link['docid']
|
|
linked_title = link['title']
|
|
already_added = any(r.get('docid') == docid for r in table.rows if r.get('docid'))
|
|
cached = is_doc_cached(docid)
|
|
|
|
with ui.row().classes('w-full items-center justify-between p-2 rounded bg-slate-800 border border-slate-700 hover:border-blue-500 transition-all'):
|
|
if cached:
|
|
with ui.row().classes('items-center gap-2 flex-grow'):
|
|
ui.icon('link', color='primary').classes('text-sm')
|
|
ui.link(linked_title, f'/document/{docid}', new_tab=True).classes('text-blue-300 hover:text-blue-200 font-semibold text-sm')
|
|
else:
|
|
with ui.row().classes('items-center gap-2 flex-grow'):
|
|
ui.icon('link', color='warning').classes('text-sm')
|
|
ui.label(linked_title).classes('text-slate-400 font-semibold line-through text-sm')
|
|
ui.label('Not Cached').classes('text-red-400 text-xs font-semibold px-2 py-0.5 rounded bg-red-950/40 border border-red-800')
|
|
def make_capture_click(d_id=docid, t_title=linked_title):
|
|
return lambda: asyncio.create_task(trigger_capture_handler({'docid': d_id, 'title': t_title}))
|
|
ui.button(icon='public', on_click=make_capture_click()).classes('text-xs text-yellow-400 hover:bg-yellow-900/20').props('flat round dense')
|
|
|
|
if already_added:
|
|
ui.label('Added').classes('text-green-400 text-xs font-semibold px-2 py-1 rounded bg-green-950/40 border border-green-800')
|
|
else:
|
|
def make_add_handler(d_id=docid, title=linked_title):
|
|
return lambda: asyncio.create_task(add_single_doc(d_id, title, row['title'], section_title, dialog))
|
|
ui.button(icon='add', on_click=make_add_handler()).classes('text-xs text-blue-400 hover:bg-blue-900/20').props('flat round dense')
|
|
|
|
with ui.row().classes('w-full justify-between items-center mt-4 pt-4 border-t border-slate-800'):
|
|
to_add = [l for l in links if not any(r.get('docid') == l['docid'] for r in table.rows if r.get('docid'))]
|
|
if to_add:
|
|
def add_all_handler():
|
|
return asyncio.create_task(add_all_docs(links, row['title'], section_title, dialog))
|
|
ui.button(f'Add All ({len(to_add)})', icon='playlist_add', on_click=add_all_handler, color='primary').classes('text-white font-semibold')
|
|
else:
|
|
ui.button('All Added', icon='check', color='positive').props('disabled').classes('text-slate-400')
|
|
|
|
ui.button('Close', on_click=dialog.close).props('outline').classes('text-slate-300 border-slate-700')
|
|
|
|
dialog.open()
|
|
|
|
async def add_single_doc(docid: str, title: str, parent_title: str, section_title: str, dialog: ui.dialog):
|
|
with table.client:
|
|
status.set_text(f'Adding {title}...')
|
|
try:
|
|
def fetch():
|
|
fm, content, doc_path = search_md.get_doc_by_id('docs_md/articles', docid)
|
|
if fm:
|
|
return {
|
|
'path': doc_path,
|
|
'title': fm.get('title') or fm.get('pageTitle') or title,
|
|
'docid': docid,
|
|
'breadcrumbs': fm.get('breadcrumbs'),
|
|
'authors': fm.get('authors'),
|
|
'pageKeywords': fm.get('pageKeywords'),
|
|
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
|
'linked_info': search_md.check_linked_sections(content, root_input.value),
|
|
'is_cached': True
|
|
}
|
|
else:
|
|
return {
|
|
'path': f"missing_{docid}",
|
|
'title': title,
|
|
'docid': docid,
|
|
'breadcrumbs': [],
|
|
'authors': [],
|
|
'pageKeywords': [],
|
|
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
|
'linked_info': None,
|
|
'is_cached': False
|
|
}
|
|
|
|
res = await anyio.to_thread.run_sync(fetch)
|
|
if res:
|
|
table.rows.append(res)
|
|
table.rows = list(table.rows)
|
|
ui.notify(f'Added "{title}" to search results', color='positive')
|
|
status.set_text(f'Added "{title}". Total rows: {len(table.rows)}')
|
|
dialog.close()
|
|
else:
|
|
ui.notify(f'Could not load document for {title}', color='warning')
|
|
except Exception as e:
|
|
ui.notify(f'Error adding document: {e}', color='negative')
|
|
|
|
async def add_all_docs(links: List[Dict[str, str]], parent_title: str, section_title: str, dialog: ui.dialog):
|
|
with table.client:
|
|
status.set_text('Adding all linked documents...')
|
|
try:
|
|
seen_docids = {r.get('docid') for r in table.rows if r.get('docid')}
|
|
|
|
def fetch_all():
|
|
res = []
|
|
for link in links:
|
|
docid = link['docid']
|
|
if docid and docid not in seen_docids:
|
|
fm, content, doc_path = search_md.get_doc_by_id('docs_md/articles', docid)
|
|
if fm:
|
|
res.append({
|
|
'path': doc_path,
|
|
'title': fm.get('title') or fm.get('pageTitle') or link['title'],
|
|
'docid': docid,
|
|
'breadcrumbs': fm.get('breadcrumbs'),
|
|
'authors': fm.get('authors'),
|
|
'pageKeywords': fm.get('pageKeywords'),
|
|
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
|
'linked_info': search_md.check_linked_sections(content, root_input.value),
|
|
'is_cached': True
|
|
})
|
|
else:
|
|
res.append({
|
|
'path': f"missing_{docid}",
|
|
'title': link['title'],
|
|
'docid': docid,
|
|
'breadcrumbs': [],
|
|
'authors': [],
|
|
'pageKeywords': [],
|
|
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
|
'linked_info': None,
|
|
'is_cached': False
|
|
})
|
|
seen_docids.add(docid)
|
|
return res
|
|
|
|
new_rows = await anyio.to_thread.run_sync(fetch_all)
|
|
if new_rows:
|
|
table.rows.extend(new_rows)
|
|
table.rows = list(table.rows)
|
|
ui.notify(f'Added {len(new_rows)} linked articles to results', color='positive')
|
|
status.set_text(f'Added {len(new_rows)} linked articles. Total rows: {len(table.rows)}')
|
|
dialog.close()
|
|
else:
|
|
ui.notify('No new articles were added', color='info')
|
|
except Exception as e:
|
|
ui.notify(f'Error adding documents: {e}', color='negative')
|
|
|
|
async def run_search_handler():
|
|
with table.client:
|
|
has_query = False
|
|
for q in queries:
|
|
if q['qval'].strip():
|
|
has_query = True
|
|
break
|
|
if not has_query:
|
|
ui.notify('Please enter a search query', color='warning')
|
|
return
|
|
|
|
status.set_text('Running search...')
|
|
search_btn.set_enabled(False)
|
|
|
|
# Save query configuration to persistent history
|
|
active_queries = [q for q in queries if q['qval'].strip()]
|
|
if active_queries:
|
|
history_label = " OR ".join([f"'{q['qval']}' ({q['mode']})" for q in active_queries])
|
|
history_item = {
|
|
'label': history_label,
|
|
'queries': json.loads(json.dumps(queries))
|
|
}
|
|
history = load_history()
|
|
history = [h for h in history if h['label'] != history_label]
|
|
history.insert(0, history_item)
|
|
save_history(history[:20])
|
|
render_history_chips()
|
|
|
|
try:
|
|
or_queries_param = []
|
|
for q in queries:
|
|
if q['qval'].strip():
|
|
or_queries_param.append({
|
|
'qkey': key_select.value,
|
|
'qval': q['qval'],
|
|
'mode': q['mode'],
|
|
'targets': [t.lower() for t in q['targets']]
|
|
})
|
|
|
|
results = await anyio.to_thread.run_sync(
|
|
search_md.run_search,
|
|
root_input.value,
|
|
'global',
|
|
'',
|
|
'exact',
|
|
None,
|
|
or_queries_param,
|
|
expand_chk.value
|
|
)
|
|
|
|
table.rows = results
|
|
table.selected.clear()
|
|
table.selected.extend(results)
|
|
table.update()
|
|
|
|
# write output if requested
|
|
if out_input.value:
|
|
try:
|
|
await anyio.to_thread.run_sync(search_md.write_output, results, out_input.value, fmt_select.value)
|
|
status.set_text(f'Found {len(results)} matches. Wrote {out_input.value}')
|
|
except Exception as e:
|
|
status.set_text(f'Found {len(results)} matches. Failed to write output: {e}')
|
|
else:
|
|
status.set_text(f'Found {len(results)} matches (not written).')
|
|
|
|
except Exception as e:
|
|
status.set_text(f'Search failed: {e}')
|
|
ui.notify(f'Search error: {e}', color='negative')
|
|
finally:
|
|
search_btn.set_enabled(True)
|
|
|
|
async def copy_selected():
|
|
with table.client:
|
|
if not table.selected:
|
|
ui.notify('No files selected', color='warning')
|
|
return
|
|
|
|
dest = copy_input.value.strip() or 'out'
|
|
status.set_text(f'Copying {len(table.selected)} files to {dest}...')
|
|
|
|
try:
|
|
os.makedirs(dest, exist_ok=True)
|
|
|
|
if copy_clear.value:
|
|
def clear_dst():
|
|
for name in os.listdir(dest):
|
|
path = os.path.join(dest, name)
|
|
try:
|
|
if os.path.isfile(path) or os.path.islink(path):
|
|
os.remove(path)
|
|
elif os.path.isdir(path):
|
|
shutil.rmtree(path)
|
|
except Exception:
|
|
pass
|
|
await anyio.to_thread.run_sync(clear_dst)
|
|
|
|
def copy_files():
|
|
for r in table.selected:
|
|
try:
|
|
dst = os.path.join(dest, os.path.basename(r['path']))
|
|
with open(r['path'], 'rb') as srcf, open(dst, 'wb') as dstf:
|
|
dstf.write(srcf.read())
|
|
except Exception:
|
|
pass
|
|
await anyio.to_thread.run_sync(copy_files)
|
|
status.set_text(status.text + f' Copied {len(table.selected)} files to {dest}')
|
|
ui.notify('Files copied successfully', color='positive')
|
|
except Exception as e:
|
|
status.set_text(status.text + f' Copy failed: {e}')
|
|
ui.notify(f'Copy failed: {e}', color='negative')
|
|
|
|
async def export_combined_selected():
|
|
with table.client:
|
|
if not table.selected:
|
|
ui.notify('No records selected', color='warning')
|
|
return
|
|
|
|
dest_dir = copy_input.value.strip() or 'out'
|
|
filename = os.path.basename(out_input.value or 'results.json')
|
|
if not filename.lower().endswith('.json'):
|
|
filename = os.path.splitext(filename)[0] + '.json'
|
|
|
|
outp = os.path.join(dest_dir, filename)
|
|
status.set_text(f'Exporting {len(table.selected)} records combined to {outp}...')
|
|
|
|
try:
|
|
os.makedirs(dest_dir, exist_ok=True)
|
|
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, 'combined_json')
|
|
status.set_text(f'Exported {len(table.selected)} records combined to {outp}.')
|
|
ui.notify('Combined export successful', color='positive')
|
|
except Exception as e:
|
|
status.set_text(f'Combined export failed: {e}')
|
|
ui.notify(f'Combined export failed: {e}', color='negative')
|
|
|
|
async def export_combined_md():
|
|
with table.client:
|
|
if not table.selected:
|
|
ui.notify('No records selected', color='warning')
|
|
return
|
|
|
|
dest_dir = copy_input.value.strip() or 'out'
|
|
|
|
active_queries = [q for q in queries if q['qval'].strip()]
|
|
if active_queries:
|
|
qvals = [q['qval'].strip() for q in active_queries]
|
|
slug = "_".join([re.sub(r'[^a-zA-Z0-9_-]', '_', qv.lower()) for qv in qvals])
|
|
slug = re.sub(r'_+', '_', slug).strip('_')
|
|
filename = f"{slug}_combined.md"
|
|
else:
|
|
filename = "combined.md"
|
|
|
|
outp = os.path.join(dest_dir, filename)
|
|
status.set_text(f'Exporting {len(table.selected)} records combined to {outp}...')
|
|
|
|
try:
|
|
os.makedirs(dest_dir, exist_ok=True)
|
|
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, 'combined_md')
|
|
status.set_text(f'Exported {len(table.selected)} records combined to {outp}.')
|
|
ui.notify('Combined markdown export successful', color='positive')
|
|
except Exception as e:
|
|
status.set_text(f'Combined markdown export failed: {e}')
|
|
ui.notify(f'Combined markdown export failed: {e}', color='negative')
|
|
|
|
async def export_selected():
|
|
with table.client:
|
|
if not table.selected:
|
|
ui.notify('No records selected', color='warning')
|
|
return
|
|
|
|
outp = out_input.value or 'results.json'
|
|
status.set_text(f'Exporting {len(table.selected)} records to {outp}...')
|
|
|
|
try:
|
|
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, fmt_select.value)
|
|
status.set_text(f'Exported {len(table.selected)} records to {outp}.')
|
|
ui.notify('Export successful', color='positive')
|
|
except Exception as e:
|
|
status.set_text(f'Export failed: {e}')
|
|
ui.notify(f'Export failed: {e}', color='negative')
|
|
|
|
def open_output_dir() -> None:
|
|
dest = copy_input.value.strip() if copy_input.value else ''
|
|
if dest:
|
|
path = os.path.abspath(dest)
|
|
else:
|
|
outp = out_input.value or 'results.json'
|
|
dirname = os.path.dirname(outp)
|
|
path = os.path.abspath(dirname) if dirname else os.getcwd()
|
|
status.set_text(f'Opening: {path} ...')
|
|
|
|
tried = []
|
|
try:
|
|
if os.name == 'posix':
|
|
opener = shutil.which('xdg-open') or shutil.which('gio') or shutil.which('gvfs-open')
|
|
if opener:
|
|
tried.append(opener)
|
|
try:
|
|
subprocess.Popen([opener, path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
status.set_text(status.text + f' Opened {path} with {opener}')
|
|
return
|
|
except Exception as e:
|
|
status.set_text(status.text + f' Failed to open with {opener}: {e}')
|
|
except Exception as e:
|
|
status.set_text(status.text + f' Server-side open check failed: {e}')
|
|
|
|
status.set_text(status.text + ' Falling back to browser open (may be blocked).')
|
|
ui.notify(f'Path: {path}', color='primary')
|
|
ui.run_javascript("window.open(" + json.dumps('file://' + path) + ")")
|
|
|
|
# Document -> Markdown conversion section
|
|
ui.separator().classes('my-4')
|
|
with ui.card().classes('w-full p-5 shadow-xl gap-3'):
|
|
with ui.row().classes('w-full items-center justify-between'):
|
|
with ui.row().classes('items-center gap-3'):
|
|
ui.icon('transform', size='sm').classes('text-purple-400')
|
|
ui.label('Convert Captured JSON → Markdown').classes('text-lg font-bold text-slate-200')
|
|
dtm_status = ui.label('').classes('text-slate-400 text-sm')
|
|
|
|
with ui.expansion('Conversion Settings', icon='tune').classes('w-full text-slate-300 border border-slate-800 rounded-lg') as dtm_config_expansion:
|
|
with ui.column().classes('w-full gap-3 p-4'):
|
|
with ui.row().classes('w-full gap-4 items-start'):
|
|
dtm_input_dir = ui.input(value='xhr_captured_async', label='Input directory (captures)').classes('flex-grow').props('outlined dense')
|
|
dtm_output_dir = ui.input(value='docs_md', label='Output directory').classes('flex-grow').props('outlined dense')
|
|
dtm_pattern = ui.input(value='*', label='Glob pattern').classes('w-40').props('outlined dense')
|
|
with ui.row().classes('w-full gap-4 items-center flex-wrap'):
|
|
dtm_overwrite = ui.checkbox('Overwrite existing', value=False).classes('text-slate-300')
|
|
dtm_verbose = ui.checkbox('Verbose', value=False).classes('text-slate-300')
|
|
dtm_clear = ui.checkbox('Clear output first', value=False).classes('text-slate-300')
|
|
dtm_copy_images = ui.checkbox('Copy annotation images', value=True).classes('text-slate-300')
|
|
|
|
dtm_log = ui.markdown('').classes('text-sm')
|
|
|
|
# Track whether scraper has been opened to highlight the button
|
|
_capture_opened = {'value': False}
|
|
|
|
async def run_dtm_handler():
|
|
run_conv_button.set_enabled(False)
|
|
dtm_status.set_text('Converting...')
|
|
|
|
argv = []
|
|
argv += ['--input-dir', dtm_input_dir.value or 'xhr_captured_async']
|
|
argv += ['--output-dir', dtm_output_dir.value or 'docs_md']
|
|
if dtm_pattern.value:
|
|
argv += ['--pattern', dtm_pattern.value]
|
|
if dtm_overwrite.value:
|
|
argv += ['--overwrite']
|
|
if dtm_verbose.value:
|
|
argv += ['--verbose']
|
|
if dtm_clear.value:
|
|
argv += ['--clear']
|
|
if not dtm_copy_images.value:
|
|
try:
|
|
document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__'] = False
|
|
except Exception:
|
|
pass
|
|
|
|
dtm_status.set_text('Running: ' + ' '.join(argv))
|
|
|
|
def run_main():
|
|
try:
|
|
return document_to_markdown.main(argv)
|
|
finally:
|
|
if '__COPY_ANNOTATION_IMAGES_OVERRIDE__' in document_to_markdown.__dict__:
|
|
del document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__']
|
|
|
|
try:
|
|
code = await anyio.to_thread.run_sync(run_main)
|
|
dtm_status.set_text(f'Conversion complete (exit code {code})')
|
|
dtm_log.set_content(f'Exit code: `{code}` — Args: `{" ".join(argv)}`')
|
|
# After successful conversion, clear doc cache so searches pick up new docs
|
|
try:
|
|
search_md.clear_doc_cache()
|
|
except Exception:
|
|
pass
|
|
# Remove highlight after running
|
|
_capture_opened['value'] = False
|
|
run_conv_button.classes(remove='ring-2 ring-amber-400 ring-offset-2 ring-offset-slate-900 animate-pulse')
|
|
except Exception as e:
|
|
dtm_status.set_text(f'Conversion failed: {e}')
|
|
dtm_log.set_content(f'Error: {e}')
|
|
finally:
|
|
run_conv_button.set_enabled(True)
|
|
|
|
run_conv_button = ui.button(
|
|
'Run Conversion', icon='play_arrow',
|
|
on_click=lambda: asyncio.create_task(run_dtm_handler()),
|
|
).classes('bg-purple-600 hover:bg-purple-700 text-white font-semibold px-6 py-2 rounded-lg shadow-md transition-all duration-300')
|
|
|
|
def highlight_conversion_button():
|
|
"""Call after capture scraper is opened to draw attention to the conversion button."""
|
|
_capture_opened['value'] = True
|
|
run_conv_button.classes(add='ring-2 ring-amber-400 ring-offset-2 ring-offset-slate-900 animate-pulse')
|
|
|
|
# Register the highlight callback so trigger_capture_handler can call it
|
|
_page_callbacks['highlight_conversion'] = highlight_conversion_button
|
|
|
|
|
|
def linkify_references(content: str, root: str = 'docs_md/articles') -> str:
|
|
def replace_anatomy(match):
|
|
raw_text = match.group(0)
|
|
uuid = match.group(1)
|
|
fm, _, _ = search_md.get_doc_by_id(root, uuid)
|
|
if fm:
|
|
title = fm.get('title') or fm.get('pageTitle') or uuid
|
|
return f'[{title}](/document/{uuid})'
|
|
return raw_text
|
|
|
|
anatomy_pattern = re.compile(r'(?:[a-zA-Z0-9\-]+/)?ANATOMY:([a-f0-9\-]{36})', re.IGNORECASE)
|
|
content = anatomy_pattern.sub(replace_anatomy, content)
|
|
|
|
def replace_ddx(match):
|
|
raw_text = match.group(0)
|
|
uuid = match.group(1)
|
|
fm, _, _ = search_md.get_doc_by_id(root, uuid)
|
|
if fm:
|
|
title = fm.get('title') or fm.get('pageTitle') or uuid
|
|
return f'[{title}](/document/{uuid})'
|
|
return raw_text
|
|
|
|
ddx_pattern = re.compile(r'DDX:([a-f0-9\-]{36})', re.IGNORECASE)
|
|
content = ddx_pattern.sub(replace_ddx, content)
|
|
|
|
def replace_standard_links(match):
|
|
label = match.group(1)
|
|
uuid = match.group(3)
|
|
fm, _, _ = search_md.get_doc_by_id(root, uuid)
|
|
if fm:
|
|
return f'[{label}](/document/{uuid})'
|
|
return match.group(0)
|
|
|
|
markdown_link_pattern = re.compile(r'\[([^\]]+)\]\(((?:https?://app\.statdx\.com)?/document/[^/]+/([a-f0-9\-]{36}))\)')
|
|
content = markdown_link_pattern.sub(replace_standard_links, content)
|
|
|
|
return content
|
|
|
|
|
|
HISTORY_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '.search_history.json')
|
|
HISTORY_FILE = os.path.abspath(HISTORY_FILE)
|
|
|
|
def load_history() -> list:
|
|
try:
|
|
if os.path.exists(HISTORY_FILE):
|
|
with open(HISTORY_FILE, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
except Exception:
|
|
pass
|
|
return []
|
|
|
|
def save_history(history_list: list):
|
|
try:
|
|
with open(HISTORY_FILE, 'w', encoding='utf-8') as f:
|
|
json.dump(history_list, f, indent=2, ensure_ascii=False)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def is_doc_cached(docid: str) -> bool:
|
|
fm, _, _ = search_md.get_doc_by_id('docs_md/articles', docid)
|
|
return fm is not None
|
|
|
|
|
|
def make_online_url(docid: str, title: str) -> str:
|
|
slug = title.lower()
|
|
slug = re.sub(r'[^a-z0-9]+', '-', slug).strip('-')
|
|
return f"https://app.statdx.com/document/{slug}/{docid}"
|
|
|
|
|
|
|
|
@ui.page('/document/{identifier}')
|
|
def render_doc_page(identifier: str):
|
|
ui.add_head_html('''
|
|
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700;800&display=swap" rel="stylesheet">
|
|
<style>
|
|
body {
|
|
font-family: 'Outfit', sans-serif;
|
|
background-color: #0b0f19;
|
|
color: #e2e8f0;
|
|
line-height: 1.6;
|
|
}
|
|
.container {
|
|
max-width: 900px;
|
|
margin: 0 auto;
|
|
padding: 2rem 1.5rem;
|
|
}
|
|
.header-card {
|
|
background: linear-gradient(135deg, #1e293b 0%, #0f172a 100%);
|
|
border: 1px solid #334155;
|
|
border-radius: 12px;
|
|
padding: 2rem;
|
|
margin-bottom: 2rem;
|
|
}
|
|
.markdown-content h1 {
|
|
font-size: 2.25rem;
|
|
font-weight: 800;
|
|
color: #3b82f6;
|
|
border-bottom: 2px solid #1e293b;
|
|
padding-bottom: 0.5rem;
|
|
margin-top: 2rem;
|
|
margin-bottom: 1rem;
|
|
}
|
|
.markdown-content h2 {
|
|
font-size: 1.75rem;
|
|
font-weight: 700;
|
|
color: #60a5fa;
|
|
margin-top: 1.5rem;
|
|
margin-bottom: 0.75rem;
|
|
}
|
|
.markdown-content h3 {
|
|
font-size: 1.35rem;
|
|
font-weight: 600;
|
|
color: #93c5fd;
|
|
margin-top: 1.25rem;
|
|
margin-bottom: 0.5rem;
|
|
}
|
|
.markdown-content p, .markdown-content li {
|
|
color: #cbd5e1;
|
|
font-size: 1.05rem;
|
|
}
|
|
.markdown-content ul {
|
|
list-style-type: disc;
|
|
padding-left: 1.5rem;
|
|
}
|
|
.markdown-content img {
|
|
max-width: 100%;
|
|
border-radius: 8px;
|
|
border: 1px solid #334155;
|
|
margin: 1.5rem 0;
|
|
display: block;
|
|
}
|
|
.markdown-content em {
|
|
color: #94a3b8;
|
|
font-style: italic;
|
|
}
|
|
.breadcrumb-chip {
|
|
background-color: rgba(59, 130, 246, 0.15);
|
|
color: #93c5fd;
|
|
border: 1px solid rgba(59, 130, 246, 0.3);
|
|
padding: 4px 10px;
|
|
border-radius: 9999px;
|
|
font-size: 0.85rem;
|
|
font-weight: 500;
|
|
}
|
|
</style>
|
|
''')
|
|
|
|
fm, content, path = search_md.get_doc_by_id('docs_md/articles', identifier)
|
|
if fm and content:
|
|
content = linkify_references(content, 'docs_md/articles')
|
|
if not fm:
|
|
with ui.column().classes('w-full items-center justify-center p-12 gap-4'):
|
|
ui.icon('warning', size='4rem', color='negative')
|
|
ui.label(f'Document "{identifier}" not found.').classes('text-2xl text-red-400 font-bold')
|
|
ui.button('Go to Search', on_click=lambda: ui.navigate.to('/')).props('outline')
|
|
return
|
|
|
|
with ui.column().classes('container gap-6'):
|
|
ui.button('Back to Explorer', icon='arrow_back', on_click=lambda: ui.navigate.to('/')).classes('self-start text-blue-400 border border-blue-800').props('outline dense')
|
|
|
|
with ui.column().classes('header-card w-full gap-4'):
|
|
bcs = fm.get('breadcrumbs') or []
|
|
if bcs:
|
|
with ui.row().classes('gap-2 items-center flex-wrap'):
|
|
for b in bcs:
|
|
ui.label(b).classes('breadcrumb-chip')
|
|
|
|
title_text = fm.get('title') or fm.get('pageTitle') or "Untitled"
|
|
ui.label(title_text).classes('text-4xl font-extrabold text-white leading-tight')
|
|
|
|
with ui.row().classes('w-full gap-6 mt-2'):
|
|
if fm.get('category'):
|
|
with ui.column().classes('gap-0.5'):
|
|
ui.label('Category').classes('text-slate-400 text-xs uppercase font-semibold')
|
|
ui.label(fm.get('category')).classes('text-slate-200 font-medium')
|
|
if fm.get('type'):
|
|
with ui.column().classes('gap-0.5'):
|
|
ui.label('Type').classes('text-slate-400 text-xs uppercase font-semibold')
|
|
ui.label(fm.get('type')).classes('text-slate-200 font-medium')
|
|
if fm.get('lastUpdated'):
|
|
with ui.column().classes('gap-0.5'):
|
|
ui.label('Last Updated').classes('text-slate-400 text-xs uppercase font-semibold')
|
|
ui.label(fm.get('lastUpdated')).classes('text-slate-200 font-medium')
|
|
|
|
authors = fm.get('authors') or []
|
|
if authors:
|
|
with ui.column().classes('gap-1 mt-2'):
|
|
ui.label('Authors').classes('text-slate-400 text-xs uppercase font-semibold')
|
|
authors_text = ", ".join([a.get('value') if isinstance(a, dict) else str(a) for a in authors])
|
|
ui.label(authors_text).classes('text-blue-300 font-medium')
|
|
|
|
with ui.card().classes('w-full p-8 shadow-xl bg-slate-900/60 border border-slate-800 rounded-xl markdown-content'):
|
|
ui.markdown(content)
|
|
|
|
|
|
def main() -> None:
|
|
port = int(os.environ.get('PORT', '8081'))
|
|
# NiceGUI's ui.run will serve the app; mount the page at /search-md
|
|
ui.run(title='search_md GUI', port=port)
|
|
|
|
|
|
if __name__ in {"__main__", "__mp_main__"}:
|
|
main()
|