ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update search

Browse Source
This commit is contained in:
Ross
2026-07-11 10:48:55 +01:00
parent 36ca884b4f
commit 9f70f3994c
5 changed files with 12500 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/search_md_gui.py
+310 -89
View File
@@ -50,102 +50,331 @@ def load_document_to_markdown_module() -> object:
document_to_markdown = load_document_to_markdown_module()
def format_result_md(results: List[Dict[str, Any]]) -> str:
if not results:
return 'No results.'
lines = [
'| Title | DocID | Path | breadcrumbs | authors | pageKeywords |',
'|---|---|---|---|---|---|',
]
for r in results:
title = (r.get('title') or '').replace('|', r'\|')
docid = r.get('docid') or ''
if docid:
doclink = f"[\u2197 {docid}](https://app.statdx.com/document/{docid})"
else:
doclink = ''
path = r.get('path') or ''
# file:// link so clicking in a browser attempts to open local file (may be blocked by browser)
path_link = f"[{os.path.basename(path)}](file://{path})" if path else ''
bcs = json.dumps(r.get('breadcrumbs') or [])
authors = json.dumps(r.get('authors') or [])
pks = r.get('pageKeywords') or ''
lines.append(f'| {title} | {doclink} | {path_link} | {bcs} | {authors} | {pks} |')
return '\n'.join(lines)
@ui.page('/')
def search_page() -> None: # build UI
ui.markdown('# Search extracted markdown (search_md)')
# Load fonts & modern dark mode stylesheet
ui.add_head_html('''
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&display=swap" rel="stylesheet">
<style>
body {
font-family: 'Outfit', sans-serif !important;
background-color: #0f172a !important;
color: #f8fafc !important;
}
.q-page-container {
background-color: #0f172a !important;
}
.q-card {
background-color: #1e293b !important;
color: #f8fafc !important;
border: 1px solid #334155 !important;
border-radius: 12px !important;
}
.q-table__container {
background-color: #1e293b !important;
color: #f8fafc !important;
border: 1px solid #334155 !important;
border-radius: 12px !important;
}
.q-table {
background-color: #1e293b !important;
color: #f8fafc !important;
}
.q-table th {
font-weight: 600 !important;
color: #94a3b8 !important;
background-color: #0f172a !important;
border-bottom: 2px solid #334155 !important;
}
.q-table tbody td {
color: #e2e8f0 !important;
border-bottom: 1px solid #1e293b !important;
}
.q-table__bottom {
background-color: #0f172a !important;
color: #94a3b8 !important;
border-top: 1px solid #334155 !important;
}
.q-field__native, .q-field__prefix, .q-field__suffix, .q-field__input {
color: #f8fafc !important;
}
.q-field__label {
color: #94a3b8 !important;
}
.q-field__marginal {
color: #94a3b8 !important;
}
/* Dropdown popups legibility overrides */
.q-menu {
background-color: #1e293b !important;
border: 1px solid #334155 !important;
color: #f8fafc !important;
box-shadow: 0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1) !important;
}
.q-item {
color: #e2e8f0 !important;
}
.q-item:hover, .q-item--active, .q-item--focused {
background-color: #334155 !important;
color: #f8fafc !important;
}
.q-item__label {
color: inherit !important;
}
.q-checkbox__bg {
border: 2px solid #64748b !important;
}
.q-item--active .q-checkbox__bg {
border: 2px solid #3b82f6 !important;
background: #3b82f6 !important;
}
.gradient-header {
background: linear-gradient(135deg, #3b82f6 0%, #a855f7 100%);
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
}
.badge-reason {
padding: 2px 8px;
border-radius: 4px;
font-size: 11px;
font-weight: 500;
display: inline-block;
margin-right: 4px;
}
.badge-title {
background-color: rgba(59, 130, 246, 0.2);
color: #93c5fd;
border: 1px solid rgba(59, 130, 246, 0.4);
}
.badge-content {
background-color: rgba(168, 85, 247, 0.2);
color: #e9d5ff;
border: 1px solid rgba(168, 85, 247, 0.4);
}
.badge-other {
background-color: rgba(148, 163, 184, 0.2);
color: #cbd5e1;
border: 1px solid rgba(148, 163, 184, 0.4);
}
</style>
''')
with ui.row().style('gap: 12px; align-items: start;'):
root_input = ui.input(value='docs_md/articles', label='Root directory', placeholder='docs_md/articles')
key_select = ui.select(['global', 'or', 'breadcrumbs', 'authors', 'pageKeywords', 'category', 'title', 'enhancedTitle', 'type'], value='breadcrumbs', label='Query key')
qval_input = ui.input(value='', label='Query value', placeholder='e.g. Brain>Diagnosis or Smith or keyword')
with ui.column().classes('w-full max-w-7xl mx-auto p-6 gap-6'):
# Header / Title Block
with ui.row().classes('items-center gap-4 w-full justify-between pb-4 border-b border-slate-800'):
with ui.column().classes('gap-1'):
ui.html('<h1 class="text-4xl font-extrabold gradient-header m-0">STATdx Article Explorer</h1>')
ui.label('Search extracted markdown files with fuzzy matching, stemming, and interactive selection.').classes('text-slate-400 text-base')
with ui.row().classes('gap-2'):
ui.button('Open Output Dir', icon='folder', on_click=lambda: open_output_dir(), color='secondary').props('outline')
with ui.row().style('gap: 12px; align-items: start; margin-top: 8px;'):
fmt_select = ui.select(['json', 'csv'], value='json', label='Output format')
out_input = ui.input(value='results.json', label='Output file (server-side)')
copy_input = ui.input(value='out', label='Copy matched files to (optional)')
copy_clear = ui.checkbox('Clear destination before copy (--copy-to-clear)', value=False)
# Main Search Box Card
with ui.card().classes('w-full p-6 shadow-xl'):
with ui.row().classes('w-full items-center gap-4'):
# Single search box
qval_input = ui.input(
label='Search Query',
placeholder='Type search terms...'
).classes('flex-grow').props('outlined label-slot')
qval_input.add_slot('append', '<q-icon name="search" />')
# Match mode select
mode_select = ui.select(
options={'exact': 'Exact Match', 'fuzzy': 'Fuzzy Match', 'stemming': 'Stemming (Related Words)'},
value='exact',
label='Match Mode'
).classes('w-56').props('outlined')
# Targets select
targets_select = ui.select(
options={
'Title': 'Title',
'Content': 'Content',
'Breadcrumbs': 'Breadcrumbs',
'Authors': 'Authors',
'Keywords': 'Keywords',
'Category': 'Category',
'Type': 'Type'
},
value=['Title', 'Content', 'Breadcrumbs', 'Keywords', 'Category', 'Type'],
label='Search Targets',
multiple=True
).classes('w-80').props('outlined use-chips')
# Search button
search_btn = ui.button('Search', icon='search', on_click=lambda: asyncio.create_task(run_search_handler())).classes('bg-blue-600 hover:bg-blue-700 text-white font-semibold py-4 px-8 rounded-lg shadow-md transition-all duration-300')
status = ui.label('')
md = ui.markdown('')
# Advanced Criteria Expansion Box
with ui.expansion('Advanced Configurations', icon='settings').classes('w-full text-slate-300 border border-slate-800 rounded-lg p-2'):
with ui.row().classes('w-full gap-4 items-start p-4'):
root_input = ui.input(value='docs_md/articles', label='Root directory').classes('w-64').props('outlined dense')
key_select = ui.select(['global', 'breadcrumbs', 'authors', 'pageKeywords', 'category', 'title', 'enhancedTitle', 'type', 'content'], value='global', label='Query key').classes('w-64').props('outlined dense')
fmt_select = ui.select(['json', 'csv'], value='json', label='Output format').classes('w-40').props('outlined dense')
out_input = ui.input(value='results.json', label='Output file (server-side)').classes('w-64').props('outlined dense')
copy_input = ui.input(value='out', label='Copy matched files to (optional)').classes('w-64').props('outlined dense')
copy_clear = ui.checkbox('Clear destination before copy', value=False).classes('self-center text-slate-300')
status = ui.label('').classes('text-blue-400 font-semibold text-lg px-2')
# Action bar for selected rows
# Hidden by default, displayed via bind_visibility_from
action_bar = ui.row().classes('w-full justify-between items-center gap-4 bg-slate-800 p-4 rounded-lg border border-slate-700')
action_bar_text = ui.label('').classes('text-white font-semibold')
with action_bar:
with ui.row().classes('gap-2'):
ui.button('Copy Selected Files', icon='content_copy', on_click=lambda: asyncio.create_task(copy_selected())).classes('bg-indigo-600 hover:bg-indigo-700 text-white font-medium px-4')
ui.button('Export Selected', icon='file_download', on_click=lambda: asyncio.create_task(export_selected())).classes('bg-emerald-600 hover:bg-emerald-700 text-white font-medium px-4')
ui.button('Deselect All', icon='close', on_click=lambda: table.selected.clear()).classes('bg-slate-700 hover:bg-slate-600 text-white font-medium px-4')
# Results table filter input
filter_card = ui.row().classes('w-full items-center gap-4')
with filter_card:
table_filter = ui.input(
placeholder='Filter results table below (live)...'
).classes('w-full').props('outlined dense clearable')
table_filter.add_slot('append', '<q-icon name="filter_list" />')
# Table definition
columns = [
{'name': 'title', 'label': 'Title', 'field': 'title', 'required': True, 'align': 'left', 'sortable': True},
{'name': 'reasons', 'label': 'Matched In', 'field': 'reasons', 'align': 'left', 'sortable': True},
{'name': 'snippet', 'label': 'Snippet', 'field': 'snippet', 'align': 'left'},
{'name': 'docid', 'label': 'DocID', 'field': 'docid', 'align': 'left'},
{'name': 'path', 'label': 'Path', 'field': 'path', 'align': 'left', 'sortable': True},
]
table = ui.table(
columns=columns,
rows=[],
row_key='path',
selection='multiple',
).classes('w-full shadow-lg')
table.bind_filter_from(table_filter, 'value')
# Bind action bar visibility and selection text
action_bar.bind_visibility_from(table, 'selected', lambda sel: len(sel) > 0)
action_bar_text.bind_text_from(table, 'selected', lambda sel: f'{len(sel)} files selected')
# Add slot overrides to format table cells nicely
table.add_slot('body-cell-title', '''
<q-td :props="props">
<a v-if="props.row.docid" :href="'https://app.statdx.com/document/' + props.row.docid" target="_blank" class="text-blue-400 hover:text-blue-300 font-semibold" style="text-decoration: none;">
{{ props.row.title }}
</a>
<span v-else class="text-slate-200 font-semibold">{{ props.row.title }}</span>
</q-td>
''')
table.add_slot('body-cell-reasons', '''
<q-td :props="props">
<span v-for="r in props.row.reasons" :key="r"
class="badge-reason"
:class="r === 'Title' ? 'badge-title' : r === 'Content' ? 'badge-content' : 'badge-other'">
{{ r }}
</span>
</q-td>
''')
table.add_slot('body-cell-path', '''
<q-td :props="props">
<span class="text-slate-400 text-xs">
{{ props.row.path.split('/').pop() }}
</span>
</q-td>
''')
async def run_search_handler():
if not qval_input.value:
ui.notify('Please enter a search query', color='warning')
return
status.set_text('Running search...')
# run blocking search in thread
results = await anyio.to_thread.run_sync(search_md.run_search, root_input.value, key_select.value, qval_input.value)
# write output if requested
if out_input.value:
try:
await anyio.to_thread.run_sync(search_md.write_output, results, out_input.value, fmt_select.value)
status.set_text(f'Found {len(results)} matches. Wrote {out_input.value}')
except Exception as e:
status.set_text(f'Found {len(results)} matches. Failed to write output: {e}')
else:
status.set_text(f'Found {len(results)} matches (not written).')
search_btn.set_enabled(False)
try:
results = await anyio.to_thread.run_sync(
search_md.run_search,
root_input.value,
key_select.value,
qval_input.value,
mode_select.value,
targets_select.value
)
table.rows = results
table.selected.clear()
# write output if requested
if out_input.value:
try:
await anyio.to_thread.run_sync(search_md.write_output, results, out_input.value, fmt_select.value)
status.set_text(f'Found {len(results)} matches. Wrote {out_input.value}')
except Exception as e:
status.set_text(f'Found {len(results)} matches. Failed to write output: {e}')
else:
status.set_text(f'Found {len(results)} matches (not written).')
except Exception as e:
status.set_text(f'Search failed: {e}')
ui.notify(f'Search error: {e}', color='negative')
finally:
search_btn.set_enabled(True)
# optionally copy matched files
if copy_input.value and results:
try:
os.makedirs(copy_input.value, exist_ok=True)
# Optionally clear the destination dir before copying
if bool(copy_clear.value):
def clear_dst():
for name in os.listdir(copy_input.value):
path = os.path.join(copy_input.value, name)
try:
if os.path.isfile(path) or os.path.islink(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
except Exception:
pass
await anyio.to_thread.run_sync(clear_dst)
def copy_files():
for r in results:
async def copy_selected():
if not table.selected:
ui.notify('No files selected', color='warning')
return
dest = copy_input.value.strip() or 'out'
status.set_text(f'Copying {len(table.selected)} files to {dest}...')
try:
os.makedirs(dest, exist_ok=True)
if copy_clear.value:
def clear_dst():
for name in os.listdir(dest):
path = os.path.join(dest, name)
try:
dst = os.path.join(copy_input.value, os.path.basename(r['path']))
with open(r['path'], 'rb') as srcf, open(dst, 'wb') as dstf:
dstf.write(srcf.read())
if os.path.isfile(path) or os.path.islink(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
except Exception:
pass
await anyio.to_thread.run_sync(copy_files)
status.set_text(status.text + f' Copied {len(results)} files to {copy_input.value}')
except Exception as e:
status.set_text(status.text + f' Copy failed: {e}')
await anyio.to_thread.run_sync(clear_dst)
def copy_files():
for r in table.selected:
try:
dst = os.path.join(dest, os.path.basename(r['path']))
with open(r['path'], 'rb') as srcf, open(dst, 'wb') as dstf:
dstf.write(srcf.read())
except Exception:
pass
await anyio.to_thread.run_sync(copy_files)
status.set_text(status.text + f' Copied {len(table.selected)} files to {dest}')
ui.notify('Files copied successfully', color='positive')
except Exception as e:
status.set_text(status.text + f' Copy failed: {e}')
ui.notify(f'Copy failed: {e}', color='negative')
md.set_content(format_result_md(results))
async def export_selected():
if not table.selected:
ui.notify('No records selected', color='warning')
return
outp = out_input.value or 'results.json'
status.set_text(f'Exporting {len(table.selected)} records to {outp}...')
try:
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, fmt_select.value)
status.set_text(f'Exported {len(table.selected)} records to {outp}.')
ui.notify('Export successful', color='positive')
except Exception as e:
status.set_text(f'Export failed: {e}')
ui.notify(f'Export failed: {e}', color='negative')
ui.button('Run search', on_click=lambda: asyncio.create_task(run_search_handler()))
ui.button('Open results.json (server)', on_click=lambda: ui.run_javascript("window.open(" + json.dumps(out_input.value) + ")"), color='secondary')
def open_output_dir() -> None:
# Prefer the copy destination if provided, otherwise use the directory of the output file
dest = copy_input.value.strip() if copy_input.value else ''
if dest:
path = os.path.abspath(dest)
@@ -155,7 +384,6 @@ def search_page() -> None: # build UI
path = os.path.abspath(dirname) if dirname else os.getcwd()
status.set_text(f'Opening: {path} ...')
# Try to open with a server-side file manager (xdg-open/gio/gvfs-open) on POSIX
tried = []
try:
if os.name == 'posix':
@@ -171,17 +399,10 @@ def search_page() -> None: # build UI
except Exception as e:
status.set_text(status.text + f' Server-side open check failed: {e}')
# If we got here, server-side open didn't work. Try client-side fallback and show path.
status.set_text(status.text + ' Falling back to browser open (may be blocked).')
# Also render the path visibly so the user can copy it
ui.notify(f'Path: {path}', color='primary')
ui.run_javascript("window.open(" + json.dumps('file://' + path) + ")")
ui.button('Open output directory', on_click=open_output_dir, color='secondary')
ui.markdown('---')
ui.markdown('Results:')
# Document -> Markdown conversion section
ui.markdown('---')
ui.markdown('# Convert captured JSON -> Markdown (document_to_markdown)')