ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
Ross
2026-07-12 18:03:18 +01:00
parent 0bd27c2714
commit 2c3adb081f
398 changed files with 14455 additions and 514 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/search_md_gui.py
+127 -75
View File
@@ -205,6 +205,18 @@ def search_page() -> None: # build UI
font-weight: 500;
display: inline-block;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.7; }
}
.animate-pulse {
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
}
.ring-2 {
box-shadow: 0 0 0 2px var(--ring-color, #fbbf24), 0 0 12px 2px rgba(251, 191, 36, 0.3);
}
.ring-amber-400 { --ring-color: #fbbf24; }
.ring-offset-2 { outline-offset: 2px; }
</style>
''')
@@ -294,17 +306,19 @@ def search_page() -> None: # build UI
# Persistent Search History Container
with ui.row().classes('w-full items-center gap-2 mt-2'):
ui.label('Recent Searches:').classes('text-slate-400 text-xs font-semibold')
history_container = ui.row().classes('gap-1 items-center')
history_container = ui.row().classes('gap-1 items-center flex-wrap')
clear_history_container = ui.row().classes('items-center')
def render_history_chips():
history_container.clear()
clear_history_container.clear()
history = load_history()
if not history:
with history_container:
ui.label('None').classes('text-slate-500 text-xs italic')
ui.label('No history yet').classes('text-slate-500 text-xs italic')
return
with history_container:
for item in history[:8]:
for idx, item in enumerate(history[:10]):
def make_click(saved=item['queries']):
def handler():
nonlocal queries
@@ -312,13 +326,31 @@ def search_page() -> None: # build UI
for sq in saved:
queries.append(json.loads(json.dumps(sq)))
render_query_blocks()
ui.notify("Search query restored!", type='info')
asyncio.create_task(run_search_handler())
return handler
def make_delete(label=item['label']):
def handler(e):
# value becomes False when chip is removed via X
if not e.value:
h = load_history()
h = [x for x in h if x['label'] != label]
save_history(h)
render_history_chips()
ui.notify('Removed from history', type='info')
return handler
ui.chip(
item['label'],
clickable=True,
on_click=make_click()
).props('dense color="slate-800" text-color="blue-300" icon="history"').classes('text-xs hover:bg-slate-700')
removable=True,
on_click=make_click(),
on_value_change=make_delete(),
).props('dense color="slate-800" text-color="blue-300" icon="history"').classes('text-xs hover:bg-slate-700 cursor-pointer')
with clear_history_container:
if history:
def clear_all_history():
save_history([])
render_history_chips()
ui.notify('Search history cleared', type='info')
ui.button(icon='delete_sweep', on_click=clear_all_history).props('flat round dense size="xs"').classes('text-slate-500 hover:text-red-400').tooltip('Clear all history')
render_history_chips()
@@ -510,6 +542,9 @@ def search_page() -> None: # build UI
ui.notify(f"Cached document: {title}", type='positive')
return
# Registry for cross-section callbacks (populated later by conversion section)
_page_callbacks = {}
async def trigger_capture_handler(args: dict):
docid = args.get('docid')
title = args.get('title')
@@ -534,6 +569,10 @@ def search_page() -> None: # build UI
if ok:
ui.notify(f"Requested capture browser to open: {title}", type='positive')
start_capture_poll_task(docid, title)
# Highlight the conversion button to remind user to run it
cb = _page_callbacks.get('highlight_conversion')
if cb:
cb()
return
ui.notify("Capture browser not responding. Cleaning up existing capture processes...", type='warning')
@@ -556,6 +595,10 @@ def search_page() -> None: # build UI
if ok_again:
ui.notify(f"Capture browser launched successfully. Opened: {title}", type='positive')
start_capture_poll_task(docid, title)
# Highlight the conversion button to remind user to run it
cb = _page_callbacks.get('highlight_conversion')
if cb:
cb()
else:
ui.notify("Capture browser process launched, but command port timed out. Try clicking again in a few seconds.", type='warning')
except Exception as e:
@@ -971,83 +1014,91 @@ def search_page() -> None: # build UI
ui.run_javascript("window.open(" + json.dumps('file://' + path) + ")")
# Document -> Markdown conversion section
ui.markdown('---')
ui.markdown('# Convert captured JSON -> Markdown (document_to_markdown)')
ui.separator().classes('my-4')
with ui.card().classes('w-full p-5 shadow-xl gap-3'):
with ui.row().classes('w-full items-center justify-between'):
with ui.row().classes('items-center gap-3'):
ui.icon('transform', size='sm').classes('text-purple-400')
ui.label('Convert Captured JSON → Markdown').classes('text-lg font-bold text-slate-200')
dtm_status = ui.label('').classes('text-slate-400 text-sm')
with ui.row().style('gap: 12px; align-items: start;'):
dtm_input_dir = ui.input(value='xhr_captured_async', label='Input directory (captures)')
dtm_output_dir = ui.input(value='docs_md', label='Output directory')
dtm_pattern = ui.input(value='*', label='Glob pattern (filename match)')
with ui.expansion('Conversion Settings', icon='tune').classes('w-full text-slate-300 border border-slate-800 rounded-lg') as dtm_config_expansion:
with ui.column().classes('w-full gap-3 p-4'):
with ui.row().classes('w-full gap-4 items-start'):
dtm_input_dir = ui.input(value='xhr_captured_async', label='Input directory (captures)').classes('flex-grow').props('outlined dense')
dtm_output_dir = ui.input(value='docs_md', label='Output directory').classes('flex-grow').props('outlined dense')
dtm_pattern = ui.input(value='*', label='Glob pattern').classes('w-40').props('outlined dense')
with ui.row().classes('w-full gap-4 items-center flex-wrap'):
dtm_overwrite = ui.checkbox('Overwrite existing', value=False).classes('text-slate-300')
dtm_verbose = ui.checkbox('Verbose', value=False).classes('text-slate-300')
dtm_clear = ui.checkbox('Clear output first', value=False).classes('text-slate-300')
dtm_copy_images = ui.checkbox('Copy annotation images', value=True).classes('text-slate-300')
with ui.row().style('gap: 12px; align-items: center; margin-top: 8px;'):
dtm_overwrite = ui.checkbox('Overwrite existing .md files (--overwrite)', value=False)
dtm_verbose = ui.checkbox('Verbose', value=False)
dtm_clear = ui.checkbox('Clear output dir before processing (--clear)', value=False)
dtm_copy_images = ui.checkbox('Copy annotation images (--copy-annotation-images)', value=True)
dtm_log = ui.markdown('').classes('text-sm')
dtm_status = ui.label('')
dtm_log = ui.markdown('')
# Track whether scraper has been opened to highlight the button
_capture_opened = {'value': False}
async def run_dtm_handler():
run_conv_button.set_enabled(False)
dtm_status.set_text('Starting conversion...')
async def run_dtm_handler():
run_conv_button.set_enabled(False)
dtm_status.set_text('Converting...')
argv = []
argv += ['--input-dir', dtm_input_dir.value or 'xhr_captured_async']
argv += ['--output-dir', dtm_output_dir.value or 'docs_md']
if dtm_pattern.value:
argv += ['--pattern', dtm_pattern.value]
if dtm_overwrite.value:
argv += ['--overwrite']
if dtm_verbose.value:
argv += ['--verbose']
if dtm_clear.value:
argv += ['--clear']
# copy-annotation-images default is True in the script; include flag only to force True/False
if dtm_copy_images.value:
# default True; nothing to do (module default preserves True)
pass
else:
# to disable, pass a flag that the script does not have; instead we'll set the attribute on the module
# The document_to_markdown.main() reads args and uses them directly; we can pass --copy-annotation-images if True
# but to disable, we append a fake flag handling by setting module default before running
try:
document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__'] = False
except Exception:
pass
dtm_status.set_text('Running document_to_markdown.main with: ' + ' '.join(argv))
def run_main():
# If the caller set an override for copy-image flag, patch the module default
try:
if document_to_markdown.__dict__.get('__COPY_ANNOTATION_IMAGES_OVERRIDE__') is False:
# monkeypatch the default used in argparse by modifying the function's default after parse
# simpler: modify module level default so code that checks args.copy_annotation_images still behaves
# but since argparse produces args, we can't change that here. Instead, we'll call main() then
# but ensure environment where images are not copied by setting an env var the script checks? It doesn't.
# As a pragmatic approach, run main normally but then remove copied images afterwards if any were copied.
argv = []
argv += ['--input-dir', dtm_input_dir.value or 'xhr_captured_async']
argv += ['--output-dir', dtm_output_dir.value or 'docs_md']
if dtm_pattern.value:
argv += ['--pattern', dtm_pattern.value]
if dtm_overwrite.value:
argv += ['--overwrite']
if dtm_verbose.value:
argv += ['--verbose']
if dtm_clear.value:
argv += ['--clear']
if not dtm_copy_images.value:
try:
document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__'] = False
except Exception:
pass
except Exception:
pass
dtm_status.set_text('Running: ' + ' '.join(argv))
def run_main():
try:
return document_to_markdown.main(argv)
finally:
if '__COPY_ANNOTATION_IMAGES_OVERRIDE__' in document_to_markdown.__dict__:
del document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__']
try:
return document_to_markdown.main(argv)
code = await anyio.to_thread.run_sync(run_main)
dtm_status.set_text(f'Conversion complete (exit code {code})')
dtm_log.set_content(f'Exit code: `{code}` — Args: `{" ".join(argv)}`')
# After successful conversion, clear doc cache so searches pick up new docs
try:
search_md.clear_doc_cache()
except Exception:
pass
# Remove highlight after running
_capture_opened['value'] = False
run_conv_button.classes(remove='ring-2 ring-amber-400 ring-offset-2 ring-offset-slate-900 animate-pulse')
except Exception as e:
dtm_status.set_text(f'Conversion failed: {e}')
dtm_log.set_content(f'Error: {e}')
finally:
# cleanup any override marker
if '__COPY_ANNOTATION_IMAGES_OVERRIDE__' in document_to_markdown.__dict__:
del document_to_markdown.__dict__['__COPY_ANNOTATION_IMAGES_OVERRIDE__']
run_conv_button.set_enabled(True)
try:
code = await anyio.to_thread.run_sync(run_main)
dtm_status.set_text(f'document_to_markdown finished with exit code {code}')
dtm_log.set_content(f'Last run exit code: {code}\n\nArgs used: `{json.dumps(argv)}`')
except Exception as e:
dtm_status.set_text(f'Conversion failed: {e}')
dtm_log.set_content(f'Conversion failed: {e}')
finally:
run_conv_button.set_enabled(True)
run_conv_button = ui.button(
'Run Conversion', icon='play_arrow',
on_click=lambda: asyncio.create_task(run_dtm_handler()),
).classes('bg-purple-600 hover:bg-purple-700 text-white font-semibold px-6 py-2 rounded-lg shadow-md transition-all duration-300')
run_conv_button = ui.button('Run conversion', on_click=lambda: asyncio.create_task(run_dtm_handler()), color='primary')
def highlight_conversion_button():
"""Call after capture scraper is opened to draw attention to the conversion button."""
_capture_opened['value'] = True
run_conv_button.classes(add='ring-2 ring-amber-400 ring-offset-2 ring-offset-slate-900 animate-pulse')
# Register the highlight callback so trigger_capture_handler can call it
_page_callbacks['highlight_conversion'] = highlight_conversion_button
def linkify_references(content: str, root: str = 'docs_md/articles') -> str:
@@ -1089,7 +1140,8 @@ def linkify_references(content: str, root: str = 'docs_md/articles') -> str:
return content
HISTORY_FILE = 'search_history.json'
HISTORY_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '.search_history.json')
HISTORY_FILE = os.path.abspath(HISTORY_FILE)
def load_history() -> list:
try: