more fixes
This commit is contained in:
Ross
+36
-5
@@ -585,7 +585,8 @@ def run_search(root: str, qkey: str, qval: str, mode: str = 'exact', targets: Li
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': sorted(list(set(reasons))),
|
||||
'snippet': snippet,
|
||||
'linked_info': check_linked_sections(content, root)
|
||||
'linked_info': check_linked_sections(content, root),
|
||||
'is_cached': True
|
||||
})
|
||||
|
||||
if expand_links:
|
||||
@@ -616,10 +617,25 @@ def run_search(root: str, qkey: str, qval: str, mode: str = 'exact', targets: Li
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': [f'Linked (Anatomy of {parent_title})'],
|
||||
'snippet': content[:300] + '...' if content else '',
|
||||
'linked_info': check_linked_sections(content, root)
|
||||
'linked_info': check_linked_sections(content, root),
|
||||
'is_cached': True
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
seen_paths.add(path)
|
||||
else:
|
||||
expanded_results.append({
|
||||
'path': f"missing_{docid}",
|
||||
'title': link['title'],
|
||||
'docid': docid,
|
||||
'breadcrumbs': [],
|
||||
'authors': [],
|
||||
'pageKeywords': [],
|
||||
'reasons': [f'Linked (Anatomy of {parent_title})'],
|
||||
'snippet': '',
|
||||
'linked_info': None,
|
||||
'is_cached': False
|
||||
})
|
||||
seen_paths.add(f"missing_{docid}")
|
||||
seen_docids.add(docid)
|
||||
|
||||
# Differential links
|
||||
for link in info['differential']['links']:
|
||||
@@ -637,10 +653,25 @@ def run_search(root: str, qkey: str, qval: str, mode: str = 'exact', targets: Li
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': [f'Linked (Diff Diag of {parent_title})'],
|
||||
'snippet': content[:300] + '...' if content else '',
|
||||
'linked_info': check_linked_sections(content, root)
|
||||
'linked_info': check_linked_sections(content, root),
|
||||
'is_cached': True
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
seen_paths.add(path)
|
||||
else:
|
||||
expanded_results.append({
|
||||
'path': f"missing_{docid}",
|
||||
'title': link['title'],
|
||||
'docid': docid,
|
||||
'breadcrumbs': [],
|
||||
'authors': [],
|
||||
'pageKeywords': [],
|
||||
'reasons': [f'Linked (Diff Diag of {parent_title})'],
|
||||
'snippet': '',
|
||||
'linked_info': None,
|
||||
'is_cached': False
|
||||
})
|
||||
seen_paths.add(f"missing_{docid}")
|
||||
seen_docids.add(docid)
|
||||
|
||||
out.extend(expanded_results)
|
||||
|
||||
|
||||
+240
-111
@@ -353,14 +353,24 @@ def search_page() -> None: # build UI
|
||||
table.add_slot('body-cell-title', '''
|
||||
<q-td :props="props">
|
||||
<div class="row items-center no-wrap q-gutter-xs">
|
||||
<a :href="'/document/' + (props.row.docid || props.row.path.split('/').pop().replace('.md', ''))" target="_blank" class="text-blue-400 hover:text-blue-300 font-semibold" style="text-decoration: none;">
|
||||
{{ props.row.title }}
|
||||
</a>
|
||||
<q-btn v-if="props.row.linked_info && ((props.row.linked_info.anatomy && props.row.linked_info.anatomy.links && props.row.linked_info.anatomy.links.length > 0) || (props.row.linked_info.differential && props.row.linked_info.differential.links && props.row.linked_info.differential.links.length > 0))"
|
||||
flat round dense size="xs" icon="alt_route" color="primary"
|
||||
@click.stop="$parent.$emit('expand_row_links', props.row.path)">
|
||||
<q-tooltip>Expand Linked Articles</q-tooltip>
|
||||
</q-btn>
|
||||
<template v-if="props.row.is_cached !== false">
|
||||
<a :href="'/document/' + (props.row.docid || props.row.path.split('/').pop().replace('.md', ''))" target="_blank" class="text-blue-400 hover:text-blue-300 font-semibold" style="text-decoration: none;">
|
||||
{{ props.row.title }}
|
||||
</a>
|
||||
<q-btn v-if="props.row.linked_info && ((props.row.linked_info.anatomy && props.row.linked_info.anatomy.links && props.row.linked_info.anatomy.links.length > 0) || (props.row.linked_info.differential && props.row.linked_info.differential.links && props.row.linked_info.differential.links.length > 0))"
|
||||
flat round dense size="xs" icon="alt_route" color="primary"
|
||||
@click.stop="$parent.$emit('expand_row_links', props.row.path)">
|
||||
<q-tooltip>Expand Linked Articles</q-tooltip>
|
||||
</q-btn>
|
||||
</template>
|
||||
<template v-else>
|
||||
<span class="text-slate-400 font-semibold line-through q-mr-xs">{{ props.row.title }}</span>
|
||||
<span class="badge-status-missing" style="background-color: rgba(239, 68, 68, 0.2); border-color: rgba(239, 68, 68, 0.4); color: #f87171;">Not Cached</span>
|
||||
<q-btn flat round dense size="xs" icon="public" color="warning" class="q-ml-xs"
|
||||
@click.stop="$parent.$emit('capture_article', { docid: props.row.docid, title: props.row.title })">
|
||||
<q-tooltip>Open in Capture Browser</q-tooltip>
|
||||
</q-btn>
|
||||
</template>
|
||||
</div>
|
||||
</q-td>
|
||||
''')
|
||||
@@ -406,12 +416,59 @@ def search_page() -> None: # build UI
|
||||
table.add_slot('body-cell-path', '''
|
||||
<q-td :props="props">
|
||||
<span class="text-slate-400 text-xs">
|
||||
{{ props.row.path.split('/').pop() }}
|
||||
{{ props.row.is_cached !== false ? props.row.path.split('/').pop() : 'N/A (Missing)' }}
|
||||
</span>
|
||||
</q-td>
|
||||
''')
|
||||
table.on('expand_row_links', lambda msg: asyncio.create_task(expand_row_links_handler(msg.args)))
|
||||
table.on('show_links_modal', lambda msg: show_links_modal_handler(msg.args))
|
||||
table.on('capture_article', lambda msg: asyncio.create_task(trigger_capture_handler(msg.args)))
|
||||
|
||||
async def trigger_capture_handler(args: dict):
|
||||
docid = args.get('docid')
|
||||
title = args.get('title')
|
||||
target_url = make_online_url(docid, title)
|
||||
|
||||
def check_and_send():
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
try:
|
||||
url = f"http://127.0.0.1:8089/open?url=" + urllib.parse.quote(target_url)
|
||||
with urllib.request.urlopen(url, timeout=2.0) as resp:
|
||||
return resp.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
with table.client:
|
||||
ui.notify(f"Checking capture browser status for: {title}...", type='info')
|
||||
ok = await asyncio.to_thread(check_and_send)
|
||||
if ok:
|
||||
ui.notify(f"Requested capture browser to open: {title}", type='positive')
|
||||
return
|
||||
|
||||
ui.notify("Capture browser not responding. Cleaning up existing capture processes...", type='warning')
|
||||
try:
|
||||
# Terminate any existing capture script processes
|
||||
subprocess.run(['pkill', '-f', 'capture_passive_playwright_async.py'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
await asyncio.sleep(1.0) # brief pause for os to clean up
|
||||
|
||||
ui.notify("Launching fresh capture browser...", type='info')
|
||||
subprocess.Popen([
|
||||
'uv', 'run', 'python', 'scrapers/capture_passive_playwright_async.py',
|
||||
'--output-dir', 'xhr_captured_async',
|
||||
'--continuous'
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
# Wait 5 seconds for the browser context to open and port to listen
|
||||
await asyncio.sleep(5.0)
|
||||
|
||||
ok_again = await asyncio.to_thread(check_and_send)
|
||||
if ok_again:
|
||||
ui.notify(f"Capture browser launched successfully. Opened: {title}", type='positive')
|
||||
else:
|
||||
ui.notify("Capture browser process launched, but command port timed out. Try clicking again in a few seconds.", type='warning')
|
||||
except Exception as e:
|
||||
ui.notify(f"Failed to start capture browser: {e}", type='negative')
|
||||
|
||||
async def expand_row_links_handler(path: str):
|
||||
with table.client:
|
||||
@@ -464,10 +521,32 @@ def search_page() -> None: # build UI
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': [origins[docid]],
|
||||
'snippet': content[:300] + '...' if content else '',
|
||||
'linked_info': search_md.check_linked_sections(content, root_input.value)
|
||||
'linked_info': search_md.check_linked_sections(content, root_input.value),
|
||||
'is_cached': True
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
seen_paths.add(doc_path)
|
||||
else:
|
||||
fallback_title = "Linked Document"
|
||||
for l in info['anatomy']['links'] + info['differential']['links']:
|
||||
if l['docid'] == docid:
|
||||
fallback_title = l['title']
|
||||
break
|
||||
fake_path = f"missing_{docid}"
|
||||
res.append({
|
||||
'path': fake_path,
|
||||
'title': fallback_title,
|
||||
'docid': docid,
|
||||
'breadcrumbs': [],
|
||||
'authors': [],
|
||||
'pageKeywords': [],
|
||||
'reasons': [origins[docid]],
|
||||
'snippet': '',
|
||||
'linked_info': None,
|
||||
'is_cached': False
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
seen_paths.add(fake_path)
|
||||
return res
|
||||
|
||||
new_rows = await anyio.to_thread.run_sync(fetch_linked)
|
||||
@@ -515,11 +594,21 @@ def search_page() -> None: # build UI
|
||||
docid = link['docid']
|
||||
linked_title = link['title']
|
||||
already_added = any(r.get('docid') == docid for r in table.rows if r.get('docid'))
|
||||
cached = is_doc_cached(docid)
|
||||
|
||||
with ui.row().classes('w-full items-center justify-between p-2 rounded bg-slate-800 border border-slate-700 hover:border-blue-500 transition-all'):
|
||||
with ui.row().classes('items-center gap-2 flex-grow'):
|
||||
ui.icon('link', color='primary').classes('text-sm')
|
||||
ui.link(linked_title, f'/document/{docid}', new_tab=True).classes('text-blue-300 hover:text-blue-200 font-semibold text-sm')
|
||||
if cached:
|
||||
with ui.row().classes('items-center gap-2 flex-grow'):
|
||||
ui.icon('link', color='primary').classes('text-sm')
|
||||
ui.link(linked_title, f'/document/{docid}', new_tab=True).classes('text-blue-300 hover:text-blue-200 font-semibold text-sm')
|
||||
else:
|
||||
with ui.row().classes('items-center gap-2 flex-grow'):
|
||||
ui.icon('link', color='warning').classes('text-sm')
|
||||
ui.label(linked_title).classes('text-slate-400 font-semibold line-through text-sm')
|
||||
ui.label('Not Cached').classes('text-red-400 text-xs font-semibold px-2 py-0.5 rounded bg-red-950/40 border border-red-800')
|
||||
def make_capture_click(d_id=docid, t_title=linked_title):
|
||||
return lambda: asyncio.create_task(trigger_capture_handler({'docid': d_id, 'title': t_title}))
|
||||
ui.button(icon='public', on_click=make_capture_click()).classes('text-xs text-yellow-400 hover:bg-yellow-900/20').props('flat round dense')
|
||||
|
||||
if already_added:
|
||||
ui.label('Added').classes('text-green-400 text-xs font-semibold px-2 py-1 rounded bg-green-950/40 border border-green-800')
|
||||
@@ -556,9 +645,21 @@ def search_page() -> None: # build UI
|
||||
'authors': fm.get('authors'),
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
||||
'linked_info': search_md.check_linked_sections(content, root_input.value)
|
||||
'linked_info': search_md.check_linked_sections(content, root_input.value),
|
||||
'is_cached': True
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'path': f"missing_{docid}",
|
||||
'title': title,
|
||||
'docid': docid,
|
||||
'breadcrumbs': [],
|
||||
'authors': [],
|
||||
'pageKeywords': [],
|
||||
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
||||
'linked_info': None,
|
||||
'is_cached': False
|
||||
}
|
||||
return None
|
||||
|
||||
res = await anyio.to_thread.run_sync(fetch)
|
||||
if res:
|
||||
@@ -593,9 +694,22 @@ def search_page() -> None: # build UI
|
||||
'authors': fm.get('authors'),
|
||||
'pageKeywords': fm.get('pageKeywords'),
|
||||
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
||||
'linked_info': search_md.check_linked_sections(content, root_input.value)
|
||||
'linked_info': search_md.check_linked_sections(content, root_input.value),
|
||||
'is_cached': True
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
else:
|
||||
res.append({
|
||||
'path': f"missing_{docid}",
|
||||
'title': link['title'],
|
||||
'docid': docid,
|
||||
'breadcrumbs': [],
|
||||
'authors': [],
|
||||
'pageKeywords': [],
|
||||
'reasons': [f'Linked ({section_title} of {parent_title})'],
|
||||
'linked_info': None,
|
||||
'is_cached': False
|
||||
})
|
||||
seen_docids.add(docid)
|
||||
return res
|
||||
|
||||
new_rows = await anyio.to_thread.run_sync(fetch_all)
|
||||
@@ -611,113 +725,116 @@ def search_page() -> None: # build UI
|
||||
ui.notify(f'Error adding documents: {e}', color='negative')
|
||||
|
||||
async def run_search_handler():
|
||||
has_query = False
|
||||
for q in queries:
|
||||
if q['qval'].strip():
|
||||
has_query = True
|
||||
break
|
||||
if not has_query:
|
||||
ui.notify('Please enter a search query', color='warning')
|
||||
return
|
||||
|
||||
status.set_text('Running search...')
|
||||
search_btn.set_enabled(False)
|
||||
|
||||
try:
|
||||
or_queries_param = []
|
||||
with table.client:
|
||||
has_query = False
|
||||
for q in queries:
|
||||
if q['qval'].strip():
|
||||
or_queries_param.append({
|
||||
'qkey': key_select.value,
|
||||
'qval': q['qval'],
|
||||
'mode': q['mode'],
|
||||
'targets': [t.lower() for t in q['targets']]
|
||||
})
|
||||
|
||||
results = await anyio.to_thread.run_sync(
|
||||
search_md.run_search,
|
||||
root_input.value,
|
||||
'global',
|
||||
'',
|
||||
'exact',
|
||||
None,
|
||||
or_queries_param,
|
||||
expand_chk.value
|
||||
)
|
||||
has_query = True
|
||||
break
|
||||
if not has_query:
|
||||
ui.notify('Please enter a search query', color='warning')
|
||||
return
|
||||
|
||||
status.set_text('Running search...')
|
||||
search_btn.set_enabled(False)
|
||||
|
||||
table.rows = results
|
||||
table.selected.clear()
|
||||
|
||||
# write output if requested
|
||||
if out_input.value:
|
||||
try:
|
||||
await anyio.to_thread.run_sync(search_md.write_output, results, out_input.value, fmt_select.value)
|
||||
status.set_text(f'Found {len(results)} matches. Wrote {out_input.value}')
|
||||
except Exception as e:
|
||||
status.set_text(f'Found {len(results)} matches. Failed to write output: {e}')
|
||||
else:
|
||||
status.set_text(f'Found {len(results)} matches (not written).')
|
||||
try:
|
||||
or_queries_param = []
|
||||
for q in queries:
|
||||
if q['qval'].strip():
|
||||
or_queries_param.append({
|
||||
'qkey': key_select.value,
|
||||
'qval': q['qval'],
|
||||
'mode': q['mode'],
|
||||
'targets': [t.lower() for t in q['targets']]
|
||||
})
|
||||
|
||||
results = await anyio.to_thread.run_sync(
|
||||
search_md.run_search,
|
||||
root_input.value,
|
||||
'global',
|
||||
'',
|
||||
'exact',
|
||||
None,
|
||||
or_queries_param,
|
||||
expand_chk.value
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
status.set_text(f'Search failed: {e}')
|
||||
ui.notify(f'Search error: {e}', color='negative')
|
||||
finally:
|
||||
search_btn.set_enabled(True)
|
||||
table.rows = results
|
||||
table.selected.clear()
|
||||
|
||||
# write output if requested
|
||||
if out_input.value:
|
||||
try:
|
||||
await anyio.to_thread.run_sync(search_md.write_output, results, out_input.value, fmt_select.value)
|
||||
status.set_text(f'Found {len(results)} matches. Wrote {out_input.value}')
|
||||
except Exception as e:
|
||||
status.set_text(f'Found {len(results)} matches. Failed to write output: {e}')
|
||||
else:
|
||||
status.set_text(f'Found {len(results)} matches (not written).')
|
||||
|
||||
except Exception as e:
|
||||
status.set_text(f'Search failed: {e}')
|
||||
ui.notify(f'Search error: {e}', color='negative')
|
||||
finally:
|
||||
search_btn.set_enabled(True)
|
||||
|
||||
async def copy_selected():
|
||||
if not table.selected:
|
||||
ui.notify('No files selected', color='warning')
|
||||
return
|
||||
with table.client:
|
||||
if not table.selected:
|
||||
ui.notify('No files selected', color='warning')
|
||||
return
|
||||
|
||||
dest = copy_input.value.strip() or 'out'
|
||||
status.set_text(f'Copying {len(table.selected)} files to {dest}...')
|
||||
|
||||
dest = copy_input.value.strip() or 'out'
|
||||
status.set_text(f'Copying {len(table.selected)} files to {dest}...')
|
||||
|
||||
try:
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
|
||||
if copy_clear.value:
|
||||
def clear_dst():
|
||||
for name in os.listdir(dest):
|
||||
path = os.path.join(dest, name)
|
||||
try:
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
|
||||
if copy_clear.value:
|
||||
def clear_dst():
|
||||
for name in os.listdir(dest):
|
||||
path = os.path.join(dest, name)
|
||||
try:
|
||||
if os.path.isfile(path) or os.path.islink(path):
|
||||
os.remove(path)
|
||||
elif os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
except Exception:
|
||||
pass
|
||||
await anyio.to_thread.run_sync(clear_dst)
|
||||
|
||||
def copy_files():
|
||||
for r in table.selected:
|
||||
try:
|
||||
if os.path.isfile(path) or os.path.islink(path):
|
||||
os.remove(path)
|
||||
elif os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
dst = os.path.join(dest, os.path.basename(r['path']))
|
||||
with open(r['path'], 'rb') as srcf, open(dst, 'wb') as dstf:
|
||||
dstf.write(srcf.read())
|
||||
except Exception:
|
||||
pass
|
||||
await anyio.to_thread.run_sync(clear_dst)
|
||||
|
||||
def copy_files():
|
||||
for r in table.selected:
|
||||
try:
|
||||
dst = os.path.join(dest, os.path.basename(r['path']))
|
||||
with open(r['path'], 'rb') as srcf, open(dst, 'wb') as dstf:
|
||||
dstf.write(srcf.read())
|
||||
except Exception:
|
||||
pass
|
||||
await anyio.to_thread.run_sync(copy_files)
|
||||
status.set_text(status.text + f' Copied {len(table.selected)} files to {dest}')
|
||||
ui.notify('Files copied successfully', color='positive')
|
||||
except Exception as e:
|
||||
status.set_text(status.text + f' Copy failed: {e}')
|
||||
ui.notify(f'Copy failed: {e}', color='negative')
|
||||
await anyio.to_thread.run_sync(copy_files)
|
||||
status.set_text(status.text + f' Copied {len(table.selected)} files to {dest}')
|
||||
ui.notify('Files copied successfully', color='positive')
|
||||
except Exception as e:
|
||||
status.set_text(status.text + f' Copy failed: {e}')
|
||||
ui.notify(f'Copy failed: {e}', color='negative')
|
||||
|
||||
async def export_selected():
|
||||
if not table.selected:
|
||||
ui.notify('No records selected', color='warning')
|
||||
return
|
||||
with table.client:
|
||||
if not table.selected:
|
||||
ui.notify('No records selected', color='warning')
|
||||
return
|
||||
|
||||
outp = out_input.value or 'results.json'
|
||||
status.set_text(f'Exporting {len(table.selected)} records to {outp}...')
|
||||
|
||||
outp = out_input.value or 'results.json'
|
||||
status.set_text(f'Exporting {len(table.selected)} records to {outp}...')
|
||||
|
||||
try:
|
||||
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, fmt_select.value)
|
||||
status.set_text(f'Exported {len(table.selected)} records to {outp}.')
|
||||
ui.notify('Export successful', color='positive')
|
||||
except Exception as e:
|
||||
status.set_text(f'Export failed: {e}')
|
||||
ui.notify(f'Export failed: {e}', color='negative')
|
||||
try:
|
||||
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, fmt_select.value)
|
||||
status.set_text(f'Exported {len(table.selected)} records to {outp}.')
|
||||
ui.notify('Export successful', color='positive')
|
||||
except Exception as e:
|
||||
status.set_text(f'Export failed: {e}')
|
||||
ui.notify(f'Export failed: {e}', color='negative')
|
||||
|
||||
def open_output_dir() -> None:
|
||||
dest = copy_input.value.strip() if copy_input.value else ''
|
||||
@@ -867,6 +984,18 @@ def linkify_references(content: str, root: str = 'docs_md/articles') -> str:
|
||||
return content
|
||||
|
||||
|
||||
def is_doc_cached(docid: str) -> bool:
|
||||
fm, _, _ = search_md.get_doc_by_id('docs_md/articles', docid)
|
||||
return fm is not None
|
||||
|
||||
|
||||
def make_online_url(docid: str, title: str) -> str:
|
||||
slug = title.lower()
|
||||
slug = re.sub(r'[^a-z0-9]+', '-', slug).strip('-')
|
||||
return f"https://app.statdx.com/document/{slug}/{docid}"
|
||||
|
||||
|
||||
|
||||
@ui.page('/document/{identifier}')
|
||||
def render_doc_page(identifier: str):
|
||||
ui.add_head_html('''
|
||||
|
||||
Reference in New Issue
Block a user