This commit is contained in:
Ross
2026-07-15 21:36:23 +01:00
parent 2c3adb081f
commit c980cc8faa
1036 changed files with 20170 additions and 454 deletions
+63 -1
View File
@@ -693,6 +693,68 @@ def write_output(results: List[Dict[str, Any]], out_path: str, fmt: str = 'json'
w.writerow(['path', 'title', 'docid', 'breadcrumbs', 'authors', 'pageKeywords'])
for r in results:
w.writerow([r['path'], r['title'], r.get('docid') or '', json.dumps(r.get('breadcrumbs') or []), json.dumps(r.get('authors') or []), r.get('pageKeywords') or ''])
elif fmt == 'combined_json':
combined_docs = []
for r in results:
path = r.get('path')
title = r.get('title', 'Untitled')
docid = r.get('docid', '')
content = ""
if path and os.path.exists(path):
try:
with open(path, 'r', encoding='utf-8') as f:
content = f.read()
except Exception as e:
content = f"Error reading file {path}: {e}"
else:
content = f"File not found: {path}"
# Format using XML tags as a clear separator that LLMs understand
doc_str = f'<article id="{docid}" title="{title}">\n{content.strip()}\n</article>'
combined_docs.append(doc_str)
combined_document = "\n\n".join(combined_docs)
output_data = {
"combined_document": combined_document,
"articles": [
{
"title": r.get('title'),
"docid": r.get('docid'),
"path": r.get('path')
}
for r in results
]
}
with open(out_path, 'w', encoding='utf-8') as f:
json.dump(output_data, f, indent=2, ensure_ascii=False)
elif fmt == 'combined_md':
combined_docs = []
for r in results:
path = r.get('path')
title = r.get('title', 'Untitled')
docid = r.get('docid', '')
content = ""
if path and os.path.exists(path):
try:
with open(path, 'r', encoding='utf-8') as f:
content = f.read()
except Exception as e:
content = f"Error reading file {path}: {e}"
else:
content = f"File not found: {path}"
# Format using XML tags as a clear separator that LLMs understand
doc_str = f'<article id="{docid}" title="{title}">\n{content.strip()}\n</article>'
combined_docs.append(doc_str)
combined_document = "\n\n".join(combined_docs)
with open(out_path, 'w', encoding='utf-8') as f:
f.write(combined_document)
def main(argv=None):
@@ -701,7 +763,7 @@ def main(argv=None):
p.add_argument('--query', required=True, help='Query. Can be key:value or a raw search string')
p.add_argument('--mode', choices=['exact', 'fuzzy', 'stemming'], default='exact', help='Search matching mode')
p.add_argument('--out', default='results.json', help='Output file')
p.add_argument('--format', choices=['json','csv'], default='json')
p.add_argument('--format', choices=['json', 'csv', 'combined_json', 'combined_md'], default='json')
p.add_argument('--copy-to', help='Optional: copy matched markdown files to this dir')
p.add_argument("--copy-to-clear", default=True, action="store_true", help="Overwrite files when copying to --copy-to")
args = p.parse_args(argv)
+57 -1
View File
@@ -365,7 +365,7 @@ def search_page() -> None: # build UI
with ui.row().classes('w-full gap-4 items-start p-4'):
root_input = ui.input(value='docs_md/articles', label='Root directory').classes('w-64').props('outlined dense')
key_select = ui.select(['global', 'breadcrumbs', 'authors', 'pageKeywords', 'category', 'title', 'enhancedTitle', 'type', 'content'], value='global', label='Query key').classes('w-64').props('outlined dense')
fmt_select = ui.select(['json', 'csv'], value='json', label='Output format').classes('w-40').props('outlined dense')
fmt_select = ui.select(['json', 'csv', 'combined_json', 'combined_md'], value='json', label='Output format').classes('w-40').props('outlined dense')
out_input = ui.input(value='results.json', label='Output file (server-side)').classes('w-64').props('outlined dense')
copy_input = ui.input(value='out', label='Copy matched files to (optional)').classes('w-64').props('outlined dense')
copy_clear = ui.checkbox('Clear destination before copy', value=False).classes('self-center text-slate-300')
@@ -379,6 +379,8 @@ def search_page() -> None: # build UI
with action_bar:
with ui.row().classes('gap-2'):
ui.button('Copy Selected Files', icon='content_copy', on_click=lambda: asyncio.create_task(copy_selected())).classes('bg-indigo-600 hover:bg-indigo-700 text-white font-medium px-4')
ui.button('Export Combined JSON', icon='library_books', on_click=lambda: asyncio.create_task(export_combined_selected())).classes('bg-teal-600 hover:bg-teal-700 text-white font-medium px-4')
ui.button('Export Combined MD', icon='article', on_click=lambda: asyncio.create_task(export_combined_md())).classes('bg-cyan-600 hover:bg-cyan-700 text-white font-medium px-4')
ui.button('Export Selected', icon='file_download', on_click=lambda: asyncio.create_task(export_selected())).classes('bg-emerald-600 hover:bg-emerald-700 text-white font-medium px-4')
ui.button('Deselect All', icon='close', on_click=lambda: table.selected.clear()).classes('bg-slate-700 hover:bg-slate-600 text-white font-medium px-4')
@@ -910,6 +912,8 @@ def search_page() -> None: # build UI
table.rows = results
table.selected.clear()
table.selected.extend(results)
table.update()
# write output if requested
if out_input.value:
@@ -967,6 +971,58 @@ def search_page() -> None: # build UI
status.set_text(status.text + f' Copy failed: {e}')
ui.notify(f'Copy failed: {e}', color='negative')
async def export_combined_selected():
with table.client:
if not table.selected:
ui.notify('No records selected', color='warning')
return
dest_dir = copy_input.value.strip() or 'out'
filename = os.path.basename(out_input.value or 'results.json')
if not filename.lower().endswith('.json'):
filename = os.path.splitext(filename)[0] + '.json'
outp = os.path.join(dest_dir, filename)
status.set_text(f'Exporting {len(table.selected)} records combined to {outp}...')
try:
os.makedirs(dest_dir, exist_ok=True)
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, 'combined_json')
status.set_text(f'Exported {len(table.selected)} records combined to {outp}.')
ui.notify('Combined export successful', color='positive')
except Exception as e:
status.set_text(f'Combined export failed: {e}')
ui.notify(f'Combined export failed: {e}', color='negative')
async def export_combined_md():
with table.client:
if not table.selected:
ui.notify('No records selected', color='warning')
return
dest_dir = copy_input.value.strip() or 'out'
active_queries = [q for q in queries if q['qval'].strip()]
if active_queries:
qvals = [q['qval'].strip() for q in active_queries]
slug = "_".join([re.sub(r'[^a-zA-Z0-9_-]', '_', qv.lower()) for qv in qvals])
slug = re.sub(r'_+', '_', slug).strip('_')
filename = f"{slug}_combined.md"
else:
filename = "combined.md"
outp = os.path.join(dest_dir, filename)
status.set_text(f'Exporting {len(table.selected)} records combined to {outp}...')
try:
os.makedirs(dest_dir, exist_ok=True)
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, 'combined_md')
status.set_text(f'Exported {len(table.selected)} records combined to {outp}.')
ui.notify('Combined markdown export successful', color='positive')
except Exception as e:
status.set_text(f'Combined markdown export failed: {e}')
ui.notify(f'Combined markdown export failed: {e}', color='negative')
async def export_selected():
with table.client:
if not table.selected: