.
This commit is contained in:
+63
-1
@@ -693,6 +693,68 @@ def write_output(results: List[Dict[str, Any]], out_path: str, fmt: str = 'json'
|
||||
w.writerow(['path', 'title', 'docid', 'breadcrumbs', 'authors', 'pageKeywords'])
|
||||
for r in results:
|
||||
w.writerow([r['path'], r['title'], r.get('docid') or '', json.dumps(r.get('breadcrumbs') or []), json.dumps(r.get('authors') or []), r.get('pageKeywords') or ''])
|
||||
elif fmt == 'combined_json':
|
||||
combined_docs = []
|
||||
for r in results:
|
||||
path = r.get('path')
|
||||
title = r.get('title', 'Untitled')
|
||||
docid = r.get('docid', '')
|
||||
|
||||
content = ""
|
||||
if path and os.path.exists(path):
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except Exception as e:
|
||||
content = f"Error reading file {path}: {e}"
|
||||
else:
|
||||
content = f"File not found: {path}"
|
||||
|
||||
# Format using XML tags as a clear separator that LLMs understand
|
||||
doc_str = f'<article id="{docid}" title="{title}">\n{content.strip()}\n</article>'
|
||||
combined_docs.append(doc_str)
|
||||
|
||||
combined_document = "\n\n".join(combined_docs)
|
||||
|
||||
output_data = {
|
||||
"combined_document": combined_document,
|
||||
"articles": [
|
||||
{
|
||||
"title": r.get('title'),
|
||||
"docid": r.get('docid'),
|
||||
"path": r.get('path')
|
||||
}
|
||||
for r in results
|
||||
]
|
||||
}
|
||||
|
||||
with open(out_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(output_data, f, indent=2, ensure_ascii=False)
|
||||
elif fmt == 'combined_md':
|
||||
combined_docs = []
|
||||
for r in results:
|
||||
path = r.get('path')
|
||||
title = r.get('title', 'Untitled')
|
||||
docid = r.get('docid', '')
|
||||
|
||||
content = ""
|
||||
if path and os.path.exists(path):
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except Exception as e:
|
||||
content = f"Error reading file {path}: {e}"
|
||||
else:
|
||||
content = f"File not found: {path}"
|
||||
|
||||
# Format using XML tags as a clear separator that LLMs understand
|
||||
doc_str = f'<article id="{docid}" title="{title}">\n{content.strip()}\n</article>'
|
||||
combined_docs.append(doc_str)
|
||||
|
||||
combined_document = "\n\n".join(combined_docs)
|
||||
|
||||
with open(out_path, 'w', encoding='utf-8') as f:
|
||||
f.write(combined_document)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
@@ -701,7 +763,7 @@ def main(argv=None):
|
||||
p.add_argument('--query', required=True, help='Query. Can be key:value or a raw search string')
|
||||
p.add_argument('--mode', choices=['exact', 'fuzzy', 'stemming'], default='exact', help='Search matching mode')
|
||||
p.add_argument('--out', default='results.json', help='Output file')
|
||||
p.add_argument('--format', choices=['json','csv'], default='json')
|
||||
p.add_argument('--format', choices=['json', 'csv', 'combined_json', 'combined_md'], default='json')
|
||||
p.add_argument('--copy-to', help='Optional: copy matched markdown files to this dir')
|
||||
p.add_argument("--copy-to-clear", default=True, action="store_true", help="Overwrite files when copying to --copy-to")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
+57
-1
@@ -365,7 +365,7 @@ def search_page() -> None: # build UI
|
||||
with ui.row().classes('w-full gap-4 items-start p-4'):
|
||||
root_input = ui.input(value='docs_md/articles', label='Root directory').classes('w-64').props('outlined dense')
|
||||
key_select = ui.select(['global', 'breadcrumbs', 'authors', 'pageKeywords', 'category', 'title', 'enhancedTitle', 'type', 'content'], value='global', label='Query key').classes('w-64').props('outlined dense')
|
||||
fmt_select = ui.select(['json', 'csv'], value='json', label='Output format').classes('w-40').props('outlined dense')
|
||||
fmt_select = ui.select(['json', 'csv', 'combined_json', 'combined_md'], value='json', label='Output format').classes('w-40').props('outlined dense')
|
||||
out_input = ui.input(value='results.json', label='Output file (server-side)').classes('w-64').props('outlined dense')
|
||||
copy_input = ui.input(value='out', label='Copy matched files to (optional)').classes('w-64').props('outlined dense')
|
||||
copy_clear = ui.checkbox('Clear destination before copy', value=False).classes('self-center text-slate-300')
|
||||
@@ -379,6 +379,8 @@ def search_page() -> None: # build UI
|
||||
with action_bar:
|
||||
with ui.row().classes('gap-2'):
|
||||
ui.button('Copy Selected Files', icon='content_copy', on_click=lambda: asyncio.create_task(copy_selected())).classes('bg-indigo-600 hover:bg-indigo-700 text-white font-medium px-4')
|
||||
ui.button('Export Combined JSON', icon='library_books', on_click=lambda: asyncio.create_task(export_combined_selected())).classes('bg-teal-600 hover:bg-teal-700 text-white font-medium px-4')
|
||||
ui.button('Export Combined MD', icon='article', on_click=lambda: asyncio.create_task(export_combined_md())).classes('bg-cyan-600 hover:bg-cyan-700 text-white font-medium px-4')
|
||||
ui.button('Export Selected', icon='file_download', on_click=lambda: asyncio.create_task(export_selected())).classes('bg-emerald-600 hover:bg-emerald-700 text-white font-medium px-4')
|
||||
ui.button('Deselect All', icon='close', on_click=lambda: table.selected.clear()).classes('bg-slate-700 hover:bg-slate-600 text-white font-medium px-4')
|
||||
|
||||
@@ -910,6 +912,8 @@ def search_page() -> None: # build UI
|
||||
|
||||
table.rows = results
|
||||
table.selected.clear()
|
||||
table.selected.extend(results)
|
||||
table.update()
|
||||
|
||||
# write output if requested
|
||||
if out_input.value:
|
||||
@@ -967,6 +971,58 @@ def search_page() -> None: # build UI
|
||||
status.set_text(status.text + f' Copy failed: {e}')
|
||||
ui.notify(f'Copy failed: {e}', color='negative')
|
||||
|
||||
async def export_combined_selected():
|
||||
with table.client:
|
||||
if not table.selected:
|
||||
ui.notify('No records selected', color='warning')
|
||||
return
|
||||
|
||||
dest_dir = copy_input.value.strip() or 'out'
|
||||
filename = os.path.basename(out_input.value or 'results.json')
|
||||
if not filename.lower().endswith('.json'):
|
||||
filename = os.path.splitext(filename)[0] + '.json'
|
||||
|
||||
outp = os.path.join(dest_dir, filename)
|
||||
status.set_text(f'Exporting {len(table.selected)} records combined to {outp}...')
|
||||
|
||||
try:
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, 'combined_json')
|
||||
status.set_text(f'Exported {len(table.selected)} records combined to {outp}.')
|
||||
ui.notify('Combined export successful', color='positive')
|
||||
except Exception as e:
|
||||
status.set_text(f'Combined export failed: {e}')
|
||||
ui.notify(f'Combined export failed: {e}', color='negative')
|
||||
|
||||
async def export_combined_md():
|
||||
with table.client:
|
||||
if not table.selected:
|
||||
ui.notify('No records selected', color='warning')
|
||||
return
|
||||
|
||||
dest_dir = copy_input.value.strip() or 'out'
|
||||
|
||||
active_queries = [q for q in queries if q['qval'].strip()]
|
||||
if active_queries:
|
||||
qvals = [q['qval'].strip() for q in active_queries]
|
||||
slug = "_".join([re.sub(r'[^a-zA-Z0-9_-]', '_', qv.lower()) for qv in qvals])
|
||||
slug = re.sub(r'_+', '_', slug).strip('_')
|
||||
filename = f"{slug}_combined.md"
|
||||
else:
|
||||
filename = "combined.md"
|
||||
|
||||
outp = os.path.join(dest_dir, filename)
|
||||
status.set_text(f'Exporting {len(table.selected)} records combined to {outp}...')
|
||||
|
||||
try:
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
await anyio.to_thread.run_sync(search_md.write_output, table.selected, outp, 'combined_md')
|
||||
status.set_text(f'Exported {len(table.selected)} records combined to {outp}.')
|
||||
ui.notify('Combined markdown export successful', color='positive')
|
||||
except Exception as e:
|
||||
status.set_text(f'Combined markdown export failed: {e}')
|
||||
ui.notify(f'Combined markdown export failed: {e}', color='negative')
|
||||
|
||||
async def export_selected():
|
||||
with table.client:
|
||||
if not table.selected:
|
||||
|
||||
Reference in New Issue
Block a user