diff --git a/nice.py b/nice.py index dd7e233..5f9b495 100644 --- a/nice.py +++ b/nice.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 +import asyncio from collections import defaultdict from pathlib import Path from bs4 import BeautifulSoup from local_file_picker import local_file_picker -from nicegui import ui, app, html, run +from nicegui import ui, app, html, run, Client from datetime import datetime @@ -25,6 +26,8 @@ from multiprocessing import Manager, Queue, freeze_support import watchfiles +from functools import partial + from typing import Optional @@ -53,6 +56,8 @@ from starlette.middleware.base import BaseHTTPMiddleware # ui.html("

Uploader tool

") # ui.button('Choose file', on_click=pick_file, icon='folder') +SOCKET_PATH = 'tcp://localhost:9975' + EXPORT_PATH = Path("./test/export") PROCESS_PATH = Path("./test/to_process") ANON_PATH = Path("./test/anon") @@ -62,38 +67,171 @@ PROCESS_PATH.mkdir(exist_ok=True, parents=True) EXPORT_PATH.mkdir(exist_ok=True, parents=True) BASE_SITE_URL = "http://localhost:8000" +BASE_SITE_URL = "https://www.penracourses.org.uk" LOGIN_URL = f"{BASE_SITE_URL}/accounts/login/" LOGIN_SUCCESS = False +LOGGED_IN_USER = "None" + + +REFRESH_TRIGGERED = False + +DELETE_FILES_ON_UPLOAD = False + rqst = None -use_zmq = False +use_zmq = True loaded_files: dict = {} +loaded_series = defaultdict(list) +loaded_series_data = {} uploaded_files: dict = {} duplicate_series = set() + if use_zmq: context = zmq.asyncio.Context() socket = context.socket(zmq.PULL) #socket.bind('tcp://localhost:5575') try: - socket.bind('tcp://localhost:9975') + socket.bind(SOCKET_PATH) except zmq.error.ZMQError as e: logger.debug("send reload") socket = context.socket(zmq.PUSH) - socket.connect('tcp://localhost:5575') + socket.connect(SOCKET_PATH) socket.send_string("loaded") sys.exit(0) + #app.shutdown() poller = zmq.asyncio.Poller() poller.register(socket, zmq.POLLIN) + async def read_loop() -> None: + while not app.is_stopped: + events = await poller.poll() + if socket in dict(events): + data = await socket.recv() + msg = data.decode() + + logger.debug(f'Received msg {msg}') + + match msg: + case "loaded": + for client in Client.instances.values(): + if not client.has_socket_connection: + continue + with client: + ui.notify("Export done", color='positive') + + global REFRESH_TRIGGERED + REFRESH_TRIGGERED = True + logger.debug("loaded") + + #with updates: + # ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}") + #logger.debug("loaded") + #load_files_start() + + + case _: + logger.error("invalid message") anonymizer = Anonymizer(ANON_PATH) +async def upload_files_start(progress, upload_queue): + global uploaded_files, duplicate_series + progress.visible = True + new_uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst) + progress.visible = False + + uploaded_files.update(new_uploaded_files) + + ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive') + ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning') + ui.notify(f"Failed {len(failed)} files", color='negative') + + if DELETE_FILES_ON_UPLOAD: + clear_anonymized_files() + + + loaded_series_ui.refresh() + loaded_files_ui.refresh() + uploaded_files_ui.refresh() + +async def load_files_start(progress_bar, queue): + global loaded_files, loaded_series, loaded_series_data + progress_bar.visible = True + new_loaded_files, new_loaded_series, new_loaded_series_data = await run.cpu_bound(load_files, queue) + + loaded_files.update(new_loaded_files) + loaded_series.update(new_loaded_series) + loaded_series_data.update(new_loaded_series_data) + + progress_bar.visible = False + #load_series_view(loaded_series, loaded_series_data) + loaded_series_ui.refresh() + loaded_files_ui.refresh() + +async def reload_anonymized_start(progress_bar, queue): + global loaded_files, loaded_series, loaded_series_data + progress_bar.visible = True + loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue) + progress_bar.visible = False + #load_series_view(loaded_series, loaded_series_data) + loaded_series_ui.refresh() + loaded_files_ui.refresh() + +@ui.refreshable +def user_info_ui(): + global LOGGED_IN_USER + user_label = ui.label(f"User: {LOGGED_IN_USER}") + #user_label.bind_visibility_from(globals(), "LOGIN_SUCCESS") + +@ui.refreshable +def loaded_series_ui() -> None: + global loaded_series, loaded_series_data + + series_title = ui.label('Series to upload').classes("text-h3") + series_title.visible = False + + series = ui.row() + + #def load_series_view(loaded_series, loaded_series_data): + logger.debug("load series view") + logger.debug(loaded_series) + for key in loaded_series: + logger.debug(f"load series: {key}") + with series: + with ui.card(): + ui.label(loaded_series_data[key][0]) + ui.label(loaded_series_data[key][1]) + ui.label(f"Images: {len(loaded_series[key])}") + + if loaded_series: + series_title.visible = True + +@ui.refreshable +def loaded_files_ui() -> None: + with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'): + ui.label(f"Items: {len(loaded_files)}") + with ui.list(): + for file, data in loaded_files.items(): + ui.item(f"{file} - {data[0]} - {data[1]}") + + +@ui.refreshable +def uploaded_files_ui() -> None: + with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'): + ui.label(f"Items: {len(uploaded_files)}") + with ui.list(): + for series in duplicate_series: + ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500") + with ui.list(): + for file, data in uploaded_files.items(): + ui.item(f"{file} - {data}") + def load_files(q: Queue): loaded_files: dict = {} loaded_series = defaultdict(list) @@ -126,7 +264,6 @@ def load_files(q: Queue): #time.sleep(0.01) return loaded_files, loaded_series, loaded_series_data - #load_series_view() @@ -163,7 +300,14 @@ def reload_anonymized(q: Queue): return loaded_files, loaded_series, loaded_series_data - #load_series_view() +def clear_anonymized_files(): + for file in ANON_PATH.iterdir(): + if file.is_file(): + file.unlink() + + logger.debug("Deleted all files in ANON_PATH") + + return def upload_files(q, rqst): #global rqst#, uploaded_files @@ -217,7 +361,6 @@ def upload_files(q, rqst): # try to upload the files for i in range(3): - logger.error(f"i: {i}") resp = upload_files_(files) if resp.status_code == 200: upload_file_list.extend(resp.json()["uploaded"]) @@ -227,6 +370,7 @@ def upload_files(q, rqst): break + logger.error(f"i: {i}") logger.debug(f"n: {n} fail (attempt {i})") q.put_nowait((n + 1) / to_process_len) @@ -267,7 +411,7 @@ def login() -> Optional[RedirectResponse]: user = username.value pw = password.value - global rqst, LOGIN_SUCCESS + global rqst, LOGIN_SUCCESS, LOGGED_IN_USER rqst = requests.session() try: rsp = rqst.get(LOGIN_URL) @@ -299,6 +443,8 @@ def login() -> Optional[RedirectResponse]: # cookies=cookies ) + print(rsp.content) + soup = BeautifulSoup(rsp.content, "html.parser") if soup.find("button") and soup.find("button").find(string="Login"): @@ -307,8 +453,10 @@ def login() -> Optional[RedirectResponse]: else: print("login success") LOGIN_SUCCESS = True - user_label.clear() - user_label.text = f"User: {user}" + LOGGED_IN_USER = user + user_info_ui.refresh() + #user_label.clear() + #user_label.text = f"User: {user}" # print(rqst.headers) rqst.headers["X-CSRFToken"] = token @@ -345,89 +493,27 @@ def login() -> Optional[RedirectResponse]: @ui.page('/') def main_page(): - def load_series_view(loaded_series, loaded_series_data): - logger.debug("load series view") - logger.debug(loaded_series) - for key in loaded_series: - logger.debug(f"load series: {key}") - with series: - with ui.card(): - ui.label(loaded_series_data[key][0]) - ui.label(loaded_series_data[key][1]) - ui.label(f"Images: {len(loaded_series[key])}") - series_title.visible = True - - if use_zmq: - async def read_loop() -> None: - while not app.is_stopped: - events = await poller.poll() - if socket in dict(events): - data = await socket.recv() - msg = data.decode() - - logger.debug(f'Received msg {msg}') - - match msg: - case "loaded": - with updates: - ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}") - logger.debug("loaded") - load_files_start() - - - case _: - logger.error("invalid message") - app.on_startup(read_loop) dark = ui.dark_mode() dark.enable() ui.page_title('Uploader tool') ui.label('Uploader tool').classes("text-h2") - user_label = ui.label("User: None") + + user_info_ui() login_button = ui.button("LOGIN", on_click=lambda: ui.navigate.to("/login")) login_button.bind_visibility_from(globals(), "LOGIN_SUCCESS", backward=lambda v: not v) updates = ui.list() - series_title = ui.label('Series to upload').classes("text-h3") - series_title.visible = False - - series = ui.row() + loaded_series_ui() - async def load_files_start(): - global loaded_files - anon_progress.visible = True - loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(load_files, queue) - anon_progress.visible = False - load_series_view(loaded_series, loaded_series_data) - loaded_files_ui.refresh() - async def reload_anonymized_start(): - global loaded_files - anon_progress.visible = True - loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue) - anon_progress.visible = False - load_series_view(loaded_series, loaded_series_data) - loaded_files_ui.refresh() - async def upload_files_start(): - global uploaded_files, duplicate_series - upload_progress.visible = True - uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst) - upload_progress.visible = False - ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive') - ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning') - ui.notify(f"Failed {len(failed)} files", color='negative') - - uploaded_files_ui.refresh() - - upload_button = ui.button("Upload", on_click=upload_files_start) - upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS") # typer_app() # Create a queue to communicate with the heavy computation process queue = Manager().Queue() @@ -439,9 +525,6 @@ def main_page(): # Update the progress bar on the main process ui.timer(0.1, callback=lambda: upload_progressbar.set_value(upload_queue.get() if not upload_queue.empty() else upload_progressbar.value)) - with ui.expansion('Extra', icon='build').classes('w-full'): - ui.button("Load files", on_click=load_files_start) - ui.button("Reload anon", on_click=reload_anonymized_start) @@ -452,6 +535,9 @@ def main_page(): upload_progressbar = ui.linear_progress(value=0).props('instant-feedback') upload_progress.visible = False + upload_button = ui.button("Upload", on_click=partial(upload_files_start, upload_progress, upload_queue)) + upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS") + anon_progress = ui.row().classes("w-full place-content-center bg-blue-900") with anon_progress: @@ -459,24 +545,9 @@ def main_page(): progressbar = ui.linear_progress(value=0).props('instant-feedback') anon_progress.visible = False - @ui.refreshable - def loaded_files_ui() -> None: - with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'): - ui.label(f"Items: {len(loaded_files)}") - with ui.list(): - for file, data in loaded_files.items(): - ui.item(f"{file} - {data[0]} - {data[1]}") - - @ui.refreshable - def uploaded_files_ui() -> None: - with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'): - ui.label(f"Items: {len(uploaded_files)}") - with ui.list(): - for series in duplicate_series: - ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500") - with ui.list(): - for file, data in uploaded_files.items(): - ui.item(f"{file} - {data}") + with ui.expansion('Extra', icon='build').classes('w-full'): + ui.button("Load files", on_click=partial(load_files_start, anon_progress, queue)) + ui.button("Reload anon", on_click=partial(reload_anonymized_start, anon_progress, queue)) with ui.expansion('File status', icon='view_list').classes('w-full'): loaded_files_ui() @@ -490,12 +561,28 @@ def main_page(): ui.label(f"Export path: {EXPORT_PATH}") ui.label(f"Process path: {PROCESS_PATH}") ui.label(f"Anon path: {ANON_PATH}") + ui.button("Clear anon path", on_click=clear_anonymized_files).tooltip("Delete all files in ANON_PATH") + + ui.notify("Welcome to the uploader tool", color='positive') + + async def watch_for_refresh(): + global REFRESH_TRIGGERED + while not app.is_stopped: + if REFRESH_TRIGGERED: + REFRESH_TRIGGERED = False + await load_files_start(anon_progress, queue) + await asyncio.sleep(1) + + asyncio.create_task(watch_for_refresh()) if __name__ in {"__main__", "__mp_main__"}: freeze_support() - ui.run(storage_secret="123456", reload=False) + if use_zmq: + app.on_startup(read_loop) + + ui.run(storage_secret="123456", reload=False, show=False)