will it compile (on windows)

nice.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
+import asyncio
 from collections import defaultdict
 from pathlib import Path
 
 from bs4 import BeautifulSoup
 from local_file_picker import local_file_picker
 
-from nicegui import ui, app, html, run
+from nicegui import ui, app, html, run, Client
 
 from datetime import datetime
 
@@ -25,6 +26,8 @@ from multiprocessing import Manager, Queue, freeze_support
 
 import watchfiles
 
+from functools import partial
+
 
 from typing import Optional
 
@@ -53,6 +56,8 @@ from starlette.middleware.base import BaseHTTPMiddleware
 #    ui.html("<h2>Uploader tool</h2>")
 #    ui.button('Choose file', on_click=pick_file, icon='folder')
 
+SOCKET_PATH = 'tcp://localhost:9975'
+
 EXPORT_PATH = Path("./test/export")
 PROCESS_PATH = Path("./test/to_process")
 ANON_PATH = Path("./test/anon")
@@ -62,38 +67,171 @@ PROCESS_PATH.mkdir(exist_ok=True, parents=True)
 EXPORT_PATH.mkdir(exist_ok=True, parents=True)
 
 BASE_SITE_URL = "http://localhost:8000"
+BASE_SITE_URL = "https://www.penracourses.org.uk"
 
 LOGIN_URL = f"{BASE_SITE_URL}/accounts/login/"
 
 LOGIN_SUCCESS = False
 
+LOGGED_IN_USER = "None"
+
+
+REFRESH_TRIGGERED = False
+
+DELETE_FILES_ON_UPLOAD = False
+
 rqst = None
 
 
-use_zmq = False
+use_zmq = True
 loaded_files: dict = {}
+loaded_series = defaultdict(list)
+loaded_series_data = {}
 uploaded_files: dict = {}
 duplicate_series = set()
 
+
 if use_zmq:
     context = zmq.asyncio.Context()
     socket = context.socket(zmq.PULL)
     #socket.bind('tcp://localhost:5575')
     try:
-        socket.bind('tcp://localhost:9975')
+        socket.bind(SOCKET_PATH)
     except zmq.error.ZMQError as e:
         logger.debug("send reload")
         socket = context.socket(zmq.PUSH)
-        socket.connect('tcp://localhost:5575')
+        socket.connect(SOCKET_PATH)
         socket.send_string("loaded")
         sys.exit(0)
+        #app.shutdown()
 
     poller = zmq.asyncio.Poller()
     poller.register(socket, zmq.POLLIN)
 
+    async def read_loop() -> None:
+        while not app.is_stopped:
+            events = await poller.poll()
+            if socket in dict(events):
+                data = await socket.recv()
+                msg = data.decode()
+
+                logger.debug(f'Received msg {msg}')
+
+                match msg:
+                    case "loaded":
+                        for client in Client.instances.values():
+                            if not client.has_socket_connection:
+                                continue
+                            with client:
+                                ui.notify("Export done", color='positive')
+                        
+                        global REFRESH_TRIGGERED
+                        REFRESH_TRIGGERED = True
+                        logger.debug("loaded")
+
+                        #with updates:
+                        #    ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}")
+                        #logger.debug("loaded")
+                        #load_files_start()
+
+                        
+                    case _:
+                        logger.error("invalid message")
 
 anonymizer = Anonymizer(ANON_PATH)
 
+async def upload_files_start(progress, upload_queue):
+    global uploaded_files, duplicate_series
+    progress.visible = True
+    new_uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst)
+    progress.visible = False
+
+    uploaded_files.update(new_uploaded_files)
+
+    ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive')
+    ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning')
+    ui.notify(f"Failed {len(failed)} files", color='negative')
+
+    if DELETE_FILES_ON_UPLOAD:
+        clear_anonymized_files()
+
+
+    loaded_series_ui.refresh()
+    loaded_files_ui.refresh()
+    uploaded_files_ui.refresh()
+
+async def load_files_start(progress_bar, queue):
+    global loaded_files, loaded_series, loaded_series_data
+    progress_bar.visible = True
+    new_loaded_files, new_loaded_series, new_loaded_series_data = await run.cpu_bound(load_files, queue)
+
+    loaded_files.update(new_loaded_files)
+    loaded_series.update(new_loaded_series)
+    loaded_series_data.update(new_loaded_series_data)
+
+    progress_bar.visible = False
+    #load_series_view(loaded_series, loaded_series_data)
+    loaded_series_ui.refresh()
+    loaded_files_ui.refresh()
+
+async def reload_anonymized_start(progress_bar, queue):
+    global loaded_files, loaded_series, loaded_series_data
+    progress_bar.visible = True
+    loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue)
+    progress_bar.visible = False
+    #load_series_view(loaded_series, loaded_series_data)
+    loaded_series_ui.refresh()
+    loaded_files_ui.refresh()
+
+@ui.refreshable
+def user_info_ui():
+    global LOGGED_IN_USER
+    user_label = ui.label(f"User: {LOGGED_IN_USER}")
+    #user_label.bind_visibility_from(globals(), "LOGIN_SUCCESS")
+
+@ui.refreshable
+def loaded_series_ui() -> None:
+    global loaded_series, loaded_series_data
+
+    series_title = ui.label('Series to upload').classes("text-h3")
+    series_title.visible = False
+
+    series = ui.row()
+
+    #def load_series_view(loaded_series, loaded_series_data):
+    logger.debug("load series view")
+    logger.debug(loaded_series)
+    for key in loaded_series:
+        logger.debug(f"load series: {key}")
+        with series:
+            with ui.card():
+                ui.label(loaded_series_data[key][0])
+                ui.label(loaded_series_data[key][1])
+                ui.label(f"Images: {len(loaded_series[key])}")
+
+    if loaded_series:
+        series_title.visible = True
+
+@ui.refreshable
+def loaded_files_ui() -> None:
+    with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'):
+        ui.label(f"Items: {len(loaded_files)}")
+        with ui.list():
+            for file, data in loaded_files.items():
+                ui.item(f"{file} - {data[0]} - {data[1]}")
+
+
+@ui.refreshable
+def uploaded_files_ui() -> None:
+    with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'):
+        ui.label(f"Items: {len(uploaded_files)}")
+        with ui.list():
+            for series in duplicate_series:
+                ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500")
+        with ui.list():
+            for file, data in uploaded_files.items():
+                ui.item(f"{file} - {data}")
+
 def load_files(q: Queue):
     loaded_files: dict = {}
     loaded_series = defaultdict(list)
@@ -126,7 +264,6 @@ def load_files(q: Queue):
         #time.sleep(0.01)
 
     return loaded_files, loaded_series, loaded_series_data 
-    #load_series_view()
 
 
 
@@ -163,7 +300,14 @@ def reload_anonymized(q: Queue):
 
     return loaded_files, loaded_series, loaded_series_data 
 
-    #load_series_view()
+def clear_anonymized_files():
+    for file in ANON_PATH.iterdir():
+        if file.is_file():
+            file.unlink()
+
+    logger.debug("Deleted all files in ANON_PATH")
+
+    return
 
 def upload_files(q, rqst):
     #global rqst#, uploaded_files
@@ -217,7 +361,6 @@ def upload_files(q, rqst):
         # try to upload the files
 
         for i in range(3):
-            logger.error(f"i: {i}")
             resp = upload_files_(files)
             if resp.status_code == 200:
                 upload_file_list.extend(resp.json()["uploaded"])
@@ -227,6 +370,7 @@ def upload_files(q, rqst):
 
                 break
 
+            logger.error(f"i: {i}")
             logger.debug(f"n: {n} fail (attempt {i})")
 
         q.put_nowait((n + 1) / to_process_len)
@@ -267,7 +411,7 @@ def login() -> Optional[RedirectResponse]:
         user = username.value
         pw = password.value
 
-        global rqst, LOGIN_SUCCESS
+        global rqst, LOGIN_SUCCESS, LOGGED_IN_USER
         rqst = requests.session()
         try: 
             rsp = rqst.get(LOGIN_URL)
@@ -299,6 +443,8 @@ def login() -> Optional[RedirectResponse]:
             # cookies=cookies
         )
 
+        print(rsp.content)
+
         soup = BeautifulSoup(rsp.content, "html.parser")
 
         if soup.find("button") and soup.find("button").find(string="Login"):
@@ -307,8 +453,10 @@ def login() -> Optional[RedirectResponse]:
         else:
             print("login success")
             LOGIN_SUCCESS = True
-            user_label.clear()
+            LOGGED_IN_USER = user
-            user_label.text = f"User: {user}"
+            user_info_ui.refresh()
+            #user_label.clear()
+            #user_label.text = f"User: {user}"
         # print(rqst.headers)
 
         rqst.headers["X-CSRFToken"] = token
@@ -345,89 +493,27 @@ def login() -> Optional[RedirectResponse]:
 
 @ui.page('/')
 def main_page():
-    def load_series_view(loaded_series, loaded_series_data):
-        logger.debug("load series view")
-        logger.debug(loaded_series)
-        for key in loaded_series:
-            logger.debug(f"load series: {key}")
-            with series:
-                with ui.card():
-                    ui.label(loaded_series_data[key][0])
-                    ui.label(loaded_series_data[key][1])
-                    ui.label(f"Images: {len(loaded_series[key])}")
 
-        series_title.visible = True
-
-    if use_zmq:
-        async def read_loop() -> None:
-            while not app.is_stopped:
-                events = await poller.poll()
-                if socket in dict(events):
-                    data = await socket.recv()
-                    msg = data.decode()
-
-                    logger.debug(f'Received msg {msg}')
-
-                    match msg:
-                        case "loaded":
-                            with updates:
-                                ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}")
-                            logger.debug("loaded")
-                            load_files_start()
-
-                            
-                        case _:
-                            logger.error("invalid message")
-        app.on_startup(read_loop)
 
     dark = ui.dark_mode()
     dark.enable()
     ui.page_title('Uploader tool')
     ui.label('Uploader tool').classes("text-h2")
-    user_label = ui.label("User: None")
+
+    user_info_ui()
 
     login_button = ui.button("LOGIN", on_click=lambda: ui.navigate.to("/login"))
     login_button.bind_visibility_from(globals(), "LOGIN_SUCCESS", backward=lambda v: not v)
 
     updates = ui.list()
 
-    series_title = ui.label('Series to upload').classes("text-h3")
+    loaded_series_ui()
-    series_title.visible = False
-
-    series = ui.row()
 
 
 
-    async def load_files_start():
-        global loaded_files
-        anon_progress.visible = True
-        loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(load_files, queue)
-        anon_progress.visible = False
-        load_series_view(loaded_series, loaded_series_data)
-        loaded_files_ui.refresh()
 
-    async def reload_anonymized_start():
-        global loaded_files
-        anon_progress.visible = True
-        loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue)
-        anon_progress.visible = False
-        load_series_view(loaded_series, loaded_series_data)
-        loaded_files_ui.refresh()
 
-    async def upload_files_start():
-        global uploaded_files, duplicate_series
-        upload_progress.visible = True
-        uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst)
-        upload_progress.visible = False
 
-        ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive')
-        ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning')
-        ui.notify(f"Failed {len(failed)} files", color='negative')
-
-        uploaded_files_ui.refresh()
-
-    upload_button = ui.button("Upload", on_click=upload_files_start)
-    upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS")
 #    typer_app()
     # Create a queue to communicate with the heavy computation process
     queue = Manager().Queue()
@@ -439,9 +525,6 @@ def main_page():
     # Update the progress bar on the main process
     ui.timer(0.1, callback=lambda: upload_progressbar.set_value(upload_queue.get() if not upload_queue.empty() else upload_progressbar.value))
 
-    with ui.expansion('Extra', icon='build').classes('w-full'):
-        ui.button("Load files", on_click=load_files_start)
-        ui.button("Reload anon", on_click=reload_anonymized_start)
 
 
 
@@ -452,6 +535,9 @@ def main_page():
         upload_progressbar = ui.linear_progress(value=0).props('instant-feedback')
     upload_progress.visible = False
 
+    upload_button = ui.button("Upload", on_click=partial(upload_files_start, upload_progress, upload_queue))
+    upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS")
+
     anon_progress = ui.row().classes("w-full place-content-center bg-blue-900")
 
     with anon_progress:
@@ -459,24 +545,9 @@ def main_page():
         progressbar = ui.linear_progress(value=0).props('instant-feedback')
     anon_progress.visible = False
 
-    @ui.refreshable
+    with ui.expansion('Extra', icon='build').classes('w-full'):
-    def loaded_files_ui() -> None:
+        ui.button("Load files", on_click=partial(load_files_start, anon_progress, queue))
-        with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'):
+        ui.button("Reload anon", on_click=partial(reload_anonymized_start, anon_progress, queue))
-            ui.label(f"Items: {len(loaded_files)}")
-            with ui.list():
-                for file, data in loaded_files.items():
-                    ui.item(f"{file} - {data[0]} - {data[1]}")
-
-    @ui.refreshable
-    def uploaded_files_ui() -> None:
-        with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'):
-            ui.label(f"Items: {len(uploaded_files)}")
-            with ui.list():
-                for series in duplicate_series:
-                    ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500")
-            with ui.list():
-                for file, data in uploaded_files.items():
-                    ui.item(f"{file} - {data}")
 
     with ui.expansion('File status', icon='view_list').classes('w-full'):
         loaded_files_ui()
@@ -490,12 +561,28 @@ def main_page():
         ui.label(f"Export path: {EXPORT_PATH}")
         ui.label(f"Process path: {PROCESS_PATH}")
         ui.label(f"Anon path: {ANON_PATH}")
+        ui.button("Clear anon path", on_click=clear_anonymized_files).tooltip("Delete all files in ANON_PATH")
 
 
     
 
+    ui.notify("Welcome to the uploader tool", color='positive')
+
+    async def watch_for_refresh():
+        global REFRESH_TRIGGERED
+        while not app.is_stopped:
+            if REFRESH_TRIGGERED:
+                REFRESH_TRIGGERED = False
+                await load_files_start(anon_progress, queue)
+            await asyncio.sleep(1)
+
+    asyncio.create_task(watch_for_refresh())
+
 if __name__ in {"__main__", "__mp_main__"}:
     freeze_support()
 
-    ui.run(storage_secret="123456", reload=False)
+    if use_zmq:
+        app.on_startup(read_loop)
+
+    ui.run(storage_secret="123456", reload=False, show=False)