ross/uploader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

will it compile (on windows)

Browse Source
This commit is contained in:
Ross
2024-12-02 11:50:13 +00:00
parent 6532801fc1
commit 530f9ed7ea
1 changed files with 184 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nice.py
+184 -97
View File
@@ -1,11 +1,12 @@
#!/usr/bin/env python3
import asyncio
from collections import defaultdict
from pathlib import Path
from bs4 import BeautifulSoup
from local_file_picker import local_file_picker
from nicegui import ui, app, html, run
from nicegui import ui, app, html, run, Client
from datetime import datetime
@@ -25,6 +26,8 @@ from multiprocessing import Manager, Queue, freeze_support
import watchfiles
from functools import partial
from typing import Optional
@@ -53,6 +56,8 @@ from starlette.middleware.base import BaseHTTPMiddleware
# ui.html("<h2>Uploader tool</h2>")
# ui.button('Choose file', on_click=pick_file, icon='folder')
SOCKET_PATH = 'tcp://localhost:9975'
EXPORT_PATH = Path("./test/export")
PROCESS_PATH = Path("./test/to_process")
ANON_PATH = Path("./test/anon")
@@ -62,38 +67,171 @@ PROCESS_PATH.mkdir(exist_ok=True, parents=True)
EXPORT_PATH.mkdir(exist_ok=True, parents=True)
BASE_SITE_URL = "http://localhost:8000"
BASE_SITE_URL = "https://www.penracourses.org.uk"
LOGIN_URL = f"{BASE_SITE_URL}/accounts/login/"
LOGIN_SUCCESS = False
LOGGED_IN_USER = "None"
REFRESH_TRIGGERED = False
DELETE_FILES_ON_UPLOAD = False
rqst = None
use_zmq = False
use_zmq = True
loaded_files: dict = {}
loaded_series = defaultdict(list)
loaded_series_data = {}
uploaded_files: dict = {}
duplicate_series = set()
if use_zmq:
context = zmq.asyncio.Context()
socket = context.socket(zmq.PULL)
#socket.bind('tcp://localhost:5575')
try:
socket.bind('tcp://localhost:9975')
socket.bind(SOCKET_PATH)
except zmq.error.ZMQError as e:
logger.debug("send reload")
socket = context.socket(zmq.PUSH)
socket.connect('tcp://localhost:5575')
socket.connect(SOCKET_PATH)
socket.send_string("loaded")
sys.exit(0)
#app.shutdown()
poller = zmq.asyncio.Poller()
poller.register(socket, zmq.POLLIN)
async def read_loop() -> None:
while not app.is_stopped:
events = await poller.poll()
if socket in dict(events):
data = await socket.recv()
msg = data.decode()
logger.debug(f'Received msg {msg}')
match msg:
case "loaded":
for client in Client.instances.values():
if not client.has_socket_connection:
continue
with client:
ui.notify("Export done", color='positive')
global REFRESH_TRIGGERED
REFRESH_TRIGGERED = True
logger.debug("loaded")
#with updates:
# ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}")
#logger.debug("loaded")
#load_files_start()
case _:
logger.error("invalid message")
anonymizer = Anonymizer(ANON_PATH)
async def upload_files_start(progress, upload_queue):
global uploaded_files, duplicate_series
progress.visible = True
new_uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst)
progress.visible = False
uploaded_files.update(new_uploaded_files)
ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive')
ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning')
ui.notify(f"Failed {len(failed)} files", color='negative')
if DELETE_FILES_ON_UPLOAD:
clear_anonymized_files()
loaded_series_ui.refresh()
loaded_files_ui.refresh()
uploaded_files_ui.refresh()
async def load_files_start(progress_bar, queue):
global loaded_files, loaded_series, loaded_series_data
progress_bar.visible = True
new_loaded_files, new_loaded_series, new_loaded_series_data = await run.cpu_bound(load_files, queue)
loaded_files.update(new_loaded_files)
loaded_series.update(new_loaded_series)
loaded_series_data.update(new_loaded_series_data)
progress_bar.visible = False
#load_series_view(loaded_series, loaded_series_data)
loaded_series_ui.refresh()
loaded_files_ui.refresh()
async def reload_anonymized_start(progress_bar, queue):
global loaded_files, loaded_series, loaded_series_data
progress_bar.visible = True
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue)
progress_bar.visible = False
#load_series_view(loaded_series, loaded_series_data)
loaded_series_ui.refresh()
loaded_files_ui.refresh()
@ui.refreshable
def user_info_ui():
global LOGGED_IN_USER
user_label = ui.label(f"User: {LOGGED_IN_USER}")
#user_label.bind_visibility_from(globals(), "LOGIN_SUCCESS")
@ui.refreshable
def loaded_series_ui() -> None:
global loaded_series, loaded_series_data
series_title = ui.label('Series to upload').classes("text-h3")
series_title.visible = False
series = ui.row()
#def load_series_view(loaded_series, loaded_series_data):
logger.debug("load series view")
logger.debug(loaded_series)
for key in loaded_series:
logger.debug(f"load series: {key}")
with series:
with ui.card():
ui.label(loaded_series_data[key][0])
ui.label(loaded_series_data[key][1])
ui.label(f"Images: {len(loaded_series[key])}")
if loaded_series:
series_title.visible = True
@ui.refreshable
def loaded_files_ui() -> None:
with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'):
ui.label(f"Items: {len(loaded_files)}")
with ui.list():
for file, data in loaded_files.items():
ui.item(f"{file} - {data[0]} - {data[1]}")
@ui.refreshable
def uploaded_files_ui() -> None:
with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'):
ui.label(f"Items: {len(uploaded_files)}")
with ui.list():
for series in duplicate_series:
ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500")
with ui.list():
for file, data in uploaded_files.items():
ui.item(f"{file} - {data}")
def load_files(q: Queue):
loaded_files: dict = {}
loaded_series = defaultdict(list)
@@ -126,7 +264,6 @@ def load_files(q: Queue):
#time.sleep(0.01)
return loaded_files, loaded_series, loaded_series_data
#load_series_view()
@@ -163,7 +300,14 @@ def reload_anonymized(q: Queue):
return loaded_files, loaded_series, loaded_series_data
#load_series_view()
def clear_anonymized_files():
for file in ANON_PATH.iterdir():
if file.is_file():
file.unlink()
logger.debug("Deleted all files in ANON_PATH")
return
def upload_files(q, rqst):
#global rqst#, uploaded_files
@@ -217,7 +361,6 @@ def upload_files(q, rqst):
# try to upload the files
for i in range(3):
logger.error(f"i: {i}")
resp = upload_files_(files)
if resp.status_code == 200:
upload_file_list.extend(resp.json()["uploaded"])
@@ -227,6 +370,7 @@ def upload_files(q, rqst):
break
logger.error(f"i: {i}")
logger.debug(f"n: {n} fail (attempt {i})")
q.put_nowait((n + 1) / to_process_len)
@@ -267,7 +411,7 @@ def login() -> Optional[RedirectResponse]:
user = username.value
pw = password.value
global rqst, LOGIN_SUCCESS
global rqst, LOGIN_SUCCESS, LOGGED_IN_USER
rqst = requests.session()
try:
rsp = rqst.get(LOGIN_URL)
@@ -299,6 +443,8 @@ def login() -> Optional[RedirectResponse]:
# cookies=cookies
)
print(rsp.content)
soup = BeautifulSoup(rsp.content, "html.parser")
if soup.find("button") and soup.find("button").find(string="Login"):
@@ -307,8 +453,10 @@ def login() -> Optional[RedirectResponse]:
else:
print("login success")
LOGIN_SUCCESS = True
user_label.clear()
user_label.text = f"User: {user}"
LOGGED_IN_USER = user
user_info_ui.refresh()
#user_label.clear()
#user_label.text = f"User: {user}"
# print(rqst.headers)
rqst.headers["X-CSRFToken"] = token
@@ -345,89 +493,27 @@ def login() -> Optional[RedirectResponse]:
@ui.page('/')
def main_page():
def load_series_view(loaded_series, loaded_series_data):
logger.debug("load series view")
logger.debug(loaded_series)
for key in loaded_series:
logger.debug(f"load series: {key}")
with series:
with ui.card():
ui.label(loaded_series_data[key][0])
ui.label(loaded_series_data[key][1])
ui.label(f"Images: {len(loaded_series[key])}")
series_title.visible = True
if use_zmq:
async def read_loop() -> None:
while not app.is_stopped:
events = await poller.poll()
if socket in dict(events):
data = await socket.recv()
msg = data.decode()
logger.debug(f'Received msg {msg}')
match msg:
case "loaded":
with updates:
ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}")
logger.debug("loaded")
load_files_start()
case _:
logger.error("invalid message")
app.on_startup(read_loop)
dark = ui.dark_mode()
dark.enable()
ui.page_title('Uploader tool')
ui.label('Uploader tool').classes("text-h2")
user_label = ui.label("User: None")
user_info_ui()
login_button = ui.button("LOGIN", on_click=lambda: ui.navigate.to("/login"))
login_button.bind_visibility_from(globals(), "LOGIN_SUCCESS", backward=lambda v: not v)
updates = ui.list()
series_title = ui.label('Series to upload').classes("text-h3")
series_title.visible = False
series = ui.row()
loaded_series_ui()
async def load_files_start():
global loaded_files
anon_progress.visible = True
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(load_files, queue)
anon_progress.visible = False
load_series_view(loaded_series, loaded_series_data)
loaded_files_ui.refresh()
async def reload_anonymized_start():
global loaded_files
anon_progress.visible = True
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue)
anon_progress.visible = False
load_series_view(loaded_series, loaded_series_data)
loaded_files_ui.refresh()
async def upload_files_start():
global uploaded_files, duplicate_series
upload_progress.visible = True
uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst)
upload_progress.visible = False
ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive')
ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning')
ui.notify(f"Failed {len(failed)} files", color='negative')
uploaded_files_ui.refresh()
upload_button = ui.button("Upload", on_click=upload_files_start)
upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS")
# typer_app()
# Create a queue to communicate with the heavy computation process
queue = Manager().Queue()
@@ -439,9 +525,6 @@ def main_page():
# Update the progress bar on the main process
ui.timer(0.1, callback=lambda: upload_progressbar.set_value(upload_queue.get() if not upload_queue.empty() else upload_progressbar.value))
with ui.expansion('Extra', icon='build').classes('w-full'):
ui.button("Load files", on_click=load_files_start)
ui.button("Reload anon", on_click=reload_anonymized_start)
@@ -452,6 +535,9 @@ def main_page():
upload_progressbar = ui.linear_progress(value=0).props('instant-feedback')
upload_progress.visible = False
upload_button = ui.button("Upload", on_click=partial(upload_files_start, upload_progress, upload_queue))
upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS")
anon_progress = ui.row().classes("w-full place-content-center bg-blue-900")
with anon_progress:
@@ -459,24 +545,9 @@ def main_page():
progressbar = ui.linear_progress(value=0).props('instant-feedback')
anon_progress.visible = False
@ui.refreshable
def loaded_files_ui() -> None:
with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'):
ui.label(f"Items: {len(loaded_files)}")
with ui.list():
for file, data in loaded_files.items():
ui.item(f"{file} - {data[0]} - {data[1]}")
@ui.refreshable
def uploaded_files_ui() -> None:
with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'):
ui.label(f"Items: {len(uploaded_files)}")
with ui.list():
for series in duplicate_series:
ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500")
with ui.list():
for file, data in uploaded_files.items():
ui.item(f"{file} - {data}")
with ui.expansion('Extra', icon='build').classes('w-full'):
ui.button("Load files", on_click=partial(load_files_start, anon_progress, queue))
ui.button("Reload anon", on_click=partial(reload_anonymized_start, anon_progress, queue))
with ui.expansion('File status', icon='view_list').classes('w-full'):
loaded_files_ui()
@@ -490,12 +561,28 @@ def main_page():
ui.label(f"Export path: {EXPORT_PATH}")
ui.label(f"Process path: {PROCESS_PATH}")
ui.label(f"Anon path: {ANON_PATH}")
ui.button("Clear anon path", on_click=clear_anonymized_files).tooltip("Delete all files in ANON_PATH")
ui.notify("Welcome to the uploader tool", color='positive')
async def watch_for_refresh():
global REFRESH_TRIGGERED
while not app.is_stopped:
if REFRESH_TRIGGERED:
REFRESH_TRIGGERED = False
await load_files_start(anon_progress, queue)
await asyncio.sleep(1)
asyncio.create_task(watch_for_refresh())
if __name__ in {"__main__", "__mp_main__"}:
freeze_support()
ui.run(storage_secret="123456", reload=False)
if use_zmq:
app.on_startup(read_loop)
ui.run(storage_secret="123456", reload=False, show=False)