ross/uploader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tidy up

Browse Source
This commit is contained in:
Ross
2024-12-09 11:13:46 +00:00
parent 8aae29f3e8
commit 66b4fffb7b
10 changed files with 227 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nice.py
+221 -210
View File
@@ -1,20 +1,17 @@
#!/usr/bin/env python3
import asyncio
import pynng
#asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
# asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
from collections import defaultdict
from pathlib import Path
from bs4 import BeautifulSoup
from local_file_picker import local_file_picker
from nicegui import ui, app, html, run, Client, native
from datetime import datetime
import zmq
import zmq.asyncio
from loguru import logger
import sys
@@ -26,10 +23,9 @@ import time
from multiprocessing import Manager, Queue, freeze_support
import watchfiles
from functools import partial
from local_file_picker import local_file_picker
from local_folder_picker import local_folder_picker
from typing import Optional
@@ -39,29 +35,7 @@ from fastapi.responses import RedirectResponse
from starlette.middleware.base import BaseHTTPMiddleware
import typer
#typer_app = typer.Typer()
#logger.add(sys.stderr, format="{time} {level} {message}", filter="my_module", level="INFO")
#async def pick_file() -> None:
# result = await local_file_picker('~', multiple=True)
# ui.notify(f'You chose {result}')
# ui.html("<h2>Uploader tool</h2>")
#
# ui.chat_message(f'{result}!',
# name='Robot',
# stamp='now',
# avatar='https://robohash.org/ui')
#
#
#@ui.page('/')
#def index():
# ui.html("<h2>Uploader tool</h2>")
# ui.button('Choose file', on_click=pick_file, icon='folder')
NATIVE = False
SOCKET_PATH = 'tcp://localhost:9976'
SOCKET_PATH = "tcp://localhost:9976"
WORK_DIR = Path("C:/uploader")
@@ -83,6 +57,7 @@ LOGGED_IN_USER = "None"
REFRESH_TRIGGERED = True
SHUTDOWN = False
DELETE_FILES_ON_UPLOAD = True
@@ -98,8 +73,6 @@ if DEBUG:
rqst = None
use_zmq = False
use_nng = True
loaded_files: dict = {}
loaded_series = defaultdict(list)
loaded_series_data = {}
@@ -107,88 +80,44 @@ uploaded_files: dict = {}
duplicate_series = set()
if use_zmq:
context = zmq.asyncio.Context()
socket = context.socket(zmq.PULL)
#socket.bind('tcp://localhost:5575')
async def read_nng_messages():
print("start nng")
try:
socket.bind(SOCKET_PATH)
except zmq.error.ZMQError as e:
logger.debug("send reload")
socket = context.socket(zmq.PUSH)
socket.connect(SOCKET_PATH)
socket.send_string("loaded")
sys.exit(0)
#app.shutdown()
poller = zmq.asyncio.Poller()
poller.register(socket, zmq.POLLIN)
async def read_loop() -> None:
while not app.is_stopped:
events = await poller.poll()
if socket in dict(events):
data = await socket.recv()
msg = data.decode()
logger.debug(f'Received msg {msg}')
match msg:
with pynng.Pair0(listen=SOCKET_PATH) as sub:
# sub.subscribe('')
while not app.is_stopped:
msg = await sub.arecv_msg()
data = msg.bytes.decode()
logger.debug(f"Received msg {data}")
match data:
case "loaded":
for client in Client.instances.values():
if not client.has_socket_connection:
continue
with client:
ui.notify("Export done", color='positive')
ui.notify("Export done", color="positive")
global REFRESH_TRIGGERED
REFRESH_TRIGGERED = True
logger.debug("loaded")
#with updates:
# ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}")
#logger.debug("loaded")
#load_files_start()
case _:
logger.error("invalid message")
except pynng.exceptions.AddressInUse:
logger.debug("Address in use")
with pynng.Pair0(dial=SOCKET_PATH) as sub:
await sub.asend(b"loaded")
if use_nng:
async def read_nng_messages():
print("start nng")
try:
with pynng.Pair0(listen=SOCKET_PATH) as sub:
#sub.subscribe('')
while not app.is_stopped:
msg = await sub.arecv_msg()
data = msg.bytes.decode()
logger.debug(f'Received msg {data}')
match data:
case "loaded":
for client in Client.instances.values():
if not client.has_socket_connection:
continue
with client:
ui.notify("Export done", color='positive')
global REFRESH_TRIGGERED
REFRESH_TRIGGERED = True
logger.debug("loaded")
case _:
logger.error("invalid message")
except pynng.exceptions.AddressInUse:
logger.debug("Address in use")
with pynng.Pair0(dial=SOCKET_PATH) as sub:
await sub.asend(b"loaded")
sys.exit(0)
#asyncio.create_task(read_nng_messages())
# Trigger shutdown from in the async loop
# This shouldn't be necessary
global SHUTDOWN
SHUTDOWN = True
return
# asyncio.create_task(read_nng_messages())
anonymizer = Anonymizer(ANON_PATH)
async def upload_files_start(progress, upload_queue):
global uploaded_files, duplicate_series
progress.visible = True
@@ -197,67 +126,91 @@ async def upload_files_start(progress, upload_queue):
progress.visible = False
if results:
new_uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = results
(
new_uploaded_files,
upload_file_list,
duplicate_file_list,
failed,
duplicate_series,
) = results
else:
ui.notify("No files to upload", color="negative")
return
uploaded_files.update(new_uploaded_files)
ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive')
ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning')
ui.notify(f"Failed {len(failed)} files", color='negative')
ui.notify(f"Uploaded {len(upload_file_list)} files", color="positive")
if duplicate_file_list:
ui.notify(f"Duplicate {len(duplicate_file_list)} files", color="warning")
if failed:
ui.notify(f"Failed {len(failed)} files", color="negative")
if DELETE_FILES_ON_UPLOAD:
clear_anonymized_files()
loaded_series_ui.refresh()
loaded_files_ui.refresh()
uploaded_files_ui.refresh()
async def load_files_start(progress_bar, queue, custom_path=None):
logger.debug("load files start")
global loaded_files, loaded_series, loaded_series_data
progress_bar.visible = True
new_loaded_files, new_loaded_series, new_loaded_series_data = await run.cpu_bound(load_files, queue, custom_path)
new_loaded_files, new_loaded_series, new_loaded_series_data = await run.cpu_bound(
load_files, queue, custom_path
)
for client in Client.instances.values():
if not client.has_socket_connection:
continue
with client:
ui.notify(f"Files loaded: {len(new_loaded_files)}, [Series: {len(new_loaded_series)}] ", color="positive")
loaded_files.update(new_loaded_files)
loaded_series.update(new_loaded_series)
loaded_series_data.update(new_loaded_series_data)
progress_bar.visible = False
#load_series_view(loaded_series, loaded_series_data)
# load_series_view(loaded_series, loaded_series_data)
loaded_series_ui.refresh()
loaded_files_ui.refresh()
async def reload_anonymized_start(progress_bar, queue):
global loaded_files, loaded_series, loaded_series_data
progress_bar.visible = True
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue)
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(
reload_anonymized, queue
)
progress_bar.visible = False
#load_series_view(loaded_series, loaded_series_data)
ui.notify(f"Files reloaded: {len(loaded_files)}, [Series: {len(loaded_series)}] ", color="positive")
# load_series_view(loaded_series, loaded_series_data)
loaded_series_ui.refresh()
loaded_files_ui.refresh()
@ui.refreshable
def user_info_ui():
global LOGGED_IN_USER
user_label = ui.label(f"User: {LOGGED_IN_USER}")
#user_label.bind_visibility_from(globals(), "LOGIN_SUCCESS")
# user_label.bind_visibility_from(globals(), "LOGIN_SUCCESS")
@ui.refreshable
def loaded_series_ui() -> None:
global loaded_series, loaded_series_data
series_title = ui.label('Series to upload').classes("text-h3")
series_title = ui.label("Series to upload").classes("text-h3")
series_title.visible = False
series = ui.row()
#def load_series_view(loaded_series, loaded_series_data):
# def load_series_view(loaded_series, loaded_series_data):
logger.debug("load series view")
logger.debug(loaded_series)
for key in loaded_series:
@@ -271,9 +224,10 @@ def loaded_series_ui() -> None:
if loaded_series:
series_title.visible = True
@ui.refreshable
def loaded_files_ui() -> None:
with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'):
with ui.expansion("Loaded files", icon="drive_folder_upload").classes("w-full"):
ui.label(f"Items: {len(loaded_files)}")
with ui.list():
for file, data in loaded_files.items():
@@ -282,15 +236,21 @@ def loaded_files_ui() -> None:
@ui.refreshable
def uploaded_files_ui() -> None:
with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'):
with ui.expansion("Uploaded files", icon="file_upload").classes("w-full"):
ui.label(f"Items: {len(uploaded_files)}")
with ui.list():
for series in duplicate_series:
ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500")
ui.item(
f"Duplicate series: {series}",
on_click=lambda: ui.navigate.to(
f"{BASE_SITE_URL}{series}", new_tab=True
),
).classes("text-red-500")
with ui.list():
for file, data in uploaded_files.items():
ui.item(f"{file} - {data}")
def load_files(q: Queue, src_path: Path | None = None):
loaded_files: dict = {}
loaded_series = defaultdict(list)
@@ -311,20 +271,26 @@ def load_files(q: Queue, src_path: Path | None = None):
to_process_len = len(to_process)
for n, file in enumerate(to_process):
dataset, output_file = anonymizer.anonymize_file(file, remove_original=True)
#with processed_files:
# with processed_files:
# ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {file.name} -> {output_file.name}")
loaded_files[output_file] = (dataset.StudyDescription, dataset.SeriesDescription, str(dataset.SeriesInstanceUID))
loaded_files[output_file] = (
dataset.StudyDescription,
dataset.SeriesDescription,
str(dataset.SeriesInstanceUID),
)
loaded_series[dataset.SeriesInstanceUID].append(output_file)
loaded_series_data[dataset.SeriesInstanceUID] = (dataset.StudyDescription, dataset.SeriesDescription)
loaded_series_data[dataset.SeriesInstanceUID] = (
dataset.StudyDescription,
dataset.SeriesDescription,
)
q.put_nowait((n + 1) / to_process_len)
#time.sleep(0.01)
return loaded_files, loaded_series, loaded_series_data
# time.sleep(0.01)
return loaded_files, loaded_series, loaded_series_data
def reload_anonymized(q: Queue):
@@ -332,33 +298,39 @@ def reload_anonymized(q: Queue):
loaded_series = defaultdict(list)
loaded_series_data = {}
#processed_files.clear()
# processed_files.clear()
to_process = list(ANON_PATH.glob("**/*.dcm"))
to_process_len = len(to_process)
for n, file in enumerate(to_process):
with pydicom.dcmread(file) as dataset:
loaded_files[file] = (dataset.StudyDescription, dataset.SeriesDescription, str(dataset.SeriesInstanceUID))
loaded_files[file] = (
dataset.StudyDescription,
dataset.SeriesDescription,
str(dataset.SeriesInstanceUID),
)
loaded_series[dataset.SeriesInstanceUID].append(file)
loaded_series_data[dataset.SeriesInstanceUID] = (dataset.StudyDescription, dataset.SeriesDescription)
loaded_series_data[dataset.SeriesInstanceUID] = (
dataset.StudyDescription,
dataset.SeriesDescription,
)
#with processed_files:
# with processed_files:
# ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {file.name} -> Reloaded")
q.put_nowait(f"{(n + 1) / to_process_len * 100:.0f}%")
logger.debug(file)
#time.sleep(0.01)
# time.sleep(0.01)
logger.debug("end reload")
logger.debug(loaded_files)
logger.debug(loaded_series)
logger.debug("-----")
return loaded_files, loaded_series, loaded_series_data
return loaded_files, loaded_series, loaded_series_data
def clear_anonymized_files():
for file in ANON_PATH.iterdir():
@@ -369,8 +341,9 @@ def clear_anonymized_files():
return
def upload_files(q, rqst):
#global rqst#, uploaded_files
# global rqst#, uploaded_files
uploaded_files = {}
@@ -387,7 +360,6 @@ def upload_files(q, rqst):
files_to_upload[i : i + n] for i in range(0, len(files_to_upload), n)
]
upload_file_list = []
duplicate_file_list = []
failed = []
@@ -447,35 +419,37 @@ def upload_files(q, rqst):
for f in failed:
uploaded_files[f] = "failed"
return uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series
return (
uploaded_files,
upload_file_list,
duplicate_file_list,
failed,
duplicate_series,
)
@ui.page('/login')
@ui.page("/login")
def login() -> Optional[RedirectResponse]:
dark = ui.dark_mode()
dark.enable()
def try_login() -> None: # local function to avoid passing username and password as arguments
logger.debug(username.value)
#if passwords.get(username.value) == password.value:
# app.storage.user.update({'username': username.value, 'authenticated': True})
# ui.navigate.to(app.storage.user.get('referrer_path', '/')) # go back to where the user wanted to go
#else:
# ui.notify('Wrong username or password', color='negative')
def try_login() -> (
None
): # local function to avoid passing username and password as arguments
# Because I haven't gotten around to implementing a proper login system yet
# we just log in via requests and store teh session in the global rqst variable
user = username.value
pw = password.value
global rqst, LOGIN_SUCCESS, LOGGED_IN_USER
rqst = requests.session()
try:
try:
rsp = rqst.get(LOGIN_URL)
except requests.exceptions.ConnectionError:
ui.notify('Connection error!', color='negative')
ui.notify("Connection error!", color="negative")
return
token = (
@@ -514,116 +488,124 @@ def login() -> Optional[RedirectResponse]:
LOGIN_SUCCESS = True
LOGGED_IN_USER = user
user_info_ui.refresh()
#user_label.clear()
#user_label.text = f"User: {user}"
# user_label.clear()
# user_label.text = f"User: {user}"
# print(rqst.headers)
rqst.headers["X-CSRFToken"] = token
print(rqst.headers)
print(rqst.cookies)
new = rqst.get(f"{BASE_SITE_URL}/accounts/api/atlas/get_cases_user")
if LOGIN_SUCCESS:
ui.notify('Logged in!', color='positive')
ui.notify("Logged in!", color="positive")
ui.navigate.to("/")
#login_button.visible = False
pass
#self.parent.upload_button.Enable()
#self.parent.login_button.Hide()
#self.parent.panel.Layout()
#self.Destroy()
# self.parent.login_button.SetBackgroundColour(wx.BLUE)
# self.parent.login_button.SetForegroundColour(wx.WHITE)
else:
ui.notify('Wrong username or password', color='negative')
pass
#self.login_fail()
ui.notify("Wrong username or password", color="negative")
#if app.storage.user.get('authenticated', False):
# if app.storage.user.get('authenticated', False):
# return RedirectResponse('/')
with ui.card().classes('absolute-center'):
username = ui.input('Username').on('keydown.enter', try_login)
password = ui.input('Password', password=True, password_toggle_button=True).on('keydown.enter', try_login)
ui.button('Log in', on_click=try_login)
with ui.card().classes("absolute-center"):
username = ui.input("Username").on("keydown.enter", try_login)
password = ui.input("Password", password=True, password_toggle_button=True).on(
"keydown.enter", try_login
)
with ui.row():
ui.button("Log in", on_click=try_login)
ui.button("Cancel", on_click=lambda: ui.navigate.to("/"), color="red")
return None
@ui.page('/')
def main_page():
@ui.page("/")
def main_page():
async def watch_for_shutdown():
global SHUTDOWN
while not app.is_stopped:
if SHUTDOWN:
logger.debug("Trigger app shutdown")
app.shutdown()
await asyncio.sleep(1)
asyncio.create_task(watch_for_shutdown())
dark = ui.dark_mode()
dark.enable()
ui.page_title('Uploader tool')
ui.label('Uploader tool').classes("text-h2")
ui.page_title("Uploader tool")
ui.label("Uploader tool").classes("text-h2")
user_info_ui()
login_button = ui.button("LOGIN", on_click=lambda: ui.navigate.to("/login"))
login_button.bind_visibility_from(globals(), "LOGIN_SUCCESS", backward=lambda v: not v)
login_button.bind_visibility_from(
globals(), "LOGIN_SUCCESS", backward=lambda v: not v
)
updates = ui.list()
loaded_series_ui()
# typer_app()
# typer_app()
# Create a queue to communicate with the heavy computation process
queue = Manager().Queue()
# Update the progress bar on the main process
ui.timer(0.1, callback=lambda: progressbar.set_value(queue.get() if not queue.empty() else progressbar.value))
ui.timer(
0.1,
callback=lambda: progressbar.set_value(
queue.get() if not queue.empty() else progressbar.value
),
)
# Create a queue to communicate with the heavy computation process
upload_queue = Manager().Queue()
# Update the progress bar on the main process
ui.timer(0.1, callback=lambda: upload_progressbar.set_value(upload_queue.get() if not upload_queue.empty() else upload_progressbar.value))
ui.timer(
0.1,
callback=lambda: upload_progressbar.set_value(
upload_queue.get() if not upload_queue.empty() else upload_progressbar.value
),
)
upload_progress = ui.row().classes("w-full place-content-center bg-blue-900")
with upload_progress:
ui.label("Uploading files")
upload_progressbar = ui.linear_progress(value=0).props('instant-feedback')
upload_progressbar = ui.linear_progress(value=0).props("instant-feedback")
upload_progress.visible = False
upload_button = ui.button("Upload", on_click=partial(upload_files_start, upload_progress, upload_queue))
upload_button = ui.button(
"Upload", on_click=partial(upload_files_start, upload_progress, upload_queue)
)
upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS")
anon_progress = ui.row().classes("w-full place-content-center bg-blue-900")
with anon_progress:
ui.label("Anonymizing files")
progressbar = ui.linear_progress(value=0).props('instant-feedback')
progressbar = ui.linear_progress(value=0).props("instant-feedback")
anon_progress.visible = False
async def load_files_from_folder() -> None:
folder = await local_folder_picker("~")
logger.error(folder)
if folder is not None:
ui.notify(f"Selected folder: {folder}")
ui.notify(f"Selected folder: {folder}")
await load_files_start(anon_progress, queue, folder)
#return dir
# return dir
else:
ui.notify("No folder selected", color='negative')
ui.notify("No folder selected", color="negative")
with ui.expansion('Extra', icon='build').classes('w-full'):
ui.button("Load files", on_click=partial(load_files_start, anon_progress, queue))
with ui.expansion("Extra", icon="build").classes("w-full"):
ui.button(
"Load files", on_click=partial(load_files_start, anon_progress, queue)
)
ui.button("Load files from folder", on_click=load_files_from_folder)
ui.button("Reload anon", on_click=partial(reload_anonymized_start, anon_progress, queue))
ui.button(
"Reload anon",
on_click=partial(reload_anonymized_start, anon_progress, queue),
)
with ui.expansion('File status', icon='view_list').classes('w-full'):
with ui.expansion("File status", icon="view_list").classes("w-full"):
loaded_files_ui()
uploaded_files_ui()
#with ui.expansion('Processed files!', icon='work').classes('w-full'):
# with ui.expansion('Processed files!', icon='work').classes('w-full'):
# processed_files = ui.list()
with ui.expansion("Settings", icon="settings").classes("w-full"):
@@ -631,13 +613,13 @@ def main_page():
ui.label(f"Export path: {EXPORT_PATH}")
ui.label(f"Process path: {PROCESS_PATH}")
ui.label(f"Anon path: {ANON_PATH}")
ui.button("Clear anon path", on_click=clear_anonymized_files).tooltip("Delete all files in ANON_PATH")
ui.button("Clear anon path", on_click=clear_anonymized_files).tooltip(
"Delete all files in ANON_PATH"
)
ui.button("Shutdown", on_click=app.shutdown)
ui.notify("Welcome to the uploader tool", color='positive')
def shutdown_app():
app.shutdown()
ui.run_javascript("window.close()")
async def watch_for_refresh():
global REFRESH_TRIGGERED
@@ -649,29 +631,58 @@ def main_page():
asyncio.create_task(watch_for_refresh())
with ui.dialog() as about_dialog:
with ui.card().classes("absolute-center"):
ui.label("Uploader tool").classes("text-h2")
ui.label("A tool to help upload files to penracourses.org.uk")
ui.image("icon/icon1.png")
ui.label("Version: 0.1")
ui.label("Author: Ross Kruger")
def launch_app(work_dir: Path = typer.Option(WORK_DIR, help="Working directory"), nng: bool = typer.Option(use_nng, help="Use nng")):
global WORK_DIR, EXPORT_PATH, PROCESS_PATH, ANON_PATH, use_nng
ui.button("Close", on_click=about_dialog.close, icon="close", color="secondary")
with ui.footer():
with ui.row():
ui.button("Shutdown", on_click=shutdown_app).props("outline color=white")
ui.button("About", on_click=about_dialog.open).props("outline color=white")
def launch_app(
work_dir: Path = typer.Option(WORK_DIR, help="Working directory"),
nng: bool = typer.Option(True, help="Use nng"),
native_mode: bool = typer.Option(False, help="Use native mode"),
):
global WORK_DIR, EXPORT_PATH, PROCESS_PATH, ANON_PATH
WORK_DIR = work_dir
EXPORT_PATH = WORK_DIR / Path("export/")
PROCESS_PATH = WORK_DIR / Path("to_process/")
ANON_PATH = WORK_DIR / Path("anon/")
logger.error(ANON_PATH)
logger.debug(f"Work dir: {WORK_DIR}")
if nng:
use_nng = True
try:
with pynng.Pair0(listen=SOCKET_PATH) as sub:
pass
except pynng.exceptions.AddressInUse:
logger.debug("Address in use")
with pynng.Pair0(dial=SOCKET_PATH) as sub:
sub.send(b"loaded")
sys.exit(0)
if use_nng:
app.on_startup(read_nng_messages)
app.on_exception(logger.debug)
try:
ui.run(reload=False, show=True, port=native.find_open_port(), native=native_mode, favicon="icon/icon1.ico")
except Exception as e:
logger.error(e)
pass
ui.run(storage_secret="123456", reload=False, show=True, port=native.find_open_port(), native=NATIVE)
if __name__ in {"__main__", "__mp_main__"}:
freeze_support()
typer.run(launch_app)