refactor so multiprocessing works on windows
This commit is contained in:
Ross Kruger
@@ -21,7 +21,7 @@ import pydicom
|
||||
import requests
|
||||
import time
|
||||
|
||||
from multiprocessing import Manager, Queue
|
||||
from multiprocessing import Manager, Queue, freeze_support
|
||||
|
||||
import watchfiles
|
||||
|
||||
@@ -128,18 +128,6 @@ def load_files(q: Queue):
|
||||
return loaded_files, loaded_series, loaded_series_data
|
||||
#load_series_view()
|
||||
|
||||
def load_series_view(loaded_series, loaded_series_data):
|
||||
logger.debug("load series view")
|
||||
logger.debug(loaded_series)
|
||||
for key in loaded_series:
|
||||
logger.debug(f"load series: {key}")
|
||||
with series:
|
||||
with ui.card():
|
||||
ui.label(loaded_series_data[key][0])
|
||||
ui.label(loaded_series_data[key][1])
|
||||
ui.label(f"Images: {len(loaded_series[key])}")
|
||||
|
||||
series_title.visible = True
|
||||
|
||||
|
||||
def reload_anonymized(q: Queue):
|
||||
@@ -260,108 +248,15 @@ def upload_files(q, rqst):
|
||||
return uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series
|
||||
|
||||
|
||||
dark = ui.dark_mode()
|
||||
dark.enable()
|
||||
|
||||
ui.page_title('Uploader tool')
|
||||
ui.label('Uploader tool').classes("text-h2")
|
||||
user_label = ui.label("User: None")
|
||||
|
||||
login_button = ui.button("LOGIN", on_click=lambda: ui.navigate.to("/login"))
|
||||
login_button.bind_visibility_from(globals(), "LOGIN_SUCCESS", backward=lambda v: not v)
|
||||
|
||||
updates = ui.list()
|
||||
|
||||
series_title = ui.label('Series to upload').classes("text-h3")
|
||||
series_title.visible = False
|
||||
|
||||
series = ui.row()
|
||||
|
||||
|
||||
async def upload_files_start():
|
||||
global uploaded_files, duplicate_series
|
||||
upload_progress.visible = True
|
||||
uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst)
|
||||
upload_progress.visible = False
|
||||
|
||||
ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive')
|
||||
ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning')
|
||||
ui.notify(f"Failed {len(failed)} files", color='negative')
|
||||
|
||||
uploaded_files_ui.refresh()
|
||||
|
||||
upload_button = ui.button("Upload", on_click=upload_files_start)
|
||||
upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS")
|
||||
|
||||
if use_zmq:
|
||||
async def read_loop() -> None:
|
||||
while not app.is_stopped:
|
||||
events = await poller.poll()
|
||||
if socket in dict(events):
|
||||
data = await socket.recv()
|
||||
msg = data.decode()
|
||||
|
||||
logger.debug(f'Received msg {msg}')
|
||||
|
||||
match msg:
|
||||
case "loaded":
|
||||
with updates:
|
||||
ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}")
|
||||
logger.debug("loaded")
|
||||
load_files_start()
|
||||
|
||||
|
||||
case _:
|
||||
logger.error("invalid message")
|
||||
app.on_startup(read_loop)
|
||||
|
||||
# Create a queue to communicate with the heavy computation process
|
||||
queue = Manager().Queue()
|
||||
# Update the progress bar on the main process
|
||||
ui.timer(0.1, callback=lambda: progressbar.set_value(queue.get() if not queue.empty() else progressbar.value))
|
||||
|
||||
# Create a queue to communicate with the heavy computation process
|
||||
upload_queue = Manager().Queue()
|
||||
# Update the progress bar on the main process
|
||||
ui.timer(0.1, callback=lambda: upload_progressbar.set_value(upload_queue.get() if not upload_queue.empty() else upload_progressbar.value))
|
||||
upload_progress = ui.row().classes("w-full place-content-center bg-blue-900")
|
||||
|
||||
with upload_progress:
|
||||
ui.label("Uploading files")
|
||||
upload_progressbar = ui.linear_progress(value=0).props('instant-feedback')
|
||||
upload_progress.visible = False
|
||||
|
||||
anon_progress = ui.row().classes("w-full place-content-center bg-blue-900")
|
||||
|
||||
with anon_progress:
|
||||
ui.label("Anonymizing files")
|
||||
progressbar = ui.linear_progress(value=0).props('instant-feedback')
|
||||
anon_progress.visible = False
|
||||
|
||||
|
||||
async def load_files_start():
|
||||
global loaded_files
|
||||
anon_progress.visible = True
|
||||
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(load_files, queue)
|
||||
anon_progress.visible = False
|
||||
load_series_view(loaded_series, loaded_series_data)
|
||||
loaded_files_ui.refresh()
|
||||
|
||||
async def reload_anonymized_start():
|
||||
global loaded_files
|
||||
anon_progress.visible = True
|
||||
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue)
|
||||
anon_progress.visible = False
|
||||
load_series_view(loaded_series, loaded_series_data)
|
||||
loaded_files_ui.refresh()
|
||||
|
||||
with ui.expansion('Extra', icon='build').classes('w-full'):
|
||||
ui.button("Load files", on_click=load_files_start)
|
||||
ui.button("Reload anon", on_click=reload_anonymized_start)
|
||||
|
||||
|
||||
@ui.page('/login')
|
||||
def login() -> Optional[RedirectResponse]:
|
||||
dark = ui.dark_mode()
|
||||
dark.enable()
|
||||
def try_login() -> None: # local function to avoid passing username and password as arguments
|
||||
logger.debug(username.value)
|
||||
#if passwords.get(username.value) == password.value:
|
||||
@@ -448,41 +343,159 @@ def login() -> Optional[RedirectResponse]:
|
||||
ui.button('Log in', on_click=try_login)
|
||||
return None
|
||||
|
||||
@ui.refreshable
|
||||
def loaded_files_ui() -> None:
|
||||
with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'):
|
||||
ui.label(f"Items: {len(loaded_files)}")
|
||||
with ui.list():
|
||||
for file, data in loaded_files.items():
|
||||
ui.item(f"{file} - {data[0]} - {data[1]}")
|
||||
@ui.page('/')
|
||||
def main_page():
|
||||
def load_series_view(loaded_series, loaded_series_data):
|
||||
logger.debug("load series view")
|
||||
logger.debug(loaded_series)
|
||||
for key in loaded_series:
|
||||
logger.debug(f"load series: {key}")
|
||||
with series:
|
||||
with ui.card():
|
||||
ui.label(loaded_series_data[key][0])
|
||||
ui.label(loaded_series_data[key][1])
|
||||
ui.label(f"Images: {len(loaded_series[key])}")
|
||||
|
||||
@ui.refreshable
|
||||
def uploaded_files_ui() -> None:
|
||||
with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'):
|
||||
ui.label(f"Items: {len(uploaded_files)}")
|
||||
with ui.list():
|
||||
for series in duplicate_series:
|
||||
ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500")
|
||||
with ui.list():
|
||||
for file, data in uploaded_files.items():
|
||||
ui.item(f"{file} - {data}")
|
||||
series_title.visible = True
|
||||
|
||||
with ui.expansion('File status', icon='view_list').classes('w-full'):
|
||||
loaded_files_ui()
|
||||
uploaded_files_ui()
|
||||
if use_zmq:
|
||||
async def read_loop() -> None:
|
||||
while not app.is_stopped:
|
||||
events = await poller.poll()
|
||||
if socket in dict(events):
|
||||
data = await socket.recv()
|
||||
msg = data.decode()
|
||||
|
||||
#with ui.expansion('Processed files!', icon='work').classes('w-full'):
|
||||
# processed_files = ui.list()
|
||||
logger.debug(f'Received msg {msg}')
|
||||
|
||||
with ui.expansion("Settings", icon="settings").classes("w-full"):
|
||||
ui.label("Settings").classes("text-h4")
|
||||
ui.label(f"Export path: {EXPORT_PATH}")
|
||||
ui.label(f"Process path: {PROCESS_PATH}")
|
||||
ui.label(f"Anon path: {ANON_PATH}")
|
||||
match msg:
|
||||
case "loaded":
|
||||
with updates:
|
||||
ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}")
|
||||
logger.debug("loaded")
|
||||
load_files_start()
|
||||
|
||||
ui.run(storage_secret="123456")
|
||||
|
||||
case _:
|
||||
logger.error("invalid message")
|
||||
app.on_startup(read_loop)
|
||||
|
||||
dark = ui.dark_mode()
|
||||
dark.enable()
|
||||
ui.page_title('Uploader tool')
|
||||
ui.label('Uploader tool').classes("text-h2")
|
||||
user_label = ui.label("User: None")
|
||||
|
||||
login_button = ui.button("LOGIN", on_click=lambda: ui.navigate.to("/login"))
|
||||
login_button.bind_visibility_from(globals(), "LOGIN_SUCCESS", backward=lambda v: not v)
|
||||
|
||||
updates = ui.list()
|
||||
|
||||
series_title = ui.label('Series to upload').classes("text-h3")
|
||||
series_title.visible = False
|
||||
|
||||
series = ui.row()
|
||||
|
||||
|
||||
|
||||
#if __name__ in {"__main__", "__mp_main__"}:
|
||||
# typer_app()
|
||||
async def load_files_start():
|
||||
global loaded_files
|
||||
anon_progress.visible = True
|
||||
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(load_files, queue)
|
||||
anon_progress.visible = False
|
||||
load_series_view(loaded_series, loaded_series_data)
|
||||
loaded_files_ui.refresh()
|
||||
|
||||
async def reload_anonymized_start():
|
||||
global loaded_files
|
||||
anon_progress.visible = True
|
||||
loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue)
|
||||
anon_progress.visible = False
|
||||
load_series_view(loaded_series, loaded_series_data)
|
||||
loaded_files_ui.refresh()
|
||||
|
||||
async def upload_files_start():
|
||||
global uploaded_files, duplicate_series
|
||||
upload_progress.visible = True
|
||||
uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst)
|
||||
upload_progress.visible = False
|
||||
|
||||
ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive')
|
||||
ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning')
|
||||
ui.notify(f"Failed {len(failed)} files", color='negative')
|
||||
|
||||
uploaded_files_ui.refresh()
|
||||
|
||||
upload_button = ui.button("Upload", on_click=upload_files_start)
|
||||
upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS")
|
||||
# typer_app()
|
||||
# Create a queue to communicate with the heavy computation process
|
||||
queue = Manager().Queue()
|
||||
# Update the progress bar on the main process
|
||||
ui.timer(0.1, callback=lambda: progressbar.set_value(queue.get() if not queue.empty() else progressbar.value))
|
||||
|
||||
# Create a queue to communicate with the heavy computation process
|
||||
upload_queue = Manager().Queue()
|
||||
# Update the progress bar on the main process
|
||||
ui.timer(0.1, callback=lambda: upload_progressbar.set_value(upload_queue.get() if not upload_queue.empty() else upload_progressbar.value))
|
||||
|
||||
with ui.expansion('Extra', icon='build').classes('w-full'):
|
||||
ui.button("Load files", on_click=load_files_start)
|
||||
ui.button("Reload anon", on_click=reload_anonymized_start)
|
||||
|
||||
|
||||
|
||||
upload_progress = ui.row().classes("w-full place-content-center bg-blue-900")
|
||||
|
||||
with upload_progress:
|
||||
ui.label("Uploading files")
|
||||
upload_progressbar = ui.linear_progress(value=0).props('instant-feedback')
|
||||
upload_progress.visible = False
|
||||
|
||||
anon_progress = ui.row().classes("w-full place-content-center bg-blue-900")
|
||||
|
||||
with anon_progress:
|
||||
ui.label("Anonymizing files")
|
||||
progressbar = ui.linear_progress(value=0).props('instant-feedback')
|
||||
anon_progress.visible = False
|
||||
|
||||
@ui.refreshable
|
||||
def loaded_files_ui() -> None:
|
||||
with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'):
|
||||
ui.label(f"Items: {len(loaded_files)}")
|
||||
with ui.list():
|
||||
for file, data in loaded_files.items():
|
||||
ui.item(f"{file} - {data[0]} - {data[1]}")
|
||||
|
||||
@ui.refreshable
|
||||
def uploaded_files_ui() -> None:
|
||||
with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'):
|
||||
ui.label(f"Items: {len(uploaded_files)}")
|
||||
with ui.list():
|
||||
for series in duplicate_series:
|
||||
ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500")
|
||||
with ui.list():
|
||||
for file, data in uploaded_files.items():
|
||||
ui.item(f"{file} - {data}")
|
||||
|
||||
with ui.expansion('File status', icon='view_list').classes('w-full'):
|
||||
loaded_files_ui()
|
||||
uploaded_files_ui()
|
||||
|
||||
#with ui.expansion('Processed files!', icon='work').classes('w-full'):
|
||||
# processed_files = ui.list()
|
||||
|
||||
with ui.expansion("Settings", icon="settings").classes("w-full"):
|
||||
ui.label("Settings").classes("text-h4")
|
||||
ui.label(f"Export path: {EXPORT_PATH}")
|
||||
ui.label(f"Process path: {PROCESS_PATH}")
|
||||
ui.label(f"Anon path: {ANON_PATH}")
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ in {"__main__", "__mp_main__"}:
|
||||
freeze_support()
|
||||
|
||||
ui.run(storage_secret="123456", reload=False)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user