diff --git a/.gitignore b/.gitignore index 3e97adc..e74c97c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ __pycache__/* build dist +test/* + *.build \ No newline at end of file diff --git a/anonymiser.py b/anonymiser.py new file mode 100644 index 0000000..12dd0eb --- /dev/null +++ b/anonymiser.py @@ -0,0 +1,41 @@ +import dicognito.anonymizer +import pydicom +from pathlib import Path +import os + +class Anonymizer: + + def __init__(self, output_dir): + self.output_dir = output_dir + self.anonymizer = dicognito.anonymizer.Anonymizer() + + self.i: int = 0 + + def reset(self): + self.i = 0 + self.anonymizer = dicognito.anonymizer.Anonymizer() + + + def anonymize_file(self, input_file: Path, output_file: Path | None = None, remove_original=True): + with pydicom.dcmread(input_file) as dataset: + + if output_file is None: + new_filepath: Path + # Get next available filename + while os.path.exists( + output_file := self.output_dir.joinpath(f"IMG_{self.i:03}.dcm") + ): + self.i += 1 + + self.anonymizer.anonymize(dataset) + dataset.save_as(output_file) + + if remove_original: + os.remove(input_file) + return dataset, output_file + + + + + + diff --git a/local_file_picker.py b/local_file_picker.py new file mode 100644 index 0000000..ff05e76 --- /dev/null +++ b/local_file_picker.py @@ -0,0 +1,82 @@ +import platform +from pathlib import Path +from typing import Optional + +from nicegui import events, ui + + +class local_file_picker(ui.dialog): + + def __init__(self, directory: str, *, + upper_limit: Optional[str] = ..., multiple: bool = False, show_hidden_files: bool = False) -> None: + """Local File Picker + + This is a simple file picker that allows you to select a file from the local filesystem where NiceGUI is running. + + :param directory: The directory to start in. + :param upper_limit: The directory to stop at (None: no limit, default: same as the starting directory). + :param multiple: Whether to allow multiple files to be selected. + :param show_hidden_files: Whether to show hidden files. + """ + super().__init__() + + self.path = Path(directory).expanduser() + if upper_limit is None: + self.upper_limit = None + else: + self.upper_limit = Path(directory if upper_limit == ... else upper_limit).expanduser() + self.show_hidden_files = show_hidden_files + + with self, ui.card(): + self.add_drives_toggle() + self.grid = ui.aggrid({ + 'columnDefs': [{'field': 'name', 'headerName': 'File'}], + 'rowSelection': 'multiple' if multiple else 'single', + }, html_columns=[0]).classes('w-96').on('cellDoubleClicked', self.handle_double_click) + with ui.row().classes('w-full justify-end'): + ui.button('Cancel', on_click=self.close).props('outline') + ui.button('Ok', on_click=self._handle_ok) + self.update_grid() + + def add_drives_toggle(self): + if platform.system() == 'Windows': + import win32api + drives = win32api.GetLogicalDriveStrings().split('\000')[:-1] + self.drives_toggle = ui.toggle(drives, value=drives[0], on_change=self.update_drive) + + def update_drive(self): + self.path = Path(self.drives_toggle.value).expanduser() + self.update_grid() + + def update_grid(self) -> None: + paths = list(self.path.glob('*')) + if not self.show_hidden_files: + paths = [p for p in paths if not p.name.startswith('.')] + paths.sort(key=lambda p: p.name.lower()) + paths.sort(key=lambda p: not p.is_dir()) + + self.grid.options['rowData'] = [ + { + 'name': f'📁 {p.name}' if p.is_dir() else p.name, + 'path': str(p), + } + for p in paths + ] + if self.upper_limit is None and self.path != self.path.parent or \ + self.upper_limit is not None and self.path != self.upper_limit: + self.grid.options['rowData'].insert(0, { + 'name': '📁 ..', + 'path': str(self.path.parent), + }) + self.grid.update() + + def handle_double_click(self, e: events.GenericEventArguments) -> None: + self.path = Path(e.args['data']['path']) + if self.path.is_dir(): + self.update_grid() + else: + self.submit([str(self.path)]) + + async def _handle_ok(self): + rows = await self.grid.get_selected_rows() + self.submit([r['path'] for r in rows]) \ No newline at end of file diff --git a/nice.py b/nice.py new file mode 100644 index 0000000..a5bc151 --- /dev/null +++ b/nice.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 +from collections import defaultdict +from pathlib import Path + +from bs4 import BeautifulSoup +from local_file_picker import local_file_picker + +from nicegui import ui, app, html, run + +from datetime import datetime + +import zmq +import zmq.asyncio + +from loguru import logger +import sys + +from anonymiser import Anonymizer +import pydicom + +import requests +import time + +from multiprocessing import Manager, Queue + +import watchfiles + + +from typing import Optional + +from fastapi import Request +from fastapi.responses import RedirectResponse +from starlette.middleware.base import BaseHTTPMiddleware +#import typer + +#typer_app = typer.Typer() + +#logger.add(sys.stderr, format="{time} {level} {message}", filter="my_module", level="INFO") + +#async def pick_file() -> None: +# result = await local_file_picker('~', multiple=True) +# ui.notify(f'You chose {result}') +# ui.html("

Uploader tool

") +# +# ui.chat_message(f'{result}!', +# name='Robot', +# stamp='now', +# avatar='https://robohash.org/ui') +# +# +#@ui.page('/') +#def index(): +# ui.html("

Uploader tool

") +# ui.button('Choose file', on_click=pick_file, icon='folder') + +EXPORT_PATH = Path("./test/export") +PROCESS_PATH = Path("./test/to_process") +ANON_PATH = Path("./test/anon") + +EXPORT_PATH.mkdir(exist_ok=True, parents=True) +PROCESS_PATH.mkdir(exist_ok=True, parents=True) +EXPORT_PATH.mkdir(exist_ok=True, parents=True) + +BASE_SITE_URL = "http://localhost:8000" + +LOGIN_URL = f"{BASE_SITE_URL}/accounts/login/" + +LOGIN_SUCCESS = False + +rqst = None + + +use_zmq = False +loaded_files: dict = {} +uploaded_files: dict = {} +duplicate_series = set() + +if use_zmq: + context = zmq.asyncio.Context() + socket = context.socket(zmq.PULL) + #socket.bind('tcp://localhost:5575') + try: + socket.bind('tcp://localhost:9975') + except zmq.error.ZMQError as e: + logger.debug("send reload") + socket = context.socket(zmq.PUSH) + socket.connect('tcp://localhost:5575') + socket.send_string("loaded") + sys.exit(0) + + poller = zmq.asyncio.Poller() + poller.register(socket, zmq.POLLIN) + + +anonymizer = Anonymizer(ANON_PATH) + +def load_files(q: Queue): + loaded_files: dict = {} + loaded_series = defaultdict(list) + loaded_series_data = {} + # Move files + src_path = EXPORT_PATH + + to_process = [] + for file in src_path.iterdir(): + logger.debug(f"Moving {file} to {PROCESS_PATH}") + + file_to_process = PROCESS_PATH / file.name + file.rename(file_to_process) + + to_process.append(file_to_process) + + to_process_len = len(to_process) + for n, file in enumerate(to_process): + dataset, output_file = anonymizer.anonymize_file(file, remove_original=True) + #with processed_files: + # ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {file.name} -> {output_file.name}") + + loaded_files[output_file] = (dataset.StudyDescription, dataset.SeriesDescription, str(dataset.SeriesInstanceUID)) + + loaded_series[dataset.SeriesInstanceUID].append(output_file) + loaded_series_data[dataset.SeriesInstanceUID] = (dataset.StudyDescription, dataset.SeriesDescription) + + q.put_nowait((n + 1) / to_process_len) + + #time.sleep(0.01) + + return loaded_files, loaded_series, loaded_series_data + #load_series_view() + +def load_series_view(loaded_series, loaded_series_data): + logger.debug("load series view") + logger.debug(loaded_series) + for key in loaded_series: + logger.debug(f"load series: {key}") + with series: + with ui.card(): + ui.label(loaded_series_data[key][0]) + ui.label(loaded_series_data[key][1]) + ui.label(f"Images: {len(loaded_series[key])}") + + series_title.visible = True + + +def reload_anonymized(q: Queue): + loaded_files = {} + loaded_series = defaultdict(list) + loaded_series_data = {} + + #processed_files.clear() + + + to_process = list(ANON_PATH.glob("*.dcm")) + to_process_len = len(to_process) + for n, file in enumerate(to_process): + with pydicom.dcmread(file) as dataset: + loaded_files[file] = (dataset.StudyDescription, dataset.SeriesDescription, str(dataset.SeriesInstanceUID)) + + loaded_series[dataset.SeriesInstanceUID].append(file) + loaded_series_data[dataset.SeriesInstanceUID] = (dataset.StudyDescription, dataset.SeriesDescription) + + #with processed_files: + # ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {file.name} -> Reloaded") + + + q.put_nowait(f"{(n + 1) / to_process_len * 100:.0f}%") + logger.debug(file) + + #time.sleep(0.01) + + logger.debug("end reload") + logger.debug(loaded_files) + logger.debug(loaded_series) + logger.debug("-----") + + return loaded_files, loaded_series, loaded_series_data + + #load_series_view() + +def upload_files(q, rqst): + #global rqst#, uploaded_files + + uploaded_files = {} + + files_to_upload = [] + for file in ANON_PATH.iterdir(): + files_to_upload.append(("files", open(str(file), "rb"))) + + if not files_to_upload: + ui.notify('No files to upload', color='negative') + return + + # chunck files + n = 10 + chunked_files = [ + files_to_upload[i : i + n] for i in range(0, len(files_to_upload), n) + ] + + + upload_file_list = [] + duplicate_file_list = [] + failed = [] + duplicate_series = set() + + logger.debug("START UPLOAD") + logger.debug(files_to_upload) + + to_process_len = len(chunked_files) + + for n, files in enumerate(chunked_files): + + def upload_files_(files): + rqst.headers["X-CSRFToken"] = rqst.cookies["csrftoken"] + # print(self.rqst.headers) + # print(self.rqst.cookies) + resp = rqst.post( + f"{BASE_SITE_URL}/api/atlas/upload_dicom", + # data=data, + files=files, + ) + + logger.debug(f"Resp: {resp}") + logger.debug(f"{resp.content}") + return resp + + # progress_dialog.Update(n, f"Uploading batch {n}/{len(chunked_files)}") + + logger.debug(f"n: {n}") + # try to upload the files + + for i in range(3): + logger.error(f"i: {i}") + resp = upload_files_(files) + if resp.status_code == 200: + upload_file_list.extend(resp.json()["uploaded"]) + duplicate_file_list.extend(resp.json()["duplicates"]) + failed.extend(resp.json()["failed"]) + duplicate_series.update(resp.json()["duplicate_series"]) + + break + + logger.debug(f"n: {n} fail (attempt {i})") + + q.put_nowait((n + 1) / to_process_len) + # progress_dialog.Destroy() + print(upload_file_list) + print("dup", duplicate_file_list) + print("failed", failed) + + for f, hash in upload_file_list: + uploaded_files[f] = "success" + + for f, hash in duplicate_file_list: + uploaded_files[f] = "duplicate" + + for f in failed: + uploaded_files[f] = "failed" + + + return uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series + + +dark = ui.dark_mode() +dark.enable() + +ui.page_title('Uploader tool') +ui.label('Uploader tool').classes("text-h2") +user_label = ui.label("User: None") + +login_button = ui.button("LOGIN", on_click=lambda: ui.navigate.to("/login")) +login_button.bind_visibility_from(globals(), "LOGIN_SUCCESS", backward=lambda v: not v) + +updates = ui.list() + +series_title = ui.label('Series to upload').classes("text-h3") +series_title.visible = False + +series = ui.row() + + +async def upload_files_start(): + global uploaded_files, duplicate_series + upload_progress.visible = True + uploaded_files, upload_file_list, duplicate_file_list, failed, duplicate_series = await run.cpu_bound(upload_files, upload_queue, rqst) + upload_progress.visible = False + + ui.notify(f"Uploaded {len(upload_file_list)} files", color='positive') + ui.notify(f"Duplicate {len(duplicate_file_list)} files", color='warning') + ui.notify(f"Failed {len(failed)} files", color='negative') + + uploaded_files_ui.refresh() + +upload_button = ui.button("Upload", on_click=upload_files_start) +upload_button.bind_visibility_from(globals(), "LOGIN_SUCCESS") + +if use_zmq: + async def read_loop() -> None: + while not app.is_stopped: + events = await poller.poll() + if socket in dict(events): + data = await socket.recv() + msg = data.decode() + + logger.debug(f'Received msg {msg}') + + match msg: + case "loaded": + with updates: + ui.item(f"{datetime.now().strftime('%H:%M:%S')} - {msg}") + logger.debug("loaded") + load_files_start() + + + case _: + logger.error("invalid message") + app.on_startup(read_loop) + +# Create a queue to communicate with the heavy computation process +queue = Manager().Queue() +# Update the progress bar on the main process +ui.timer(0.1, callback=lambda: progressbar.set_value(queue.get() if not queue.empty() else progressbar.value)) + +# Create a queue to communicate with the heavy computation process +upload_queue = Manager().Queue() +# Update the progress bar on the main process +ui.timer(0.1, callback=lambda: upload_progressbar.set_value(upload_queue.get() if not upload_queue.empty() else upload_progressbar.value)) +upload_progress = ui.row().classes("w-full place-content-center bg-blue-900") + +with upload_progress: + ui.label("Uploading files") + upload_progressbar = ui.linear_progress(value=0).props('instant-feedback') +upload_progress.visible = False + +anon_progress = ui.row().classes("w-full place-content-center bg-blue-900") + +with anon_progress: + ui.label("Anonymizing files") + progressbar = ui.linear_progress(value=0).props('instant-feedback') +anon_progress.visible = False + + +async def load_files_start(): + global loaded_files + anon_progress.visible = True + loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(load_files, queue) + anon_progress.visible = False + load_series_view(loaded_series, loaded_series_data) + loaded_files_ui.refresh() + +async def reload_anonymized_start(): + global loaded_files + anon_progress.visible = True + loaded_files, loaded_series, loaded_series_data = await run.cpu_bound(reload_anonymized, queue) + anon_progress.visible = False + load_series_view(loaded_series, loaded_series_data) + loaded_files_ui.refresh() + +with ui.expansion('Extra', icon='build').classes('w-full'): + ui.button("Load files", on_click=load_files_start) + ui.button("Reload anon", on_click=reload_anonymized_start) + + +@ui.page('/login') +def login() -> Optional[RedirectResponse]: + def try_login() -> None: # local function to avoid passing username and password as arguments + logger.debug(username.value) + #if passwords.get(username.value) == password.value: + # app.storage.user.update({'username': username.value, 'authenticated': True}) + # ui.navigate.to(app.storage.user.get('referrer_path', '/')) # go back to where the user wanted to go + #else: + # ui.notify('Wrong username or password', color='negative') + user = username.value + pw = password.value + + global rqst, LOGIN_SUCCESS + rqst = requests.session() + try: + rsp = rqst.get(LOGIN_URL) + except requests.exceptions.ConnectionError: + ui.notify('Connection error!', color='negative') + return + + token = ( + BeautifulSoup(rsp.content, "html.parser").find("input").attrs["value"] + ) # , attr={"name": "csrfmiddlewaretoken"}).attrs("value") + + # token = rsp.cookies["csrftoken"] + # header = {"X-CSRFToken": token} + # cookies = {"csrftoken": token} + + data = { + "username": user, + "password": pw, + "csrfmiddlewaretoken": token, + "next": "/", + } + + rqst.headers["Referer"] = LOGIN_URL + + rsp = rqst.post( + LOGIN_URL, + data=data, + # headers=header, + # cookies=cookies + ) + + soup = BeautifulSoup(rsp.content, "html.parser") + + if soup.find("button") and soup.find("button").find(string="Login"): + print("login fail") + pass + else: + print("login success") + LOGIN_SUCCESS = True + user_label.clear() + user_label.text = f"User: {user}" + # print(rqst.headers) + + rqst.headers["X-CSRFToken"] = token + + print(rqst.headers) + print(rqst.cookies) + new = rqst.get(f"{BASE_SITE_URL}/accounts/api/atlas/get_cases_user") + + + if LOGIN_SUCCESS: + ui.notify('Logged in!', color='positive') + ui.navigate.to("/") + #login_button.visible = False + pass + #self.parent.upload_button.Enable() + #self.parent.login_button.Hide() + #self.parent.panel.Layout() + #self.Destroy() + + # self.parent.login_button.SetBackgroundColour(wx.BLUE) + # self.parent.login_button.SetForegroundColour(wx.WHITE) + else: + ui.notify('Wrong username or password', color='negative') + pass + #self.login_fail() + + #if app.storage.user.get('authenticated', False): + # return RedirectResponse('/') + with ui.card().classes('absolute-center'): + username = ui.input('Username').on('keydown.enter', try_login) + password = ui.input('Password', password=True, password_toggle_button=True).on('keydown.enter', try_login) + ui.button('Log in', on_click=try_login) + return None + +@ui.refreshable +def loaded_files_ui() -> None: + with ui.expansion('Loaded files', icon='drive_folder_upload').classes('w-full'): + ui.label(f"Items: {len(loaded_files)}") + with ui.list(): + for file, data in loaded_files.items(): + ui.item(f"{file} - {data[0]} - {data[1]}") + +@ui.refreshable +def uploaded_files_ui() -> None: + with ui.expansion('Uploaded files', icon='file_upload').classes('w-full'): + ui.label(f"Items: {len(uploaded_files)}") + with ui.list(): + for series in duplicate_series: + ui.item(f"Duplicate series: {series}", on_click=lambda: ui.navigate.to(f'{BASE_SITE_URL}{series}', new_tab=True)).classes("text-red-500") + with ui.list(): + for file, data in uploaded_files.items(): + ui.item(f"{file} - {data}") + +with ui.expansion('File status', icon='view_list').classes('w-full'): + loaded_files_ui() + uploaded_files_ui() + + #with ui.expansion('Processed files!', icon='work').classes('w-full'): + # processed_files = ui.list() + +with ui.expansion("Settings", icon="settings").classes("w-full"): + ui.label("Settings").classes("text-h4") + ui.label(f"Export path: {EXPORT_PATH}") + ui.label(f"Process path: {PROCESS_PATH}") + ui.label(f"Anon path: {ANON_PATH}") + +ui.run(storage_secret="123456") + + + +#if __name__ in {"__main__", "__mp_main__"}: +# typer_app() \ No newline at end of file diff --git a/zmq-server.py b/zmq-server.py new file mode 100644 index 0000000..4eefd23 --- /dev/null +++ b/zmq-server.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import asyncio +import random + +import zmq +import zmq.asyncio + +context = zmq.asyncio.Context() +socket = context.socket(zmq.PUSH) +socket.connect('tcp://localhost:5575') + + +async def send_loop(): + #print(f'Sending number {number}') + print("Send hello") + await socket.send_string("loaded") + #await asyncio.sleep(0.1) + + print("end") + + + +asyncio.run(send_loop()) \ No newline at end of file