ross/uploader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

numerous improvements

Browse Source
This commit is contained in:
Ross
2024-02-19 11:15:15 +00:00
parent 1e3318dfd6
commit e4c5847fd1
1 changed files with 162 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
anon_gui.py
+162 -95
View File
@@ -61,7 +61,6 @@ except ImportError: # if it's not there locally, try the wxPython lib.
import wx.lib.agw.pygauge as PG
import wx.lib.agw.thumbnailctrl as TC
import wx.lib.agw.ultimatelistctrl as ULC
@@ -192,13 +191,12 @@ def annonymise(
anonymize_file(file, new_filepath)
# progress_dialog.Destroy()
return new_filepath
def anonymize_file(file: Path, new_filepath: Path):
def anonymize_file(file: Path, new_filepath: Path, remove_original: bool = False):
logger.debug(f"Annonymise file, in={file}, out={new_filepath}")
# Ensure the output folder exists
new_filepath.parents[0].mkdir(parents=True, exist_ok=True)
@@ -210,6 +208,9 @@ def anonymize_file(file: Path, new_filepath: Path):
with pydicom.dcmread(file) as dataset:
anonymizer.anonymize(dataset)
dataset.save_as(new_filepath)
if remove_original:
os.remove(file)
return dataset, new_filepath
@@ -298,7 +299,7 @@ class LoginDialog(wx.Dialog):
main_sizer.Add(user_sizer, 0, wx.ALL, 5)
main_sizer.Add(p_sizer, 0, wx.ALL, 5)
btn = wx.Button(self, label="Login")
btn = wx.Button(self, id=wx.ID_OK, label="Login")
btn.Bind(wx.EVT_BUTTON, self.on_login)
main_sizer.Add(btn, 0, wx.ALL | wx.CENTER, 5)
@@ -372,7 +373,7 @@ class LoginDialog(wx.Dialog):
print(new.content)
if self.parent.login_success:
self.parent.upload_button.Show()
self.parent.upload_button.Enable()
self.parent.login_button.Hide()
self.parent.panel.Layout()
self.Destroy()
@@ -477,6 +478,7 @@ class AnonGui(wx.App):
clear_files=True,
clear_files_on_close=True,
dev=False,
remove_files_on_annonymisation=False,
):
self.dir = output_dir
self.input_dir = input_dir
@@ -500,6 +502,7 @@ class AnonGui(wx.App):
self.silent_fail = silent_fail
self.clear_files_on_start = clear_files
self.clear_files_on_close = clear_files_on_close
self.remove_files_on_annonymisation = remove_files_on_annonymisation
self.rqst = None
@@ -575,7 +578,7 @@ class AnonGui(wx.App):
self.link_rapid_upload = hl.HyperLinkCtrl(
self.panel,
wx.ID_ANY,
"Rapid Uploader",
"Rapid",
URL=f"{self.base_site_url}/rapids/create/",
)
# self.panel.sizer.Add(self.link_rapid_upload, 0, wx.ALL, 10)
@@ -586,10 +589,17 @@ class AnonGui(wx.App):
"Long Series",
URL=f"{self.base_site_url}/longs/create/series",
)
self.link_atlas_case = hl.HyperLinkCtrl(
self.panel,
wx.ID_ANY,
"Atlas Case",
URL=f"{self.base_site_url}/atlas/case/create/",
)
# self.panel.sizer.Add(self.link_long_upload, 0, wx.ALL, 10)
hoz_links = wx.BoxSizer(wx.HORIZONTAL)
hoz_links.Add(self.link_rapid_upload, 1, wx.EXPAND)
hoz_links.Add(self.link_long_upload, 1, wx.EXPAND)
hoz_links.Add(self.link_atlas_case, 1, wx.EXPAND)
self.panel.sizer.Add(hoz_links, 1, wx.EXPAND)
# self.dirctrl = wx.GenericDirCtrl(self.panel, wx.ID_ANY,
@@ -598,13 +608,18 @@ class AnonGui(wx.App):
# wx.DIRCTRL_3D_INTERNAL |
# wx.DIRCTRL_MULTIPLE,
# filter="Dicom files (*.dcm)|*.dcm")
upload_drag_sizer = wx.BoxSizer(wx.HORIZONTAL)
self.drag_button = wx.Button(
self.panel, id=wx.ID_ANY, label="Click here to drag files"
self.panel, id=wx.ID_ANY, label="Drag"
)
self.drag_button.SetToolTip("Click here to drag files")
self.upload_button = wx.Button(
self.panel, id=wx.ID_ANY, label="Click here to upload files"
)
self.upload_button.Hide()
self.upload_button.SetToolTip("Upload files to the server. You must be logged in.")
self.upload_button.Disable()
upload_drag_sizer.Add(self.upload_button, 4, wx.EXPAND)
upload_drag_sizer.Add(self.drag_button, 1, wx.EXPAND)
# Use some sizers to see layout options
# self.panel.sizer.Add(self.dirctrl, 1, wx.EXPAND)
@@ -630,20 +645,25 @@ class AnonGui(wx.App):
self.list_ctrl.InsertColumn(6, "Hash")
self.list_ctrl.InsertColumn(7, "Series Instance UID")
# self.UpdateListCtrl()
self.login_button = wx.Button(self.panel, id=wx.ID_ANY, label="Login")
self.login_button.SetForegroundColour(wx.BLUE)
self.login_button.SetBackgroundColour(wx.WHITE)
self.login_button.SetToolTip("Click to login to the server")
self.clear_files_button = wx.Button(
self.panel, id=wx.ID_ANY, label="Clear files"
)
self.clear_files_button.SetToolTip("Click to clear all files.")
self.refresh_files_button = wx.Button(self.panel, id=wx.ID_ANY, label="Refresh")
self.reorder_files_button = wx.Button(self.panel, id=wx.ID_ANY, label="Reorder")
self.reorder_files_button.SetToolTip("Click to reorder files as they are shown below. This will sequentially rename the files. Please note this is usually better done once the files have been uploaded.")
self.reannonymise_files_button = wx.Button(
self.panel, id=wx.ID_ANY, label="Reannonymise"
)
self.reannonymise_files_button.SetToolTip("Click to reannonymise the currently loaded files.")
self.manual_load_button = wx.Button(self.panel, id=wx.ID_ANY, label="Load")
self.manual_load_button.SetToolTip("Click to manually load dicom from a directory.")
# wx.Bind(self.clear_files_button, )
self.Bind(wx.EVT_BUTTON, self.Login, self.login_button)
@@ -651,11 +671,12 @@ class AnonGui(wx.App):
self.Bind(wx.EVT_BUTTON, lambda evt: self.LoadDir(), self.refresh_files_button)
self.Bind(wx.EVT_BUTTON, self.ReorderFiles, self.reorder_files_button)
self.Bind(wx.EVT_BUTTON, self.ReAnnonymiseFiles, self.reannonymise_files_button)
self.Bind(
wx.EVT_BUTTON,
lambda evt: self.AnnonymiseFilesInDir(self.input_dir, self.dir),
self.manual_load_button,
)
#self.Bind(
# wx.EVT_BUTTON,
# lambda evt: self.AnnonymiseFilesInDir(self.input_dir, self.dir),
# self.manual_load_button,
#)
self.Bind(wx.EVT_BUTTON, self.import_from_dir, self.manual_load_button)
# Use some sizers to see layout options
# self.panel.sizer.Add(self.dirctrl, 1, wx.EXPAND)
@@ -672,13 +693,16 @@ class AnonGui(wx.App):
self.panel.sizer.Add(self.series_grid, 2, wx.EXPAND)
self.panel.sizer.Add(self.list_ctrl, 8, wx.EXPAND)
self.panel.sizer.Add(self.drag_button, 2, wx.EXPAND)
self.panel.sizer.Add(self.upload_button, 2, wx.EXPAND)
#self.panel.sizer.Add(self.drag_button, 2, wx.EXPAND)
#self.panel.sizer.Add(self.upload_button, 2, wx.EXPAND)
self.panel.sizer.Add(upload_drag_sizer, 2, wx.EXPAND)
self.progress_sizer = wx.BoxSizer(wx.VERTICAL)
self.progress_bar = wx.Gauge(self.panel, style=wx.GA_HORIZONTAL|wx.GA_PROGRESS)
#self.progress_bar = PG.PyGauge(self.panel, style=wx.GA_HORIZONTAL|wx.GA_PROGRESS)
#self.progress_bar.SetDrawValue(draw=True, drawPercent=True, font=None, colour=wx.RED, formatString=None)
self.progress_bar = wx.Gauge(
self.panel, style=wx.GA_HORIZONTAL | wx.GA_PROGRESS
)
# self.progress_bar = PG.PyGauge(self.panel, style=wx.GA_HORIZONTAL|wx.GA_PROGRESS)
# self.progress_bar.SetDrawValue(draw=True, drawPercent=True, font=None, colour=wx.RED, formatString=None)
self.progress_text = wx.StaticText(self.panel, label="Text")
self.progress_sizer.Add(self.progress_bar, 2, wx.EXPAND)
self.progress_sizer.Add(self.progress_text, 2, wx.EXPAND)
@@ -691,18 +715,21 @@ class AnonGui(wx.App):
self.panel.sizer.Fit(self.panel)
self.frame.Show()
# self.spinner.Hide()
wx.CallAfter(self.post_init)
return super().OnInit()
def post_init(self):
self.AnnonymiseFilesInDir(self.input_dir, self.dir)
self.Bind(wx.EVT_KEY_DOWN, self.onKeyPress)
self.image_hash_check_in_progress = False
self.duplicate_check_timer = wx.Timer()
self.Bind(wx.EVT_TIMER, self.check_image_hashes, self.duplicate_check_timer)
self.duplicate_check_timer.Start(2000)
return super().OnInit()
def populate_series_grid(self):
self.series_grid.series = []
@@ -717,6 +744,15 @@ class AnonGui(wx.App):
self.series_grid.display_series()
def import_from_dir(self, event):
default_path = str(self.input_dir.resolve())
with wx.DirDialog(
self.frame, "Choose a directory", style=wx.DD_DEFAULT_STYLE, defaultPath=default_path
) as dlg:
if dlg.ShowModal() == wx.ID_OK:
load_path = Path(dlg.GetPath())
self.AnnonymiseFilesInDir(load_path, self.dir)
def AnnonymiseFilesInDir(self, input_dir, output_dir):
logger.debug("AnnonymiseFilesInDir")
logger.debug(f"input: {input_dir} / output {output_dir}")
@@ -726,21 +762,26 @@ class AnonGui(wx.App):
else:
return
if not files:
logger.debug("No files to annonymise")
wx.adv.NotificationMessage("Annonymise files", "No files to annonymise").Show()
return
logger.debug(files)
#files = order_files_by_dicom_attribute(files)
# files = order_files_by_dicom_attribute(files)
#progress_dialog = wx.ProgressDialog(
# progress_dialog = wx.ProgressDialog(
# "Annonymise files",
# "Removing data...",
# maximum=len(files),
# style=wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME | wx.PD_ESTIMATED_TIME,
#)
# )
with self.progress(len(files), "Annonymise files") as p:
#p.SetRange(len(files))
# p.SetRange(len(files))
for n, file in enumerate(files):
#progress_dialog.Update(n)
# progress_dialog.Update(n)
p.update(n)
new_filepath: Path
# Get next available filename
@@ -751,36 +792,39 @@ class AnonGui(wx.App):
i += 1
# new_filepath = Path(output_dir).joinpath(f"IMG_{i:03}.dcm")
dataset, filepath = anonymize_file(file, new_filepath)
dataset, filepath = anonymize_file(file, new_filepath, remove_original=self.remove_files_on_annonymisation)
self.add_datails_to_file_detail_map(dataset, filepath)
#progress_dialog.Destroy()
# progress_dialog.Destroy()
self.LoadDir(self.dir)
@contextlib.contextmanager
def progress(self, total: int, text: str="Working"):
#self.progress_sizer.Show()
def progress(self, total: int, text: str = "Working"):
# self.progress_sizer.Show()
self.progress_text.SetLabelText(text)
self.progress_bar.SetRange(total)
self.progress_sizer.ShowItems(show=True)
self.panel.Layout()
#self.panel.sizer.Fit(self.panel)
self.panel.sizer.Fit(self.panel)
try:
class P():
class P:
def __init__(self, gui, total, text):
self.gui = gui
self.gui = gui
self.text = text
self.total = total
def update(self, n: int):
self.gui.progress_bar.SetValue(n)
self.gui.progress_text.SetLabelText(f"{self.text} : {n}")
yield P(self, total, text)
finally:
#self.progress_sizer.Hide()
# self.progress_sizer.Hide()
self.progress_sizer.ShowItems(show=False)
self.panel.Layout()
#self.panel.SetAutoLayout(1)
#self.panel.sizer.Fit(self.panel)
# self.panel.SetAutoLayout(1)
# self.panel.sizer.Fit(self.panel)
def onKeyPress(self, event):
keycode = event.GetKeyCode()
@@ -822,6 +866,7 @@ class AnonGui(wx.App):
return super().OnExit()
def LoadDir(self, dir=None, refresh=True, clear_file_detail_map=False):
"""Loads data from the output directory (annonymised files)"""
if clear_file_detail_map:
self.file_detail_map = {}
self.file_duplicate_check = {}
@@ -871,12 +916,20 @@ class AnonGui(wx.App):
for i in range(self.list_ctrl.GetItemCount()):
if self.list_ctrl.IsSelected(i):
path = self.list_ctrl.GetItemPyData(i)["file"]
if path in self.file_duplicate_check and self.file_duplicate_check[path]["id"]:
if (
path in self.file_duplicate_check
and self.file_duplicate_check[path]["id"]
):
menu.Append(3, "View on site")
menu.Bind(wx.EVT_MENU, lambda evt: webbrowser.open(f"{self.base_site_url}{self.file_duplicate_check[path]['url']}"), id=3)
menu.Bind(
wx.EVT_MENU,
lambda evt: webbrowser.open(
f"{self.base_site_url}{self.file_duplicate_check[path]['url']}"
),
id=3,
)
break
self.frame.PopupMenu(menu)
def ViewDicom(self, event):
@@ -900,13 +953,15 @@ class AnonGui(wx.App):
tempdir = Path(tempfile.gettempdir())
to_move = []
for i in range(self.list_ctrl.GetItemCount()):
path = self.list_ctrl.GetItemPyData(i)["file"]
logger.debug("--", path)
# paths.append(path)
new_path = annonymise(path, tempdir, run_gui=False)
os.remove(path)
to_move.append((new_path, Path(self.dir, new_path.name)))
with self.progress(self.list_ctrl.GetItemCount(), "Reannonymise files") as p:
for i in range(self.list_ctrl.GetItemCount()):
p.update(i)
path = self.list_ctrl.GetItemPyData(i)["file"]
logger.debug("--", path)
# paths.append(path)
new_path = annonymise(path, tempdir, run_gui=False)
os.remove(path)
to_move.append((new_path, Path(self.dir, new_path.name)))
for path, new_path in to_move:
logger.debug("**", path, new_path)
@@ -956,11 +1011,11 @@ class AnonGui(wx.App):
dlg.ShowModal()
def ClearFiles(self, event=None, load_dir: bool=True):
def ClearFiles(self, event=None, load_dir: bool = True):
files = Path(self.dir).glob("*")
for f in files:
os.remove(f)
if load_dir:
self.LoadDir(clear_file_detail_map=True)
@@ -985,7 +1040,6 @@ class AnonGui(wx.App):
slice_location = str(dataset.SliceLocation)
except:
slice_location = "..."
series_instances_uid = str(dataset.SeriesInstanceUID)
d2 = {
@@ -995,36 +1049,36 @@ class AnonGui(wx.App):
"name": patient_name,
}
#md5 = hashlib.md5()
#first = True
#for i in dataset.pixel_array.astype(str).flatten():
# md5 = hashlib.md5()
# first = True
# for i in dataset.pixel_array.astype(str).flatten():
# if first:
# first = False
# md5.update(f"{i}".encode())
# else:
# md5.update(f",{i}".encode())
#md5.update(dataset.PixelData)
#md5.update("".join(dataset.pixel_array.astype(str).flatten()).encode())
#md5.update(np.char.join("", dataset.pixel_array.astype(str).flatten()).encode())
#hash = md5.hexdigest()
#print(hash)
# md5.update(dataset.PixelData)
# md5.update("".join(dataset.pixel_array.astype(str).flatten()).encode())
# md5.update(np.char.join("", dataset.pixel_array.astype(str).flatten()).encode())
# hash = md5.hexdigest()
# print(hash)
#md5 = hashlib.md5()
# md5 = hashlib.md5()
##hash = md5.hexdigest()
##md5 = hashlib.md5()
##first = True
##s = ",".join([f"{i}" for i in dataset.pixel_array.astype(str).flatten()]).encode()
##print(s)
#print(dataset.PixelData[:20])
#print(len(dataset.PixelData))
#md5.update(dataset.PixelData)
# print(dataset.PixelData[:20])
# print(len(dataset.PixelData))
# md5.update(dataset.PixelData)
#hash = md5.hexdigest()
# hash = md5.hexdigest()
hasher = blake3()
hasher.update(dataset.PixelData)
hash = hasher.hexdigest()
#dataset.PixelData
# dataset.PixelData
d = (
file.name,
@@ -1034,23 +1088,23 @@ class AnonGui(wx.App):
series_desc,
slice_location,
hash,
series_instances_uid
series_instances_uid,
)
#hasher = blake3()
#first = True
#for i in dataset.pixel_array.astype(str).flatten():
# hasher = blake3()
# first = True
# for i in dataset.pixel_array.astype(str).flatten():
# if first:
# first = False
# hasher.update(f"{i}".encode())
# else:
# hasher.update(f",{i}".encode())
#hash = hasher.digest()
# hash = hasher.digest()
#hash = get_file_hash(file, hasher="md5")
# hash = get_file_hash(file, hasher="md5")
#hash = None # If we don't use the hash...
# hash = None # If we don't use the hash...
self.file_detail_map[file] = d, hash, d2
return d, hash, d2
@@ -1061,14 +1115,14 @@ class AnonGui(wx.App):
n = 0
DATA = {}
#progress_dialog = wx.ProgressDialog(
# progress_dialog = wx.ProgressDialog(
# "Loading files",
# "Extracting data from files",
# maximum=len(self.files),
# style=wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME | wx.PD_ESTIMATED_TIME,
#)
# )
with self.progress((len(self.files)), "Loading files") as p:
#p.SetRange(len(self.files))
# p.SetRange(len(self.files))
self.series_map = defaultdict(list)
self.series_map_details = defaultdict(dict)
@@ -1076,7 +1130,7 @@ class AnonGui(wx.App):
hashes = []
for n, f in enumerate(self.files):
#progress_dialog.Update(n)
# progress_dialog.Update(n)
p.update(n)
# dataset = pydicom.read_file(f)
@@ -1094,16 +1148,15 @@ class AnonGui(wx.App):
hashes.append(hash)
series_instances_uid = d2["series_instances_uid"]
try:
self.series_map[series_instances_uid].append(f)
self.series_map_details[series_instances_uid][
"series_description"
] = d2["series_description"]
self.series_map_details[series_instances_uid]["study_description"] = d2[
self.series_map_details[series_instances_uid][
"study_description"
]
] = d2["study_description"]
self.series_map_details[series_instances_uid]["name"] = d2["name"]
except:
logger.debug(f"Unable to get Series Instance UID: {f.name}")
@@ -1122,10 +1175,9 @@ class AnonGui(wx.App):
self.list_ctrl.UpdateData(DATA)
self.populate_series_grid()
# progress_dialog.Destroy()
#progress_dialog.Destroy()
#if hashes:
# if hashes:
# self.check_image_hashes(hashes)
def check_image_hashes(self, evt):
@@ -1133,15 +1185,12 @@ class AnonGui(wx.App):
self.image_hash_check_in_progress = True
to_check = self.file_detail_map.keys() - self.file_duplicate_check.keys()
print(to_check)
if to_check:
hash_file_map = {}
for file in to_check:
hash_file_map[self.file_detail_map[file][1]] = file
hashes = list(hash_file_map.keys())
self.rqst.headers["X-CSRFToken"] = self.rqst.cookies["csrftoken"]
@@ -1150,7 +1199,7 @@ class AnonGui(wx.App):
resp = self.rqst.post(
f"{self.base_site_url}/api/atlas/check_image_hashes/",
data=json.dumps(hashes),
#files=files,
# files=files,
)
for hash, dup in resp.json().items():
@@ -1180,14 +1229,14 @@ class AnonGui(wx.App):
files_to_upload[i : i + n] for i in range(0, len(files_to_upload), n)
]
#progress_dialog = wx.ProgressDialog(
# progress_dialog = wx.ProgressDialog(
# "Uploading files",
# "Uploading files. This is done in batches...",
# maximum=len(chunked_files),
# style=wx.PD_ELAPSED_TIME | wx.PD_ESTIMATED_TIME,
#)
# )
with self.progress(len(chunked_files), "Uploading files (in batches)") as p:
#p.SetRange(len(chunked_files))
# p.SetRange(len(chunked_files))
upload_file_list = []
duplicate_file_list = []
@@ -1209,7 +1258,7 @@ class AnonGui(wx.App):
return resp
p.update(n)
#progress_dialog.Update(n, f"Uploading batch {n}/{len(chunked_files)}")
# progress_dialog.Update(n, f"Uploading batch {n}/{len(chunked_files)}")
logger.debug(f"n: {n}")
# try to upload the files
@@ -1228,11 +1277,13 @@ class AnonGui(wx.App):
duplicate_file_list.extend(resp.json()["duplicates"])
continue
#progress_dialog.Destroy()
# progress_dialog.Destroy()
print(upload_file_list)
print(duplicate_file_list)
wx.CallAfter(
self.upload_complete_message, upload_file_list, duplicate_file_list
)
@@ -1263,26 +1314,41 @@ class AnonGui(wx.App):
def runGUI(
ctx: typer.Context,
input_dir: Path = "C:\\Temp\\",
#input_dir: Path = "/home/ross/temp/in/",
output_dir: Path = "C:\\temp_upload\\",
#output_dir: Path = "/home/ross/temp/out/",
silent_fail: bool = False,
clear_files: bool = True,
clear_files_on_close: bool = True,
remove_files_on_annonymisation: bool = True,
dev: bool = False,
):
if "linux" in sys.platform:
input_dir = Path("/home/ross/temp/in/")
output_dir = Path("/home/ross/temp/out/")
if ctx.invoked_subcommand is None:
try:
typer.echo("run gui")
except AttributeError: # No console
pass
launchGUI(
input_dir, output_dir, silent_fail, clear_files, clear_files_on_close, dev
input_dir,
output_dir,
silent_fail,
clear_files,
clear_files_on_close,
dev,
remove_files_on_annonymisation,
)
def launchGUI(
input_dir, output_dir, silent_fail, clear_files, clear_files_on_close, dev
input_dir,
output_dir,
silent_fail,
clear_files,
clear_files_on_close,
dev,
remove_files_on_annonymisation,
):
app = AnonGui(
input_dir=input_dir,
@@ -1291,6 +1357,7 @@ def launchGUI(
clear_files=clear_files,
clear_files_on_close=clear_files_on_close,
dev=dev,
remove_files_on_annonymisation=remove_files_on_annonymisation,
)
app.MainLoop()