ross/uploader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

few updates

Browse Source
This commit is contained in:
Ross
2024-01-03 19:15:47 +00:00
parent 7812e803c9
commit 393075178a
3 changed files with 204 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
anon_gui.py
+201 -81
View File
@@ -1,9 +1,12 @@
from collections import defaultdict
import json
import os
import threading
from typing import Iterable
from bs4 import BeautifulSoup
import numpy as np
import requests
import webbrowser
# from blake3 import blake3
@@ -91,7 +94,7 @@ try:
except TypeError:
pass
try:
logger.add(os.path.join(application_path, "anon_gui_{time}.log"))
logger.add(os.path.join(application_path, "logs", "anon_gui_{time}.log"))
except TypeError:
pass
@@ -144,7 +147,11 @@ app = typer.Typer(add_completion=False)
@app.command()
@logger.catch()
def annonymise(
path: Path, output_dir: Path, run_gui: bool = True, file_list: bool = False
path: Path,
output_dir: Path,
run_gui: bool = True,
file_list: bool = False,
dev: bool = False,
):
logger.debug(locals())
if run_gui: # Let the GUI handle it all
@@ -162,7 +169,7 @@ def annonymise(
elif path.is_file():
if file_list:
with open(path) as f:
files = f.read().splitlines()
files = [Path(p) for p in f.read().splitlines()]
# files = [path]
else:
files = [path]
@@ -182,6 +189,7 @@ def annonymise(
anonymize_file(file, new_filepath)
# progress_dialog.Destroy()
return new_filepath
@@ -199,7 +207,7 @@ def anonymize_file(file: Path, new_filepath: Path):
with pydicom.dcmread(file) as dataset:
anonymizer.anonymize(dataset)
dataset.save_as(new_filepath)
return new_filepath
return dataset, new_filepath
def order_files_by_dicom_attribute(
@@ -307,7 +315,7 @@ class LoginDialog(wx.Dialog):
"""
Check credentials and login
"""
LOGIN_URL = f"{self.parent.base_site_url}accounts/login/"
LOGIN_URL = f"{self.parent.base_site_url}/accounts/login/"
user = self.user.GetValue()
password = self.password.GetValue()
@@ -355,7 +363,7 @@ class LoginDialog(wx.Dialog):
print(rqst.headers)
print(rqst.cookies)
new = rqst.get(f"{self.parent.base_site_url}accounts/api/atlas/get_cases_user")
new = rqst.get(f"{self.parent.base_site_url}/accounts/api/atlas/get_cases_user")
print("CASESE", new)
print(new.content)
@@ -473,10 +481,10 @@ class AnonGui(wx.App):
logger.debug(f"Input dir: {input_dir}")
logger.debug(f"Output dir: {output_dir}")
self.base_site_url = "https://www.penracourses.org.uk/"
self.base_site_url = "https://www.penracourses.org.uk"
if dev:
self.base_site_url = "http://localhost:8000/"
self.base_site_url = "http://localhost:8000"
self.login_success = False
@@ -548,6 +556,7 @@ class AnonGui(wx.App):
self.panel = wx.Panel(self.frame, wx.ID_ANY)
self.file_detail_map = {}
self.file_duplicate_check = {}
self.list_ctrl = SortableList(
self.panel,
@@ -564,7 +573,7 @@ class AnonGui(wx.App):
self.panel,
wx.ID_ANY,
"Rapid Uploader",
URL=f"{self.base_site_url}rapids/create/",
URL=f"{self.base_site_url}/rapids/create/",
)
# self.panel.sizer.Add(self.link_rapid_upload, 0, wx.ALL, 10)
@@ -572,7 +581,7 @@ class AnonGui(wx.App):
self.panel,
wx.ID_ANY,
"Long Series",
URL=f"{self.base_site_url}longs/create/series",
URL=f"{self.base_site_url}/longs/create/series",
)
# self.panel.sizer.Add(self.link_long_upload, 0, wx.ALL, 10)
hoz_links = wx.BoxSizer(wx.HORIZONTAL)
@@ -615,6 +624,8 @@ class AnonGui(wx.App):
self.list_ctrl.InsertColumn(3, "Name")
self.list_ctrl.InsertColumn(4, "Series Description")
self.list_ctrl.InsertColumn(5, "Slice location")
self.list_ctrl.InsertColumn(6, "Hash")
self.list_ctrl.InsertColumn(7, "Series Instance UID")
self.AnnonymiseFilesInDir(self.input_dir, self.dir)
@@ -662,7 +673,6 @@ class AnonGui(wx.App):
self.panel.sizer.Add(self.drag_button, 2, wx.EXPAND)
self.panel.sizer.Add(self.upload_button, 2, wx.EXPAND)
# Layout sizers
self.panel.SetSizer(self.panel.sizer)
self.panel.SetAutoLayout(1)
@@ -672,6 +682,12 @@ class AnonGui(wx.App):
self.Bind(wx.EVT_KEY_DOWN, self.onKeyPress)
self.image_hash_check_in_progress = False
self.duplicate_check_timer = wx.Timer()
self.Bind(wx.EVT_TIMER, self.check_image_hashes, self.duplicate_check_timer)
self.duplicate_check_timer.Start(2000)
return super().OnInit()
def populate_series_grid(self):
@@ -699,7 +715,7 @@ class AnonGui(wx.App):
logger.debug(files)
files = order_files_by_dicom_attribute(files)
#files = order_files_by_dicom_attribute(files)
progress_dialog = wx.ProgressDialog(
"Annonymise files",
@@ -719,7 +735,8 @@ class AnonGui(wx.App):
i += 1
# new_filepath = Path(output_dir).joinpath(f"IMG_{i:03}.dcm")
anonymize_file(file, new_filepath)
dataset, filepath = anonymize_file(file, new_filepath)
self.add_datails_to_file_detail_map(dataset, filepath)
progress_dialog.Destroy()
self.LoadDir(self.dir)
@@ -766,6 +783,7 @@ class AnonGui(wx.App):
def LoadDir(self, dir=None, refresh=True, clear_file_detail_map=False):
if clear_file_detail_map:
self.file_detail_map = {}
self.file_duplicate_check = {}
if dir is None:
dir = self.dir
if refresh:
@@ -808,13 +826,23 @@ class AnonGui(wx.App):
menu.Bind(wx.EVT_MENU, self.DeleteItems, id=1)
menu.Append(2, "View Dicom")
menu.Bind(wx.EVT_MENU, self.ViewDicom, id=2)
for i in range(self.list_ctrl.GetItemCount()):
if self.list_ctrl.IsSelected(i):
path = self.list_ctrl.GetItemPyData(i)["file"]
if path in self.file_duplicate_check and self.file_duplicate_check[path]["id"]:
menu.Append(3, "View on site")
menu.Bind(wx.EVT_MENU, lambda evt: webbrowser.open(f"{self.base_site_url}{self.file_duplicate_check[path]['url']}"), id=3)
break
self.frame.PopupMenu(menu)
def ViewDicom(self, event):
selectedItems = []
for i in range(self.list_ctrl.GetItemCount()):
if self.list_ctrl.IsSelected(i):
#path = self.list_ctrl.GetItemPyData(i["file"])
# path = self.list_ctrl.GetItemPyData(i["file"])
path = self.list_ctrl.GetItemPyData(i)["file"]
print(path)
@@ -832,7 +860,7 @@ class AnonGui(wx.App):
tempdir = Path(tempfile.gettempdir())
to_move = []
for i in range(self.list_ctrl.GetItemCount()):
path = self.list_ctrl.GetItemPyData(i["file"])
path = self.list_ctrl.GetItemPyData(i)["file"]
logger.debug("--", path)
# paths.append(path)
new_path = annonymise(path, tempdir, run_gui=False)
@@ -853,7 +881,7 @@ class AnonGui(wx.App):
tempdir = Path(tempfile.gettempdir())
to_move = []
for i in range(self.list_ctrl.GetItemCount()):
path = self.list_ctrl.GetItemPyData(i["file"])
path = self.list_ctrl.GetItemPyData(i)["file"]
logger.debug("--", path)
# paths.append(path)
@@ -873,7 +901,7 @@ class AnonGui(wx.App):
selectedItems = []
for i in range(self.list_ctrl.GetItemCount()):
if self.list_ctrl.IsSelected(i):
selectedItems.append(self.list_ctrl.GetItemPyData(i["file"]))
selectedItems.append(self.list_ctrl.GetItemPyData(i)["file"])
logger.debug(selectedItems)
for f in selectedItems:
@@ -893,6 +921,97 @@ class AnonGui(wx.App):
os.remove(f)
self.LoadDir(clear_file_detail_map=True)
def add_datails_to_file_detail_map(self, dataset, file):
try:
study_desc = dataset.StudyDescription
except:
study_desc = "..."
try:
series_desc = dataset.SeriesDescription
except:
series_desc = "..."
try:
instance_number = f"{dataset.InstanceNumber:03}"
except:
instance_number = "..."
try:
patient_name = str(dataset.PatientName)
except:
patient_name = "..."
try:
slice_location = str(dataset.SliceLocation)
except:
slice_location = "..."
series_instances_uid = str(dataset.SeriesInstanceUID)
d2 = {
"series_instances_uid": series_instances_uid,
"series_description": series_desc,
"study_description": study_desc,
"name": patient_name,
}
#md5 = hashlib.md5()
#first = True
#for i in dataset.pixel_array.astype(str).flatten():
# if first:
# first = False
# md5.update(f"{i}".encode())
# else:
# md5.update(f",{i}".encode())
#md5.update(dataset.PixelData)
#md5.update("".join(dataset.pixel_array.astype(str).flatten()).encode())
#md5.update(np.char.join("", dataset.pixel_array.astype(str).flatten()).encode())
#hash = md5.hexdigest()
#print(hash)
#md5 = hashlib.md5()
##hash = md5.hexdigest()
##md5 = hashlib.md5()
##first = True
##s = ",".join([f"{i}" for i in dataset.pixel_array.astype(str).flatten()]).encode()
##print(s)
#print(dataset.PixelData[:20])
#print(len(dataset.PixelData))
#md5.update(dataset.PixelData)
#hash = md5.hexdigest()
hasher = blake3()
hasher.update(dataset.PixelData)
hash = hasher.hexdigest()
#dataset.PixelData
d = (
file.name,
study_desc,
instance_number,
patient_name,
series_desc,
slice_location,
hash,
series_instances_uid
)
#hasher = blake3()
#first = True
#for i in dataset.pixel_array.astype(str).flatten():
# if first:
# first = False
# hasher.update(f"{i}".encode())
# else:
# hasher.update(f",{i}".encode())
#hash = hasher.digest()
#hash = get_file_hash(file, hasher="md5")
#hash = None # If we don't use the hash...
self.file_detail_map[file] = d, hash, d2
return d, hash, d2
def UpdateListCtrl(self, event=None):
logger.debug("update list ctrl")
self.list_ctrl.DeleteAllItems()
@@ -909,69 +1028,26 @@ class AnonGui(wx.App):
self.series_map = defaultdict(list)
self.series_map_details = defaultdict(dict)
hashes = []
for n, f in enumerate(self.files):
progress_dialog.Update(n)
# dataset = pydicom.read_file(f)
if f in self.file_detail_map:
d, hash, d2 = self.file_detail_map[f]
print("in map")
else:
try:
dataset = pydicom.dcmread(f)
except FileNotFoundError:
return
try:
study_desc = dataset.StudyDescription
except:
study_desc = "..."
try:
series_desc = dataset.SeriesDescription
except:
series_desc = "..."
try:
instance_number = f"{dataset.InstanceNumber:03}"
except:
instance_number = "..."
try:
patient_name = str(dataset.PatientName)
except:
patient_name = "..."
try:
slice_location = str(dataset.SliceLocation)
except:
slice_location = "..."
print("recalc")
d, hash, d2 = self.add_datails_to_file_detail_map(dataset, f)
d = (
f.name,
study_desc,
instance_number,
patient_name,
series_desc,
slice_location,
)
hashes.append(hash)
series_instances_uid = str(dataset.SeriesInstanceUID)
d2 = {
"series_instances_uid": series_instances_uid,
"series_description": series_desc,
"study_description": study_desc,
"name": patient_name,
}
#md5 = hashlib.md5()
#first = True
#for i in dataset.pixel_array.astype(str).flatten():
# if first:
# first = False
# md5.update(f"{i}".encode())
# else:
# md5.update(f",{i}".encode())
#hash = md5.hexdigest()
hash = None # If we don't use the hash...
self.file_detail_map[f] = d, hash, d2
series_instances_uid = d2["series_instances_uid"]
try:
@@ -991,12 +1067,53 @@ class AnonGui(wx.App):
self.list_ctrl.Append(d)
self.list_ctrl.SetItemPyData(n, {"file": f, "hash": hash})
self.list_ctrl.SetItemData(n, n)
if f in self.file_duplicate_check:
if self.file_duplicate_check[f]["id"]:
self.list_ctrl.SetItemBackgroundColour(n, wx.RED)
n = n + 1
self.list_ctrl.UpdateData(DATA)
self.populate_series_grid()
progress_dialog.Destroy()
#if hashes:
# self.check_image_hashes(hashes)
def check_image_hashes(self, evt):
if self.login_success and not self.image_hash_check_in_progress:
self.image_hash_check_in_progress = True
to_check = self.file_detail_map.keys() - self.file_duplicate_check.keys()
print(to_check)
if to_check:
hash_file_map = {}
for file in to_check:
hash_file_map[self.file_detail_map[file][1]] = file
hashes = list(hash_file_map.keys())
self.rqst.headers["X-CSRFToken"] = self.rqst.cookies["csrftoken"]
# print(self.rqst.headers)
# print(self.rqst.cookies)
resp = self.rqst.post(
f"{self.base_site_url}/api/atlas/check_image_hashes/",
data=json.dumps(hashes),
#files=files,
)
for hash, dup in resp.json().items():
self.file_duplicate_check[hash_file_map[hash]] = dup
print(self.file_duplicate_check)
wx.CallAfter(self.UpdateListCtrl)
self.image_hash_check_in_progress = False
def OnColRightClick(self, event):
item = self.list_ctrl.GetColumn(event.GetColumn())
logger.debug(item)
@@ -1013,14 +1130,16 @@ class AnonGui(wx.App):
# chunck files
n = 10
chunked_files = [files_to_upload[i:i + n] for i in range(0, len(files_to_upload), n)]
chunked_files = [
files_to_upload[i : i + n] for i in range(0, len(files_to_upload), n)
]
progress_dialog = wx.ProgressDialog(
"Uploading files",
"Uploading files. This is done in batches...",
maximum=len(chunked_files),
style=wx.PD_ELAPSED_TIME | wx.PD_ESTIMATED_TIME,
)
)
upload_file_list = []
duplicate_file_list = []
@@ -1029,10 +1148,10 @@ class AnonGui(wx.App):
def upload_files(files):
self.rqst.headers["X-CSRFToken"] = self.rqst.cookies["csrftoken"]
#print(self.rqst.headers)
#print(self.rqst.cookies)
# print(self.rqst.headers)
# print(self.rqst.cookies)
resp = self.rqst.post(
f"{self.base_site_url}api/atlas/upload_dicom",
f"{self.base_site_url}/api/atlas/upload_dicom",
# data=data,
files=files,
)
@@ -1041,8 +1160,6 @@ class AnonGui(wx.App):
logger.debug(f"{resp.content}")
return resp
progress_dialog.Update(n, f"Uploading batch {n}/{len(chunked_files)}")
logger.debug(f"n: {n}")
@@ -1052,7 +1169,6 @@ class AnonGui(wx.App):
upload_file_list.extend(resp.json()["uploaded"])
duplicate_file_list.extend(resp.json()["duplicates"])
continue
logger.debug(f"n: {n} fail")
@@ -1068,11 +1184,15 @@ class AnonGui(wx.App):
print(upload_file_list)
print(duplicate_file_list)
wx.CallAfter(self.upload_complete_message, upload_file_list, duplicate_file_list)
wx.CallAfter(
self.upload_complete_message, upload_file_list, duplicate_file_list
)
def upload_complete_message(self, upload_file_list, duplicate_file_list):
wx.MessageBox(f"Files uploaded: {len(upload_file_list)}\nDuplicate files (not uploaded): {len(duplicate_file_list)}")
wx.MessageBox(
f"Files uploaded: {len(upload_file_list)}\nDuplicate files (not uploaded): {len(duplicate_file_list)}"
)
self.file_duplicate_check = {}
def OnDragInit(self, event):
my_data = wx.FileDataObject()
@@ -1093,10 +1213,10 @@ class AnonGui(wx.App):
@app.callback(invoke_without_command=True)
def runGUI(
ctx: typer.Context,
input_dir: Path = "C:\\Temp\\",
# input_dir: Path = "/home/ross/temp/in/",
output_dir: Path = "C:\\temp_upload\\",
# output_dir: Path = "/home/ross/temp/out/",
#input_dir: Path = "C:\\Temp\\",
input_dir: Path = "/home/ross/temp/in/",
#output_dir: Path = "C:\\temp_upload\\",
output_dir: Path = "/home/ross/temp/out/",
silent_fail: bool = False,
clear_files: bool = True,
clear_files_on_close: bool = True,