diff --git a/.gitignore b/.gitignore index 15f1ea8..d9e76a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .env/* .venv/* +logs/* + .pyc \ No newline at end of file diff --git a/anon_gui.py b/anon_gui.py index 71def97..5e38493 100644 --- a/anon_gui.py +++ b/anon_gui.py @@ -1,9 +1,12 @@ from collections import defaultdict +import json import os import threading from typing import Iterable from bs4 import BeautifulSoup +import numpy as np import requests +import webbrowser # from blake3 import blake3 @@ -91,7 +94,7 @@ try: except TypeError: pass try: - logger.add(os.path.join(application_path, "anon_gui_{time}.log")) + logger.add(os.path.join(application_path, "logs", "anon_gui_{time}.log")) except TypeError: pass @@ -144,7 +147,11 @@ app = typer.Typer(add_completion=False) @app.command() @logger.catch() def annonymise( - path: Path, output_dir: Path, run_gui: bool = True, file_list: bool = False + path: Path, + output_dir: Path, + run_gui: bool = True, + file_list: bool = False, + dev: bool = False, ): logger.debug(locals()) if run_gui: # Let the GUI handle it all @@ -162,7 +169,7 @@ def annonymise( elif path.is_file(): if file_list: with open(path) as f: - files = f.read().splitlines() + files = [Path(p) for p in f.read().splitlines()] # files = [path] else: files = [path] @@ -182,6 +189,7 @@ def annonymise( anonymize_file(file, new_filepath) + # progress_dialog.Destroy() return new_filepath @@ -199,7 +207,7 @@ def anonymize_file(file: Path, new_filepath: Path): with pydicom.dcmread(file) as dataset: anonymizer.anonymize(dataset) dataset.save_as(new_filepath) - return new_filepath + return dataset, new_filepath def order_files_by_dicom_attribute( @@ -307,7 +315,7 @@ class LoginDialog(wx.Dialog): """ Check credentials and login """ - LOGIN_URL = f"{self.parent.base_site_url}accounts/login/" + LOGIN_URL = f"{self.parent.base_site_url}/accounts/login/" user = self.user.GetValue() password = self.password.GetValue() @@ -355,7 +363,7 @@ class LoginDialog(wx.Dialog): print(rqst.headers) print(rqst.cookies) - new = rqst.get(f"{self.parent.base_site_url}accounts/api/atlas/get_cases_user") + new = rqst.get(f"{self.parent.base_site_url}/accounts/api/atlas/get_cases_user") print("CASESE", new) print(new.content) @@ -473,10 +481,10 @@ class AnonGui(wx.App): logger.debug(f"Input dir: {input_dir}") logger.debug(f"Output dir: {output_dir}") - self.base_site_url = "https://www.penracourses.org.uk/" + self.base_site_url = "https://www.penracourses.org.uk" if dev: - self.base_site_url = "http://localhost:8000/" + self.base_site_url = "http://localhost:8000" self.login_success = False @@ -548,6 +556,7 @@ class AnonGui(wx.App): self.panel = wx.Panel(self.frame, wx.ID_ANY) self.file_detail_map = {} + self.file_duplicate_check = {} self.list_ctrl = SortableList( self.panel, @@ -564,7 +573,7 @@ class AnonGui(wx.App): self.panel, wx.ID_ANY, "Rapid Uploader", - URL=f"{self.base_site_url}rapids/create/", + URL=f"{self.base_site_url}/rapids/create/", ) # self.panel.sizer.Add(self.link_rapid_upload, 0, wx.ALL, 10) @@ -572,7 +581,7 @@ class AnonGui(wx.App): self.panel, wx.ID_ANY, "Long Series", - URL=f"{self.base_site_url}longs/create/series", + URL=f"{self.base_site_url}/longs/create/series", ) # self.panel.sizer.Add(self.link_long_upload, 0, wx.ALL, 10) hoz_links = wx.BoxSizer(wx.HORIZONTAL) @@ -615,6 +624,8 @@ class AnonGui(wx.App): self.list_ctrl.InsertColumn(3, "Name") self.list_ctrl.InsertColumn(4, "Series Description") self.list_ctrl.InsertColumn(5, "Slice location") + self.list_ctrl.InsertColumn(6, "Hash") + self.list_ctrl.InsertColumn(7, "Series Instance UID") self.AnnonymiseFilesInDir(self.input_dir, self.dir) @@ -662,7 +673,6 @@ class AnonGui(wx.App): self.panel.sizer.Add(self.drag_button, 2, wx.EXPAND) self.panel.sizer.Add(self.upload_button, 2, wx.EXPAND) - # Layout sizers self.panel.SetSizer(self.panel.sizer) self.panel.SetAutoLayout(1) @@ -672,6 +682,12 @@ class AnonGui(wx.App): self.Bind(wx.EVT_KEY_DOWN, self.onKeyPress) + + self.image_hash_check_in_progress = False + self.duplicate_check_timer = wx.Timer() + self.Bind(wx.EVT_TIMER, self.check_image_hashes, self.duplicate_check_timer) + self.duplicate_check_timer.Start(2000) + return super().OnInit() def populate_series_grid(self): @@ -699,7 +715,7 @@ class AnonGui(wx.App): logger.debug(files) - files = order_files_by_dicom_attribute(files) + #files = order_files_by_dicom_attribute(files) progress_dialog = wx.ProgressDialog( "Annonymise files", @@ -719,7 +735,8 @@ class AnonGui(wx.App): i += 1 # new_filepath = Path(output_dir).joinpath(f"IMG_{i:03}.dcm") - anonymize_file(file, new_filepath) + dataset, filepath = anonymize_file(file, new_filepath) + self.add_datails_to_file_detail_map(dataset, filepath) progress_dialog.Destroy() self.LoadDir(self.dir) @@ -766,6 +783,7 @@ class AnonGui(wx.App): def LoadDir(self, dir=None, refresh=True, clear_file_detail_map=False): if clear_file_detail_map: self.file_detail_map = {} + self.file_duplicate_check = {} if dir is None: dir = self.dir if refresh: @@ -808,13 +826,23 @@ class AnonGui(wx.App): menu.Bind(wx.EVT_MENU, self.DeleteItems, id=1) menu.Append(2, "View Dicom") menu.Bind(wx.EVT_MENU, self.ViewDicom, id=2) + + for i in range(self.list_ctrl.GetItemCount()): + if self.list_ctrl.IsSelected(i): + path = self.list_ctrl.GetItemPyData(i)["file"] + if path in self.file_duplicate_check and self.file_duplicate_check[path]["id"]: + menu.Append(3, "View on site") + menu.Bind(wx.EVT_MENU, lambda evt: webbrowser.open(f"{self.base_site_url}{self.file_duplicate_check[path]['url']}"), id=3) + break + + self.frame.PopupMenu(menu) def ViewDicom(self, event): selectedItems = [] for i in range(self.list_ctrl.GetItemCount()): if self.list_ctrl.IsSelected(i): - #path = self.list_ctrl.GetItemPyData(i["file"]) + # path = self.list_ctrl.GetItemPyData(i["file"]) path = self.list_ctrl.GetItemPyData(i)["file"] print(path) @@ -832,7 +860,7 @@ class AnonGui(wx.App): tempdir = Path(tempfile.gettempdir()) to_move = [] for i in range(self.list_ctrl.GetItemCount()): - path = self.list_ctrl.GetItemPyData(i["file"]) + path = self.list_ctrl.GetItemPyData(i)["file"] logger.debug("--", path) # paths.append(path) new_path = annonymise(path, tempdir, run_gui=False) @@ -853,7 +881,7 @@ class AnonGui(wx.App): tempdir = Path(tempfile.gettempdir()) to_move = [] for i in range(self.list_ctrl.GetItemCount()): - path = self.list_ctrl.GetItemPyData(i["file"]) + path = self.list_ctrl.GetItemPyData(i)["file"] logger.debug("--", path) # paths.append(path) @@ -873,7 +901,7 @@ class AnonGui(wx.App): selectedItems = [] for i in range(self.list_ctrl.GetItemCount()): if self.list_ctrl.IsSelected(i): - selectedItems.append(self.list_ctrl.GetItemPyData(i["file"])) + selectedItems.append(self.list_ctrl.GetItemPyData(i)["file"]) logger.debug(selectedItems) for f in selectedItems: @@ -893,6 +921,97 @@ class AnonGui(wx.App): os.remove(f) self.LoadDir(clear_file_detail_map=True) + def add_datails_to_file_detail_map(self, dataset, file): + try: + study_desc = dataset.StudyDescription + except: + study_desc = "..." + try: + series_desc = dataset.SeriesDescription + except: + series_desc = "..." + try: + instance_number = f"{dataset.InstanceNumber:03}" + except: + instance_number = "..." + try: + patient_name = str(dataset.PatientName) + except: + patient_name = "..." + try: + slice_location = str(dataset.SliceLocation) + except: + slice_location = "..." + + + series_instances_uid = str(dataset.SeriesInstanceUID) + d2 = { + "series_instances_uid": series_instances_uid, + "series_description": series_desc, + "study_description": study_desc, + "name": patient_name, + } + + #md5 = hashlib.md5() + #first = True + #for i in dataset.pixel_array.astype(str).flatten(): + # if first: + # first = False + # md5.update(f"{i}".encode()) + # else: + # md5.update(f",{i}".encode()) + #md5.update(dataset.PixelData) + #md5.update("".join(dataset.pixel_array.astype(str).flatten()).encode()) + #md5.update(np.char.join("", dataset.pixel_array.astype(str).flatten()).encode()) + #hash = md5.hexdigest() + #print(hash) + + #md5 = hashlib.md5() + ##hash = md5.hexdigest() + ##md5 = hashlib.md5() + ##first = True + ##s = ",".join([f"{i}" for i in dataset.pixel_array.astype(str).flatten()]).encode() + ##print(s) + #print(dataset.PixelData[:20]) + #print(len(dataset.PixelData)) + #md5.update(dataset.PixelData) + + #hash = md5.hexdigest() + + hasher = blake3() + hasher.update(dataset.PixelData) + hash = hasher.hexdigest() + #dataset.PixelData + + d = ( + file.name, + study_desc, + instance_number, + patient_name, + series_desc, + slice_location, + hash, + series_instances_uid + ) + + #hasher = blake3() + #first = True + #for i in dataset.pixel_array.astype(str).flatten(): + # if first: + # first = False + # hasher.update(f"{i}".encode()) + # else: + # hasher.update(f",{i}".encode()) + + #hash = hasher.digest() + + #hash = get_file_hash(file, hasher="md5") + + #hash = None # If we don't use the hash... + self.file_detail_map[file] = d, hash, d2 + + return d, hash, d2 + def UpdateListCtrl(self, event=None): logger.debug("update list ctrl") self.list_ctrl.DeleteAllItems() @@ -909,69 +1028,26 @@ class AnonGui(wx.App): self.series_map = defaultdict(list) self.series_map_details = defaultdict(dict) + hashes = [] + for n, f in enumerate(self.files): progress_dialog.Update(n) # dataset = pydicom.read_file(f) if f in self.file_detail_map: d, hash, d2 = self.file_detail_map[f] + print("in map") else: try: dataset = pydicom.dcmread(f) except FileNotFoundError: return - try: - study_desc = dataset.StudyDescription - except: - study_desc = "..." - try: - series_desc = dataset.SeriesDescription - except: - series_desc = "..." - try: - instance_number = f"{dataset.InstanceNumber:03}" - except: - instance_number = "..." - try: - patient_name = str(dataset.PatientName) - except: - patient_name = "..." - try: - slice_location = str(dataset.SliceLocation) - except: - slice_location = "..." + print("recalc") + d, hash, d2 = self.add_datails_to_file_detail_map(dataset, f) - d = ( - f.name, - study_desc, - instance_number, - patient_name, - series_desc, - slice_location, - ) + hashes.append(hash) - series_instances_uid = str(dataset.SeriesInstanceUID) - d2 = { - "series_instances_uid": series_instances_uid, - "series_description": series_desc, - "study_description": study_desc, - "name": patient_name, - } - - #md5 = hashlib.md5() - #first = True - #for i in dataset.pixel_array.astype(str).flatten(): - # if first: - # first = False - # md5.update(f"{i}".encode()) - # else: - # md5.update(f",{i}".encode()) - - #hash = md5.hexdigest() - - hash = None # If we don't use the hash... - self.file_detail_map[f] = d, hash, d2 series_instances_uid = d2["series_instances_uid"] try: @@ -991,12 +1067,53 @@ class AnonGui(wx.App): self.list_ctrl.Append(d) self.list_ctrl.SetItemPyData(n, {"file": f, "hash": hash}) self.list_ctrl.SetItemData(n, n) + + if f in self.file_duplicate_check: + if self.file_duplicate_check[f]["id"]: + self.list_ctrl.SetItemBackgroundColour(n, wx.RED) + n = n + 1 self.list_ctrl.UpdateData(DATA) self.populate_series_grid() + progress_dialog.Destroy() + #if hashes: + # self.check_image_hashes(hashes) + + def check_image_hashes(self, evt): + if self.login_success and not self.image_hash_check_in_progress: + self.image_hash_check_in_progress = True + + to_check = self.file_detail_map.keys() - self.file_duplicate_check.keys() + print(to_check) + + if to_check: + hash_file_map = {} + for file in to_check: + hash_file_map[self.file_detail_map[file][1]] = file + + + + hashes = list(hash_file_map.keys()) + + self.rqst.headers["X-CSRFToken"] = self.rqst.cookies["csrftoken"] + # print(self.rqst.headers) + # print(self.rqst.cookies) + resp = self.rqst.post( + f"{self.base_site_url}/api/atlas/check_image_hashes/", + data=json.dumps(hashes), + #files=files, + ) + + for hash, dup in resp.json().items(): + self.file_duplicate_check[hash_file_map[hash]] = dup + + print(self.file_duplicate_check) + wx.CallAfter(self.UpdateListCtrl) + self.image_hash_check_in_progress = False + def OnColRightClick(self, event): item = self.list_ctrl.GetColumn(event.GetColumn()) logger.debug(item) @@ -1013,14 +1130,16 @@ class AnonGui(wx.App): # chunck files n = 10 - chunked_files = [files_to_upload[i:i + n] for i in range(0, len(files_to_upload), n)] + chunked_files = [ + files_to_upload[i : i + n] for i in range(0, len(files_to_upload), n) + ] progress_dialog = wx.ProgressDialog( "Uploading files", "Uploading files. This is done in batches...", maximum=len(chunked_files), style=wx.PD_ELAPSED_TIME | wx.PD_ESTIMATED_TIME, - ) + ) upload_file_list = [] duplicate_file_list = [] @@ -1029,10 +1148,10 @@ class AnonGui(wx.App): def upload_files(files): self.rqst.headers["X-CSRFToken"] = self.rqst.cookies["csrftoken"] - #print(self.rqst.headers) - #print(self.rqst.cookies) + # print(self.rqst.headers) + # print(self.rqst.cookies) resp = self.rqst.post( - f"{self.base_site_url}api/atlas/upload_dicom", + f"{self.base_site_url}/api/atlas/upload_dicom", # data=data, files=files, ) @@ -1041,8 +1160,6 @@ class AnonGui(wx.App): logger.debug(f"{resp.content}") return resp - - progress_dialog.Update(n, f"Uploading batch {n}/{len(chunked_files)}") logger.debug(f"n: {n}") @@ -1052,7 +1169,6 @@ class AnonGui(wx.App): upload_file_list.extend(resp.json()["uploaded"]) duplicate_file_list.extend(resp.json()["duplicates"]) - continue logger.debug(f"n: {n} fail") @@ -1068,11 +1184,15 @@ class AnonGui(wx.App): print(upload_file_list) print(duplicate_file_list) - wx.CallAfter(self.upload_complete_message, upload_file_list, duplicate_file_list) - + wx.CallAfter( + self.upload_complete_message, upload_file_list, duplicate_file_list + ) def upload_complete_message(self, upload_file_list, duplicate_file_list): - wx.MessageBox(f"Files uploaded: {len(upload_file_list)}\nDuplicate files (not uploaded): {len(duplicate_file_list)}") + wx.MessageBox( + f"Files uploaded: {len(upload_file_list)}\nDuplicate files (not uploaded): {len(duplicate_file_list)}" + ) + self.file_duplicate_check = {} def OnDragInit(self, event): my_data = wx.FileDataObject() @@ -1093,10 +1213,10 @@ class AnonGui(wx.App): @app.callback(invoke_without_command=True) def runGUI( ctx: typer.Context, - input_dir: Path = "C:\\Temp\\", - # input_dir: Path = "/home/ross/temp/in/", - output_dir: Path = "C:\\temp_upload\\", - # output_dir: Path = "/home/ross/temp/out/", + #input_dir: Path = "C:\\Temp\\", + input_dir: Path = "/home/ross/temp/in/", + #output_dir: Path = "C:\\temp_upload\\", + output_dir: Path = "/home/ross/temp/out/", silent_fail: bool = False, clear_files: bool = True, clear_files_on_close: bool = True, diff --git a/wxDicomViewer.py b/wxDicomViewer.py index 1adaefd..4787f1b 100644 --- a/wxDicomViewer.py +++ b/wxDicomViewer.py @@ -210,7 +210,7 @@ class ViewerFrame(wx.Frame): if self.loadCentered: bmpX0 = (windowsize[0] - self.bitmap.Width) / 2 bmpY0 = (windowsize[1] - self.bitmap.Height) / 2 - dc.DrawBitmap(self.bitmap, bmpX0, bmpY0, False) + dc.DrawBitmap(self.bitmap, int(bmpX0), int(bmpY0), False) # ------------------------------------------------------------ # ViewerFrame.ConvertWXToPIL()