improve duplicate handling

anon_gui.py

@@ -512,6 +512,7 @@ class AnonGui(wx.App):
         self.silent_fail = silent_fail
         self.clear_files_on_start = clear_files
         self.clear_files_on_close = clear_files_on_close
+        self.clear_files_on_upload = True
         self.remove_files_on_annonymisation = remove_files_on_annonymisation
 
         self.rqst = None
@@ -1027,14 +1028,24 @@ base site url: {self.base_site_url}
                     path in self.file_duplicate_check
                     and self.file_duplicate_check[path]["id"]
                 ):
-                    menu.Append(3, "View on site")
-                    menu.Bind(
-                        wx.EVT_MENU,
-                        lambda evt: webbrowser.open(
-                            f"{self.base_site_url}{self.file_duplicate_check[path]['url']}"
-                        ),
-                        id=3,
-                    )
+                    if self.file_duplicate_check[path]["type"] == "series":
+                        menu.Append(3, "View on site (series)")
+                        menu.Bind(
+                            wx.EVT_MENU,
+                            lambda evt: webbrowser.open(
+                                f"{self.base_site_url}{self.file_duplicate_check[path]['url']}"
+                            ),
+                            id=3,
+                        )
+                    else:
+                        menu.Append(3, "View on site (uncategorised)")
+                        menu.Bind(
+                            wx.EVT_MENU,
+                            lambda evt: webbrowser.open(
+                                f"{self.base_site_url}{self.file_duplicate_check[path]['url']}"
+                            ),
+                            id=3,
+                        )
                 break
 
         self.frame.PopupMenu(menu)
@@ -1118,8 +1129,12 @@ base site url: {self.base_site_url}
 
         dlg.ShowModal()
 
-    def ClearFiles(self, event=None, load_dir: bool = True):
-        files = Path(self.dir).glob("*")
+    def ClearFiles(self, event=None, load_dir: bool = True, files:None| list[(str, str)] = None):
+        if files is None:
+            files = Path(self.dir).glob("*")
+        else:
+            files = [Path(self.dir, f) for f, file_hash in files]
+
         for f in files:
             os.remove(f)
 
@@ -1243,7 +1258,7 @@ base site url: {self.base_site_url}
 
                 if f in self.file_detail_map:
                     d, hash, d2 = self.file_detail_map[f]
-                    print("in map")
+                    print(f"{n}: in map")
                 else:
                     try:
                         dataset = pydicom.dcmread(f)
@@ -1329,9 +1344,14 @@ base site url: {self.base_site_url}
 
     def upload(self):
         files_to_upload = []
+        duplicate_files_not_uploaded = []
         logger.debug("Upload files")
         for f in self.files:
-            files_to_upload.append(("files", open(str(f), "rb")))
+            if f in self.file_duplicate_check:
+                # duplicate the api return format so ClearFiles works
+                duplicate_files_not_uploaded.append((f.name, "..."))
+            else:
+                files_to_upload.append(("files", open(str(f), "rb")))
 
         # chunck files
         n = 10
@@ -1350,6 +1370,7 @@ base site url: {self.base_site_url}
 
             upload_file_list = []
             duplicate_file_list = []
+            failed = []
 
             for n, files in enumerate(chunked_files):
 
@@ -1372,35 +1393,41 @@ base site url: {self.base_site_url}
 
                 logger.debug(f"n: {n}")
                 # try to upload the files
-                resp = upload_files(files)
-                if resp.status_code == 200:
-                    upload_file_list.extend(resp.json()["uploaded"])
-                    duplicate_file_list.extend(resp.json()["duplicates"])
 
-                    continue
+                for i in range(3):
+                    resp = upload_files(files)
+                    if resp.status_code == 200:
+                        upload_file_list.extend(resp.json()["uploaded"])
+                        duplicate_file_list.extend(resp.json()["duplicates"])
+                        failed.extend(resp.json()["failed"])
 
-                logger.debug(f"n: {n} fail")
-                # try again if we fail
-                resp = upload_files(files)
-                if resp.status_code == 200:
-                    upload_file_list.extend(resp.json()["uploaded"])
-                    duplicate_file_list.extend(resp.json()["duplicates"])
-                    continue
+                        continue
+
+                    logger.debug(f"n: {n} fail (attempt {i})")
 
         # progress_dialog.Destroy()
 
         print(upload_file_list)
-        print(duplicate_file_list)
+        print("dup", duplicate_file_list)
+        print("failed", failed)
 
 
 
         wx.CallAfter(
-            self.upload_complete_message, upload_file_list, duplicate_file_list
+            self.upload_complete_message, upload_file_list, duplicate_file_list, failed, duplicate_files_not_uploaded
         )
 
-    def upload_complete_message(self, upload_file_list, duplicate_file_list):
+        if self.clear_files_on_upload:
+            all_files = upload_file_list + duplicate_file_list + duplicate_files_not_uploaded
+            print(all_files)
+            self.ClearFiles(event=None, load_dir=True, files=all_files)
+
+    def upload_complete_message(self, upload_file_list, duplicate_file_list, failed, duplicate_files_not_uploaded):
         wx.MessageBox(
-            f"Files uploaded: {len(upload_file_list)}\nDuplicate files (not uploaded): {len(duplicate_file_list)}"
+            f"""Files uploaded: {len(upload_file_list)}
+Duplicate files (not uploaded): {len(duplicate_file_list)}
+Failed to upload: {len(failed)}
+Duplicate files (not uploaded): {len(duplicate_files_not_uploaded)}"""
         )
         self.file_duplicate_check = {}