From fa613ae1165652dcf70c6cc391ecb4bde13839de Mon Sep 17 00:00:00 2001
From: Ross <xkjq@ymail.com>
Date: Mon, 22 Apr 2024 15:17:31 +0100
Subject: [PATCH] improve uploading

---
 anatomy/static/js/upload_form_helpers.js |   2 +-
 atlas/api.py                             |   2 -
 atlas/models.py                          |   1 -
 atlas/templates/atlas/help.html          |   2 +
 atlas/templates/atlas/new_uploads.html   | 224 +++++++++++++++++++++--
 atlas/templates/atlas/user_uploads.html  |   1 +
 rad/static/css/anatomy.css               | 145 ++++++++++++++-
 7 files changed, 356 insertions(+), 21 deletions(-)

diff --git a/anatomy/static/js/upload_form_helpers.js b/anatomy/static/js/upload_form_helpers.js
index f5c04474..4e74615e 100644
--- a/anatomy/static/js/upload_form_helpers.js
+++ b/anatomy/static/js/upload_form_helpers.js
@@ -108,7 +108,7 @@
             uploading_el = $(`img[data-input-id='${input.id}'], div[data-input-id='${input.id}']`)
             uploading_el.removeClass("image-ident-loading");
             console.log("found text", l);
-            if (l.includes("accesion") || l.includes("number") || l.search(
+            if (l.includes("accession") || l.includes("number") || l.search(
                     /(ref|rk9|rh8|ra9|rbz)\d+/g) > -1) {
                 console.log("Match found ", input);
                 $(input).addClass("image-ident-warning");
diff --git a/atlas/api.py b/atlas/api.py
index c9d0a6c6..991188ac 100644
--- a/atlas/api.py
+++ b/atlas/api.py
@@ -54,7 +54,6 @@ class CaseSchema(ModelSchema):
         model_fields = ["id", "title"]
 
 
-@logger.catch()
 @router.post("/upload_dicom", auth=django_auth)
 def upload_dicom(request, files: List[UploadedFile] = File(...)):
     uploaded = []
@@ -69,7 +68,6 @@ def upload_dicom(request, files: List[UploadedFile] = File(...)):
 
             uploaded.append((file.name, ud.image_blake3_hash))
         except DuplicateDicom:
-            print(ud.check_for_duplicates(ud.image_blake3_hash))
             duplicate.append((file.name, ud.image_blake3_hash))
             pass
         except InvalidDicomError:
diff --git a/atlas/models.py b/atlas/models.py
index dc156059..2d861d52 100644
--- a/atlas/models.py
+++ b/atlas/models.py
@@ -1110,7 +1110,6 @@ class UncategorisedDicom(models.Model):
 
         return duplicate
 
-    @logger.catch()
     def save(self, *args, **kwargs):
         """Override save method to add image hash"""
         if self.image:
diff --git a/atlas/templates/atlas/help.html b/atlas/templates/atlas/help.html
index 075901a9..e4830554 100644
--- a/atlas/templates/atlas/help.html
+++ b/atlas/templates/atlas/help.html
@@ -30,5 +30,7 @@
     Available <a href="https://www.penracourses.org.uk/uploader/insight_rad_up.xml">here</a>
     <h4>Step 3: Integrate the uploader with PACs</h4>
     Whilst in PACs open the settings menu with the keyboard shortcut 'Ctrl+Shift+C'. In the 'General' tab, select 'External Tools' and click 'Add'. Press the Import button and select the xml file you downloaded in Step 2. Modify the file to run path to point the uploader exe downloaded in step 1 (clicking on the 3 dots will allow you to browse and select this file). The remaining fields should have been set via the xml file and should not require modification.
+    <h3>Uploading without the uploader tool</h3>
+    If you have <b>annonymised</b> dicom files already exported you can uploading them directly using this <a href='{% url "atlas:new_uploads" %}'>form</a>.
 
 {% endblock %}
diff --git a/atlas/templates/atlas/new_uploads.html b/atlas/templates/atlas/new_uploads.html
index 75988bbd..a5bb971a 100644
--- a/atlas/templates/atlas/new_uploads.html
+++ b/atlas/templates/atlas/new_uploads.html
@@ -10,26 +10,156 @@
         Please note this form will not annonymise files. Please ensure that you have done so prior to using it.
     </div>
 
-    <div>This form allows you to quickly upload dicom files to the server. Select the folder containing the dicom files below.</div>
+    <div>
+        <p>This form allows you to quickly upload dicom files to the server. Select the folder containing the dicom files below. If you wish to upload files directly from Insight you may rather use the <a href='{% url "atlas:help" %}'>Uploader tool.</a></p>
+        <details>
+            <summary>Help</summary>
+            <p>Only valid dicom files will be uploaded. </p>
+            <p>It is possible to upload multiple files by selecting the parent (containing) folder. Folders within will then be searched recursively for any valid dicom files.</p>
+            <p>Files to be uploaded will be displayed below. </p>
+        </details>
+    </div>
 
     <form method="post" enctype="multipart/form-data">
         {% csrf_token %}
         <input type="file" name="dicom_files" id="filepicker" webkitdirectory multiple>
-        <input type="submit" value="Upload">
+        <button type="button" id="uploadButton">Upload</button>
 
     </form>
 
-    <div>Selected files:</div>
-    <div id="single-dicom-viewer" class="dicom-viewer" data-images="" data-annotations=''></div>
-    <ul id="listing"></ul>
+    <details id="upload-results"><summary>Upload results</summary>
+        <div id="upload-results-list">
+            <h5>Uploaded</h5>
+            <ul id="uploaded-files"></ul>
+            <h5>Duplicates</h5>
+            <ul id="duplicate-files"></ul>
+            <h5>Failed</h5>
+            <ul id="failed-files"></ul>
+        </div>
+    </details>
 
 
+    <h4>Selected files:</h4>
+    <div id="single-dicom-viewer" class="dicom-viewer" data-images="" data-annotations=''></div>
+    <ol id="listing"></ol>
+
+
+    <div id="loading" class="fullscreen-overlay">
+        <div class="progress-block">
+            <div id="progress"></div>
+            <div class="sk-cube-grid full">
+                <div class="sk-cube sk-cube1"></div>
+                <div class="sk-cube sk-cube2"></div>
+                <div class="sk-cube sk-cube3"></div>
+                <div class="sk-cube sk-cube4"></div>
+                <div class="sk-cube sk-cube5"></div>
+                <div class="sk-cube sk-cube6"></div>
+                <div class="sk-cube sk-cube7"></div>
+                <div class="sk-cube sk-cube8"></div>
+                <div class="sk-cube sk-cube9"></div>
+            </div>
+            <div id="progress-text">Uploading...</div>
+        </div>
+    </div>
+
 {% endblock %}
 
 
 {% block js %}
     <script src="{% static 'js/upload_form_helpers.js' %}"></script>
     <script>
+        window.to_upload = [];
+        window.upload_count = 1;
+
+        function* chunks(arr, n) {
+            for (let i = 0; i < arr.length; i += n) {
+                yield arr.slice(i, i + n);
+            }
+        }
+
+        function onUploadButtonClicked(event) {
+            event.preventDefault();
+            const fileInput = document.getElementById("filepicker");
+            const selectedFiles = fileInput.files;
+    // Check if any files are selected
+            if (selectedFiles.length === 0) {
+                alert("Please select at least one file to upload.");
+                return;
+            }
+            //alert(`${window.to_upload.length} files selected. Uploading...`);
+
+            window.upload_results = {
+                "uploaded": [],
+                "duplicates": [],
+                "failed": []
+            };
+
+            chunked_files = [...chunks(window.to_upload, 10)];
+
+
+            $("#loading").show();
+            $("#progress-text").text(`Uploading..`);
+            for (let i = 0; i < chunked_files.length; i++) {
+                uploadFiles(chunked_files[i], i, chunked_files.length);
+
+            }
+        }
+
+        function uploadFiles(files_to_upload, current_chunk, total_chunks){
+            console.log("uploading", files_to_upload)
+
+  // Create a FormData object to store the form data
+            const formData = new FormData();
+  // Append each selected file to the FormData object
+            for (let i = 0; i < files_to_upload.length; i++) {
+                formData.append("files", files_to_upload[i]);
+            }
+            console.log("formdata", formData)
+            const xhr = new XMLHttpRequest();
+            xhr.open("POST", "{% url 'api-1:upload_dicom' %}", true);
+            xhr.setRequestHeader("X-CSRFToken", "{{ csrf_token }}");
+            xhr.onreadystatechange = function () {
+                $("#progress-text").text(`Uploading ${window.upload_count} of ${total_chunks} chunks`);
+                if (xhr.readyState === XMLHttpRequest.DONE) {
+                    window.upload_count += 1;
+                    if (xhr.status === 200) {
+                    // Handle successful response from the server
+                        console.log('Files uploaded successfully!');
+                        console.log(xhr.response);
+                        window.upload_results["uploaded"].push(...JSON.parse(xhr.response)["uploaded"]);
+                        window.upload_results["duplicates"].push(...JSON.parse(xhr.response)["duplicates"]);
+                        window.upload_results["failed"].push(...JSON.parse(xhr.response)["failed"]);
+
+                        for (let i = 0; i < JSON.parse(xhr.response)["uploaded"].length; i++) {
+                            let item = document.createElement("li");
+                            item.textContent = JSON.parse(xhr.response)["uploaded"][i];
+                            $("#uploaded-files").append(item);
+                        }
+
+                        for (let i = 0; i < JSON.parse(xhr.response)["duplicates"].length; i++) {
+                            let item = document.createElement("li");
+                            item.textContent = JSON.parse(xhr.response)["duplicates"][i];
+                            $("#duplicate-files").append(item);
+                        }
+
+                        for (let i = 0; i < JSON.parse(xhr.response)["failed"].length; i++) {
+                            let item = document.createElement("li");
+                            item.textContent = JSON.parse(xhr.response)["failed"][i];
+                            $("#failed-files").append(item);
+                        }
+                    } else {
+                   // Handle error response from the server
+                        console.error('Failed to upload files.');
+                        alert("Error occurred during file upload. Please try again.");
+                    }
+                    if (current_chunk + 1 == total_chunks) {
+                        $("#loading").hide();
+                        alert("Uploading complete")
+                    }
+                }
+            };
+            xhr.send(formData);
+        }
 
         function checkAnnonymisation(byteArray) {
         // We need to setup a try/catch block because parseDicom will throw an exception
@@ -63,11 +193,11 @@
                 for (let i = 0; i < site_codes.length; i++) {
                     if (
                         accession_number.startsWith(site_codes[i]) ||
-                        patient_id.startsWith(site_codes[i]) ||
-                        patient_id.startsWith("pat"))
-                         {
-                            not_anon = true;
-                        }
+                        patient_id.startsWith(site_codes[i])
+                    )
+                    {
+                        not_anon = true;
+                    }
                 }
 
                 if (not_anon) {
@@ -88,6 +218,7 @@
                         "showMethod": "fadeIn",
                         "hideMethod": "fadeOut",
                     });
+                    $("uploadButton").prop("disabled", true);
                 }
 
             // TODO: fix
@@ -176,12 +307,19 @@
 
             let output = document.getElementById("listing");
             images = [];
+            window.to_upload = [];
+            $("#loading").show();
+            $("#progress-text").text(`Processing..`);
+            n = 0;
             for (const file of file_set) {
+                n += 1;
+                $("#progress-text").text(`Processing ${file.name} (${n} of ${file_set.length})`);
                     //await readFile(file);
                 dicom = await readDicomFile(file);
 
                 if (dicom) {
                     let item = document.createElement("li");
+                    item.className = "dicom-file";
                     item.textContent = file.webkitRelativePath;
                     output.appendChild(item);
                         //await readFile(file, item);
@@ -191,21 +329,23 @@
                     console.log(url)
                     imageId = `wadouri:${url}`;
                     images.push(imageId);
+                    window.to_upload.push(file);
 
                     const element = $(`<div class='temp-thumb' src=${imageId}></div>`).get(0)
                     item.appendChild(element);
                     cornerstone.enable(element);
                     cornerstone.loadAndCacheImage(imageId).then(function (image) {
                         cornerstone.displayImage(element, image);
+
+                        $(element).addClass("temp-thumb-large");
                         cornerstone.resize(element)
+                        setTimeout(function () {
+                            ocr($(element).find("canvas").get(0).toDataURL(), element);
+                            $(element).removeClass("temp-thumb-large");
+                            cornerstone.resize(element)
+                        }, 500);
 
 
-
-                        let bytesView = image.getPixelData()
-                        let str = new TextDecoder("utf-8").decode(bytesView, );
-
-                    //extractDicomDetails(image.data.byteArray)
-
                     });
                 }
             }
@@ -217,11 +357,58 @@
                 "detail": {"images": images, "annontations": undefined}
             });
 
+            $("#loading").hide()
             window.dispatchEvent(event);
 
 
 
         }
+        function ocr(url, input) {
+            console.log("OCR", input)
+            Tesseract.recognize(
+                url,
+                'eng',
+                //{ logger: m => console.log(m) }
+            ).then(({
+                data: {
+                    text
+                }
+            }) => {
+                l = text.toLowerCase();
+                uploading_el = $(`img[data-input-id='${input.id}'], div[data-input-id='${input.id}']`)
+                uploading_el.removeClass("image-ident-loading");
+                console.log("found text", l);
+                if (l.includes("accession") || l.includes("number") || l.search(
+                        /(ref|rk9|rh8|ra9|rbz)\d+/g) > -1) {
+                    console.log("Match found ", input);
+                    $(input).addClass("image-ident-warning");
+                    uploading_el.addClass("image-ident-warning");
+                    $("uploadButton").prop("disabled", true);
+                    toastr.warning(`File does not appear to be annonymised<br/>Image data burned in<br/>${element}`, 'Anonymisation warning', {
+                        "closeButton": false,
+                        "debug": false,
+                        "newestOnTop": false,
+                        "progressBar": false,
+                        "positionClass": "toast-top-full-width",
+                        "preventDuplicates": false,
+                        "onclick": null,
+                        "showDuration": "0",
+                        "hideDuration": "0",
+                        "timeOut": 0,
+                        "extendedTimeOut": 0,
+                        "showEasing": "swing",
+                        "hideEasing": "linear",
+                        "showMethod": "fadeIn",
+                        "hideMethod": "fadeOut",
+                    });
+                } else {
+                    $(input).addClass("image-ident-ok");
+
+                }
+            })
+
+        }
+
         $(document).ready(function () {
             document.getElementById("filepicker").addEventListener(
                 "change",
@@ -237,6 +424,11 @@
                 },
                 false,
             );
+
+
+        // Add event listener to the button element
+            const uploadButton = document.getElementById("uploadButton");
+            uploadButton.addEventListener("click", onUploadButtonClicked);
         });
 
     </script>
diff --git a/atlas/templates/atlas/user_uploads.html b/atlas/templates/atlas/user_uploads.html
index 2afdd848..a1d2cda2 100644
--- a/atlas/templates/atlas/user_uploads.html
+++ b/atlas/templates/atlas/user_uploads.html
@@ -3,6 +3,7 @@
 {% block content %}
     <h2>Uploaded dicoms</h2>
     User: {{user}}<br/>
+    <a href="{% url 'atlas:new_uploads' %}">Upload more dicoms</a><br/>
     {% if not case %}
         <div class="alert alert-info" role="alert">
             Series imported from here will be orphaned (not associated with a case). It is usually better to create a case first and then import directly into the case.
diff --git a/rad/static/css/anatomy.css b/rad/static/css/anatomy.css
index 23ae4471..6da3aad5 100644
--- a/rad/static/css/anatomy.css
+++ b/rad/static/css/anatomy.css
@@ -130,6 +130,12 @@ button:hover {
   background-color: #52057b;
 }
 
+button:disabled {
+  color: gray;
+  background-color: #110119;
+  border-color: #110119;
+}
+
 
 button a {
   text-decoration: none;
@@ -1122,4 +1128,141 @@ tr:has(> td > a) {
 .no-list-style {
   list-style-type: none;
   padding-left: 1rem;
-}
\ No newline at end of file
+}
+
+#listing .dicom-file {
+  clear: both;
+}
+
+/* Some fancy loading CSS */
+.sk-cube-grid {
+    width: 60px;
+    height: 60px;
+}
+
+.sk-cube-grid .sk-cube {
+    width: 33%;
+    height: 33%;
+    background-color: #4527A0;
+    float: left;
+    -webkit-animation: sk-cubeGridScaleDelay 1.3s infinite ease-in-out;
+    animation: sk-cubeGridScaleDelay 1.3s infinite ease-in-out;
+}
+
+.sk-cube-grid .sk-cube1 {
+    -webkit-animation-delay: 0.2s;
+    animation-delay: 0.2s;
+}
+
+.sk-cube-grid .sk-cube2 {
+    -webkit-animation-delay: 0.3s;
+    animation-delay: 0.3s;
+}
+
+.sk-cube-grid .sk-cube3 {
+    -webkit-animation-delay: 0.4s;
+    animation-delay: 0.4s;
+}
+
+.sk-cube-grid .sk-cube4 {
+    -webkit-animation-delay: 0.1s;
+    animation-delay: 0.1s;
+}
+
+.sk-cube-grid .sk-cube5 {
+    -webkit-animation-delay: 0.2s;
+    animation-delay: 0.2s;
+}
+
+.sk-cube-grid .sk-cube6 {
+    -webkit-animation-delay: 0.3s;
+    animation-delay: 0.3s;
+}
+
+.sk-cube-grid .sk-cube7 {
+    -webkit-animation-delay: 0s;
+    animation-delay: 0s;
+}
+
+.sk-cube-grid .sk-cube8 {
+    -webkit-animation-delay: 0.1s;
+    animation-delay: 0.1s;
+}
+
+.sk-cube-grid .sk-cube9 {
+    -webkit-animation-delay: 0.2s;
+    animation-delay: 0.2s;
+}
+
+@-webkit-keyframes sk-cubeGridScaleDelay {
+
+    0%,
+    70%,
+    100% {
+        -webkit-transform: scale3D(1, 1, 1);
+        transform: scale3D(1, 1, 1);
+    }
+
+    35% {
+        -webkit-transform: scale3D(0, 0, 1);
+        transform: scale3D(0, 0, 1);
+    }
+}
+
+@keyframes sk-cubeGridScaleDelay {
+
+    0%,
+    70%,
+    100% {
+        -webkit-transform: scale3D(1, 1, 1);
+        transform: scale3D(1, 1, 1);
+    }
+
+    35% {
+        -webkit-transform: scale3D(0, 0, 1);
+        transform: scale3D(0, 0, 1);
+    }
+}
+
+.progress-block {
+    position: fixed;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    font-size: smaller;
+}
+
+#progress-text {
+  position: absolute;
+}
+
+.fullscreen-overlay {
+    position: fixed;
+    display: none;
+    width: 100%;
+    height: 100%;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background-color: rgba(0, 0, 0, 0.9);
+    z-index: 2;
+    cursor: pointer;
+    color: lightgray;
+    overflow: auto;
+}
+
+.fullscreen-overlay-transparent {
+    position: fixed;
+    display: none;
+    width: 100%;
+    height: 100%;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    z-index: 2;
+    cursor: pointer;
+    color: lightgray;
+    overflow: auto;
+}