improve uploading

anatomy/static/js/upload_form_helpers.js

@@ -108,7 +108,7 @@
             uploading_el = $(`img[data-input-id='${input.id}'], div[data-input-id='${input.id}']`)
             uploading_el.removeClass("image-ident-loading");
             console.log("found text", l);
-            if (l.includes("accesion") || l.includes("number") || l.search(
+            if (l.includes("accession") || l.includes("number") || l.search(
                     /(ref|rk9|rh8|ra9|rbz)\d+/g) > -1) {
                 console.log("Match found ", input);
                 $(input).addClass("image-ident-warning");

atlas/api.py

@@ -54,7 +54,6 @@ class CaseSchema(ModelSchema):
         model_fields = ["id", "title"]
 
 
-@logger.catch()
 @router.post("/upload_dicom", auth=django_auth)
 def upload_dicom(request, files: List[UploadedFile] = File(...)):
     uploaded = []
@@ -69,7 +68,6 @@ def upload_dicom(request, files: List[UploadedFile] = File(...)):
 
             uploaded.append((file.name, ud.image_blake3_hash))
         except DuplicateDicom:
-            print(ud.check_for_duplicates(ud.image_blake3_hash))
             duplicate.append((file.name, ud.image_blake3_hash))
             pass
         except InvalidDicomError:

atlas/models.py

@@ -1110,7 +1110,6 @@ class UncategorisedDicom(models.Model):
 
         return duplicate
 
-    @logger.catch()
     def save(self, *args, **kwargs):
         """Override save method to add image hash"""
         if self.image:

atlas/templates/atlas/help.html

@@ -30,5 +30,7 @@
     Available <a href="https://www.penracourses.org.uk/uploader/insight_rad_up.xml">here</a>
     <h4>Step 3: Integrate the uploader with PACs</h4>
     Whilst in PACs open the settings menu with the keyboard shortcut 'Ctrl+Shift+C'. In the 'General' tab, select 'External Tools' and click 'Add'. Press the Import button and select the xml file you downloaded in Step 2. Modify the file to run path to point the uploader exe downloaded in step 1 (clicking on the 3 dots will allow you to browse and select this file). The remaining fields should have been set via the xml file and should not require modification.
+    <h3>Uploading without the uploader tool</h3>
+    If you have <b>annonymised</b> dicom files already exported you can uploading them directly using this <a href='{% url "atlas:new_uploads" %}'>form</a>.
 
 {% endblock %}

atlas/templates/atlas/new_uploads.html

@@ -10,26 +10,156 @@
         Please note this form will not annonymise files. Please ensure that you have done so prior to using it.
     </div>
 
-    <div>This form allows you to quickly upload dicom files to the server. Select the folder containing the dicom files below.</div>
+    <div>
+        <p>This form allows you to quickly upload dicom files to the server. Select the folder containing the dicom files below. If you wish to upload files directly from Insight you may rather use the <a href='{% url "atlas:help" %}'>Uploader tool.</a></p>
+        <details>
+            <summary>Help</summary>
+            <p>Only valid dicom files will be uploaded. </p>
+            <p>It is possible to upload multiple files by selecting the parent (containing) folder. Folders within will then be searched recursively for any valid dicom files.</p>
+            <p>Files to be uploaded will be displayed below. </p>
+        </details>
+    </div>
 
     <form method="post" enctype="multipart/form-data">
         {% csrf_token %}
         <input type="file" name="dicom_files" id="filepicker" webkitdirectory multiple>
-        <input type="submit" value="Upload">
+        <button type="button" id="uploadButton">Upload</button>
 
     </form>
 
-    <div>Selected files:</div>
+    <details id="upload-results"><summary>Upload results</summary>
-    <div id="single-dicom-viewer" class="dicom-viewer" data-images="" data-annotations=''></div>
+        <div id="upload-results-list">
-    <ul id="listing"></ul>
+            <h5>Uploaded</h5>
+            <ul id="uploaded-files"></ul>
+            <h5>Duplicates</h5>
+            <ul id="duplicate-files"></ul>
+            <h5>Failed</h5>
+            <ul id="failed-files"></ul>
+        </div>
+    </details>
 
 
+    <h4>Selected files:</h4>
+    <div id="single-dicom-viewer" class="dicom-viewer" data-images="" data-annotations=''></div>
+    <ol id="listing"></ol>
+
+
+    <div id="loading" class="fullscreen-overlay">
+        <div class="progress-block">
+            <div id="progress"></div>
+            <div class="sk-cube-grid full">
+                <div class="sk-cube sk-cube1"></div>
+                <div class="sk-cube sk-cube2"></div>
+                <div class="sk-cube sk-cube3"></div>
+                <div class="sk-cube sk-cube4"></div>
+                <div class="sk-cube sk-cube5"></div>
+                <div class="sk-cube sk-cube6"></div>
+                <div class="sk-cube sk-cube7"></div>
+                <div class="sk-cube sk-cube8"></div>
+                <div class="sk-cube sk-cube9"></div>
+            </div>
+            <div id="progress-text">Uploading...</div>
+        </div>
+    </div>
+
 {% endblock %}
 
 
 {% block js %}
     <script src="{% static 'js/upload_form_helpers.js' %}"></script>
     <script>
+        window.to_upload = [];
+        window.upload_count = 1;
+
+        function* chunks(arr, n) {
+            for (let i = 0; i < arr.length; i += n) {
+                yield arr.slice(i, i + n);
+            }
+        }
+
+        function onUploadButtonClicked(event) {
+            event.preventDefault();
+            const fileInput = document.getElementById("filepicker");
+            const selectedFiles = fileInput.files;
+    // Check if any files are selected
+            if (selectedFiles.length === 0) {
+                alert("Please select at least one file to upload.");
+                return;
+            }
+            //alert(`${window.to_upload.length} files selected. Uploading...`);
+
+            window.upload_results = {
+                "uploaded": [],
+                "duplicates": [],
+                "failed": []
+            };
+
+            chunked_files = [...chunks(window.to_upload, 10)];
+
+
+            $("#loading").show();
+            $("#progress-text").text(`Uploading..`);
+            for (let i = 0; i < chunked_files.length; i++) {
+                uploadFiles(chunked_files[i], i, chunked_files.length);
+
+            }
+        }
+
+        function uploadFiles(files_to_upload, current_chunk, total_chunks){
+            console.log("uploading", files_to_upload)
+
+  // Create a FormData object to store the form data
+            const formData = new FormData();
+  // Append each selected file to the FormData object
+            for (let i = 0; i < files_to_upload.length; i++) {
+                formData.append("files", files_to_upload[i]);
+            }
+            console.log("formdata", formData)
+            const xhr = new XMLHttpRequest();
+            xhr.open("POST", "{% url 'api-1:upload_dicom' %}", true);
+            xhr.setRequestHeader("X-CSRFToken", "{{ csrf_token }}");
+            xhr.onreadystatechange = function () {
+                $("#progress-text").text(`Uploading ${window.upload_count} of ${total_chunks} chunks`);
+                if (xhr.readyState === XMLHttpRequest.DONE) {
+                    window.upload_count += 1;
+                    if (xhr.status === 200) {
+                    // Handle successful response from the server
+                        console.log('Files uploaded successfully!');
+                        console.log(xhr.response);
+                        window.upload_results["uploaded"].push(...JSON.parse(xhr.response)["uploaded"]);
+                        window.upload_results["duplicates"].push(...JSON.parse(xhr.response)["duplicates"]);
+                        window.upload_results["failed"].push(...JSON.parse(xhr.response)["failed"]);
+
+                        for (let i = 0; i < JSON.parse(xhr.response)["uploaded"].length; i++) {
+                            let item = document.createElement("li");
+                            item.textContent = JSON.parse(xhr.response)["uploaded"][i];
+                            $("#uploaded-files").append(item);
+                        }
+
+                        for (let i = 0; i < JSON.parse(xhr.response)["duplicates"].length; i++) {
+                            let item = document.createElement("li");
+                            item.textContent = JSON.parse(xhr.response)["duplicates"][i];
+                            $("#duplicate-files").append(item);
+                        }
+
+                        for (let i = 0; i < JSON.parse(xhr.response)["failed"].length; i++) {
+                            let item = document.createElement("li");
+                            item.textContent = JSON.parse(xhr.response)["failed"][i];
+                            $("#failed-files").append(item);
+                        }
+                    } else {
+                   // Handle error response from the server
+                        console.error('Failed to upload files.');
+                        alert("Error occurred during file upload. Please try again.");
+                    }
+                    if (current_chunk + 1 == total_chunks) {
+                        $("#loading").hide();
+                        alert("Uploading complete")
+                    }
+                }
+            };
+            xhr.send(formData);
+        }
 
         function checkAnnonymisation(byteArray) {
         // We need to setup a try/catch block because parseDicom will throw an exception
@@ -63,11 +193,11 @@
                 for (let i = 0; i < site_codes.length; i++) {
                     if (
                         accession_number.startsWith(site_codes[i]) ||
-                        patient_id.startsWith(site_codes[i]) ||
+                        patient_id.startsWith(site_codes[i])
-                        patient_id.startsWith("pat"))
+                    )
-                         {
+                    {
-                            not_anon = true;
+                        not_anon = true;
-                        }
+                    }
                 }
 
                 if (not_anon) {
@@ -88,6 +218,7 @@
                         "showMethod": "fadeIn",
                         "hideMethod": "fadeOut",
                     });
+                    $("uploadButton").prop("disabled", true);
                 }
 
             // TODO: fix
@@ -176,12 +307,19 @@
 
             let output = document.getElementById("listing");
             images = [];
+            window.to_upload = [];
+            $("#loading").show();
+            $("#progress-text").text(`Processing..`);
+            n = 0;
             for (const file of file_set) {
+                n += 1;
+                $("#progress-text").text(`Processing ${file.name} (${n} of ${file_set.length})`);
                     //await readFile(file);
                 dicom = await readDicomFile(file);
 
                 if (dicom) {
                     let item = document.createElement("li");
+                    item.className = "dicom-file";
                     item.textContent = file.webkitRelativePath;
                     output.appendChild(item);
                         //await readFile(file, item);
@@ -191,21 +329,23 @@
                     console.log(url)
                     imageId = `wadouri:${url}`;
                     images.push(imageId);
+                    window.to_upload.push(file);
 
                     const element = $(`<div class='temp-thumb' src=${imageId}></div>`).get(0)
                     item.appendChild(element);
                     cornerstone.enable(element);
                     cornerstone.loadAndCacheImage(imageId).then(function (image) {
                         cornerstone.displayImage(element, image);
+
+                        $(element).addClass("temp-thumb-large");
                         cornerstone.resize(element)
+                        setTimeout(function () {
+                            ocr($(element).find("canvas").get(0).toDataURL(), element);
+                            $(element).removeClass("temp-thumb-large");
+                            cornerstone.resize(element)
+                        }, 500);
 
 
-
-                        let bytesView = image.getPixelData()
-                        let str = new TextDecoder("utf-8").decode(bytesView, );
-
-                    //extractDicomDetails(image.data.byteArray)
-
                     });
                 }
             }
@@ -217,11 +357,58 @@
                 "detail": {"images": images, "annontations": undefined}
             });
 
+            $("#loading").hide()
             window.dispatchEvent(event);
 
 
 
         }
+        function ocr(url, input) {
+            console.log("OCR", input)
+            Tesseract.recognize(
+                url,
+                'eng',
+                //{ logger: m => console.log(m) }
+            ).then(({
+                data: {
+                    text
+                }
+            }) => {
+                l = text.toLowerCase();
+                uploading_el = $(`img[data-input-id='${input.id}'], div[data-input-id='${input.id}']`)
+                uploading_el.removeClass("image-ident-loading");
+                console.log("found text", l);
+                if (l.includes("accession") || l.includes("number") || l.search(
+                        /(ref|rk9|rh8|ra9|rbz)\d+/g) > -1) {
+                    console.log("Match found ", input);
+                    $(input).addClass("image-ident-warning");
+                    uploading_el.addClass("image-ident-warning");
+                    $("uploadButton").prop("disabled", true);
+                    toastr.warning(`File does not appear to be annonymised<br/>Image data burned in<br/>${element}`, 'Anonymisation warning', {
+                        "closeButton": false,
+                        "debug": false,
+                        "newestOnTop": false,
+                        "progressBar": false,
+                        "positionClass": "toast-top-full-width",
+                        "preventDuplicates": false,
+                        "onclick": null,
+                        "showDuration": "0",
+                        "hideDuration": "0",
+                        "timeOut": 0,
+                        "extendedTimeOut": 0,
+                        "showEasing": "swing",
+                        "hideEasing": "linear",
+                        "showMethod": "fadeIn",
+                        "hideMethod": "fadeOut",
+                    });
+                } else {
+                    $(input).addClass("image-ident-ok");
+
+                }
+            })
+
+        }
+
         $(document).ready(function () {
             document.getElementById("filepicker").addEventListener(
                 "change",
@@ -237,6 +424,11 @@
                 },
                 false,
             );
+
+
+        // Add event listener to the button element
+            const uploadButton = document.getElementById("uploadButton");
+            uploadButton.addEventListener("click", onUploadButtonClicked);
         });
 
     </script>

atlas/templates/atlas/user_uploads.html

@@ -3,6 +3,7 @@
 {% block content %}
     <h2>Uploaded dicoms</h2>
     User: {{user}}<br/>
+    <a href="{% url 'atlas:new_uploads' %}">Upload more dicoms</a><br/>
     {% if not case %}
         <div class="alert alert-info" role="alert">
             Series imported from here will be orphaned (not associated with a case). It is usually better to create a case first and then import directly into the case.

rad/static/css/anatomy.css

@@ -130,6 +130,12 @@ button:hover {
   background-color: #52057b;
 }
 
+button:disabled {
+  color: gray;
+  background-color: #110119;
+  border-color: #110119;
+}
+
 
 button a {
   text-decoration: none;
@@ -1122,4 +1128,141 @@ tr:has(> td > a) {
 .no-list-style {
   list-style-type: none;
   padding-left: 1rem;
-}
+}
+
+#listing .dicom-file {
+  clear: both;
+}
+
+/* Some fancy loading CSS */
+.sk-cube-grid {
+    width: 60px;
+    height: 60px;
+}
+
+.sk-cube-grid .sk-cube {
+    width: 33%;
+    height: 33%;
+    background-color: #4527A0;
+    float: left;
+    -webkit-animation: sk-cubeGridScaleDelay 1.3s infinite ease-in-out;
+    animation: sk-cubeGridScaleDelay 1.3s infinite ease-in-out;
+}
+
+.sk-cube-grid .sk-cube1 {
+    -webkit-animation-delay: 0.2s;
+    animation-delay: 0.2s;
+}
+
+.sk-cube-grid .sk-cube2 {
+    -webkit-animation-delay: 0.3s;
+    animation-delay: 0.3s;
+}
+
+.sk-cube-grid .sk-cube3 {
+    -webkit-animation-delay: 0.4s;
+    animation-delay: 0.4s;
+}
+
+.sk-cube-grid .sk-cube4 {
+    -webkit-animation-delay: 0.1s;
+    animation-delay: 0.1s;
+}
+
+.sk-cube-grid .sk-cube5 {
+    -webkit-animation-delay: 0.2s;
+    animation-delay: 0.2s;
+}
+
+.sk-cube-grid .sk-cube6 {
+    -webkit-animation-delay: 0.3s;
+    animation-delay: 0.3s;
+}
+
+.sk-cube-grid .sk-cube7 {
+    -webkit-animation-delay: 0s;
+    animation-delay: 0s;
+}
+
+.sk-cube-grid .sk-cube8 {
+    -webkit-animation-delay: 0.1s;
+    animation-delay: 0.1s;
+}
+
+.sk-cube-grid .sk-cube9 {
+    -webkit-animation-delay: 0.2s;
+    animation-delay: 0.2s;
+}
+
+@-webkit-keyframes sk-cubeGridScaleDelay {
+
+    0%,
+    70%,
+    100% {
+        -webkit-transform: scale3D(1, 1, 1);
+        transform: scale3D(1, 1, 1);
+    }
+
+    35% {
+        -webkit-transform: scale3D(0, 0, 1);
+        transform: scale3D(0, 0, 1);
+    }
+}
+
+@keyframes sk-cubeGridScaleDelay {
+
+    0%,
+    70%,
+    100% {
+        -webkit-transform: scale3D(1, 1, 1);
+        transform: scale3D(1, 1, 1);
+    }
+
+    35% {
+        -webkit-transform: scale3D(0, 0, 1);
+        transform: scale3D(0, 0, 1);
+    }
+}
+
+.progress-block {
+    position: fixed;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    font-size: smaller;
+}
+
+#progress-text {
+  position: absolute;
+}
+
+.fullscreen-overlay {
+    position: fixed;
+    display: none;
+    width: 100%;
+    height: 100%;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background-color: rgba(0, 0, 0, 0.9);
+    z-index: 2;
+    cursor: pointer;
+    color: lightgray;
+    overflow: auto;
+}
+
+.fullscreen-overlay-transparent {
+    position: fixed;
+    display: none;
+    width: 100%;
+    height: 100%;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    z-index: 2;
+    cursor: pointer;
+    color: lightgray;
+    overflow: auto;
+}