From 63c342ff21a9c70ce2692d3c91db828ac18549a5 Mon Sep 17 00:00:00 2001
From: Ross <xkjq@ymail.com>
Date: Mon, 11 Aug 2025 10:42:08 +0100
Subject: [PATCH] Add duplicate check functionality to upload form with user
 alerts for detected duplicates

---
 atlas/templates/atlas/new_uploads.html | 240 +++++++++++++++++++------
 1 file changed, 187 insertions(+), 53 deletions(-)

diff --git a/atlas/templates/atlas/new_uploads.html b/atlas/templates/atlas/new_uploads.html
index da663660..a31d6009 100644
--- a/atlas/templates/atlas/new_uploads.html
+++ b/atlas/templates/atlas/new_uploads.html
@@ -25,6 +25,7 @@
             <input type="checkbox" id="check-annonymisation-ocr"><label for="check-annonymisation-ocr">Check for annonymisation (OCR)</label>
             <input type="checkbox" id="show-thumbnails"><label for="Show thumbnails">Show thumbnails</label>
             <input type="checkbox" id="show-viewer"><label for="Show Viewer">Show viewer</label>
+            <input type="checkbox" id="check-duplicates" checked><label for="check-duplicates">Check for duplicates before upload</label>
         </details>
     </div>
 
@@ -58,41 +59,175 @@
         <details><summary>Override annonymisation check</summary>
             <p>By clicking the button below you can override the annonymisation check and upload the files anyway. Only do this if you are sure this is a false positive.</p>
             <button id="override-anon" _="on click remove @disabled from #uploadButton then alert('Upload enabled')">Override annonymisation check</button>
-    </div>
-
-    <h4>Selected files:</h4>
-    <div id="single-dicom-viewer" class="dicom-viewer" data-images="" data-annotations=''></div>
-    <ol id="listing"></ol>
-
-
-    <div id="loading" class="fullscreen-overlay">
-        <div class="progress-block">
-            <div id="progress"></div>
-            <div class="sk-cube-grid full">
-                <div class="sk-cube sk-cube1"></div>
-                <div class="sk-cube sk-cube2"></div>
-                <div class="sk-cube sk-cube3"></div>
-                <div class="sk-cube sk-cube4"></div>
-                <div class="sk-cube sk-cube5"></div>
-                <div class="sk-cube sk-cube6"></div>
-                <div class="sk-cube sk-cube7"></div>
-                <div class="sk-cube sk-cube8"></div>
-                <div class="sk-cube sk-cube9"></div>
-            </div>
-            <div id="progress-text">Uploading...</div>
         </div>
-    </div>
+
+        <h4>Selected files:</h4>
+        <div id="duplicate-progress"></div>
+        <div id="single-dicom-viewer" class="dicom-viewer" data-images="" data-annotations=''></div>
+        <ol id="listing"></ol>
+
+
+        <div id="loading" class="fullscreen-overlay">
+            <div class="progress-block">
+                <div id="progress"></div>
+                <div class="sk-cube-grid full">
+                    <div class="sk-cube sk-cube1"></div>
+                    <div class="sk-cube sk-cube2"></div>
+                    <div class="sk-cube sk-cube3"></div>
+                    <div class="sk-cube sk-cube4"></div>
+                    <div class="sk-cube sk-cube5"></div>
+                    <div class="sk-cube sk-cube6"></div>
+                    <div class="sk-cube sk-cube7"></div>
+                    <div class="sk-cube sk-cube8"></div>
+                    <div class="sk-cube sk-cube9"></div>
+                </div>
+                <div id="progress-text">Uploading...</div>
+            </div>
+        </div>
 
 {% endblock %}
 
 
 {% block js %}
+    <script src="https://cdn.jsdelivr.net/npm/hash-wasm@4"></script>
+
     <script src="{% static 'js/upload_form_helpers.js' %}"></script>
     <script>
         window.to_upload = [];
         window.upload_count = 1;
         window.duplicate_series = new Set()
 
+async function calculateBlake3Hash(file) {
+    // Read file as ArrayBuffer
+    const buffer = await file.arrayBuffer();
+    // Parse as DICOM
+    let pixelDataElement;
+    try {
+        const byteArray = new Uint8Array(buffer);
+        const dataSet = dicomParser.parseDicom(byteArray);
+        pixelDataElement = dataSet.elements.x7fe00010; // Pixel Data tag
+        if (!pixelDataElement) {
+            throw new Error("No Pixel Data found in DICOM file.");
+        }
+        // Extract pixel data bytes
+        const pixelData = byteArray.subarray(pixelDataElement.dataOffset, pixelDataElement.dataOffset + pixelDataElement.length);
+        // Hash just the pixel data
+        return await hashwasm.blake3(pixelData);
+    } catch (err) {
+        console.error("DICOM parse/hash error:", err, file.name);
+        // Fallback: hash the whole file if pixel data is missing or parsing fails
+        return await hashwasm.blake3(new Uint8Array(buffer));
+    }
+}
+        async function checkForDuplicates(files) {
+            const hashes = [];
+            for (const file of files) {
+                const hash = await calculateBlake3Hash(file);
+                hashes.push(hash);
+                file._blake3_hash = hash; // Store for later use if needed
+            }
+    // Call the API
+            return fetch("{% url 'api-1:check_images_hashes' %}", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "X-CSRFToken": "{{ csrf_token }}"
+                },
+                body: JSON.stringify(hashes)
+            }).then(res => res.json());
+        }
+
+
+async function processFilesWithDuplicateCheck() {
+    const file_set = document.getElementById("filepicker").files;
+    let hasDuplicates = false;
+    const $dupProgress = $("#duplicate-progress");
+    $dupProgress.empty();
+
+    if ($("#check-duplicates").is(":checked")) {
+        $dupProgress.html('<span style="color:#0d6efd;">Calculating hashes...</span>');
+        $("#progress-text").text("Checking for duplicates...");
+
+        // Calculate hashes and update progress
+        const hashes = [];
+        let n = 0;
+        for (const file of file_set) {
+            n += 1;
+            $dupProgress.html(`<span style="color:#0d6efd;">Hashing: ${file.webkitRelativePath} (${n} of ${file_set.length})</span>`);
+            const hash = await calculateBlake3Hash(file);
+            hashes.push(hash);
+            file._blake3_hash = hash;
+        }
+
+        $dupProgress.html('<span style="color:#0d6efd;">Checking hashes with server...</span>');
+        const result = await fetch("{% url 'api-1:check_images_hashes' %}", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "X-CSRFToken": "{{ csrf_token }}"
+            },
+            body: JSON.stringify(hashes)
+        }).then(res => res.json());
+
+        // Build a map from hash to file for quick lookup
+        const hashToFile = {};
+        for (const file of file_set) {
+            if (file._blake3_hash) {
+                hashToFile[file._blake3_hash] = file;
+            }
+        }
+
+        let duplicateCount = 0;
+        // Map of unique series url to {url, id}
+        const duplicateSeriesMap = {};
+        for (const hash in result) {
+            if (result[hash].id) {
+                hasDuplicates = true;
+                duplicateCount += 1;
+                // Highlight the file in the listing
+                const file = hashToFile[hash];
+                if (file) {
+                    const selector = `.dicom-file[data-path='${file.webkitRelativePath}']`;
+                    const li = document.querySelector(selector);
+                    if (li) {
+                        li.classList.add("duplicate-file-highlight");
+                    }
+                }
+                // Collect unique series by url
+                if (result[hash].url) {
+                    duplicateSeriesMap[result[hash].url] = {
+                        url: result[hash].url,
+                        id: result[hash].id
+                    };
+                }
+            }
+        }
+
+        if (hasDuplicates) {
+            $("#uploadButton").prop("disabled", true);
+            // Show summary in red and list duplicate series as links if any
+            let html = `<div style="color:#dc3545;font-weight:bold;">Found ${duplicateCount} duplicate file(s). Please review before uploading.</div>`;
+            const seriesUrls = Object.values(duplicateSeriesMap);
+            if (seriesUrls.length > 0) {
+                html += `<div style="margin-top:8px;">Duplicate series:</div><ul>`;
+                seriesUrls.forEach(series => {
+                    html += `<li><a href="${series.url}" target="_blank" style="color:#dc3545;text-decoration:underline;">Series ${series.id}</a></li>`;
+                });
+                html += `</ul>`;
+            }
+            $dupProgress.html(html);
+            toastr.error("Duplicates detected. Please review before uploading.");
+        } else {
+            $("#uploadButton").prop("disabled", false);
+            $dupProgress.html('<span style="color:#198754;">No duplicates detected.</span>');
+        }
+    } else {
+        $("#uploadButton").prop("disabled", false);
+        $dupProgress.empty();
+    }
+}
+
+
         function* chunks(arr, n) {
             for (let i = 0; i < arr.length; i += n) {
                 yield arr.slice(i, i + n);
@@ -188,16 +323,16 @@
                                     "timeOut": 7000
                                 }
                             );
-                        window.duplicate_series.forEach((element) => {
-                            const wrapper = document.createElement("div");
-                            wrapper.className = "alert alert-warning duplicate-series-highlight";
-                            const link = document.createElement("a");
-                            link.href = element;
-                            link.textContent = `Series exists: ${element}`;
-                            link.target = "_blank";
-                            wrapper.appendChild(link);
-                            $("#duplicate-series").append(wrapper);
-                        });
+                            window.duplicate_series.forEach((element) => {
+                                const wrapper = document.createElement("div");
+                                wrapper.className = "alert alert-warning duplicate-series-highlight";
+                                const link = document.createElement("a");
+                                link.href = element;
+                                link.textContent = `Series exists: ${element}`;
+                                link.target = "_blank";
+                                wrapper.appendChild(link);
+                                $("#duplicate-series").append(wrapper);
+                            });
                         };
                         //$("#duplicate-series").append([...window.duplicate_series].join("<br/>"));  
                         alert("Uploading complete")
@@ -243,7 +378,6 @@
                         errors.push(`Patient ID starts with ${site_codes[i]} (${patient_id})`);
                     }
                 }
-                console.log(errors)
 
                 if (not_anon) {
                     toastr.warning(`File does not appear to be annonymised<br/>Accession: ${accession_number} <br/>Patient ID: ${patient_id}<br/>`, 'Anonymisation warning', {
@@ -267,11 +401,11 @@
                     $(".anon-alert").show();
 
                     setTimeout(
-                    function(errors) {
-                        console.log(errors)
+                        function(errors) {
+                            console.log(errors)
                         // We delay this a few seconds otherwise it does not work
-                    $(`.dicom-file[data-path='${file.webkitRelativePath}']`).addClass("image-ident-warning").append($(`<div>${errors.join("<br/>")}</div>`));
-                    }, 2000, errors
+                            $(`.dicom-file[data-path='${file.webkitRelativePath}']`).addClass("image-ident-warning").append($(`<div>${errors.join("<br/>")}</div>`));
+                        }, 2000, errors
                     )
                 }
 
@@ -476,20 +610,10 @@
         }
 
         $(document).ready(function () {
-            document.getElementById("filepicker").addEventListener(
-                "change",
-                (event) => {
-                    //let output = document.getElementById("listing");
-                    //for (const file of event.target.files) {
-                    //    let item = document.createElement("li");
-                    //    item.textContent = file.webkitRelativePath;
-                    //    output.appendChild(item);
-                    //}
-
-                    loadDicomViewerUploads();
-                },
-                false,
-            );
+            document.getElementById("filepicker").addEventListener("change", async () => {
+                await loadDicomViewerUploads();
+                await processFilesWithDuplicateCheck();
+            });
 
 
         // Add event listener to the button element
@@ -501,7 +625,17 @@
     <script type="text/hyperscript">
 
         init
-         add @disabled to #uploadButton
+        add @disabled to #uploadButton
         end
     </script>
+{% endblock %}
+
+{% block css %}
+<style>
+.duplicate-file-highlight {
+    background: #fff3cd !important;
+    border-left: 5px solid #dc3545 !important;
+    color: #856404 !important;
+}
+</style>
 {% endblock %}
\ No newline at end of file