Add duplicate check functionality to upload form with user alerts for detected duplicates
This commit is contained in:
Ross
@@ -25,6 +25,7 @@
|
||||
<input type="checkbox" id="check-annonymisation-ocr"><label for="check-annonymisation-ocr">Check for annonymisation (OCR)</label>
|
||||
<input type="checkbox" id="show-thumbnails"><label for="Show thumbnails">Show thumbnails</label>
|
||||
<input type="checkbox" id="show-viewer"><label for="Show Viewer">Show viewer</label>
|
||||
<input type="checkbox" id="check-duplicates" checked><label for="check-duplicates">Check for duplicates before upload</label>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
@@ -61,6 +62,7 @@
|
||||
</div>
|
||||
|
||||
<h4>Selected files:</h4>
|
||||
<div id="duplicate-progress"></div>
|
||||
<div id="single-dicom-viewer" class="dicom-viewer" data-images="" data-annotations=''></div>
|
||||
<ol id="listing"></ol>
|
||||
|
||||
@@ -87,12 +89,145 @@
|
||||
|
||||
|
||||
{% block js %}
|
||||
<script src="https://cdn.jsdelivr.net/npm/hash-wasm@4"></script>
|
||||
|
||||
<script src="{% static 'js/upload_form_helpers.js' %}"></script>
|
||||
<script>
|
||||
window.to_upload = [];
|
||||
window.upload_count = 1;
|
||||
window.duplicate_series = new Set()
|
||||
|
||||
async function calculateBlake3Hash(file) {
|
||||
// Read file as ArrayBuffer
|
||||
const buffer = await file.arrayBuffer();
|
||||
// Parse as DICOM
|
||||
let pixelDataElement;
|
||||
try {
|
||||
const byteArray = new Uint8Array(buffer);
|
||||
const dataSet = dicomParser.parseDicom(byteArray);
|
||||
pixelDataElement = dataSet.elements.x7fe00010; // Pixel Data tag
|
||||
if (!pixelDataElement) {
|
||||
throw new Error("No Pixel Data found in DICOM file.");
|
||||
}
|
||||
// Extract pixel data bytes
|
||||
const pixelData = byteArray.subarray(pixelDataElement.dataOffset, pixelDataElement.dataOffset + pixelDataElement.length);
|
||||
// Hash just the pixel data
|
||||
return await hashwasm.blake3(pixelData);
|
||||
} catch (err) {
|
||||
console.error("DICOM parse/hash error:", err, file.name);
|
||||
// Fallback: hash the whole file if pixel data is missing or parsing fails
|
||||
return await hashwasm.blake3(new Uint8Array(buffer));
|
||||
}
|
||||
}
|
||||
async function checkForDuplicates(files) {
|
||||
const hashes = [];
|
||||
for (const file of files) {
|
||||
const hash = await calculateBlake3Hash(file);
|
||||
hashes.push(hash);
|
||||
file._blake3_hash = hash; // Store for later use if needed
|
||||
}
|
||||
// Call the API
|
||||
return fetch("{% url 'api-1:check_images_hashes' %}", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-CSRFToken": "{{ csrf_token }}"
|
||||
},
|
||||
body: JSON.stringify(hashes)
|
||||
}).then(res => res.json());
|
||||
}
|
||||
|
||||
|
||||
async function processFilesWithDuplicateCheck() {
|
||||
const file_set = document.getElementById("filepicker").files;
|
||||
let hasDuplicates = false;
|
||||
const $dupProgress = $("#duplicate-progress");
|
||||
$dupProgress.empty();
|
||||
|
||||
if ($("#check-duplicates").is(":checked")) {
|
||||
$dupProgress.html('<span style="color:#0d6efd;">Calculating hashes...</span>');
|
||||
$("#progress-text").text("Checking for duplicates...");
|
||||
|
||||
// Calculate hashes and update progress
|
||||
const hashes = [];
|
||||
let n = 0;
|
||||
for (const file of file_set) {
|
||||
n += 1;
|
||||
$dupProgress.html(`<span style="color:#0d6efd;">Hashing: ${file.webkitRelativePath} (${n} of ${file_set.length})</span>`);
|
||||
const hash = await calculateBlake3Hash(file);
|
||||
hashes.push(hash);
|
||||
file._blake3_hash = hash;
|
||||
}
|
||||
|
||||
$dupProgress.html('<span style="color:#0d6efd;">Checking hashes with server...</span>');
|
||||
const result = await fetch("{% url 'api-1:check_images_hashes' %}", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-CSRFToken": "{{ csrf_token }}"
|
||||
},
|
||||
body: JSON.stringify(hashes)
|
||||
}).then(res => res.json());
|
||||
|
||||
// Build a map from hash to file for quick lookup
|
||||
const hashToFile = {};
|
||||
for (const file of file_set) {
|
||||
if (file._blake3_hash) {
|
||||
hashToFile[file._blake3_hash] = file;
|
||||
}
|
||||
}
|
||||
|
||||
let duplicateCount = 0;
|
||||
// Map of unique series url to {url, id}
|
||||
const duplicateSeriesMap = {};
|
||||
for (const hash in result) {
|
||||
if (result[hash].id) {
|
||||
hasDuplicates = true;
|
||||
duplicateCount += 1;
|
||||
// Highlight the file in the listing
|
||||
const file = hashToFile[hash];
|
||||
if (file) {
|
||||
const selector = `.dicom-file[data-path='${file.webkitRelativePath}']`;
|
||||
const li = document.querySelector(selector);
|
||||
if (li) {
|
||||
li.classList.add("duplicate-file-highlight");
|
||||
}
|
||||
}
|
||||
// Collect unique series by url
|
||||
if (result[hash].url) {
|
||||
duplicateSeriesMap[result[hash].url] = {
|
||||
url: result[hash].url,
|
||||
id: result[hash].id
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hasDuplicates) {
|
||||
$("#uploadButton").prop("disabled", true);
|
||||
// Show summary in red and list duplicate series as links if any
|
||||
let html = `<div style="color:#dc3545;font-weight:bold;">Found ${duplicateCount} duplicate file(s). Please review before uploading.</div>`;
|
||||
const seriesUrls = Object.values(duplicateSeriesMap);
|
||||
if (seriesUrls.length > 0) {
|
||||
html += `<div style="margin-top:8px;">Duplicate series:</div><ul>`;
|
||||
seriesUrls.forEach(series => {
|
||||
html += `<li><a href="${series.url}" target="_blank" style="color:#dc3545;text-decoration:underline;">Series ${series.id}</a></li>`;
|
||||
});
|
||||
html += `</ul>`;
|
||||
}
|
||||
$dupProgress.html(html);
|
||||
toastr.error("Duplicates detected. Please review before uploading.");
|
||||
} else {
|
||||
$("#uploadButton").prop("disabled", false);
|
||||
$dupProgress.html('<span style="color:#198754;">No duplicates detected.</span>');
|
||||
}
|
||||
} else {
|
||||
$("#uploadButton").prop("disabled", false);
|
||||
$dupProgress.empty();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function* chunks(arr, n) {
|
||||
for (let i = 0; i < arr.length; i += n) {
|
||||
yield arr.slice(i, i + n);
|
||||
@@ -243,7 +378,6 @@
|
||||
errors.push(`Patient ID starts with ${site_codes[i]} (${patient_id})`);
|
||||
}
|
||||
}
|
||||
console.log(errors)
|
||||
|
||||
if (not_anon) {
|
||||
toastr.warning(`File does not appear to be annonymised<br/>Accession: ${accession_number} <br/>Patient ID: ${patient_id}<br/>`, 'Anonymisation warning', {
|
||||
@@ -476,20 +610,10 @@
|
||||
}
|
||||
|
||||
$(document).ready(function () {
|
||||
document.getElementById("filepicker").addEventListener(
|
||||
"change",
|
||||
(event) => {
|
||||
//let output = document.getElementById("listing");
|
||||
//for (const file of event.target.files) {
|
||||
// let item = document.createElement("li");
|
||||
// item.textContent = file.webkitRelativePath;
|
||||
// output.appendChild(item);
|
||||
//}
|
||||
|
||||
loadDicomViewerUploads();
|
||||
},
|
||||
false,
|
||||
);
|
||||
document.getElementById("filepicker").addEventListener("change", async () => {
|
||||
await loadDicomViewerUploads();
|
||||
await processFilesWithDuplicateCheck();
|
||||
});
|
||||
|
||||
|
||||
// Add event listener to the button element
|
||||
@@ -505,3 +629,13 @@
|
||||
end
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
{% block css %}
|
||||
<style>
|
||||
.duplicate-file-highlight {
|
||||
background: #fff3cd !important;
|
||||
border-left: 5px solid #dc3545 !important;
|
||||
color: #856404 !important;
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
Reference in New Issue
Block a user