From b24eac862b49be51189865ea3def9e41582b32cc Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 11 May 2026 11:53:18 +0100 Subject: [PATCH] feat: Implement duplicate check functionality during DICOM uploads and update viewer element ID --- atlas/templates/atlas/new_uploads.html | 210 ++++++++++++++++++------- 1 file changed, 152 insertions(+), 58 deletions(-) diff --git a/atlas/templates/atlas/new_uploads.html b/atlas/templates/atlas/new_uploads.html index e670b031..72fa1d0b 100644 --- a/atlas/templates/atlas/new_uploads.html +++ b/atlas/templates/atlas/new_uploads.html @@ -93,7 +93,7 @@
-
+
@@ -176,11 +176,15 @@ forceAnonymisationOverride: false, ocrWorker: null, currentTagsRecord: null, + duplicateCheckInProgress: false, + duplicateCheckRunId: 0, + duplicateCheckTimer: null, }; const SITE_PREFIXES = ["ref", "rk9", "ra9", "rh8", "rbz", "rba"]; const OCR_MAX_SERIES_SAMPLES = 40; const OCR_CANVAS_MAX = 768; + const DUPLICATE_BATCH_SIZE = 50; function setProgress(text) { document.getElementById("progress-text").textContent = text; @@ -241,6 +245,7 @@ ocrWarning: false, duplicate: false, duplicateUrl: "", + duplicateChecked: false, blake3Hash: "", objectUrl: URL.createObjectURL(file), }; @@ -320,6 +325,9 @@ const hasSelected = window.uploadPreview.records.some(r => r.include); const hasWarnings = hasAnonymisationWarnings(); const hasDuplicates = window.uploadPreview.records.some(r => r.include && r.duplicate); + const duplicatesEnabled = document.getElementById("check-duplicates")?.checked; + const hasPendingDuplicateChecks = duplicatesEnabled && hasSelected && + window.uploadPreview.records.some((r) => r.include && !r.duplicateChecked); if (!hasSelected) { uploadButton.disabled = true; @@ -333,10 +341,36 @@ return; } + if (hasPendingDuplicateChecks || window.uploadPreview.duplicateCheckInProgress) { + uploadButton.disabled = true; + anonAlert.classList.add("d-none"); + return; + } + anonAlert.classList.add("d-none"); uploadButton.disabled = !!hasDuplicates; } + function getSeriesDuplicateCheckBadge(series) { + const duplicatesEnabled = document.getElementById("check-duplicates")?.checked; + if (!duplicatesEnabled) { + return 'Duplicate check off'; + } + + const selectedFiles = series.files.filter((f) => f.include); + if (!selectedFiles.length) { + return 'Not selected'; + } + + const checkedCount = selectedFiles.filter((f) => f.duplicateChecked).length; + if (checkedCount < selectedFiles.length) { + const badgeClass = window.uploadPreview.duplicateCheckInProgress ? "text-bg-info" : "text-bg-warning"; + return `Duplicate check: ${checkedCount}/${selectedFiles.length}`; + } + + return 'Duplicate check complete'; + } + function getSeriesStatusClass(series) { const hasTagWarnings = series.files.some(f => f.tagWarnings.length > 0); const hasOcrWarnings = series.files.some(f => f.ocrWarning); @@ -392,6 +426,7 @@
${tagWarningCount ? `Tag warnings: ${tagWarningCount}` : ""} ${ocrWarningCount ? `OCR warnings: ${ocrWarningCount}` : ""} + ${getSeriesDuplicateCheckBadge(series)} ${duplicateCount ? `Duplicates: ${duplicateCount}` : ""}
@@ -577,83 +612,133 @@ } } + function scheduleDuplicateCheck() { + if (window.uploadPreview.duplicateCheckTimer) { + clearTimeout(window.uploadPreview.duplicateCheckTimer); + } + window.uploadPreview.duplicateCheckTimer = setTimeout(async () => { + await processDuplicates(); + renderGroupedPreview(); + }, 100); + } + async function processDuplicates() { const dupProgress = document.getElementById("duplicate-progress"); dupProgress.innerHTML = ""; - window.uploadPreview.records.forEach(r => { + window.uploadPreview.records.forEach((r) => { r.duplicate = false; r.duplicateUrl = ""; + if (r.include) r.duplicateChecked = false; }); + document.getElementById("duplicate-series").innerHTML = ""; + window.uploadPreview.duplicateSeriesUrls = new Set(); if (!document.getElementById("check-duplicates").checked) { - return; - } - - const selected = window.uploadPreview.records.filter(r => r.include); - if (!selected.length) return; - - const hashes = []; - let idx = 0; - for (const record of selected) { - idx += 1; - if (!record.blake3Hash) { - setProgress(`Hashing ${idx} of ${selected.length}: ${record.path}`); - record.blake3Hash = await calculateBlake3Hash(record.file); - } - hashes.push(record.blake3Hash); - } - - dupProgress.innerHTML = 'Checking hashes with server...'; - - let result = {}; - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 30000); - const res = await fetch("{% url 'api-1:check_images_hashes' %}", { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-CSRFToken": "{{ csrf_token }}", - }, - body: JSON.stringify(hashes), - signal: controller.signal, + window.uploadPreview.records.forEach((r) => { + r.duplicateChecked = true; }); - clearTimeout(timeoutId); - if (res.ok) result = await res.json(); - else throw new Error(`Server returned ${res.status}`); - } catch (err) { - if (err.name === "AbortError") { - dupProgress.innerHTML = 'Duplicate check timed out. Proceeding without server verification.'; - } else { - console.error(err); - dupProgress.innerHTML = 'Could not verify duplicates. Proceeding.'; - } + window.uploadPreview.duplicateCheckInProgress = false; + updateUploadButtonState(); return; } + const selected = window.uploadPreview.records.filter((r) => r.include); + if (!selected.length) { + window.uploadPreview.duplicateCheckInProgress = false; + updateUploadButtonState(); + return; + } + + const runId = window.uploadPreview.duplicateCheckRunId + 1; + window.uploadPreview.duplicateCheckRunId = runId; + window.uploadPreview.duplicateCheckInProgress = true; + updateUploadButtonState(); + + let processed = 0; const duplicateSeriesMap = {}; - for (const record of selected) { - const info = result[record.blake3Hash]; - if (!info || !info.id) continue; - record.duplicate = true; - if (info.url) { - record.duplicateUrl = info.url; - duplicateSeriesMap[info.url] = info.id; + + try { + for (let offset = 0; offset < selected.length; offset += DUPLICATE_BATCH_SIZE) { + if (runId !== window.uploadPreview.duplicateCheckRunId) return; + + const batch = selected.slice(offset, offset + DUPLICATE_BATCH_SIZE); + const hashes = []; + + for (const record of batch) { + if (runId !== window.uploadPreview.duplicateCheckRunId) return; + if (!record.blake3Hash) { + setProgress(`Hashing ${processed + 1} of ${selected.length}: ${record.path}`); + record.blake3Hash = await calculateBlake3Hash(record.file); + } + hashes.push(record.blake3Hash); + processed += 1; + } + + dupProgress.innerHTML = `Checking duplicate batch ${Math.floor(offset / DUPLICATE_BATCH_SIZE) + 1} of ${Math.ceil(selected.length / DUPLICATE_BATCH_SIZE)} (${processed}/${selected.length})...`; + + let result = {}; + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 30000); + const res = await fetch("{% url 'api-1:check_images_hashes' %}", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-CSRFToken": "{{ csrf_token }}", + }, + body: JSON.stringify(hashes), + signal: controller.signal, + }); + clearTimeout(timeoutId); + if (res.ok) result = await res.json(); + else throw new Error(`Server returned ${res.status}`); + } catch (err) { + if (err.name === "AbortError") { + dupProgress.innerHTML = 'Duplicate check batch timed out. Upload is locked until checks complete.'; + } else { + console.error(err); + dupProgress.innerHTML = 'Duplicate check failed. Upload is locked until checks complete.'; + } + return; + } + + for (const record of batch) { + const info = result[record.blake3Hash]; + if (info && info.id) { + record.duplicate = true; + if (info.url) { + record.duplicateUrl = info.url; + duplicateSeriesMap[info.url] = info.id; + } + } + record.duplicateChecked = true; + } + + renderGroupedPreview(); } + } finally { + if (runId === window.uploadPreview.duplicateCheckRunId) { + window.uploadPreview.duplicateCheckInProgress = false; + } + } + + if (runId !== window.uploadPreview.duplicateCheckRunId) { + return; } if (Object.keys(duplicateSeriesMap).length) { dupProgress.innerHTML = `Duplicates detected. Remove duplicates before upload.`; const duplicateSeries = document.getElementById("duplicate-series"); - duplicateSeries.innerHTML = ""; Object.entries(duplicateSeriesMap).forEach(([url, id]) => { const item = document.createElement("li"); item.innerHTML = `Series ${id}`; duplicateSeries.appendChild(item); }); } else { - dupProgress.innerHTML = 'No duplicates detected.'; + dupProgress.innerHTML = 'All duplicate checks complete. No duplicates detected.'; } + + updateUploadButtonState(); } function collectSelectedRecords() { @@ -734,10 +819,9 @@ renderGroupedPreview(); await runOcrChecks(); - await processDuplicates(); - renderGroupedPreview(); showLoading(false); + scheduleDuplicateCheck(); } function* chunks(arr, n) { @@ -831,6 +915,12 @@ return; } + if (document.getElementById("check-duplicates").checked && + window.uploadPreview.records.some((r) => r.include && !r.duplicateChecked)) { + alert("Duplicate checks are still running or incomplete. Please wait for all checks to finish before uploading."); + return; + } + resetUploadResults(); showLoading(true, "Uploading..."); @@ -872,7 +962,7 @@ }); } - const viewerElement = document.getElementById('upload-dicom-viewer'); + const viewerElement = document.getElementById('uploadviewer'); if (!viewerElement) { console.warn('Viewer element not found'); return; @@ -881,6 +971,10 @@ //console.log('Clearing viewer element and loading files'); //// Clear any old viewer content before mounting dv3d //viewerElement.innerHTML = ''; + if (window.clearViewer_uploadviewer && typeof window.clearViewer_uploadviewer === 'function') { + console.log('Clearing viewer using clearViewer_uploadviewer'); + //window.clearViewer_uploadviewer(); + } // Try to load files using the dv3d viewer API if (window.loadDicomStackFromFiles && typeof window.loadDicomStackFromFiles === 'function') { @@ -1014,7 +1108,7 @@ if (includeSeries) { applyIncludeToSeries(includeSeries.dataset.seriesUid, includeSeries.checked); buildSelectionSummary(); - await processDuplicates(); + scheduleDuplicateCheck(); renderGroupedPreview(); return; } @@ -1022,7 +1116,7 @@ const includeStudy = e.target.closest(".include-study"); if (includeStudy) { applyIncludeToStudy(includeStudy.dataset.studyUid, includeStudy.dataset.value === "1"); - await processDuplicates(); + scheduleDuplicateCheck(); renderGroupedPreview(); return; } @@ -1055,7 +1149,7 @@ }); bindIfExists("check-duplicates", "change", async () => { - await processDuplicates(); + scheduleDuplicateCheck(); renderGroupedPreview(); });