ross/penracourses
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Implement duplicate check functionality during DICOM uploads and update viewer element ID

Browse Source
This commit is contained in:
Ross
2026-05-11 11:53:18 +01:00
parent 84b032addb
commit b24eac862b
1 changed files with 152 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
atlas/templates/atlas/new_uploads.html
+116 -22
View File
@@ -93,7 +93,7 @@
</div> </div>
</div> </div>
<div class="card-body" style="position: relative; min-height: 450px;"> <div class="card-body" style="position: relative; min-height: 450px;">
<div id="upload-dicom-viewer" class="dicom-viewer-root" style="position: absolute; top: 0; left: 0; right: 0; bottom: 0; width: 100%; height: 100%;"></div> <div id="uploadviewer" class="dicom-viewer-root" style="position: absolute; top: 0; left: 0; right: 0; bottom: 0; width: 100%; height: 100%;"></div>
</div> </div>
</div> </div>
@@ -176,11 +176,15 @@
forceAnonymisationOverride: false, forceAnonymisationOverride: false,
ocrWorker: null, ocrWorker: null,
currentTagsRecord: null, currentTagsRecord: null,
duplicateCheckInProgress: false,
duplicateCheckRunId: 0,
duplicateCheckTimer: null,
}; };
const SITE_PREFIXES = ["ref", "rk9", "ra9", "rh8", "rbz", "rba"]; const SITE_PREFIXES = ["ref", "rk9", "ra9", "rh8", "rbz", "rba"];
const OCR_MAX_SERIES_SAMPLES = 40; const OCR_MAX_SERIES_SAMPLES = 40;
const OCR_CANVAS_MAX = 768; const OCR_CANVAS_MAX = 768;
const DUPLICATE_BATCH_SIZE = 50;
function setProgress(text) { function setProgress(text) {
document.getElementById("progress-text").textContent = text; document.getElementById("progress-text").textContent = text;
@@ -241,6 +245,7 @@
ocrWarning: false, ocrWarning: false,
duplicate: false, duplicate: false,
duplicateUrl: "", duplicateUrl: "",
duplicateChecked: false,
blake3Hash: "", blake3Hash: "",
objectUrl: URL.createObjectURL(file), objectUrl: URL.createObjectURL(file),
}; };
@@ -320,6 +325,9 @@
const hasSelected = window.uploadPreview.records.some(r => r.include); const hasSelected = window.uploadPreview.records.some(r => r.include);
const hasWarnings = hasAnonymisationWarnings(); const hasWarnings = hasAnonymisationWarnings();
const hasDuplicates = window.uploadPreview.records.some(r => r.include && r.duplicate); const hasDuplicates = window.uploadPreview.records.some(r => r.include && r.duplicate);
const duplicatesEnabled = document.getElementById("check-duplicates")?.checked;
const hasPendingDuplicateChecks = duplicatesEnabled && hasSelected &&
window.uploadPreview.records.some((r) => r.include && !r.duplicateChecked);
if (!hasSelected) { if (!hasSelected) {
uploadButton.disabled = true; uploadButton.disabled = true;
@@ -333,10 +341,36 @@
return; return;
} }
if (hasPendingDuplicateChecks || window.uploadPreview.duplicateCheckInProgress) {
uploadButton.disabled = true;
anonAlert.classList.add("d-none");
return;
}
anonAlert.classList.add("d-none"); anonAlert.classList.add("d-none");
uploadButton.disabled = !!hasDuplicates; uploadButton.disabled = !!hasDuplicates;
} }
function getSeriesDuplicateCheckBadge(series) {
const duplicatesEnabled = document.getElementById("check-duplicates")?.checked;
if (!duplicatesEnabled) {
return '<span class="badge text-bg-secondary">Duplicate check off</span>';
}
const selectedFiles = series.files.filter((f) => f.include);
if (!selectedFiles.length) {
return '<span class="badge text-bg-secondary">Not selected</span>';
}
const checkedCount = selectedFiles.filter((f) => f.duplicateChecked).length;
if (checkedCount < selectedFiles.length) {
const badgeClass = window.uploadPreview.duplicateCheckInProgress ? "text-bg-info" : "text-bg-warning";
return `<span class="badge ${badgeClass}">Duplicate check: ${checkedCount}/${selectedFiles.length}</span>`;
}
return '<span class="badge text-bg-success">Duplicate check complete</span>';
}
function getSeriesStatusClass(series) { function getSeriesStatusClass(series) {
const hasTagWarnings = series.files.some(f => f.tagWarnings.length > 0); const hasTagWarnings = series.files.some(f => f.tagWarnings.length > 0);
const hasOcrWarnings = series.files.some(f => f.ocrWarning); const hasOcrWarnings = series.files.some(f => f.ocrWarning);
@@ -392,6 +426,7 @@
<div class="series-flags mt-2 small"> <div class="series-flags mt-2 small">
${tagWarningCount ? `<span class="badge text-bg-warning">Tag warnings: ${tagWarningCount}</span>` : ""} ${tagWarningCount ? `<span class="badge text-bg-warning">Tag warnings: ${tagWarningCount}</span>` : ""}
${ocrWarningCount ? `<span class="badge text-bg-warning">OCR warnings: ${ocrWarningCount}</span>` : ""} ${ocrWarningCount ? `<span class="badge text-bg-warning">OCR warnings: ${ocrWarningCount}</span>` : ""}
${getSeriesDuplicateCheckBadge(series)}
${duplicateCount ? `<span class="badge text-bg-danger">Duplicates: ${duplicateCount}</span>` : ""} ${duplicateCount ? `<span class="badge text-bg-danger">Duplicates: ${duplicateCount}</span>` : ""}
</div> </div>
<div class="series-thumbnail mt-2" data-series-thumb="${series.uid}"></div> <div class="series-thumbnail mt-2" data-series-thumb="${series.uid}"></div>
@@ -577,33 +612,69 @@
} }
} }
function scheduleDuplicateCheck() {
if (window.uploadPreview.duplicateCheckTimer) {
clearTimeout(window.uploadPreview.duplicateCheckTimer);
}
window.uploadPreview.duplicateCheckTimer = setTimeout(async () => {
await processDuplicates();
renderGroupedPreview();
}, 100);
}
async function processDuplicates() { async function processDuplicates() {
const dupProgress = document.getElementById("duplicate-progress"); const dupProgress = document.getElementById("duplicate-progress");
dupProgress.innerHTML = ""; dupProgress.innerHTML = "";
window.uploadPreview.records.forEach(r => { window.uploadPreview.records.forEach((r) => {
r.duplicate = false; r.duplicate = false;
r.duplicateUrl = ""; r.duplicateUrl = "";
if (r.include) r.duplicateChecked = false;
}); });
document.getElementById("duplicate-series").innerHTML = "";
window.uploadPreview.duplicateSeriesUrls = new Set();
if (!document.getElementById("check-duplicates").checked) { if (!document.getElementById("check-duplicates").checked) {
window.uploadPreview.records.forEach((r) => {
r.duplicateChecked = true;
});
window.uploadPreview.duplicateCheckInProgress = false;
updateUploadButtonState();
return; return;
} }
const selected = window.uploadPreview.records.filter(r => r.include); const selected = window.uploadPreview.records.filter((r) => r.include);
if (!selected.length) return; if (!selected.length) {
window.uploadPreview.duplicateCheckInProgress = false;
updateUploadButtonState();
return;
}
const runId = window.uploadPreview.duplicateCheckRunId + 1;
window.uploadPreview.duplicateCheckRunId = runId;
window.uploadPreview.duplicateCheckInProgress = true;
updateUploadButtonState();
let processed = 0;
const duplicateSeriesMap = {};
try {
for (let offset = 0; offset < selected.length; offset += DUPLICATE_BATCH_SIZE) {
if (runId !== window.uploadPreview.duplicateCheckRunId) return;
const batch = selected.slice(offset, offset + DUPLICATE_BATCH_SIZE);
const hashes = []; const hashes = [];
let idx = 0;
for (const record of selected) { for (const record of batch) {
idx += 1; if (runId !== window.uploadPreview.duplicateCheckRunId) return;
if (!record.blake3Hash) { if (!record.blake3Hash) {
setProgress(`Hashing ${idx} of ${selected.length}: ${record.path}`); setProgress(`Hashing ${processed + 1} of ${selected.length}: ${record.path}`);
record.blake3Hash = await calculateBlake3Hash(record.file); record.blake3Hash = await calculateBlake3Hash(record.file);
} }
hashes.push(record.blake3Hash); hashes.push(record.blake3Hash);
processed += 1;
} }
dupProgress.innerHTML = '<span class="text-info">Checking hashes with server...</span>'; dupProgress.innerHTML = `<span class="text-info">Checking duplicate batch ${Math.floor(offset / DUPLICATE_BATCH_SIZE) + 1} of ${Math.ceil(selected.length / DUPLICATE_BATCH_SIZE)} (${processed}/${selected.length})...</span>`;
let result = {}; let result = {};
try { try {
@@ -623,37 +694,51 @@
else throw new Error(`Server returned ${res.status}`); else throw new Error(`Server returned ${res.status}`);
} catch (err) { } catch (err) {
if (err.name === "AbortError") { if (err.name === "AbortError") {
dupProgress.innerHTML = '<span class="text-warning">Duplicate check timed out. Proceeding without server verification.</span>'; dupProgress.innerHTML = '<span class="text-warning">Duplicate check batch timed out. Upload is locked until checks complete.</span>';
} else { } else {
console.error(err); console.error(err);
dupProgress.innerHTML = '<span class="text-warning">Could not verify duplicates. Proceeding.</span>'; dupProgress.innerHTML = '<span class="text-warning">Duplicate check failed. Upload is locked until checks complete.</span>';
} }
return; return;
} }
const duplicateSeriesMap = {}; for (const record of batch) {
for (const record of selected) {
const info = result[record.blake3Hash]; const info = result[record.blake3Hash];
if (!info || !info.id) continue; if (info && info.id) {
record.duplicate = true; record.duplicate = true;
if (info.url) { if (info.url) {
record.duplicateUrl = info.url; record.duplicateUrl = info.url;
duplicateSeriesMap[info.url] = info.id; duplicateSeriesMap[info.url] = info.id;
} }
} }
record.duplicateChecked = true;
}
renderGroupedPreview();
}
} finally {
if (runId === window.uploadPreview.duplicateCheckRunId) {
window.uploadPreview.duplicateCheckInProgress = false;
}
}
if (runId !== window.uploadPreview.duplicateCheckRunId) {
return;
}
if (Object.keys(duplicateSeriesMap).length) { if (Object.keys(duplicateSeriesMap).length) {
dupProgress.innerHTML = `<span class="text-danger fw-semibold">Duplicates detected. Remove duplicates before upload.</span>`; dupProgress.innerHTML = `<span class="text-danger fw-semibold">Duplicates detected. Remove duplicates before upload.</span>`;
const duplicateSeries = document.getElementById("duplicate-series"); const duplicateSeries = document.getElementById("duplicate-series");
duplicateSeries.innerHTML = "";
Object.entries(duplicateSeriesMap).forEach(([url, id]) => { Object.entries(duplicateSeriesMap).forEach(([url, id]) => {
const item = document.createElement("li"); const item = document.createElement("li");
item.innerHTML = `<a href="${url}" target="_blank">Series ${id}</a>`; item.innerHTML = `<a href="${url}" target="_blank">Series ${id}</a>`;
duplicateSeries.appendChild(item); duplicateSeries.appendChild(item);
}); });
} else { } else {
dupProgress.innerHTML = '<span class="text-success">No duplicates detected.</span>'; dupProgress.innerHTML = '<span class="text-success">All duplicate checks complete. No duplicates detected.</span>';
} }
updateUploadButtonState();
} }
function collectSelectedRecords() { function collectSelectedRecords() {
@@ -734,10 +819,9 @@
renderGroupedPreview(); renderGroupedPreview();
await runOcrChecks(); await runOcrChecks();
await processDuplicates();
renderGroupedPreview(); renderGroupedPreview();
showLoading(false); showLoading(false);
scheduleDuplicateCheck();
} }
function* chunks(arr, n) { function* chunks(arr, n) {
@@ -831,6 +915,12 @@
return; return;
} }
if (document.getElementById("check-duplicates").checked &&
window.uploadPreview.records.some((r) => r.include && !r.duplicateChecked)) {
alert("Duplicate checks are still running or incomplete. Please wait for all checks to finish before uploading.");
return;
}
resetUploadResults(); resetUploadResults();
showLoading(true, "Uploading..."); showLoading(true, "Uploading...");
@@ -872,7 +962,7 @@
}); });
} }
const viewerElement = document.getElementById('upload-dicom-viewer'); const viewerElement = document.getElementById('uploadviewer');
if (!viewerElement) { if (!viewerElement) {
console.warn('Viewer element not found'); console.warn('Viewer element not found');
return; return;
@@ -881,6 +971,10 @@
//console.log('Clearing viewer element and loading files'); //console.log('Clearing viewer element and loading files');
//// Clear any old viewer content before mounting dv3d //// Clear any old viewer content before mounting dv3d
//viewerElement.innerHTML = ''; //viewerElement.innerHTML = '';
if (window.clearViewer_uploadviewer && typeof window.clearViewer_uploadviewer === 'function') {
console.log('Clearing viewer using clearViewer_uploadviewer');
//window.clearViewer_uploadviewer();
}
// Try to load files using the dv3d viewer API // Try to load files using the dv3d viewer API
if (window.loadDicomStackFromFiles && typeof window.loadDicomStackFromFiles === 'function') { if (window.loadDicomStackFromFiles && typeof window.loadDicomStackFromFiles === 'function') {
@@ -1014,7 +1108,7 @@
if (includeSeries) { if (includeSeries) {
applyIncludeToSeries(includeSeries.dataset.seriesUid, includeSeries.checked); applyIncludeToSeries(includeSeries.dataset.seriesUid, includeSeries.checked);
buildSelectionSummary(); buildSelectionSummary();
await processDuplicates(); scheduleDuplicateCheck();
renderGroupedPreview(); renderGroupedPreview();
return; return;
} }
@@ -1022,7 +1116,7 @@
const includeStudy = e.target.closest(".include-study"); const includeStudy = e.target.closest(".include-study");
if (includeStudy) { if (includeStudy) {
applyIncludeToStudy(includeStudy.dataset.studyUid, includeStudy.dataset.value === "1"); applyIncludeToStudy(includeStudy.dataset.studyUid, includeStudy.dataset.value === "1");
await processDuplicates(); scheduleDuplicateCheck();
renderGroupedPreview(); renderGroupedPreview();
return; return;
} }
@@ -1055,7 +1149,7 @@
}); });
bindIfExists("check-duplicates", "change", async () => { bindIfExists("check-duplicates", "change", async () => {
await processDuplicates(); scheduleDuplicateCheck();
renderGroupedPreview(); renderGroupedPreview();
}); });