diff --git a/atlas/templates/atlas/new_uploads.html b/atlas/templates/atlas/new_uploads.html
index e670b031..72fa1d0b 100644
--- a/atlas/templates/atlas/new_uploads.html
+++ b/atlas/templates/atlas/new_uploads.html
@@ -93,7 +93,7 @@
@@ -176,11 +176,15 @@
forceAnonymisationOverride: false,
ocrWorker: null,
currentTagsRecord: null,
+ duplicateCheckInProgress: false,
+ duplicateCheckRunId: 0,
+ duplicateCheckTimer: null,
};
const SITE_PREFIXES = ["ref", "rk9", "ra9", "rh8", "rbz", "rba"];
const OCR_MAX_SERIES_SAMPLES = 40;
const OCR_CANVAS_MAX = 768;
+ const DUPLICATE_BATCH_SIZE = 50;
function setProgress(text) {
document.getElementById("progress-text").textContent = text;
@@ -241,6 +245,7 @@
ocrWarning: false,
duplicate: false,
duplicateUrl: "",
+ duplicateChecked: false,
blake3Hash: "",
objectUrl: URL.createObjectURL(file),
};
@@ -320,6 +325,9 @@
const hasSelected = window.uploadPreview.records.some(r => r.include);
const hasWarnings = hasAnonymisationWarnings();
const hasDuplicates = window.uploadPreview.records.some(r => r.include && r.duplicate);
+ const duplicatesEnabled = document.getElementById("check-duplicates")?.checked;
+ const hasPendingDuplicateChecks = duplicatesEnabled && hasSelected &&
+ window.uploadPreview.records.some((r) => r.include && !r.duplicateChecked);
if (!hasSelected) {
uploadButton.disabled = true;
@@ -333,10 +341,36 @@
return;
}
+ if (hasPendingDuplicateChecks || window.uploadPreview.duplicateCheckInProgress) {
+ uploadButton.disabled = true;
+ anonAlert.classList.add("d-none");
+ return;
+ }
+
anonAlert.classList.add("d-none");
uploadButton.disabled = !!hasDuplicates;
}
+ function getSeriesDuplicateCheckBadge(series) {
+ const duplicatesEnabled = document.getElementById("check-duplicates")?.checked;
+ if (!duplicatesEnabled) {
+ return 'Duplicate check off';
+ }
+
+ const selectedFiles = series.files.filter((f) => f.include);
+ if (!selectedFiles.length) {
+ return 'Not selected';
+ }
+
+ const checkedCount = selectedFiles.filter((f) => f.duplicateChecked).length;
+ if (checkedCount < selectedFiles.length) {
+ const badgeClass = window.uploadPreview.duplicateCheckInProgress ? "text-bg-info" : "text-bg-warning";
+ return `Duplicate check: ${checkedCount}/${selectedFiles.length}`;
+ }
+
+ return 'Duplicate check complete';
+ }
+
function getSeriesStatusClass(series) {
const hasTagWarnings = series.files.some(f => f.tagWarnings.length > 0);
const hasOcrWarnings = series.files.some(f => f.ocrWarning);
@@ -392,6 +426,7 @@
${tagWarningCount ? `Tag warnings: ${tagWarningCount}` : ""}
${ocrWarningCount ? `OCR warnings: ${ocrWarningCount}` : ""}
+ ${getSeriesDuplicateCheckBadge(series)}
${duplicateCount ? `Duplicates: ${duplicateCount}` : ""}
@@ -577,83 +612,133 @@
}
}
+ function scheduleDuplicateCheck() {
+ if (window.uploadPreview.duplicateCheckTimer) {
+ clearTimeout(window.uploadPreview.duplicateCheckTimer);
+ }
+ window.uploadPreview.duplicateCheckTimer = setTimeout(async () => {
+ await processDuplicates();
+ renderGroupedPreview();
+ }, 100);
+ }
+
async function processDuplicates() {
const dupProgress = document.getElementById("duplicate-progress");
dupProgress.innerHTML = "";
- window.uploadPreview.records.forEach(r => {
+ window.uploadPreview.records.forEach((r) => {
r.duplicate = false;
r.duplicateUrl = "";
+ if (r.include) r.duplicateChecked = false;
});
+ document.getElementById("duplicate-series").innerHTML = "";
+ window.uploadPreview.duplicateSeriesUrls = new Set();
if (!document.getElementById("check-duplicates").checked) {
- return;
- }
-
- const selected = window.uploadPreview.records.filter(r => r.include);
- if (!selected.length) return;
-
- const hashes = [];
- let idx = 0;
- for (const record of selected) {
- idx += 1;
- if (!record.blake3Hash) {
- setProgress(`Hashing ${idx} of ${selected.length}: ${record.path}`);
- record.blake3Hash = await calculateBlake3Hash(record.file);
- }
- hashes.push(record.blake3Hash);
- }
-
- dupProgress.innerHTML = 'Checking hashes with server...';
-
- let result = {};
- try {
- const controller = new AbortController();
- const timeoutId = setTimeout(() => controller.abort(), 30000);
- const res = await fetch("{% url 'api-1:check_images_hashes' %}", {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- "X-CSRFToken": "{{ csrf_token }}",
- },
- body: JSON.stringify(hashes),
- signal: controller.signal,
+ window.uploadPreview.records.forEach((r) => {
+ r.duplicateChecked = true;
});
- clearTimeout(timeoutId);
- if (res.ok) result = await res.json();
- else throw new Error(`Server returned ${res.status}`);
- } catch (err) {
- if (err.name === "AbortError") {
- dupProgress.innerHTML = 'Duplicate check timed out. Proceeding without server verification.';
- } else {
- console.error(err);
- dupProgress.innerHTML = 'Could not verify duplicates. Proceeding.';
- }
+ window.uploadPreview.duplicateCheckInProgress = false;
+ updateUploadButtonState();
return;
}
+ const selected = window.uploadPreview.records.filter((r) => r.include);
+ if (!selected.length) {
+ window.uploadPreview.duplicateCheckInProgress = false;
+ updateUploadButtonState();
+ return;
+ }
+
+ const runId = window.uploadPreview.duplicateCheckRunId + 1;
+ window.uploadPreview.duplicateCheckRunId = runId;
+ window.uploadPreview.duplicateCheckInProgress = true;
+ updateUploadButtonState();
+
+ let processed = 0;
const duplicateSeriesMap = {};
- for (const record of selected) {
- const info = result[record.blake3Hash];
- if (!info || !info.id) continue;
- record.duplicate = true;
- if (info.url) {
- record.duplicateUrl = info.url;
- duplicateSeriesMap[info.url] = info.id;
+
+ try {
+ for (let offset = 0; offset < selected.length; offset += DUPLICATE_BATCH_SIZE) {
+ if (runId !== window.uploadPreview.duplicateCheckRunId) return;
+
+ const batch = selected.slice(offset, offset + DUPLICATE_BATCH_SIZE);
+ const hashes = [];
+
+ for (const record of batch) {
+ if (runId !== window.uploadPreview.duplicateCheckRunId) return;
+ if (!record.blake3Hash) {
+ setProgress(`Hashing ${processed + 1} of ${selected.length}: ${record.path}`);
+ record.blake3Hash = await calculateBlake3Hash(record.file);
+ }
+ hashes.push(record.blake3Hash);
+ processed += 1;
+ }
+
+ dupProgress.innerHTML = `Checking duplicate batch ${Math.floor(offset / DUPLICATE_BATCH_SIZE) + 1} of ${Math.ceil(selected.length / DUPLICATE_BATCH_SIZE)} (${processed}/${selected.length})...`;
+
+ let result = {};
+ try {
+ const controller = new AbortController();
+ const timeoutId = setTimeout(() => controller.abort(), 30000);
+ const res = await fetch("{% url 'api-1:check_images_hashes' %}", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "X-CSRFToken": "{{ csrf_token }}",
+ },
+ body: JSON.stringify(hashes),
+ signal: controller.signal,
+ });
+ clearTimeout(timeoutId);
+ if (res.ok) result = await res.json();
+ else throw new Error(`Server returned ${res.status}`);
+ } catch (err) {
+ if (err.name === "AbortError") {
+ dupProgress.innerHTML = 'Duplicate check batch timed out. Upload is locked until checks complete.';
+ } else {
+ console.error(err);
+ dupProgress.innerHTML = 'Duplicate check failed. Upload is locked until checks complete.';
+ }
+ return;
+ }
+
+ for (const record of batch) {
+ const info = result[record.blake3Hash];
+ if (info && info.id) {
+ record.duplicate = true;
+ if (info.url) {
+ record.duplicateUrl = info.url;
+ duplicateSeriesMap[info.url] = info.id;
+ }
+ }
+ record.duplicateChecked = true;
+ }
+
+ renderGroupedPreview();
}
+ } finally {
+ if (runId === window.uploadPreview.duplicateCheckRunId) {
+ window.uploadPreview.duplicateCheckInProgress = false;
+ }
+ }
+
+ if (runId !== window.uploadPreview.duplicateCheckRunId) {
+ return;
}
if (Object.keys(duplicateSeriesMap).length) {
dupProgress.innerHTML = `Duplicates detected. Remove duplicates before upload.`;
const duplicateSeries = document.getElementById("duplicate-series");
- duplicateSeries.innerHTML = "";
Object.entries(duplicateSeriesMap).forEach(([url, id]) => {
const item = document.createElement("li");
item.innerHTML = `Series ${id}`;
duplicateSeries.appendChild(item);
});
} else {
- dupProgress.innerHTML = 'No duplicates detected.';
+ dupProgress.innerHTML = 'All duplicate checks complete. No duplicates detected.';
}
+
+ updateUploadButtonState();
}
function collectSelectedRecords() {
@@ -734,10 +819,9 @@
renderGroupedPreview();
await runOcrChecks();
- await processDuplicates();
-
renderGroupedPreview();
showLoading(false);
+ scheduleDuplicateCheck();
}
function* chunks(arr, n) {
@@ -831,6 +915,12 @@
return;
}
+ if (document.getElementById("check-duplicates").checked &&
+ window.uploadPreview.records.some((r) => r.include && !r.duplicateChecked)) {
+ alert("Duplicate checks are still running or incomplete. Please wait for all checks to finish before uploading.");
+ return;
+ }
+
resetUploadResults();
showLoading(true, "Uploading...");
@@ -872,7 +962,7 @@
});
}
- const viewerElement = document.getElementById('upload-dicom-viewer');
+ const viewerElement = document.getElementById('uploadviewer');
if (!viewerElement) {
console.warn('Viewer element not found');
return;
@@ -881,6 +971,10 @@
//console.log('Clearing viewer element and loading files');
//// Clear any old viewer content before mounting dv3d
//viewerElement.innerHTML = '';
+ if (window.clearViewer_uploadviewer && typeof window.clearViewer_uploadviewer === 'function') {
+ console.log('Clearing viewer using clearViewer_uploadviewer');
+ //window.clearViewer_uploadviewer();
+ }
// Try to load files using the dv3d viewer API
if (window.loadDicomStackFromFiles && typeof window.loadDicomStackFromFiles === 'function') {
@@ -1014,7 +1108,7 @@
if (includeSeries) {
applyIncludeToSeries(includeSeries.dataset.seriesUid, includeSeries.checked);
buildSelectionSummary();
- await processDuplicates();
+ scheduleDuplicateCheck();
renderGroupedPreview();
return;
}
@@ -1022,7 +1116,7 @@
const includeStudy = e.target.closest(".include-study");
if (includeStudy) {
applyIncludeToStudy(includeStudy.dataset.studyUid, includeStudy.dataset.value === "1");
- await processDuplicates();
+ scheduleDuplicateCheck();
renderGroupedPreview();
return;
}
@@ -1055,7 +1149,7 @@
});
bindIfExists("check-duplicates", "change", async () => {
- await processDuplicates();
+ scheduleDuplicateCheck();
renderGroupedPreview();
});