feat: Optimize image hash checks with bulk queries and add database indexing for performance

atlas/api.py

@@ -583,41 +583,57 @@ def check_image_hash(request, hash: str):
 
 @router.post("/check_image_hashes/", auth=TokenAuth())
 def check_images_hashes(request, hashes: List[str]):
-    """Checks a list of image hashes and returns the series id / url if found
+    """Checks a list of image hashes and returns the series id / url if found.
+
+    Uses two bulk queries (one for SeriesImage, one for UncategorisedDicom) to
+    handle large batches efficiently regardless of batch size.
 
     Return format
     { "hash_id": {"id": "series_id|false", "url": "series_url|false"}, ...}
     """
 
     try:
-        hash_status = {}
+        hash_status: dict = {h: {"id": False, "url": False} for h in hashes}
 
-        for hash in hashes:
+        # Single bulk query for all SeriesImage matches
-            # Prefer filter().first() to avoid MultipleObjectsReturned
+        series_images = (
-            series_image = SeriesImage.objects.filter(image_blake3_hash=hash).select_related("series").first()
+            SeriesImage.objects.filter(image_blake3_hash__in=hashes)
-            if series_image:
+            .select_related("series")
-                try:
+            .only("pk", "image_blake3_hash", "series__slug", "series__pk")
-                    url = series_image.series.get_absolute_url()
+        )
-                except Exception:
+        matched_hashes: set = set()
-                    url = None
+        for si in series_images:
-                data = {"id": series_image.pk, "url": url, "type": "series"}
+            if si.image_blake3_hash in matched_hashes:
-                hash_status[hash] = data
+                continue  # keep first match per hash
-                continue
+            try:
+                url = si.series.get_absolute_url()
+            except Exception:
+                url = None
+            hash_status[si.image_blake3_hash] = {"id": si.pk, "url": url, "type": "series"}
+            matched_hashes.add(si.image_blake3_hash)
 
-            uncategorised = UncategorisedDicom.objects.filter(image_blake3_hash=hash).first()
+        # Single bulk query for remaining hashes in UncategorisedDicom
-            if uncategorised:
+        remaining = [h for h in hashes if h not in matched_hashes]
-                try:
+        if remaining:
-                    uploads_url = reverse("atlas:user_uploads")
+            try:
-                except Exception:
+                uploads_url = reverse("atlas:user_uploads")
-                    uploads_url = None
+            except Exception:
-                data = {"id": uncategorised.pk, "url": uploads_url, "type": "uncategorised"}
+                uploads_url = None
-                hash_status[hash] = data
+            uncategorised_qs = (
-                continue
+                UncategorisedDicom.objects.filter(image_blake3_hash__in=remaining)
-
+                .only("pk", "image_blake3_hash")
-            hash_status[hash] = {"id": False, "url": False}
+            )
+            for uc in uncategorised_qs:
+                if uc.image_blake3_hash not in matched_hashes:
+                    hash_status[uc.image_blake3_hash] = {
+                        "id": uc.pk,
+                        "url": uploads_url,
+                        "type": "uncategorised",
+                    }
+                    matched_hashes.add(uc.image_blake3_hash)
 
         return hash_status
-    except Exception as e:
+    except Exception:
         logger.exception("check_images_hashes failed")
         return JsonResponse({"detail": "Internal server error"}, status=500)
 

atlas/migrations/0102_alter_seriesimage_image_blake3_hash_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.7 on 2026-05-11 11:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0101_series_study_instance_uid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='seriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, db_index=True, max_length=64, null=True),
+        ),
+        migrations.AlterField(
+            model_name='uncategoriseddicom',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, db_index=True, max_length=64, null=True),
+        ),
+    ]

atlas/models.py

@@ -2173,7 +2173,7 @@ class UncategorisedDicom(models.Model):
 
     series_instance_uid = models.CharField(max_length=255, blank=True, null=True)
 
-    image_blake3_hash = models.CharField(max_length=64, null=True, blank=True)
+    image_blake3_hash = models.CharField(max_length=64, null=True, blank=True, db_index=True)
 
     created_date = models.DateTimeField(default=timezone.now)
 

atlas/templates/atlas/new_uploads.html

@@ -184,7 +184,7 @@
         const SITE_PREFIXES = ["ref", "rk9", "ra9", "rh8", "rbz", "rba"];
         const OCR_MAX_SERIES_SAMPLES = 40;
         const OCR_CANVAS_MAX = 768;
-        const DUPLICATE_BATCH_SIZE = 50;
+        const DUPLICATE_BATCH_SIZE = 500;
 
         function setProgress(text) {
             document.getElementById("progress-text").textContent = text;

generic/models.py

@@ -298,7 +298,7 @@ class SeriesImageBase(models.Model):
 
     # The second is a blake3 hash of the direct pixel data
     # This is much faster when using pydicom
-    image_blake3_hash = models.CharField(max_length=64, null=True, blank=True)
+    image_blake3_hash = models.CharField(max_length=64, null=True, blank=True, db_index=True)
 
     is_dicom = models.BooleanField(default=False)
 

longs/migrations/0035_alter_longseriesimage_image_blake3_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.7 on 2026-05-11 11:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('longs', '0034_exam_created_exam_updated'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='longseriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, db_index=True, max_length=64, null=True),
+        ),
+    ]