diff --git a/atlas/api.py b/atlas/api.py index 08215f04..6a10373b 100644 --- a/atlas/api.py +++ b/atlas/api.py @@ -583,41 +583,57 @@ def check_image_hash(request, hash: str): @router.post("/check_image_hashes/", auth=TokenAuth()) def check_images_hashes(request, hashes: List[str]): - """Checks a list of image hashes and returns the series id / url if found + """Checks a list of image hashes and returns the series id / url if found. + + Uses two bulk queries (one for SeriesImage, one for UncategorisedDicom) to + handle large batches efficiently regardless of batch size. Return format { "hash_id": {"id": "series_id|false", "url": "series_url|false"}, ...} """ try: - hash_status = {} + hash_status: dict = {h: {"id": False, "url": False} for h in hashes} - for hash in hashes: - # Prefer filter().first() to avoid MultipleObjectsReturned - series_image = SeriesImage.objects.filter(image_blake3_hash=hash).select_related("series").first() - if series_image: - try: - url = series_image.series.get_absolute_url() - except Exception: - url = None - data = {"id": series_image.pk, "url": url, "type": "series"} - hash_status[hash] = data - continue + # Single bulk query for all SeriesImage matches + series_images = ( + SeriesImage.objects.filter(image_blake3_hash__in=hashes) + .select_related("series") + .only("pk", "image_blake3_hash", "series__slug", "series__pk") + ) + matched_hashes: set = set() + for si in series_images: + if si.image_blake3_hash in matched_hashes: + continue # keep first match per hash + try: + url = si.series.get_absolute_url() + except Exception: + url = None + hash_status[si.image_blake3_hash] = {"id": si.pk, "url": url, "type": "series"} + matched_hashes.add(si.image_blake3_hash) - uncategorised = UncategorisedDicom.objects.filter(image_blake3_hash=hash).first() - if uncategorised: - try: - uploads_url = reverse("atlas:user_uploads") - except Exception: - uploads_url = None - data = {"id": uncategorised.pk, "url": uploads_url, "type": "uncategorised"} - hash_status[hash] = data - continue - - hash_status[hash] = {"id": False, "url": False} + # Single bulk query for remaining hashes in UncategorisedDicom + remaining = [h for h in hashes if h not in matched_hashes] + if remaining: + try: + uploads_url = reverse("atlas:user_uploads") + except Exception: + uploads_url = None + uncategorised_qs = ( + UncategorisedDicom.objects.filter(image_blake3_hash__in=remaining) + .only("pk", "image_blake3_hash") + ) + for uc in uncategorised_qs: + if uc.image_blake3_hash not in matched_hashes: + hash_status[uc.image_blake3_hash] = { + "id": uc.pk, + "url": uploads_url, + "type": "uncategorised", + } + matched_hashes.add(uc.image_blake3_hash) return hash_status - except Exception as e: + except Exception: logger.exception("check_images_hashes failed") return JsonResponse({"detail": "Internal server error"}, status=500) diff --git a/atlas/migrations/0102_alter_seriesimage_image_blake3_hash_and_more.py b/atlas/migrations/0102_alter_seriesimage_image_blake3_hash_and_more.py new file mode 100644 index 00000000..42ea790d --- /dev/null +++ b/atlas/migrations/0102_alter_seriesimage_image_blake3_hash_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.7 on 2026-05-11 11:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('atlas', '0101_series_study_instance_uid'), + ] + + operations = [ + migrations.AlterField( + model_name='seriesimage', + name='image_blake3_hash', + field=models.CharField(blank=True, db_index=True, max_length=64, null=True), + ), + migrations.AlterField( + model_name='uncategoriseddicom', + name='image_blake3_hash', + field=models.CharField(blank=True, db_index=True, max_length=64, null=True), + ), + ] diff --git a/atlas/models.py b/atlas/models.py index 3367accd..a0974144 100644 --- a/atlas/models.py +++ b/atlas/models.py @@ -2173,7 +2173,7 @@ class UncategorisedDicom(models.Model): series_instance_uid = models.CharField(max_length=255, blank=True, null=True) - image_blake3_hash = models.CharField(max_length=64, null=True, blank=True) + image_blake3_hash = models.CharField(max_length=64, null=True, blank=True, db_index=True) created_date = models.DateTimeField(default=timezone.now) diff --git a/atlas/templates/atlas/new_uploads.html b/atlas/templates/atlas/new_uploads.html index f1671790..3db7a236 100644 --- a/atlas/templates/atlas/new_uploads.html +++ b/atlas/templates/atlas/new_uploads.html @@ -184,7 +184,7 @@ const SITE_PREFIXES = ["ref", "rk9", "ra9", "rh8", "rbz", "rba"]; const OCR_MAX_SERIES_SAMPLES = 40; const OCR_CANVAS_MAX = 768; - const DUPLICATE_BATCH_SIZE = 50; + const DUPLICATE_BATCH_SIZE = 500; function setProgress(text) { document.getElementById("progress-text").textContent = text; diff --git a/generic/models.py b/generic/models.py index a51de4ac..e80228c4 100644 --- a/generic/models.py +++ b/generic/models.py @@ -298,7 +298,7 @@ class SeriesImageBase(models.Model): # The second is a blake3 hash of the direct pixel data # This is much faster when using pydicom - image_blake3_hash = models.CharField(max_length=64, null=True, blank=True) + image_blake3_hash = models.CharField(max_length=64, null=True, blank=True, db_index=True) is_dicom = models.BooleanField(default=False) diff --git a/longs/migrations/0035_alter_longseriesimage_image_blake3_hash.py b/longs/migrations/0035_alter_longseriesimage_image_blake3_hash.py new file mode 100644 index 00000000..f40c6052 --- /dev/null +++ b/longs/migrations/0035_alter_longseriesimage_image_blake3_hash.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.7 on 2026-05-11 11:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('longs', '0034_exam_created_exam_updated'), + ] + + operations = [ + migrations.AlterField( + model_name='longseriesimage', + name='image_blake3_hash', + field=models.CharField(blank=True, db_index=True, max_length=64, null=True), + ), + ]