improve image hashing

atlas/api.py

@@ -58,10 +58,10 @@ def upload_dicom(request, files: List[UploadedFile] = File(...)):
 
             ud.save()
 
-            uploaded.append((file.name, ud.image_md5_hash))
+            uploaded.append((file.name, ud.image_blake3_hash))
         except DuplicateDicom:
-            print(ud.check_for_duplicates(ud.image_md5_hash))
-            duplicate.append((file.name, ud.image_md5_hash))
+            print(ud.check_for_duplicates(ud.image_blake3_hash))
+            duplicate.append((file.name, ud.image_blake3_hash))
             pass
 
     return {"uploaded": uploaded, "duplicates": duplicate}
@@ -91,7 +91,7 @@ def uncategorised_dicoms(request):
         tags = d.get_basic_dicom_tags()
 
         data[tags["SeriesInstanceUID"]].append(tags)
-        data["image_hash"].append(d.image_md5_hash)
+        data["image_hash"].append(d.image_blake3_hash)
 
     return data
 
@@ -137,7 +137,7 @@ def import_dicoms_helper(request, case_id: int | None = None):
                 image=dicom.image,
                 series=series,
                 is_dicom=True,
-                image_md5_hash=dicom.image_md5_hash,
+                image_blake3_hash=dicom.image_blake3_hash,
             )
             series_image.save()
             # dicom.image.delete()
@@ -171,9 +171,9 @@ def series_remove_duplicate_images(request, series_id: int):
     img_ids = set()
     dupes = set()
     for series_image in series.images.all():
-        if series_image.image_md5_hash in img_ids:
+        if series_image.image_blake3_hash in img_ids:
             dupes.add(series_image.id)
-        img_ids.add(series_image.image_md5_hash)
+        img_ids.add(series_image.image_blake3_hash)
 
     if dupes:
         SeriesImage.objects.filter(id__in=dupes).delete()
@@ -210,7 +210,7 @@ def get_cases_user(request):
 def check_image_hash(request, hash: str):
     try:
         print(hash)
-        series_image = SeriesImage.objects.get(image_md5_hash=hash)
+        series_image = SeriesImage.objects.get(image_blake3_hash=hash)
         data = {
             "status": "success",
             "id": series_image.pk,
@@ -234,7 +234,7 @@ def check_images_hashes(request, hashes: List[str]):
     for hash in hashes:
         try:
             print(hash)
-            series_image = SeriesImage.objects.get(image_md5_hash=hash)
+            series_image = SeriesImage.objects.get(image_blake3_hash=hash)
             data = {
                 "id": series_image.pk,
                 "url": series_image.series.get_absolute_url(),
@@ -250,7 +250,7 @@ def check_images_hashes(request, hashes: List[str]):
 def generate_image_hash(request, id: int):
         series_image = SeriesImage.objects.get(pk=id)
 
-        series_image.image_md5_hash = ""
+        series_image.image_blake3_hash = ""
 
         series_image.save()
 
@@ -259,7 +259,7 @@ def generate_image_hash(request, id: int):
 @router.get("/view_dicom_tags/{hash}", auth=django_auth)
 def view_dicom_tags(request, hash: str):
 
-    item = SeriesImage.objects.get(image_md5_hash=hash)
+    item = SeriesImage.objects.get(image_blake3_hash=hash)
     return item.get_dicom_json()
 
 @router.get("/series_split_by_dicom_tag/{series_id}/{dicom_tag}", auth=django_auth)

atlas/forms.py

@@ -444,7 +444,7 @@ CaseCollectionCaseFormSet = inlineformset_factory(
 SeriesImageFormSet = inlineformset_factory(
     Series,
     SeriesImage,
-    exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "is_dicom"],
+    exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "image_blake3_hash", "is_dicom"],
     can_delete=True,
     extra=0,
     max_num=2000,

atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0034_uncategoriseddicom_basic_dicom_tags_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='uncategoriseddicom',
+            name='image_md5_hash',
+        ),
+        migrations.AddField(
+            model_name='seriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+        migrations.AddField(
+            model_name='uncategoriseddicom',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+        migrations.AlterField(
+            model_name='uncategoriseddicom',
+            name='basic_dicom_tags',
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0035_remove_uncategoriseddicom_image_md5_hash_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='uncategoriseddicom',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=64, null=True),
+        ),
+    ]

atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0036_alter_uncategoriseddicom_image_blake3_hash'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='seriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=64, null=True),
+        ),
+    ]

atlas/models.py

@@ -892,7 +892,7 @@ class UncategorisedDicom(models.Model):
 
     series_instance_uid = models.CharField(max_length=255, blank=True, null=True)
 
-    image_md5_hash = models.CharField(max_length=32, null=True, blank=True)
+    image_blake3_hash = models.CharField(max_length=64, null=True, blank=True)
 
     created_date = models.DateTimeField(default=timezone.now)
 
@@ -901,12 +901,13 @@ class UncategorisedDicom(models.Model):
     def check_for_duplicates(self, image_hash):
         duplicate = None
 
-        if obj := UncategorisedDicom.objects.filter(image_md5_hash=image_hash).first():
-            duplicate = obj
+        if obj := UncategorisedDicom.objects.filter(image_blake3_hash=image_hash).first():
+            #duplicate = obj
+            return obj
 
         print(image_hash)
 
-        if obj := SeriesImage.objects.filter(image_md5_hash=image_hash).first():
+        if obj := SeriesImage.objects.filter(image_blake3_hash=image_hash).first():
             duplicate = obj
 
         return duplicate
@@ -920,11 +921,11 @@ class UncategorisedDicom(models.Model):
 
             # if self.check_for_duplicates():
             #    return DuplicateDicom
-            image_hash = get_image_dicom_hash(None, dataset=ds)
+            image_blake3_hash = get_image_dicom_hash(None, dataset=ds, hash_type="blake3", direct_pixel_data=True)
 
-            self.image_md5_hash = image_hash
+            self.image_blake3_hash = image_blake3_hash
 
-            duplicate = self.check_for_duplicates(image_hash)
+            duplicate = self.check_for_duplicates(image_blake3_hash)
 
             self.basic_dicom_tags = self.get_basic_dicom_tags(dataset=ds)
 
@@ -932,10 +933,8 @@ class UncategorisedDicom(models.Model):
                 if duplicate != self:
                     raise DuplicateDicom
 
-            
-
         # Hack for tests
-        if image_hash != "1234":
+        if image_blake3_hash != "1234":
             super().save(*args, **kwargs)  # Call the "real" save() method.
 
     def get_dicom_info(self):

atlas/templates/atlas/series_viewer.html

@@ -100,7 +100,7 @@
                 [{{image.image.size|filesizeformat}}]
             {% endif %}
 
-            {{image.image_md5_hash}} ({{image.is_dicom}})
+            {{image.image_md5_hash}} ({{image.is_dicom}}) {{image.image_blake3_hash}}
 
             <br />
         {% comment %} {{image.get_dicom_info|safe}}<br /> {% endcomment %}