improve image hashing

anatomy/static/js/upload_form_helpers.js

@@ -222,6 +222,12 @@
 
                     console.log("image", image)
 
+                    let bytesView = image.getPixelData()
+                    let str = new TextDecoder("utf-8").decode(bytesView, ); 
+
+                    console.log("bytes", bytesView)
+                    console.log("str", str)
+
                     extractDicomDetails(image.data.byteArray)
 
                 });

atlas/api.py

@@ -58,10 +58,10 @@ def upload_dicom(request, files: List[UploadedFile] = File(...)):
 
             ud.save()
 
-            uploaded.append((file.name, ud.image_md5_hash))
+            uploaded.append((file.name, ud.image_blake3_hash))
         except DuplicateDicom:
-            print(ud.check_for_duplicates(ud.image_md5_hash))
-            duplicate.append((file.name, ud.image_md5_hash))
+            print(ud.check_for_duplicates(ud.image_blake3_hash))
+            duplicate.append((file.name, ud.image_blake3_hash))
             pass
 
     return {"uploaded": uploaded, "duplicates": duplicate}
@@ -91,7 +91,7 @@ def uncategorised_dicoms(request):
         tags = d.get_basic_dicom_tags()
 
         data[tags["SeriesInstanceUID"]].append(tags)
-        data["image_hash"].append(d.image_md5_hash)
+        data["image_hash"].append(d.image_blake3_hash)
 
     return data
 
@@ -137,7 +137,7 @@ def import_dicoms_helper(request, case_id: int | None = None):
                 image=dicom.image,
                 series=series,
                 is_dicom=True,
-                image_md5_hash=dicom.image_md5_hash,
+                image_blake3_hash=dicom.image_blake3_hash,
             )
             series_image.save()
             # dicom.image.delete()
@@ -171,9 +171,9 @@ def series_remove_duplicate_images(request, series_id: int):
     img_ids = set()
     dupes = set()
     for series_image in series.images.all():
-        if series_image.image_md5_hash in img_ids:
+        if series_image.image_blake3_hash in img_ids:
             dupes.add(series_image.id)
-        img_ids.add(series_image.image_md5_hash)
+        img_ids.add(series_image.image_blake3_hash)
 
     if dupes:
         SeriesImage.objects.filter(id__in=dupes).delete()
@@ -210,7 +210,7 @@ def get_cases_user(request):
 def check_image_hash(request, hash: str):
     try:
         print(hash)
-        series_image = SeriesImage.objects.get(image_md5_hash=hash)
+        series_image = SeriesImage.objects.get(image_blake3_hash=hash)
         data = {
             "status": "success",
             "id": series_image.pk,
@@ -234,7 +234,7 @@ def check_images_hashes(request, hashes: List[str]):
     for hash in hashes:
         try:
             print(hash)
-            series_image = SeriesImage.objects.get(image_md5_hash=hash)
+            series_image = SeriesImage.objects.get(image_blake3_hash=hash)
             data = {
                 "id": series_image.pk,
                 "url": series_image.series.get_absolute_url(),
@@ -250,7 +250,7 @@ def check_images_hashes(request, hashes: List[str]):
 def generate_image_hash(request, id: int):
         series_image = SeriesImage.objects.get(pk=id)
 
-        series_image.image_md5_hash = ""
+        series_image.image_blake3_hash = ""
 
         series_image.save()
 
@@ -259,7 +259,7 @@ def generate_image_hash(request, id: int):
 @router.get("/view_dicom_tags/{hash}", auth=django_auth)
 def view_dicom_tags(request, hash: str):
 
-    item = SeriesImage.objects.get(image_md5_hash=hash)
+    item = SeriesImage.objects.get(image_blake3_hash=hash)
     return item.get_dicom_json()
 
 @router.get("/series_split_by_dicom_tag/{series_id}/{dicom_tag}", auth=django_auth)

atlas/forms.py

@@ -444,7 +444,7 @@ CaseCollectionCaseFormSet = inlineformset_factory(
 SeriesImageFormSet = inlineformset_factory(
     Series,
     SeriesImage,
-    exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "is_dicom"],
+    exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "image_blake3_hash", "is_dicom"],
     can_delete=True,
     extra=0,
     max_num=2000,

atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0034_uncategoriseddicom_basic_dicom_tags_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='uncategoriseddicom',
+            name='image_md5_hash',
+        ),
+        migrations.AddField(
+            model_name='seriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+        migrations.AddField(
+            model_name='uncategoriseddicom',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+        migrations.AlterField(
+            model_name='uncategoriseddicom',
+            name='basic_dicom_tags',
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0035_remove_uncategoriseddicom_image_md5_hash_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='uncategoriseddicom',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=64, null=True),
+        ),
+    ]

atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0036_alter_uncategoriseddicom_image_blake3_hash'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='seriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=64, null=True),
+        ),
+    ]

atlas/models.py

@@ -892,7 +892,7 @@ class UncategorisedDicom(models.Model):
 
     series_instance_uid = models.CharField(max_length=255, blank=True, null=True)
 
-    image_md5_hash = models.CharField(max_length=32, null=True, blank=True)
+    image_blake3_hash = models.CharField(max_length=64, null=True, blank=True)
 
     created_date = models.DateTimeField(default=timezone.now)
 
@@ -901,12 +901,13 @@ class UncategorisedDicom(models.Model):
     def check_for_duplicates(self, image_hash):
         duplicate = None
 
-        if obj := UncategorisedDicom.objects.filter(image_md5_hash=image_hash).first():
-            duplicate = obj
+        if obj := UncategorisedDicom.objects.filter(image_blake3_hash=image_hash).first():
+            #duplicate = obj
+            return obj
 
         print(image_hash)
 
-        if obj := SeriesImage.objects.filter(image_md5_hash=image_hash).first():
+        if obj := SeriesImage.objects.filter(image_blake3_hash=image_hash).first():
             duplicate = obj
 
         return duplicate
@@ -920,11 +921,11 @@ class UncategorisedDicom(models.Model):
 
             # if self.check_for_duplicates():
             #    return DuplicateDicom
-            image_hash = get_image_dicom_hash(None, dataset=ds)
+            image_blake3_hash = get_image_dicom_hash(None, dataset=ds, hash_type="blake3", direct_pixel_data=True)
 
-            self.image_md5_hash = image_hash
+            self.image_blake3_hash = image_blake3_hash
 
-            duplicate = self.check_for_duplicates(image_hash)
+            duplicate = self.check_for_duplicates(image_blake3_hash)
 
             self.basic_dicom_tags = self.get_basic_dicom_tags(dataset=ds)
 
@@ -932,10 +933,8 @@ class UncategorisedDicom(models.Model):
                 if duplicate != self:
                     raise DuplicateDicom
 
-            
-
         # Hack for tests
-        if image_hash != "1234":
+        if image_blake3_hash != "1234":
             super().save(*args, **kwargs)  # Call the "real" save() method.
 
     def get_dicom_info(self):

atlas/templates/atlas/series_viewer.html

@@ -100,7 +100,7 @@
                 [{{image.image.size|filesizeformat}}]
             {% endif %}
 
-            {{image.image_md5_hash}} ({{image.is_dicom}})
+            {{image.image_md5_hash}} ({{image.is_dicom}}) {{image.image_blake3_hash}}
 
             <br />
         {% comment %} {{image.get_dicom_info|safe}}<br /> {% endcomment %}

generic/models.py

@@ -200,7 +200,18 @@ class SeriesImageBase(models.Model):
         null=True,
     )
 
+    # We store two hashes of the dicom data to uniquely identify the file
+    # and allow duplicate detection
+
+    # The first is a md5 hash of the pixel array data
+    # this is used because I can't get the direct pixel data to match from cornerstone
+    # (and struggle to get blake3 working in the browser)
     image_md5_hash = models.CharField(max_length=32, null=True, blank=True)
+
+    # The second is a blake3 hash of the direct pixel data
+    # This is much faster when using pydicom
+    image_blake3_hash = models.CharField(max_length=64, null=True, blank=True)
+
     is_dicom = models.BooleanField(default=False)
 
     removed = models.BooleanField(
@@ -231,14 +242,18 @@ class SeriesImageBase(models.Model):
         """Override save method to add image hash"""
         if self.image:
             if not self.image_md5_hash:
-                image_hash, is_dicom = get_image_hash(self.image)
+                image_hash, is_dicom = get_image_hash(self.image, hash_type="md5", direct_pixel_data=False)
 
                 self.is_dicom = is_dicom
                 self.image_md5_hash = image_hash
 
-                # Hack for tests
-                if image_hash != "12345ABCD":
-                    super().save(*args, **kwargs)  # Call the "real" save() method.
+            if not self.image_blake3_hash:
+                image_blake3_hash, is_dicom = get_image_hash(self.image, hash_type="blake3", direct_pixel_data=True)
+                self.image_blake3_hash = image_blake3_hash
+
+                ## Hack for tests
+                #if image_hash != "12345ABCD":
+                #    super().save(*args, **kwargs)  # Call the "real" save() method.
 
         super().save(*args, **kwargs)  # Call the "real" save() method.
 

helpers/images.py

@@ -5,6 +5,7 @@ import mimetypes
 from io import BytesIO
 
 from PIL import Image, ImageFile
+from blake3 import blake3
 
 import pydicom
 import sys
@@ -153,38 +154,53 @@ def print_dicom(dataset, indent=0, include_tag_ids=True):
                 l.append(f"{data_element.tag}: {indent_string:s} {data_element.name:s} = {repr_value:s}")
     return "\n<br/>".join(l)
 
-def get_image_dicom_hash(img, dataset=None) -> (str, bool):
+def get_image_dicom_hash(img, dataset=None, hash_type="md5", direct_pixel_data=True) -> (str, bool):
     if dataset is None:
         dataset = pydicom.dcmread(img)
 
     # TODO: improve?
-    md5 = hashlib.md5()
-    first = True
-    for i in dataset.pixel_array.astype(str).flatten():
-        if first:
-            first = False
-            md5.update(f"{i}".encode())
-        else:
-            md5.update(f",{i}".encode())
+    match hash_type:
+        case "md5":
+            hasher = hashlib.md5()
+        case "blake3":
+            hasher = blake3()
+        case _:
+            raise NotImplemented
 
-    hash = md5.hexdigest()
+    if direct_pixel_data:
+        hasher.update(dataset.PixelData)
+    else:
+        first = True
+        for i in dataset.pixel_array.astype(str).flatten():
+            if first:
+                first = False
+                hasher.update(f"{i}".encode())
+            else:
+                hasher.update(f",{i}".encode())
+
+    hash = hasher.hexdigest()
     return hash
 
-def get_image_hash(img) -> (str, bool):
+def get_image_hash(img, hash_type="blake3", direct_pixel_data=True) -> (str, bool):
     is_dicom = False
     # Try and read the file as a dicom
     hash = "1234"
     try:
         # and generate a hash from the pixel data
-        hash = get_image_dicom_hash(img)
+        hash = get_image_dicom_hash(img, hash_type=hash_type, direct_pixel_data=direct_pixel_data)
         is_dicom = True
         # ----
 
     except pydicom.errors.InvalidDicomError as e:
+        print("dicom error")
         try:  # This is horrible (but needed for current unit tests)
             # (we use a temporary file that breaks here)
             img.file.open()
-            hash = hashlib.md5(img.read()).hexdigest()
+            match hash_type:
+                case "md5":
+                    hash = hashlib.md5(img.read()).hexdigest()
+                case "blake3":
+                    hash = blake3(img.read()).hexdigest()
         except AttributeError:
             return hash, False
 

longs/migrations/0012_longseriesimage_image_blake3_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('longs', '0011_longseries_modified_longseriesimage_removed'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='longseriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+    ]

longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('longs', '0012_longseriesimage_image_blake3_hash'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='longseriesimage',
+            name='image_blake3_hash',
+            field=models.CharField(blank=True, max_length=64, null=True),
+        ),
+    ]

rapids/models.py

@@ -581,7 +581,7 @@ class RapidImage(models.Model):
     def save(self, *args, **kwargs):
         """Override save method to add image hash"""
         if self.image:
-            image_hash, is_dicom = get_image_hash(self.image)
+            image_hash, is_dicom = get_image_hash(self.image, hash_type="md5", direct_pixel_data=False)
             self.is_dicom = is_dicom
             self.image_md5_hash = image_hash
 

requirements.txt

@@ -41,3 +41,4 @@ ipython
 django-htmx-autocomplete
 faker
 pytest-faker
+blake3