diff --git a/anatomy/static/js/upload_form_helpers.js b/anatomy/static/js/upload_form_helpers.js
index 8fff2006..fe00eb11 100644
--- a/anatomy/static/js/upload_form_helpers.js
+++ b/anatomy/static/js/upload_form_helpers.js
@@ -222,6 +222,12 @@
console.log("image", image)
+ let bytesView = image.getPixelData()
+ let str = new TextDecoder("utf-8").decode(bytesView, );
+
+ console.log("bytes", bytesView)
+ console.log("str", str)
+
extractDicomDetails(image.data.byteArray)
});
diff --git a/atlas/api.py b/atlas/api.py
index c979b1ed..c3dd5157 100644
--- a/atlas/api.py
+++ b/atlas/api.py
@@ -58,10 +58,10 @@ def upload_dicom(request, files: List[UploadedFile] = File(...)):
ud.save()
- uploaded.append((file.name, ud.image_md5_hash))
+ uploaded.append((file.name, ud.image_blake3_hash))
except DuplicateDicom:
- print(ud.check_for_duplicates(ud.image_md5_hash))
- duplicate.append((file.name, ud.image_md5_hash))
+ print(ud.check_for_duplicates(ud.image_blake3_hash))
+ duplicate.append((file.name, ud.image_blake3_hash))
pass
return {"uploaded": uploaded, "duplicates": duplicate}
@@ -91,7 +91,7 @@ def uncategorised_dicoms(request):
tags = d.get_basic_dicom_tags()
data[tags["SeriesInstanceUID"]].append(tags)
- data["image_hash"].append(d.image_md5_hash)
+ data["image_hash"].append(d.image_blake3_hash)
return data
@@ -137,7 +137,7 @@ def import_dicoms_helper(request, case_id: int | None = None):
image=dicom.image,
series=series,
is_dicom=True,
- image_md5_hash=dicom.image_md5_hash,
+ image_blake3_hash=dicom.image_blake3_hash,
)
series_image.save()
# dicom.image.delete()
@@ -171,9 +171,9 @@ def series_remove_duplicate_images(request, series_id: int):
img_ids = set()
dupes = set()
for series_image in series.images.all():
- if series_image.image_md5_hash in img_ids:
+ if series_image.image_blake3_hash in img_ids:
dupes.add(series_image.id)
- img_ids.add(series_image.image_md5_hash)
+ img_ids.add(series_image.image_blake3_hash)
if dupes:
SeriesImage.objects.filter(id__in=dupes).delete()
@@ -210,7 +210,7 @@ def get_cases_user(request):
def check_image_hash(request, hash: str):
try:
print(hash)
- series_image = SeriesImage.objects.get(image_md5_hash=hash)
+ series_image = SeriesImage.objects.get(image_blake3_hash=hash)
data = {
"status": "success",
"id": series_image.pk,
@@ -234,7 +234,7 @@ def check_images_hashes(request, hashes: List[str]):
for hash in hashes:
try:
print(hash)
- series_image = SeriesImage.objects.get(image_md5_hash=hash)
+ series_image = SeriesImage.objects.get(image_blake3_hash=hash)
data = {
"id": series_image.pk,
"url": series_image.series.get_absolute_url(),
@@ -250,7 +250,7 @@ def check_images_hashes(request, hashes: List[str]):
def generate_image_hash(request, id: int):
series_image = SeriesImage.objects.get(pk=id)
- series_image.image_md5_hash = ""
+ series_image.image_blake3_hash = ""
series_image.save()
@@ -259,7 +259,7 @@ def generate_image_hash(request, id: int):
@router.get("/view_dicom_tags/{hash}", auth=django_auth)
def view_dicom_tags(request, hash: str):
- item = SeriesImage.objects.get(image_md5_hash=hash)
+ item = SeriesImage.objects.get(image_blake3_hash=hash)
return item.get_dicom_json()
@router.get("/series_split_by_dicom_tag/{series_id}/{dicom_tag}", auth=django_auth)
diff --git a/atlas/forms.py b/atlas/forms.py
index 241547d3..d59ec7a9 100755
--- a/atlas/forms.py
+++ b/atlas/forms.py
@@ -444,7 +444,7 @@ CaseCollectionCaseFormSet = inlineformset_factory(
SeriesImageFormSet = inlineformset_factory(
Series,
SeriesImage,
- exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "is_dicom"],
+ exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "image_blake3_hash", "is_dicom"],
can_delete=True,
extra=0,
max_num=2000,
diff --git a/atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py b/atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py
new file mode 100644
index 00000000..efd2937f
--- /dev/null
+++ b/atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py
@@ -0,0 +1,32 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('atlas', '0034_uncategoriseddicom_basic_dicom_tags_and_more'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='uncategoriseddicom',
+ name='image_md5_hash',
+ ),
+ migrations.AddField(
+ model_name='seriesimage',
+ name='image_blake3_hash',
+ field=models.CharField(blank=True, max_length=32, null=True),
+ ),
+ migrations.AddField(
+ model_name='uncategoriseddicom',
+ name='image_blake3_hash',
+ field=models.CharField(blank=True, max_length=32, null=True),
+ ),
+ migrations.AlterField(
+ model_name='uncategoriseddicom',
+ name='basic_dicom_tags',
+ field=models.JSONField(blank=True, null=True),
+ ),
+ ]
diff --git a/atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py b/atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py
new file mode 100644
index 00000000..d99fd179
--- /dev/null
+++ b/atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('atlas', '0035_remove_uncategoriseddicom_image_md5_hash_and_more'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='uncategoriseddicom',
+ name='image_blake3_hash',
+ field=models.CharField(blank=True, max_length=64, null=True),
+ ),
+ ]
diff --git a/atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py b/atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py
new file mode 100644
index 00000000..5e88fc69
--- /dev/null
+++ b/atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('atlas', '0036_alter_uncategoriseddicom_image_blake3_hash'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='seriesimage',
+ name='image_blake3_hash',
+ field=models.CharField(blank=True, max_length=64, null=True),
+ ),
+ ]
diff --git a/atlas/models.py b/atlas/models.py
index 566436e3..f0dccb76 100644
--- a/atlas/models.py
+++ b/atlas/models.py
@@ -892,7 +892,7 @@ class UncategorisedDicom(models.Model):
series_instance_uid = models.CharField(max_length=255, blank=True, null=True)
- image_md5_hash = models.CharField(max_length=32, null=True, blank=True)
+ image_blake3_hash = models.CharField(max_length=64, null=True, blank=True)
created_date = models.DateTimeField(default=timezone.now)
@@ -901,12 +901,13 @@ class UncategorisedDicom(models.Model):
def check_for_duplicates(self, image_hash):
duplicate = None
- if obj := UncategorisedDicom.objects.filter(image_md5_hash=image_hash).first():
- duplicate = obj
+ if obj := UncategorisedDicom.objects.filter(image_blake3_hash=image_hash).first():
+ #duplicate = obj
+ return obj
print(image_hash)
- if obj := SeriesImage.objects.filter(image_md5_hash=image_hash).first():
+ if obj := SeriesImage.objects.filter(image_blake3_hash=image_hash).first():
duplicate = obj
return duplicate
@@ -920,11 +921,11 @@ class UncategorisedDicom(models.Model):
# if self.check_for_duplicates():
# return DuplicateDicom
- image_hash = get_image_dicom_hash(None, dataset=ds)
+ image_blake3_hash = get_image_dicom_hash(None, dataset=ds, hash_type="blake3", direct_pixel_data=True)
- self.image_md5_hash = image_hash
+ self.image_blake3_hash = image_blake3_hash
- duplicate = self.check_for_duplicates(image_hash)
+ duplicate = self.check_for_duplicates(image_blake3_hash)
self.basic_dicom_tags = self.get_basic_dicom_tags(dataset=ds)
@@ -932,10 +933,8 @@ class UncategorisedDicom(models.Model):
if duplicate != self:
raise DuplicateDicom
-
-
# Hack for tests
- if image_hash != "1234":
+ if image_blake3_hash != "1234":
super().save(*args, **kwargs) # Call the "real" save() method.
def get_dicom_info(self):
diff --git a/atlas/templates/atlas/series_viewer.html b/atlas/templates/atlas/series_viewer.html
index 1b37fc8e..3371d271 100755
--- a/atlas/templates/atlas/series_viewer.html
+++ b/atlas/templates/atlas/series_viewer.html
@@ -100,7 +100,7 @@
[{{image.image.size|filesizeformat}}]
{% endif %}
- {{image.image_md5_hash}} ({{image.is_dicom}})
+ {{image.image_md5_hash}} ({{image.is_dicom}}) {{image.image_blake3_hash}}
{% comment %} {{image.get_dicom_info|safe}}
{% endcomment %}
diff --git a/generic/models.py b/generic/models.py
index b6b90169..05aac5a7 100644
--- a/generic/models.py
+++ b/generic/models.py
@@ -200,7 +200,18 @@ class SeriesImageBase(models.Model):
null=True,
)
+ # We store two hashes of the dicom data to uniquely identify the file
+ # and allow duplicate detection
+
+ # The first is a md5 hash of the pixel array data
+ # this is used because I can't get the direct pixel data to match from cornerstone
+ # (and struggle to get blake3 working in the browser)
image_md5_hash = models.CharField(max_length=32, null=True, blank=True)
+
+ # The second is a blake3 hash of the direct pixel data
+ # This is much faster when using pydicom
+ image_blake3_hash = models.CharField(max_length=64, null=True, blank=True)
+
is_dicom = models.BooleanField(default=False)
removed = models.BooleanField(
@@ -231,14 +242,18 @@ class SeriesImageBase(models.Model):
"""Override save method to add image hash"""
if self.image:
if not self.image_md5_hash:
- image_hash, is_dicom = get_image_hash(self.image)
+ image_hash, is_dicom = get_image_hash(self.image, hash_type="md5", direct_pixel_data=False)
self.is_dicom = is_dicom
self.image_md5_hash = image_hash
- # Hack for tests
- if image_hash != "12345ABCD":
- super().save(*args, **kwargs) # Call the "real" save() method.
+ if not self.image_blake3_hash:
+ image_blake3_hash, is_dicom = get_image_hash(self.image, hash_type="blake3", direct_pixel_data=True)
+ self.image_blake3_hash = image_blake3_hash
+
+ ## Hack for tests
+ #if image_hash != "12345ABCD":
+ # super().save(*args, **kwargs) # Call the "real" save() method.
super().save(*args, **kwargs) # Call the "real" save() method.
diff --git a/helpers/images.py b/helpers/images.py
index ec3083d9..df21e232 100644
--- a/helpers/images.py
+++ b/helpers/images.py
@@ -5,6 +5,7 @@ import mimetypes
from io import BytesIO
from PIL import Image, ImageFile
+from blake3 import blake3
import pydicom
import sys
@@ -153,38 +154,53 @@ def print_dicom(dataset, indent=0, include_tag_ids=True):
l.append(f"{data_element.tag}: {indent_string:s} {data_element.name:s} = {repr_value:s}")
return "\n
".join(l)
-def get_image_dicom_hash(img, dataset=None) -> (str, bool):
+def get_image_dicom_hash(img, dataset=None, hash_type="md5", direct_pixel_data=True) -> (str, bool):
if dataset is None:
dataset = pydicom.dcmread(img)
# TODO: improve?
- md5 = hashlib.md5()
- first = True
- for i in dataset.pixel_array.astype(str).flatten():
- if first:
- first = False
- md5.update(f"{i}".encode())
- else:
- md5.update(f",{i}".encode())
+ match hash_type:
+ case "md5":
+ hasher = hashlib.md5()
+ case "blake3":
+ hasher = blake3()
+ case _:
+ raise NotImplemented
- hash = md5.hexdigest()
+ if direct_pixel_data:
+ hasher.update(dataset.PixelData)
+ else:
+ first = True
+ for i in dataset.pixel_array.astype(str).flatten():
+ if first:
+ first = False
+ hasher.update(f"{i}".encode())
+ else:
+ hasher.update(f",{i}".encode())
+
+ hash = hasher.hexdigest()
return hash
-def get_image_hash(img) -> (str, bool):
+def get_image_hash(img, hash_type="blake3", direct_pixel_data=True) -> (str, bool):
is_dicom = False
# Try and read the file as a dicom
hash = "1234"
try:
# and generate a hash from the pixel data
- hash = get_image_dicom_hash(img)
+ hash = get_image_dicom_hash(img, hash_type=hash_type, direct_pixel_data=direct_pixel_data)
is_dicom = True
# ----
except pydicom.errors.InvalidDicomError as e:
+ print("dicom error")
try: # This is horrible (but needed for current unit tests)
# (we use a temporary file that breaks here)
img.file.open()
- hash = hashlib.md5(img.read()).hexdigest()
+ match hash_type:
+ case "md5":
+ hash = hashlib.md5(img.read()).hexdigest()
+ case "blake3":
+ hash = blake3(img.read()).hexdigest()
except AttributeError:
return hash, False
diff --git a/longs/migrations/0012_longseriesimage_image_blake3_hash.py b/longs/migrations/0012_longseriesimage_image_blake3_hash.py
new file mode 100644
index 00000000..077e2f4d
--- /dev/null
+++ b/longs/migrations/0012_longseriesimage_image_blake3_hash.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('longs', '0011_longseries_modified_longseriesimage_removed'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='longseriesimage',
+ name='image_blake3_hash',
+ field=models.CharField(blank=True, max_length=32, null=True),
+ ),
+ ]
diff --git a/longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py b/longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py
new file mode 100644
index 00000000..ea8c385c
--- /dev/null
+++ b/longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.1.4 on 2023-12-26 16:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('longs', '0012_longseriesimage_image_blake3_hash'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='longseriesimage',
+ name='image_blake3_hash',
+ field=models.CharField(blank=True, max_length=64, null=True),
+ ),
+ ]
diff --git a/rapids/models.py b/rapids/models.py
index bfac3b6b..d4afa7bc 100644
--- a/rapids/models.py
+++ b/rapids/models.py
@@ -581,7 +581,7 @@ class RapidImage(models.Model):
def save(self, *args, **kwargs):
"""Override save method to add image hash"""
if self.image:
- image_hash, is_dicom = get_image_hash(self.image)
+ image_hash, is_dicom = get_image_hash(self.image, hash_type="md5", direct_pixel_data=False)
self.is_dicom = is_dicom
self.image_md5_hash = image_hash
diff --git a/requirements.txt b/requirements.txt
index d57ac46b..18ecaf87 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,4 +40,5 @@ django-rich
ipython
django-htmx-autocomplete
faker
-pytest-faker
\ No newline at end of file
+pytest-faker
+blake3
\ No newline at end of file