From 714f472da6ef699075a57d034d86a39acff834c6 Mon Sep 17 00:00:00 2001 From: Ross Date: Tue, 26 Dec 2023 17:11:02 +0000 Subject: [PATCH] improve image hashing --- anatomy/static/js/upload_form_helpers.js | 6 +++ atlas/api.py | 22 +++++----- atlas/forms.py | 2 +- ...ategoriseddicom_image_md5_hash_and_more.py | 32 ++++++++++++++ ...er_uncategoriseddicom_image_blake3_hash.py | 18 ++++++++ ...037_alter_seriesimage_image_blake3_hash.py | 18 ++++++++ atlas/models.py | 19 ++++----- atlas/templates/atlas/series_viewer.html | 2 +- generic/models.py | 23 ++++++++-- helpers/images.py | 42 +++++++++++++------ .../0012_longseriesimage_image_blake3_hash.py | 18 ++++++++ ...alter_longseriesimage_image_blake3_hash.py | 18 ++++++++ rapids/models.py | 2 +- requirements.txt | 3 +- 14 files changed, 183 insertions(+), 42 deletions(-) create mode 100644 atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py create mode 100644 atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py create mode 100644 atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py create mode 100644 longs/migrations/0012_longseriesimage_image_blake3_hash.py create mode 100644 longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py diff --git a/anatomy/static/js/upload_form_helpers.js b/anatomy/static/js/upload_form_helpers.js index 8fff2006..fe00eb11 100644 --- a/anatomy/static/js/upload_form_helpers.js +++ b/anatomy/static/js/upload_form_helpers.js @@ -222,6 +222,12 @@ console.log("image", image) + let bytesView = image.getPixelData() + let str = new TextDecoder("utf-8").decode(bytesView, ); + + console.log("bytes", bytesView) + console.log("str", str) + extractDicomDetails(image.data.byteArray) }); diff --git a/atlas/api.py b/atlas/api.py index c979b1ed..c3dd5157 100644 --- a/atlas/api.py +++ b/atlas/api.py @@ -58,10 +58,10 @@ def upload_dicom(request, files: List[UploadedFile] = File(...)): ud.save() - uploaded.append((file.name, ud.image_md5_hash)) + uploaded.append((file.name, ud.image_blake3_hash)) except DuplicateDicom: - print(ud.check_for_duplicates(ud.image_md5_hash)) - duplicate.append((file.name, ud.image_md5_hash)) + print(ud.check_for_duplicates(ud.image_blake3_hash)) + duplicate.append((file.name, ud.image_blake3_hash)) pass return {"uploaded": uploaded, "duplicates": duplicate} @@ -91,7 +91,7 @@ def uncategorised_dicoms(request): tags = d.get_basic_dicom_tags() data[tags["SeriesInstanceUID"]].append(tags) - data["image_hash"].append(d.image_md5_hash) + data["image_hash"].append(d.image_blake3_hash) return data @@ -137,7 +137,7 @@ def import_dicoms_helper(request, case_id: int | None = None): image=dicom.image, series=series, is_dicom=True, - image_md5_hash=dicom.image_md5_hash, + image_blake3_hash=dicom.image_blake3_hash, ) series_image.save() # dicom.image.delete() @@ -171,9 +171,9 @@ def series_remove_duplicate_images(request, series_id: int): img_ids = set() dupes = set() for series_image in series.images.all(): - if series_image.image_md5_hash in img_ids: + if series_image.image_blake3_hash in img_ids: dupes.add(series_image.id) - img_ids.add(series_image.image_md5_hash) + img_ids.add(series_image.image_blake3_hash) if dupes: SeriesImage.objects.filter(id__in=dupes).delete() @@ -210,7 +210,7 @@ def get_cases_user(request): def check_image_hash(request, hash: str): try: print(hash) - series_image = SeriesImage.objects.get(image_md5_hash=hash) + series_image = SeriesImage.objects.get(image_blake3_hash=hash) data = { "status": "success", "id": series_image.pk, @@ -234,7 +234,7 @@ def check_images_hashes(request, hashes: List[str]): for hash in hashes: try: print(hash) - series_image = SeriesImage.objects.get(image_md5_hash=hash) + series_image = SeriesImage.objects.get(image_blake3_hash=hash) data = { "id": series_image.pk, "url": series_image.series.get_absolute_url(), @@ -250,7 +250,7 @@ def check_images_hashes(request, hashes: List[str]): def generate_image_hash(request, id: int): series_image = SeriesImage.objects.get(pk=id) - series_image.image_md5_hash = "" + series_image.image_blake3_hash = "" series_image.save() @@ -259,7 +259,7 @@ def generate_image_hash(request, id: int): @router.get("/view_dicom_tags/{hash}", auth=django_auth) def view_dicom_tags(request, hash: str): - item = SeriesImage.objects.get(image_md5_hash=hash) + item = SeriesImage.objects.get(image_blake3_hash=hash) return item.get_dicom_json() @router.get("/series_split_by_dicom_tag/{series_id}/{dicom_tag}", auth=django_auth) diff --git a/atlas/forms.py b/atlas/forms.py index 241547d3..d59ec7a9 100755 --- a/atlas/forms.py +++ b/atlas/forms.py @@ -444,7 +444,7 @@ CaseCollectionCaseFormSet = inlineformset_factory( SeriesImageFormSet = inlineformset_factory( Series, SeriesImage, - exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "is_dicom"], + exclude=["dicom_tags_ohif", "replaced", "image_md5_hash", "image_blake3_hash", "is_dicom"], can_delete=True, extra=0, max_num=2000, diff --git a/atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py b/atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py new file mode 100644 index 00000000..efd2937f --- /dev/null +++ b/atlas/migrations/0035_remove_uncategoriseddicom_image_md5_hash_and_more.py @@ -0,0 +1,32 @@ +# Generated by Django 4.1.4 on 2023-12-26 16:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('atlas', '0034_uncategoriseddicom_basic_dicom_tags_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='uncategoriseddicom', + name='image_md5_hash', + ), + migrations.AddField( + model_name='seriesimage', + name='image_blake3_hash', + field=models.CharField(blank=True, max_length=32, null=True), + ), + migrations.AddField( + model_name='uncategoriseddicom', + name='image_blake3_hash', + field=models.CharField(blank=True, max_length=32, null=True), + ), + migrations.AlterField( + model_name='uncategoriseddicom', + name='basic_dicom_tags', + field=models.JSONField(blank=True, null=True), + ), + ] diff --git a/atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py b/atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py new file mode 100644 index 00000000..d99fd179 --- /dev/null +++ b/atlas/migrations/0036_alter_uncategoriseddicom_image_blake3_hash.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.4 on 2023-12-26 16:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('atlas', '0035_remove_uncategoriseddicom_image_md5_hash_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='uncategoriseddicom', + name='image_blake3_hash', + field=models.CharField(blank=True, max_length=64, null=True), + ), + ] diff --git a/atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py b/atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py new file mode 100644 index 00000000..5e88fc69 --- /dev/null +++ b/atlas/migrations/0037_alter_seriesimage_image_blake3_hash.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.4 on 2023-12-26 16:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('atlas', '0036_alter_uncategoriseddicom_image_blake3_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='seriesimage', + name='image_blake3_hash', + field=models.CharField(blank=True, max_length=64, null=True), + ), + ] diff --git a/atlas/models.py b/atlas/models.py index 566436e3..f0dccb76 100644 --- a/atlas/models.py +++ b/atlas/models.py @@ -892,7 +892,7 @@ class UncategorisedDicom(models.Model): series_instance_uid = models.CharField(max_length=255, blank=True, null=True) - image_md5_hash = models.CharField(max_length=32, null=True, blank=True) + image_blake3_hash = models.CharField(max_length=64, null=True, blank=True) created_date = models.DateTimeField(default=timezone.now) @@ -901,12 +901,13 @@ class UncategorisedDicom(models.Model): def check_for_duplicates(self, image_hash): duplicate = None - if obj := UncategorisedDicom.objects.filter(image_md5_hash=image_hash).first(): - duplicate = obj + if obj := UncategorisedDicom.objects.filter(image_blake3_hash=image_hash).first(): + #duplicate = obj + return obj print(image_hash) - if obj := SeriesImage.objects.filter(image_md5_hash=image_hash).first(): + if obj := SeriesImage.objects.filter(image_blake3_hash=image_hash).first(): duplicate = obj return duplicate @@ -920,11 +921,11 @@ class UncategorisedDicom(models.Model): # if self.check_for_duplicates(): # return DuplicateDicom - image_hash = get_image_dicom_hash(None, dataset=ds) + image_blake3_hash = get_image_dicom_hash(None, dataset=ds, hash_type="blake3", direct_pixel_data=True) - self.image_md5_hash = image_hash + self.image_blake3_hash = image_blake3_hash - duplicate = self.check_for_duplicates(image_hash) + duplicate = self.check_for_duplicates(image_blake3_hash) self.basic_dicom_tags = self.get_basic_dicom_tags(dataset=ds) @@ -932,10 +933,8 @@ class UncategorisedDicom(models.Model): if duplicate != self: raise DuplicateDicom - - # Hack for tests - if image_hash != "1234": + if image_blake3_hash != "1234": super().save(*args, **kwargs) # Call the "real" save() method. def get_dicom_info(self): diff --git a/atlas/templates/atlas/series_viewer.html b/atlas/templates/atlas/series_viewer.html index 1b37fc8e..3371d271 100755 --- a/atlas/templates/atlas/series_viewer.html +++ b/atlas/templates/atlas/series_viewer.html @@ -100,7 +100,7 @@ [{{image.image.size|filesizeformat}}] {% endif %} - {{image.image_md5_hash}} ({{image.is_dicom}}) + {{image.image_md5_hash}} ({{image.is_dicom}}) {{image.image_blake3_hash}}
{% comment %} {{image.get_dicom_info|safe}}
{% endcomment %} diff --git a/generic/models.py b/generic/models.py index b6b90169..05aac5a7 100644 --- a/generic/models.py +++ b/generic/models.py @@ -200,7 +200,18 @@ class SeriesImageBase(models.Model): null=True, ) + # We store two hashes of the dicom data to uniquely identify the file + # and allow duplicate detection + + # The first is a md5 hash of the pixel array data + # this is used because I can't get the direct pixel data to match from cornerstone + # (and struggle to get blake3 working in the browser) image_md5_hash = models.CharField(max_length=32, null=True, blank=True) + + # The second is a blake3 hash of the direct pixel data + # This is much faster when using pydicom + image_blake3_hash = models.CharField(max_length=64, null=True, blank=True) + is_dicom = models.BooleanField(default=False) removed = models.BooleanField( @@ -231,14 +242,18 @@ class SeriesImageBase(models.Model): """Override save method to add image hash""" if self.image: if not self.image_md5_hash: - image_hash, is_dicom = get_image_hash(self.image) + image_hash, is_dicom = get_image_hash(self.image, hash_type="md5", direct_pixel_data=False) self.is_dicom = is_dicom self.image_md5_hash = image_hash - # Hack for tests - if image_hash != "12345ABCD": - super().save(*args, **kwargs) # Call the "real" save() method. + if not self.image_blake3_hash: + image_blake3_hash, is_dicom = get_image_hash(self.image, hash_type="blake3", direct_pixel_data=True) + self.image_blake3_hash = image_blake3_hash + + ## Hack for tests + #if image_hash != "12345ABCD": + # super().save(*args, **kwargs) # Call the "real" save() method. super().save(*args, **kwargs) # Call the "real" save() method. diff --git a/helpers/images.py b/helpers/images.py index ec3083d9..df21e232 100644 --- a/helpers/images.py +++ b/helpers/images.py @@ -5,6 +5,7 @@ import mimetypes from io import BytesIO from PIL import Image, ImageFile +from blake3 import blake3 import pydicom import sys @@ -153,38 +154,53 @@ def print_dicom(dataset, indent=0, include_tag_ids=True): l.append(f"{data_element.tag}: {indent_string:s} {data_element.name:s} = {repr_value:s}") return "\n
".join(l) -def get_image_dicom_hash(img, dataset=None) -> (str, bool): +def get_image_dicom_hash(img, dataset=None, hash_type="md5", direct_pixel_data=True) -> (str, bool): if dataset is None: dataset = pydicom.dcmread(img) # TODO: improve? - md5 = hashlib.md5() - first = True - for i in dataset.pixel_array.astype(str).flatten(): - if first: - first = False - md5.update(f"{i}".encode()) - else: - md5.update(f",{i}".encode()) + match hash_type: + case "md5": + hasher = hashlib.md5() + case "blake3": + hasher = blake3() + case _: + raise NotImplemented - hash = md5.hexdigest() + if direct_pixel_data: + hasher.update(dataset.PixelData) + else: + first = True + for i in dataset.pixel_array.astype(str).flatten(): + if first: + first = False + hasher.update(f"{i}".encode()) + else: + hasher.update(f",{i}".encode()) + + hash = hasher.hexdigest() return hash -def get_image_hash(img) -> (str, bool): +def get_image_hash(img, hash_type="blake3", direct_pixel_data=True) -> (str, bool): is_dicom = False # Try and read the file as a dicom hash = "1234" try: # and generate a hash from the pixel data - hash = get_image_dicom_hash(img) + hash = get_image_dicom_hash(img, hash_type=hash_type, direct_pixel_data=direct_pixel_data) is_dicom = True # ---- except pydicom.errors.InvalidDicomError as e: + print("dicom error") try: # This is horrible (but needed for current unit tests) # (we use a temporary file that breaks here) img.file.open() - hash = hashlib.md5(img.read()).hexdigest() + match hash_type: + case "md5": + hash = hashlib.md5(img.read()).hexdigest() + case "blake3": + hash = blake3(img.read()).hexdigest() except AttributeError: return hash, False diff --git a/longs/migrations/0012_longseriesimage_image_blake3_hash.py b/longs/migrations/0012_longseriesimage_image_blake3_hash.py new file mode 100644 index 00000000..077e2f4d --- /dev/null +++ b/longs/migrations/0012_longseriesimage_image_blake3_hash.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.4 on 2023-12-26 16:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('longs', '0011_longseries_modified_longseriesimage_removed'), + ] + + operations = [ + migrations.AddField( + model_name='longseriesimage', + name='image_blake3_hash', + field=models.CharField(blank=True, max_length=32, null=True), + ), + ] diff --git a/longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py b/longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py new file mode 100644 index 00000000..ea8c385c --- /dev/null +++ b/longs/migrations/0013_alter_longseriesimage_image_blake3_hash.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.4 on 2023-12-26 16:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('longs', '0012_longseriesimage_image_blake3_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='longseriesimage', + name='image_blake3_hash', + field=models.CharField(blank=True, max_length=64, null=True), + ), + ] diff --git a/rapids/models.py b/rapids/models.py index bfac3b6b..d4afa7bc 100644 --- a/rapids/models.py +++ b/rapids/models.py @@ -581,7 +581,7 @@ class RapidImage(models.Model): def save(self, *args, **kwargs): """Override save method to add image hash""" if self.image: - image_hash, is_dicom = get_image_hash(self.image) + image_hash, is_dicom = get_image_hash(self.image, hash_type="md5", direct_pixel_data=False) self.is_dicom = is_dicom self.image_md5_hash = image_hash diff --git a/requirements.txt b/requirements.txt index d57ac46b..18ecaf87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,5 @@ django-rich ipython django-htmx-autocomplete faker -pytest-faker \ No newline at end of file +pytest-faker +blake3 \ No newline at end of file