from rad.settings import REMOTE_URL
from django.db import models
from django.http.response import JsonResponse
from django.shortcuts import get_object_or_404
from django.utils import timezone
import tagulous
import tagulous.models
from django.core.files.storage import FileSystemStorage
from django.conf import settings
from django.utils.html import format_html
from django.urls import reverse
from django.utils.translation import ugettext_lazy as _
from django.utils.html import mark_safe
from sortedm2m.fields import SortedManyToManyField
import string
from collections import defaultdict
from helpers.images import image_as_base64
from django.contrib.contenttypes.fields import GenericRelation
from generic.models import CidUser, Site, Condition, Sign, ExamBase, QuestionNote
import reversion
import json
import hashlib
import pydicom
import pydicom.errors
image_storage = FileSystemStorage(
# Physical file location ROOT
location="{0}".format(settings.MEDIA_ROOT),
# Url for file
base_url="{0}".format(settings.MEDIA_URL),
)
def image_directory_path(instance, filename):
# return u"{0}".format(filename)
return "rapids/picture/{0}".format(filename)
def get_answer_compare(s):
s = s.strip().lower()
s = s.translate(str.maketrans("", "", string.punctuation.replace("#", "")))
return s
class Abnormality(models.Model):
name = models.CharField(
max_length=200, unique=True, help_text="Primary abnormality on the film(s)"
)
def __str__(self):
return self.name
class Meta:
ordering = ("name",)
class Region(models.Model):
name = models.CharField(
max_length=200,
unique=True,
help_text="Region of the abnormality (not including lateralitiy)",
blank=True,
)
def __str__(self):
return self.name
class Meta:
ordering = ("name",)
class Examination(models.Model):
examination = models.CharField(max_length=200)
def __str__(self):
return self.examination
class Meta:
ordering = ("examination",)
class Answer(models.Model):
question = models.ForeignKey(
"Rapid", related_name="answers", on_delete=models.CASCADE
)
answer = models.TextField(max_length=500)
answer_compare = models.TextField(max_length=500)
class MarkOptions(models.TextChoices):
UNMARKED = "", _("Unmarked")
INCORRECT = "0", _("Incorrect")
HALF_MARK = "1", _("Half mark")
CORRECT = "2", _("Correct")
status = models.CharField(
max_length=1, choices=MarkOptions.choices, default=MarkOptions.UNMARKED
)
proposed = models.BooleanField(default=False)
def __str__(self):
return self.answer
def save(self, *args, **kwargs):
self.clean()
return super(Answer, self).save(*args, **kwargs)
def clean(self):
if self.answer:
self.answer_compare = get_answer_compare(self.answer)
class Rapid(models.Model):
# author = models.ForeignKey('auth.User', on_delete=models.CASCADE)
# image = models.ImageField()
question = models.TextField(null=True, blank=True)
feedback = models.TextField(null=True, blank=True)
normal = models.BooleanField(default=False, help_text="Tick if true")
NONE = "NONE"
LEFT = "LEFT"
RIGHT = "RIGHT"
BILATERAL = "BILAT"
LATERALITY_CHOICES = (
(NONE, "None"),
(LEFT, "Left"),
(RIGHT, "Right"),
(BILATERAL, "Bilateral"),
)
abnormality = models.ManyToManyField(
Abnormality,
blank=True,
help_text="The abnormality (laterality and region independent). Used for categorisation but does not affect the answer",
)
region = models.ManyToManyField(
Region,
blank=True,
help_text="Region of the abnormality (laterality independent)",
)
examination = models.ManyToManyField(
Examination, help_text="Name of the (primary) examination"
)
laterality = models.CharField(
max_length=20,
choices=LATERALITY_CHOICES,
default=NONE,
help_text="Applies to the answer, not the examination",
)
# condition = tagulous.models.TagField(
# to=Condition,
# blank=True,
# help_text=
# "Associated condition. Will allow searching / filtering and tips / hints to be displayed. Conditions with spaces must be enclosed in quotes \"...\""
# )
# sign = tagulous.models.TagField(
# to=Sign, blank=True, help_text='Radiological signs in the question')
DEFAULT_SITE_ID = 1
# site = models.ManyToManyField(
# Site,
# related_name="site_rapid",
# blank=True,
# default=DEFAULT_SITE_ID,
# help_text=
# "If we know the source of the image")
verified = models.BooleanField(default=False)
created_date = models.DateTimeField(default=timezone.now)
published_date = models.DateTimeField(blank=True, null=True)
author = models.ManyToManyField(
settings.AUTH_USER_MODEL,
blank=True,
help_text="Author of question",
related_name="rapid_authored_questions",
)
scrapped = models.BooleanField(
default=False, help_text="Question has been scrapped and will not be shown"
)
open_access = models.BooleanField(
help_text="If a question should be freely available to browse", default=True
)
notes = GenericRelation(QuestionNote)
def get_absolute_url(self):
return reverse("rapids:question_detail", kwargs={"pk": self.pk})
def get_authors(self):
"""Returns a comma seperated text list of authors"""
authors = ", ".join([i.username for i in self.author.all()])
return authors
def get_regions(self):
"""Returns a comma seperated text list of regions"""
regions = ", ".join([i.name for i in self.region.all()])
return regions
def get_abnormalities(self):
"""Returns a comma seperated text list of regions"""
abnormalities = ", ".join([i.name for i in self.abnormality.all()])
return abnormalities
def get_examinations(self):
"""Returns a comma seperated text list of regions"""
examinations = ", ".join([i.examination for i in self.examination.all()])
return examinations
def __str__(self):
n = "Normal"
if not self.normal:
# n = self.answers.first()
n = "{}/{} : {}".format(
self.pk, self.abnormality.first(), self.region.first()
)
exams = self.get_examinations()
lat = ""
if self.laterality != "NONE":
lat = self.laterality
return "{}/{} : {} {}".format(self.pk, exams, lat, n)
def get_primary_answer(self):
if self.normal:
return "Normal"
elif (
self.answers.filter(
proposed=False, status=Answer.MarkOptions.CORRECT
).count()
> 0
):
return (
self.answers.filter(proposed=False, status=Answer.MarkOptions.CORRECT)
.first()
.answer
)
else:
return "None yet..."
def get_exams(self):
e = self.exams.all().values_list("name", flat=True)
exams = ", ".join(e)
return exams
def get_unmarked_user_answer_string(self, exam_pk=None):
unmarked_answers = self.get_unmarked_user_answers(exam_pk)
if not unmarked_answers:
return "No answers to mark"
return format_html(
"{} answer unmarked: {}".format(
len(unmarked_answers), ", ".join(unmarked_answers)
)
)
def get_unmarked_user_answers(self, exam_pk=None, marker=None):
# If normal no answers to mark (they will be automarked)
if self.normal:
return []
if exam_pk is None:
user_answer_queryset = self.cid_user_answers.all()
else:
user_answer_queryset = self.cid_user_answers.filter(exam__id=exam_pk)
marked_answers = self.get_marked_answers()
unmarked_answers = set(
[
i
for i in user_answer_queryset
if i.normal == False and i.answer_compare not in marked_answers
]
)
return [i.answer_compare for i in unmarked_answers]
# Marker code below is not used
if marker is None:
return [i.answer_compare for i in unmarked_answers]
# If marker is specified we check for what they have marked
marker_unmarked = []
for answer in unmarked_answers:
if answer.mark.filter(marker=marker).count() < 1:
marker_unmarked.append(answer.answer_compare)
return marker_unmarked
def get_unmarked_user_answer_count(self, exam_pk=None, marker=None):
if exam_pk is None:
return self.cid_user_answers.all().count()
return len(self.get_unmarked_user_answers(exam_pk, marker=marker))
def get_user_answers(self, exam_pk=None, include_normal=True):
if exam_pk is None:
queryset = self.cid_user_answers.all()
else:
queryset = self.cid_user_answers.filter(exam__id=exam_pk)
if not include_normal:
queryset = queryset.filter(normal=False)
user_answers = set([i.answer_compare for i in queryset])
return user_answers
def get_compare_answers(self):
return set([i.answer_compare for i in self.answers.filter()])
def get_marked_answers(self):
return set(
[
i.answer_compare
for i in self.answers.filter(proposed=False)
if i.status != i.MarkOptions.UNMARKED
]
)
correct_answers = set(
[i.answer.get_compare_string() for i in self.answers.all()]
)
half_mark_answers = set(
[i.answer.get_compare_string() for i in self.half_mark_answers.all()]
)
incorrect_answers = set(
[i.answer.get_compare_string() for i in self.incorrect_answers.all()]
)
marked_answers = correct_answers | half_mark_answers | incorrect_answers
return marked_answers
def get_correct_unstripped_answers(self):
return set(
[
str(i)
for i in self.answers.filter(proposed=False)
if i.status == i.MarkOptions.CORRECT
]
)
def get_images(self, feedback=False):
qs = self.images.all()
if not feedback:
images = [i.image for i in qs if not i.feedback_image]
else:
images = [i.image for i in qs]
return images
def get_image_urls(self):
return ",".join([f"{REMOTE_URL}{i.url}" for i in self.get_images()])
def get_image_url_array(self):
return json.dumps([f"{REMOTE_URL}{i.url}" for i in self.get_images()])
# def GetNonFeedbackQuestionImages(self):
# return self.get_images()
def get_image_annotations(self):
return json.dumps(
[i.image_annotations for i in self.images.all() if not i.feedback_image]
)
def get_laterality_string(self):
if self.laterality == self.NONE:
s = ""
elif self.laterality == self.BILATERAL:
s = "bilateral "
elif self.laterality == self.RIGHT:
s = "right "
elif self.laterality == self.LEFT:
s = "left "
return s
def get_suggested_answers(self):
answers = []
for r in self.region.all():
laterality = self.get_laterality_string()
for a in self.abnormality.all():
answers.append("{}{} {}".format(laterality, r.name, a.name))
answers.append("{} {}{}".format(a.name, laterality, r.name))
answers.append("{} {}".format(r.name, a.name))
answers.append("{} {}".format(a.name, r.name))
compare_answers = self.get_compare_answers()
new_answers = [
a for a in answers if get_answer_compare(a) not in compare_answers
]
return new_answers
def get_question_json(self, based=True):
"""Returns json"""
# Loop through rapidimage associations
images = []
annotations = []
feedback_images = []
for i in self.images.all():
annotations.append(i.image_annotations)
if i.feedback_image == True:
if based:
feedback_images.append(image_as_base64(i.image))
else:
feedback_images.append(
"{}/{}".format(settings.REMOTE_URL, i.image.url)
)
feedback_images.append("{}/{}".format(settings.REMOTE_URL, i.image.url))
else:
if based:
images.append(image_as_base64(i.image))
else:
images.append("{}/{}".format(settings.REMOTE_URL, i.image.url))
json = {
"images": images,
# "feedback_image": [],
"annotations": annotations,
"type": "rapid",
}
json["normal"] = self.normal
json["feedback_images"] = feedback_images
json["answers"] = list(self.get_correct_unstripped_answers())
return json
# reversion.register(Rapid, follow=["images"])
# @reversion.register
class RapidImage(models.Model):
rapid = models.ForeignKey(Rapid, related_name="images", on_delete=models.CASCADE)
image = models.FileField(upload_to=image_directory_path)
image_annotations = models.TextField(
blank=True,
null=True,
help_text="Stores a JSON representation of annotations to be applied by cornerstonetools",
)
feedback_image = models.BooleanField(default=False)
image_md5_hash = models.CharField(max_length=32, null=True, blank=True)
is_dicom = models.BooleanField(default=False)
def image_tag(self):
if self.image:
return mark_safe(
'
Click and hold to zoom'.format(
self.image.url
)
)
else:
return ""
image_tag.short_description = "Image"
def save(self, *args, **kwargs):
"""Override save method to add image hash"""
# TODO: consider moving to signal to reuse across apps
if self.image:
# Try and read the file as a dicom
try:
# and generate a hash from the pixel data
# TODO: improve?
dataset = pydicom.dcmread(self.image)
# flatten = dataset.pixel_array.astype(str).flatten()
# print("flatteded")
# pre_join = ",".join(flatten)
# print(pre_join)
# hash = hashlib.md5(pre_join.encode()).hexdigest()
# ----
md5 = hashlib.md5()
first = True
for i in dataset.pixel_array.astype(str).flatten():
if first:
first = False
md5.update(f"{i}".encode())
else:
md5.update(f",{i}".encode())
hash = md5.hexdigest()
# ----
self.is_dicom = True
except pydicom.errors.InvalidDicomError:
self.image.file.open()
hash = hashlib.md5(self.image.read()).hexdigest()
self.image_md5_hash = hash
super().save(*args, **kwargs) # Call the "real" save() method.
class RapidCreationDefault(models.Model):
# author = models.OneToOneField(User,
author = models.OneToOneField(
settings.AUTH_USER_MODEL, related_name="rapid_default", on_delete=models.CASCADE
)
# site = models.ManyToManyField(
# Site,
# related_name="site_rapid_creation_default",
# blank=True,
# default=1,
# help_text="Default site to use when creating a new rapid")
def get_absolute_url(self):
return reverse("rapids:rapid_create")
# @reversion.register
class Exam(ExamBase):
app_name = "rapids"
exam_questions = SortedManyToManyField(Rapid, related_name="exams", blank="true")
time_limit = models.IntegerField(
help_text="Exam time limit (in seconds). Default is 2100 secondse (35 minutes)",
default=2100,
)
author = models.ManyToManyField(
settings.AUTH_USER_MODEL,
blank=True,
help_text="Author of exam",
related_name="rapid_exam_author",
)
valid_users = models.ManyToManyField(
CidUser, blank=True, related_name="rapid_exams"
)
def get_normal_abnormal_breakdown(self):
# Inefficient but more extendible
questions = self.exam_questions.all()
normal = []
abnormal = []
for q in questions:
if q.normal:
normal.append(q)
else:
abnormal.append(q)
return len(normal)
def get_exam_json(self, based=True):
questions = self.exam_questions.all()
exam_questions = defaultdict(dict)
exam_order = []
for q in questions:
exam_order.append(q.id)
# Loop through rapidimage associations
images = []
feedback_images = []
for i in q.images.all():
if i.feedback_image == True:
if based:
feedback_images.append(image_as_base64(i.image))
else:
feedback_images.append(
"{}/{}".format(settings.REMOTE_URL, i.image.url)
)
else:
if based:
images.append(image_as_base64(i.image))
else:
images.append("{}/{}".format(settings.REMOTE_URL, i.image.url))
exam_questions[q.id] = {
"images": images,
# "feedback_image": [],
# "annotations": [str(q.image_annotations)],
"type": "rapid",
}
if not self.exam_mode:
exam_questions[q.id]["normal"] = q.normal
exam_questions[q.id]["feedback_images"] = feedback_images
exam_questions[q.id]["answers"] = list(
q.get_correct_unstripped_answers()
)
# if feedback_images:
# exam_questions[q.id]["feedback_image"] = feedback_images
exam_json = {
"eid": "rapid/{}".format(self.id),
"cached": False,
"exam_type": "rapid",
"exam_name": self.name,
"exam_mode": self.exam_mode,
"exam_order": exam_order,
"questions": exam_questions,
}
if self.time_limit:
exam_json["exam_time"] = self.time_limit
return exam_json
def get_absolute_url(self):
return reverse("rapids:exam_overview", kwargs={"pk": self.pk})
@reversion.register
class CidUserAnswer(models.Model):
"""User answers by candidate"""
question = models.ForeignKey(
Rapid, related_name="cid_user_answers", on_delete=models.CASCADE
)
# For rapids the answer can be normal in which case the below field is true
normal = models.BooleanField(default=False)
answer = models.TextField(max_length=500, blank=True)
answer_compare = models.TextField(max_length=500, blank=True)
cid = models.BigIntegerField(
blank=True,
null=True,
help_text="Candidate ID (limitied by BigIntegerField size)",
)
# Each user answer is associated with a particular exam
exam = models.ForeignKey(
Exam, related_name="cid_user_answers", on_delete=models.CASCADE, null=True
)
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
def __str__(self):
try:
exam = self.exam
except (Exam.DoesNotExist, KeyError) as e:
exam = "None"
return "{}/{}/{}: {}".format(
exam, self.cid, self.question.get_primary_answer(), self.answer
)
def save(self, *args, **kwargs):
self.clean()
return super(CidUserAnswer, self).save(*args, **kwargs)
def clean(self):
if self.answer:
self.answer = self.answer.strip()
s = self.answer.lower()
s = s.translate(str.maketrans("", "", string.punctuation.replace("#", "")))
self.answer_compare = s
# def get_compare_string(self):
# # strip here should be unneccasry (providing clean is now working)
# s = self.answer.lower().strip()
# s = s.translate(str.maketrans('', '', string.punctuation))
# return s
def get_absolute_url(self):
return reverse("rapids:user_answer_view", kwargs={"pk": self.pk})
def get_answer_score(self):
q = self.question
# First step we check that the normal/abnormal states match
if q.normal != self.normal:
# If they don't match the answer is wrong (score is 0)
return 0
# If both are normal full marks
elif q.normal and self.normal:
return 2
# Then compare answer strings (as per anatomy questions)
ans = self.answer_compare
# TODO: this should be cleaned up as we don't want duplicates...
# try:
# marked_ans = q.answers.get(answer_compare__iexact=ans)
# except Answer.DoesNotExist:
# marked_ans = None
marked_ans = q.answers.filter(answer_compare__iexact=ans).first()
mark = "unmarked"
if marked_ans is not None:
if marked_ans.status == Answer.MarkOptions.CORRECT:
mark = 2
elif marked_ans.status == Answer.MarkOptions.HALF_MARK:
mark = 1
elif marked_ans.status == Answer.MarkOptions.INCORRECT:
mark = 0
return mark
if (
q.answers.filter(
answer_compare__iexact=ans, status=Answer.MarkOptions.CORRECT
).first()
is not None
):
mark = 1
elif (
q.answers.filter(
answer_compare__iexact=ans, status=Answer.MarkOptions.HALF_MARK
).first()
is not None
):
mark = 0.5
elif q.answers.filter(
answer_compare__iexact=ans, status=Answer.MarkOptions.INCORRECT
).first():
mark = 0
else:
mark = "unmarked"
return mark