basic rapid integration testing

Dockerfile

@@ -1,5 +1,5 @@
 # pull official base image
-FROM python:3.10.4-slim
+FROM python:3.11.1-slim
 
 # set work directory
 WORKDIR /usr/src/rad

anatomy/models.py

@@ -265,13 +265,11 @@ class AnatomyQuestion(models.Model):
             ]
         )
 
-    def get_question_json(self, based=True, feedback=False, answers=True):
+    def get_question_json(self, based: bool=True, answers: bool=True):
         """Returns a json representation of the question"""
 
-        # Loop through rapidimage associations
         images = []
         annotations = []
-        feedback_images = []
 
         if based:
             images.append(image_as_base64(self.image))

atlas/migrations/0048_auto_20230116_1119.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.13 on 2023-01-16 11:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atlas', '0047_alter_condition_parent_alter_condition_synonym_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='condition',
+            name='parent',
+            field=models.ManyToManyField(blank=True, related_name='_atlas_condition_parent_+', to='atlas.Condition'),
+        ),
+        migrations.AlterField(
+            model_name='condition',
+            name='synonym',
+            field=models.ManyToManyField(blank=True, related_name='_atlas_condition_synonym_+', to='atlas.Condition'),
+        ),
+        migrations.AlterField(
+            model_name='finding',
+            name='synonym',
+            field=models.ManyToManyField(blank=True, related_name='_atlas_finding_synonym_+', to='atlas.Finding'),
+        ),
+        migrations.AlterField(
+            model_name='structure',
+            name='synonym',
+            field=models.ManyToManyField(blank=True, related_name='_atlas_structure_synonym_+', to='atlas.Structure'),
+        ),
+    ]

generic/migrations/0049_auto_20230116_1119.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.13 on 2023-01-16 11:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('generic', '0048_alter_userusergroup_archive_alter_userusergroup_name_and_more'),
+    ]
+
+    operations = [
+    ]

generic/templates/generic/exam_overview_headers.html

@@ -37,7 +37,10 @@
 
 This exam has {{question_number}} questions. Time limit: {{exam.time_limit}} seconds.
 Exam mode: {{ exam.exam_mode }}<br />
-Cid candidates: <a href="{{exam.get_cid_edit_url}}">{{candidate_count.0}}</a>, User candidates: <a href="{{exam.get_user_edit_url}}">{{candidate_count.1}}</a><br />
+
+{% if exam.exam_mode %}
+    Cid candidates: <a href="{{exam.get_cid_edit_url}}">{{candidate_count.0}}</a>, User candidates: <a href="{{exam.get_user_edit_url}}">{{candidate_count.1}}</a><br />
+{% endif %}
 Open access: {{ exam.open_access }}<br />
 
 <div class="parent-help" title="Click to enable / disable the exam">

generic/views.py

@@ -1371,6 +1371,7 @@ class ExamViews(View, LoginRequiredMixin):
             n = 0
 
             for answer in json.loads(request.POST.get("answers")):
+                print("ANSWER", answer)
 
                 eid = int(answer["eid"].split("/")[1])
                 uid = False
@@ -1454,6 +1455,8 @@ class ExamViews(View, LoginRequiredMixin):
                             ans = existing_answers[0]
                             if answer["qidn"] == "1":
                                 ans.normal = normal
+                                # Only wipe the answer text if the new answer is "Normal"
+                                if normal:
                                     ans.answer = ""
                             elif answer["qidn"] == "2" and not ans.normal:
                                 ans.answer = posted_answer

rapids/models.py

@@ -48,6 +48,20 @@ import hashlib
 import pydicom
 import pydicom.errors
 
+from typing import List, TypedDict, Required, NotRequired
+
+
+class QuestionData(TypedDict):
+    images: Required[list[str]]
+    annotations: Required[List[str]]
+    type: Required[str]
+    history: NotRequired[str]
+    normal: Required[bool]
+    answers: NotRequired[List[str]]
+    feedback: NotRequired[str]
+    feedback_images: NotRequired[List[str]]
+
+
 image_storage = FileSystemStorage(
     # Physical file location ROOT
     location="{0}".format(settings.MEDIA_ROOT),
@@ -136,6 +150,15 @@ class Answer(models.Model):
 
 
 class Rapid(models.Model):
+    """
+    Django model that defines a rapid question.
+
+    Images are help by a seperate model: RapidImage
+    which has a foreign key to this model
+
+
+    """
+
     question = models.TextField(null=True, blank=True)
     history = models.TextField(null=True, blank=True)
     feedback = models.TextField(null=True, blank=True)
@@ -293,7 +316,7 @@ class Rapid(models.Model):
         exams = ", ".join(e)
         return exams
 
-    def get_unmarked_user_answer_string(self, exam_pk=None):
+    def get_unmarked_user_answer_string(self, exam_pk: int=None):
         unmarked_answers = self.get_unmarked_user_answers(exam_pk)
 
         if not unmarked_answers:
@@ -304,7 +327,7 @@ class Rapid(models.Model):
             )
         )
 
-    def get_unmarked_user_answers(self, exam_pk=None, marker=None):
+    def get_unmarked_user_answers(self, exam_pk: int|None=None, marker=None):
         # If normal no answers to mark (they will be automarked)
         if self.normal:
             return []
@@ -446,8 +469,6 @@ class Rapid(models.Model):
                 answers.append("{} {}".format(a.name, laterality))
                 answers.append("{}".format(a.name))
 
-
-
         compare_answers = self.get_compare_answers()
 
         new_answers = [
@@ -456,16 +477,21 @@ class Rapid(models.Model):
 
         return new_answers
 
-    def get_question_json(self, based=True, feedback=False):
+    def get_question_json(
-        """Returns json"""
+        self, based: bool = True, feedback: bool = False, answers: bool = True, history: bool = False, annotations: bool = False
+    ) -> QuestionData:
+        """Returns a json representation of the question"""
 
         # Loop through rapidimage associations
         images = []
         annotations = []
         feedback_images = []
         for i in self.images.all():
-            annotations.append(i.image_annotations)
+            # TODO: add anotations
+
             if i.feedback_image == True:
+                if feedback:
+                    annotations.append(i.image_annotations)
                     if based:
                         feedback_images.append(image_as_base64(i.image))
                     else:
@@ -474,6 +500,7 @@ class Rapid(models.Model):
                         )
                     # feedback_images.append("{}/{}".format(settings.REMOTE_URL, i.image.url))
             else:
+                annotations.append(i.image_annotations)
                 if based:
                     images.append(image_as_base64(i.image))
                 else:
@@ -482,16 +509,21 @@ class Rapid(models.Model):
         json = {
             "images": images,
             # "feedback_image": [],
-            "annotations": annotations,
             "type": "rapid",
         }
 
+        if annotations: 
+            json["annotations"] = self.annotations
+
+        if history:
             json["history"] = self.history
+
+        if answers:
             json["normal"] = self.normal
-        json["feedback_images"] = feedback_images
             json["answers"] = list(self.get_correct_unstripped_answers())
 
         if feedback:
+            json["feedback_images"] = feedback_images
             json["feedback"] = self.feedback
 
         return json
@@ -537,8 +569,6 @@ class RapidImage(models.Model):
 
     is_dicom = models.BooleanField(default=False)
 
-
-
     def image_tag(self):
         if self.image:
             return mark_safe(
@@ -580,8 +610,12 @@ class RapidImage(models.Model):
 
                 self.is_dicom = True
             except pydicom.errors.InvalidDicomError:
+                try:  # This is horrible (but needed for current unit tests)
+                    # (we use a temporary file that breaks here)
                     self.image.file.open()
                     hash = hashlib.md5(self.image.read()).hexdigest()
+                except AttributeError:
+                    return
 
             self.image_md5_hash = hash
         super().save(*args, **kwargs)  # Call the "real" save() method.
@@ -633,15 +667,14 @@ class Exam(ExamBase):
         CidUserGroup,
         blank=True,
         help_text="These groups define which candidates are able to be added to the exams/collection.",
-        related_name="rapid_cid_user_groups"
+        related_name="rapid_cid_user_groups",
-
     )
 
     user_user_groups = models.ManyToManyField(
         UserUserGroup,
         blank=True,
         help_text="These groups define which candidates are able to be added to the exams/collection.",
-        related_name="rapid_user_user_groups"
+        related_name="rapid_user_user_groups",
     )
 
     exam_user_status = GenericRelation(ExamUserStatus)
@@ -667,6 +700,7 @@ class Exam(ExamBase):
 
         exam_order = []
 
+        q: Rapid
         for q in questions:
             exam_order.append(q.id)
 

rapids/templates/rapids/exam_scores_user.html

@@ -19,11 +19,18 @@
         {% include 'user_score_header.html' %}
         <ul class="score-answer-list">
             {% for ans, score, correct_answer in answers_and_marks %}
-                <li class="user-answer-li">Question {{forloop.counter}} - Correct answer: <span class="correct-answer">{{ correct_answer }}</span></li>
+                <li class="user-answer-li" data-question-number="{{forloop.counter}}">Question {{forloop.counter}} - Correct answer: {% if exam.publish_results %}<span class="correct-answer">{{correct_answer}}</span>{% endif %}
-                <span class="user-answer-score user-answer-score-{{score}} rapid-ans">
+                    <br/>
-                    <pre>{{ans}}</pre> ({{score}})
+                    <span class="user-answer-score user-answer-score-{{score}}">
+                        <span class="submitted-user-answer">
+                            <pre>{{ans}}</pre>
+                        </span>
+                        {% if exam.publish_results %}
+                            <span class="answer-score">{{score}}</span>
+                        {% endif %}
                     </span>
                     <span class="view-question-link" data-qn={{forloop.counter0}}>View</span>
+                </li>
             {% endfor %}
         </ul>
         {% include 'user_scores_footer.html' %}

rapids/tests/test_rapids_exams.py

@@ -15,37 +15,31 @@ from bs4 import BeautifulSoup
 
 from rapids.views import GenericExamViews as RapidExamViews
 
-from rapids.models import Answer, Exam, Examination, Rapid as Question
+from rapids.models import Answer, CidUserAnswer, Exam, Examination, Rapid as Question, RapidImage
 
-from generic.models import CidUser, CidUserGroup
+from generic.models import CidUser, CidUserGroup, UserUserGroup
+
+from django.contrib.auth.models import User
+
+from rad.test_helpers import AssertNotFound, create_cid_user_and_groups
 
 APP_NAME = "rapids"
 
 
-def create_cid_user_and_groups(db):
-    group1 = CidUserGroup.objects.create(name="Group1")
-    group2 = CidUserGroup.objects.create(name="Group2")
-
-    cid_user_1000 = CidUser.objects.create(cid=1000, passcode="ABCD", group=group1)
-    cid_user_1001 = CidUser.objects.create(cid=1001, passcode="EFGH", group=group1)
-    cid_user_2001 = CidUser.objects.create(cid=2001, passcode="EFGH", group=group2)
-
-    assert cid_user_1000.cid == 1000
-    assert cid_user_1001.cid == 1001
-
-
 def create_exam(db):
     exam: Exam = RapidExamViews.Exam.objects.create(
         name="Cid Exam", exam_mode=True, active=True
     )
 
     group1: CidUserGroup = CidUserGroup.objects.get(name="Group1")
+    usergroup1: UserUserGroup = UserUserGroup.objects.get(name="Group1")
 
     exam.cid_user_groups.add(group1)
 
     exam.valid_cid_users.add(*group1.ciduser_set.all())
+    exam.valid_user_users.add(*usergroup1.users.all())
 
-    assert exam in group1.rapids_cid_user_groups.all()
+    assert exam in group1.rapid_cid_user_groups.all()
 
     assert exam
 
@@ -59,22 +53,29 @@ def create_examination(db):
     examination.save()
 
 
-def create_rapid_image(db, rapid):
+def create_rapid_image(question):
-    rapid_image = RapidImage()
-
-
-def create_question(exam, answer=None):
-    # Just assign random stuff
-
     image = tempfile.NamedTemporaryFile(
         dir=settings.MEDIA_ROOT, suffix=".jpg", delete=False
     )
 
+    rapid_image = RapidImage(image=image, rapid=question)
+
+    rapid_image.save()
+
+
+def create_question(exam, answer=None, normal=False):
+    # Just assign random stuff
+
     question: Question = Question.objects.create(
-        laterality=Question.LATERALITY_CHOICES.NONE,
+        laterality=Question.NONE,
-        examination=Examination.objects.get(pk=1),
+        # examination=Examination.objects.get(pk=1),
-        image=image.name,
+        # image=imagohe.name,
+        history="TEST history",
+        normal=normal,
     )
+
+    create_rapid_image(question=question)
+
     question.exams.set([exam])
 
     # Create an answer for the question
@@ -82,50 +83,86 @@ def create_question(exam, answer=None):
         answer_text = "testanswer"
     else:
         answer_text = answer
-    answer = Answer(question=question, answer=answer_text, answer_compare=answer_text, status=Answer.MarkOptions.CORRECT)
+    answer = Answer(
+        question=question,
+        answer=answer_text,
+        answer_compare=answer_text,
+        status=Answer.MarkOptions.CORRECT,
+    )
     answer.save()
 
     return question
 
 
 def test_exams(db, client):
-    return
     create_cid_user_and_groups(db)
     exam = create_exam(db)
 
-
+    # create_structures(db)
-    create_structures(db)
+    # create_question_types(db)
-    create_question_types(db)
+    create_examination(db)
 
     question1 = create_question(exam)
-    create_question(exam)
+    generated_questions = [question1, create_question(exam), create_question(exam)]
-    create_question(exam)
 
     active_exams = client.get(reverse(f"{APP_NAME}:active_exams"))
     assert active_exams.status_code == 200
     assert len(json.loads(active_exams.content)["exams"]) == 0
 
+    cid_user_1000 = CidUser.objects.get(cid=1000)
     cid_user_1001 = CidUser.objects.get(cid=1001)
+    cid_user_2001 = CidUser.objects.get(cid=2001)
+    user1 = User.objects.get(username="user1")
+
+    # Test exam access urls
+
+    # Check that we can't access without logging in
+    assert client.get(exam.get_json_url()).status_code == 404
+    assert (
+        client.get(
+            exam.get_json_url(cid=cid_user_2001.cid, passcode=cid_user_2001.passcode)
+        ).status_code
+        == 404
+    )
+    # Valid user but incorrect passcode
+    assert (
+        client.get(
+            exam.get_json_url(cid=cid_user_1001.cid, passcode=cid_user_2001.passcode)
+        ).status_code
+        == 404
+    )
+
+    users_tested = 1
+    for cid_user in [cid_user_1001, cid_user_1000, user1]:
+        if isinstance(cid_user, CidUser):
+            testing_cid_user = True
+        else:
+            testing_cid_user = False
+            current_user = cid_user
+            client.force_login(cid_user)
+
+        exam.active = True
+        exam.publish_results = False
+        exam.save()
 
-    for cid_user in [cid_user_1001]:
         # Test active exams sections works as intended
-        active_exams_cid = client.get(
+        if testing_cid_user:
-            reverse(
+            active_exams_url = reverse(
                 f"active_exams_cid",
                 kwargs={"cid": cid_user.cid, "passcode": cid_user.passcode},
             )
+        else:
+            active_exams_url = reverse(
+                f"active_exams",
             )
 
+        active_exams_cid = client.get(active_exams_url)
+
+        print(cid_user)
         assert active_exams_cid.status_code == 200
         assert len(json.loads(active_exams_cid.content)["exams"]) == 1
 
-        active_exams_cid = client.get(
+        if testing_cid_user:
-            reverse(f"active_exams_cid", kwargs={"cid": 1001, "passcode": "EFGH"})
-        )
-
-        assert active_exams_cid.status_code == 200
-        assert len(json.loads(active_exams_cid.content)["exams"]) == 1
-
             no_active_exams_cid = client.get(
                 reverse(f"active_exams_cid", kwargs={"cid": 2001, "passcode": "EGFH"})
             )
@@ -134,7 +171,10 @@ def test_exams(db, client):
             assert json.loads(no_active_exams_cid.content)["status"] == "invalid"
 
             invalid_passcode = client.get(
-            reverse(f"active_exams_cid", kwargs={"cid": 1001, "passcode": "ABCD"})
+                reverse(
+                    f"active_exams_cid",
+                    kwargs={"cid": cid_user.cid, "passcode": "AAAA"},
+                )
             )
             assert invalid_passcode.status_code == 401
             assert json.loads(invalid_passcode.content)["status"] == "invalid"
@@ -148,6 +188,7 @@ def test_exams(db, client):
         res = client.get(exam_metadata["url"])
         assert res.status_code == 200
         exam_json = json.loads(res.content)
+        print(exam_json)
 
         assert (
             question1.get_question_json(answers=False)
@@ -165,9 +206,14 @@ def test_exams(db, client):
         exam.active = True
         exam.save()
 
+        if testing_cid_user:
+            post_cid = cid_user.cid
+        else:
+            post_cid = f"u-{current_user.pk}"
+
         post_json = {
-            "eid": f"anatomy/{exam.pk}",
+            "eid": f"rapid/{exam.pk}",
-            "cid": cid_user_1001.cid,
+            "cid": post_cid,
             "start_time": "1659618141.731",
             "answers": [[]],
         }
@@ -182,8 +228,8 @@ def test_exams(db, client):
         assert json_res["question_count"] == 0
 
         post_json = {
-            "eid": f"anatomy/{exam.pk+10}",
+            "eid": f"rapid/{exam.pk+10}",
-            "cid": cid_user_1001.cid,
+            "cid": post_cid,
             "start_time": "1659618141.731",
             "answers": [[]],
         }
@@ -193,7 +239,7 @@ def test_exams(db, client):
 
         # Test invalid cid
         post_json = {
-            "eid": f"anatomy/{exam.pk+10}",
+            "eid": f"rapid/{exam.pk+10}",
             "cid": 1,
             "start_time": "1659618141.731",
             "answers": [[]],
@@ -203,26 +249,39 @@ def test_exams(db, client):
         assert json_res["success"] == False
         assert json_res["error"] == "Invalid data"
 
+
+        # Rapid answers currently send a seperate object for each component
         valid_answers = []
+        valid_answers_extra = []
 
         for questions in exam_json["questions"]:
             for qid in questions:
-                valid_answers.append(
+                post_data = {
-                    {
                     "eid": exam_json["eid"],
-                        "cid": cid_user_1001.cid,
+                    "cid": post_cid,
-                        "passcode": cid_user_1001.passcode,
                     "qid": qid,
                     "qidn": "1",
+                    "ans": f"Abnormal",
+                }
+                post_data2 = {
+                    "eid": exam_json["eid"],
+                    "cid": post_cid,
+                    "qid": qid,
+                    "qidn": "2",
                     "ans": f"answer {qid}",
                 }
-                )
+                if testing_cid_user:
+                    post_data["passcode"] = cid_user.passcode
+                    post_data2["passcode"] = cid_user.passcode
+
+                valid_answers_extra.append(post_data)
+                valid_answers.append(post_data2)
 
         post_json = {
             "eid": exam_json["eid"],
-            "cid": cid_user_1001.cid,
+            "cid": post_cid,
             "start_time": "1659618141.731",
-            "answers": json.dumps(valid_answers),
+            "answers": json.dumps([*valid_answers, *valid_answers_extra]),
         }
 
         # Test submission
@@ -232,70 +291,142 @@ def test_exams(db, client):
         json_res = json.loads(res.content)
 
         assert json_res["success"] == True
-        assert json_res["question_count"] == 3
+        assert json_res["question_count"] == 6
+
+        # Check the answers now exist in the database
+        if testing_cid_user:
+            answers = CidUserAnswer.objects.filter(cid=cid_user.cid)
+        else:
+            answers = CidUserAnswer.objects.filter(user=cid_user)
+
+        assert answers.count() == 3
 
         # We can then check the answers have been submitted (and can be viewed by the user)
 
-        # Get overview scores page
+        if testing_cid_user:
-        cid_scores_res = client.get(
+            # Get overview scores page (invalid passcode)
-            reverse(f"cid_scores", kwargs={"cid": 1001, "passcode": "EFGH"})
+            AssertNotFound(
+                client,
+                reverse(
+                    f"cid_scores", kwargs={"cid": cid_user.cid, "passcode": "AAAA"}
+                ),
             )
 
+            cid_scores_url = reverse(
+                f"cid_scores",
+                kwargs={"cid": cid_user.cid, "passcode": cid_user.passcode},
+            )
+        else:
+            cid_scores_url = reverse(f"user_scores")
+
+        # Get overview scores page
+        cid_scores_res = client.get(cid_scores_url)
+
         cid_scores_soup = BeautifulSoup(cid_scores_res.content, "html.parser")
 
-        print(cid_scores_soup.find("div", {"id" : "exam-assigned"}))
+        search_exam = (
-        search_exam = cid_scores_soup.find("div", {"id" : "exam-assigned"}).find("ul", {"class" : exam.app_name}).find_all("li", attrs={"data-exam-id": exam.pk})
+            cid_scores_soup.find("div", {"id": "exam-assigned"})
+            .find("ul", {"class": exam.app_name})
+            .find_all("li", attrs={"data-exam-id": exam.pk})
+        )
         assert search_exam
         assert exam.name in str(search_exam)
         assert "Active" in str(search_exam)
-        #assert "Active" in assigned_exams # check it is active
+        # assert "Active" in assigned_exams # check it is active
 
-        invalid_exam = cid_scores_soup.find_all("li", attrs={"data-exam-id": exam.pk+5})
+        invalid_exam = cid_scores_soup.find_all(
+            "li", attrs={"data-exam-id": exam.pk + 5}
+        )
         assert not invalid_exam  # Check we can't find an invalid exam
 
         # Check that we have a results link
-        results_exam = cid_scores_soup.find("div", {"id" : "exam-results"}).find("ul", {"class" : exam.app_name}).find_all("li", attrs={"data-exam-id": exam.pk})
+        results_exam = (
+            cid_scores_soup.find("div", {"id": "exam-results"})
+            .find("ul", {"class": exam.app_name})
+            .find_all("li", attrs={"data-exam-id": exam.pk})
+        )
         assert results_exam
         assert exam.name in str(results_exam)
-        assert exam.name+" test" not in str(results_exam)
+        assert exam.name + " test" not in str(results_exam)
-        assert "Results Published" not in str(results_exam) # It should not be published yet
+        assert "Results Published" not in str(
+            results_exam
+        )  # It should not be published yet
 
+        if testing_cid_user:
+            AssertNotFound(
+                client,
+                reverse(
+                    f"{exam.app_name}:exam_scores_cid_user",
+                    kwargs={"pk": exam.pk, "cid": cid_user.cid, "passcode": "AAAA"},
+                ),
+            )
+            AssertNotFound(
+                client,
+                reverse(
+                    f"{exam.app_name}:exam_scores_cid_user",
+                    kwargs={
+                        "pk": exam.pk,
+                        "cid": cid_user_2001.cid,
+                        "passcode": cid_user_2001.passcode,
+                    },
+                ),
+            )
+
+            exam_scores_url = reverse(
+                f"{exam.app_name}:exam_scores_cid_user",
+                kwargs={
+                    "pk": exam.pk,
+                    "cid": cid_user.cid,
+                    "passcode": cid_user.passcode,
+                },
+            )
+        else:
+            exam_scores_url = reverse(
+                f"{exam.app_name}:exam_scores_user",
+                kwargs={"pk": exam.pk},
+            )
 
         # Load the exam results page
-        cid_exam_scores_res = client.get(
+        cid_exam_scores_res = client.get(exam_scores_url)
-            reverse(f"{exam.app_name}:exam_scores_cid_user", kwargs={"pk": exam.pk, "cid": 1001, "passcode": "EFGH"})
+        # Load the exam results page
-        )
         cid_exam_scores_soup = BeautifulSoup(cid_exam_scores_res.content, "html.parser")
 
         # Check that we have an alert that exams are not published
         alert = cid_exam_scores_soup.find("div", {"class": "alert"})
         assert "Results are not currently published." in str(alert)
 
-
+        answer_lis = cid_exam_scores_soup.find(
-        answer_lis = cid_exam_scores_soup.find("ul", {"class": "score-answer-list"}).find_all("li")
+            "ul", {"class": "score-answer-list"}
+        ).find_all("li")
 
         assert len(answer_lis) == len(valid_answers)
 
-        for n in range(len(valid_answers)): # Check no answers are visible if results not published
+        for n in range(
+            len(valid_answers)
+        ):  # Check no answers are visible if results not published
             answer = valid_answers[n]
             li = answer_lis[n]
 
             assert answer["ans"] in str(li)  # Check that the saved answer is shown
 
-            assert li.find("span", {"class": "correct-answer"}).string == "*****" # and that the correct answer isn't
+            assert not li.find(
+                "span", {"class": "correct-answer"}
+            )  # and that the correct answer isn't
 
-        assert cid_exam_scores_soup.find("div", {"class": "score-overview"}) is None # check we hide the score overview if not published
+        assert (
+            cid_exam_scores_soup.find("div", {"class": "score-overview"}) is None
+        )  # check we hide the score overview if not published
 
         # Add another question to the exam
         create_question(exam)
 
         # Load the exam results page (again)
-        cid_exam_scores_res = client.get(
+        cid_exam_scores_res = client.get(exam_scores_url)
-            reverse(f"{exam.app_name}:exam_scores_cid_user", kwargs={"pk": exam.pk, "cid": 1001, "passcode": "EFGH"})
-        )
         cid_exam_scores_soup = BeautifulSoup(cid_exam_scores_res.content, "html.parser")
 
-        answer_lis = cid_exam_scores_soup.find("ul", {"class": "score-answer-list"}).find_all("li")
+        answer_lis = cid_exam_scores_soup.find(
+            "ul", {"class": "score-answer-list"}
+        ).find_all("li")
         assert len(answer_lis) == len(valid_answers) + 1
         assert "Not answered" in str(answer_lis[-1])
 
@@ -304,20 +435,24 @@ def test_exams(db, client):
         exam.save()
 
         # Load the exam results page (again)
-        cid_exam_scores_res = client.get(
+        cid_exam_scores_res = client.get(exam_scores_url)
-            reverse(f"{exam.app_name}:exam_scores_cid_user", kwargs={"pk": exam.pk, "cid": 1001, "passcode": "EFGH"})
-        )
         cid_exam_scores_soup = BeautifulSoup(cid_exam_scores_res.content, "html.parser")
 
-        assert cid_exam_scores_soup.find("div", {"class": "alert"}) is None # Check our warning banner is gone
+        assert (
+            cid_exam_scores_soup.find("div", {"class": "alert"}) is None
+        )  # Check our warning banner is gone
 
-        answer_lis = cid_exam_scores_soup.find("ul", {"class": "score-answer-list"}).find_all("li")
+        answer_lis = cid_exam_scores_soup.find(
+            "ul", {"class": "score-answer-list"}
+        ).find_all("li")
 
         assert len(answer_lis) == len(valid_answers) + 1
         assert "Not answered" in str(answer_lis[-1])
 
         # Repetition....
-        for n in range(len(valid_answers)): # Check answers are visible if results published
+        for n in range(
+            len(valid_answers)
+        ):  # Check answers are visible if results published
             answer = valid_answers[n]
             li = answer_lis[n]
 
@@ -325,20 +460,46 @@ def test_exams(db, client):
 
             assert li.find("span", {"class": "correct-answer"}).string == "testanswer"
 
-        assert cid_exam_scores_soup.find("div", {"class": "score-overview"}).find("span", {"id": "total-score"}).string == "0 (3 unmarked)"
+        if users_tested == 1:
-
+            # First user the answers will not have been saved
+            assert (
+                cid_exam_scores_soup.find("div", {"class": "score-overview"})
+                .find("span", {"id": "total-score"})
+                .string
+                == "0 (3 unmarked)"
+            )
+        else:
+            # Subsequent users should already have answers marked (apart from the last question which we have deleted)
+            assert (
+                cid_exam_scores_soup.find("div", {"class": "score-overview"})
+                .find("span", {"id": "total-score"})
+                .string
+                == "4 (1 unmarked)"
+            )
 
         for question in exam.exam_questions.all():
+            print(question)
+            print(question.get_compare_answers())
             try:
-                new_ans = Answer(question=question, answer=question.get_unmarked_user_answers().pop(), status=Answer.MarkOptions.CORRECT)
+                new_ans = Answer(
+                    question=question,
+                    answer=question.get_unmarked_user_answers().pop(),
+                    status=Answer.MarkOptions.CORRECT,
+                )
                 new_ans.save()
-            except KeyError: # Final question does not have an answer
+            except IndexError:  # Final question does not have an answer
                 pass
 
         # Load the exam results page (again)
-        cid_exam_scores_res = client.get(
+        cid_exam_scores_res = client.get(exam_scores_url)
-            reverse(f"{exam.app_name}:exam_scores_cid_user", kwargs={"pk": exam.pk, "cid": 1001, "passcode": "EFGH"})
-        )
         cid_exam_scores_soup = BeautifulSoup(cid_exam_scores_res.content, "html.parser")
 
-        assert cid_exam_scores_soup.find("div", {"class": "score-overview"}).find("span", {"id": "total-score"}).string == "6"
+        assert (
+            cid_exam_scores_soup.find("div", {"class": "score-overview"})
+            .find("span", {"id": "total-score"})
+            .string
+            == "6"
+        )
+
+        generated_questions.pop(-1).delete()
+        users_tested = users_tested + 1