from collections import defaultdict, Counter
from pathlib import Path
import secrets
import statistics
import threading
from typing import Any
from dal import autocomplete
from django.contrib.auth.models import User, Group
from django.contrib.contenttypes.models import ContentType
from django.forms.models import model_to_dict
from django.db.models.functions import Coalesce, Lower
from django.shortcuts import render, get_object_or_404, redirect

from django.contrib.auth.decorators import login_required, user_passes_test
from django.views.decorators.http import require_POST
from django.urls import reverse_lazy
from django.urls.base import reverse
from django.utils import timezone
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.views.decorators.csrf import csrf_exempt
from django.core.exceptions import PermissionDenied, FieldError

from django.http import Http404, HttpRequest, JsonResponse
from django.http import HttpResponseRedirect, HttpResponse
from django.template.loader import render_to_string

from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin

from django.core.exceptions import ValidationError
from django.core.validators import validate_email


from django.utils.decorators import method_decorator


from django.core.cache import cache


import json
import urllib
import urllib.parse

from django.views import View
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django_filters.views import FilterView
from django_filters import (
    FilterSet,
    OrderingFilter,
    ModelMultipleChoiceFilter,
    DateFromToRangeFilter,
)
from django_filters.widgets import DateRangeWidget
from django_tables2.views import SingleTableMixin
from reversion.views import RevisionMixin
from atlas.models import CaseCollection, CaseDetail
from generic.decorators import user_is_cid_user_manager
from generic.filters import CidUserFilter, ExaminationFilter, SupervisorFilter
from generic.models import UserUserGroup, CidUserExam

from generic.tables import (
    CidUserExamTable,
    CidUserTable,
    ExaminationTable,
    SupervisorTable,
)
from generic.mixins import CheckCanEditMixin, SuperuserRequiredMixin

import zipfile
from django.core.files.base import ContentFile
from django.db.models import Q, Count, F

from .forms import (
    CidGroupExamForm,
    CidUserForm,
    CidUserExamForm,
    ExamCollectionCloneForm,
    ExamCollectionForm,
    QuestionReviewForm,
    ExaminationForm,
    CidUserGroupForm,
    ExaminationMergeForm,
    SupervisorForm,
    TraineeForm,
    UserGroupExamForm,
    CidGroupExamForm,
    UserUserForm,
    UserUserGroupForm,
    UsersAutocompleteForm,  # Added import
    ExamCollectionBulkAddGroupsForm,
)

from loguru import logger

from .models import (
    CidUser,
    CidUserExam,
    CidUserGroup,
    CimarCase,
    ExamBase,
    ExamCollection,
    ExamUserStatus,
    Examination,
    QuestionBase,
    QuestionNote,
    Supervisor,
    UserGrades,
    UserProfile,
    UserUserGroup,
    get_next_cid,
    QuestionReview,
)


@user_is_cid_user_manager
def cid_user_exam_partial(request, app_name: str, exam_id: int, cid: int):
    """Return an HTMX partial showing details for a CidUserExam record.

    Matches by content_type app_label and object_id (exam id), and cid value.
    """
    # Find the matching CidUserExam
    try:
        cux = CidUserExam.objects.select_related("cid_user", "user_user", "content_type").filter(
            content_type__app_label=app_name, object_id=exam_id, cid_user__cid=cid
        ).first()
    except Exception:
        cux = None

    if not cux:
        return HttpResponse("No record found", status=404)

    return render(request, "generic/partials/ciduserexam_detail_partial.html", {"cid_user_exam": cux})


@login_required
def exam_collection_exams_partial(request, collection_id: int, exam_type: str):
    """Return the exam-list partial for a given collection and exam type.

    Scope: authenticated users only.
    Functionality: HTMX lazy-loading endpoint that returns the exam list partial
    for a specific exam type within a collection. Annotates candidate counts and
    prefetches authors to avoid N+1 DB queries in the template.
    """
    collection = get_object_or_404(ExamCollection, pk=collection_id)

    # Map exam_type to related name and model
    EXAM_MAP = {
        "anatomy": ("anatomy_exams", "anatomy"),
        "longs": ("longs_exams", "longs"),
        "rapids": ("rapids_exams", "rapids"),
        "shorts": ("shorts_exams", "shorts"),
        "physics": ("physics_exams", "physics"),
        "sbas": ("sbas_exams", "sbas"),
    }

    if exam_type not in EXAM_MAP:
        return HttpResponse("Unknown exam type", status=400)

    rel_name, app_label = EXAM_MAP[exam_type]

    # Attempt to import the model class for the app to annotate counts
    ExamModel = None
    try:
        if exam_type == "anatomy":
            from anatomy.models import Exam as AnatomyExam

            ExamModel = AnatomyExam
        elif exam_type == "longs":
            from longs.models import Exam as LongsExam

            ExamModel = LongsExam
        elif exam_type == "rapids":
            from rapids.models import Exam as RapidsExam

            ExamModel = RapidsExam
        elif exam_type == "shorts":
            from shorts.models import Exam as ShortsExam

            ExamModel = ShortsExam
        elif exam_type == "physics":
            from physics.models import Exam as PhysicsExam

            ExamModel = PhysicsExam
        elif exam_type == "sbas":
            from sbas.models import Exam as SbasExam

            ExamModel = SbasExam
    except Exception:
        ExamModel = None

    # Build queryset on the related manager; filter archived false by default
    related_qs = getattr(collection, rel_name).all().filter(archive=False)

    if ExamModel is not None:
        # Avoid evaluating the related manager multiple times; use ids
        pks = list(related_qs.values_list("pk", flat=True))
        related_qs = (
            ExamModel.objects.filter(pk__in=pks)
            .annotate(
                valid_cid_users_count=Count("valid_cid_users", distinct=True),
                valid_user_users_count=Count("valid_user_users", distinct=True),
            )
            .prefetch_related("author")
            .order_by("name")
        )

    context = {
        "exams": related_qs,
        "app_name": app_label,
        "marking": request.user.has_perm("generic.can_mark") if request.user.is_authenticated else False,
        "collection": collection,
    }

    # Render only the fragment template which includes the `exam-list` partialdef
    html = render_to_string(
        "generic/partials/exam_list_fragment.html", context, request=request
    )
    return HttpResponse(html)


from rapids.models import (
    Rapid as RapidQuestion,
    ExamQuestionDetail as RapidsExamQuestionDetail,
)
from rapids.models import Exam as RapidsExam
from rapids.models import UserAnswer as RapidsUserAnswer
from longs.models import (
    Long as LongQuestion,
    LongSeries,
    ExamQuestionDetail as LongsExamQuestionDetail,
)
from longs.models import Exam as LongsExam
from longs.models import UserAnswer as LongsUserAnswer
from anatomy.models import (
    AnatomyQuestion as AnatomyQuestion,
    ExamQuestionDetail as AnatomyExamQuestionDetail,
)
from anatomy.models import Exam as AnatomyExam
from sbas.models import (
    Question as SbasQuestion,
    ExamQuestionDetail as SbasExamQuestionDetail,
)
from anatomy.models import UserAnswer as AnatomyUserAnswer
from sbas.models import Exam as SbasExam
from sbas.models import UserAnswer as SbasUserAnswer
from physics.models import (
    Question as PhysicsQuestion,
    ExamQuestionDetail as PhysicsExamQuestionDetail,
)
from physics.models import Exam as PhysicsExam
from physics.models import UserAnswer as PhysicsUserAnswer

from shorts.models import Question as ShortsQuestion, ExamQuestionDetail as ShortsExamQuestionDetail, UserAnswer as ShortsUserAnswer, Exam as ShortsExam

from django.db.models import Case, When
from django.conf import settings
from django.db import transaction

from datetime import datetime
from django.utils import timezone
import os
import string
import random
import plotly.express as px

# from rad.views import get_question_and_content_type
from django.db.models import Prefetch


from loguru import logger


@login_required
@user_passes_test(lambda u: u.is_superuser)
def bulk_delete_questions(request):
    """Generic bulk delete for question-like models.

    Expects POST with 'app' (app_name) and 'selected' (list or comma-separated ids).
    Returns a small HTML fragment summarising deletions (suitable for HTMX).
    """
    if request.method != "POST":
        return HttpResponse(status=405)

    logger.debug(f"bulk_delete_questions called with POST: {request.POST}")

    app = request.POST.get("app")

    # Accept different submission patterns: checkboxes may be sent as 'selection' or 'selected'
    selected_list = []
    # common variants
    for key in ("selected", "selection", "selected[]", "selection[]"):
        vals = request.POST.getlist(key)
        if vals:
            # extend list; we'll parse ints below
            selected_list.extend(vals)

    # Also accept a raw CSV string under 'selected'
    selected_raw = request.POST.get("selected")

    ids = set()
    if selected_list:
        for s in selected_list:
            # some checkbox libraries may give empty strings or 'on' for checked boxes
            try:
                ids.add(int(s))
            except Exception:
                # ignore non-int vals
                continue
    elif selected_raw:
        for part in selected_raw.split(','):
            try:
                part = part.strip()
                if not part:
                    continue
                ids.add(int(part))
            except Exception:
                continue

    # Map app name to model class
    APP_MODEL_MAP = {
        "sbas": SbasQuestion,
        "physics": PhysicsQuestion,
        "anatomy": AnatomyQuestion,
        "rapids": RapidQuestion,
        "rapid": RapidQuestion,
        "shorts": ShortsQuestion,
        "longs": LongQuestion,
        "long": LongQuestion,
    }

    Model = APP_MODEL_MAP.get(app)
    if Model is None:
        logger.error(f"bulk_delete_questions called with unknown app: {app}")
        return HttpResponse(f"Unknown app: {app}")

    deleted = 0
    errors = []
    try:
        qs = Model.objects.filter(pk__in=list(ids))
        deleted = qs.count()
        qs.delete()
    except Exception as e:
        logger.exception("bulk_delete failed for app=%s ids=%s", app, ids)
        return HttpResponse("Unable to deleted items")

    # Render a small fragment summarising deletions
    context = {"deleted": deleted, "errors": errors}
    return render(request, "generic/partials/bulk_delete_result.html", context)


class RedirectMixin:
    def get_success_url(self) -> str:
        if "redirect" in self.request.GET:
            return self.request.GET["redirect"]
        return super().get_success_url()


class AuthorRequiredMixin(object):
    def get_object(self, *args, **kwargs):
        obj = super().get_object(*args, **kwargs)
        if self.request.user.is_superuser:
            return obj

        if self.request.user not in obj.author.all():
            raise PermissionDenied()  # or Http404
        return obj


class CidManagerRequiredMixin(UserPassesTestMixin):
    def test_func(self):
        return self.request.user.groups.filter(name="cid_user_manager").exists()


#    def get_object(self, *args, **kwargs):
#        obj = super().get_object(*args, **kwargs)
#        if self.request.user.groups.filter(name="cid_user_manager").exists():
#            return obj
#        raise PermissionDenied()  # or Http404


def get_exam_attempt_score(exam_type, exam, user):
    from django.contrib.contenttypes.models import ContentType
    from physics.models import UserAnswer as PhysicsUserAnswer
    from anatomy.models import UserAnswer as AnatomyUserAnswer
    from rapids.models import UserAnswer as RapidsUserAnswer
    from shorts.models import UserAnswer as ShortsUserAnswer
    from longs.models import UserAnswer as LongsUserAnswer
    from sbas.models import UserAnswer as SbasUserAnswer

    MAP = {
        "physics": PhysicsUserAnswer,
        "anatomy": AnatomyUserAnswer,
        "rapids": RapidsUserAnswer,
        "shorts": ShortsUserAnswer,
        "longs": LongsUserAnswer,
        "sbas": SbasUserAnswer,
    }
    UserAnswer = MAP.get(exam_type)
    if not UserAnswer:
        return None, None

    questions = list(exam.get_questions())
    if not questions:
        return 0, 0
    user_answers = UserAnswer.objects.filter(user=user, exam=exam).select_related("question")
    user_answers_map = {ua.question_id: ua for ua in user_answers}

    total_score = 0
    for q in questions:
        ua = user_answers_map.get(q.pk)
        if ua:
            total_score += ua.get_answer_score()
        else:
            score, _ = q.get_unanswered_mark_and_text()
            total_score += score

    # Max score
    match exam_type:
        case "rapids" | "anatomy":
            max_score = len(questions) * 2
        case "longs":
            max_score = len(questions) * 8
        case "physics" | "shorts":
            max_score = len(questions) * 5
        case _:
            max_score = len(questions)

    return total_score, max_score


def get_supervisor_attempts(supervisor, trainees=None):
    from django.contrib.contenttypes.models import ContentType
    from generic.models import CidUserExam
    from atlas.models import CaseCollection
    from django.urls import reverse
    from django.utils import timezone

    if trainees is None:
        trainees = User.objects.filter(userprofile__supervisor=supervisor)

    attempts = CidUserExam.objects.filter(user_user__in=trainees).select_related("user_user", "content_type")

    parsed_attempts = []
    for att in attempts:
        exam = att.exam
        if not exam:
            continue

        friendly_type = att.content_type.app_label
        if friendly_type == "atlas":
            friendly_type = "casecollection"

        exam_supervisor_visible = getattr(exam, "results_supervisor_visible", False)
        is_shared = att.share_with_supervisor or exam_supervisor_visible

        score_str = "N/A"
        detail_url = None

        if is_shared:
            if friendly_type == "casecollection":
                total_cases = exam.casedetail_set.count()
                reviewed_cases = att.selfreview_set.count()
                score_str = f"{reviewed_cases} / {total_cases} cases"
                detail_url = reverse(
                    "atlas:collection_history_user",
                    kwargs={
                        "exam_id": exam.pk,
                        "user_pk": att.user_user.pk,
                    },
                )
            else:
                total_score, max_score = get_exam_attempt_score(friendly_type, exam, att.user_user)
                if total_score is not None:
                    pct = int((total_score / max_score) * 100) if max_score > 0 else 0
                    score_str = f"{total_score} / {max_score} ({pct}%)"
                else:
                    score_str = "No answers"
                detail_url = reverse(
                    f"{friendly_type}:exam_scores_user_supervisor",
                    kwargs={"pk": exam.pk, "user_id": att.user_user.pk},
                )
        else:
            score_str = "Private"

        parsed_attempts.append({
            "attempt": att,
            "exam": exam,
            "trainee": att.user_user,
            "type": friendly_type,
            "type_display": friendly_type.title() if friendly_type != "sbas" else "SBAs",
            "date": att.end_time or att.start_time,
            "is_shared": is_shared,
            "score_str": score_str,
            "detail_url": detail_url,
            "completed": att.completed,
        })
    parsed_attempts.sort(key=lambda x: x["date"] or timezone.now(), reverse=True)
    return parsed_attempts


def get_user_exams(user, supervisor_view=False):
    EXAM_ANSWER_MAP = {
        "physics": (PhysicsUserAnswer, PhysicsExam, "physics_exams"),
        "anatomy": (AnatomyUserAnswer, AnatomyExam, "anatomy_exams"),
        "rapids": (RapidsUserAnswer, RapidsExam, "rapids_exams"),
        "shorts": (ShortsUserAnswer, ShortsExam, "shorts_exams"),
        "longs": (LongsUserAnswer, LongsExam, "longs_exams"),
        "sbas": (SbasUserAnswer, SbasExam, "sbas_exams"),
    }
    kwargs = {"exam_mode": True, "archive": False}

    exams = []

    if supervisor_view:

        kwargs["results_supervisor_visible"] = True
        del kwargs["archive"]

    for exam_type in EXAM_ANSWER_MAP:
        UserAnswer, Exam, cid_rel = EXAM_ANSWER_MAP[exam_type]
        exam_answers = UserAnswer.objects.filter(user=user)
        exam_ids = exam_answers.values_list("exam").distinct()
        exams_to_add = Exam.objects.filter(id__in=exam_ids, **kwargs)
        if supervisor_view:
            extra_exam_ids = CidUserExam.objects.filter(
                user_user=user,
                share_with_supervisor=True,
                content_type=ContentType.objects.get_for_model(Exam),
            ).values_list(cid_rel)

            exams_to_add = exams_to_add | Exam.objects.filter(id__in=extra_exam_ids)
        if exams_to_add:
            exams.append((exam_type, exams_to_add))

    return exams


def get_exam_model_from_app_name(app_name: str) -> ExamBase:
    EXAM_MAP = {
        "physics": PhysicsExam,
        "anatomy": AnatomyExam,
        "rapids": RapidsExam,
        "shorts": ShortsExam,
        "longs": LongsExam,
        "sbas": SbasExam,
    }

    return EXAM_MAP[app_name]


def normaliseRapidsScore(score):
    if score == 49:
        return 4.5
    elif score in [50, 51]:
        return 5
    elif score in [52, 53]:
        return 5.5
    elif score in [54]:
        return 6
    elif score in [55, 56]:
        return 6.5
    elif score in [57, 58]:
        return 7
    elif score in [59]:
        return 7.5
    elif score in [60]:
        return 8

    return 4


def get_question_and_content_type(question_type):
    if question_type == "rapid":
        question = RapidQuestion
        content_type = ContentType.objects.get(model="rapid")
    elif question_type == "shorts":
        question = ShortsQuestion
        content_type = ContentType.objects.get(model="rapid")
    elif question_type == "anatomy":
        question = AnatomyQuestion
        content_type = ContentType.objects.get(model="anatomyquestion")
    elif question_type == "long":
        question = LongQuestion
        content_type = ContentType.objects.get(model="long")
    elif question_type == "sbas":
        question = SbasQuestion
        content_type = ContentType.objects.get(app_label="sbas", model="question")
    elif question_type == "physics":
        question = PhysicsQuestion
        content_type = ContentType.objects.get(app_label="physics", model="question")
    else:
        raise PermissionError()

    return question, content_type


# Create your views here.
@login_required
def create_examination(request):
    form = ExaminationForm(request.POST or None)
    if form.is_valid():
        instance = form.save()
        return HttpResponse(
            '<script>opener.closePopup(window, "%s", "%s", "#id_examination");</script>'
            % (instance.pk, instance)
        )
    return render(
        request, "rapids/create_simple.html", {"form": form, "name": "Examination"}
    )


@login_required
@user_passes_test(lambda u: u.is_superuser or u.groups.filter(name="cid_user_manager").exists())
def exam_collection_bulk_add_groups(request, collection_id):
    """HTMX endpoint: show form (GET) and process bulk-adding of CID/User groups (POST).

    Renders small partials suitable for HTMX swaps.
    """
    collection = get_object_or_404(ExamCollection, pk=collection_id)

    logger.debug(f"exam_collection_bulk_add_groups called with method={request.method} by user={request.user.username}")
    logger.debug(request)


    if request.method == "GET":
        logger.debug(f"exam_collection_bulk_add_groups GET request by user={request.user.username}")
        form = ExamCollectionBulkAddGroupsForm(user=request.user)
        return render(request, "generic/partials/examcollection_bulk_add_groups_form.html", {"form": form, "collection": collection})

    # POST
    form = ExamCollectionBulkAddGroupsForm(request.POST, user=request.user)
    if not form.is_valid():
        logger.debug(f"exam_collection_bulk_add_groups form invalid: {form.errors}")
        return render(request, "generic/partials/examcollection_bulk_add_groups_form.html", {"form": form, "collection": collection})

    cid_groups = form.cleaned_data.get("cid_user_groups")
    user_groups = form.cleaned_data.get("user_user_groups")
    exam_types = form.cleaned_data.get("exam_types")
    # If 'all' is selected (or no selection), treat as None -> all exams
    if exam_types:
        if "all" in exam_types:
            exam_types = None
    else:
        exam_types = None

    # Debug: log what was submitted so we can trace why groups may not be applied
    try:
        cid_pks = [g.pk for g in cid_groups] if cid_groups is not None else []
    except Exception:
        cid_pks = []
    try:
        user_pks = [g.pk for g in user_groups] if user_groups is not None else []
    except Exception:
        user_pks = []
    logger.debug(
        "ExamCollection bulk-add POST: collection=%s cid_groups=%s user_groups=%s exam_types=%s",
        collection.pk,
        cid_pks,
        user_pks,
        exam_types,
    )

    total_user = 0
    total_cid = 0
    try:
        with transaction.atomic():
            if user_groups:
                total_user = collection.bulk_add_user_user_groups(user_groups, exam_types=exam_types)
            if cid_groups:
                total_cid = collection.bulk_add_cid_user_groups(cid_groups, exam_types=exam_types)
    except Exception as e:
        # render an error fragment
        return render(request, "generic/partials/examcollection_bulk_add_groups_result.html", {"error": str(e), "collection": collection})

    return render(request, "generic/partials/examcollection_bulk_add_groups_result.html", {"total_user": total_user, "total_cid": total_cid, "collection": collection})


@login_required
@user_passes_test(lambda u: u.is_superuser)
def examination_detail(request, pk):
    examination = get_object_or_404(Examination, pk=pk)

    merge_form = ExaminationMergeForm()
    # logging.debug(atlas.subspecialty.first().name.all())
    return render(
        request,
        "generic/examination_detail.html",
        {"examination": examination, "merge_form": merge_form},
    )


@login_required
@user_passes_test(lambda u: u.is_superuser)
def examination_merge(request, pk):
    examination = get_object_or_404(Examination, pk=pk)

    form = ExaminationMergeForm(request.POST)

    if form.is_valid():
        examination_to_merge_into = form.cleaned_data["examination_to_merge_into"]

        success = False
        if examination.merge_into(examination_to_merge_into):
            success = True

    else:
        examination_to_merge_into = False
    return render(
        request,
        "generic/examination_merge.html",
        {
            "form": form,
            "examination": examination,
            "examination_to_merge_into": examination_to_merge_into,
            "success": success,
        },
    )

    # logging.debug(atlas.subspecialty.first().name.all())
    return render(
        request,
        "generic/examination_merge.html",
        {
            "examination": examination,
        },
    )


@login_required
def get_examination_id(request):
    """Return the primary key for an Examination looked up by name.

    Scope: authenticated users only (admin popup helper).
    Functionality: looks up an Examination by exact name and returns its ID
    as JSON; used by inline admin form popups.
    """
    if request.accepts("application/json"):
        examination_name = request.GET.get("examination_name", "")
        examination_id = Examination.objects.get(name=examination_name).id
        data = {
            "examination_id": examination_id,
        }
        return HttpResponse(json.dumps(data), content_type="application/json")
    return HttpResponse("/")


# Generic view to dispatch to indivuals app views
@login_required
def generic_exam_json_edit(request):
    # GET: return a small fragment with an exam selector for the given type
    if request.method == "GET":
        question_type = request.GET.get("type")
        if not question_type:
            return JsonResponse({"status": "error", "message": "missing type"}, status=400)
        try:
            Exam = get_exam_model_from_app_name(question_type)
        except Exception:
            return JsonResponse({"status": "error", "message": "unknown type"}, status=400)

        # Limit to exams that are not archived and are exam_mode
        exams = Exam.objects.filter(archive=False, exam_mode=True).order_by("name")[:200]
        return render(request, "generic/partials/exam_select_fragment.html", {"exams": exams, "type": question_type})

    # Support POST additions via either an explicit JSON 'add_exam_questions' or
    # via a list of checkboxes named 'selection' sent by HTMX from the question list.
    if request.method == "POST":
        question_type = request.POST.get("type")

        if question_type == "rapids":
            Exam = RapidsExam
            Question = RapidQuestion
        elif question_type == "shorts":
            Exam = ShortsExam
            Question = ShortsQuestion
        elif question_type == "longs":
            Exam = LongsExam
            Question = LongQuestion
        elif question_type == "anatomy":
            Exam = AnatomyExam
            Question = AnatomyQuestion
        elif question_type == "physics":
            Exam = PhysicsExam
            Question = PhysicsQuestion
        elif question_type == "sbas":
            Exam = SbasExam
            Question = SbasQuestion
        else:
            data = {"status": "error"}
            return JsonResponse(data, status=400)

        exam = get_object_or_404(Exam, pk=request.POST.get("exam_id"))

        # if "exam_toggle_active" in request.POST:
        #    active = json.loads(request.POST.get("active"))

        #    data = {"status": "success"}
        #    return JsonResponse(data, status=200)

        if "add_exam_questions" in request.POST:
            question_ids = json.loads(request.POST.get("add_exam_questions"))
        else:
            # Support HTMX flow where checkboxes named 'selection' are posted
            # (multiple values). Convert to list of ints.
            sel = request.POST.getlist("selection") or request.POST.getlist("selection[]")
            question_ids = []
            for s in sel:
                try:
                    question_ids.append(int(s))
                except Exception:
                    continue
            # question_objects = Question.objects.filter(pk__in=question_ids)

            add_count = 0
            for i in question_ids:
                # We get an integrity error if we try to add an object that already exists
                if not exam.exam_questions.filter(pk=i).exists():
                    add_count += 1
                    exam.exam_questions.add(i)

            if add_count > 0:
                exam.save()
            data = {"status": "success", "questions added": add_count}
            return JsonResponse(data, status=200)

        if "set_exam_order" in request.POST:
            new_order = json.loads(request.POST.get("set_exam_order"))

            preserved = Case(
                *[When(pk=pk, then=pos) for pos, pk in enumerate(new_order)]
            )
            objects = Question.objects.filter(pk__in=new_order).order_by(preserved)

            if len(objects) != exam.exam_questions.count():
                data = {"status": "error, count does not match"}
                return JsonResponse(data, status=400)

            exam.exam_questions.set(objects)
            exam.save()
            data = {"status": "success"}
            return JsonResponse(data, status=200)

        # Only checkers can do the following
        if (
            question_type == "rapid"
            and not request.user.groups.filter(name="rapid_checker").exists()
        ):
            data = {"status": "error, permission denied"}
            return JsonResponse(data, status=400)
        if (
            question_type == "anatomy"
            and not request.user.groups.filter(name="anatomy_checker").exists()
        ):
            data = {"status": "error, permission denied"}
            return JsonResponse(data, status=400)
        if (
            question_type == "long"
            and not request.user.groups.filter(name="long_checker").exists()
        ):
            data = {"status": "error, permission denied"}
            return JsonResponse(data, status=400)

        if "set_open_access" in request.POST:
            if request.POST.get("set_open_access") == "true":
                state = True
            else:
                state = False

            questions_changed = []
            for q in exam.exam_questions.all():
                q.open_access = state
                q.save()
                questions_changed.append(q.pk)

            data = {"status": "success", "questions": questions_changed, "state": state}
            return JsonResponse(data, status=200)

        data = {"status": "error, unknown request"}
        return JsonResponse(data, status=400)
    else:
        data = {"status": "error"}
        return JsonResponse(data, status=400)


@login_required
def generic_exam_htmx(request):
    """HTMX-specific endpoint for selecting an exam and adding selected questions.

    GET: returns the exam selection fragment (same as earlier partial)
    POST: expects 'type' and checkbox list 'selection' (or selection[]) and 'exam_id'
          adds selected question ids to the chosen exam and returns a small HTML fragment
          or JSON suitable for HTMX swapping.
    """
    if request.method == "GET":
        question_type = request.GET.get("type")
        if not question_type:
            return HttpResponse("Missing type", status=400)
        try:
            Exam = get_exam_model_from_app_name(question_type)
        except Exception:
            return HttpResponse("Unknown type", status=400)

        # For non-superusers, only show exams they can edit or that they authored
        if request.user.is_superuser:
            exams = Exam.objects.filter(archive=False).order_by("name")
        else:
            # authors or markers or assigned exams
            exams = (
                Exam.objects.filter(author__id=request.user.id)
                | Exam.objects.filter(markers__id=request.user.id)
            )
            exams = exams.filter(archive=False).order_by("name")

        return render(request, "generic/partials/exam_select_fragment.html", {"exams": exams, "type": question_type})

    if request.method == "POST":
        logger.debug(f"generic_exam_htmx POST: {request.POST}")
        question_type = request.POST.get("type")
        if not question_type:
            return JsonResponse({"status": "error", "message": "missing type"}, status=400)

        try:
            Exam = get_exam_model_from_app_name(question_type)
        except Exception:
            return JsonResponse({"status": "error", "message": "unknown type"}, status=400)

        exam = get_object_or_404(Exam, pk=request.POST.get("exam_id"))

        # read checkbox selections
        sel = request.POST.getlist("selection") or request.POST.getlist("selection[]")
        question_ids = []
        for s in sel:
            try:
                question_ids.append(int(s))
            except Exception:
                continue

        add_count = 0
        for i in question_ids:
            if not exam.exam_questions.filter(pk=i).exists():
                add_count += 1
                exam.exam_questions.add(i)

        if add_count > 0:
            exam.save()

        # return a small HTMX-friendly fragment summarising the result
        return render(request, "generic/partials/exam_select_result.html", {"added": add_count, "exam": exam})

    return HttpResponse(status=405)


@login_required
def generic_exam_set_open_access(request):
    """HTMX endpoint to set open_access on questions in an exam.

    POST params:
      - type: app name (sbas, rapids, anatomy, physics, shorts, longs)
      - exam_id: id of the exam
      - set_open_access: 'true' or 'false'
      - selection / selection[]: optional list of question ids to apply to (if omitted apply to all)

    Only questions that request.user can edit (question.can_edit(user)) or superusers will be modified.
    Returns a small HTML fragment summarising the count of changed questions.
    """
    if request.method != "POST":
        return HttpResponse(status=405)

    question_type = request.POST.get("type")
    if not question_type:
        return JsonResponse({"status": "error", "message": "missing type"}, status=400)

    try:
        Exam = get_exam_model_from_app_name(question_type)
    except Exception:
        return JsonResponse({"status": "error", "message": "unknown type"}, status=400)

    exam = get_object_or_404(Exam, pk=request.POST.get("exam_id"))

    # permission to edit exam required
    # Many exam edits are limited to exam authors / checkers; reuse existing check where possible
    # For now require the user to have edit access (ExamViews.check_user_edit_access equivalent)
    # We'll do a lightweight check: if user is superuser OR user in exam.get_author_objects()
    if not (request.user.is_superuser or request.user in exam.get_author_objects()):
        return HttpResponse(mark_safe('<div class="alert alert-danger">Permission denied</div>'))

    set_val = True if request.POST.get("set_open_access") == "true" else False

    sel = request.POST.getlist("selection") or request.POST.getlist("selection[]")
    if sel:
        try:
            ids = [int(s) for s in sel]
        except Exception:
            ids = []
        qs = exam.exam_questions.filter(pk__in=ids)
    else:
        qs = exam.exam_questions.all()

    changed = 0
    for q in qs:
        try:
            # prefer per-question permission check if available
            can_edit = request.user.is_superuser or getattr(q, 'can_edit', lambda u: False)(request.user)
        except Exception:
            can_edit = request.user.is_superuser

        if not can_edit:
            continue

        if q.open_access != set_val:
            q.open_access = set_val
            q.save()
            changed += 1

    return render(request, "generic/partials/set_open_access_result.html", {"changed": changed, "exam": exam, "state": set_val})


class ExamViews(View, LoginRequiredMixin):
    def __init__(
        self,
        exam,
        question,
        answer,
        cid_user_answer,
        app,
        question_type,
        normalise_score=None,
    ):
        self.Exam: ExamBase = exam
        self.Question = question
        self.Answer = answer
        self.UserAnswer = cid_user_answer
        self.app_name = app
        self.question_type = question_type
        self.normalise_score = normalise_score

        # THis may be better than check_user_access below
        # group_map = {"rapids" : "rapid_checker", "anatomy" : "anatomy_checker", "longs":"long_checker"}
        # exam_group = group_map[self.app_name]

        class BasicExamFilter(FilterSet):
            sort_order = OrderingFilter(
                fields=(("name", "name"), ("exam_mode", "exam_mode"))
            )

            # Allow filtering by start/end date ranges (render as paired date inputs)
            start_date = DateFromToRangeFilter(
                field_name="start_date",
                label="Start date",
                widget=DateRangeWidget(attrs={"class": "datepicker", "autocomplete": "off", "placeholder": "dd/mm/yyyy"}),
            )
            end_date = DateFromToRangeFilter(
                field_name="end_date",
                label="End date",
                widget=DateRangeWidget(attrs={"class": "datepicker", "autocomplete": "off", "placeholder": "dd/mm/yyyy"}),
            )

            class Meta:
                model = exam
                fields = {
                    "name": ["contains"],
                    "exam_mode": ["exact"],
                    "active": ["exact"],
                    "archive": ["exact"],
                    "open_access": ["exact"],
                }

            def __init__(self, data=None, *args, **kwargs):
                # if filterset is bound, use initial values as defaults
                if data is not None:
                    # get a mutable copy of the QueryDict
                    data = data.copy()

                    # for name, f in self.base_filters.items():
                    # filter param is either missing or empty, use initial as default
                    if not data.get("archive"):
                        data["archive"] = False
                    if not data.get("sort_order"):
                        data["sort_order"] = "name"

                super().__init__(data, *args, **kwargs)

        self.BasicExamFilter = BasicExamFilter

        # We give some users the ability to filter by authors
        class ExtraExamFilter(BasicExamFilter):
            author = ModelMultipleChoiceFilter(
                queryset=User.objects.all(), null_label="No author"
            )

        self.ExtraExamFilter = ExtraExamFilter

    # TODO: these may be better implemented as decorators
    def check_user_access(self, user: User, exam_id: int = None, marker=False):
        """Check if a user should be able to access a view

        Args:
            user ([user]): user object
            exam_id ([int], optional): exam id. Defaults to None.

            marker ([boolean], optional): Allow access if the user is a marker. Defaults to False. Requires that exam_id is set.

        Returns:
            [boolean]: True if the user has access
        """
        logger.debug("check_user_access called for user={} exam_id={}", user, exam_id)
        if user.is_superuser:
            return True

        if marker and exam_id is None:
            # raise error if exam_id is not provided
            raise ValueError("exam_id must be provided if marker checking for a marker")

        if exam_id is not None:
            exam: ExamBase = get_object_or_404(self.Exam, pk=exam_id)
            if (exam.open_access and exam.active) or user in exam.get_author_objects():
                return True

            if (
                user.is_authenticated
                and exam.active
                and exam.cid_user_exam.filter(user_user=user)
            ):
                return True

            if marker and user in exam.markers.all():
                return True

            if exam.authors_only:
                return False

            if exam.open_access:
                return True

        if user.groups.filter(name="cid_user_manager").exists():
            return True

        if (
            self.app_name == "rapids"
            and user.groups.filter(name="rapid_checker").exists()
        ):
            return True
        if (
            self.app_name == "shorts"
            and user.groups.filter(name="shorts").exists()
        ):
            return True
        if (
            self.app_name == "anatomy"
            and user.groups.filter(
            name__in=("anatomy_checker", "anatomy_marker")
            ).exists()
        ):
            return True
        if (
            self.app_name == "longs"
            and user.groups.filter(
            name__in=("long_checker", "long_marker")
            ).exists()
        ):
            return True
        if (
            self.app_name == "physics"
            and user.groups.filter(name="physics_checker").exists()
        ):
            return True
        if (
            self.app_name == "sbas"
            and user.groups.filter(name="sba_checker").exists()
        ):
            return True
        if (
            self.app_name == "atlas"
            and user.groups.filter(name="casecollection_checker").exists()
        ):
            return True
        return False

    def check_user_marker_access(self, user: User, exam_id: int = None):
        return self.check_user_access(user, exam_id, marker=True)

    def check_user_edit_access(self, user, exam_id=None):
        """Check if a user should be able to access a view

        Args:
            user ([user]): user object

        Returns:
            [boolean]: True if the user has access
        """
        if user.is_superuser:
            return True

        # If a user is an exam author they should have acccess
        if exam_id is not None:
            exam = get_object_or_404(self.Exam, pk=exam_id)
            # Remove open_access check for the momement
            # if exam.open_access or user in exam.get_author_objects():
            #    return True
            if user in exam.get_author_objects():
                return True

            if exam.authors_only:
                return False

        if (
            self.app_name == "rapids"
            and not user.groups.filter(name="rapid_checker").exists()
        ):
            return False
        if (
            self.app_name == "anatomy"
            and not user.groups.filter(name="anatomy_checker").exists()
        ):
            return False
        if (
            self.app_name == "longs"
            and not user.groups.filter(name__in=("long_checker",)).exists()
        ):
            return False
        if (
            self.app_name == "physics"
            and not user.groups.filter(name="physics_checker").exists()
        ):
            return False
        if (
            self.app_name == "sbas"
            and not user.groups.filter(name="sba_checker").exists()
        ):
            return False
        if (
            self.app_name == "casecollection"
            and not user.groups.filter(name="casecollection_checker").exists()
        ):
            return False
        return False

    @method_decorator(login_required)
    def exam_toggle_archived(self, request, pk):
        if request.method == "POST":
            if not self.check_user_edit_access(request.user):
                data = {"status": "error, insufficient permission"}
                return JsonResponse(data, status=400)

            exam = get_object_or_404(self.Exam, pk=pk)

            # Support both legacy JSON POSTs and HTMX button toggles.
            # If this is an HTMX request, toggle the archive flag and
            # return the `exam-archived` partial for in-place replacement.
            is_htmx = request.headers.get("HX-Request") == "true"
            if is_htmx:
                exam.archive = not exam.archive
            else:
                # Legacy POST expects 'archive' == 'true'
                exam.archive = True if request.POST.get("archive") == "true" else False

            exam.save()

            if is_htmx:
                return render(
                    request,
                    "generic/partials/exams/exam_status.html#exam-archived",
                    {"exam": exam, "collection": exam},
                )

            data = {
                "status": "success",
                "archive": exam.archive,
                "name": str(exam),
                "id": exam.id,
            }
            return JsonResponse(data, status=200)
        else:
            data = {"status": "error"}
            return JsonResponse(data, status=400)

    @method_decorator(login_required)
    def exam_toggle_results_published_htmx(self, request, pk):  
        if request.method == "POST":
            if not self.check_user_edit_access(request.user, exam_id=pk):
                return HttpResponse(
                    mark_safe(
                        '<div class="alert alert-danger" role="alert">Error toggling results published.</div>'
                    )
                )

            exam = get_object_or_404(self.Exam, pk=pk)

            exam.publish_results = not exam.publish_results
            exam.save()
            return render(
                request, "generic/partials/exams/exam_status.html#publish-results", context={
                    "exam": exam,
                    "collection": exam
                })
        else:
            HttpResponse(
                mark_safe(
                    '<div class="alert alert-danger" role="alert">Error toggling results published.</div>'
                ) 
            )

    @method_decorator(login_required)
    def exam_toggle_active_htmx(self, request, pk):  
        if request.method == "POST":
            if not self.check_user_edit_access(request.user, exam_id=pk):
                return HttpResponse(
                    mark_safe(
                        '<div class="alert alert-danger" role="alert">Error toggling exam active.</div>'
                    )
                )

            exam = get_object_or_404(self.Exam, pk=pk)

            exam.active = not exam.active
            exam.save()
            return render(
                request, "generic/partials/exams/exam_status.html#exam-active", context={
                    "exam": exam,
                    "collection": exam
                })
        else:
            HttpResponse(
                mark_safe(
                    '<div class="alert alert-danger" role="alert">Error toggling exam active.</div>'
                ) 
            )

    @method_decorator(login_required)
    def exam_bulk_update(self, request):
        """HTMX endpoint to bulk-update exams (currently supports updating the date).

        Expects POST with 'exam_ids' (multiple) and optional 'date' (YYYY-MM-DD).
        Returns the updated exam list partial.
        """
        if request.method != "POST":
            return HttpResponse(status=405)

        exam_ids = request.POST.getlist("exam_ids")
        date_str = request.POST.get("date", "")

        # Parse date if provided
        new_date = None
        if date_str:
            try:
                from datetime import date as _date

                new_date = _date.fromisoformat(date_str)
            except Exception:
                new_date = None

        # Find the queryset for the current user as index() does
        if (
            request.user.is_superuser
            or (
                self.app_name == "rapids"
                and request.user.groups.filter(name="rapid_checker").exists()
            )
            or (
                self.app_name == "anatomy"
                and request.user.groups.filter(name="anatomy_checker").exists()
            )
            or (
                self.app_name == "longs"
                and request.user.groups.filter(name="long_checker").exists()
            )
        ):
            exams_qs = self.Exam.objects.all()
            filter = self.ExtraExamFilter(request.GET, queryset=exams_qs)
        else:
            exams_qs = self.Exam.objects.filter(author__id=request.user.id) | self.Exam.objects.filter(open_access=True)
            filter = self.BasicExamFilter(request.GET, queryset=exams_qs)

        # Apply changes
        for eid in exam_ids:
            try:
                exam = self.Exam.objects.get(pk=eid)
            except self.Exam.DoesNotExist:
                continue

            if not self.check_user_edit_access(request.user, exam_id=exam.pk):
                # Skip exams the user cannot edit
                continue

            if new_date is not None:
                exam.start_date = new_date
                exam.save()

        return render(request, "generic/partials/_exam_list.html", {"filter": filter, "app_name": self.app_name})

    @method_decorator(login_required)
    def exam_bulk_delete(self, request):
        """HTMX endpoint to bulk-delete selected exams. Expects POST 'exam_ids'.

        Returns the updated exam list partial.
        """
        if request.method != "POST":
            return HttpResponse(status=405)

        exam_ids = request.POST.getlist("exam_ids")

        if (
            request.user.is_superuser
            or (
                self.app_name == "rapids"
                and request.user.groups.filter(name="rapid_checker").exists()
            )
            or (
                self.app_name == "anatomy"
                and request.user.groups.filter(name="anatomy_checker").exists()
            )
            or (
                self.app_name == "longs"
                and request.user.groups.filter(name="long_checker").exists()
            )
        ):
            exams_qs = self.Exam.objects.all()
            filter = self.ExtraExamFilter(request.GET, queryset=exams_qs)
        else:
            exams_qs = self.Exam.objects.filter(author__id=request.user.id) | self.Exam.objects.filter(open_access=True)
            filter = self.BasicExamFilter(request.GET, queryset=exams_qs)

        for eid in exam_ids:
            try:
                exam = self.Exam.objects.get(pk=eid)
            except self.Exam.DoesNotExist:
                continue

            if not self.check_user_edit_access(request.user, exam_id=exam.pk):
                continue

            try:
                exam.delete()
            except Exception:
                # ignore individual delete failures
                continue

        return render(request, "generic/partials/_exam_list.html", {"filter": filter, "app_name": self.app_name})

    @method_decorator(login_required)
    def exam_bulk_archive(self, request):
        """HTMX endpoint to bulk-archive or unarchive selected exams.

        Expects POST with 'exam_ids' (multiple) and 'archive' ('true' or 'false').
        Returns the updated exam list partial.
        """
        if request.method != "POST":
            return HttpResponse(status=405)

        exam_ids = request.POST.getlist("exam_ids")
        archive_val = request.POST.get("archive", "true")
        set_archive = True if archive_val == "true" else False

        if (
            request.user.is_superuser
            or (
                self.app_name == "rapids"
                and request.user.groups.filter(name="rapid_checker").exists()
            )
            or (
                self.app_name == "anatomy"
                and request.user.groups.filter(name="anatomy_checker").exists()
            )
            or (
                self.app_name == "longs"
                and request.user.groups.filter(name="long_checker").exists()
            )
        ):
            exams_qs = self.Exam.objects.all()
            filter = self.ExtraExamFilter(request.GET, queryset=exams_qs)
        else:
            exams_qs = self.Exam.objects.filter(author__id=request.user.id) | self.Exam.objects.filter(open_access=True)
            filter = self.BasicExamFilter(request.GET, queryset=exams_qs)

        for eid in exam_ids:
            try:
                exam = self.Exam.objects.get(pk=eid)
            except self.Exam.DoesNotExist:
                continue

            if not self.check_user_edit_access(request.user, exam_id=exam.pk):
                continue

            try:
                exam.archive = set_archive
                exam.save()
            except Exception:
                continue

        return render(request, "generic/partials/_exam_list.html", {"filter": filter, "app_name": self.app_name})

    @method_decorator(login_required)
    def exam_toggle_results_published(self, request, pk):
        if request.method == "POST":
            if not self.check_user_edit_access(request.user, exam_id=pk):
                data = {"status": "error, insufficient permission"}
                return JsonResponse(data, status=400)

            exam = get_object_or_404(self.Exam, pk=pk)

            exam.publish_results = (
                True if request.POST.get("publish_results") == "true" else False
            )
            exam.save()
            data = {
                "status": "success",
                "publish_results": exam.publish_results,
                "name": str(exam),
                "id": exam.id,
            }
            return JsonResponse(data, status=200)
        else:
            data = {"status": "error"}
            return JsonResponse(data, status=400)

    @method_decorator(login_required)
    def exam_toggle_active(self, request, pk):
        if request.method == "POST":
            if not self.check_user_edit_access(request.user, exam_id=pk):
                data = {"status": "error, insufficient permission"}
                return JsonResponse(data, status=400)

            exam = get_object_or_404(self.Exam, pk=pk)

            exam.active = True if request.POST.get("active") == "true" else False
            exam.save()
            data = {
                "status": "success",
                "active": exam.active,
                "name": str(exam),
                "id": exam.id,
            }
            return JsonResponse(data, status=200)
        else:
            data = {"status": "error"}
            return JsonResponse(data, status=400)

    @method_decorator(login_required)
    def exam_review_start(self, request, pk):
        """Render an overview page for per-question review.

        This shows a compact summary for every question in the exam (response counts,
        percent answered / percent correct where applicable) and colour-codes each
        question so the reviewer can quickly triage items. From here the reviewer
        can jump to an individual question's review page or start the sequential
        review (existing behavior).
        """
        # Ensure the user can edit the exam (authors only)
        if not self.check_user_edit_access(request.user, exam_id=pk):
            raise PermissionDenied

        exam = get_object_or_404(self.Exam, pk=pk)

        # Materialise ordered questions
        questions = list(exam.get_questions())
        if not questions:
            return render(request, "generic/exam_review_complete.html", {"exam": exam, "app_name": self.app_name})

        # Build per-question summary metrics
        question_summaries = []

        # Candidate pool size (used to compute percent answered)
        candidate_total = 0
        try:
            candidate_total = (
                exam.valid_cid_users.count() + exam.valid_user_users.count()
            )
        except Exception:
            candidate_total = 0

        for i, q in enumerate(questions):
            try:
                ua_qs = q.cid_user_answers.filter(exam=exam)
            except Exception:
                # If relationship is different for some app types, try a fallback
                ua_qs = self.UserAnswer.objects.filter(question=q, exam=exam)

            total_responses = ua_qs.count()

            counts = {}
            # Some UserAnswer models (e.g. physics) do not have a single 'answer'
            # field. Try the simple values_list first; on failure fall back to
            # iterating objects and using get_answer() or composing from
            # per-field parts (a,b,c,d,e, or longs sections).
            try:
                for ans in ua_qs.values_list("answer", flat=True):
                    counts[ans] = counts.get(ans, 0) + 1
            except Exception:
                for ua in ua_qs:
                    ans_val = None
                    # Prefer model helper if available
                    try:
                        if hasattr(ua, "get_answer"):
                            ans_val = ua.get_answer()
                    except Exception:
                        ans_val = None

                    if not ans_val:
                        parts = []
                        # Common single-letter fields used by physics
                        for fld in ("a", "b", "c", "d", "e"):
                            if hasattr(ua, fld):
                                v = getattr(ua, fld)
                                if v is None:
                                    continue
                                if isinstance(v, (list, tuple)):
                                    parts.extend([str(x) for x in v])
                                else:
                                    parts.append(str(v))

                        # Long-form answer fields (longs)
                        for fld in (
                            "answer_observations",
                            "answer_interpretation",
                            "answer_principle_diagnosis",
                            "answer_differential_diagnosis",
                            "answer_management",
                        ):
                            if hasattr(ua, fld):
                                v = getattr(ua, fld)
                                if v:
                                    parts.append(str(v))

                        if parts:
                            ans_val = ", ".join(parts)

                    # Final fallback
                    if ans_val is None:
                        ans_val = ""

                    counts[ans_val] = counts.get(ans_val, 0) + 1

            if total_responses:
                pcts = {k: round(100.0 * v / total_responses, 1) for k, v in counts.items()}
            else:
                pcts = {k: 0.0 for k in counts.keys()}

            # Compute percent correct using per-answer scoring when available.
            # Different apps use different scoring scales (e.g. anatomy/rapids=2, longs=8,
            # physics/shorts=5, sbas=1). We determine the per-question maximum based on
            # the app and count user answers whose get_answer_score() equals that max.
            correct_pct = None
            correct_count = None
            try:
                # Determine per-question maximum score for this app
                if self.app_name in ("rapids", "anatomy"):
                    per_question_max = 2
                elif self.app_name == "longs":
                    per_question_max = 8
                elif self.app_name in ("physics", "shorts"):
                    per_question_max = 5
                else:
                    per_question_max = 1

                # If we have responses, iterate and count fully-correct answers using
                # the model's get_answer_score() which encapsulates per-app logic.
                if total_responses:
                    # Special-case physics: count each true/false sub-question individually.
                    if self.app_name == "physics":
                        # Determine number of sub-items per question
                        try:
                            n_sub = len(q.get_questions())
                        except Exception:
                            n_sub = None

                        total_subitems = 0
                        correct_subitems = 0
                        for ua in ua_qs:
                            try:
                                score = ua.get_answer_score()
                            except Exception:
                                continue
                            # score is expected to be iterable (tuple/list) for physics
                            if isinstance(score, (list, tuple)) and score:
                                # count numeric-typed positive/1 values as correct
                                for s in score:
                                    if isinstance(s, (int, float)) and s > 0:
                                        correct_subitems += 1
                                total_subitems += len(score)
                            else:
                                # fallback: treat scalar score as single subitem
                                if isinstance(score, (int, float)):
                                    if score > 0:
                                        correct_subitems += 1
                                    total_subitems += 1

                        # Compute percent over all subitems seen
                        correct_count = correct_subitems
                        if total_subitems:
                            correct_pct = round(100.0 * correct_subitems / total_subitems, 1)
                        else:
                            correct_pct = 0.0
                    else:
                        cnt = 0
                        for ua in ua_qs:
                            try:
                                score = ua.get_answer_score()
                            except Exception:
                                # If get_answer_score fails for any answer, skip it
                                continue
                            if score == per_question_max:
                                cnt += 1
                        correct_count = cnt
                        correct_pct = round(100.0 * correct_count / total_responses, 1) if total_responses else 0.0
                else:
                    correct_count = 0
                    correct_pct = 0.0
            except Exception:
                # Best-effort: fall back to None if something unexpected happens
                correct_pct = None
                correct_count = None

            # Percent answered relative to candidate pool (if known)
            answered_pct = None
            if candidate_total:
                try:
                    answered_pct = round(100.0 * total_responses / candidate_total, 1)
                except Exception:
                    answered_pct = None

            # Pick a simple colour classification: prefer percent-correct when available,
            # otherwise percent-answered.
            pct_for_colour = None
            if correct_pct is not None:
                pct_for_colour = correct_pct
            elif answered_pct is not None:
                pct_for_colour = answered_pct

            colour = "secondary"
            if pct_for_colour is not None:
                if pct_for_colour >= 75:
                    colour = "success"
                elif pct_for_colour >= 50:
                    colour = "warning"
                else:
                    colour = "danger"

            # Compute top answer (most common)
            top_answer = None
            top_count = 0
            for k, v in counts.items():
                if v > top_count:
                    top_count = v
                    top_answer = k
            # Build a simple breakdown list of (answer, count, pct) for template rendering
            breakdown = []
            for k, v in counts.items():
                pct_val = pcts.get(k, 0.0) if isinstance(pcts, dict) else 0.0
                breakdown.append((k, v, pct_val))

            # Determine the canonical correct answer representation for display.
            correct_answer = None
            try:
                if self.app_name == "sbas":
                    correct_answer = q.get_correct_answer()
                elif self.app_name == "physics":
                    correct_answer = q.get_answers()
                else:
                    # get_primary_answer is used elsewhere as the generic primary answer
                    try:
                        correct_answer = q.get_primary_answer()
                    except Exception:
                        correct_answer = getattr(q, "best_answer", None)

                # Normalise iterable answers into a readable string
                if isinstance(correct_answer, (list, tuple)):
                    try:
                        correct_answer = ", ".join([str(x) for x in correct_answer])
                    except Exception:
                        correct_answer = str(correct_answer)
                elif correct_answer is not None:
                    correct_answer = str(correct_answer)
            except Exception:
                correct_answer = None

            question_summaries.append(
                {
                    "question": q,
                    "q_index": i,
                    "total_responses": total_responses,
                    "counts": counts,
                    "pcts": pcts,
                    "correct_pct": correct_pct,
                    "correct_answer": correct_answer,
                    "answered_pct": answered_pct,
                    "colour": colour,
                    "top_answer": top_answer,
                    "breakdown": breakdown,
                }
            )

        return render(
            request,
            "generic/exam_review_start.html",
            {
                "exam": exam,
                "question_summaries": question_summaries,
                "question_number": len(questions),
                "app_name": self.app_name,
            },
        )

    @method_decorator(login_required)
    def exam_review_question(self, request, pk, q_index=0):
        """Render a single question from an exam for review with navigation."""
        if not self.check_user_edit_access(request.user, exam_id=pk):
            raise PermissionDenied

        exam = get_object_or_404(self.Exam, pk=pk)

        questions = list(exam.get_questions())
        total = len(questions)

        try:
            q_index = int(q_index)
        except Exception:
            q_index = 0

        if q_index < 0 or q_index >= total:
            # Out of range — render complete
            return render(request, "generic/exam_review_complete.html", {"exam": exam, "app_name": self.app_name})

        question = questions[q_index]

        prev_index = q_index - 1 if q_index > 0 else None
        next_index = q_index + 1 if q_index < total - 1 else None

        context = {
            "exam": exam,
            "question": question,
            "q_index": q_index,
            "total": total,
            "prev_index": prev_index,
            "next_index": next_index,
            "app_name": self.app_name,
            "can_edit": self.check_user_edit_access(request.user, exam_id=pk),
        }

        # Compute aggregated response summary for this question within the current exam.
        # Add counts and percentages to the context so fragments (e.g. SBAS) can display them.
        try:
            if hasattr(question, "cid_user_answers"):
                ua_qs = question.cid_user_answers.filter(exam=exam)
                total_responses = ua_qs.count()
                counts = {}
                try:
                    for ans in ua_qs.values_list("answer", flat=True):
                        counts[ans] = counts.get(ans, 0) + 1
                except Exception:
                    for ua in ua_qs:
                        ans_val = None
                        try:
                            if hasattr(ua, "get_answer"):
                                ans_val = ua.get_answer()
                        except Exception:
                            ans_val = None

                        if not ans_val:
                            parts = []
                            for fld in ("a", "b", "c", "d", "e"):
                                if hasattr(ua, fld):
                                    v = getattr(ua, fld)
                                    if v is None:
                                        continue
                                    if isinstance(v, (list, tuple)):
                                        parts.extend([str(x) for x in v])
                                    else:
                                        parts.append(str(v))

                            for fld in (
                                "answer_observations",
                                "answer_interpretation",
                                "answer_principle_diagnosis",
                                "answer_differential_diagnosis",
                                "answer_management",
                            ):
                                if hasattr(ua, fld):
                                    v = getattr(ua, fld)
                                    if v:
                                        parts.append(str(v))

                            if parts:
                                ans_val = ", ".join(parts)

                        if ans_val is None:
                            ans_val = ""

                        counts[ans_val] = counts.get(ans_val, 0) + 1

                # Compute percentages per answer key
                pcts = {}
                if total_responses:
                    for k, v in counts.items():
                        pcts[k] = round(100.0 * v / total_responses, 1)
                else:
                    for k in counts.keys():
                        pcts[k] = 0.0

                # Compute correct count using per-answer scoring when available.
                # Different apps use different scoring scales (e.g. anatomy/rapids=2,
                # longs=8, physics/shorts=5, sbas=1). Determine per-question max
                # and count answers where get_answer_score() equals that max.
                correct_count = None
                correct_pct = None
                try:
                    if self.app_name in ("rapids", "anatomy"):
                        per_question_max = 2
                    elif self.app_name == "longs":
                        per_question_max = 8
                    elif self.app_name in ("physics", "shorts"):
                        per_question_max = 5
                    else:
                        per_question_max = 1

                    if total_responses:
                        cnt = 0
                        for ua in ua_qs:
                            try:
                                score = ua.get_answer_score()
                            except Exception:
                                continue

                            # Handle tuple/list scores (physics) by summing parts
                            if isinstance(score, (list, tuple)):
                                try:
                                    ssum = sum([s for s in score if isinstance(s, (int, float))])
                                except Exception:
                                    continue
                                if ssum == per_question_max:
                                    cnt += 1
                            else:
                                # Numeric scores only; skip 'unmarked' / None
                                if isinstance(score, (int, float)) and score == per_question_max:
                                    cnt += 1

                        correct_count = cnt
                        correct_pct = round(100.0 * correct_count / total_responses, 1) if total_responses else 0.0
                    else:
                        correct_count = 0
                        correct_pct = 0.0
                except Exception:
                    correct_count = None
                    correct_pct = None

                context.update(
                    {
                        "exam_response_counts": counts,
                        "exam_response_pcts": pcts,
                        "exam_response_total": total_responses,
                        "exam_response_correct_count": correct_count,
                        "exam_response_correct_pct": correct_pct,
                    }
                )
        except Exception:
            # Best-effort: don't fail the review page if aggregation errors occur
            pass

        # Choose an app-specific fragment if present, otherwise fall back to
        # the generic fragment. Expose the chosen fragment name in the
        # context so the full-page render can include it; return the fragment
        # directly for HTMX requests.
        from django.template import loader

        fragment_candidates = [
            f"{self.app_name}/partials/exam_review_question_fragment.html",
            "generic/partials/exam_review_question_fragment.html",
        ]
        template = loader.select_template(fragment_candidates)
        fragment_template_name = template.template.name
        context["fragment_template"] = fragment_template_name

        # Detect HTMX: prefer the request.htmx attribute when available, otherwise
        # check the HX-Request header.
        is_htmx = getattr(request, "htmx", False)
        if not is_htmx:
            try:
                is_htmx = request.headers.get("HX-Request", "").lower() == "true"
            except Exception:
                is_htmx = request.META.get("HTTP_HX_REQUEST", "") == "true"

        if is_htmx:
            return render(request, fragment_template_name, context)

        return render(request, "generic/exam_review_question.html", context)

    @method_decorator(login_required)
    def exam_json_recreate(self, request, pk):
        exam = get_object_or_404(self.Exam, pk=pk)

        exam.recreate_json = True
        exam.save()

        # exam.get_exam_json()

        # TODO: check if mememory leak?
        t = threading.Thread(target=self.exam_json, args=(request, pk))
        t.setDaemon(True)
        t.start()

        return redirect("{}:exam_overview".format(self.app_name), pk=pk)

    @method_decorator(login_required)
    def question_json_recreate(self, request, pk):
        question = get_object_or_404(self.Question, pk=pk)

        question.recreate_json = True
        question.save()

        return redirect("{}:question_detail".format(self.app_name), pk=pk)

    @method_decorator(login_required)
    def exam_list_all(self, request):
        return self.exam_list(request, all=True)

    def order_questions(self, request, exam_id):
        if not self.check_user_edit_access(request.user, exam_id=exam_id):
            return HttpResponse("Permission denied")

        exam = get_object_or_404(self.Exam, pk=exam_id)

        exam.order_questions()

        return HttpResponse("Done")

    @method_decorator(login_required)
    def exam_list_collection(self, request, collection_id):
        collection = get_object_or_404(ExamCollection, pk=collection_id)

        if not collection.is_author(request.user):
            raise PermissionDenied

        return self.exam_list(request, all=True, collection=collection)


    @method_decorator(login_required)
    def exam_list(self, request, all=False, collection=None):
        # Build a single queryset and annotate candidate counts to avoid per-item queries
        if collection is None:
            if not self.check_user_access(request.user):
                # Use a single queryset with Q to include authored or marked exams
                exam_list = (
                    self.Exam.objects.filter(
                        (Q(author__id=request.user.id) | Q(markers__id=request.user.id)),
                        exam_mode=True,
                    )
                    .distinct()
                    .order_by("name")
                )
            else:
                exam_list = self.Exam.objects.filter(exam_mode=True).order_by("name")
        else:
            exam_list = (
                self.Exam.objects.filter(exam_mode=True, examcollection__in=[collection])
                .order_by("name")
            )

        # Annotate counts for template usage so `.count()` is not called per-object.
        # Use unfiltered counts to match the per-exam `exam_overview` counts.
        exam_list = exam_list.annotate(
            valid_cid_users_count=Count("valid_cid_users", distinct=True),
            valid_user_users_count=Count("valid_user_users", distinct=True),
        ).select_related("examcollection").prefetch_related("author")

        if not all:
            exam_list = exam_list.filter(archive=False).order_by("name")

        marking = False

        if self.app_name in ("rapids", "anatomy", "longs"):
            marking = True

        return render(
            request,
            "exam_list.html".format(self.app_name),
            {
                "exams": exam_list,
                "app_name": self.app_name,
                "view_all": all,
                "marking": marking,
                "collection": collection,
            },
        )

    @method_decorator(login_required)
    def exam_user(self, request, exam_id, user_id):
        exam = get_object_or_404(
            self.Exam.objects.prefetch_related("author"), pk=exam_id
        )

        if request.user not in exam.author.all():
            if not self.check_user_access(request.user, exam_id):
                raise PermissionDenied

        user = User.objects.get(pk=user_id)
        user_exam = exam.get_or_create_cid_user_exam(user_user=user)

        return JsonResponse(model_to_dict(user_exam))

    @method_decorator(login_required)
    def exam_report_email_status(self, request, exam_id):
        exam = get_object_or_404(
            self.Exam.objects.prefetch_related("author"), pk=exam_id
        )

        if request.user not in exam.author.all():
            if not self.check_user_access(request.user, exam_id):
                raise PermissionDenied

        users = exam.get_user_users_with_scores()

        status = {}
        for user in users:
            # user = User.objects.get(pk=u)
            user_exam = exam.get_or_create_cid_user_exam(user_user=user)
            if user_exam.results_emailed_status:
                status[user] = json.loads(user_exam.results_emailed_status)

        return render(
            request,
            "generic/exam_report_email_status.html",
            {
                "exam": exam,
                "status": status,
                "app_name": self.app_name,
                "users": users,
            },
        )
        return JsonResponse(status)

    @method_decorator(login_required)
    def exam_report_email_unsent(self, request, exam_id):
        return self.exam_report_email(request, exam_id, unsent_only=True)

    @method_decorator(login_required)
    def exam_report_email(self, request, exam_id, unsent_only=False, users=None):
        exam = get_object_or_404(
            self.Exam.objects.prefetch_related("author"), pk=exam_id
        )

        if request.user not in exam.author.all():
            if not self.check_user_access(request.user, exam_id):
                raise PermissionDenied

        additional_email = False
        if request.htmx.prompt is not None:
            additional_email = request.htmx.prompt

        logger.debug("exam_send_email: additional_email={}", additional_email)

        # check addition email is valid
        if additional_email:
            try:
                validate_email(additional_email)
            except ValidationError:
                return JsonResponse("Invalid additional email")

        # We only need to send emails to those who have scores
        # (who should be in exam.user_scores)
        if users is None:
            users = exam.get_user_users_with_scores()

        time = timezone.now()

        email_results = {}
        for user in users:
            # user = User.objects.get(pk=u)
            user_exam = exam.get_or_create_cid_user_exam(user_user=user)

            if unsent_only and user_exam.results_emailed_status:
                if json.loads(user_exam.results_emailed_status)[0] == True:
                    continue

            if additional_email:
                email_results[user] = exam.email_user_results(
                    user, additional_emails=[additional_email]
                )
            else:
                email_results[user] = exam.email_user_results(user)
            email_results[user].append(f"{time}")
            user_exam.results_emailed_status = json.dumps(email_results[user])

            user_exam.save()

        exam.exam_results_emailed = time
        exam.save()

        return render(
            request,
            "generic/exam_report_email_status.html",
            {
                "exam": exam,
                "status": email_results,
                "app_name": self.app_name,
            },
        )

        return JsonResponse(email_results)

    @method_decorator(login_required)
    def exam_user_report_email(self, request, exam_id, user_id):
        return self.exam_report_email(request, exam_id, users=[user_id])
        # print(Exam.objects.all())
        # exams = Exam.objects.all()
        # print("test", Exam.objects.all().get(id=pk))
        # exam = get_object_or_404(self.Exam.objects.prefetch_related("author"), pk=exam_id)

        # u = get_object_or_404(User, pk=user_id)

        # if request.user not in exam.author.all():
        #    if not self.check_user_access(request.user, exam_id):
        #        raise PermissionDenied

        # res = exam.email_user_results(u)

        # return HttpResponse(res)

    @method_decorator(login_required)
    def exam_user_report(self, request, exam_id, user_id):
        # print(Exam.objects.all())
        # exams = Exam.objects.all()
        # print("test", Exam.objects.all().get(id=pk))
        exam = get_object_or_404(
            self.Exam.objects.prefetch_related("author"), pk=exam_id
        )

        u = get_object_or_404(User, pk=user_id)

        if request.user not in exam.author.all():
            if not self.check_user_access(request.user, exam_id):
                raise PermissionDenied

        res = exam.generate_user_report(u)
        logger.debug("generate_user_report result for exam={}: {}", exam_id, res)

        return HttpResponse(res)

    @method_decorator(login_required)
    def exam_overview(self, request, pk):
        # print(Exam.objects.all())
        # exams = Exam.objects.all()
        # print("test", Exam.objects.all().get(id=pk))
        exam = get_object_or_404(self.Exam.objects.prefetch_related("author"), pk=pk)

        if request.user not in exam.author.all():
            # We also let markers view the exam
            if not self.check_user_access(request.user, pk, marker=True):
                raise PermissionDenied

        can_edit = (
            self.check_user_edit_access(request.user, pk) | request.user.is_superuser
        )

        notes = []
        if can_edit:
            notes = self.exam_notes(request, pk)

        if self.app_name == "anatomy":
            questions = (
                exam.exam_questions.select_related(
                    "modality",
                    "structure",
                    "body_part",
                    "region",
                    "examination",
                    "question_type",
                )
                .all()
                .prefetch_related(
                    Prefetch(
                        "answers",
                        queryset=self.Answer.objects.filter(
                            proposed=False, status=self.Answer.MarkOptions.CORRECT
                        ),
                        to_attr="prefetched_primary_answer",
                        # queryset=self.Answer.objects.filter(),
                    ),
                )
                .order_by("examquestiondetail__sort_order")
            )
        elif self.app_name == "longs":
            questions = (
                exam.exam_questions.prefetch_related(
                    # Prefetch(
                    #    "series",
                    #    queryset=LongSeries.objects.select_related(
                    #        "modality", "examination", "plane", "contrast"
                    #    ).prefetch_related(Prefetch("images")),
                    # ),
                    "author",
                    # "examquestiondetail",
                    # "longsseries",
                    "series",
                    # "seriesdetail",
                    "series__images",
                    "series__plane",
                    "series__examination",
                    # to_attr="prefetched_primary_answer"
                    # queryset=self.Answer.objects.filter(),
                    # "series",
                    # "abnormality",
                    # "region",
                    # "examination",
                )
                .all()
                .order_by("examquestiondetail__sort_order")
            )
        elif self.app_name == "rapids":
            questions = (
                exam.exam_questions.select_related()
                .all()
                .prefetch_related(
                    Prefetch(
                        "answers",
                        queryset=self.Answer.objects.filter(
                            proposed=False, status=self.Answer.MarkOptions.CORRECT
                        ),
                        to_attr="prefetched_primary_answer",
                        # queryset=self.Answer.objects.filter(),
                    ),
                    "images",
                    "abnormality",
                    "region",
                    "examination",
                    "author",
                )
                .order_by("examquestiondetail__sort_order")
            )
            # questions = (
            #    exam.exam_questions.select_related()
            #    .all()
            #    .prefetch_related("images", "abnormality", "region", "examination")
            # )
        elif self.app_name == "physics":
            questions = (
                exam.exam_questions.select_related("category")
                .all()
                .order_by("examquestiondetail__sort_order")
                # .prefetch_related("images", "abnormality", "region", "examination")
            )
        else:
            questions = (
                exam.exam_questions.select_related()
                .all()
                .order_by("examquestiondetail__sort_order")
            )

        question_number = len(questions)

        cid_candidates = exam.valid_cid_users.count()
        users_candidates = exam.valid_user_users.count()

        # question_orders = [q.examquestiondetail.sort_order for q in questions]

        # Exam order can be missing when, this can be checked with
        # exam.exam_questions.select_related().all().order_by("examquestiondetail__sort_order").values_list("examquestiondetail__sort_order")
        # TODO decide if / how this should be flagged to a user (updating teh exam order will fix)

        return render(
            request,
            "{}/exam_overview.html".format(self.app_name),
            {
                "exam": exam,
                "questions": questions,
                "question_number": question_number,
                "can_edit": can_edit,
                "notes": notes,
                "app_name": self.app_name,
                "candidate_count": (cid_candidates, users_candidates),
            },
        )



    def exam_cids(self, request, exam_id):
        """View that displays an overview of users that have been added to the exam.

        Args:
            request (_type_): _description_
            exam_id (_type_): _description_

        Raises:
            PermissionDenied: _description_

        Returns:
            _type_: _description_
        """
        exam = get_object_or_404(self.Exam, pk=exam_id)

        # if not request.user.groups.filter(name="cid_user_manager").exists():
        #    raise PermissionDenied

        if request.user not in exam.author.all():
            if not request.user.is_superuser:
                raise PermissionDenied

        # Avoid loading large querysets here — the detailed lists are
        # rendered by HTMX partials which will fetch the data separately.
        cid_user_count = exam.valid_cid_users.count()
        user_user_count = exam.valid_user_users.count()

        # Provide a lightweight indicator if there are any submissions so the
        # template can decide whether to load the submitted-list partial.
        has_submissions = exam.cid_users.exists()

        context = {
            "exam": exam,
            "cid_user_count": cid_user_count,
            "user_user_count": user_user_count,
            "has_submissions": has_submissions,
            "app_name": self.app_name,
        }

        if self.app_name == "atlas":
            context["collection"] = exam

        return render(request, "exam_cids.html", context)

    def exam_cids_cid_list(self, request, exam_id):
        """HTMX partial returning the CID candidates list for an exam."""
        exam = get_object_or_404(self.Exam, pk=exam_id)
        if request.user not in exam.author.all() and not request.user.is_superuser:
            raise PermissionDenied

        cid_users = exam.valid_cid_users.all()

        return render(request, "generic/partials/exam_cids_cid_list.html", {"cid_users": cid_users, "exam": exam})

    def exam_cids_user_list(self, request, exam_id):
        """HTMX partial returning the User candidates list for an exam."""
        exam = get_object_or_404(self.Exam, pk=exam_id)
        if request.user not in exam.author.all() and not request.user.is_superuser:
            raise PermissionDenied

        user_users = exam.valid_user_users.all()

        return render(request, "generic/partials/exam_cids_user_list.html", {"user_users": user_users, "exam": exam})

    def exam_cids_submitted_list(self, request, exam_id):
        """HTMX partial returning the submitted candidates summary."""
        exam = get_object_or_404(self.Exam, pk=exam_id)
        if request.user not in exam.author.all() and not request.user.is_superuser:
            raise PermissionDenied

        # Explicitly query the CidUserExam entries for this exam to avoid any
        # ambiguity from GenericRelation managers and to ensure we only return
        # submissions that belong to this exam instance.
        from django.contrib.contenttypes.models import ContentType

        ct = ContentType.objects.get_for_model(self.Exam)
        user_exam_data = (
            CidUserExam.objects.filter(content_type=ct, object_id=exam.pk)
            .select_related("cid_user", "user_user")
            .order_by("start_time")
        )

        # Determine which submitted entries correspond to candidates/users
        # that are actually added to the exam so we can highlight those that
        # are not. Collect valid IDs once to avoid per-row queries.
        valid_cid_pks = set(exam.valid_cid_users.values_list("pk", flat=True))
        valid_user_pks = set(exam.valid_user_users.values_list("pk", flat=True))

        for ue in user_exam_data:
            in_exam = False
            try:
                if ue.cid_user_id and ue.cid_user_id in valid_cid_pks:
                    in_exam = True
                if ue.user_user_id and ue.user_user_id in valid_user_pks:
                    in_exam = True
            except Exception:
                in_exam = False

            # Attach a dynamic attribute the template can read
            setattr(ue, "in_exam", in_exam)

        return render(
            request,
            "generic/partials/exam_cids_submitted_list.html",
            {"user_exam_data": user_exam_data, "exam": exam},
        )

    # def exam_groups_edit(self, request, exam_id):
    #    exam = get_object_or_404(self.Exam, pk=exam_id)

    #    if not request.user.groups.filter(name="cid_user_manager").exists():
    #        # raise PermissionDenied
    #        if request.user not in exam.author.all():
    #            raise PermissionDenied

    #

    #    context = {
    #        "exam": exam,
    #        #"groups": exam_groups,
    #    }
    #    return render(request, "exam_groups_edit.html", context)

    def exam_users_edit(self, request, exam_id):
        exam = get_object_or_404(self.Exam, pk=exam_id)

        if not request.user.groups.filter(name="cid_user_manager").exists():
            # raise PermissionDenied
            if request.user not in exam.author.all():
                raise PermissionDenied

        current_user_users = exam.valid_user_users.all()

        exam_groups = exam.user_user_groups.all()

        available_user_users = list(
            User.objects.filter(user_groups__in=exam_groups).difference(
                current_user_users
            )
        )
        logger.debug("available_user_users count: {}", len(available_user_users))
        # available_user_users = User.objects.all()

        context = {
            "exam": exam,
            "groups": exam_groups,
            "current_user_users": current_user_users,
            "available_user_users": available_user_users,
            "app_name": self.app_name,
        }

        # Provide a searchable widget to allow adding arbitrary users
        try:
            from generic.widgets import UserSearchSelectMultipleWidget

            widget_html = UserSearchSelectMultipleWidget("add_users", []).render()
            context["user_search_widget"] = mark_safe(widget_html)
        except Exception:
            context["user_search_widget"] = ""

        if self.app_name == "atlas":
            context["collection"] = exam

        return render(request, "exam_users_edit.html", context)

    def exam_reset_answers(self, request, exam_id):
        if request.htmx:
            exam = get_object_or_404(self.Exam, pk=exam_id)

            if not self.check_user_edit_access(request.user, exam_id):
                raise PermissionDenied

            ## Delete all answers
            exam.cid_user_answers.all().delete()

            # User statuses
            exam.exam_user_status.all().delete()

            # CidUserExams
            exam.cid_users.all().delete()

            return HttpResponse("<b>Answers reset</b>")

        else:
            raise Http404("Invalid request")

    def exam_cids_edit(self, request, exam_id):
        exam = get_object_or_404(self.Exam, pk=exam_id)

        if not request.user.groups.filter(name="cid_user_manager").exists():
            # raise PermissionDenied
            if request.user not in exam.author.all():
                raise PermissionDenied

        current_cid_users = exam.valid_cid_users.all()

        exam_groups = exam.cid_user_groups.all()

        available_cid_users = CidUser.objects.filter(group__in=exam_groups).difference(
            current_cid_users
        )

        context = {
            "exam": exam,
            "groups": exam_groups,
            "current_cid_users": current_cid_users,
            "available_cid_users": available_cid_users,
            "app_name": self.app_name,
        }

        if self.app_name == "atlas":
            context["collection"] = exam

        return render(request, "exam_cids_edit.html", context)

    # @method_decorator(login_required)
    def exam_notes(self, request, pk):
        exam = get_object_or_404(self.Exam, pk=pk)

        if request.user not in exam.author.all():
            if not self.check_user_access(request.user, pk):
                raise PermissionDenied

        q, content_type = get_question_and_content_type(self.question_type)

        question_pks = exam.exam_questions.values_list("pk", flat=True)

        # Get active notes for type
        notes = QuestionNote.objects.filter(
            content_type=content_type, object_id__in=question_pks, complete=False
        )

        return notes

    def exam_user_status_cid(self, request, pk, cid):
        return self.exam_user_status(request, pk, cid=cid)

    def exam_user_status_user(self, request, pk, user_id):
        return self.exam_user_status(request, pk, user_id=user_id)

    def exam_user_status(self, request, pk, cid=None, user_id=None):
        exam = get_object_or_404(self.Exam, pk=pk)

        # Restrict just to exam authors
        if request.user not in exam.author.all():
            raise PermissionDenied

        # Robustly parse parameters from GET query string
        get_cid = request.GET.get("cid")
        get_user_id = request.GET.get("user_id")

        if get_cid not in (None, "", "None"):
            try:
                cid = int(get_cid)
            except ValueError:
                pass
        if get_user_id not in (None, "", "None"):
            try:
                user_id = int(get_user_id)
            except ValueError:
                pass

        if cid is not None:
            statuses = exam.exam_user_status.filter(cid_user_exam__cid_user__cid=cid).order_by("datetime")
        elif user_id is not None:
            statuses = exam.exam_user_status.filter(
                cid_user_exam__user_user__id=user_id
            ).order_by("datetime")
        else:
            statuses = exam.exam_user_status.all().order_by("datetime")

        return render(
            request, "exam_user_status.html", {"exam": exam, "statuses": statuses}
        )

        # if not self.check_user_access(request.user, pk):
        #    raise PermissionDenied

        # q, content_type = get_question_and_content_type(self.question_type)

        ## Get active notes for type
        # statuses = ExamUserStatus.objects.filter(
        #    content_type=content_type, object_id__in=pk
        # )

        # return statuses

    @method_decorator(login_required)
    def exam_json_edit(self, request, pk):
        if request.method == "POST":
            if request.user.groups.filter(name="cid_user_manager").exists():
                # This may give more permissions than we actually want
                pass

            elif not self.check_user_edit_access(request.user, exam_id=pk):
                data = {"status": "invalid permisions"}
                return JsonResponse(data, status=403)

            exam = get_object_or_404(self.Exam, pk=pk)

            if "edit_cid_user" in request.POST:
                user_id = request.POST.get("edit_cid_user")

                add = request.POST.get("add") == "true"

                cid_user = CidUser.objects.get(pk=user_id)

                app_exam_map = {}
                app_exam_map["rapids"] = cid_user.rapid_exams
                app_exam_map["shorts"] = cid_user.shorts_exams
                app_exam_map["anatomy"] = cid_user.anatomy_exams
                app_exam_map["longs"] = cid_user.longs_exams
                app_exam_map["physics"] = cid_user.physics_exams
                app_exam_map["sbas"] = cid_user.sba_exams
                app_exam_map["atlas"] = cid_user.casecollection_exams

                if add:
                    app_exam_map[self.app_name].add(pk)
                else:
                    app_exam_map[self.app_name].remove(pk)

                # Always return the rendered list-item HTML so HTMX (or our fetch fallback)
                # can swap the updated `<li>` into the DOM. Also provide an `HX-Trigger`
                # header so client-side toasts/handlers can respond.
                html = render_to_string(
                    'generic/partials/cid_user_li.html',
                    {'cid': cid_user, 'exam': exam, 'current': add},
                    request=request,
                )
                response = HttpResponse(html, content_type='text/html')
                response['HX-Trigger'] = json.dumps({'cid_toggled': {'pk': cid_user.pk, 'added': add}})
                return response

            if "edit_user_user" in request.POST:
                user_id = request.POST.get("edit_user_user")

                add = request.POST.get("add") == "true"

                user_user = User.objects.get(pk=user_id)

                app_exam_map = {}
                app_exam_map["rapids"] = user_user.user_rapid_exams
                app_exam_map["shorts"] = user_user.user_shorts_exams
                app_exam_map["anatomy"] = user_user.user_anatomy_exams
                app_exam_map["longs"] = user_user.user_longs_exams
                app_exam_map["physics"] = user_user.user_physics_exams
                app_exam_map["sbas"] = user_user.user_sba_exams
                app_exam_map["atlas"] = user_user.user_casecollection_exams

                if add:
                    app_exam_map[self.app_name].add(pk)
                else:
                    app_exam_map[self.app_name].remove(pk)

                # Return rendered list-item HTML for HTMX (or fetch fallback) and an HX-Trigger
                html = render_to_string(
                    'generic/partials/user_user_li.html',
                    {'user': user_user, 'exam': exam, 'current': add},
                    request=request,
                )
                response = HttpResponse(html, content_type='text/html')
                response['HX-Trigger'] = json.dumps({'user_toggled': {'pk': user_user.pk, 'added': add}})
                return response

            if "add_exam_questions" in request.POST:
                question_ids = json.loads(request.POST.get("add_exam_questions"))
                question_objects = self.Question.objects.filter(pk__in=question_ids)

                exam.exam_questions.add(question_objects)
                exam.save()
                data = {"status": "success"}
                return JsonResponse(data, status=200)

            if "set_exam_order" in request.POST:
                new_order = json.loads(request.POST.get("set_exam_order"))

                preserved = Case(
                    *[When(pk=pk, then=pos) for pos, pk in enumerate(new_order)]
                )
                objects = self.Question.objects.filter(pk__in=new_order).order_by(
                    preserved
                )

                # We now allow deleting exam questions
                # if len(objects) != exam.exam_questions.count():
                #    data = {"status": "error, count does not match"}
                #    return JsonResponse(data, status=400)

                if self.app_name == "atlas":
                    exam.cases.set(objects)
                    for n, case in enumerate(objects):
                        case_detail = CaseDetail.objects.get(case=case, collection=exam)
                        case_detail.sort_order = n
                        case_detail.save()

                else:
                    exam.exam_questions.set(objects)
                    for n, question in enumerate(objects):
                        match self.app_name:
                            case "anatomy":
                                question_detail = AnatomyExamQuestionDetail.objects.get(
                                    anatomyquestion=question, exam=exam
                                )
                            case "rapids":
                                question_detail = RapidsExamQuestionDetail.objects.get(
                                    rapid=question, exam=exam
                                )
                            case "shorts":
                                question_detail = ShortsExamQuestionDetail.objects.get(
                                    question=question, exam=exam
                                )
                            case "longs":
                                question_detail = LongsExamQuestionDetail.objects.get(
                                    question=question, exam=exam
                                )
                            case "physics":
                                question_detail = PhysicsExamQuestionDetail.objects.get(
                                    question=question, exam=exam
                                )
                            case "sbas":
                                question_detail = SbasExamQuestionDetail.objects.get(
                                    question=question, exam=exam
                                )
                            case _:
                                data = {"status": "error"}
                                return JsonResponse(data, status=200)
                        question_detail.sort_order = n
                        question_detail.save()

                exam.save()
                data = {"status": "success"}
                return JsonResponse(data, status=200)

            if not self.check_user_access(request.user, pk):
                raise PermissionDenied

            if "set_open_access" in request.POST:
                if request.POST.get("set_open_access") == "true":
                    state = True
                else:
                    state = False

                questions_changed = []
                for q in exam.exam_questions.all():
                    q.open_access = state
                    q.save()
                    questions_changed.append(q.pk)

                data = {
                    "status": "success",
                    "questions": questions_changed,
                    "state": state,
                }
                return JsonResponse(data, status=200)

            data = {"status": "error, unknown request"}
            return JsonResponse(data, status=400)
        else:
            data = {"status": "error"}
            return JsonResponse(data, status=400)

    @method_decorator(login_required)
    def exam_cid_bulk_edit(self, request, pk):
        """Bulk add/remove CidUsers to/from an exam. Expects POST with 'bulk_pks' (JSON list)
        and 'add' ('true'/'false'). Returns a rendered list partial for HTMX swaps.
        """
        if request.method != 'POST':
            return JsonResponse({'status': 'error, invalid method'}, status=400)

        if request.user.groups.filter(name="cid_user_manager").exists():
            pass
        elif not self.check_user_edit_access(request.user, exam_id=pk):
            data = {"status": "invalid permisions"}
            return JsonResponse(data, status=403)

        exam = get_object_or_404(self.Exam, pk=pk)

        bulk_pks_raw = request.POST.get('bulk_pks') or request.POST.get('bulk_pks[]')
        items = []
        if bulk_pks_raw:
            try:
                items = json.loads(bulk_pks_raw)
            except Exception:
                # attempt to parse comma separated
                try:
                    items = [int(x) for x in bulk_pks_raw.split(',') if x.strip()]
                except Exception:
                    items = []
        else:
            sel = request.POST.getlist('selection') or request.POST.getlist('selection[]')
            for s in sel:
                try:
                    items.append(int(s))
                except Exception:
                    continue

        # 'add' may be provided for uniform bulk operations; otherwise items may be a list of {pk, add}
        add_flag = None
        if 'add' in request.POST:
            add_flag = request.POST.get('add') == 'true'

        changed = []
        for entry in items:
            # entry may be an int PK or a dict {pk, add}
            if isinstance(entry, dict):
                uid = entry.get('pk')
                this_add = bool(entry.get('add'))
            else:
                uid = entry
                this_add = add_flag if add_flag is not None else True

            try:
                cid_user = CidUser.objects.get(pk=uid)
            except Exception:
                continue

            app_exam_map = {}
            app_exam_map["rapids"] = cid_user.rapid_exams
            app_exam_map["shorts"] = cid_user.shorts_exams
            app_exam_map["anatomy"] = cid_user.anatomy_exams
            app_exam_map["longs"] = cid_user.longs_exams
            app_exam_map["physics"] = cid_user.physics_exams
            app_exam_map["sbas"] = cid_user.sba_exams
            app_exam_map["atlas"] = cid_user.casecollection_exams

            try:
                if this_add:
                    app_exam_map[self.app_name].add(pk)
                else:
                    app_exam_map[self.app_name].remove(pk)
                changed.append(cid_user.pk)
            except Exception:
                continue

        # Rebuild lists for rendering
        current_cid_users = exam.valid_cid_users.all()
        exam_groups = exam.cid_user_groups.all()
        available_cid_users = CidUser.objects.filter(group__in=exam_groups).difference(current_cid_users)

        html = render_to_string('generic/partials/cid_user_list.html', {
            'exam': exam,
            'current_cid_users': current_cid_users,
            'available_cid_users': available_cid_users,
        }, request=request)

        response = HttpResponse(html, content_type='text/html')
        # add_flag may be None for mixed add/remove operations
        response['HX-Trigger'] = json.dumps({'cid_bulk_toggled': {'changed': changed, 'add': add_flag, 'mixed': add_flag is None}})
        return response

    @method_decorator(login_required)
    def exam_user_bulk_edit(self, request, pk=None, exam_id=None):
        """Bulk add/remove User objects to/from an exam. Expects POST with 'bulk_pks' (JSON list)
        and 'add' ('true'/'false'). Returns a rendered user list partial for HTMX swaps.
        """
        # Support both `pk` (generic urls) and `exam_id` (app-specific urls like atlas)
        exam_pk = pk if pk is not None else exam_id

        if request.method != 'POST':
            return JsonResponse({'status': 'error, invalid method'}, status=400)

        if request.user.groups.filter(name="cid_user_manager").exists():
            pass
        elif not self.check_user_edit_access(request.user, exam_id=exam_pk):
            data = {"status": "invalid permisions"}
            return JsonResponse(data, status=403)

        exam = get_object_or_404(self.Exam, pk=exam_pk)

        bulk_pks_raw = request.POST.get('bulk_pks') or request.POST.get('bulk_pks[]')
        items = []
        if bulk_pks_raw:
            try:
                items = json.loads(bulk_pks_raw)
            except Exception:
                try:
                    items = [int(x) for x in bulk_pks_raw.split(',') if x.strip()]
                except Exception:
                    items = []
        else:
            sel = request.POST.getlist('selection') or request.POST.getlist('selection[]')
            for s in sel:
                try:
                    items.append(int(s))
                except Exception:
                    continue

        add_flag = None
        if 'add' in request.POST:
            add_flag = request.POST.get('add') == 'true'

        changed = []
        for entry in items:
            if isinstance(entry, dict):
                uid = entry.get('pk')
                this_add = bool(entry.get('add'))
            else:
                uid = entry
                this_add = add_flag if add_flag is not None else True

            try:
                user_obj = User.objects.get(pk=uid)
            except Exception:
                continue

            app_exam_map = {}
            app_exam_map["rapids"] = user_obj.user_rapid_exams
            app_exam_map["shorts"] = user_obj.user_shorts_exams
            app_exam_map["anatomy"] = user_obj.user_anatomy_exams
            app_exam_map["longs"] = user_obj.user_longs_exams
            app_exam_map["physics"] = user_obj.user_physics_exams
            app_exam_map["sbas"] = user_obj.user_sba_exams
            app_exam_map["atlas"] = user_obj.user_casecollection_exams

            try:
                if this_add:
                    app_exam_map[self.app_name].add(exam_pk)
                else:
                    app_exam_map[self.app_name].remove(exam_pk)
                changed.append(user_obj.pk)
            except Exception:
                continue

        # Rebuild lists for rendering
        current_user_users = exam.valid_user_users.all()
        exam_groups = exam.user_user_groups.all()
        available_user_users = User.objects.filter(user_groups__in=exam_groups).difference(current_user_users)

        html = render_to_string('generic/partials/user_user_list.html', {
            'exam': exam,
            'current_user_users': current_user_users,
            'available_user_users': available_user_users,
        }, request=request)

        response = HttpResponse(html, content_type='text/html')
        response['HX-Trigger'] = json.dumps({'user_bulk_toggled': {'changed': changed, 'add': add_flag, 'mixed': add_flag is None}})
        return response

    @method_decorator(login_required)
    def mark_overview(self, request, pk):
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        if not exam.exam_mode:
            raise Http404("Packet not in exam mode")

        if request.user not in exam.author.all():
            if not self.check_user_marker_access(request.user, pk):
                raise PermissionDenied

        if self.app_name == "anatomy":
            questions = (
                exam.exam_questions.select_related(
                    "modality",
                    "structure",
                    "body_part",
                    "region",
                    "examination",
                    "question_type",
                )
                .all()
                .prefetch_related(
                    # "cid_user_answers",
                    Prefetch(
                        "answers",
                        queryset=self.Answer.objects.filter(
                            proposed=False, status=self.Answer.MarkOptions.CORRECT
                        ),
                        to_attr="prefetched_primary_answer",
                        # queryset=self.Answer.objects.filter(),
                    ),
                    # Prefetch(
                    #   "cid_user_answers",
                    #   queryset=self.UserAnswer.objects.filter(score=self.Answer.MarkOptions.UNMARKED, exam__id=pk),
                    #   to_attr="prefetched_unmarked_cid_answers"
                    #   #queryset=self.Answer.objects.filter(),
                    # ),
                )
                .order_by("examquestiondetail__sort_order")
            )
        else:
            questions = exam.get_questions()

        # Handle exams that require double marking
        try:
            if exam.double_mark:
                question_unmarked_map = []
                for q in questions:
                    question_unmarked_map.append(
                        (
                            q,
                            int(q.get_unmarked_user_answer_count(exam_pk=exam.pk)),
                            int(
                                q.get_unmarked_user_answer_count(
                                    exam_pk=exam.pk, marker=request.user
                                )
                            ),
                        )
                    )

                return render(
                    request,
                    "{}/mark_overview.html".format(self.app_name),
                    {"exam": exam, "question_unmarked_map": question_unmarked_map},
                )
        except AttributeError:
            pass

        question_unmarked_map = []
        for q in questions:
            count = int(q.get_unmarked_user_answer_count(exam_pk=exam.pk))
            question_unmarked_map.append((q, count, count))

        return render(
            request,
            "{}/mark_overview.html".format(self.app_name),
            {"exam": exam, "question_unmarked_map": question_unmarked_map},
        )

    @method_decorator(login_required)
    def index(self, request):
        # self.check_user_access(request.user, exam.pk)

        if (
            request.user.is_superuser
            or (
                self.app_name == "rapids"
                and request.user.groups.filter(name="rapid_checker").exists()
            )
            or (
                self.app_name == "anatomy"
                and request.user.groups.filter(name="anatomy_checker").exists()
            )
            or (
                self.app_name == "longs"
                and request.user.groups.filter(name="long_checker").exists()
            )
        ):
            exams = self.Exam.objects.all()
            filter = self.ExtraExamFilter(request.GET, queryset=exams)
        else:
            exams = self.Exam.objects.filter(
                author__id=request.user.id
            ) | self.Exam.objects.filter(open_access=True)
            filter = self.BasicExamFilter(request.GET, queryset=exams)

        return render(
            request,
            "generic/frcr_index.html",
            {"filter": filter, "app_name": self.app_name},
        )

    @method_decorator(login_required)
    def exam_question_user_answer(
        self, request, pk: int, sk: int, c_or_u: str, user_or_cid: str
    ):
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        if c_or_u == "u":
            user = get_object_or_404(User, pk=user_or_cid)
            answer = exam.get_question_user_user_answer(sk, user)
        elif c_or_u == "c":
            # cid_user = CidUser.objects.filter(cid=user_or_cid)
            answer = exam.get_question_cid_user_answer(sk, user_or_cid)
        else:
            raise Http404

        logger.debug("exam_question_user_answer result: {}", answer)
        return HttpResponse(answer)

    @method_decorator(login_required)
    def exam_question_cid_user_answer(self, request, pk: int, sk: int, cid: str):
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        cid_user = CidUser.objects.filter(cid=cid)
        answer = exam.get_question_cid_user_answer(sk, cid_user)

        logger.debug("exam_question_cid_user_answer result: {}", answer)

    @method_decorator(login_required)
    def exam_question_detail(self, request, pk, sk):
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        if request.user not in exam.author.all():
            if not self.check_user_access(request.user, pk, marker=True):
                raise PermissionDenied

        # question = exam.get_questions()[sk]
        question = exam.get_question_by_index(sk)

        exam_length = len(exam.exam_questions.all())

        # pos = exam.get_question_index(question) + 1
        pos = sk + 1

        previous = -1
        if sk > 0:
            previous = sk - 1
        next = sk + 1
        if sk == exam_length - 1:
            next = False

        view_feedback = False
        if (
            # request.user.groups.filter(name=self.checker_group).exists() or
            request.user.groups.filter(name="feedback_checker").exists()
            or request.user in question.author.all()
        ):
            view_feedback = True

        can_edit = question.can_edit(request.user)

        return render(
            request,
            "{}/question_detail.html".format(self.app_name),
            {
                "question": question,
                "exam": exam,
                "next": next,
                "previous": previous,
                "exam_length": exam_length,
                "pos": pos,
                "view_feedback": view_feedback,
                "can_edit": can_edit,
                "remote_url": settings.REMOTE_URL,
                "app_name": self.app_name,
            },
        )

    def active_exams(
        self,
        request: HttpRequest,
        json: bool = True,
        based: bool = True,
        cid: int | None = None,
        passcode: str | None = None,
    ):
        exams = self.Exam.objects.filter(archive=False)

        active_exams = {"exams": []}

        if cid is not None:
            cid_user = CidUser.objects.filter(cid=cid).first()
            if not cid_user or cid_user.passcode != passcode:
                if json == False:
                    return active_exams["exams"]
                return JsonResponse({"status": "invalid"}, status=401)

        exam: ExamBase
        for exam in exams:
            if exam.active or self.check_user_access(request.user, exam.pk):
                if exam.exam_mode and not exam.check_cid_user(
                    cid, passcode, user=request.user, user_id=request.user.pk
                ):
                    continue

                if exam.json_creation_time:
                    creation_time = exam.json_creation_time.isoformat()
                else:
                    creation_time = "None"

                url = request.build_absolute_uri(
                    exam.get_json_url(cid=cid, passcode=passcode)
                )
                # hacky
                if not based:
                    url = url + "/unbased"

                obj = {
                    "name": exam.get_exam_name(),
                    "url": url,
                    "type": self.question_type,
                    "eid": "{}/{}".format(self.question_type, exam.pk),
                    "json_creation_time": creation_time,
                    "exam_json_id": exam.exam_json_id,
                    "exam_active": exam.active,
                    "exam_mode": exam.exam_mode,
                }

                if self.question_type == "long":
                    h = {}
                    for question in exam.exam_questions.all():
                        # Generate json if needed
                        if question.json_creation_time is None:
                            question.get_question_json()
                        h[question.pk] = question.question_json_id
                    obj["multi_question_json"] = h
                active_exams["exams"].append(obj)

        if json is False:
            return active_exams["exams"]

        return JsonResponse(active_exams)

    @method_decorator(csrf_exempt)
    def post_exam_answers(self, request):
        if request.method == "POST":
            root_cid = request.POST.get("cid")
            if not root_cid:
                return JsonResponse({"success": False, "error": "cid not defined"})

            if request.user.is_authenticated:
                expected_cid = f"u-{request.user.id}"
                if root_cid != expected_cid:
                    return JsonResponse({"success": False, "error": "user / cid mismatch"})
            else:
                if root_cid.startswith("u-"):
                    return JsonResponse(
                        {"success": False, "error": "authentication required for user submissions"}
                    )

            n = 0
            for answer in json.loads(request.POST.get("answers")):
                logger.debug("Processing answer: {}", answer)

                eid = int(answer["eid"].split("/")[1])
                uid = False
                passcode = None

                ans_cid = answer.get("cid")
                if not ans_cid:
                    return JsonResponse({"success": False, "error": "answer cid not defined"})

                if request.user.is_authenticated:
                    if ans_cid != f"u-{request.user.id}":
                        return JsonResponse({"success": False, "error": "user / uid mismatch"})
                    uid = request.user.id
                    cid = False
                    passcode = None
                else:
                    if str(ans_cid).startswith("u-"):
                        return JsonResponse(
                            {"success": False, "error": "authentication required"}
                        )
                    try:
                        cid = int(ans_cid)
                        passcode = answer["passcode"]
                        uid = False
                    except (ValueError, KeyError):
                        return JsonResponse(
                            {"success": False, "error": "invalid cid or passcode"}
                        )

                if not uid:
                    # As access is not restricted make sure the data appears valid
                    if (not isinstance(cid, int)) or (
                        not isinstance(eid, int) or (not isinstance(answer["ans"], str))
                    ):
                        return JsonResponse(
                            {
                                "success": False,
                                "error": "cid or eid or answers not defined",
                            }
                        )

                exam: ExamBase = get_object_or_404(self.Exam, pk=eid)

                if not exam.check_cid_user(cid, passcode, user_id=uid):
                    return JsonResponse(
                        {"success": False, "error": "invalid cid / passcode"}
                    )

                if uid:
                    existing_answers = self.UserAnswer.objects.filter(
                        question__id=answer["qid"], exam__id=eid, user__id=uid
                    )
                else:
                    existing_answers = self.UserAnswer.objects.filter(
                        question__id=answer["qid"], exam__id=eid, cid=cid
                    )
                posted_answer = answer["ans"]

                match self.app_name:
                    case "rapids":
                        # Normal answers are just posted with the answer "Normal"
                        normal = False
                        if posted_answer == "Normal":
                            normal = True

                        if not existing_answers:
                            if uid:
                                ans = self.UserAnswer(
                                    answer=posted_answer,
                                    normal=normal,
                                    user=User.objects.get(id=uid),
                                )
                            else:
                                ans = self.UserAnswer(
                                    answer=posted_answer, normal=normal, cid=cid
                                )

                            ans.question_id = answer["qid"]
                            ans.exam_id = eid
                        else:
                            # Update an existing answer
                            # should never be more than one (famous last words)
                            ans = existing_answers[0]
                            if answer["qidn"] == "1":
                                ans.normal = normal
                                # Only wipe the answer text if the new answer is "Normal"
                                if normal:
                                    ans.answer = ""
                            elif answer["qidn"] == "2" and not ans.normal:
                                ans.answer = posted_answer
                    case "shorts":
                        if not existing_answers:
                            if uid:
                                ans = self.UserAnswer(
                                    answer=posted_answer, user=User.objects.get(id=uid)
                                )
                            else:
                                ans = self.UserAnswer(answer=posted_answer, cid=cid)

                            ans.question_id = answer["qid"]
                            ans.exam_id = eid
                            ans.score = None
                        else:
                            # Update an existing answer
                            # should never be more than one (famous last words)
                            ans = existing_answers[0]
                            ans.answer = posted_answer
                            ans.score = None
                    case "anatomy":
                        if not existing_answers:
                            if uid:
                                ans = self.UserAnswer(
                                    answer=posted_answer, user=User.objects.get(id=uid)
                                )
                            else:
                                ans = self.UserAnswer(answer=posted_answer, cid=cid)

                            ans.question_id = answer["qid"]
                            ans.exam_id = eid
                            ans.score = self.Answer.MarkOptions.UNMARKED
                        else:
                            # Update an existing answer
                            # should never be more than one (famous last words)
                            ans = existing_answers[0]
                            ans.answer = posted_answer
                            ans.score = self.Answer.MarkOptions.UNMARKED
                    case "longs":
                        # Long cases seperate sections by qidn
                        qidn = answer["qidn"]

                        # If the user answer does not exist
                        if not existing_answers:
                            if uid:
                                ans = self.UserAnswer(user=User.objects.get(id=uid))
                            else:
                                ans = self.UserAnswer(cid=cid)
                            ans.question_id = answer["qid"]
                            ans.exam_id = eid

                        # If the answer already exists or we have started populating it
                        else:
                            # Update an existing answer
                            # should never be more than one (famous last words)
                            ans = existing_answers[0]

                        if qidn == "1":
                            ans.answer_observations = posted_answer
                        elif qidn == "2":
                            ans.answer_interpretation = posted_answer
                        elif qidn == "3":
                            ans.answer_principle_diagnosis = posted_answer
                        elif qidn == "4":
                            ans.answer_differential_diagnosis = posted_answer
                        elif qidn == "5":
                            ans.answer_management = posted_answer

                        # Mark as unmarked
                        ans.score = ans.ScoreOptions.UNMARKED

                time_answered_str = answer.get("time_answered")
                if time_answered_str:
                    from django.utils.dateparse import parse_datetime
                    parsed_time = parse_datetime(time_answered_str)
                    if parsed_time:
                        if timezone.is_naive(parsed_time):
                            parsed_time = timezone.make_aware(parsed_time)
                        ans.time_answered = parsed_time

                ans.full_clean()
                ans.save()

                # if ret is not True:
                #    return ret

                n = n + 1

                # print(UserAnswer.objects.filter(exam__id=q["eid"]))
            # print(request.urlencode())
            exam_type, exam_id = request.POST.get("eid").split("/")

            exam = self.Exam.objects.get(pk=exam_id)
            t = timezone.make_aware(
                datetime.fromtimestamp(float(request.POST.get("start_time")))
            )

            # We currently store the submission time in two different places
            if request.POST.get("cid").startswith("u-"):
                cid_user_exam = exam.get_or_create_cid_user_exam(
                    user_user=request.user, start_time=t
                )
            else:
                c = CidUser.objects.filter(cid=request.POST.get("cid")).first()
                cid_user_exam = exam.get_or_create_cid_user_exam(
                    cid_user=c, start_time=t
                )
            in_progress = request.POST.get("in_progress") == "true"
            if in_progress:
                exam.exam_user_status.create(
                    cid_user_exam=cid_user_exam,
                    status="autosaved",
                    extra="autosaved {} answers".format(n),
                )
            else:
                exam.exam_user_status.create(
                    cid_user_exam=cid_user_exam,
                    status="submitted",
                    extra="manual submission",
                )

                cid_user_exam.end_time = timezone.now()
                cid_user_exam.save()

            return JsonResponse({"success": True, "question_count": n})
        return JsonResponse({"success": False, "error": "Invalid data"})

    def exam_json_cid(self, request, pk, cid, passcode):
        return self.exam_json(request, pk, cid, passcode)

    def exam_json_cid_unbased(self, request, pk, cid, passcode):
        return self.exam_json_unbased(request, pk, cid, passcode)

    def exam_json(self, request, pk, cid=None, passcode=None):
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        # Check access (for logged in users when inactive)
        if not exam.active and not self.check_user_access(request.user, pk):
            raise Http404("No available exam")

        # TODO: check access for logged in users
        logger.debug("exam_question_json: cid={}", cid)

        # Check access for users
        if request.user.is_anonymous:
            user_id = None
        else:
            user_id = request.user.pk
        if exam.exam_mode and not exam.check_cid_user(
            cid, passcode, request.user, user_id
        ):
            raise Http404("No available exam")

        # exam_json_cache = cache.get("{}_exam_json_{}".format(self.app_name, pk))

        # Log exam access
        if exam.exam_mode:
            cid_user_exam = exam.get_or_create_cid_user_exam(
                cid=cid, user_user=request.user
            )
            exam.exam_user_status.create(
                cid_user_exam=cid_user_exam, status="downloaded"
            )

        path = "{0}{1}/exam/{2}.json".format(settings.MEDIA_ROOT, self.app_name, pk)
        url = "{0}{1}/exam/{2}.json".format(settings.MEDIA_URL, self.app_name, pk)

        Path(path).parent.mkdir(parents=True, exist_ok=True)

        if os.path.isfile(path) and not exam.recreate_json:
            # exam_json_cache["cached"] = True
            # Make sure we pass on an argument if we are forcing a reload
            if request.GET.get("_"):
                query_string = urllib.parse.urlencode({"_": request.GET.get("_")})
                url = "{}?{}".format(url, query_string)
                return redirect(url)
            return redirect(url)
            return JsonResponse(exam_json_cache)

        time = timezone.now()

        exam.exam_json_id += 1

        with open(path, "w+") as f:
            exam_json = exam.get_exam_json()
            exam_json["generated"] = time.isoformat()
            exam_json["exam_json_id"] = exam.exam_json_id
            f.write(json.dumps(exam_json))

        exam.recreate_json = False

        exam.json_creation_time = time
        exam.save(recreate_json=False)

        # We also try to clear the cache of any associated questions (longs)
        # see exam_question_json
        # n = exam.exam_questions.count()
        # question_keys = ["{}_exam_json_{}_{}".format(self.app_name, pk, i) for i in range(1, n)]
        # cache.delete_many(question_keys)

        # cache.set("{}_exam_json_{}".format(self.app_name, pk), exam_json, 3600)

        return JsonResponse(exam_json)

    def exam_json_unbased(self, request, pk, cid=None, passcode=None):
        """
        No (file based) caching is enabled for unbased exams
        """
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        if (
            exam.exam_mode
            and not exam.active
            and not self.check_user_access(request.user, pk)
        ):
            raise Http404("No available exam")

        # Check access for users
        if request.user.is_anonymous:
            user_id = None
        else:
            user_id = request.user.pk
        if not exam.check_cid_user(cid, passcode, request.user, user_id):
            raise Http404("No available exam")

        time = timezone.now()

        exam_json = exam.get_exam_json(based=False)
        exam_json["generated"] = time.isoformat()
        exam_json["exam_json_id"] = exam.exam_json_id
        # exam_json["exam_active"] = False

        # Log exam access
        if exam.exam_mode:
            cid_user_exam = exam.get_or_create_cid_user_exam(
                cid=cid, user_user=request.user
            )
            exam.exam_user_status.create(
                cid_user_exam=cid_user_exam, status="downloaded", extra="unbased"
            )

        return JsonResponse(exam_json)

    def exam_question_json_cid(self, request, pk, sk, cid, passcode):
        return self.exam_question_json(request, pk, sk, cid, passcode)

    def exam_question_json(self, request, pk, sk, cid=None, passcode=None):
        question = get_object_or_404(self.Question, pk=sk)
        exam = get_object_or_404(self.Exam, pk=pk)

        if not exam.active and not self.check_user_access(request.user, pk):
            raise Http404("No available exam")

        # Check access for users
        if request.user.is_anonymous:
            user_id = None
        else:
            user_id = request.user.pk
        if not exam.check_cid_user(cid, passcode, request.user, user_id):
            raise Http404("No available exam")

        if request.GET.get("_"):
            base_url = redirect("{}:question_json".format(self.app_name), pk=sk)
            query_string = urllib.parse.urlencode({"_": request.GET.get("_")})
            url = "{}?{}".format(base_url, query_string)
            return redirect(url)

        return redirect("{}:question_json".format(self.app_name), pk=sk)

    def exam_question_json_unbased_cid(self, request, pk, sk, cid, passcode):
        return self.exam_question_json_unbased(request, pk, sk, cid, passcode)

    def exam_question_json_unbased(self, request, pk, sk, cid=None, passcode=None):
        question = get_object_or_404(self.Question, pk=sk)
        exam = get_object_or_404(self.Exam, pk=pk)

        logger.debug("exam_question_json_unbased: cid={}", cid)

        if not exam.active and not self.check_user_access(request.user, pk):
            raise Http404("No available exam")

        # Check access for users
        if request.user.is_anonymous:
            user_id = None
        else:
            user_id = request.user.pk

        if not exam.check_cid_user(cid, passcode, request.user, user_id):
            raise Http404("No available exam")

        return redirect("{}:question_json_unbased".format(self.app_name), pk=sk)
        redirect()

        question_json_cache = cache.get("{}_question_json_{}".format(self.app_name, sk))

        if question_json_cache is not None and not question.recreate_json:
            question_json_cache["cached"] = True
            return JsonResponse(question_json_cache)

        question_json = exam.get_exam_question_json(sk)

        cache.set("{}_question_json_{}".format(self.app_name, sk), question_json, 3600)

        return JsonResponse(question_json)

    @method_decorator(login_required)
    def exam_scores_user(self, request, pk):
        return self.exam_scores_cid_user(request, pk)

    @method_decorator(user_passes_test(lambda u: u.is_superuser))
    def exam_scores_user_admin(self, request, pk, user_id):
        user = User.objects.get(id=user_id)
        return self.exam_scores_cid_user(request, pk, user=user)

    def exam_scores_user_supervisor(self, request, pk, user_id):
        from django.contrib.contenttypes.models import ContentType
        from generic.models import CidUserExam

        user = User.objects.get(id=user_id)
        if not request.user.is_superuser:
            try:
                if not request.user.supervisor == user.userprofile.supervisor:
                    raise PermissionDenied
            except Exception:
                raise PermissionDenied

        exam = get_object_or_404(self.Exam, pk=pk)
        ct = ContentType.objects.get_for_model(self.Exam)
        cid_user_exam = CidUserExam.objects.filter(
            content_type=ct,
            object_id=exam.pk,
            user_user=user
        ).first()

        is_visible = False
        if exam.results_supervisor_visible:
            is_visible = True
        elif cid_user_exam and cid_user_exam.share_with_supervisor:
            is_visible = True

        if not is_visible:
            raise PermissionDenied("This exam attempt has not been shared with the supervisor.")

        return self.exam_scores_cid_user(
            request, pk, user=user, supervisor=user.userprofile.supervisor.user
        )

    @method_decorator(user_passes_test(lambda u: u.is_superuser))
    def exam_scores_cid_user_admin(self, request, pk, cid):
        user = CidUser.objects.get(cid=cid)
        return self.exam_scores_cid_user(request, pk, cid, user.passcode)

    def exam_scores_cid_user(
        self, request, pk, cid=None, passcode="", user=None, supervisor=None
    ):
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        if not exam.exam_mode:
            raise Http404("Packet not in exam mode")

        view_all_results = False
        if request.user.groups.filter(name="view_all_results").exists():
            view_all_results = True

        if supervisor is None:
            if not exam.check_cid_user(cid, passcode, request.user):
                # TODO clear this up
                # Should users be able to access results for exams they do not have access to (if they have results)?
                if not self.check_user_access(request.user, pk):
                    # raise PermissionDenied
                    # if request.user.supervisor == user.userprofile.supervisor:
                    #    pass

                    # check for supervisor access
                    raise Http404("Error accessing exam")
        else:
            if user is None:
                raise Exception("user must be defined")
            view_all_results = True

        if user is None:
            user = request.user

        # Load questions; prefetch 'answers' for apps that use them to avoid repeated DB hits
        questions_qs = exam.get_questions()
        if self.app_name == "anatomy":
            questions_qs = questions_qs.prefetch_related("answers")

        # Materialize questions list for iteration
        questions = list(questions_qs)

        # Fetch all relevant UserAnswer objects up-front to avoid per-question queries
        if cid is not None:
            user_answers_qs = self.UserAnswer.objects.filter(cid=cid, exam__id=pk).select_related("question")
        else:
            user_answers_qs = self.UserAnswer.objects.filter(user=user, exam__id=pk).select_related("question")

        if self.app_name in ("anatomy", "rapids"):
            user_answers_qs = user_answers_qs.prefetch_related("question__answers")

        # Map question_id -> UserAnswer for O(1) lookup
        user_answers_map = {ua.question_id: ua for ua in user_answers_qs}

        answers_and_marks = []
        answers_marks = []
        answers = []

        for q in questions:
            # Get user answer from the in-memory map (avoids N+1 queries)
            user_answer = user_answers_map.get(q.pk)
            # user_answer = cid_user_answers_q_map[q]

            feedback = None
            if not user_answer or user_answer is None:
                # skip if no answer
                score, text = q.get_unanswered_mark_and_text()
                # answers_marks.append(score)
                # answers.append("")
                answer_score = score
                ans = text
            else:
                ans = user_answer.get_answer()
                answer_score = user_answer.get_answer_score()

                if self.app_name in ("longs", "shorts"):
                    feedback = user_answer.candidate_feedback

            match self.app_name:
                case "sbas":
                    correct_answer = q.get_correct_answer()

                    if not exam.publish_results and not view_all_results:
                        correct_answer = "*****"
                        answer_score = 0
                    answers.append(ans)
                    answers_marks.append(answer_score)

                    merged_ans = (ans, answer_score, correct_answer)
                    chosen_answer = q.get_answer_by_choice(ans)
                    answers_and_marks.append((q, *merged_ans, chosen_answer))
                case "physics":
                    correct_answer = q.get_answers()

                    if not exam.publish_results and not view_all_results:
                        correct_answer = ("*****", "*****", "*****", "*****", "*****")
                        answer_score = (0, 0, 0, 0, 0)
                    answers.append(ans)
                    answers_marks.append(answer_score)

                    logger.debug("physics answer row: questions={} ans={} score={} correct={}", q.get_questions(), ans, answer_score, correct_answer)
                    merged_ans = zip(
                        q.get_questions(), ans, answer_score, correct_answer
                    )
                    answers_and_marks.append((q, merged_ans))
                case _:
                    correct_answer = q.get_primary_answer()

                    if not exam.publish_results and not view_all_results:
                        correct_answer = "*****"
                        answer_score = 0
                    logger.debug(f"{ans=} {answer_score=} {correct_answer=}")
                    answers.append(ans)
                    answers_marks.append(answer_score)

                    if self.app_name in ("longs", "shorts"):
                        answers_and_marks.append(
                            (ans, answer_score, correct_answer, feedback)
                        )
                    else:
                        answers_and_marks.append((ans, answer_score, correct_answer))

        match self.app_name:
            case "physics":
                total_score = sum(sum(i) for i in answers_marks)
            case _:
                if "unmarked" in answers_marks or None in answers_marks:
                    answered = [i for i in answers_marks if type(i) == int]
                    total_score = sum(answered)
                    unmarked_number = len(answers_marks) - len(answered)
                    total_score = "{} ({} unmarked)".format(
                        total_score, unmarked_number
                    )
                else:
                    total_score = sum(answers_marks)

        max_score = self.get_max_score(questions)

        #cid_user_exam = exam.cid_user_exam.filter(user_user=user).first()
        #print(user)
        #print(f"{cid_user_exam=}")

        template_context = {
            "exam": exam,
            "cid": cid,
            "passcode": passcode,
            "questions": questions,
            "answers": answers,
            "answers_marks": answers_marks,
            "total_score": total_score,
            "max_score": max_score,
            "answers_and_marks": answers_and_marks,
            "user_answers_map": user_answers_map,
            "view_all_results": view_all_results,
            "supervisor": supervisor,
            #"cid_user_exam": cid_user_exam,
        }

        template_context["normalised_score"] = None
        if self.normalise_score is not None:
            normalised_score = self.normalise_score(total_score)
            template_context["normalised_score"] = normalised_score

        return render(
            request,
            f"{self.app_name}/exam_scores_user.html",
            template_context,
        )

    def get_max_score(self, questions):
        match self.app_name:
            case "rapids" | "anatomy":
                max_score = len(questions) * 2
            case "longs":
                max_score = len(questions) * 8
            case "physics" | "shorts":
                max_score = len(questions) * 5
            case _:
                max_score = len(questions)
        return max_score

    def exam_scores_all(self, request, pk):
        """The exam scores pages. Displays all user scores in a tabular format."""
        exam: ExamBase = get_object_or_404(self.Exam, pk=pk)

        if not exam.exam_mode:
            raise Http404("Packet not in exam mode")

        # Restrict access to the scores page just to exam authors
        if request.user not in exam.author.all():
            raise PermissionDenied

        table_only = request.GET.get("table_only") == "true" or request.headers.get("HX-Target") == "answers-table-container"
        stats_only = request.GET.get("stats_only") == "true" or request.headers.get("HX-Target") == "stats-container"

        user_answers_marks = defaultdict(list)
        by_question = defaultdict(dict)
        unmarked = set()
        cached_scores = False

        valid_cid_users = set(exam.valid_cid_users.all().values_list("cid", flat=True))
        valid_user_users = set(exam.valid_user_users.all().values_list("pk", flat=True))

        if self.app_name in ("rapids",):
            user_answers_callstates = defaultdict(list)
            user_answers_callstates_counted = {}

        if self.app_name in ("physics", "sbas", "longs", "shorts"):
            questions_qs = exam.get_questions()
        else:
            questions_qs = exam.get_questions().prefetch_related("answers")

        # Materialize questions into a list so we can index and build a question->index map
        questions = list(questions_qs)
        question_index_map = {q: i for i, q in enumerate(questions)}

        # Fetch and prefetch the UserAnswers
        cid_user_answers_qs = (
            self.UserAnswer.objects.select_related("question", "user")
            .filter(question__in=questions, exam__id=pk)
        )
        if self.app_name in ("anatomy", "rapids"):
            cid_user_answers_qs = cid_user_answers_qs.prefetch_related("question__answers")

        cid_user_answers = list(cid_user_answers_qs)
        question_number = len(questions)

        cids = set()
        plain_cids = set()
        user_ids = set()
        cids_user_id_map = {}

        # Loop through all candidates
        for cid_user_answer in cid_user_answers:
            if cid_user_answer.user is None:
                cid = f"c/{cid_user_answer.cid}"
                cids_user_id_map[cid] = cid
                cids.add(cid)
                plain_cids.add(cid_user_answer.cid)
            else:
                cid = f"u/{cid_user_answer.user.pk}"
                name = f"{cid_user_answer.user.first_name} {cid_user_answer.user.last_name}"
                if not name.strip():
                    name = cid_user_answer.user.username
                cids_user_id_map[cid] = name
                cids.add(cid)
                user_ids.add(cid_user_answer.user.pk)

            s = cid_user_answer
            q = cid_user_answer.question

            answer_score = s.get_answer_score()
            if answer_score == "unmarked":
                index = question_index_map.get(q)
                if index is not None:
                    unmarked.add(index)

            user_answers_marks[cid].append(answer_score)

            if self.app_name == "rapids":
                callstate = s.get_answer_callstate()
                user_answers_callstates[cid].append(callstate)

            if table_only:
                ans = s.get_answer_string()
                if self.app_name == "rapids":
                    by_question[q][cid] = (ans, answer_score, callstate)
                elif self.app_name in ("anatomy", "sbas", "longs", "shorts"):
                    by_question[q][cid] = (ans, answer_score)
                else:
                    zipped_ans_scores = zip(ans, answer_score)
                    by_question[q][cid] = zipped_ans_scores

        user_scores = {}
        user_scores_normalised = {}
        user_answer_count = {}
        for user in user_answers_marks:
            if self.app_name == "longs":
                missing_count = question_number - len(user_answers_marks[user])
                for _ in range(missing_count):
                    user_answers_marks[user].append(3.0)
                if table_only:
                    for question in questions:
                        if user not in by_question[question]:
                            by_question[question][user] = ("Not answered", 3.0)

            if self.app_name in ("physics",):
                user_scores[user] = sum(
                    [sum(i) for i in user_answers_marks[user] if i != "unmarked"]
                )
            else:
                user_scores[user] = sum(
                    [i for i in user_answers_marks[user] if i != "unmarked" and i != None]
                )

            if self.app_name == "rapids":
                user_scores_normalised[user] = normaliseRapidsScore(
                    sum([i for i in user_answers_marks[user] if i != "unmarked"])
                )
            else:
                user_scores_normalised[user] = user_scores[user]

            user_answer_count[user] = len(user_answers_marks[user])

        # If table_only and rapids, populate user_answers_callstates_counted
        if table_only and self.app_name == "rapids":
            for u in cids:
                user_answers_callstates_counted[u] = dict(Counter(user_answers_callstates[u]))

        missing_user_ids = valid_user_users - user_ids
        missing_users = []
        if missing_user_ids:
            missing_users = User.objects.filter(pk__in=missing_user_ids)

        max_score = self.get_max_score(questions)

        from django.contrib.contenttypes.models import ContentType
        from generic.models import CidUserExam
        ct = ContentType.objects.get_for_model(self.Exam)

        excluded_keys = set()
        for cux in CidUserExam.objects.filter(content_type=ct, object_id=exam.pk, exclude_from_stats=True):
            if cux.user_user_id:
                excluded_keys.add(f"u/{cux.user_user_id}")
            elif cux.cid_user_id:
                excluded_keys.add(f"c/{cux.cid_user.cid}")

        pass_mark_threshold = exam.pass_mark if exam.pass_mark is not None else (max_score / 2)

        if stats_only:
            user_scores_list = [score for user_key, score in user_scores.items() if user_key not in excluded_keys and score > 0]
            mean = 0
            median = 0
            mode = 0
            fig_html = ""
            std_dev = 0
            q1 = 0
            q3 = 0
            min_score = 0
            max_score_achieved = 0
            passed_candidates_count = 0
            pass_rate = 0.0

            if len(user_scores_list) >= 1:

                mean = round(statistics.mean(user_scores_list), 2)
                median = round(statistics.median(user_scores_list), 2)
                try:
                    mode = statistics.mode(user_scores_list)
                    if isinstance(mode, (int, float)):
                        mode = round(mode, 2)
                except statistics.StatisticsError:
                    mode = "No unique mode"

                std_dev = round(statistics.pstdev(user_scores_list), 2)

                if len(user_scores_list) >= 2:
                    quants = statistics.quantiles(user_scores_list, n=4)
                    q1 = round(quants[0], 2)
                    q3 = round(quants[2], 2)
                else:
                    q1 = round(user_scores_list[0], 2)
                    q3 = round(user_scores_list[0], 2)

                min_score = round(min(user_scores_list), 2)
                max_score_achieved = round(max(user_scores_list), 2)

                passed_candidates_count = sum(1 for user_key, score in user_scores.items() if user_key not in excluded_keys and score >= pass_mark_threshold)
                stats_cids = [c for c in cids if c not in excluded_keys]
                pass_rate = round((passed_candidates_count / len(stats_cids)) * 100, 1) if stats_cids else 0.0

                df = user_scores_list
                fig = px.histogram(
                    df,
                    x=0,
                    title="{}: distribution of scores".format(exam),
                    labels={"0": "Score"},
                    height=400,
                    width=600,
                )
                fig_html = fig.to_html()

                exam.stats_mean = mean
                exam.stats_median = median
                exam.stats_mode = mode

                exam.stats_candidates = len(user_scores_list)
                exam.stats_max_possible = max_score

                exam.stats_min = min_score
                exam.stats_max = max_score_achieved

                exam.stats_graph = fig_html

                user_data = {}
                for u in cids:
                    user_data[u] = {
                        "score": user_scores[u],
                        "normalised_score": user_scores_normalised[u],
                    }

                    if self.app_name == "rapids":
                        counted_callstates = dict(Counter(user_answers_callstates[u]))
                        user_data[u]["callstates"] = str(counted_callstates)

                exam.user_scores = user_data
                exam.save()

            stats_cids = [c for c in cids if c not in excluded_keys]
            expected_answer_count = len(stats_cids) * question_number
            user_answer_count_total = sum(user_answer_count[u] for u in stats_cids if u in user_answer_count)
            completion_rate = round((user_answer_count_total / expected_answer_count) * 100, 1) if expected_answer_count > 0 else 0.0

            if hasattr(self.Answer, "MarkOptions") and hasattr(self.UserAnswer, "score"):
                unmarked_count = sum(1 for c in cid_user_answers if c.score == self.Answer.MarkOptions.UNMARKED or c.score is None)
            elif hasattr(self.UserAnswer, "is_marked"):
                unmarked_count = sum(1 for c in cid_user_answers if not c.is_marked())
            elif hasattr(self.UserAnswer, "score"):
                unmarked_count = sum(1 for c in cid_user_answers if c.score == "" or c.score is None)
            else:
                unmarked_count = 0

            context = {
                "exam": exam,
                "cids": sorted(stats_cids),
                "question_number": question_number,
                "max_score": max_score,
                "mean": mean,
                "median": median,
                "mode": mode,
                "std_dev": std_dev,
                "q1": q1,
                "q3": q3,
                "min_score": min_score,
                "max_score_achieved": max_score_achieved,
                "passed_candidates_count": passed_candidates_count,
                "pass_rate": pass_rate,
                "expected_answer_count": expected_answer_count,
                "user_answer_count_total": user_answer_count_total,
                "completion_rate": completion_rate,
                "unmarked_count": unmarked_count,
                "plot": fig_html,
                "pass_mark_threshold": pass_mark_threshold,
            }
            return render(request, "generic/partials/_exam_scores_stats.html", context)

        pass_status = {}
        for cid, score in user_scores.items():
            try:
                val = float(score)
                pass_status[cid] = "Pass" if val >= pass_mark_threshold else "Fail"
            except (ValueError, TypeError):
                pass_status[cid] = "Fail"

        excluded_map = {k: True for k in excluded_keys}

        template_variables = {
            "cids": sorted(cids),
            "missing_cids": valid_cid_users - plain_cids,
            "missing_users": missing_users,
            "exam": exam,
            "unmarked": unmarked,
            "questions": questions,
            "question_number": question_number,
            "by_question": by_question,
            "user_scores": user_scores,
            "user_scores_normalised": user_scores_normalised,
            "user_answer_count": user_answer_count,
            "cids_user_id_map": cids_user_id_map,
            "max_score": max_score,
            "can_edit": exam.check_user_can_edit(request.user),
            "base_template": "generic/exam_scores_partial_base.html" if table_only else "generic/exam_scores_base.html",
            "excluded_map": excluded_map,
            "pass_status": pass_status,
            "pass_mark_threshold": pass_mark_threshold,
            "default_pass_mark": max_score / 2,
        }

        if self.app_name == "rapids":
            template_variables["user_answers_callstates"] = (
                user_answers_callstates_counted
            )

        return render(
            request,
            f"{self.app_name}/exam_scores.html",
            template_variables,
        )

    def exam_stats(self, request, exam_id):
        exam: ExamBase = get_object_or_404(self.Exam, pk=exam_id)

        if not self.check_user_edit_access(request.user, exam_id):
            raise PermissionDenied()

        if not exam.exam_mode:
            raise Http404("Packet not in exam mode")

        template_variables = {"exam": exam}

        return render(
            request,
            "exam_stats.html",
            template_variables,
        )

    def update_pass_mark(self, request, pk):
        exam = get_object_or_404(self.Exam, pk=pk)
        if not exam.check_user_can_edit(request.user):
            raise PermissionDenied

        pass_mark_str = request.POST.get("pass_mark")
        if pass_mark_str == "" or pass_mark_str is None:
            exam.pass_mark = None
        else:
            try:
                exam.pass_mark = float(pass_mark_str)
            except ValueError:
                return HttpResponse("Invalid pass mark", status=400)
        exam.save()

        from django.urls import reverse
        url = reverse(f"{self.app_name}:exam_scores_all", kwargs={"pk": pk})
        if request.htmx:
            # Return 200 OK with HX-Redirect header to trigger client-side redirect in HTMX
            return HttpResponse("", headers={"HX-Redirect": url})
        
        # Standard non-HTMX redirect
        from django.http import HttpResponseRedirect
        return HttpResponseRedirect(url)

    def toggle_candidate_stats_exclusion(self, request, pk):
        exam = get_object_or_404(self.Exam, pk=pk)
        if not exam.check_user_can_edit(request.user):
            raise PermissionDenied

        candidate_key = request.GET.get("candidate")
        if not candidate_key:
            return HttpResponse("Candidate ID required", status=400)

        from django.contrib.contenttypes.models import ContentType
        from generic.models import CidUserExam, CidUser

        ct = ContentType.objects.get_for_model(self.Exam)

        cux = None
        if candidate_key.startswith("u/"):
            try:
                user_id = int(candidate_key[2:])
                cux, created = CidUserExam.objects.get_or_create(
                    content_type=ct,
                    object_id=exam.pk,
                    user_user_id=user_id,
                )
            except ValueError:
                return HttpResponse("Invalid user ID", status=400)
        elif candidate_key.startswith("c/"):
            try:
                cid = int(candidate_key[2:])
                cid_user = CidUser.objects.filter(cid=cid).first()
                if cid_user:
                    cux, created = CidUserExam.objects.get_or_create(
                        content_type=ct,
                        object_id=exam.pk,
                        cid_user=cid_user,
                    )
            except ValueError:
                return HttpResponse("Invalid CID", status=400)
        else:
            return HttpResponse("Invalid candidate key format", status=400)

        if cux:
            cux.exclude_from_stats = not cux.exclude_from_stats
            cux.save()

            response = render(
                request,
                "generic/partials/_candidate_stats_checkbox.html",
                {
                    "cid": candidate_key,
                    "is_excluded": cux.exclude_from_stats,
                    "exam": exam,
                }
            )
            response["HX-Trigger"] = "refreshStats"
            return response

        return HttpResponse("Candidate not found", status=404)


class GenericViewBase:
    def __init__(self, app_name, QuestionObject, UserAnswerObject, ExamObject):
        self.question_object = QuestionObject
        self.cid_user_answer_object = UserAnswerObject
        self.exam_object = ExamObject
        self.app_name = app_name

        g = {
            "rapids": "rapid_checker",
            "anatomy": "anatomy_checker",
            "longs": "longs_checker",
            "sbas": "sba_checker",
            "physics": "physics_checker",
            "shorts": "shorts_checker",
        }

        self.checker_group = g[app_name]

    def check_user_edit_access(self, user, exam_id=None):
        """Check if a user should be able to access editing/reviewing an exam.

        This mirrors the older permission helper used elsewhere in the file but
        references `self.exam_object` and `self.app_name` (since GenericViewBase
        is instantiated per app).
        """
        if user.is_superuser:
            return True

        # If a user is an exam author they should have access
        if exam_id is not None:
            exam = get_object_or_404(self.exam_object, pk=exam_id)
            if user in exam.get_author_objects():
                return True

            if getattr(exam, "authors_only", False):
                return False

        # App-specific group checks (require checker role for edit/review)
        if (
            self.app_name == "rapids"
            and not user.groups.filter(name="rapid_checker").exists()
        ):
            return False
        if (
            self.app_name == "anatomy"
            and not user.groups.filter(name="anatomy_checker").exists()
        ):
            return False
        if (
            self.app_name == "longs"
            and not user.groups.filter(name__in=("long_checker",)).exists()
        ):
            return False
        if (
            self.app_name == "physics"
            and not user.groups.filter(name="physics_checker").exists()
        ):
            return False
        if (
            self.app_name == "sbas"
            and not user.groups.filter(name="sba_checker").exists()
        ):
            return False
        if (
            self.app_name == "casecollection"
            and not user.groups.filter(name="casecollection_checker").exists()
        ):
            return False
        return False

    def help(self, request):
        from django.template.loader import TemplateDoesNotExist, get_template
        try:
            get_template(f"{self.app_name}/help.html")
            template_name = f"{self.app_name}/help.html"
        except TemplateDoesNotExist:
            template_name = "generic/help.html"
        return render(request, template_name, {"app_name": self.app_name})

    def question_review(self, request, pk):
        """
        Return a json representation of the question when the exam is published
        """
        exam: ExamBase = get_object_or_404(self.exam_object, pk=pk)

        logger.debug("question_published_json: question_number={}", request.POST.get("question_number"))

        if not exam.publish_results:
            raise Http404

        question = exam.get_questions()[int(request.POST["question_number"])]

        question_json = question.get_question_json(based=False, feedback=True)
        return JsonResponse(question_json)

    def question_set_review(self, request, pk):
        """HTMX endpoint to get/set the latest QuestionReview for a question.

        GET: renders the current review block or the form (if ?edit=1)
        POST: creates a new QuestionReview and returns the rendered block
        """

        logger.debug("question_set_review called")

        question = get_object_or_404(self.question_object, pk=pk)

        ct = ContentType.objects.get_for_model(question)
        latest = (
            QuestionReview.objects.filter(content_type=ct, object_id=question.pk)
            .order_by("-created_on").first()
        )

        # If GET and edit requested, return the form. If `new` is set, render with no latest so the
        # form is blank for creating a fresh review.
        if request.method == "GET" and request.GET.get("edit"):
            if request.GET.get("new"):
                form = QuestionReviewForm()
                return render(request, "generic/partials/question_review_form.html", {"question": question, "latest": None, "form": form, "app_name": self.app_name})
            # Prefill the form with the latest review values (but we'll create a new review on POST)
            if latest:
                form = QuestionReviewForm(initial={"status": latest.status, "comment": latest.comment})
            else:
                form = QuestionReviewForm()
            return render(request, "generic/partials/question_review_form.html", {"question": question, "latest": latest, "form": form, "app_name": self.app_name})

        if request.method == "POST":
            # Require login to post a review
            if not request.user.is_authenticated:
                return HttpResponse(status=403)

            form = QuestionReviewForm(request.POST, user=request.user)
            if not form.is_valid():
                # Render the form back with errors
                return render(request, "generic/partials/question_review_form.html", {"question": question, "latest": latest, "form": form, "app_name": self.app_name})

            review = QuestionReview.objects.create(
                content_type=ct,
                object_id=question.pk,
                author=request.user,
                status=form.cleaned_data["status"],
                comment=form.cleaned_data.get("comment", ""),
            )
            # Render the updated review block and include an HX-Trigger so the client
            # can close the modal (HTMX will dispatch the event named below).
            resp = render(request, "generic/partials/question_review_block.html", {"review": review, "question": question, "app_name": self.app_name})
            # Trigger a client-side event; base template will listen for this and hide the modal
            resp["HX-Trigger"] = "reviewSaved"
            return resp

        # Default: render the block (latest if exists, otherwise form)
        if latest:
            return render(request, "generic/partials/question_review_block.html", {"review": latest, "question": question, "app_name": self.app_name})
        else:
            # Ensure a form is available for the template
            if latest:
                form = QuestionReviewForm(initial={"status": latest.status, "comment": latest.comment})
            else:
                form = QuestionReviewForm()
            return render(request, "generic/partials/question_review_form.html", {"question": question, "latest": latest, "form": form, "app_name": self.app_name})

    def question_reviews_list(self, request, pk):
        """Return a partial listing all QuestionReview instances for a question."""

        question = get_object_or_404(self.question_object, pk=pk)

        ct = ContentType.objects.get_for_model(question)
        reviews = (
            QuestionReview.objects.filter(content_type=ct, object_id=question.pk)
            .order_by("-created_on")
        )

        return render(request, "generic/partials/question_reviews_list.html", {"question": question, "reviews": reviews, "app_name": self.app_name})

    def question_review_start(self, request):

        # Prepare category choices if the question model exposes a FK named 'category'
        categories = None
        try:
            field = self.question_object._meta.get_field("category")
            # Only handle FK-like fields
            if hasattr(field, "related_model") and field.related_model is not None:
                CategoryModel = field.related_model
                # prefer 'category' or 'name' display field if present
                order_by = "category" if hasattr(CategoryModel, "category") else "name" if hasattr(CategoryModel, "name") else "pk"
                categories = CategoryModel.objects.all().order_by(order_by)
        except Exception:
            categories = None

        # Status options map: label -> status code (or special 'UNREVIEWED')
        status_options = [
            ("ANY", "Any"),
            ("UNREVIEWED", "Unreviewed"),
            ("AC", "Accepted"),
            ("OD", "Outdated"),
            ("ER", "Error"),
            ("RJ", "Rejected"),
            ("IP", "In Progress"),
        ]

        return render(
            request,
            f"{self.app_name}/question_review_start.html",
            {"app_name": self.app_name, "categories": categories, "status_options": status_options},
        )


    def question_review_next(self, request):
        """Find and redirect to the next question matching the supplied filters.

        Accepts POST or GET parameters:
        - category: integer primary key of category (optional)
        - status: one of the status codes (AC, OD, ER, RJ, IP), or 'UNREVIEWED' or 'ANY'
        """

        # Read filters
        category = request.POST.get("category") or request.GET.get("category")
        status = request.POST.get("status") or request.GET.get("status") or "ANY"
        # How many matching questions to skip (useful for Skip button). Expected integer >= 0.
        try:
            skip = int(request.POST.get("skip") or request.GET.get("skip") or 0)
            if skip < 0:
                skip = 0
        except Exception:
            skip = 0

        # Build base queryset of questions for this app
        qs = self.question_object.objects.all().order_by("pk")

        # Optionally restrict iteration to questions reviewed by the current user
        reviewed_filter = (request.POST.get("reviewed") or request.GET.get("reviewed")) == "me"
        if reviewed_filter:
            try:
                from django.db.models import Exists, OuterRef

                # use the module-level ContentType (imported near file top)
                ct = ContentType.objects.get_for_model(self.question_object)
                reviewed_exists = QuestionReview.objects.filter(
                    content_type=ct, object_id=OuterRef("pk"), author=request.user
                )
                qs = qs.annotate(reviewed_by_me=Exists(reviewed_exists))
            except Exception:
                reviewed_filter = True

        # Restrict by category if provided and the model has such a relation
        if category:
            try:
                qs = qs.filter(category__id=int(category))
            except Exception:
                pass

        # Apply permission filter similar to filters elsewhere: non-checkers see only open_access or their own
        if not request.user.groups.filter(name=self.checker_group).exists():
            try:
                qs = qs.filter(open_access=True) | qs.filter(author__id=request.user.id)
            except Exception:
                # If model doesn't have those fields, ignore
                pass

        # Iterate questions and pick first matching status
        for question in qs:
            # Skip authors_only questions unless user is author or superuser
            try:
                if question.authors_only and not (
                    request.user.is_superuser or request.user in question.author.all()
                ):
                    continue
            except Exception:
                pass

            ct = ContentType.objects.get_for_model(question)
            latest = (
                QuestionReview.objects.filter(content_type=ct, object_id=question.pk)
                .order_by("-created_on").first()
            )

            match = False
            if status == "ANY":
                match = True
            elif status == "UNREVIEWED":
                match = latest is None
            else:
                # status is expected to be a QuestionReview.StatusChoices code
                if latest is not None and latest.status == status:
                    match = True

            if match:
                # If caller asked us to skip N matches, decrement and continue until skip==0
                if skip and skip > 0:
                    skip -= 1
                    continue

                form = QuestionReviewForm()
                return render(request, f"{self.app_name}/question_review_question.html", {"question": question, "form": form, "app_name": self.app_name, "filters": {"category": category, "status": status}})

        # Nothing found - render complete page
        return render(request, f"{self.app_name}/question_review_complete.html", {"app_name": self.app_name})


    def question_review_list(self, request):
        """Return a page listing questions that match the supplied filters.

        Accepts query parameters similar to question_review_next: category, status.
        """

        # Read filters
        category = request.GET.get("category") or request.POST.get("category")
        status = request.GET.get("status") or request.POST.get("status") or "ANY"

        # Build base queryset of questions for this app
        qs = self.question_object.objects.all().order_by("pk")

        # Optionally filter to questions reviewed by the current user.
        # Use an Exists subquery to avoid fetching extra rows.
        reviewed_filter = (request.GET.get("reviewed") or request.POST.get("reviewed")) == "me"
        if reviewed_filter:
            try:
                from django.db.models import Exists, OuterRef

                # use the module-level ContentType (imported near file top)
                ct = ContentType.objects.get_for_model(self.question_object)
                reviewed_exists = QuestionReview.objects.filter(
                    content_type=ct, object_id=OuterRef("pk"), author=request.user
                )
                qs = qs.annotate(reviewed_by_me=Exists(reviewed_exists))
            except Exception:
                # If annotation fails for any reason, fall back to no-op and we'll filter in Python loop
                reviewed_filter = True

        # Restrict by category if provided
        if category:
            try:
                qs = qs.filter(category__id=int(category))
            except Exception:
                pass

        # Apply permission filter similar to question_review_next
        if not request.user.groups.filter(name=self.checker_group).exists():
            try:
                qs = qs.filter(open_access=True) | qs.filter(author__id=request.user.id)
            except Exception:
                pass

        # Annotate with latest review status using a DB subquery for efficiency
        try:
            from django.db.models import OuterRef, Subquery
            ct = ContentType.objects.get_for_model(self.question_object)
            latest_status_subq = (
                QuestionReview.objects.filter(content_type=ct, object_id=OuterRef("pk"))
                .order_by("-created_on")
                .values("status")[:1]
            )
            qs = qs.annotate(latest_review_status=Subquery(latest_status_subq))
        except Exception:
            # annotation failed — we'll fall back to per-object lookup below
            pass

        # Optionally filter by review status
        results = []
        for question in qs:
            try:
                if question.authors_only and not (
                    request.user.is_superuser or request.user in question.author.all()
                ):
                    continue
            except Exception:
                pass

            # Determine latest status — prefer annotated value if present
            latest_status = getattr(question, "latest_review_status", None)
            match = False
            if status == "ANY":
                match = True
            elif status == "UNREVIEWED":
                match = latest_status is None
            else:
                if latest_status is not None and latest_status == status:
                    match = True

            if match:
                # If the caller requested only questions reviewed by the current user,
                # skip any question that does not have a matching review.
                if reviewed_filter:
                    # If annotation was applied this attribute will be present on the instance.
                    if hasattr(question, "reviewed_by_me"):
                        if not question.reviewed_by_me:
                            continue
                    else:
                        # Last-resort check via DB lookup
                        ct = ContentType.objects.get_for_model(question)
                        if not QuestionReview.objects.filter(content_type=ct, object_id=question.pk, author=request.user).exists():
                            continue

                results.append(question)

        # Provide human-readable labels for statuses to the template
        status_choices = {code: label for code, label in QuestionReview.StatusChoices.choices}

        return render(
            request,
            f"{self.app_name}/question_review_list.html",
            {
                "questions": results,
                "app_name": self.app_name,
                "filters": {"category": category, "status": status},
                "status_choices": status_choices,
            },
        )


    @method_decorator(user_passes_test(lambda u: u.is_superuser))
    def user_answer_delete_multiple(self, request):
        answer_ids = json.loads(request.POST["answer_ids"])

        # We could probably delete them all at once....
        for id in answer_ids:
            self.cid_user_answer_object.objects.get(pk=id).delete()
        return JsonResponse({"status": "success"})

    @method_decorator(login_required)
    def question_thumbnail_fail(self, request, pk):
        return self.question_thumbnail(request, pk=pk, fail_loudly=True)

    @method_decorator(login_required)
    def question_thumbnail(self, request, pk, fail_loudly=False):
        question = get_object_or_404(self.question_object, pk=pk)

        thumbnail, _ = question.get_thumbnail(recreate=True, fail_loudly=fail_loudly)

        return HttpResponse(thumbnail)

    @method_decorator(login_required)
    def question_detail(self, request, pk):
        question: QuestionBase = get_object_or_404(self.question_object, pk=pk)

        if request.user.is_superuser:
            pass
        elif not question.open_access:
            if (
                not request.user.groups.filter(name=self.checker_group).exists()
                and request.user not in question.author.all()
            ):
                raise PermissionDenied()
        # if request.user not in rapid.author.all():
        #    raise PermissionDenied

        view_feedback = False
        if (
            request.user.groups.filter(name=self.checker_group).exists()
            or request.user.groups.filter(name="feedback_checker").exists()
            or request.user in question.author.all()
        ):
            view_feedback = True

        # logging.debug(rapid.subspecialty.first().name.all())
        return render(
            request,
            f"{self.app_name}/question_detail.html",
            {
                "question": question,
                "view_feedback": view_feedback,
                "remote_url": settings.REMOTE_URL,
                "can_edit": question.can_edit(request.user),
                "app_name": self.app_name,
            },
        )

    # TODO: improve permissions on these
    @method_decorator(login_required)
    def question_user_answers_by_compare(self, request, pk, answer_compare):
        answer_compare = urllib.parse.unquote(answer_compare)
        return self.question_user_answers(request, pk, answer_compare)

    @method_decorator(login_required)
    def question_user_answers(self, request, pk, answer_compare=None):
        question = get_object_or_404(self.question_object, pk=pk)

        if answer_compare is None:
            answers = self.cid_user_answer_object.objects.filter(
                question__id=question.pk
            ).prefetch_related("question", "exam", "user")
        else:
            answers = self.cid_user_answer_object.objects.filter(
                question__id=question.pk, answer_compare=answer_compare
            ).prefetch_related("question", "exam", "user")

        return render(
            request,
            f"{self.app_name}/question_user_answers.html",
            {
                "question": question,
                "answers": answers,
                "answer_compare": answer_compare,
            },
        )

    @method_decorator(login_required)
    def author_detail(self, request, pk):
        # logging.debug(Author.objects.all())
        author = get_object_or_404(User, pk=pk)

        questions = self.question_object.objects.filter(author=pk)

        return render(
            request,
            f"{self.app_name}/category_detail.html",
            {"category": author, "questions": questions},
        )

    @method_decorator(login_required)
    def author_list(self, request):
        authors = User.objects.all()

        return render(
            request, f"{self.app_name}/author_list.html", {"authors": authors}
        )


class ExamCloneMixin:
    def get_template_names(self) -> list[str]:
        return "exam_clone_form.html"
        # return super().get_template_names()

    def get_initial(self):
        old_object = get_object_or_404(self.model, pk=self.kwargs["exam_id"])

        if not old_object.check_user_can_edit(self.request.user):
            raise PermissionDenied()  # or Http404

        initial_data = model_to_dict(old_object, exclude=["id"])

        # We manually transfer the forign keys / m2m relationships
        questions = old_object.exam_questions.all().values_list("id", flat=True)
        authors = old_object.author.all().values_list("id", flat=True)

        # clear the associated groups
        initial_data["valid_user_users"] = []
        initial_data["valid_cid_users"] = []
        initial_data["cid_user_groups"] = []
        initial_data["user_user_groups"] = []

        initial_data["active"] = False
        initial_data["archive"] = False
        initial_data["publish_results"] = False

        self.exam_questions = list(questions)
        self.author = list(authors)

        self.old_object = old_object

        return initial_data

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        context["can_edit"] = self.old_object.check_user_can_edit(self.request.user)
        context["exam"] = self.old_object

        return context

    def form_valid(self, form):
        object = form.save()
        # Reapply these otherwise they get lost?
        object.exam_questions.set(self.exam_questions)
        object.author.set(self.author)
        object.save()
        object.order_questions()
        return HttpResponseRedirect(object.get_absolute_url())


class ExaminationAutocomplete(autocomplete.Select2QuerySetView):
    def get_queryset(self):
        # Don't forget to filter out results depending on the visitor !
        if not self.request.user.is_authenticated:
            return Examination.objects.none()

        qs = Examination.objects.all()

        if self.q:
            # This raises a fielderror which breaks creating a new item if not caught
            try:
                qs = qs.filter(examination__icontains=self.q)
            except FieldError:
                return Examination.objects.none()

        return qs


class SupervisorAutocomplete(autocomplete.Select2QuerySetView):
    def get_queryset(self):
        # TODO: we should probably filter this to only
        # allow access by trainees / other suprevisors
        if not self.request.user.is_authenticated:
            return Supervisor.objects.none()

        qs = Supervisor.objects.all().order_by("name")

        if self.q:
            # This raises a fielderror which breaks creating a new item if not caught
            try:
                qs = qs.filter(Q(email__icontains=self.q) | Q(name__icontains=self.q))
            except FieldError:
                return Supervisor.objects.none()

        return qs


class CidUserExamView(CidManagerRequiredMixin, SingleTableMixin, FilterView):
    model = CidUser
    table_class = CidUserExamTable
    template_name = "generic/cid_view.html"

    filterset_class = CidUserFilter

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        # user = self.request.user

        filters = {"archive": False, "exam_mode": True}

        physics_exams = [(i.name, i.pk) for i in PhysicsExam.objects.filter(**filters)]
        rapid_exams = [(i.name, i.pk) for i in RapidsExam.objects.filter(**filters)]
        shorts_exams = [(i.name, i.pk) for i in ShortsExam.objects.filter(**filters)]
        sba_exams = [(i.name, i.pk) for i in SbasExam.objects.filter(**filters)]
        longs_exams = [(i.name, i.pk) for i in LongsExam.objects.filter(**filters)]
        anatomy_exams = [(i.name, i.pk) for i in AnatomyExam.objects.filter(**filters)]
        casecollection_exams = [
            (i.name, i.pk)
            for i in CaseCollection.objects.filter(archive=False, collection_type__gt=0)
        ]

        cid_user_groups = [
            (i.name, i.pk) for i in CidUserGroup.objects.filter(archive=False)
        ]

        context["physics_exams"] = physics_exams
        context["rapid_exams"] = rapid_exams
        context["shorts_exams"] = shorts_exams
        context["sba_exams"] = sba_exams
        context["longs_exams"] = longs_exams
        context["anatomy_exams"] = anatomy_exams
        context["casecollection_exams"] = casecollection_exams
        context["cid_user_groups"] = cid_user_groups
        context["exams"] = True
        return context


class CidUserView(CidManagerRequiredMixin, SingleTableMixin, FilterView):
    model = CidUser
    table_class = CidUserTable
    template_name = "generic/cid_view.html"

    filterset_class = CidUserFilter

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        # user = self.request.user

        filters = {"archive": False, "exam_mode": True}

        physics_exams = [(i.name, i.pk) for i in PhysicsExam.objects.filter(**filters)]
        rapid_exams = [(i.name, i.pk) for i in RapidsExam.objects.filter(**filters)]
        shorts_exams = [(i.name, i.pk) for i in ShortsExam.objects.filter(**filters)]
        sba_exams = [(i.name, i.pk) for i in SbasExam.objects.filter(**filters)]
        longs_exams = [(i.name, i.pk) for i in LongsExam.objects.filter(**filters)]
        anatomy_exams = [(i.name, i.pk) for i in AnatomyExam.objects.filter(**filters)]
        casecollection_exams = [
            (i.name, i.pk)
            for i in CaseCollection.objects.filter(archive=False, collection_type__gt=0)
        ]

        context["physics_exams"] = physics_exams
        context["rapid_exams"] = rapid_exams
        context["shorts_exams"] = shorts_exams
        context["sba_exams"] = sba_exams
        context["longs_exams"] = longs_exams
        context["anatomy_exams"] = anatomy_exams
        context["casecollection_exams"] = casecollection_exams

        cid_user_groups = [
            (i.name, i.pk) for i in CidUserGroup.objects.filter(archive=False)
        ]

        context["cid_user_groups"] = cid_user_groups
        return context


@user_is_cid_user_manager
def cid_group_view(request):
    groups = CidUserGroup.objects.filter(archive=False)

    return render(
        request,
        "generic/cid_group_view.html",
        {"groups": groups},
    )


@user_is_cid_user_manager
def import_exam_answers(request):
    """
    Imports manual JSON exam answers files exported from RTS when direct API submission failed.
    Supports a preview step comparing file answers with existing database entries.
    """
    from django.contrib import messages
    
    if request.method == "POST":
        confirm = request.POST.get("confirm") == "1"
        
        if confirm:
            data = request.session.get("pending_import_data")
            if not data:
                messages.error(request, "Import session expired. Please upload the file again.")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            eid_str = data.get("eid")
            cid_str = data.get("cid")
            start_time_val = data.get("start_time")
            answers_data = data.get("answers")
            
            try:
                exam_type, exam_id_str = eid_str.split("/")
                eid = int(exam_id_str)
            except Exception:
                messages.error(request, "Invalid exam ID format in stored session.")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            EXAM_IMPORT_MAP = {
                "physics": (PhysicsExam, PhysicsUserAnswer, "physics"),
                "anatomy": (AnatomyExam, AnatomyUserAnswer, "anatomy"),
                "rapid": (RapidsExam, RapidsUserAnswer, "rapids"),
                "rapids": (RapidsExam, RapidsUserAnswer, "rapids"),
                "short": (ShortsExam, ShortsUserAnswer, "shorts"),
                "shorts": (ShortsExam, ShortsUserAnswer, "shorts"),
                "long": (LongsExam, LongsUserAnswer, "longs"),
                "longs": (LongsExam, LongsUserAnswer, "longs"),
                "sba": (SbasExam, SbasUserAnswer, "sbas"),
                "sbas": (SbasExam, SbasUserAnswer, "sbas"),
            }
            
            mapping = EXAM_IMPORT_MAP.get(exam_type.lower())
            if not mapping:
                messages.error(request, f"Unsupported exam type: {exam_type}")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            ExamModel, UserAnswerModel, app_name = mapping
            
            uid = False
            cid = False
            if str(cid_str).startswith("u-"):
                uid = int(cid_str[2:])
            else:
                cid = int(cid_str)
                
            try:
                exam = ExamModel.objects.get(pk=eid)
            except ExamModel.DoesNotExist:
                messages.error(request, f"Exam {eid} does not exist.")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
                
            if isinstance(answers_data, str):
                answers_list = json.loads(answers_data)
            else:
                answers_list = answers_data
                
            created_count = 0
            updated_count = 0
            
            for answer in answers_list:
                try:
                    qid = int(answer["qid"])
                    posted_answer = answer["ans"]
                    qidn = str(answer.get("qidn", "1"))
                except (KeyError, ValueError, TypeError):
                    continue
                
                if uid:
                    existing_answers = UserAnswerModel.objects.filter(
                        question__id=qid, exam__id=eid, user__id=uid
                    )
                else:
                    existing_answers = UserAnswerModel.objects.filter(
                        question__id=qid, exam__id=eid, cid=cid
                    )
                    
                ans = None
                is_new = len(existing_answers) == 0
                
                if app_name == "rapids":
                    normal = (posted_answer == "Normal")
                    if is_new:
                        if uid:
                            ans = UserAnswerModel(answer=posted_answer, normal=normal, user=User.objects.get(id=uid))
                        else:
                            ans = UserAnswerModel(answer=posted_answer, normal=normal, cid=cid)
                        ans.question_id = qid
                        ans.exam_id = eid
                    else:
                        ans = existing_answers[0]
                        if qidn == "1":
                            ans.normal = normal
                            if normal:
                                ans.answer = ""
                        elif qidn == "2" and not ans.normal:
                            ans.answer = posted_answer
                elif app_name == "shorts":
                    if is_new:
                        if uid:
                            ans = UserAnswerModel(answer=posted_answer, user=User.objects.get(id=uid))
                        else:
                            ans = UserAnswerModel(answer=posted_answer, cid=cid)
                        ans.question_id = qid
                        ans.exam_id = eid
                        ans.score = None
                    else:
                        ans = existing_answers[0]
                        ans.answer = posted_answer
                        ans.score = None
                elif app_name == "anatomy":
                    from anatomy.models import Answer as AnatomyAnswer
                    if is_new:
                        if uid:
                            ans = UserAnswerModel(answer=posted_answer, user=User.objects.get(id=uid))
                        else:
                            ans = UserAnswerModel(answer=posted_answer, cid=cid)
                        ans.question_id = qid
                        ans.exam_id = eid
                        ans.score = AnatomyAnswer.MarkOptions.UNMARKED
                    else:
                        ans = existing_answers[0]
                        ans.answer = posted_answer
                        ans.score = AnatomyAnswer.MarkOptions.UNMARKED
                elif app_name == "longs":
                    if is_new:
                        if uid:
                            ans = UserAnswerModel(user=User.objects.get(id=uid))
                        else:
                            ans = UserAnswerModel(cid=cid)
                        ans.question_id = qid
                        ans.exam_id = eid
                    else:
                        ans = existing_answers[0]
                    
                    if qidn == "1":
                        ans.answer_observations = posted_answer
                    elif qidn == "2":
                        ans.answer_interpretation = posted_answer
                    elif qidn == "3":
                        ans.answer_principle_diagnosis = posted_answer
                    elif qidn == "4":
                        ans.answer_differential_diagnosis = posted_answer
                    elif qidn == "5":
                        ans.answer_management = posted_answer
                    ans.score = ans.ScoreOptions.UNMARKED
                    
                if ans:
                    time_answered_str = answer.get("time_answered")
                    if time_answered_str:
                        from django.utils.dateparse import parse_datetime
                        parsed_time = parse_datetime(time_answered_str)
                        if parsed_time:
                            if timezone.is_naive(parsed_time):
                                parsed_time = timezone.make_aware(parsed_time)
                            ans.time_answered = parsed_time
                    try:
                        ans.full_clean()
                        ans.save()
                        if is_new:
                            created_count += 1
                        else:
                            updated_count += 1
                    except Exception as e:
                        messages.warning(request, f"Failed to save answer for question {qid}: {e}")
                        
            # Create CidUserExam if user attempt
            try:
                if uid:
                    t = timezone.now()
                    if start_time_val:
                        t = timezone.datetime.fromtimestamp(start_time_val, tz=timezone.utc)
                    exam.get_or_create_cid_user_exam(
                        user_user=User.objects.get(id=uid), start_time=t
                    )
            except Exception:
                pass
                
            del request.session["pending_import_data"]
            
            return render(request, "generic/import_exam_answers_success.html", {
                "created_count": created_count,
                "updated_count": updated_count,
                "total_count": created_count + updated_count,
                "cid_str": cid_str,
                "eid_str": eid_str,
            })
            
        else:
            # First stage: parse and preview
            json_file = request.FILES.get("answers_file")
            if not json_file:
                messages.error(request, "No file uploaded")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            try:
                data = json.load(json_file)
            except Exception as e:
                messages.error(request, f"Failed to parse JSON: {e}")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            eid_str = data.get("eid")
            cid_str = data.get("cid")
            answers_data = data.get("answers")
            
            if not eid_str or not cid_str or answers_data is None:
                messages.error(request, "Uploaded JSON is missing required fields (eid, cid, answers)")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            try:
                exam_type, exam_id_str = eid_str.split("/")
                eid = int(exam_id_str)
            except Exception:
                messages.error(request, f"Invalid eid format: {eid_str}")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            EXAM_IMPORT_MAP = {
                "physics": (PhysicsExam, PhysicsUserAnswer, "physics"),
                "anatomy": (AnatomyExam, AnatomyUserAnswer, "anatomy"),
                "rapid": (RapidsExam, RapidsUserAnswer, "rapids"),
                "rapids": (RapidsExam, RapidsUserAnswer, "rapids"),
                "short": (ShortsExam, ShortsUserAnswer, "shorts"),
                "shorts": (ShortsExam, ShortsUserAnswer, "shorts"),
                "long": (LongsExam, LongsUserAnswer, "longs"),
                "longs": (LongsExam, LongsUserAnswer, "longs"),
                "sba": (SbasExam, SbasUserAnswer, "sbas"),
                "sbas": (SbasExam, SbasUserAnswer, "sbas"),
            }

            mapping = EXAM_IMPORT_MAP.get(exam_type.lower())
            if not mapping:
                messages.error(request, f"Unsupported exam type: {exam_type}")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            ExamModel, UserAnswerModel, app_name = mapping
            
            uid = False
            cid = False
            if str(cid_str).startswith("u-"):
                try:
                    uid = int(cid_str[2:])
                except ValueError:
                    messages.error(request, f"Invalid user CID format: {cid_str}")
                    return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            else:
                try:
                    cid = int(cid_str)
                except ValueError:
                    messages.error(request, f"Invalid candidate CID format: {cid_str}")
                    return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            
            try:
                exam = ExamModel.objects.get(pk=eid)
            except ExamModel.DoesNotExist:
                messages.error(request, f"Exam {eid} of type {exam_type} does not exist")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
                
            if isinstance(answers_data, str):
                try:
                    answers_list = json.loads(answers_data)
                except Exception as e:
                    messages.error(request, f"Failed to parse answers string JSON: {e}")
                    return HttpResponseRedirect(reverse("generic:import_exam_answers"))
            elif isinstance(answers_data, list):
                answers_list = answers_data
            else:
                messages.error(request, "answers field must be a list or a JSON string representation of a list")
                return HttpResponseRedirect(reverse("generic:import_exam_answers"))
                
            request.session["pending_import_data"] = data
            
            preview_items = []
            for answer in answers_list:
                try:
                    qid = int(answer["qid"])
                    posted_answer = answer["ans"]
                    qidn = str(answer.get("qidn", "1"))
                except (KeyError, ValueError, TypeError):
                    continue
                
                if uid:
                    existing = UserAnswerModel.objects.filter(
                        question__id=qid, exam__id=eid, user__id=uid
                    ).first()
                else:
                    existing = UserAnswerModel.objects.filter(
                        question__id=qid, exam__id=eid, cid=cid
                    ).first()
                    
                existing_str = ""
                action = "Create"
                if existing:
                    action = "Update"
                    if app_name == "longs":
                        if qidn == "1":
                            existing_str = existing.answer_observations
                        elif qidn == "2":
                            existing_str = existing.answer_interpretation
                        elif qidn == "3":
                            existing_str = existing.answer_principle_diagnosis
                        elif qidn == "4":
                            existing_str = existing.answer_differential_diagnosis
                        elif qidn == "5":
                            existing_str = existing.answer_management
                    else:
                        existing_str = existing.answer
                        
                preview_items.append({
                    "qid": qid,
                    "qidn": qidn,
                    "posted": posted_answer,
                    "existing": existing_str,
                    "action": action,
                })
                
            return render(request, "generic/import_exam_answers_preview.html", {
                "eid_str": eid_str,
                "cid_str": cid_str,
                "preview_items": preview_items,
                "answers_count": len(preview_items),
            })
            
    # GET request
    if "pending_import_data" in request.session:
        del request.session["pending_import_data"]
    return render(request, "generic/import_exam_answers.html")


def compact_history_for_object(obj):
    """
    Helper function to delete all historical revisions for a registered object,
    leaving only the most recent version.
    """
    from reversion.models import Version
    versions = Version.objects.get_for_object(obj).order_by("-revision__date_created")
    count = versions.count()
    if count > 1:
        # Keep the latest version, delete older versions and their revisions
        recent_version = versions[0]
        old_versions = list(versions[1:])
        for v in old_versions:
            revision = v.revision
            v.delete()
            if not revision.version_set.exists():
                revision.delete()
        return count - 1
    return 0


@login_required
def view_answer_history(request, app_name, model_name, pk):
    """
    View to display the revision history for a user/candidate answer,
    and compact history (individual and bulk).
    """
    from django.contrib import messages
    from django.apps import apps
    from reversion.models import Version
    
    try:
        Model = apps.get_model(app_name, model_name)
    except LookupError:
        raise Http404("Model not found")
        
    answer = get_object_or_404(Model, pk=pk)
    
    # Check permissions
    can_view = (
        request.user.is_staff 
        or request.user.is_superuser
        or request.user.groups.filter(name__in=[
            "cid_user_manager", "anatomy_checker", "rapid_checker", "long_checker"
        ]).exists()
    )
    
    exam = getattr(answer, "exam", None)
    collection = None
    if hasattr(answer, "question") and hasattr(answer.question, "collection"):
        collection = answer.question.collection
        
    if not can_view:
        if exam and hasattr(exam, "author") and request.user in exam.author.all():
            can_view = True
        if collection and hasattr(collection, "author") and request.user in collection.author.all():
            can_view = True
            
    if not can_view:
        raise PermissionDenied("You do not have permission to view this answer's history")
        
    if request.method == "POST":
        if not (request.user.is_superuser or request.user.is_staff):
            raise PermissionDenied("Only admin users can compact history")
            
        action = request.POST.get("action")
        if action == "compact_bulk":
            compacted_total = 0
            if exam:
                answers_qs = Model.objects.filter(exam=exam)
                for ans in answers_qs:
                    compacted_total += compact_history_for_object(ans)
                messages.success(request, f"Bulk compacted history for all answers in Exam '{exam}'. Removed {compacted_total} old revisions.")
            elif collection:
                from atlas.models import CidReportAnswer, UserReportAnswer
                cids_qs = CidReportAnswer.objects.filter(question__collection=collection)
                users_qs = UserReportAnswer.objects.filter(question__collection=collection)
                for ans in cids_qs:
                    compacted_total += compact_history_for_object(ans)
                for ans in users_qs:
                    compacted_total += compact_history_for_object(ans)
                messages.success(request, f"Bulk compacted history for all answers in Case Collection '{collection}'. Removed {compacted_total} old revisions.")
            return HttpResponseRedirect(request.path)
        else:
            raise PermissionDenied("Invalid compaction action or action not allowed on an individual answer basis.")
            
    # GET request
    versions = Version.objects.get_for_object(answer).select_related("revision__user").order_by("-revision__date_created")
    
    history_data = []
    for v in versions:
        fields = v.field_dict
        
        # Format answer contents nicely based on model/app
        ans_text = fields.get("answer", "")
        if not ans_text:
            if model_name.lower() == "useranswer" and app_name.lower() == "longs":
                ans_text = (
                    f"Observations: {fields.get('answer_observations', '')}\n"
                    f"Interpretation: {fields.get('answer_interpretation', '')}\n"
                    f"Diagnosis: {fields.get('answer_principle_diagnosis', '')}\n"
                    f"Differential: {fields.get('answer_differential_diagnosis', '')}\n"
                    f"Management: {fields.get('answer_management', '')}"
                )
            elif model_name.lower() == "useranswer" and app_name.lower() == "physics":
                ans_text = f"A: {fields.get('a')}, B: {fields.get('b')}, C: {fields.get('c')}, D: {fields.get('d')}, E: {fields.get('e')}"

                
        history_data.append({
            "revision_id": v.revision_id,
            "date": v.revision.date_created,
            "user": v.revision.user,
            "answer_text": ans_text,
            "score": fields.get("score", ""),
            "comment": v.revision.comment,
            "time_answered": fields.get("time_answered", ""),
        })
        
    context = {
        "answer": answer,
        "history": history_data,
        "exam": exam,
        "collection": collection,
        "app_name": app_name,
        "model_name": model_name,
    }
    
    return render(request, "generic/answer_history.html", context)


@user_is_cid_user_manager
def user_group_add_by_email_user(request, group_id):
    if request.htmx:
        group = get_object_or_404(UserUserGroup, pk=group_id)
        emails = request.POST.get("emails")

        if not emails:
            return HttpResponse("No emails provided", content_type="text/plain")


        added = []
        invalid_emails = []
        for email in emails.split():
            try:
                validate_email(email)

                try:
                    user = User.objects.get(email=email)

                    group.users.add(user)
                    added.append(user.username)
                except User.DoesNotExist:
                    invalid_emails.append(email)
            except ValidationError:
                # Ignore invalid emails
                pass

        res = f"<div class='alert alert-primary'>Users added to {group.name} [{','.join(added)} ({len(added)})]</div>"

        if invalid_emails:
            res += f"<br/><div class='alert alert-warning'>Invalid emails [{','.join(invalid_emails)}]</div>"


        return HttpResponse(res, content_type="text/html")



    
    raise Http404



@user_is_cid_user_manager
def cid_group_view_detail(request, group_id):
    group = get_object_or_404(CidUserGroup, pk=group_id)

    users = group.ciduser_set.filter(active=True)

    return render(
        request,
        "generic/cid_group_view_detail.html",
        {"group": group, "users": users, "group_type": "cid"},
    )


@user_is_cid_user_manager
def cid_group_generate_preview(request, group_id):
    """Parse submitted rows and return a preview fragment for HTMX."""
    if not request.htmx:
        raise Http404()

    group = get_object_or_404(CidUserGroup, pk=group_id)
    rows_text = request.POST.get('rows', '')
    original = rows_text
    parsed = []
    for line in rows_text.splitlines():
        line = line.strip()
        if not line:
            continue
        # Accept CSV or tab-separated: name,email or just email
        parts = [p.strip() for p in line.replace('\t', ',').split(',') if p.strip()]
        if len(parts) == 1:
            email = parts[0]
            name = ''
        else:
            name = parts[0]
            email = parts[1]
        parsed.append({'name': name, 'email': email})

    html = render_to_string('generic/partials/cid_group_generate_preview.html', {'rows': parsed, 'original': original, 'group': group}, request=request)
    return HttpResponse(html)


@user_is_cid_user_manager
def cid_group_create_bulk(request, group_id):
    """Create CidUser rows from posted rows and add them to the group, returning updated users list HTML."""
    if not request.htmx:
        raise Http404()

    group = get_object_or_404(CidUserGroup, pk=group_id)
    rows_text = request.POST.get('rows', '')
    created = []
    import secrets

    for line in rows_text.splitlines():
        line = line.strip()
        if not line:
            continue
        parts = [p.strip() for p in line.replace('\t', ',').split(',') if p.strip()]
        if len(parts) == 1:
            email = parts[0]
            name = ''
        else:
            name = parts[0]
            email = parts[1]

        # generate cid and passcode
        try:
            cid_val = get_next_cid()
        except Exception:
            # fallback: max+1
            last = CidUser.objects.order_by('-cid').first()
            cid_val = (last.cid if last and last.cid else 0) + 1

        passcode = secrets.token_hex(3)

        cu = CidUser.objects.create(cid=cid_val, passcode=passcode, name=name, email=email, group=group)
        created.append(cu)

    # re-render the users list
    users = group.ciduser_set.filter(active=True)
    inner = render_to_string('generic/partials/group_users_list.html', {'users': users, 'group_type': 'cid'}, request=request)
    # return the wrapper so HTMX can replace the container even if it didn't exist before
    html = f"<div id=\"group-users-wrap\">{inner}</div>"
    return HttpResponse(html)


@user_is_cid_user_manager
def cid_group_view_all(request):
    groups = CidUserGroup.objects.filter()

    return render(
        request,
        "generic/cid_group_view.html",
        {"groups": groups, "view_all": True},
    )

@user_is_cid_user_manager
def cid_group_add_candidates_to_exams(request, group_id):
    if request.htmx:
        group = get_object_or_404(CidUserGroup, pk=group_id)

        exam_id = request.POST.get("exam_id")
        exam_type = request.POST.get("exam_type")

        ExamModel = get_exam_model_from_app_name(exam_type)

        exam = get_object_or_404(ExamModel, pk=exam_id)

        if request.POST.get("action") == "remove":
            exam.valid_cid_users.remove(*group.ciduser_set.all())
            return HttpResponse(f"Candidates removed")
        else:
            exam.valid_cid_users.add(*group.ciduser_set.all())
            return HttpResponse(f"Candidates added")
        # exam.save()

    raise PermissionDenied()  # or Http404

@user_is_cid_user_manager
def user_group_add_candidates_to_exams(request, group_id):
    if request.htmx:
        group = get_object_or_404(UserUserGroup, pk=group_id)

        exam_id = request.POST.get("exam_id")
        exam_type = request.POST.get("exam_type")

        ExamModel = get_exam_model_from_app_name(exam_type)

        exam = get_object_or_404(ExamModel, pk=exam_id)

        if request.POST.get("action") == "remove":
            exam.valid_user_users.remove(*group.users.all())
            return HttpResponse(f"Candidates removed")
        else:
            exam.valid_user_users.add(*group.users.all())
            return HttpResponse(f"Candidates added")
        # exam.save()

    raise PermissionDenied()  # or Http404
    pass


@user_is_cid_user_manager
def user_group_view_detail(request, group_id):
    group = get_object_or_404(UserUserGroup, pk=group_id)

    users = group.users.filter()  # Filter inactive users?

    return render(
        request,
        "generic/cid_group_view_detail.html",
        {"group": group, "users": users, "group_type": "user"},
    )


@user_is_cid_user_manager
def users_bulk_delete_supervisors(request):
    if "selection" in request.POST:
        selected_users = request.POST.getlist("selection")
        user_models = User.objects.filter(id__in=selected_users)

        for u in user_models:
            u.userprofile.supervisor = None
            u.save()

        modified_users = ", ".join([f"{user.first_name} {user.last_name}" for user in user_models])

        return HttpResponse(
            f"""
            <div class="alert alert-success alert-dismissible fade show" role="alert">
                <strong>Success!</strong> Supervisors removed from: {modified_users}. Refresh page to see update.
                <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
            </div>
            """,
            content_type="text/html",
        )

    else:
        return HttpResponse(
            """
            <div class="alert alert-warning alert-dismissible fade show" role="alert">
                <strong>Warning!</strong> No users selected.
                <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
            </div>
            """,
            content_type="text/html",
        )


class NoUsersSelected(Exception):
    pass


def get_user_selection_from_request(request):
    selected_users = request.POST.getlist("selection")

    if not selected_users:
        raise NoUsersSelected
    user_models = User.objects.filter(id__in=selected_users)
    return user_models


@user_is_cid_user_manager
def user_not_trainee(request, user_id):
    user = get_object_or_404(User, pk=user_id)

    user.userprofile.peninsula_trainee = False
    user.userprofile.save()

    if request.htmx:
        return HttpResponse("")
    return HttpResponse(
        f"{user.username} is no longer a trainee", content_type="text/plain"
    )


@user_is_cid_user_manager
def users_bulk_edit(request):
    logger.debug("users_bulk_edit: htmx trigger={} name={}", request.htmx.trigger, request.htmx.trigger_name)

    try:
        match request.htmx.trigger:
            case r if r is None:
                html = "None"
            case "bulk-deactivate-user-button":
                user_models = get_user_selection_from_request(request)

                # u: User
                for u in user_models:
                    u.is_active = False
                    u.save()

                html = """
                <div class="alert alert-success alert-dismissible fade show" role="alert">
                    <strong>Success!</strong> Selected users have been deactivated.
                    <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
                </div>
                """

            case "bulk-edit-grade-button":
                user_grades = UserGrades.objects.all()

                html = "".join(
                    [
                        f"""<button class='change-grade-button' id='add-grade--{grade.id}'
                    hx-post="{reverse('generic:users_bulk_edit')}"
                    hx-include="[name='selection']"
                    hx-confirm="This will modify grades of all selected users, are you sure you wish to continue?"
                    hx-target="#htmx-info"
                    _='on click put "" into #htmx-options'
                >{grade.name}</button>"""
                        for grade in user_grades
                    ]
                )
                # TODO: work out how to actually use hyperscript
                html = (
                    html
                    + f"""<button class='change-grade-button' id='add-grade--remove'
                    hx-post="{reverse('generic:users_bulk_edit')}"
                    hx-include="[name='selection']"
                    hx-confirm="This will remove grades of all selected users, are you sure you wish to continue?"
                    hx-target="#htmx-info"
                    _='on click put "" into #htmx-options'
                >Remove</button>"""
                )
                html = (
                    html
                    + "<button class='cancel-button' _='on click for el in .change-grade-button remove el end then remove me'>Cancel</button>"
                )

            case r if r.startswith("add-grade"):
                user_models = get_user_selection_from_request(request)

                grade = r.split("--")[-1]

                if grade == "remove":
                    grade = None

                # u: User
                for u in user_models:
                    u.userprofile.grade_id = grade
                    u.save()
                html = """
                <div class="alert alert-success alert-dismissible fade show" role="alert">
                    <strong>Success!</strong> Grades updated successfully.
                    <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
                </div>
                """
            case "bulk-add-group-button":
                user_groups = UserUserGroup.objects.all()

                html = "".join(
                    [
                        f"""<button class='add-group-button' id='add-group--{group.id}'
                    hx-post="{reverse('generic:users_bulk_edit')}"
                    hx-include="[name='selection']"
                    hx-confirm="This will add the group '{group.name}' to all selected users, are you sure you wish to continue?"
                    hx-target="#htmx-info"
                    _='on click put "" into #htmx-options'
                >{group.name}</button>"""
                        for group in user_groups
                    ]
                )
                # TODO: work out how to actually use hyperscript
                html = (
                    html
                    + "<button  class='cancel-button'_='on click for el in .add-group-button remove el end then remove me'>Cancel</button>"
                )

            case "bulk-remove-group-button":
                user_groups = UserUserGroup.objects.all()

                html = "".join(
                    [
                        f"""<button class='remove-group-button' id='remove-group--{group.id}'
                    hx-post="{reverse('generic:users_bulk_edit')}"
                    hx-include="[name='selection']"
                    hx-confirm="This will remove the group '{group.name}' from all selected users, are you sure you wish to continue?"
                    hx-target="#htmx-info"
                    _='on click put "" into #htmx-options'
                >{group.name}</button>"""
                        for group in user_groups
                    ]
                )
                # TODO: work out how to actually use hyperscript
                html = (
                    html
                    + "<button  class='cancel-button'_='on click for el in .add-group-button remove el end then remove me'>Cancel</button>"
                )

            case r if r.startswith("add-group"):
                user_models = get_user_selection_from_request(request)

                # u: User
                for u in user_models:
                    pass
                    u.user_groups.add(r.split("--")[-1])
                    u.save()
                html = """
                <div class="alert alert-success alert-dismissible fade show" role="alert">
                    <strong>Success!</strong> Group added to all selected users.
                    <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
                </div>
                """
            case r if r.startswith("remove-group"):
                user_models = get_user_selection_from_request(request)

                # u: User
                for u in user_models:
                    group_id = r.split("--")[-1]
                    u.user_groups.remove(group_id)
                html = """
                <div class="alert alert-success alert-dismissible fade show" role="alert">
                    <strong>Success!</strong> Group removed from all selected users.
                    <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
                </div>
                """
            case _:
                html = "None"
        return HttpResponse(html, content_type="text/html")
    except NoUsersSelected:
        return HttpResponse(
            """
            <div class="alert alert-warning alert-dismissible fade show" role="alert">
                <strong>Warning!</strong> No users selected.
                <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
            </div>
            """,
            content_type="text/html",
        )


@user_is_cid_user_manager
def user_group_view(request):
    groups = UserUserGroup.objects.filter(archive=False)

    return render(
        request,
        "generic/user_group_view.html",
        {"groups": groups},
    )


@user_is_cid_user_manager
def user_group_view_all(request):
    groups = UserUserGroup.objects.filter()

    return render(
        request,
        "generic/user_group_view.html",
        {"groups": groups, "view_all": True},
    )


@user_is_cid_user_manager
def user_group_toggle_archive(request, pk):
    """HTMX endpoint to archive or unarchive a UserUserGroup.

    POST params:
      - archive: 'true' or 'false'
      - view_all: optional '1' to indicate the caller is viewing the all-groups list

    Returns an HTML fragment for the updated group card, or an empty
    response when the card should be removed from the page (e.g. archiving
    from the active-only view).
    """
    if request.method != "POST":
        return HttpResponse(status=405)

    group = get_object_or_404(UserUserGroup, pk=pk)

    archive_val = request.POST.get("archive", "true")
    group.archive = True if archive_val == "true" else False
    group.save()

    view_all = True if request.POST.get("view_all") == "1" else False

    # If the group is now archived and the caller is not viewing archived
    # groups, remove the card from the page by returning an empty fragment.
    if group.archive and not view_all:
        return HttpResponse("")

    return render(request, "generic/partials/user_group_card.html", {"group": group, "view_all": view_all})


@login_required
def user_toggle_group(request, user_id, group_id):
    """Toggle membership of a user in either a UserUserGroup or an auth Group.

    Visible to superusers and users in the `cid_user_manager` group. The
    caller may pass a querystring `?type=auth` to toggle a Django auth Group; by
    default it toggles the project's UserUserGroup.

    Returns the rendered groups fragment for HTMX swaps.
    """
    if request.method != "POST":
        return HttpResponse("Method not allowed", status=405)

    # Allow superusers or members of the cid_user_manager group
    if not (
        request.user.is_superuser
        or request.user.groups.filter(name="cid_user_manager").exists()
    ):
        return HttpResponse("Forbidden", status=403)

    user = get_object_or_404(User, pk=user_id)

    gtype = request.GET.get("type", "custom")

    if gtype == "auth":
        # Toggle Django auth Group membership
        auth_group = get_object_or_404(Group, pk=group_id)
        if auth_group.user_set.filter(pk=user.pk).exists():
            auth_group.user_set.remove(user)
        else:
            auth_group.user_set.add(user)
    else:
        # Toggle project UserUserGroup membership
        group = get_object_or_404(UserUserGroup, pk=group_id)
        if group.users.filter(pk=user.pk).exists():
            group.users.remove(user)
        else:
            group.users.add(user)

    # Prepare available lists for the fragment
    available_groups = UserUserGroup.objects.filter(archive=False).exclude(
        pk__in=user.user_groups.values_list("pk", flat=True)
    )
    available_auth_groups = Group.objects.exclude(
        pk__in=user.groups.values_list("pk", flat=True)
    )

    is_cid_user_manager = request.user.groups.filter(name="cid_user_manager").exists()

    if gtype == "auth":
        return render(
            request,
            "generic/partials/auth_groups_fragment.html",
            {
                "user": user,
                "available_auth_groups": available_auth_groups,
                "is_cid_user_manager": is_cid_user_manager,
            },
        )

    return render(
        request,
        "generic/partials/user_groups_fragment.html",
        {
            "user": user,
            "available_groups": available_groups,
            "available_auth_groups": available_auth_groups,
            "is_cid_user_manager": is_cid_user_manager,
        },
    )


@login_required
@user_is_cid_user_manager
def group_email_resend(request, pk):
    return group_email(request, pk, resend=True)


@login_required
@user_is_cid_user_manager
def group_email_results_resend(request, pk):
    return group_email_results(request, pk, resend=True)


@login_required
@user_is_cid_user_manager
def group_email(request, pk, resend=False):
    group = get_object_or_404(CidUserGroup, pk=pk)

    if resend:
        users = group.ciduser_set.filter(active=True)
    else:
        users = group.ciduser_set.filter(active=True, login_email_sent=False)

    return render(
        request,
        "generic/group_emailed.html",
        # {"sent": sent, "not_sent": not_sent, "users_count": users.count()},
        {"users": users, "users_count": users.count(), "results": False},
    )


@user_is_cid_user_manager
def group_email_results(request, pk, resend=False):
    group = get_object_or_404(CidUserGroup, pk=pk)

    if resend:
        users = group.ciduser_set.filter(active=True, internal_candidate=True)
    else:
        users = group.ciduser_set.filter(
            active=True, internal_candidate=True, results_email_sent=False
        )

    # sent = []
    # not_sent = []
    # for u in users:
    #    success, msg = u.email_results(resend=resend)
    #    if success:
    #        sent.append(u)
    #    else:
    #        not_sent.append((u, msg))

    return render(
        request,
        "generic/group_emailed.html",
        # {"sent": sent, "not_sent": not_sent, "users_count": users.count()},
        {"users": users, "users_count": users.count(), "results": True},
    )


@user_is_cid_user_manager
def candidate_email_details(request, cid, resend=False):
    user = get_object_or_404(CidUser, cid=cid)

    # Allow caller to request a resend via querystring or POST param
    resend_flag = False
    try:
        if request.method == 'POST':
            resend_flag = str(request.POST.get('resend', '')).lower() in ('1', 'true', 'yes')
        else:
            resend_flag = str(request.GET.get('resend', '')).lower() in ('1', 'true', 'yes')
    except Exception:
        resend_flag = False

    # Allow a preview mode which returns the generated email body without
    # actually sending the email. Useful for reviewing content in the UI.
    preview_flag = False
    try:
        if request.method == 'POST':
            preview_flag = str(request.POST.get('preview', '')).lower() in ('1', 'true', 'yes')
        else:
            preview_flag = str(request.GET.get('preview', '')).lower() in ('1', 'true', 'yes')
    except Exception:
        preview_flag = False

    if preview_flag:
        # Return plaintext body for preview (opens cleanly in a new tab/window)
        msg = user.generate_email_details_msg()
        return HttpResponse(msg, content_type='text/plain')

    email_sent, comment = user.email_details(resend=resend_flag)

    logger.debug(f"Email details sent to CID {cid}: sent={email_sent}, comment={comment}")

    return JsonResponse({"sent": email_sent, "comment": str(comment)})


@user_is_cid_user_manager
def candidate_email_results(request, cid, resend=False):
    user = CidUser.objects.get(cid=cid)

    email_sent, comment = user.email_results(
        additional_emails=["priya.suresh@nhs.net"], resend=True
    )

    return JsonResponse({"sent": email_sent, "comment": comment})


@user_is_cid_user_manager
def candidate_email_results_resend(request, cid, resend=True):
    return candidate_email_results(request, cid, resend=True)


@user_is_cid_user_manager
def create_cid_email(request):
    pass


@user_is_cid_user_manager
def cid_details(request, cid: int):
    """Return a brief display string for a CID user (HTMX partial).

    Scope: cid_user_manager and superusers only.
    Functionality: returns the CID user's name and email as plain text for
    HTMX inline display.
    """
    if request.htmx:
        cid_user = get_object_or_404(CidUser, cid=cid)
        return HttpResponse(f"{cid_user.name} ({cid_user.email})")

    raise PermissionDenied()  # or Http404


@user_is_cid_user_manager
def manage_cid_users(request):
    if request.method == "POST":
        physics_exams = json.loads(request.POST.get("physics_exams"))
        rapid_exams = json.loads(request.POST.get("rapid_exams"))
        sba_exams = json.loads(request.POST.get("sba_exams"))
        longs_exams = json.loads(request.POST.get("longs_exams"))
        anatomy_exams = json.loads(request.POST.get("anatomy_exams"))
        shorts_exams = json.loads(request.POST.get("shorts_exams"))
        casecollection_exams = json.loads(request.POST.get("casecollection_exams"))
        cid_groups = json.loads(request.POST.get("cid_groups"))

        selected_cids = json.loads(request.POST.get("selected_cids"))

        emails = json.loads(request.POST.get("emails"))

        number_to_create = int(request.POST.get("number_to_create"))

        add_group = request.POST.get("add_group")
        delete = request.POST.get("delete")
        activate = request.POST.get("activate")
        deactivate = request.POST.get("deactivate")
        add_exams = request.POST.get("add_exams")
        change_exams = request.POST.get("change_exams")
        clear_exams = request.POST.get("clear_exams")
        toggle_internal = request.POST.get("toggle_internal")

        if add_group == "true":
            cids = CidUser.objects.filter(pk__in=selected_cids)
            for c in cids:
                c.group_id = cid_groups
                c.save()
            return JsonResponse({"status": "success"})

        if delete == "true":
            if not request.user.is_superuser:
                return JsonResponse({"status": "fail"})

            cids = CidUser.objects.filter(pk__in=selected_cids).delete()
            return JsonResponse({"status": "success"})

        if activate == "true":
            cids = CidUser.objects.filter(pk__in=selected_cids)
            for c in cids:
                c.active = True
                c.save()
            return JsonResponse({"status": "success"})

        if toggle_internal == "true":
            cids = CidUser.objects.filter(pk__in=selected_cids)
            for c in cids:
                c.internal_candidate = not c.internal_candidate
                c.save()
            return JsonResponse({"status": "success"})

        if deactivate == "true":
            cids = CidUser.objects.filter(pk__in=selected_cids)
            for c in cids:
                c.active = False
                c.save()
            return JsonResponse({"status": "success"})

        if clear_exams == "true":
            cids = CidUser.objects.filter(pk__in=selected_cids)
            for c in cids:
                c.physics_exams.clear()
                c.rapid_exams.clear()
                c.sba_exams.clear()
                c.longs_exams.clear()
                c.anatomy_exams.clear()
                c.shorts_exams.clear()
                c.casecollection_exams.clear()
            return JsonResponse({"status": "success"})

        if add_exams == "true":
            cids = CidUser.objects.filter(pk__in=selected_cids)
            for c in cids:
                for exam in physics_exams:
                    c.physics_exams.add(exam)
                for exam in rapid_exams:
                    c.rapid_exams.add(exam)
                for exam in sba_exams:
                    c.sba_exams.add(exam)
                for exam in longs_exams:
                    c.longs_exams.add(exam)
                for exam in anatomy_exams:
                    c.anatomy_exams.add(exam)
                for exam in shorts_exams:
                    c.shorts_exams.add(exam)
                for exam in casecollection_exams:
                    c.casecollection_exams.add(exam)
                c.save()
            return JsonResponse({"status": "success"})

        if change_exams == "true":
            cids = CidUser.objects.filter(pk__in=selected_cids)
            for c in cids:
                c.physics_exams.set(physics_exams)
                c.rapid_exams.set(rapid_exams)
                c.sba_exams.set(sba_exams)
                c.longs_exams.set(longs_exams)
                c.anatomy_exams.set(anatomy_exams)
                c.shorts_exams.set(shorts_exams)
                c.casecollection_exams.set(casecollection_exams)
                c.save()
            return JsonResponse({"status": "success"})

        try:
            new_cid = get_next_cid()
        except IndexError:
            new_cid = 10000

        if "add_new_emails" in request.POST:
            email_list = [i.strip() for i in emails.split()]

            # Verify emails are all valid
            invalid_emails = []
            for email in email_list:
                try:
                    validate_email(email)
                except ValidationError as e:
                    invalid_emails.append(f"Invalid email: {email}")

            logger.debug("bulk_create_cid_users: email_list count={}", len(email_list))
            if not email_list:
                return JsonResponse({"status": "fail", "details": "No valid emails"})
            if invalid_emails or not email_list:
                e = "<br/>".join(invalid_emails)
                return JsonResponse(
                    {"status": "fail", "details": f"Unable to create users<hr>{e}"}
                )

            for e in email_list:
                passcode = "".join(random.choices(string.ascii_uppercase, k=4))
                c = CidUser(cid=new_cid, passcode=passcode, email=e)
                c.save()
                if cid_groups:
                    c.group_id = cid_groups[0]

                if physics_exams:
                    c.physics_exams.set(physics_exams)
                if rapid_exams:
                    c.rapid_exams.set(rapid_exams)
                if sba_exams:
                    c.sba_exams.set(sba_exams)
                if longs_exams:
                    c.longs_exams.set(longs_exams)
                if anatomy_exams:
                    c.anatomy_exams.set(anatomy_exams)
                if shorts_exams:
                    c.shorts_exams.set(shorts_exams)
                if casecollection_exams:
                    c.casecollection_exams.set(casecollection_exams)
                c.save()
                new_cid = new_cid + 1

        for n in range(number_to_create):
            passcode = "".join(random.choices(string.ascii_uppercase, k=4))
            c = CidUser(cid=new_cid, passcode=passcode)
            c.save()
            if cid_groups:
                c.group_id = cid_groups[0]

            if physics_exams:
                c.physics_exams.set(physics_exams)
            if rapid_exams:
                c.rapid_exams.set(rapid_exams)
            if sba_exams:
                c.sba_exams.set(sba_exams)
            if longs_exams:
                c.longs_exams.set(longs_exams)
            if anatomy_exams:
                c.anatomy_exams.set(anatomy_exams)
            if shorts_exams:
                c.shorts_exams.set(shorts_exams)
            if casecollection_exams:
                c.casecollection_exams.set(casecollection_exams)
            c.save()
            new_cid = new_cid + 1

        return JsonResponse({"status": "success"})
    return JsonResponse({"status": "fail"})


class CidUserCreate(CidManagerRequiredMixin, CreateView):
    model = CidUser
    form_class = CidUserForm


class CidUserUpdate(CidManagerRequiredMixin, UpdateView):
    model = CidUser
    form_class = CidUserForm


class CidUserExamUpdate(CidManagerRequiredMixin, UpdateView):
    # This view edits the CidUser (which stores which exams the CID is registered for).
    # Keep model as `CidUser` so existing URL `update_cid_exams` continues to work.
    model = CidUser
    form_class = CidUserExamForm
    template_name = "generic/ciduserexam_form.html"

    # def get_context_data(self, *args, **kwargs):
    #    context = super().get_context_data(**kwargs)
    #    context["test"] = "HELLO"
    #    return context


class CidUserGroupDelete(RevisionMixin, CidManagerRequiredMixin, DeleteView):
    model = CidUserGroup
    template_name = "confirm_delete.html"
    success_url = reverse_lazy("generic:cid_group_view")


class CidUserGroupCreate(RevisionMixin, CidManagerRequiredMixin, CreateView):
    model = CidUserGroup
    form_class = CidUserGroupForm


class CidUserExamRecordUpdate(CidManagerRequiredMixin, UpdateView):
    """Update a CidUserExam record (individual exam record for a CID/user).

    Separate from `CidUserExamUpdate` which edits a `CidUser`'s exam memberships.
    """
    model = CidUserExam
    form_class = None
    template_name = "generic/ciduserexam_record_form.html"

    def get_form_class(self):
        # import locally to avoid circular imports at module import time
        from .forms import CidUserExamRecordForm

        return CidUserExamRecordForm


class CidUserGroupUpdate(RevisionMixin, CidManagerRequiredMixin, UpdateView):
    model = CidUserGroup
    form_class = CidUserGroupForm

    def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
        context = super().get_context_data(**kwargs)
        context["group_type"] = "cid"
        return context


class UserUserGroupCreate(RevisionMixin, CidManagerRequiredMixin, CreateView):
    model = UserUserGroup
    form_class = UserUserGroupForm


class UserUserGroupUpdate(RevisionMixin, CidManagerRequiredMixin, UpdateView):
    model = UserUserGroup
    form_class = UserUserGroupForm

    def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
        context = super().get_context_data(**kwargs)
        context["group_type"] = "user"
        return context


class UserUserGroupDelete(RevisionMixin, CidManagerRequiredMixin, DeleteView):
    model = UserUserGroup
    template_name = "confirm_delete.html"
    success_url = reverse_lazy("generic:user_group_view")


class UserGroupExamUpdate(CidManagerRequiredMixin, UpdateView):
    model = UserUserGroup
    form_class = UserGroupExamForm

    def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
        context = super().get_context_data(**kwargs)
        context["group_type"] = "user"
        return context


class CidGroupExamUpdate(CidManagerRequiredMixin, UpdateView):
    model = CidUserGroup
    form_class = CidGroupExamForm

    def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
        context = super().get_context_data(**kwargs)
        context["group_type"] = "cid"
        return context


class ExamCreateBase(RevisionMixin, LoginRequiredMixin, CreateView):
    template_name = "exam_create_form.html"

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.save()

        form.instance.author.add(self.request.user.id)
        return super().form_valid(form)

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super().get_context_data(**kwargs)
        return context

    def get_form_kwargs(self):
        kwargs = super(ExamCreateBase, self).get_form_kwargs()
        kwargs.update({"user": self.request.user})
        return kwargs


class ExamUpdateBase(RevisionMixin, CheckCanEditMixin, LoginRequiredMixin, UpdateView):
    template_name = "exam_update_form.html"

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.save()

        form.instance.author.add(self.request.user.id)

        return super().form_valid(form)

    def get_form_kwargs(self):
        kwargs = super(ExamUpdateBase, self).get_form_kwargs()
        kwargs.update({"user": self.request.user})
        return kwargs


class ExamDeleteBase(RevisionMixin, CheckCanEditMixin, DeleteView):
    template_name = "exam_confirm_delete.html"

    def get_success_url(self) -> str:
        # return super().get_success_url()(self):
        return reverse_lazy(f"{self.model.app_name}:index")


def exam_inactive(request, context):
    return render(request, "exam_inactive.html", context)


# class UserUserCreate(CidManagerRequiredMixin, CreateView):
#    model = User
#    form_class = UserUserForm
#    template_name: str = "generic/user_creation_form.html"
#
#    success_url = reverse_lazy("accounts_list")


@user_is_cid_user_manager
def trainees(request, grade: None | str = None):

    trainees = (
        UserProfile.objects.filter(peninsula_trainee=True, user__is_active=True)
        .order_by("grade__name", Lower("user__last_name"))
        .prefetch_related("supervisor", "grade", "user")
    )

    if grade is not None:
        trainees = trainees.filter(grade__name=grade)

    context = {"trainees": trainees, "grade": grade}

    return render(request, "generic/trainees.html", context)


@user_is_cid_user_manager
def trainees_bulk_update_from_spreadsheet(request):
    """Bulk update trainees based on pasted spreadsheet-like data.

    The view works in two stages:
    - Preview: parse the pasted text and show rows with any matched user / supervisor
    - Apply: apply grade and supervisor updates for matched users
    """

    import re

    preview = []
    report = None

    if request.method == "POST":
        action = request.POST.get("action", "preview")
        data = request.POST.get("data", "")

        lines = [l.strip() for l in data.splitlines()]

        current_grade = None

        for idx, line in enumerate(lines):
            if not line:
                continue

            # detect grade headings like 'Year 1 - Group 1' -> ST1
            m = re.search(r"Year\s*(\d+)", line, re.IGNORECASE)
            if m:
                num = m.group(1)
                current_grade = f"ST{num}"
                continue

            # split columns by tab or two+ spaces
            parts = re.split(r"\t| {2,}", line)
            if len(parts) < 2:
                # maybe single-space separated; try splitting on tab or single tab
                parts = [p.strip() for p in line.split("\t") if p.strip()]

            if not parts:
                continue

            name_part = parts[0].strip()
            supervisor_part = parts[1].strip() if len(parts) > 1 else ""

            # normalize name (remove parenthetical annotations)
            norm_name = re.sub(r"\s*\([^)]*\)", "", name_part).strip()
            # split into tokens; assume first token is first name, last token is last name
            tokens = [t for t in re.split(r"\s+", norm_name) if t]
            first = tokens[0] if tokens else ""
            last = tokens[-1] if tokens else ""

            matched_user = None
            if first and last:
                # Prefer an exact first+last match
                qs = User.objects.filter(first_name__iexact=first, last_name__iexact=last)
                if qs.exists():
                    matched_user = qs.first()
                else:
                    # Fallback: only match by last name when the last-name query returns
                    # a single candidate AND the candidate's first name reasonably
                    # matches the provided first token. This avoids ambiguous matches
                    # (e.g. Maria vs Stephanie both 'Bailey'). We accept a match when
                    # the candidate first name startswith the first 3 chars of the
                    # provided first token, or contains it as a substring (case-ins).
                    candidates = User.objects.filter(last_name__iexact=last)
                    if candidates.count() == 1:
                        candidate = candidates.first()
                        cand_first = (candidate.first_name or "").strip()
                        if cand_first:
                            # take at most 3 chars for prefix comparison
                            prefix = first[:3].lower()
                            cand_first_low = cand_first.lower()
                            if cand_first_low.startswith(prefix) or first.lower() in cand_first_low:
                                matched_user = candidate

            matched_supervisor = None
            if supervisor_part:
                sup_qs = Supervisor.objects.filter(name__icontains=supervisor_part)
                if sup_qs.exists():
                    matched_supervisor = sup_qs.first()

            preview.append(
                {
                    "raw": line,
                    "row_index": idx,
                    "name": name_part,
                    "norm_name": norm_name,
                    "first": first,
                    "last": last,
                    "supervisor_text": supervisor_part,
                    "matched_user": matched_user,
                    "matched_supervisor": matched_supervisor,
                    "grade": current_grade,
                }
            )

        if action == "apply":
            updated = 0
            skipped = 0
            for row in preview:
                # Check for a manual override from the submitted form
                manual_key = f"manual_user_{row.get('row_index')}"
                manual_val = request.POST.get(manual_key)

                user = None
                if manual_val:
                    try:
                        user = User.objects.get(pk=int(manual_val))
                    except Exception:
                        user = None

                # Fall back to automatic match
                if not user:
                    user = row.get("matched_user")
                if not user:
                    skipped += 1
                    continue

                profile = user.userprofile

                # apply grade if present
                grade_name = row.get("grade")
                if grade_name:
                    grade_obj, _ = UserGrades.objects.get_or_create(name=grade_name)
                    profile.grade = grade_obj

                # apply supervisor if matched
                sup = row.get("matched_supervisor")
                if sup:
                    profile.supervisor = sup

                profile.save()
                updated += 1

            report = {"updated": updated, "skipped": skipped}

    return render(
        request,
        "generic/trainees_bulk_update.html",
        {"preview": preview, "report": report},
    )


def create_trainee(request, context=None):
    return create_user(request, context, trainee=True)


@user_is_cid_user_manager
def create_user(request, context=None, trainee: bool = False):
    if trainee:
        form_template = "generic/trainee_creation_form.html"
    else:
        form_template = "generic/user_creation_form.html"
    # if this is a POST request we need to process the form data
    if request.method == "POST":
        # create a form instance and populate it with data from the request:
        if trainee:
            form = TraineeForm(request.POST)
        else:
            form = UserUserForm(request.POST)
        # check whether it's valid:

        if form.is_valid():
            try:
                username = form.cleaned_data["username"].strip().lower()
                existing_user = User.objects.filter(
                    Q(username__iexact=username) | Q(email__iexact=username)
                ).first()

                user_dict = {
                    "username": username,
                    "first_name": form.cleaned_data["first_name"],
                    "last_name": form.cleaned_data["last_name"],
                    "email": username,
                    "password": secrets.token_hex(nbytes=16),
                }

                with transaction.atomic():
                    if existing_user is not None:
                        existing_user.username = user_dict["username"]
                        existing_user.email = user_dict["email"]
                        existing_user.first_name = user_dict["first_name"]
                        existing_user.last_name = user_dict["last_name"]
                        existing_user.save(
                            update_fields=["username", "email", "first_name", "last_name"]
                        )
                        new_user = existing_user
                    else:
                        new_user = User.objects.create_user(**user_dict)
            except Exception as error:
                errors = f"Unable to create account: {error}"
                return render(
                    request,
                    form_template,
                    {"form": form, "errors": errors},
                )

            try:
                user_profile, _ = UserProfile.objects.get_or_create(user=new_user)
                user_profile.peninsula_trainee = trainee

                if form.cleaned_data.get("grade"):
                    grade = form.cleaned_data["grade"]
                    user_profile.grade = grade

                if trainee and form.cleaned_data.get("supervisor"):
                    supervisor = form.cleaned_data["supervisor"]
                    user_profile.supervisor = supervisor
                user_profile.save()

                if not trainee and form.cleaned_data.get("create_supervisor"):
                    from generic.models import Supervisor
                    supervisor, created = Supervisor.objects.get_or_create(
                        email=new_user.email,
                        defaults={
                            "name": f"{new_user.first_name} {new_user.last_name}".strip(),
                            "user": new_user,
                        }
                    )
                    if not created and not supervisor.user:
                        supervisor.user = new_user
                        supervisor.name = f"{new_user.first_name} {new_user.last_name}".strip()
                        supervisor.save(update_fields=["user", "name"])

            except Exception as error:
                errors = f"Unable to create account profile {error}"
                return render(
                    request,
                    form_template,
                    {"form": form, "errors": errors},
                )

            return HttpResponseRedirect(reverse_lazy("accounts_list"))
            return reverse_lazy("accounts_list")

    # if a GET (or any other method) we'll create a blank form
    else:
        if trainee:
            form = TraineeForm()
        else:
            form = UserUserForm()

    return render(request, form_template, {"form": form})


class SupervisorDetail(DetailView):
    model = Supervisor

    def get_object(self, queryset=None):
        """Return the supervisor if the requesting user is authorised.

        - Superusers and ``cid_user_manager`` group members may view any supervisor.
        - A supervisor linked to the requesting user may view their own record.
        - All other authenticated users receive PermissionDenied (403).
        - Unauthenticated users are redirected to the login page by
          ``LoginRequiredMixin`` (enforced via the URLconf decorator or mixin).
        """
        obj = super().get_object(queryset)
        if (
            self.request.user.is_superuser
            or self.request.user.groups.filter(name="cid_user_manager").exists()
            or (hasattr(self.request.user, "supervisor") and obj.user == self.request.user)
        ):
            return obj
        raise PermissionDenied


@user_is_cid_user_manager
def supervisor_trainee_search(request, pk):
    """HTMX endpoint: search for users to add as trainees to a supervisor.
    Scope: managers and the supervisor themselves.
    """
    supervisor = get_object_or_404(Supervisor, pk=pk)

    if not (
        request.user.is_superuser
        or request.user.groups.filter(name="cid_user_manager").exists()
        or supervisor.user == request.user
    ):
        raise PermissionDenied

    q = request.GET.get("q", "").strip()
    if not q:
        return render(
            request,
            "generic/partials/supervisor_trainee_search_results.html",
            {"supervisor": supervisor, "results": []},
        )

    already_trainees = supervisor.trainee.values_list("user_id", flat=True)
    results = (
        User.objects.filter(
            Q(first_name__icontains=q)
            | Q(last_name__icontains=q)
            | Q(username__icontains=q)
            | Q(email__icontains=q)
        )
        .exclude(pk__in=already_trainees)
        .order_by("first_name", "last_name", "username")[:20]
    )

    return render(
        request,
        "generic/partials/supervisor_trainee_search_results.html",
        {"supervisor": supervisor, "results": results},
    )


@require_POST
@user_is_cid_user_manager
def supervisor_trainee_add(request, pk, user_id):
    """Add a trainee to a supervisor.
    Scope: managers and the supervisor themselves.
    """
    supervisor = get_object_or_404(Supervisor, pk=pk)
    trainee_user = get_object_or_404(User, pk=user_id)

    if not (
        request.user.is_superuser
        or request.user.groups.filter(name="cid_user_manager").exists()
        or supervisor.user == request.user
    ):
        raise PermissionDenied

    profile = trainee_user.userprofile
    profile.supervisor = supervisor
    profile.save()

    if request.htmx:
        return render(
            request,
            "generic/partials/supervisor_trainees_list.html",
            {"supervisor": supervisor},
        )
    return redirect(supervisor.get_absolute_url())


@require_POST
@user_is_cid_user_manager
def supervisor_trainee_remove(request, pk, user_id):
    """Remove a trainee from a supervisor.
    Scope: managers and the supervisor themselves.
    """
    supervisor = get_object_or_404(Supervisor, pk=pk)
    trainee_user = get_object_or_404(User, pk=user_id)

    if not (
        request.user.is_superuser
        or request.user.groups.filter(name="cid_user_manager").exists()
        or supervisor.user == request.user
    ):
        raise PermissionDenied

    profile = trainee_user.userprofile
    if profile.supervisor == supervisor:
        profile.supervisor = None
        profile.save()

    if request.htmx:
        return render(
            request,
            "generic/partials/supervisor_trainees_list.html",
            {"supervisor": supervisor},
        )
    return redirect(supervisor.get_absolute_url())


class SupervisorDelete(CidManagerRequiredMixin, DeleteView):
    model = Supervisor
    template_name: str = "confirm_delete.html"
    success_url = reverse_lazy("generic:supervisor")


class SupervisorUpdate(RedirectMixin, CidManagerRequiredMixin, UpdateView):
    model = Supervisor
    form_class = SupervisorForm
    # success_url = reverse_lazy("generic:supervisor_detail", kwargs={'pk': self.pk})


class SupervisorCreate(CidManagerRequiredMixin, CreateView):
    model = Supervisor
    form_class = SupervisorForm
    # success_url = reverse_lazy("generic:supervisor_detail")


# class SupervisorList(CidManagerRequiredMixin, ListView):
#    model = Supervisor


class SupervisorList(CidManagerRequiredMixin, SingleTableMixin, FilterView):
    model = Supervisor
    table_class = SupervisorTable
    template_name = "generic/supervisor_list.html"

    filterset_class = SupervisorFilter


class ExamCollectionList(LoginRequiredMixin, ListView):
    model = ExamCollection
    
    def get_queryset(self):
        """Annotate exam counts per collection and prefetch authors to avoid N+1 queries in templates."""
        # allow toggling whether archived collections are included via ?show_archived=1
        show_archived = self.request.GET.get("show_archived")

        # If user is anonymous return empty set; otherwise limit to collections
        # where the current user is listed as an author.
        if not self.request.user.is_authenticated:
            return ExamCollection.objects.none()

        base_qs = ExamCollection.objects.filter(author=self.request.user)
        if not show_archived:
            base_qs = base_qs.filter(archive=False)

        # Build annotation kwargs dynamically so counts respect the show_archived toggle.
        # Count exams regardless of exam_mode; include archived depending on the toggle.
        if show_archived:
            # include archived exams in counts (count all exams for each app)
            counts = {
                "anatomy_count": Count("anatomy_exams", distinct=True),
                "longs_count": Count("longs_exams", distinct=True),
                "physics_count": Count("physics_exams", distinct=True),
                "rapids_count": Count("rapids_exams", distinct=True),
                "sbas_count": Count("sbas_exams", distinct=True),
                "shorts_count": Count("shorts_exams", distinct=True),
            }
        else:
            # only count non-archived exams
            counts = {
                "anatomy_count": Count(
                    "anatomy_exams",
                    filter=Q(anatomy_exams__archive=False),
                    distinct=True,
                ),
                "longs_count": Count(
                    "longs_exams",
                    filter=Q(longs_exams__archive=False),
                    distinct=True,
                ),
                "physics_count": Count(
                    "physics_exams",
                    filter=Q(physics_exams__archive=False),
                    distinct=True,
                ),
                "rapids_count": Count(
                    "rapids_exams",
                    filter=Q(rapids_exams__archive=False),
                    distinct=True,
                ),
                "sbas_count": Count(
                    "sbas_exams",
                    filter=Q(sbas_exams__archive=False),
                    distinct=True,
                ),
                "shorts_count": Count(
                    "shorts_exams",
                    filter=Q(shorts_exams__archive=False),
                    distinct=True,
                ),
            }

        qs = (
            base_qs
            .annotate(**counts)
            .annotate(
                total_count=F("anatomy_count")
                + F("longs_count")
                + F("physics_count")
                + F("rapids_count")
                + F("sbas_count")
                + F("shorts_count")
            )
            .prefetch_related("author")
            .order_by("name")
        )

        return qs


class ExamCollectionDetail(AuthorRequiredMixin, DetailView):
    model = ExamCollection
    # Annotate counts for each exam type (non-archived) so template can check presence
    # without evaluating entire related querysets.
    def get_queryset(self):
        qs = super().get_queryset()
        counts = {
            "anatomy_count": Count(
                "anatomy_exams", filter=Q(anatomy_exams__archive=False), distinct=True
            ),
            "longs_count": Count(
                "longs_exams", filter=Q(longs_exams__archive=False), distinct=True
            ),
            "physics_count": Count(
                "physics_exams", filter=Q(physics_exams__archive=False), distinct=True
            ),
            "rapids_count": Count(
                "rapids_exams", filter=Q(rapids_exams__archive=False), distinct=True
            ),
            "sbas_count": Count(
                "sbas_exams", filter=Q(sbas_exams__archive=False), distinct=True
            ),
            "shorts_count": Count(
                "shorts_exams", filter=Q(shorts_exams__archive=False), distinct=True
            ),
        }
        try:
            return qs.annotate(**counts)
        except Exception:
            return qs


class ExamCollectionEdit(AuthorRequiredMixin, UpdateView):
    model = ExamCollection
    form_class = ExamCollectionForm

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        # Ensure we have a bound form to extract bound fields
        form = kwargs.get('form') or self.get_form()
        groups = []
        for label, field_name, model in getattr(self.form_class, 'GROUP_TYPES', []):
            try:
                bound = form[field_name]
            except Exception:
                bound = None
            groups.append((label, field_name, bound))
        context['exam_groups'] = groups
        return context


class ExamCollectionCreate(AuthorRequiredMixin, CreateView):
    model = ExamCollection
    form_class = ExamCollectionForm

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        form = kwargs.get('form') or self.get_form()
        groups = []
        for label, field_name, model in getattr(self.form_class, 'GROUP_TYPES', []):
            try:
                bound = form[field_name]
            except Exception:
                bound = None
            groups.append((label, field_name, bound))
        context['exam_groups'] = groups
        return context


class ExamCollectionClone(AuthorRequiredMixin, CreateView):
    model = ExamCollection
    template_name = "generic/examcollection_clone_form.html"
    form_class = ExamCollectionCloneForm

    def get_initial(self):
        old_object = get_object_or_404(self.model, pk=self.kwargs["pk"])
        self.initial_object = old_object

        initial_data = model_to_dict(old_object, exclude=["id", "date"])

        ## We manually transfer the forign keys / m2m relationships
        # questions = old_object.exam_questions.all().values_list("id", flat=True)
        authors = old_object.author.all().values_list("id", flat=True)

        ## clear the associated groups
        # initial_data["valid_user_users"] = []
        # initial_data["valid_cid_users"] = []
        # initial_data["cid_user_groups"] = []
        # initial_data["user_user_groups"] = []

        # initial_data["active"] = False
        # initial_data["publish_results"] = False

        # self.exam_questions = list(questions)
        self.author = list(authors)

        return initial_data

    def form_valid(self, form):
        object = form.save()
        # Reapply these otherwise they get lost?
        # object.exam_questions.set(self.exam_questions)

        GROUP_TYPES = (
            ("Rapid Exams", "rapids_exams", RapidsExam),
            ("Short Exams", "shorts_exams", ShortsExam),
            ("Long Exams", "longs_exams", LongsExam),
            ("SBA Exams", "sbas_exams", SbasExam),
            ("Physic Exams", "physics_exams", PhysicsExam),
            ("Anatomy Exams", "anatomy_exams", AnatomyExam),
        )

        object.author.set(self.author)
        object.save()

        # Clone exams linked to the original collection into the new one.
        # For each supported exam app, find exams related to the old collection
        # and create shallow copies (questions/authors re-applied, but
        # active/archive/publish flags cleared).
        if hasattr(self, "initial_object") and self.initial_object:
            for _label, rel_name, ExamModel in GROUP_TYPES:
                try:
                    old_exams = list(getattr(self.initial_object, rel_name).all())
                except Exception:
                    old_exams = []

                for old_exam in old_exams:
                    # Build a dict of field values excluding the PK
                    data = model_to_dict(old_exam, exclude=["id"])

                    # Clear group membership and flags for cloned exams
                    data["valid_user_users"] = [] if "valid_user_users" in data else []
                    data["valid_cid_users"] = [] if "valid_cid_users" in data else []
                    data["cid_user_groups"] = [] if "cid_user_groups" in data else []
                    data["user_user_groups"] = [] if "user_user_groups" in data else []

                    data["active"] = False
                    # some models use `archive` or `archived` - try both safely
                    if "archive" in data:
                        data["archive"] = False
                    if "archived" in data:
                        data["archived"] = False
                    data["publish_results"] = False if "publish_results" in data else data.get("publish_results", False)

                    # Keep a copy of M2M fields to reapply after create
                    m2m_backup = {}
                    for m2m_field in ("exam_questions", "author", "markers", "valid_user_users", "valid_cid_users", "cid_user_groups", "user_user_groups"):
                        if m2m_field in data:
                            m2m_backup[m2m_field] = data.pop(m2m_field)

                    # If the user supplied a date on the new collection, apply it to
                    # cloned exams so the cloned exams have the requested date.
                    if getattr(object, "date", None):
                        data["date"] = object.date

                    # Set the new examcollection foreign key to the newly created collection
                    # model_to_dict returns the FK id; replace with instance for create()
                    data["examcollection"] = object

                    try:
                        new_exam = ExamModel.objects.create(**data)
                    except Exception:
                        # If creation failed for any model-specific field, skip cloning this exam
                        continue

                    # Reapply M2M relationships where possible
                    for field_name, ids in m2m_backup.items():
                        try:
                            getattr(new_exam, field_name).set(ids)
                        except Exception:
                            # If the field doesn't exist on this model, ignore
                            pass

                    try:
                        # Some exam models provide an ordering helper
                        if hasattr(new_exam, "order_questions"):
                            new_exam.order_questions()
                    except Exception:
                        pass

        return HttpResponseRedirect(object.get_absolute_url())


class ExamCollectionDelete(AuthorRequiredMixin, DeleteView):
    model = ExamCollection
    template_name = "confirm_delete.html"
    success_url = reverse_lazy("generic:examcollection_list")


class ExaminationView(SuperuserRequiredMixin, SingleTableMixin, FilterView):
    model = Examination
    table_class = ExaminationTable
    template_name = "atlas/view.html"

    filterset_class = ExaminationFilter


class ExaminationDelete(RevisionMixin, SuperuserRequiredMixin, DeleteView):
    model = Examination
    template_name = "confirm_delete.html"
    success_url = reverse_lazy("generic:examination_view")


class ExaminationUpdate(SuperuserRequiredMixin, UpdateView):
    model = Examination
    form_class = ExaminationForm


class SeriesImagesZipViewBase(SuperuserRequiredMixin, View):
    """Download all images from an image series"""

    http_method_names = ["get"]
    series_object = None

    def get_files(self):
        series = self.series_object.objects.get(pk=self.kwargs["pk"])

        return [i.image.file for i in series.get_images()]

    def get_archive_name(self, request):
        # series = Series.objects.get(pk=self.kwargs["pk"])
        return f"Series {self.kwargs['pk']}.zip"

    def get(self, request, *args, **kwargs):
        temp_file = ContentFile(b"", name=self.get_archive_name(request))
        with zipfile.ZipFile(
            temp_file, mode="w", compression=zipfile.ZIP_DEFLATED
        ) as zip_file:
            files = self.get_files()
            for file_ in files:
                path = os.path.split(file_.name)[-1]
                zip_file.writestr(path, file_.read())

        file_size = temp_file.tell()
        temp_file.seek(0)

        response = HttpResponse(temp_file, content_type="application/zip")
        response["Content-Disposition"] = (
            "attachment; filename=%s" % self.get_archive_name(request)
        )
        response["Content-Length"] = file_size
        return response


class ExamGroupsUpdateBase(
    RevisionMixin,
    CheckCanEditMixin,
    LoginRequiredMixin,
    AuthorRequiredMixin,
    UpdateView,
):
    template_name = "generic/exam_groups_edit.html"

    def get_success_url(self) -> str:
        return reverse_lazy(
            f"{self.object.get_app_name()}:exam_cids",
            kwargs={"exam_id": self.object.pk},
        )

    def get_form_kwargs(self):
        kwargs = super(ExamGroupsUpdateBase, self).get_form_kwargs()
        kwargs.update({"user": self.request.user})
        return kwargs


class UpdateQuestionMixin(RedirectMixin, RevisionMixin, UpdateView):

    def get_form_kwargs(self):
        kwargs = super().get_form_kwargs()
        kwargs.update({"user": self.request.user})

        return kwargs

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["question"] = context["object"]

        # Do permission checks here

        obj = context["object"]
        if self.request.user in obj.get_author_objects():
            return context
        elif (
            self.request.user.groups.filter(name="long_checker").exists()
            or self.request.user.is_superuser
        ):
            return context

        review_group = f"{obj.get_app_name()}_reviewer"
        logger.debug(f"Checking for group membership: {review_group}")
        if self.request.user.groups.filter(name=review_group).exists():
            return context
        raise PermissionDenied()  # or Http404


@login_required
def supervisor_overview(request, pk):
    supervisor = get_object_or_404(Supervisor, pk=pk)

    if not request.user.is_superuser:
        try:
            if not request.user.supervisor == supervisor:
                raise PermissionDenied()
        except User.supervisor.RelatedObjectDoesNotExist:
            raise PermissionDenied()

    trainees = User.objects.filter(userprofile__supervisor=supervisor).select_related("userprofile__grade", "userprofile__site")
    attempts = get_supervisor_attempts(supervisor, trainees)

    # Calculate statistics for trainees
    trainee_stats = []
    for trainee in trainees:
        trainee_attempts = [a for a in attempts if a["trainee"] == trainee]
        total_count = len(trainee_attempts)
        shared_count = sum(1 for a in trainee_attempts if a["is_shared"])
        private_count = total_count - shared_count
        
        last_active = None
        if trainee_attempts:
            last_active = trainee_attempts[0]["date"]
            
        trainee_stats.append({
            "trainee": trainee,
            "total_count": total_count,
            "shared_count": shared_count,
            "private_count": private_count,
            "last_active": last_active,
        })

    return render(
        request,
        "generic/supervisor_overview.html",
        {
            "supervisor": supervisor,
            "trainees": trainees,
            "trainee_stats": trainee_stats,
            "attempts": attempts,
        },
    )


@login_required
def supervisor_trainee(request, pk, trainee_id):
    supervisor = get_object_or_404(Supervisor, pk=pk)

    if not request.user.is_superuser:
        try:
            if not request.user.supervisor == supervisor:
                raise PermissionDenied()
        except User.supervisor.RelatedObjectDoesNotExist:
            raise PermissionDenied()

    trainee = User.objects.get(userprofile__supervisor=supervisor, pk=trainee_id)
    attempts = get_supervisor_attempts(supervisor, [trainee])

    return render(
        request,
        "generic/supervisor_trainee.html",
        {
            "supervisor": supervisor,
            "trainee": trainee,
            "attempts": attempts,
        },
    )


@login_required
def supervisor_collection_feedback(request, pk, trainee_id, collection_id):
    from atlas.models import CaseCollection, CaseDetail
    from atlas.views import _build_collection_feedback_rows
    from django.contrib.contenttypes.models import ContentType

    supervisor = get_object_or_404(Supervisor, pk=pk)

    if not request.user.is_superuser:
        try:
            if not request.user.supervisor == supervisor:
                raise PermissionDenied()
        except User.supervisor.RelatedObjectDoesNotExist:
            raise PermissionDenied()

    trainee = get_object_or_404(User, userprofile__supervisor=supervisor, pk=trainee_id)
    collection = get_object_or_404(CaseCollection, pk=collection_id)

    ct = ContentType.objects.get_for_model(CaseCollection)
    cid_user_exam = get_object_or_404(
        CidUserExam, user_user=trainee, content_type=ct, object_id=collection.pk
    )

    if not (cid_user_exam.share_with_supervisor or collection.results_supervisor_visible):
        raise PermissionDenied("This collection attempt has not been shared with the supervisor.")

    casedetails = list(
        CaseDetail.objects.filter(collection=collection).select_related("case").order_by("sort_order")
    )
    feedback_rows, outstanding_case_count, total_outstanding_feedback = _build_collection_feedback_rows(
        collection=collection,
        cid_user_exam=cid_user_exam,
        casedetails=casedetails,
    )

    outstanding_rows = [row for row in feedback_rows if row["stats"].get("has_outstanding_feedback")]
    history_rows = [row for row in feedback_rows if row["stats"].get("has_messages") and not row["stats"].get("has_outstanding_feedback")]

    return render(
        request,
        "generic/supervisor_collection_feedback.html",
        {
            "supervisor": supervisor,
            "trainee": trainee,
            "collection": collection,
            "cid_user_exam": cid_user_exam,
            "feedback_rows": feedback_rows,
            "outstanding_rows": outstanding_rows,
            "history_rows": history_rows,
            "outstanding_case_count": outstanding_case_count,
            "total_outstanding_feedback": total_outstanding_feedback,
        },
    )


def supervisor_request_account(request):
    if request.method == "POST":
        email = request.POST.get("email")
        try:
            validate_email(email)
        except ValidationError:
            return HttpResponse(f"Invalid email address ({email})")

        try:
            supervisor = Supervisor.objects.get(email=email)
        except Supervisor.DoesNotExist:
            return HttpResponse(f"Invalid email address ({email})")

        if supervisor.user:
            return HttpResponse(
                format_html(
                    "User already exists. Reset your password <a href='{}'>here</a>",
                    reverse("password_reset"),
                )
            )

        try:
            user_account = User.objects.get(email=email)

        except User.DoesNotExist:

            try:
                first, last = supervisor.name.split(" ", 1)
            except ValueError:
                first, last = supervisor.name, ""
            user_account = User.objects.create_user(
                email=email,
                username=email,
                password=secrets.token_hex(nbytes=16),
                first_name=first,
                last_name=last,
            )

        supervisor.user = user_account
        supervisor.save()

        return HttpResponse(
            format_html(
                "Account created, you now need to reset your password <a href='{}'>here</a>",
                reverse("password_reset"),
            )
        )

    else:

        return render(request, "generic/supervisor_request_account.html", {})


@login_required
def toggle_share_with_supervisor(request, pk):
    """Toggle the share_with_supervisor flag on a CidUserExam record.

    Scope: authenticated owner of the CidUserExam record only.
    Functionality: HTMX endpoint that toggles whether the exam result is
    shared with the user's supervisor.
    """
    if request.htmx:
        cid_user_exam = get_object_or_404(CidUserExam, pk=pk)

        if request.user != cid_user_exam.user_user:
            return HttpResponse("Incorrect permissions")

        cid_user_exam.share_with_supervisor = not cid_user_exam.share_with_supervisor
        cid_user_exam.save()

        return HttpResponse(
            f"Shared with supervisor: {cid_user_exam.share_with_supervisor}"
        )
    else:
        raise PermissionDenied()


def exam_collection_add_author(request, collection_id):
    if request.method != "POST":
        form = UsersAutocompleteForm()

        # Render the fixed form which properly includes form data in HTMX POSTs
        return render(
            request,
            "generic/user_form_fixed.html",
            {"form": form, "collection_id": collection_id},
        )
        return


    collection = get_object_or_404(ExamCollection, pk=collection_id)

    if not request.user in collection.author.all():
        raise PermissionDenied

    #user = get_object_or_404(User, pk=request.POST.get("user"))
    users = User.objects.filter(pk__in=request.POST.getlist("users"))

    collection.add_author_to_exams(users)

    return HttpResponse("Author added to all exams")


def exam_collection_add_marker(request, collection_id, exam_type):
    if request.method != "POST":
        form = UsersAutocompleteForm()

        return render(
            request,
            "generic/marker_form.html",
            {"form": form, "collection_id": collection_id, "exam_type": exam_type},
        )
        return


    collection = get_object_or_404(ExamCollection, pk=collection_id)

    if not request.user in collection.author.all():
        raise PermissionDenied

    #user = get_object_or_404(User, pk=request.POST.get("user"))
    users = User.objects.filter(pk__in=request.POST.getlist("users"))

    collection.add_marker_to_exams(users, exam_types=(exam_type,))

    return HttpResponse("Marker added to exams")


def cimar_case_refresh(request, uuid):
    try:
        CimarCase.objects.get(uuid=uuid).refresh_study()
    except CimarCase.DoesNotExist:
        CimarCase(uuid=uuid).save()



    return HttpResponse("Case refreshed")


@login_required
def user_search(request):
    """HTMX endpoint to search users for the trainees bulk-update UI.

    Expects GET params:
      - q: query string
      - row: optional row index (passed back to JS)

    Returns a small HTML fragment listing up to 10 matching users. Each
    result calls `setManualUser(row, id, text)` when clicked (see template
    JS in `trainees_bulk_update.html`).
    """
    q = (request.GET.get("q") or "").strip()
    row = request.GET.get("row")
    if not q:
        return HttpResponse("")

    users_qs = User.objects.filter(
        Q(first_name__icontains=q)
        | Q(last_name__icontains=q)
        | Q(email__icontains=q)
        | Q(username__icontains=q)
    ).order_by("last_name", "first_name")[:10]

    results = [{"id": u.pk, "text": f"{u.get_full_name() or u.username} - {u.email or ''}"} for u in users_qs]

    return render(request, "generic/partials/user_search_results.html", {"results": results, "row": row, "q": q})


@login_required
def user_search_widget(request):
    """HTMX/JS endpoint to search users for generic selectors.

    Accepts GET params:
      - q: query string
      - field: the form field name to return with the results (so the
               template can call addUserToField(field, id, text)).

    Returns a small HTML fragment listing up to 10 matching users. Each
    result will call `addUserToField(field, id, text)` when clicked.
    """
    import re

    q_raw = (request.GET.get("q") or "").strip()
    field = request.GET.get("field") or request.GET.get("row")
    if not q_raw:
        return HttpResponse("")

    # Support wildcard '*' (return many results) and field-scoped queries like 'name:smith' or 'email:foo'
    q = q_raw

    # If user explicitly asks for wildcard, return an unfiltered queryset (limited)
    if q in ("*", "%"):
        users_qs = User.objects.all()
    else:
        # field-specific search syntax: field:term
        m = re.match(r"^(?P<field>\w+):(?P<term>.+)$", q)
        if m:
            f = m.group("field").lower()
            term = m.group("term").strip()
            if term in ("*", "%"):
                users_qs = User.objects.all()
            else:
                if f in ("name", "fullname", "full_name"):
                    users_qs = User.objects.filter(
                        Q(first_name__icontains=term) | Q(last_name__icontains=term)
                    )
                elif f in ("first", "first_name"):
                    users_qs = User.objects.filter(first_name__icontains=term)
                elif f in ("last", "last_name"):
                    users_qs = User.objects.filter(last_name__icontains=term)
                elif f == "email":
                    users_qs = User.objects.filter(email__icontains=term)
                elif f == "username":
                    users_qs = User.objects.filter(username__icontains=term)
                elif f in ("id", "pk"):
                    try:
                        users_qs = User.objects.filter(pk=int(term))
                    except Exception:
                        users_qs = User.objects.none()
                elif f == "grade":
                    # allow grade name or id
                    try:
                        gid = int(term)
                        users_qs = User.objects.filter(userprofile__grade_id=gid)
                    except Exception:
                        users_qs = User.objects.filter(userprofile__grade__name__icontains=term)
                else:
                    # unknown field: fallback to general search
                    users_qs = User.objects.filter(
                        Q(first_name__icontains=term)
                        | Q(last_name__icontains=term)
                        | Q(email__icontains=term)
                        | Q(username__icontains=term)
                    )
        else:
            # default substring search (icontains)
            users_qs = User.objects.filter(
                Q(first_name__icontains=q)
                | Q(last_name__icontains=q)
                | Q(email__icontains=q)
                | Q(username__icontains=q)
            )

    # optional grade filter (UserProfile.grade FK) still applies and will narrow results
    grade = request.GET.get("grade")
    if grade:
        try:
            users_qs = users_qs.filter(userprofile__grade_id=int(grade))
        except Exception:
            pass

    # Limit results to avoid returning too many rows; wildcard returns up to 50
    limit = 50 if q in ("*", "%") or q_raw in ("*", "%") else 10
    users_qs = users_qs.order_by("last_name", "first_name")[:limit]

    results = []
    for u in users_qs:
        grade = ""
        try:
            up = getattr(u, "userprofile", None)
            if up and getattr(up, "grade", None):
                # grade may be a FK object or a simple string depending on model
                g = up.grade
                grade = getattr(g, "name", str(g)) if g is not None else ""
        except Exception:
            grade = ""

        results.append(
            {
                "id": u.pk,
                "text": f"{u.get_full_name() or u.username} - {u.email or ''}",
                "grade": grade,
            }
        )

    return render(
        request,
        "generic/partials/user_search_widget_results.html",
        {"results": results, "field": field, "q": q_raw},
    )


@login_required
def exam_search_widget(request):
    """HTMX/JS endpoint to search exams for the exam-selection widget.

    GET params:
      - q: query string
      - field: the form field name to return with the results (so the
               template can call addExamToField(field, id, text)).
      - model: optional model label ("app_label.ModelName") to restrict
               search to a specific exam model.

    Returns a small HTML fragment listing matching exams. Each result will
    include a button with class `.exam-add-btn` and `data-exam-id`/`data-exam-text`.
    """
    import re
    from django.apps import apps

    q_raw = (request.GET.get("q") or "").strip()
    field = request.GET.get("field") or request.GET.get("row")
    model_label = request.GET.get("model")
    if not q_raw:
        return HttpResponse("")

    # Support wildcard queries
    q = q_raw

    try:
        model = apps.get_model(model_label) if model_label else None
    except Exception:
        return HttpResponse("")

    # Helper to parse optional boolean GET params
    def _get_bool_param(name):
        v = request.GET.get(name)
        if v is None:
            return None
        return str(v).lower() in ("1", "true", "yes", "on")
    # Render combined results, but restrict to exams the user can access
    def user_can_view_exam(exam, user):
        if user.is_superuser:
            return True
        if exam.open_access:
            return True
        authors = exam.get_author_objects()
        if user in authors:
            return True
        if hasattr(exam, "markers") and user in exam.markers.all():
            return True
        return False

    # Build queryset
    def _field_exists(mdl, field_name):
        try:
            mdl._meta.get_field(field_name)
            return True
        except Exception:
            return False

    qs = model.objects.none()

    # Support field-prefixed queries like `name:term`, `code:term`, `id:123`, `date:YYYY-MM-DD`.
    # Treat '*' or '%' as a wildcard meaning "match everything" for the model.
    field_match = re.match(r'^(?P<field>name|code|id|pk|date|type):\s*(?P<term>.+)$', q, flags=re.I)

    # Wildcard query: return all objects for this model (filters still apply later)
    if q in ("*", "%"):
        qs = model.objects.all()
    else:
        if field_match:
            f = field_match.group('field').lower()
            term = field_match.group('term').strip()
            try:
                if f in ('id', 'pk'):
                    qs = model.objects.filter(pk=int(term))
                elif f == 'name':
                    qs = model.objects.filter(name__icontains=term)
                elif f == 'code':
                    # try a few plausible code field names if present
                    lookups = []
                    for fn in ('code', 'short_code', 'identifier'):
                        if _field_exists(model, fn):
                            lookups.append(f"{fn}__icontains")
                    if lookups:
                        from django.db.models import Q
                        qobj = Q()
                        for lk in lookups:
                            qobj |= Q(**{lk: term})
                        qs = model.objects.filter(qobj)
                    else:
                        qs = model.objects.none()
                elif f == 'date':
                    # try a few common date field names
                    date_lookups = []
                    for fn in ('date', 'exam_date', 'start_date'):
                        if _field_exists(model, fn):
                            date_lookups.append(fn)
                    if date_lookups:
                        from django.db.models import Q
                        qobj = Q()
                        for fn in date_lookups:
                            qobj |= Q(**{f"{fn}__startswith": term})
                        qs = model.objects.filter(qobj)
                    else:
                        qs = model.objects.none()
                elif f == 'type':
                    # restrict by app_label or model name match
                    if term.lower() in (model._meta.app_label.lower(), model._meta.model_name.lower()):
                        qs = model.objects.all()
                    else:
                        qs = model.objects.none()
            except Exception:
                qs = model.objects.none()
        else:
            # Default: search name and, where available, code-like fields
            field_lookups = ["name__icontains"]
            for fn in ('code', 'short_code', 'identifier'):
                if _field_exists(model, fn):
                    field_lookups.append(f"{fn}__icontains")

            from django.db.models import Q
            qobj = Q()
            for lk in field_lookups:
                qobj |= Q(**{lk: q})
            qs = model.objects.filter(qobj)

    # Apply optional filters from GET params to the per-model queryset
    # Only include filters that were supplied in GET _and_ where the model
    # actually exposes the field. This prevents models without a given
    # attribute from being accidentally excluded when a default filter
    # value is always sent by the widget JS.
    extra_filters = {}
    for pname in ("archive", "open_access", "exam_mode", "candidates_only"):
        val = _get_bool_param(pname)
        if val is None:
            continue
        # Only include the filter if the model defines the field
        if not _field_exists(model, pname):
            continue
        extra_filters[pname] = val

    # Apply filters at the queryset level for efficiency
    if extra_filters:
        try:
            qs = qs.filter(**extra_filters)
        except Exception:
            # If a filter cannot be applied at DB level, fall back to leaving qs unchanged
            pass

    # Limit results (wildcard returns up to 50)
    limit = 50 if q in ("*", "%") else 10
    qs = qs.order_by("name")[:limit]

    results = []
    logger.error(f"Exam search widget: model={model}, q='{q}', extra_filters={extra_filters}, post_filter_count={qs.count() if qs is not None else 'N/A'}")
    for e in qs:
        if not user_can_view_exam(e, request.user):
            continue
        results.append({
            "id": e.pk,
            "text": str(e),
            "archive": getattr(e, "archive", False),
            "open_access": getattr(e, "open_access", False),
            "exam_mode": getattr(e, "exam_mode", False),
            "candidates_only": getattr(e, "candidates_only", False),
        })

    logger.error(f"Exam search widget: returning {len(results)} results after filtering")

    return render(request, "generic/partials/exam_search_widget_results.html", {"results": results, "field": field, "q": q_raw})


@login_required
def supervisor_search(request):
    """HTMX endpoint to search supervisors for the trainees bulk-update UI.

    Expects GET params:
      - q: query string
      - row: optional row index (passed back to JS)

    Returns a small HTML fragment listing up to 10 matching supervisors. Each
    result calls `setManualSupervisor(row, id, text)` when clicked (see
    template JS in `trainees_bulk_update.html`).
    """
    q = (request.GET.get("q") or "").strip()
    row = request.GET.get("row")
    if not q:
        return HttpResponse("")

    # Search Supervisor records by name or email and render a small partial
    try:
        from django.db.models import Q
        qs = Supervisor.objects.filter(Q(name__icontains=q) | Q(email__icontains=q)).order_by("name")[:10]
    except Exception:
        qs = Supervisor.objects.none()

    results = []
    for s in qs:
        results.append({"id": s.pk, "text": str(s)})

    return render(request, "generic/partials/supervisor_search_results.html", {"results": results, "row": row})


@login_required
@user_passes_test(lambda u: u.is_superuser or u.groups.filter(name__in=["content_moderator", "atlas_editor"]).exists())
def user_content_list(request):
    """
    Shows a list of all users with the ability to load the content they have created via htmx requests
    """
    users = User.objects.annotate(
        content_count=Count("user_anatomy_exams", filter=Q(user_anatomy_exams__archive=False))
        + Count("user_longs_exams", filter=Q(user_longs_exams__archive=False))
        + Count("user_physics_exams", filter=Q(user_physics_exams__archive=False))
        + Count("user_rapid_exams", filter=Q(user_rapid_exams__archive=False))
        + Count("user_sba_exams", filter=Q(user_sba_exams__archive=False))
        + Count("user_shorts_exams", filter=Q(user_shorts_exams__archive=False))
    ).filter(content_count__gt=0)

    return render(request, "generic/user_content_list.html", {"users": users})



@login_required
@user_passes_test(lambda u: u.is_superuser or u.groups.filter(name__in=["content_moderator", "atlas_editor"]).exists())
def user_content_review(request, user_id):
    """
    HTMX partial to show content authored by a given user.
    Includes authored questions and authored exams for each app,
    plus atlas cases and case collections.
    """
    user_obj = get_object_or_404(User, pk=user_id)

    # Mapping of app -> (question_related_name, exam_author_related_name, display_label)
    APP_MAP = {
        "anatomy": ("anatomy_authored_questions", "anatomy_exam_author", "Anatomy"),
        "physics": ("physics_authored_questions", "physics_exam_author", "Physics"),
        "rapids": ("rapid_authored_questions", "rapid_exam_author", "Rapids"),
        "shorts": ("shorts_authored_questions", "shorts_exam_author", "Shorts"),
        "longs": ("long_authored_questions", "long_exam_author", "Longs"),
        "sbas": ("sbas_authored_questions", "sba_exam_author", "SBAs"),
    }

    content = {}

    for app_key, (q_rel, e_rel, label) in APP_MAP.items():
        questions = []
        exams = []
        try:
            questions = list(getattr(user_obj, q_rel).all().order_by("-pk")[:50])
        except Exception:
            questions = []

        try:
            exams = list(getattr(user_obj, e_rel).all().filter(archive=False).order_by("-pk")[:50])
        except Exception:
            exams = []

        content[app_key] = {
            "label": label,
            "questions": questions,
            "exams": exams,
        }

    # Atlas authored content
    atlas_cases = []
    atlas_collections = []
    try:
        atlas_cases = list(user_obj.atlas_authored_cases.all().order_by("-pk")[:50])
    except Exception:
        atlas_cases = []
    try:
        atlas_collections = list(user_obj.casecollection_authored_cases.all().order_by("-pk")[:50])
    except Exception:
        atlas_collections = []

    context = {
        "user_obj": user_obj,
        "content": content,
        "atlas_cases": atlas_cases,
        "atlas_collections": atlas_collections,
    }

    # Render HTMX partial (or full page if not HTMX)
    template = "generic/partials/user_content_review.html"
    return render(request, template, context)


@login_required
@require_POST
def toggle_hide_item(request):
    """Toggle the hidden status of an exam or collection for the current user.

    Scope: Authenticated end users.
    Functionality: Adds or removes a UserHiddenItem record for the given
    content_type_id and object_id, then triggers a page reload via HX-Refresh.
    """
    from django.contrib.contenttypes.models import ContentType
    from generic.models import UserHiddenItem

    content_type_id = request.POST.get("content_type_id")
    object_id = request.POST.get("object_id")

    if not content_type_id or not object_id:
        return HttpResponse("Missing parameters", status=400)

    try:
        content_type = ContentType.objects.get(pk=int(content_type_id))
        object_id = int(object_id)
    except (ContentType.DoesNotExist, ValueError):
        return HttpResponse("Invalid parameters", status=400)

    hidden_item, created = UserHiddenItem.objects.get_or_create(
        user=request.user,
        content_type=content_type,
        object_id=object_id,
    )

    if not created:
        hidden_item.delete()

    response = HttpResponse("")
    response["HX-Refresh"] = "true"
    return response

 
