Add LLM question import functionality and enhance question model with new fields

sbas/views.py

@@ -29,6 +29,250 @@ from django.http import Http404, HttpResponseBadRequest, JsonResponse
 from django.http import HttpResponseRedirect, HttpResponse
 
 from .models import Question, Category, Exam, UserAnswer
+from django.views.decorators.http import require_http_methods
+from django.db import transaction
+import re
+import logging
+
+logger = logging.getLogger(__name__)
+
+from loguru import logger
+
+try:
+    import jsonschema
+except Exception:
+    jsonschema = None
+
+
+class LLMImportForm(forms.Form):
+    file = forms.FileField(required=False, help_text="Upload a JSON or JSONL file matching the SBA import schema")
+    raw = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={"rows": 10, "cols": 80}),
+        help_text="Or paste JSON, JSONL (newline-delimited), or multiple JSON documents separated by blank lines",
+    )
+
+
+def _resolve_m2m(model_class, value):
+    """Resolve an item which may be an int (pk) or a string (name/slug).
+    Returns a queryset or empty list of matching objects.
+    """
+    if value is None:
+        return []
+    if isinstance(value, int):
+        try:
+            return [model_class.objects.get(pk=value)]
+        except model_class.DoesNotExist:
+            return []
+    if isinstance(value, str):
+        # try exact name field, then case-insensitive contains
+        qs = model_class.objects.filter(name__iexact=value)
+        if not qs.exists():
+            qs = model_class.objects.filter(name__icontains=value)
+        return list(qs[:5])
+    return []
+
+
+@login_required
+@user_passes_test(lambda u: u.is_superuser)
+@require_http_methods(["GET", "POST"])
+def import_llm_questions(request):
+    """Upload a JSON/JSONL file of LLM-produced questions and import them.
+
+    Only superusers may use this view. The view validates each object against
+    the agreed JSON schema (draft-07) and attempts to resolve M2M references
+    by numeric id or by name. Returns a JSON report of created items and errors.
+    """
+    schema = {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "title": "SBA Question",
+        "type": "object",
+        "additionalProperties": False,
+        "required": [
+            "title",
+            "stem",
+            "a_answer",
+            "b_answer",
+            "c_answer",
+            "d_answer",
+            "e_answer",
+            "best_answer",
+            "category",
+        ],
+        "properties": {
+            "title": {"type": "string"},
+            "stem": {"type": "string"},
+            "a_answer": {"type": "string"},
+            "a_feedback": {"type": "string"},
+            "b_answer": {"type": "string"},
+            "b_feedback": {"type": "string"},
+            "c_answer": {"type": "string"},
+            "c_feedback": {"type": "string"},
+            "d_answer": {"type": "string"},
+            "d_feedback": {"type": "string"},
+            "e_answer": {"type": "string"},
+            "e_feedback": {"type": "string"},
+            "feedback": {"type": "string"},
+            "best_answer": {"type": "string", "enum": ["a", "b", "c", "d", "e"]},
+            "category": {"oneOf": [{"type": "string"}]},
+            "finding": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
+            "structure": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
+            "condition": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
+            "presentation": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
+            "subspecialty": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
+        },
+    }
+
+    if request.method == "GET":
+        form = LLMImportForm()
+        return render(request, "sbas/import_llm_questions.html", {"form": form})
+
+    # POST
+    form = LLMImportForm(request.POST, request.FILES)
+    if not form.is_valid():
+        return JsonResponse({"ok": False, "errors": ["No file uploaded or invalid form"]}, status=400)
+
+    raw_text = None
+    if form.cleaned_data.get("raw"):
+        raw_text = form.cleaned_data.get("raw")
+    elif form.cleaned_data.get("file"):
+        f = form.cleaned_data["file"]
+        raw_text = f.read().decode("utf-8")
+    else:
+        return JsonResponse({"ok": False, "errors": ["No file uploaded or text provided"]}, status=400)
+
+    # support: single JSON object, JSON array, JSONL (one JSON object per line),
+    # or multiple JSON documents separated by one or more blank lines.
+    candidates = []
+    # sanitize raw_text by escaping backslashes that are not part of a valid
+    # JSON escape sequence. This handles inputs containing LaTeX-style
+    # sequences such as "\ge" which are invalid JSON (Invalid \escape).
+    def _sanitize_backslashes(s: str):
+        # valid escapes after backslash in JSON: \" \\ \/ \b \f \n \r \t and \uXXXX
+        # This regex finds backslashes not followed by ", \\ , /, b, f, n, r, t, or u
+        pattern = re.compile(r"\\(?!(?:[\"\\/bfnrtu]))")
+        new_s, n = pattern.subn(r"\\\\", s)
+        return new_s, n
+
+    raw_text_stripped = raw_text.strip()
+    raw_text, sanitized_count = _sanitize_backslashes(raw_text_stripped)
+    logger.debug("import_llm_questions: sanitized input length %d (escaped %d backslashes)", len(raw_text), sanitized_count)
+    # Try whole-text JSON first (object or array)
+    parse_error = None
+    try:
+        parsed = json.loads(raw_text)
+        if isinstance(parsed, list):
+            candidates = parsed
+        elif isinstance(parsed, dict):
+            candidates = [parsed]
+    except Exception as e_outer:
+        parse_error = e_outer
+        # Try JSONL: each non-empty line is a JSON object
+        for i, line in enumerate(raw_text.splitlines()):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                candidates.append(json.loads(line))
+            except Exception:
+                # not JSONL or some lines are multi-line JSON; fallthrough
+                candidates = []
+                break
+
+        # If JSONL didn't produce candidates, try splitting by blank lines
+        if not candidates:
+            blocks = [b.strip() for b in re.split(r"\n\s*\n", raw_text) if b.strip()]
+            if len(blocks) > 1:
+                for i, block in enumerate(blocks):
+                    try:
+                        candidates.append(json.loads(block))
+                    except Exception as e_block:
+                        return JsonResponse({"ok": False, "errors": [f"Invalid JSON in block {i+1}: {e_block}"]}, status=400)
+
+    if not candidates:
+        # If still empty, return the original parse error if present
+        msg = str(parse_error) if parse_error is not None else "No JSON objects found"
+        logger.debug(f"Failed to parse input: {msg}")
+        return JsonResponse({"ok": False, "errors": [f"Failed to parse input: {msg}"]}, status=400)
+
+    report = {"total": len(candidates), "created": 0, "errors": [], "sanitized_backslashes": sanitized_count}
+
+    if jsonschema is None:
+        return JsonResponse({"ok": False, "errors": ["jsonschema library not available in environment"]}, status=500)
+
+    validator = jsonschema.Draft7Validator(schema)
+
+    from atlas.models import Finding, Structure, Condition, Presentation, Subspecialty
+
+    for idx, payload in enumerate(candidates):
+        errors = []
+        for err in validator.iter_errors(payload):
+            errors.append(err.message)
+        if errors:
+            logger.debug(f"Validation errors for item {idx}: {errors}")
+            report["errors"].append({"index": idx, "errors": errors})
+            continue
+
+        # create the Question inside a transaction; partial failures should not leave half-written objects
+        try:
+            with transaction.atomic():
+                q = Question()
+                q.stem = payload.get("stem", "").strip()
+                # map title -> not present in model; use as stem prefix or store in feedback
+                title = payload.get("title")
+                if title:
+                    q.stem = f"<strong>{title}</strong>\n" + q.stem
+
+                q.a_answer = payload.get("a_answer", "").strip()
+                q.a_feedback = payload.get("a_feedback", "")
+                q.b_answer = payload.get("b_answer", "").strip()
+                q.b_feedback = payload.get("b_feedback", "")
+                q.c_answer = payload.get("c_answer", "").strip()
+                q.c_feedback = payload.get("c_feedback", "")
+                q.d_answer = payload.get("d_answer", "").strip()
+                q.d_feedback = payload.get("d_feedback", "")
+                q.e_answer = payload.get("e_answer", "").strip()
+                q.e_feedback = payload.get("e_feedback", "")
+                q.feedback = payload.get("feedback", "")
+                q.best_answer = payload.get("best_answer")
+                # category resolve by name
+                cat_val = payload.get("category")
+                if isinstance(cat_val, str):
+                    cat_obj, created = Category.objects.get_or_create(category=cat_val)
+                    q.category = cat_obj
+                q.save()
+
+                # Resolve M2Ms
+                for key, model_cls in (
+                    ("finding", Finding),
+                    ("structure", Structure),
+                    ("condition", Condition),
+                    ("presentation", Presentation),
+                    ("subspecialty", Subspecialty),
+                ):
+                    vals = payload.get(key) or []
+                    resolved = []
+                    for v in vals:
+                        if isinstance(v, int):
+                            try:
+                                resolved.append(model_cls.objects.get(pk=v))
+                            except model_cls.DoesNotExist:
+                                # skip unknown
+                                continue
+                        elif isinstance(v, str):
+                            qs = model_cls.objects.filter(name__iexact=v)
+                            if not qs.exists():
+                                qs = model_cls.objects.filter(name__icontains=v)
+                            if qs.exists():
+                                resolved.extend(list(qs[:5]))
+                    if resolved:
+                        getattr(q, key).add(*[o.pk for o in resolved])
+
+                report["created"] += 1
+        except Exception as e:
+            report["errors"].append({"index": idx, "errors": [str(e)]})
+
+    return JsonResponse({"ok": True, "report": report})
 
 from django_tables2 import SingleTableView, SingleTableMixin
 from django_filters.views import FilterView
@@ -436,3 +680,7 @@ def exam_clone2(request, exam_id):
     new_exam = exam.clone_model()
     return redirect("sbas:exam_update", pk=new_exam.id)
 
+
+
+def llm_prompt_view(request):
+    return render(request, "sbas/llm_prompt_view.html", {})