From 5e84ba3178391e0741bc828b109d9f6dc8b6a0d3 Mon Sep 17 00:00:00 2001
From: Ross <xkjq@ymail.com>
Date: Mon, 20 Oct 2025 22:03:36 +0100
Subject: [PATCH] Enhance validation process in import_llm_questions to handle
 additionalProperties errors and clean payloads for improved data integrity

---
 .../sbas/partials/import_preview.html         |  5 ++
 sbas/views.py                                 | 57 ++++++++++++++++---
 2 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/sbas/templates/sbas/partials/import_preview.html b/sbas/templates/sbas/partials/import_preview.html
index b48bad3e..eed25cec 100644
--- a/sbas/templates/sbas/partials/import_preview.html
+++ b/sbas/templates/sbas/partials/import_preview.html
@@ -15,6 +15,11 @@
             </ul>
           </div>
         {% endif %}
+        {% if item.dropped_fields %}
+          <div class="alert alert-info" role="alert">
+            <strong>Note:</strong> The following unsupported fields were dropped: <em>{{ item.dropped_fields|join:", " }}</em>
+          </div>
+        {% endif %}
         {% if item.payload %}
           <p><strong>Title:</strong> {{ item.payload.title|default:'(No title)'|safe }}</p>
           <p><strong>Stem:</strong> {{ item.payload.stem|default:''|safe }}</p>
diff --git a/sbas/views.py b/sbas/views.py
index 4095cdff..4d8cf00e 100644
--- a/sbas/views.py
+++ b/sbas/views.py
@@ -637,14 +637,55 @@ def import_llm_questions(request):
 
     for idx, payload in enumerate(candidates):
         item_report = {"index": idx, "status": None, "errors": [], "missing_m2m": [], "resolved_m2m": {}, "payload": payload}
-        for err in validator.iter_errors(payload):
-            item_report["errors"].append(err.message)
-        if item_report["errors"]:
-            logger.debug(f"Validation errors for item {idx}: {item_report['errors']}")
-            report["errors"].append({"index": idx, "errors": item_report["errors"]})
-            item_report["status"] = "invalid"
-            report["per_item"].append(item_report)
-            continue
+
+        # Collect validation errors
+        validation_errors = list(validator.iter_errors(payload))
+        if validation_errors:
+            # Separate additionalProperties errors from others
+            addl_errors = [e for e in validation_errors if getattr(e, 'validator', '') == 'additionalProperties']
+            other_errors = [e for e in validation_errors if getattr(e, 'validator', '') != 'additionalProperties']
+
+            if addl_errors and not other_errors:
+                # If the only problems are extra fields, drop them and re-validate the cleaned payload
+                allowed_keys = set(schema.get('properties', {}).keys())
+                extra_keys = [k for k in payload.keys() if k not in allowed_keys]
+                if extra_keys:
+                    logger.debug("Validation: item %s has extra keys %s, dropping them for import", idx, extra_keys)
+                    # Record dropped fields in the report for user visibility
+                    item_report.setdefault('dropped_fields', []).extend(extra_keys)
+                    # Create a cleaned payload without extra keys and use it from here on
+                    cleaned_payload = {k: v for k, v in payload.items() if k in allowed_keys}
+                    # Re-run validation on cleaned payload to be safe
+                    cleaned_errors = list(validator.iter_errors(cleaned_payload))
+                    if cleaned_errors:
+                        # If cleaning still leaves errors, mark invalid
+                        item_report['errors'] = [e.message for e in cleaned_errors]
+                        logger.debug("Validation errors after cleaning for item %s: %s", idx, item_report['errors'])
+                        report['errors'].append({"index": idx, "errors": item_report['errors']})
+                        item_report['status'] = 'invalid'
+                        report['per_item'].append(item_report)
+                        continue
+                    else:
+                        # Replace payload used downstream with cleaned version
+                        payload = cleaned_payload
+                        item_report['status'] = 'cleaned'
+                        # do not append here; allow the normal processing path to append the full item_report later
+                else:
+                    # no extra keys found, treat as invalid to be safe
+                    item_report['errors'] = [e.message for e in validation_errors]
+                    logger.debug("Validation errors for item %s: %s", idx, item_report['errors'])
+                    report['errors'].append({"index": idx, "errors": item_report['errors']})
+                    item_report['status'] = 'invalid'
+                    report['per_item'].append(item_report)
+                    continue
+            else:
+                # Other validation problems exist; report and skip
+                item_report['errors'] = [e.message for e in validation_errors]
+                logger.debug("Validation errors for item %s: %s", idx, item_report['errors'])
+                report['errors'].append({"index": idx, "errors": item_report['errors']})
+                item_report['status'] = 'invalid'
+                report['per_item'].append(item_report)
+                continue
 
         try:
             # Prepare category resolution