From bddccd576888be9c722c179382b35879b52ac25d Mon Sep 17 00:00:00 2001
From: Ross <xkjq@ymail.com>
Date: Mon, 20 Oct 2025 12:01:07 +0100
Subject: [PATCH] Enhance import preview and backend handling for M2M
 relationships, adding sources to schema and improving missing values mapping
 for better template rendering and debugging.

---
 .../sbas/partials/import_preview.html         | 148 +++++++++---------
 sbas/views.py                                 |  21 ++-
 2 files changed, 98 insertions(+), 71 deletions(-)

diff --git a/sbas/templates/sbas/partials/import_preview.html b/sbas/templates/sbas/partials/import_preview.html
index 21c6d617..20d60510 100644
--- a/sbas/templates/sbas/partials/import_preview.html
+++ b/sbas/templates/sbas/partials/import_preview.html
@@ -54,86 +54,94 @@
               <div class="small text-info">Would create: {{ item.missing_map.Finding|join:", " }}</div>
             {% endif %}
           </div>
-          </div>
+        </div>
 
           <!-- Structure -->
-          <div class="m2m-group" data-model="Structure"><strong>Structure:</strong>
-            {% if item.resolved_m2m.structure %}
-              {% for val in item.resolved_m2m.structure %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Structure" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            {% if item.missing_map.Structure %}
-              {% for val in item.missing_map.Structure %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Structure" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            <span class="text-muted m2m-none" style="display:none">(none)</span>
-            {% if item.missing_map.Structure %}
-              <div class="small text-info">Would create: {{ item.missing_map.Structure|join:", " }}</div>
-            {% endif %}
-          </div>
+        <div class="m2m-group" data-model="Structure"><strong>Structure:</strong>
+          {% if item.resolved_m2m.structure %}
+            {% for val in item.resolved_m2m.structure %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Structure" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          {% if item.missing_map.Structure %}
+            {% for val in item.missing_map.Structure %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Structure" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          <span class="text-muted m2m-none" style="display:none">(none)</span>
+          {% if item.missing_map.Structure %}
+            <div class="small text-info">Would create: {{ item.missing_map.Structure|join:", " }}</div>
+          {% endif %}
+        </div>
 
           <!-- Condition -->
-          <div class="m2m-group" data-model="Condition"><strong>Condition:</strong>
-            {% if item.resolved_m2m.condition %}
-              {% for val in item.resolved_m2m.condition %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Condition" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            {% if item.missing_map.Condition %}
-              {% for val in item.missing_map.Condition %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Condition" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            <span class="text-muted m2m-none" style="display:none">(none)</span>
-            {% if item.missing_map.Condition %}
-              <div class="small text-info">Would create: {{ item.missing_map.Condition|join:", " }}</div>
-            {% endif %}
-          </div>
+        <div class="m2m-group" data-model="Condition"><strong>Condition:</strong>
+          {% if item.resolved_m2m.condition %}
+            {% for val in item.resolved_m2m.condition %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Condition" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          {% if item.missing_map.Condition %}
+            {% for val in item.missing_map.Condition %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Condition" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          <span class="text-muted m2m-none" style="display:none">(none)</span>
+          {% if item.missing_map.Condition %}
+            <div class="small text-info">Would create: {{ item.missing_map.Condition|join:", " }}</div>
+          {% endif %}
+        </div>
 
           <!-- Presentation -->
-          <div class="m2m-group" data-model="Presentation"><strong>Presentation:</strong>
-            {% if item.resolved_m2m.presentation %}
-              {% for val in item.resolved_m2m.presentation %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Presentation" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            {% if item.missing_map.Presentation %}
-              {% for val in item.missing_map.Presentation %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Presentation" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            <span class="text-muted m2m-none" style="display:none">(none)</span>
-            {% if item.missing_map.Presentation %}
-              <div class="small text-info">Would create: {{ item.missing_map.Presentation|join:", " }}</div>
-            {% endif %}
-          </div>
+        <div class="m2m-group" data-model="Presentation"><strong>Presentation:</strong>
+          {% if item.resolved_m2m.presentation %}
+            {% for val in item.resolved_m2m.presentation %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Presentation" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          {% if item.missing_map.Presentation %}
+            {% for val in item.missing_map.Presentation %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Presentation" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          <span class="text-muted m2m-none" style="display:none">(none)</span>
+          {% if item.missing_map.Presentation %}
+            <div class="small text-info">Would create: {{ item.missing_map.Presentation|join:", " }}</div>
+          {% endif %}
+        </div>
 
           <!-- Subspecialty -->
-          <div class="m2m-group" data-model="Subspecialty"><strong>Subspecialty:</strong>
-            {% if item.resolved_m2m.subspecialty %}
-              {% for val in item.resolved_m2m.subspecialty %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Subspecialty" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            {% if item.missing_map.Subspecialty %}
-              {% for val in item.missing_map.Subspecialty %}
-                <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Subspecialty" data-value="{{ val }}">{{ val }}</button>
-              {% endfor %}
-            {% endif %}
-            <span class="text-muted m2m-none" style="display:none">(none)</span>
-            {% if item.missing_map.Subspecialty %}
-              <div class="small text-info">Would create: {{ item.missing_map.Subspecialty|join:", " }}</div>
-            {% endif %}
-          </div>
+        <div class="m2m-group" data-model="Subspecialty"><strong>Subspecialty:</strong>
+          {% if item.resolved_m2m.subspecialty %}
+            {% for val in item.resolved_m2m.subspecialty %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Subspecialty" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          {% if item.missing_map.Subspecialty %}
+            {% for val in item.missing_map.Subspecialty %}
+              <button type="button" class="m2m-toggle btn btn-sm btn-success ms-1" data-idx="{{ item.index }}" data-model="Subspecialty" data-value="{{ val }}">{{ val }}</button>
+            {% endfor %}
+          {% endif %}
+          <span class="text-muted m2m-none" style="display:none">(none)</span>
+          {% if item.missing_map.Subspecialty %}
+            <div class="small text-info">Would create: {{ item.missing_map.Subspecialty|join:", " }}</div>
+          {% endif %}
         </div>
-        <div>
-          <label><input type="checkbox" name="selected" value="{{ item.index }}" checked> Include</label>
-        </div>
-        <!-- container for hidden exclude inputs for this item -->
-        <div id="exclude-container-{{ item.index }}"></div>
       </div>
+      <!-- DEBUG: show raw resolved and missing maps for troubleshooting -->
+      <div class="card-footer text-monospace small">
+        <details>
+          <summary>Debug: resolved_m2m / missing_map (click to expand)</summary>
+          <pre style="white-space:pre-wrap;">resolved_m2m: {{ item.resolved_m2m|default:"{}" }}
+missing_map: {{ item.missing_map|default:"{}" }}</pre>
+        </details>
+      </div>
+      <div>
+        <label><input type="checkbox" name="selected" value="{{ item.index }}" checked> Include</label>
+      </div>
+        <!-- container for hidden exclude inputs for this item -->
+      <div id="exclude-container-{{ item.index }}"></div>
+    </div>
     </div>
   {% endfor %}
 
diff --git a/sbas/views.py b/sbas/views.py
index c1165c93..e7ac15da 100644
--- a/sbas/views.py
+++ b/sbas/views.py
@@ -535,6 +535,7 @@ def import_llm_questions(request):
             "condition": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
             "presentation": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
             "subspecialty": {"type": "array", "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, "uniqueItems": True},
+            "sources": {"type": "array", "items": {"type": "string"}},
         },
     }
 
@@ -685,11 +686,29 @@ def import_llm_questions(request):
 
                 m2m_map[key] = {"resolved": resolved, "missing": missing}
                 item_report["resolved_m2m"][key] = [o.name for o in resolved]
+                # Always include an entry for this M2M (values may be empty)
+                item_report["missing_m2m"].append({"model": model_cls.__name__, "values": missing})
                 if missing:
-                    item_report["missing_m2m"].extend([{"model": model_cls.__name__, "values": missing}])
                     for v in missing:
                         aggregated_would_create[model_cls.__name__].add(v)
 
+            # Build a simple map of missing values per model (stringified) for robust template rendering
+            # Provide both lowercase and capitalized keys so templates referencing either form will work
+            missing_map = {}
+            for k, v in m2m_map.items():
+                vals = [str(x) for x in v["missing"]]
+                missing_map[k] = vals
+                missing_map[k.capitalize()] = vals
+            item_report["missing_map"] = missing_map
+
+            # Debug: log missing_map and resolved_m2m for troubleshooting
+            logger.debug(
+                "import_llm_questions: item %s missing_map_keys=%s resolved_keys=%s",
+                idx,
+                list(missing_map.keys()),
+                list((item_report.get("resolved_m2m") or {}).keys()),
+            )
+
             # If dry-run, do not save; just report what would happen
             if dry_run:
                 item_report["status"] = "would_create"