feat(worker): Enhance db_worker script for better error handling and add systemd service example

README.md

@@ -54,9 +54,19 @@ Manual worker startup (non-Docker production)
 - To run it in the background and keep logs:
 
   ```sh
-  nohup DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 &
+  nohup env DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 &
   ```
 
+- In fish shell, follow with `disown` so the shell does not keep the job attached:
+
+  ```sh
+  nohup env DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 &
+  disown
+  ```
+
+- For server reliability, prefer running the same script under `systemd` (auto-start on reboot, restart-on-failure).
+- A starter unit file is provided at `scripts/db-worker.service.example`.
+
 - To verify it is still running:
 
   ```sh

atlas/tasks.py

@@ -213,104 +213,109 @@ def series_reconstruct_task(
     if volume_for_recon.shape[0] < 1:
         raise ValueError("No reconstruction slices were generated")
 
-    base_z_offset = float(recon_centers_mm[0]) if len(recon_centers_mm) > 0 else 0.0
+    # Determine source acquisition plane from normal vector.
+    dominant_normal_axis = int(np.argmax(np.abs(normal_dir)))
+    source_plane = {0: "sagittal", 1: "coronal", 2: "axial"}.get(dominant_normal_axis, "unknown")
 
-    def build_position(origin, row_vector, col_vector, stack_vector, row_index=0, col_index=0, stack_offset=0.0):
+    # Reorient the reconstructed volume into patient axes order [z, y, x] so
+    # generated plane names are anatomically correct regardless of source plane.
+    source_axis_dirs = [normal_dir, row_dir, col_dir]
+    source_axis_spacings = [float(target_spacing), float(native_row_spacing), float(native_col_spacing)]
+    source_axis_sizes = [int(volume_for_recon.shape[0]), int(volume_for_recon.shape[1]), int(volume_for_recon.shape[2])]
+
+    source_for_patient_axis = {}
+    sign_for_patient_axis = {}
+    patient_axes_claimed = set()
+
+    for src_axis, vec in enumerate(source_axis_dirs):
+        patient_axis = int(np.argmax(np.abs(vec)))
+        if patient_axis in patient_axes_claimed:
+            raise ValueError("Could not infer unique source orientation axes for reconstruction")
+        patient_axes_claimed.add(patient_axis)
+        source_for_patient_axis[patient_axis] = src_axis
+        sign_for_patient_axis[patient_axis] = 1 if float(vec[patient_axis]) >= 0 else -1
+
+    src_x = source_for_patient_axis[0]
+    src_y = source_for_patient_axis[1]
+    src_z = source_for_patient_axis[2]
+
+    spacing_x = source_axis_spacings[src_x]
+    spacing_y = source_axis_spacings[src_y]
+    spacing_z = source_axis_spacings[src_z]
+
+    vol_zyx = np.transpose(volume_for_recon, (src_z, src_y, src_x))
+
+    if sign_for_patient_axis[2] < 0:
+        vol_zyx = np.flip(vol_zyx, axis=0)
+    if sign_for_patient_axis[1] < 0:
+        vol_zyx = np.flip(vol_zyx, axis=1)
+    if sign_for_patient_axis[0] < 0:
+        vol_zyx = np.flip(vol_zyx, axis=2)
+
+    def source_indices_from_patient_zyx(iz, iy, ix):
+        src_indices = [0, 0, 0]
+
+        z_index = int(iz)
+        y_index = int(iy)
+        x_index = int(ix)
+
+        if sign_for_patient_axis[2] < 0:
+            z_index = source_axis_sizes[src_z] - 1 - z_index
+        if sign_for_patient_axis[1] < 0:
+            y_index = source_axis_sizes[src_y] - 1 - y_index
+        if sign_for_patient_axis[0] < 0:
+            x_index = source_axis_sizes[src_x] - 1 - x_index
+
+        src_indices[src_z] = z_index
+        src_indices[src_y] = y_index
+        src_indices[src_x] = x_index
+        return src_indices
+
+    def position_from_patient_zyx(iz, iy, ix):
+        src_k, src_r, src_c = source_indices_from_patient_zyx(iz, iy, ix)
         pos = (
-            np.asarray(origin, dtype=float)
+            np.asarray(origin_ipp, dtype=float)
-            + (np.asarray(stack_vector, dtype=float) * float(stack_offset))
+            + (np.asarray(normal_dir, dtype=float) * (float(src_k) * float(target_spacing)))
-            + (np.asarray(row_vector, dtype=float) * (float(row_index) * float(native_row_spacing)))
+            + (np.asarray(row_dir, dtype=float) * (float(src_r) * float(native_row_spacing)))
-            + (np.asarray(col_vector, dtype=float) * (float(col_index) * float(native_col_spacing)))
+            + (np.asarray(col_dir, dtype=float) * (float(src_c) * float(native_col_spacing)))
         )
         return [float(pos[0]), float(pos[1]), float(pos[2])]
 
-    plane_slices = {}
+    plane_builders = {}
     for plane in recon_planes:
         plane_norm = plane.lower()
         if plane_norm == "axial":
-            recon_slices = [volume_for_recon[i, :, :] for i in range(volume_for_recon.shape[0])]
+            plane_builders[plane_norm] = {
-            pixel_spacing_out = [native_row_spacing, native_col_spacing]
+                "count": int(vol_zyx.shape[0]),
-            spacing_between_slices_out = float(target_spacing)
+                "slice_fn": lambda idx: vol_zyx[idx, :, :],
-            image_orientation_out = [
+                "pixel_spacing_out": [float(spacing_y), float(spacing_x)],
-                float(row_dir[0]),
+                "spacing_between_slices_out": float(spacing_z),
-                float(row_dir[1]),
+                "image_orientation_out": [0.0, 1.0, 0.0, 1.0, 0.0, 0.0],
-                float(row_dir[2]),
+                "position_fn": lambda idx: position_from_patient_zyx(idx, 0, 0),
-                float(col_dir[0]),
+            }
-                float(col_dir[1]),
-                float(col_dir[2]),
-            ]
-
-            def position_for_index(idx):
-                return build_position(
-                    origin_ipp,
-                    row_dir,
-                    col_dir,
-                    normal_dir,
-                    row_index=0,
-                    col_index=0,
-                    stack_offset=float(recon_centers_mm[idx]),
-                )
         elif plane_norm == "coronal":
-            recon_slices = [volume_for_recon[:, i, :] for i in range(volume_for_recon.shape[1])]
+            plane_builders[plane_norm] = {
-            pixel_spacing_out = [target_spacing, native_col_spacing]
+                "count": int(vol_zyx.shape[1]),
-            spacing_between_slices_out = float(native_row_spacing)
+                "slice_fn": lambda idx: vol_zyx[:, idx, :],
-            image_orientation_out = [
+                "pixel_spacing_out": [float(spacing_z), float(spacing_x)],
-                float(normal_dir[0]),
+                "spacing_between_slices_out": float(spacing_y),
-                float(normal_dir[1]),
+                "image_orientation_out": [0.0, 0.0, 1.0, 1.0, 0.0, 0.0],
-                float(normal_dir[2]),
+                "position_fn": lambda idx: position_from_patient_zyx(0, idx, 0),
-                float(col_dir[0]),
+            }
-                float(col_dir[1]),
-                float(col_dir[2]),
-            ]
-
-            def position_for_index(idx):
-                return build_position(
-                    origin_ipp,
-                    row_dir,
-                    col_dir,
-                    normal_dir,
-                    row_index=idx,
-                    col_index=0,
-                    stack_offset=base_z_offset,
-                )
         elif plane_norm == "sagittal":
-            recon_slices = [volume_for_recon[:, :, i] for i in range(volume_for_recon.shape[2])]
+            plane_builders[plane_norm] = {
-            pixel_spacing_out = [target_spacing, native_row_spacing]
+                "count": int(vol_zyx.shape[2]),
-            spacing_between_slices_out = float(native_col_spacing)
+                "slice_fn": lambda idx: vol_zyx[:, :, idx],
-            image_orientation_out = [
+                "pixel_spacing_out": [float(spacing_z), float(spacing_y)],
-                float(normal_dir[0]),
+                "spacing_between_slices_out": float(spacing_x),
-                float(normal_dir[1]),
+                "image_orientation_out": [0.0, 0.0, 1.0, 0.0, 1.0, 0.0],
-                float(normal_dir[2]),
+                "position_fn": lambda idx: position_from_patient_zyx(0, 0, idx),
-                float(row_dir[0]),
-                float(row_dir[1]),
-                float(row_dir[2]),
-            ]
-
-            def position_for_index(idx):
-                return build_position(
-                    origin_ipp,
-                    row_dir,
-                    col_dir,
-                    normal_dir,
-                    row_index=0,
-                    col_index=idx,
-                    stack_offset=base_z_offset,
-                )
-        else:
-            continue
-
-        plane_slices[plane_norm] = {
-            "recon_slices": recon_slices,
-            "pixel_spacing_out": pixel_spacing_out,
-            "spacing_between_slices_out": spacing_between_slices_out,
-            "image_orientation_out": image_orientation_out,
-            "position_for_index": position_for_index,
             }
 
-    if not plane_slices:
+    if not plane_builders:
         raise ValueError("No valid reconstruction planes selected")
 
-    total_slices = sum(len(v["recon_slices"]) for v in plane_slices.values())
+    total_slices = sum(v["count"] for v in plane_builders.values())
     cache.set(
         progress_key,
         {"current": 0, "total": total_slices, "message": "Reconstruction started"},
@@ -320,12 +325,13 @@ def series_reconstruct_task(
     processed = 0
     created_series = []
 
-    for plane_norm, cfg in plane_slices.items():
+    for plane_norm, cfg in plane_builders.items():
-        recon_series = atlas_views._create_series_derivative(series, f"Recon {plane_norm.title()}")
+        recon_series = atlas_views._create_series_derivative(series, f"Recon {plane_norm.title()} ({source_plane.title()} src)")
         recon_series.series_instance_uid = generate_uid()
         recon_series.save(update_fields=["series_instance_uid"])
 
-        for idx, arr2d in enumerate(cfg["recon_slices"]):
+        for idx in range(int(cfg["count"])):
+            arr2d = cfg["slice_fn"](idx)
             ds_new = copy.deepcopy(template_ds)
             arr2d = np.asarray(arr2d, dtype=dicom_items[0][2].dtype)
 
@@ -398,4 +404,5 @@ def series_reconstruct_task(
         "target_spacing": float(target_spacing),
         "slab_thickness": float(slab_thickness),
         "mode": recon_thickness_mode,
+        "source_plane": source_plane,
     }

scripts/db-worker.service.example

@@ -0,0 +1,16 @@
+[Unit]
+Description=PENRA django-tasks db_worker
+After=network.target
+
+[Service]
+Type=simple
+User=www-data
+Group=www-data
+WorkingDirectory=/home/ross/web/rad
+Environment=DJANGO_SETTINGS_MODULE=rad.settings
+ExecStart=/home/ross/web/rad/scripts/run-db-worker.sh
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=multi-user.target

scripts/run-db-worker.sh

@@ -23,10 +23,16 @@ echo "Python: $PYTHON_BIN"
 
 while true; do
   echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Launching db_worker"
-  "$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker
+  if "$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker; then
+    exit_code=0
+  else
     exit_code=$?
+  fi
 
   echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker exited with code $exit_code"
+  if [ "$exit_code" -eq 137 ] || [ "$exit_code" -eq 9 ]; then
+    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker appears to have been SIGKILL'ed (likely OOM)." >&2
+  fi
 
   if [ "$MAX_RESTARTS" -gt 0 ] && [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
     echo "Max restarts reached ($MAX_RESTARTS). Exiting." >&2