diff --git a/README.md b/README.md index 2a9f3869..4e078f6f 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,19 @@ Manual worker startup (non-Docker production) - To run it in the background and keep logs: ```sh - nohup DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 & + nohup env DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 & ``` +- In fish shell, follow with `disown` so the shell does not keep the job attached: + + ```sh + nohup env DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 & + disown + ``` + +- For server reliability, prefer running the same script under `systemd` (auto-start on reboot, restart-on-failure). +- A starter unit file is provided at `scripts/db-worker.service.example`. + - To verify it is still running: ```sh diff --git a/atlas/tasks.py b/atlas/tasks.py index 17cd386c..8e355f8e 100644 --- a/atlas/tasks.py +++ b/atlas/tasks.py @@ -213,104 +213,109 @@ def series_reconstruct_task( if volume_for_recon.shape[0] < 1: raise ValueError("No reconstruction slices were generated") - base_z_offset = float(recon_centers_mm[0]) if len(recon_centers_mm) > 0 else 0.0 + # Determine source acquisition plane from normal vector. + dominant_normal_axis = int(np.argmax(np.abs(normal_dir))) + source_plane = {0: "sagittal", 1: "coronal", 2: "axial"}.get(dominant_normal_axis, "unknown") - def build_position(origin, row_vector, col_vector, stack_vector, row_index=0, col_index=0, stack_offset=0.0): + # Reorient the reconstructed volume into patient axes order [z, y, x] so + # generated plane names are anatomically correct regardless of source plane. + source_axis_dirs = [normal_dir, row_dir, col_dir] + source_axis_spacings = [float(target_spacing), float(native_row_spacing), float(native_col_spacing)] + source_axis_sizes = [int(volume_for_recon.shape[0]), int(volume_for_recon.shape[1]), int(volume_for_recon.shape[2])] + + source_for_patient_axis = {} + sign_for_patient_axis = {} + patient_axes_claimed = set() + + for src_axis, vec in enumerate(source_axis_dirs): + patient_axis = int(np.argmax(np.abs(vec))) + if patient_axis in patient_axes_claimed: + raise ValueError("Could not infer unique source orientation axes for reconstruction") + patient_axes_claimed.add(patient_axis) + source_for_patient_axis[patient_axis] = src_axis + sign_for_patient_axis[patient_axis] = 1 if float(vec[patient_axis]) >= 0 else -1 + + src_x = source_for_patient_axis[0] + src_y = source_for_patient_axis[1] + src_z = source_for_patient_axis[2] + + spacing_x = source_axis_spacings[src_x] + spacing_y = source_axis_spacings[src_y] + spacing_z = source_axis_spacings[src_z] + + vol_zyx = np.transpose(volume_for_recon, (src_z, src_y, src_x)) + + if sign_for_patient_axis[2] < 0: + vol_zyx = np.flip(vol_zyx, axis=0) + if sign_for_patient_axis[1] < 0: + vol_zyx = np.flip(vol_zyx, axis=1) + if sign_for_patient_axis[0] < 0: + vol_zyx = np.flip(vol_zyx, axis=2) + + def source_indices_from_patient_zyx(iz, iy, ix): + src_indices = [0, 0, 0] + + z_index = int(iz) + y_index = int(iy) + x_index = int(ix) + + if sign_for_patient_axis[2] < 0: + z_index = source_axis_sizes[src_z] - 1 - z_index + if sign_for_patient_axis[1] < 0: + y_index = source_axis_sizes[src_y] - 1 - y_index + if sign_for_patient_axis[0] < 0: + x_index = source_axis_sizes[src_x] - 1 - x_index + + src_indices[src_z] = z_index + src_indices[src_y] = y_index + src_indices[src_x] = x_index + return src_indices + + def position_from_patient_zyx(iz, iy, ix): + src_k, src_r, src_c = source_indices_from_patient_zyx(iz, iy, ix) pos = ( - np.asarray(origin, dtype=float) - + (np.asarray(stack_vector, dtype=float) * float(stack_offset)) - + (np.asarray(row_vector, dtype=float) * (float(row_index) * float(native_row_spacing))) - + (np.asarray(col_vector, dtype=float) * (float(col_index) * float(native_col_spacing))) + np.asarray(origin_ipp, dtype=float) + + (np.asarray(normal_dir, dtype=float) * (float(src_k) * float(target_spacing))) + + (np.asarray(row_dir, dtype=float) * (float(src_r) * float(native_row_spacing))) + + (np.asarray(col_dir, dtype=float) * (float(src_c) * float(native_col_spacing))) ) return [float(pos[0]), float(pos[1]), float(pos[2])] - plane_slices = {} + plane_builders = {} for plane in recon_planes: plane_norm = plane.lower() if plane_norm == "axial": - recon_slices = [volume_for_recon[i, :, :] for i in range(volume_for_recon.shape[0])] - pixel_spacing_out = [native_row_spacing, native_col_spacing] - spacing_between_slices_out = float(target_spacing) - image_orientation_out = [ - float(row_dir[0]), - float(row_dir[1]), - float(row_dir[2]), - float(col_dir[0]), - float(col_dir[1]), - float(col_dir[2]), - ] - - def position_for_index(idx): - return build_position( - origin_ipp, - row_dir, - col_dir, - normal_dir, - row_index=0, - col_index=0, - stack_offset=float(recon_centers_mm[idx]), - ) + plane_builders[plane_norm] = { + "count": int(vol_zyx.shape[0]), + "slice_fn": lambda idx: vol_zyx[idx, :, :], + "pixel_spacing_out": [float(spacing_y), float(spacing_x)], + "spacing_between_slices_out": float(spacing_z), + "image_orientation_out": [0.0, 1.0, 0.0, 1.0, 0.0, 0.0], + "position_fn": lambda idx: position_from_patient_zyx(idx, 0, 0), + } elif plane_norm == "coronal": - recon_slices = [volume_for_recon[:, i, :] for i in range(volume_for_recon.shape[1])] - pixel_spacing_out = [target_spacing, native_col_spacing] - spacing_between_slices_out = float(native_row_spacing) - image_orientation_out = [ - float(normal_dir[0]), - float(normal_dir[1]), - float(normal_dir[2]), - float(col_dir[0]), - float(col_dir[1]), - float(col_dir[2]), - ] - - def position_for_index(idx): - return build_position( - origin_ipp, - row_dir, - col_dir, - normal_dir, - row_index=idx, - col_index=0, - stack_offset=base_z_offset, - ) + plane_builders[plane_norm] = { + "count": int(vol_zyx.shape[1]), + "slice_fn": lambda idx: vol_zyx[:, idx, :], + "pixel_spacing_out": [float(spacing_z), float(spacing_x)], + "spacing_between_slices_out": float(spacing_y), + "image_orientation_out": [0.0, 0.0, 1.0, 1.0, 0.0, 0.0], + "position_fn": lambda idx: position_from_patient_zyx(0, idx, 0), + } elif plane_norm == "sagittal": - recon_slices = [volume_for_recon[:, :, i] for i in range(volume_for_recon.shape[2])] - pixel_spacing_out = [target_spacing, native_row_spacing] - spacing_between_slices_out = float(native_col_spacing) - image_orientation_out = [ - float(normal_dir[0]), - float(normal_dir[1]), - float(normal_dir[2]), - float(row_dir[0]), - float(row_dir[1]), - float(row_dir[2]), - ] + plane_builders[plane_norm] = { + "count": int(vol_zyx.shape[2]), + "slice_fn": lambda idx: vol_zyx[:, :, idx], + "pixel_spacing_out": [float(spacing_z), float(spacing_y)], + "spacing_between_slices_out": float(spacing_x), + "image_orientation_out": [0.0, 0.0, 1.0, 0.0, 1.0, 0.0], + "position_fn": lambda idx: position_from_patient_zyx(0, 0, idx), + } - def position_for_index(idx): - return build_position( - origin_ipp, - row_dir, - col_dir, - normal_dir, - row_index=0, - col_index=idx, - stack_offset=base_z_offset, - ) - else: - continue - - plane_slices[plane_norm] = { - "recon_slices": recon_slices, - "pixel_spacing_out": pixel_spacing_out, - "spacing_between_slices_out": spacing_between_slices_out, - "image_orientation_out": image_orientation_out, - "position_for_index": position_for_index, - } - - if not plane_slices: + if not plane_builders: raise ValueError("No valid reconstruction planes selected") - total_slices = sum(len(v["recon_slices"]) for v in plane_slices.values()) + total_slices = sum(v["count"] for v in plane_builders.values()) cache.set( progress_key, {"current": 0, "total": total_slices, "message": "Reconstruction started"}, @@ -320,12 +325,13 @@ def series_reconstruct_task( processed = 0 created_series = [] - for plane_norm, cfg in plane_slices.items(): - recon_series = atlas_views._create_series_derivative(series, f"Recon {plane_norm.title()}") + for plane_norm, cfg in plane_builders.items(): + recon_series = atlas_views._create_series_derivative(series, f"Recon {plane_norm.title()} ({source_plane.title()} src)") recon_series.series_instance_uid = generate_uid() recon_series.save(update_fields=["series_instance_uid"]) - for idx, arr2d in enumerate(cfg["recon_slices"]): + for idx in range(int(cfg["count"])): + arr2d = cfg["slice_fn"](idx) ds_new = copy.deepcopy(template_ds) arr2d = np.asarray(arr2d, dtype=dicom_items[0][2].dtype) @@ -398,4 +404,5 @@ def series_reconstruct_task( "target_spacing": float(target_spacing), "slab_thickness": float(slab_thickness), "mode": recon_thickness_mode, + "source_plane": source_plane, } \ No newline at end of file diff --git a/scripts/db-worker.service.example b/scripts/db-worker.service.example new file mode 100644 index 00000000..4cad4315 --- /dev/null +++ b/scripts/db-worker.service.example @@ -0,0 +1,16 @@ +[Unit] +Description=PENRA django-tasks db_worker +After=network.target + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=/home/ross/web/rad +Environment=DJANGO_SETTINGS_MODULE=rad.settings +ExecStart=/home/ross/web/rad/scripts/run-db-worker.sh +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/run-db-worker.sh b/scripts/run-db-worker.sh index 0e1ca641..592fd64c 100755 --- a/scripts/run-db-worker.sh +++ b/scripts/run-db-worker.sh @@ -23,10 +23,16 @@ echo "Python: $PYTHON_BIN" while true; do echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Launching db_worker" - "$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker - exit_code=$? + if "$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker; then + exit_code=0 + else + exit_code=$? + fi echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker exited with code $exit_code" + if [ "$exit_code" -eq 137 ] || [ "$exit_code" -eq 9 ]; then + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker appears to have been SIGKILL'ed (likely OOM)." >&2 + fi if [ "$MAX_RESTARTS" -gt 0 ] && [ "$restart_count" -ge "$MAX_RESTARTS" ]; then echo "Max restarts reached ($MAX_RESTARTS). Exiting." >&2