ross/penracourses
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(worker): Enhance db_worker script for better error handling and add systemd service example

Browse Source
This commit is contained in:
Ross
2026-05-17 21:36:46 +01:00
parent c2d8376e22
commit d77c5149f0
4 changed files with 130 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+11 -1
View File
@@ -54,9 +54,19 @@ Manual worker startup (non-Docker production)
- To run it in the background and keep logs:
```sh
nohup DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 &
nohup env DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 &
```
- In fish shell, follow with `disown` so the shell does not keep the job attached:
```sh
nohup env DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 &
disown
```
- For server reliability, prefer running the same script under `systemd` (auto-start on reboot, restart-on-failure).
- A starter unit file is provided at `scripts/db-worker.service.example`.
- To verify it is still running:
```sh
atlas/tasks.py
+95 -88
View File
@@ -213,104 +213,109 @@ def series_reconstruct_task(
if volume_for_recon.shape[0] < 1:
raise ValueError("No reconstruction slices were generated")
base_z_offset = float(recon_centers_mm[0]) if len(recon_centers_mm) > 0 else 0.0
# Determine source acquisition plane from normal vector.
dominant_normal_axis = int(np.argmax(np.abs(normal_dir)))
source_plane = {0: "sagittal", 1: "coronal", 2: "axial"}.get(dominant_normal_axis, "unknown")
def build_position(origin, row_vector, col_vector, stack_vector, row_index=0, col_index=0, stack_offset=0.0):
# Reorient the reconstructed volume into patient axes order [z, y, x] so
# generated plane names are anatomically correct regardless of source plane.
source_axis_dirs = [normal_dir, row_dir, col_dir]
source_axis_spacings = [float(target_spacing), float(native_row_spacing), float(native_col_spacing)]
source_axis_sizes = [int(volume_for_recon.shape[0]), int(volume_for_recon.shape[1]), int(volume_for_recon.shape[2])]
source_for_patient_axis = {}
sign_for_patient_axis = {}
patient_axes_claimed = set()
for src_axis, vec in enumerate(source_axis_dirs):
patient_axis = int(np.argmax(np.abs(vec)))
if patient_axis in patient_axes_claimed:
raise ValueError("Could not infer unique source orientation axes for reconstruction")
patient_axes_claimed.add(patient_axis)
source_for_patient_axis[patient_axis] = src_axis
sign_for_patient_axis[patient_axis] = 1 if float(vec[patient_axis]) >= 0 else -1
src_x = source_for_patient_axis[0]
src_y = source_for_patient_axis[1]
src_z = source_for_patient_axis[2]
spacing_x = source_axis_spacings[src_x]
spacing_y = source_axis_spacings[src_y]
spacing_z = source_axis_spacings[src_z]
vol_zyx = np.transpose(volume_for_recon, (src_z, src_y, src_x))
if sign_for_patient_axis[2] < 0:
vol_zyx = np.flip(vol_zyx, axis=0)
if sign_for_patient_axis[1] < 0:
vol_zyx = np.flip(vol_zyx, axis=1)
if sign_for_patient_axis[0] < 0:
vol_zyx = np.flip(vol_zyx, axis=2)
def source_indices_from_patient_zyx(iz, iy, ix):
src_indices = [0, 0, 0]
z_index = int(iz)
y_index = int(iy)
x_index = int(ix)
if sign_for_patient_axis[2] < 0:
z_index = source_axis_sizes[src_z] - 1 - z_index
if sign_for_patient_axis[1] < 0:
y_index = source_axis_sizes[src_y] - 1 - y_index
if sign_for_patient_axis[0] < 0:
x_index = source_axis_sizes[src_x] - 1 - x_index
src_indices[src_z] = z_index
src_indices[src_y] = y_index
src_indices[src_x] = x_index
return src_indices
def position_from_patient_zyx(iz, iy, ix):
src_k, src_r, src_c = source_indices_from_patient_zyx(iz, iy, ix)
pos = (
np.asarray(origin, dtype=float)
+ (np.asarray(stack_vector, dtype=float) * float(stack_offset))
+ (np.asarray(row_vector, dtype=float) * (float(row_index) * float(native_row_spacing)))
+ (np.asarray(col_vector, dtype=float) * (float(col_index) * float(native_col_spacing)))
np.asarray(origin_ipp, dtype=float)
+ (np.asarray(normal_dir, dtype=float) * (float(src_k) * float(target_spacing)))
+ (np.asarray(row_dir, dtype=float) * (float(src_r) * float(native_row_spacing)))
+ (np.asarray(col_dir, dtype=float) * (float(src_c) * float(native_col_spacing)))
)
return [float(pos[0]), float(pos[1]), float(pos[2])]
plane_slices = {}
plane_builders = {}
for plane in recon_planes:
plane_norm = plane.lower()
if plane_norm == "axial":
recon_slices = [volume_for_recon[i, :, :] for i in range(volume_for_recon.shape[0])]
pixel_spacing_out = [native_row_spacing, native_col_spacing]
spacing_between_slices_out = float(target_spacing)
image_orientation_out = [
float(row_dir[0]),
float(row_dir[1]),
float(row_dir[2]),
float(col_dir[0]),
float(col_dir[1]),
float(col_dir[2]),
]
def position_for_index(idx):
return build_position(
origin_ipp,
row_dir,
col_dir,
normal_dir,
row_index=0,
col_index=0,
stack_offset=float(recon_centers_mm[idx]),
)
plane_builders[plane_norm] = {
"count": int(vol_zyx.shape[0]),
"slice_fn": lambda idx: vol_zyx[idx, :, :],
"pixel_spacing_out": [float(spacing_y), float(spacing_x)],
"spacing_between_slices_out": float(spacing_z),
"image_orientation_out": [0.0, 1.0, 0.0, 1.0, 0.0, 0.0],
"position_fn": lambda idx: position_from_patient_zyx(idx, 0, 0),
}
elif plane_norm == "coronal":
recon_slices = [volume_for_recon[:, i, :] for i in range(volume_for_recon.shape[1])]
pixel_spacing_out = [target_spacing, native_col_spacing]
spacing_between_slices_out = float(native_row_spacing)
image_orientation_out = [
float(normal_dir[0]),
float(normal_dir[1]),
float(normal_dir[2]),
float(col_dir[0]),
float(col_dir[1]),
float(col_dir[2]),
]
def position_for_index(idx):
return build_position(
origin_ipp,
row_dir,
col_dir,
normal_dir,
row_index=idx,
col_index=0,
stack_offset=base_z_offset,
)
plane_builders[plane_norm] = {
"count": int(vol_zyx.shape[1]),
"slice_fn": lambda idx: vol_zyx[:, idx, :],
"pixel_spacing_out": [float(spacing_z), float(spacing_x)],
"spacing_between_slices_out": float(spacing_y),
"image_orientation_out": [0.0, 0.0, 1.0, 1.0, 0.0, 0.0],
"position_fn": lambda idx: position_from_patient_zyx(0, idx, 0),
}
elif plane_norm == "sagittal":
recon_slices = [volume_for_recon[:, :, i] for i in range(volume_for_recon.shape[2])]
pixel_spacing_out = [target_spacing, native_row_spacing]
spacing_between_slices_out = float(native_col_spacing)
image_orientation_out = [
float(normal_dir[0]),
float(normal_dir[1]),
float(normal_dir[2]),
float(row_dir[0]),
float(row_dir[1]),
float(row_dir[2]),
]
plane_builders[plane_norm] = {
"count": int(vol_zyx.shape[2]),
"slice_fn": lambda idx: vol_zyx[:, :, idx],
"pixel_spacing_out": [float(spacing_z), float(spacing_y)],
"spacing_between_slices_out": float(spacing_x),
"image_orientation_out": [0.0, 0.0, 1.0, 0.0, 1.0, 0.0],
"position_fn": lambda idx: position_from_patient_zyx(0, 0, idx),
}
def position_for_index(idx):
return build_position(
origin_ipp,
row_dir,
col_dir,
normal_dir,
row_index=0,
col_index=idx,
stack_offset=base_z_offset,
)
else:
continue
plane_slices[plane_norm] = {
"recon_slices": recon_slices,
"pixel_spacing_out": pixel_spacing_out,
"spacing_between_slices_out": spacing_between_slices_out,
"image_orientation_out": image_orientation_out,
"position_for_index": position_for_index,
}
if not plane_slices:
if not plane_builders:
raise ValueError("No valid reconstruction planes selected")
total_slices = sum(len(v["recon_slices"]) for v in plane_slices.values())
total_slices = sum(v["count"] for v in plane_builders.values())
cache.set(
progress_key,
{"current": 0, "total": total_slices, "message": "Reconstruction started"},
@@ -320,12 +325,13 @@ def series_reconstruct_task(
processed = 0
created_series = []
for plane_norm, cfg in plane_slices.items():
recon_series = atlas_views._create_series_derivative(series, f"Recon {plane_norm.title()}")
for plane_norm, cfg in plane_builders.items():
recon_series = atlas_views._create_series_derivative(series, f"Recon {plane_norm.title()} ({source_plane.title()} src)")
recon_series.series_instance_uid = generate_uid()
recon_series.save(update_fields=["series_instance_uid"])
for idx, arr2d in enumerate(cfg["recon_slices"]):
for idx in range(int(cfg["count"])):
arr2d = cfg["slice_fn"](idx)
ds_new = copy.deepcopy(template_ds)
arr2d = np.asarray(arr2d, dtype=dicom_items[0][2].dtype)
@@ -398,4 +404,5 @@ def series_reconstruct_task(
"target_spacing": float(target_spacing),
"slab_thickness": float(slab_thickness),
"mode": recon_thickness_mode,
"source_plane": source_plane,
}
scripts/db-worker.service.example
+16
View File
@@ -0,0 +1,16 @@
[Unit]
Description=PENRA django-tasks db_worker
After=network.target
[Service]
Type=simple
User=www-data
Group=www-data
WorkingDirectory=/home/ross/web/rad
Environment=DJANGO_SETTINGS_MODULE=rad.settings
ExecStart=/home/ross/web/rad/scripts/run-db-worker.sh
Restart=always
RestartSec=3
[Install]
WantedBy=multi-user.target
scripts/run-db-worker.sh
+8 -2
View File
@@ -23,10 +23,16 @@ echo "Python: $PYTHON_BIN"
while true; do
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Launching db_worker"
"$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker
exit_code=$?
if "$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker; then
exit_code=0
else
exit_code=$?
fi
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker exited with code $exit_code"
if [ "$exit_code" -eq 137 ] || [ "$exit_code" -eq 9 ]; then
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker appears to have been SIGKILL'ed (likely OOM)." >&2
fi
if [ "$MAX_RESTARTS" -gt 0 ] && [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
echo "Max restarts reached ($MAX_RESTARTS). Exiting." >&2