From c2d8376e223cb99646677af9af890b1ceb5ee2c7 Mon Sep 17 00:00:00 2001 From: Ross Date: Sun, 17 May 2026 21:13:40 +0100 Subject: [PATCH] feat(worker): Add db_worker service with automatic restart and logging feat(task_overview): Display generated series in task overview docs(README): Update documentation for background task workers and troubleshooting --- README.md | 53 ++++++++++++++++++++++++ atlas/templates/atlas/task_overview.html | 12 +++++- atlas/views.py | 25 ++++++++++- docker/docker-compose.dev.yml | 14 +++++++ docker/docker-compose.local.yml | 17 ++++++++ docker/docker-compose.prod.yml | 24 +++++++++++ scripts/run-db-worker.sh | 39 +++++++++++++++++ templates/base.html | 5 +++ 8 files changed, 186 insertions(+), 3 deletions(-) create mode 100755 scripts/run-db-worker.sh diff --git a/README.md b/README.md index f10b7633..2a9f3869 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,59 @@ Running locally (development) COMPOSE_ENV=dev docker compose -f docker/docker-compose.prod.yml -f docker/docker-compose.dev.yml up -d --build ``` +Background task workers (automatic) +- The compose stack now includes a dedicated `worker` service which runs: + + ```sh + python manage.py db_worker + ``` + +- This means workers start automatically with `docker compose up` and restart on failure. +- To inspect worker logs: + + ```sh + docker compose logs -f worker + ``` + +- To scale workers (for higher throughput): + + ```sh + docker compose up -d --scale worker=2 + ``` + +Troubleshooting `SIGKILL` when running manually +- A `SIGKILL` on `db_worker` is usually the kernel OOM killer (out-of-memory), not a Django exception. +- Check kernel OOM events: + + ```sh + dmesg -T | grep -i -E "killed process|out of memory|oom" + ``` + +- If OOM is confirmed, prefer running worker inside compose (managed restart + container limits), reduce concurrency/parallel services, or add memory/swap. + +Manual worker startup (non-Docker production) +- If you are not yet running production in Docker, start the worker using the restart wrapper script: + + ```sh + source .venv/bin/activate + DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh + ``` + +- This script restarts `db_worker` if it exits unexpectedly. + +- To run it in the background and keep logs: + + ```sh + nohup DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 & + ``` + +- To verify it is still running: + + ```sh + ps aux | grep "manage.py db_worker" | grep -v grep + tail -f logs/db_worker.log + ``` + - By default the development nginx ports are set in `.env.dev` to avoid colliding with a host nginx. The defaults now are: - `NGINX_HTTP_PORT=8080` - `NGINX_HTTPS_PORT=8444` diff --git a/atlas/templates/atlas/task_overview.html b/atlas/templates/atlas/task_overview.html index 9f76620b..5cb97242 100644 --- a/atlas/templates/atlas/task_overview.html +++ b/atlas/templates/atlas/task_overview.html @@ -57,6 +57,7 @@ Atlas Task Overview Task Status + Generated Series Queue Enqueued Started @@ -83,6 +84,15 @@ Atlas Task Overview {{ task.status }} {% endif %} + + {% if task.generated_series %} + {% for series in task.generated_series %} + {{ series.description }} + {% endfor %} + {% else %} + - + {% endif %} + {{ task.queue_name }} {{ task.enqueued_at|date:"Y-m-d H:i:s" }} {{ task.started_at|date:"Y-m-d H:i:s" }} @@ -92,7 +102,7 @@ Atlas Task Overview {% empty %} - No tasks found for this filter. + No tasks found for this filter. {% endfor %} diff --git a/atlas/views.py b/atlas/views.py index c9497fcd..a342d3f8 100755 --- a/atlas/views.py +++ b/atlas/views.py @@ -828,9 +828,30 @@ def task_overview(request): base_qs = DBTaskResult.objects.all().order_by("-enqueued_at") if selected_status in status_map: - tasks = base_qs.filter(status__in=status_map[selected_status])[:300] + tasks = list(base_qs.filter(status__in=status_map[selected_status])[:300]) else: - tasks = base_qs[:300] + tasks = list(base_qs[:300]) + + for task in tasks: + generated_series = [] + payload = task.return_value + + if isinstance(payload, str): + try: + payload = json.loads(payload) + except Exception: + payload = None + + if isinstance(payload, dict): + for item in payload.get("created_series", []) or []: + if not isinstance(item, dict): + continue + url = item.get("url") + description = item.get("description") or str(item.get("id", "Series")) + if url: + generated_series.append({"url": url, "description": description}) + + task.generated_series = generated_series counts = { "active": DBTaskResult.objects.filter(status__in=["READY", "RUNNING"]).count(), diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 5945db2e..1b0ea3b5 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -17,6 +17,20 @@ services: - ../:/usr/src/app:cached # Mount the app log directory so logs are visible on the host at ./logs - ../logs:/var/log/rad + + worker: + env_file: + - ../.env.dev + volumes: + - ../:/usr/src/app:cached + - ../logs:/var/log/rad + entrypoint: + [ + "sh", + "-c", + "python manage.py migrate --noinput && exec python manage.py db_worker", + ] + command: [] nginx: # Development nginx override: mount a simplified config that does not # reference LetsEncrypt certs so the container can start without real diff --git a/docker/docker-compose.local.yml b/docker/docker-compose.local.yml index 823c9e52..0b9d74e2 100644 --- a/docker/docker-compose.local.yml +++ b/docker/docker-compose.local.yml @@ -30,6 +30,23 @@ services: - 3459:3459 env_file: - ./.env.dev.local + + worker: + build: + context: .. + dockerfile: rad/Dockerfile + command: python manage.py db_worker + restart: unless-stopped + volumes: + - ../:/usr/src/app + - ../backups:/usr/src/app/backups + - ../media:/usr/src/app/media + - ../static:/usr/src/app/static + env_file: + - ./.env.dev.local + depends_on: + - web + - db db: image: postgres:14.2-alpine volumes: diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 68f0e361..4b303123 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -35,6 +35,30 @@ services: - "8000" command: ["/usr/src/app/entrypoint.sh"] + worker: + build: + context: .. + dockerfile: rad/Dockerfile.prod + restart: always + env_file: + - ../.env.${COMPOSE_ENV:-prod} + environment: + - DJANGO_SETTINGS_MODULE=rad.settings + depends_on: + redis: + condition: service_healthy + web: + condition: service_started + volumes: + - ../deploy/settings_local.py:/usr/src/app/rad/settings_local.py:ro + - ../logs:/var/log/rad:rw + entrypoint: + [ + "sh", + "-c", + "python manage.py migrate --noinput && exec python manage.py db_worker", + ] + nginx: image: nginx:stable-alpine restart: always diff --git a/scripts/run-db-worker.sh b/scripts/run-db-worker.sh new file mode 100755 index 00000000..0e1ca641 --- /dev/null +++ b/scripts/run-db-worker.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env sh +set -eu + +ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +PYTHON_BIN="${PYTHON_BIN:-$ROOT_DIR/.venv/bin/python}" +DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-rad.settings}" +RESTART_DELAY="${RESTART_DELAY:-3}" +MAX_RESTARTS="${MAX_RESTARTS:-0}" + +if [ ! -x "$PYTHON_BIN" ]; then + echo "Python binary not found or not executable: $PYTHON_BIN" >&2 + exit 1 +fi + +export DJANGO_SETTINGS_MODULE + +restart_count=0 + +echo "Starting persistent db_worker loop" +echo "Root: $ROOT_DIR" +echo "Settings: $DJANGO_SETTINGS_MODULE" +echo "Python: $PYTHON_BIN" + +while true; do + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Launching db_worker" + "$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker + exit_code=$? + + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker exited with code $exit_code" + + if [ "$MAX_RESTARTS" -gt 0 ] && [ "$restart_count" -ge "$MAX_RESTARTS" ]; then + echo "Max restarts reached ($MAX_RESTARTS). Exiting." >&2 + exit "$exit_code" + fi + + restart_count=$((restart_count + 1)) + echo "Restarting in ${RESTART_DELAY}s (restart #$restart_count)" + sleep "$RESTART_DELAY" +done diff --git a/templates/base.html b/templates/base.html index d16a4570..4ef055ba 100644 --- a/templates/base.html +++ b/templates/base.html @@ -333,6 +333,11 @@ Unused media cleanup {% endif %} + {% if request.user.is_superuser or request.user|has_group:"atlas_editor" %} +
  • + Atlas tasks overview +
    • + {% endif %} {% endif %}