feat(worker): Add db_worker service with automatic restart and logging

feat(task_overview): Display generated series in task overview
docs(README): Update documentation for background task workers and troubleshooting

README.md

@@ -11,6 +11,59 @@ Running locally (development)
   COMPOSE_ENV=dev docker compose -f docker/docker-compose.prod.yml -f docker/docker-compose.dev.yml up -d --build
   ```
 
+Background task workers (automatic)
+- The compose stack now includes a dedicated `worker` service which runs:
+
+  ```sh
+  python manage.py db_worker
+  ```
+
+- This means workers start automatically with `docker compose up` and restart on failure.
+- To inspect worker logs:
+
+  ```sh
+  docker compose logs -f worker
+  ```
+
+- To scale workers (for higher throughput):
+
+  ```sh
+  docker compose up -d --scale worker=2
+  ```
+
+Troubleshooting `SIGKILL` when running manually
+- A `SIGKILL` on `db_worker` is usually the kernel OOM killer (out-of-memory), not a Django exception.
+- Check kernel OOM events:
+
+  ```sh
+  dmesg -T | grep -i -E "killed process|out of memory|oom"
+  ```
+
+- If OOM is confirmed, prefer running worker inside compose (managed restart + container limits), reduce concurrency/parallel services, or add memory/swap.
+
+Manual worker startup (non-Docker production)
+- If you are not yet running production in Docker, start the worker using the restart wrapper script:
+
+  ```sh
+  source .venv/bin/activate
+  DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh
+  ```
+
+- This script restarts `db_worker` if it exits unexpectedly.
+
+- To run it in the background and keep logs:
+
+  ```sh
+  nohup DJANGO_SETTINGS_MODULE=rad.settings ./scripts/run-db-worker.sh > logs/db_worker.log 2>&1 &
+  ```
+
+- To verify it is still running:
+
+  ```sh
+  ps aux | grep "manage.py db_worker" | grep -v grep
+  tail -f logs/db_worker.log
+  ```
+
 - By default the development nginx ports are set in `.env.dev` to avoid colliding with a host nginx. The defaults now are:
   - `NGINX_HTTP_PORT=8080`
   - `NGINX_HTTPS_PORT=8444`

atlas/templates/atlas/task_overview.html

@@ -57,6 +57,7 @@ Atlas Task Overview
                 <tr>
                     <th>Task</th>
                     <th>Status</th>
+                    <th>Generated Series</th>
                     <th>Queue</th>
                     <th>Enqueued</th>
                     <th>Started</th>
@@ -83,6 +84,15 @@ Atlas Task Overview
                                 <span class="badge bg-secondary">{{ task.status }}</span>
                             {% endif %}
                         </td>
+                        <td class="small">
+                            {% if task.generated_series %}
+                                {% for series in task.generated_series %}
+                                    <a class="d-block" target="_blank" href="{{ series.url }}">{{ series.description }}</a>
+                                {% endfor %}
+                            {% else %}
+                                <span class="text-muted">-</span>
+                            {% endif %}
+                        </td>
                         <td>{{ task.queue_name }}</td>
                         <td class="small">{{ task.enqueued_at|date:"Y-m-d H:i:s" }}</td>
                         <td class="small">{{ task.started_at|date:"Y-m-d H:i:s" }}</td>
@@ -92,7 +102,7 @@ Atlas Task Overview
                     </tr>
                 {% empty %}
                     <tr>
-                        <td colspan="8" class="text-muted">No tasks found for this filter.</td>
+                        <td colspan="9" class="text-muted">No tasks found for this filter.</td>
                     </tr>
                 {% endfor %}
             </tbody>

atlas/views.py

@@ -828,9 +828,30 @@ def task_overview(request):
 
     base_qs = DBTaskResult.objects.all().order_by("-enqueued_at")
     if selected_status in status_map:
-        tasks = base_qs.filter(status__in=status_map[selected_status])[:300]
+        tasks = list(base_qs.filter(status__in=status_map[selected_status])[:300])
     else:
-        tasks = base_qs[:300]
+        tasks = list(base_qs[:300])
+
+    for task in tasks:
+        generated_series = []
+        payload = task.return_value
+
+        if isinstance(payload, str):
+            try:
+                payload = json.loads(payload)
+            except Exception:
+                payload = None
+
+        if isinstance(payload, dict):
+            for item in payload.get("created_series", []) or []:
+                if not isinstance(item, dict):
+                    continue
+                url = item.get("url")
+                description = item.get("description") or str(item.get("id", "Series"))
+                if url:
+                    generated_series.append({"url": url, "description": description})
+
+        task.generated_series = generated_series
 
     counts = {
         "active": DBTaskResult.objects.filter(status__in=["READY", "RUNNING"]).count(),

docker/docker-compose.dev.yml

@@ -17,6 +17,20 @@ services:
       - ../:/usr/src/app:cached
       # Mount the app log directory so logs are visible on the host at ./logs
       - ../logs:/var/log/rad
+
+  worker:
+    env_file:
+      - ../.env.dev
+    volumes:
+      - ../:/usr/src/app:cached
+      - ../logs:/var/log/rad
+    entrypoint:
+      [
+        "sh",
+        "-c",
+        "python manage.py migrate --noinput && exec python manage.py db_worker",
+      ]
+    command: []
   nginx:
     # Development nginx override: mount a simplified config that does not
     # reference LetsEncrypt certs so the container can start without real

docker/docker-compose.local.yml

@@ -30,6 +30,23 @@ services:
       - 3459:3459
     env_file:
       - ./.env.dev.local
+
+  worker:
+    build:
+      context: ..
+      dockerfile: rad/Dockerfile
+    command: python manage.py db_worker
+    restart: unless-stopped
+    volumes:
+      - ../:/usr/src/app
+      - ../backups:/usr/src/app/backups
+      - ../media:/usr/src/app/media
+      - ../static:/usr/src/app/static
+    env_file:
+      - ./.env.dev.local
+    depends_on:
+      - web
+      - db
   db:
     image: postgres:14.2-alpine
     volumes:

docker/docker-compose.prod.yml

@@ -35,6 +35,30 @@ services:
       - "8000"
     command: ["/usr/src/app/entrypoint.sh"]
 
+  worker:
+    build:
+      context: ..
+      dockerfile: rad/Dockerfile.prod
+    restart: always
+    env_file:
+      - ../.env.${COMPOSE_ENV:-prod}
+    environment:
+      - DJANGO_SETTINGS_MODULE=rad.settings
+    depends_on:
+      redis:
+        condition: service_healthy
+      web:
+        condition: service_started
+    volumes:
+      - ../deploy/settings_local.py:/usr/src/app/rad/settings_local.py:ro
+      - ../logs:/var/log/rad:rw
+    entrypoint:
+      [
+        "sh",
+        "-c",
+        "python manage.py migrate --noinput && exec python manage.py db_worker",
+      ]
+
   nginx:
     image: nginx:stable-alpine
     restart: always

scripts/run-db-worker.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env sh
+set -eu
+
+ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+PYTHON_BIN="${PYTHON_BIN:-$ROOT_DIR/.venv/bin/python}"
+DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-rad.settings}"
+RESTART_DELAY="${RESTART_DELAY:-3}"
+MAX_RESTARTS="${MAX_RESTARTS:-0}"
+
+if [ ! -x "$PYTHON_BIN" ]; then
+  echo "Python binary not found or not executable: $PYTHON_BIN" >&2
+  exit 1
+fi
+
+export DJANGO_SETTINGS_MODULE
+
+restart_count=0
+
+echo "Starting persistent db_worker loop"
+echo "Root: $ROOT_DIR"
+echo "Settings: $DJANGO_SETTINGS_MODULE"
+echo "Python: $PYTHON_BIN"
+
+while true; do
+  echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Launching db_worker"
+  "$PYTHON_BIN" "$ROOT_DIR/manage.py" db_worker
+  exit_code=$?
+
+  echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] db_worker exited with code $exit_code"
+
+  if [ "$MAX_RESTARTS" -gt 0 ] && [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
+    echo "Max restarts reached ($MAX_RESTARTS). Exiting." >&2
+    exit "$exit_code"
+  fi
+
+  restart_count=$((restart_count + 1))
+  echo "Restarting in ${RESTART_DELAY}s (restart #$restart_count)"
+  sleep "$RESTART_DELAY"
+done

templates/base.html

@@ -333,6 +333,11 @@
                                             <a class="dropdown-item" href="{% url 'media_cleanup' %}">Unused media cleanup</a>
                                         </li>
                                     {% endif %}
+                                    {% if request.user.is_superuser or request.user|has_group:"atlas_editor" %}
+                                        <li>
+                                            <a class="dropdown-item" href="{% url 'atlas:task_overview' %}">Atlas tasks overview</a>
+                                        </li>
+                                    {% endif %}
                                 </ul>
                             </li>
                         {% endif %}