Add functionality to reset stale RUNNING tasks to READY in task overview

atlas/admin.py

@@ -32,6 +32,8 @@ from reversion.admin import VersionAdmin
 from django.forms.widgets import RadioSelect
 from django.utils.html import format_html
 from django.urls import reverse
+from django.utils import timezone
+from datetime import timedelta
 
 from tinymce.widgets import TinyMCE
 
@@ -226,6 +228,24 @@ if DBTaskResult is not None:
             msg += f" Skipped {blocked} running task(s)."
         self.message_user(request, msg, level=messages.SUCCESS)
 
+    @admin.action(description="Reset stale RUNNING tasks to READY (>30 min)")
+    def reset_stale_running_to_ready(self, request, queryset):
+        cutoff = timezone.now() - timedelta(minutes=30)
+        updated = queryset.filter(status="RUNNING", started_at__lt=cutoff).update(
+            status="READY",
+            started_at=None,
+            finished_at=None,
+            worker_ids="",
+            exception_class_path="",
+            traceback="",
+            return_value=None,
+        )
+        self.message_user(
+            request,
+            f"Reset {updated} stale RUNNING task(s) to READY (older than 30 minutes).",
+            level=messages.SUCCESS,
+        )
+
     @admin.action(description="Delete selected COMPLETED tasks")
     def delete_selected_completed(self, request, queryset):
         completed_qs = queryset.filter(status="SUCCESSFUL")
@@ -278,6 +298,7 @@ if DBTaskResult is not None:
         admin_cls.actions = (
             "reset_failed_to_ready",
             "reset_selected_to_ready",
+            "reset_stale_running_to_ready",
             "delete_selected_completed",
             "delete_selected_failed",
         )
@@ -339,6 +360,7 @@ if DBTaskResult is not None:
         admin_cls.traceback_pretty = traceback_pretty
         admin_cls.reset_failed_to_ready = reset_failed_to_ready
         admin_cls.reset_selected_to_ready = reset_selected_to_ready
+        admin_cls.reset_stale_running_to_ready = reset_stale_running_to_ready
         admin_cls.delete_selected_completed = delete_selected_completed
         admin_cls.delete_selected_failed = delete_selected_failed
         admin_cls.has_add_permission = has_add_permission

atlas/templates/atlas/task_overview.html

@@ -82,10 +82,22 @@
                     <option value="">Choose task action...</option>
                     <option value="reset_failed">Reset FAILED to READY</option>
                     <option value="reset_selected">Reset selected to READY (skip RUNNING)</option>
+                    <option value="reset_stale_running">Reset stale RUNNING to READY</option>
                     <option value="delete_completed">Delete selected COMPLETED</option>
                     <option value="delete_failed">Delete selected FAILED</option>
                     <option value="delete_selected_all">Delete ALL selected (including RUNNING)</option>
                 </select>
+                <input
+                    class="form-control form-control-sm"
+                    type="number"
+                    name="stale_minutes"
+                    min="1"
+                    step="1"
+                    value="30"
+                    style="max-width: 150px;"
+                    title="Stale cutoff in minutes for reset stale RUNNING action"
+                    aria-label="Stale minutes"
+                >
                 <button type="submit" class="btn btn-sm btn-primary">Apply to selected</button>
                 <div class="form-check ms-2">
                     <input class="form-check-input" type="checkbox" id="select-all-tasks">

atlas/views.py

@@ -4,6 +4,7 @@ import csv
 import random
 import copy
 import io
+from datetime import timedelta
 from typing import Any
 from dal import autocomplete
 from django.db.models.base import Model as Model
@@ -25,6 +26,7 @@ from django.template.defaultfilters import filesizeformat
 # from django.contrib.auth.models import User
 from django.contrib.auth.decorators import login_required, user_passes_test
 from django.contrib.auth.models import User
+from django.contrib import messages
 from django.core.exceptions import (
     FieldError,
     ObjectDoesNotExist,
@@ -882,7 +884,7 @@ def series_reconstruct_status_htmx(request, series_id, task_id):
 @login_required
 @user_is_atlas_editor
 def task_overview(request):
-    """Atlas admin-style task overview page with active/completed/failed task summaries."""
+    """Atlas editor task control page for inspecting and recovering queued/background tasks."""
     if DBTaskResult is None:
         return render(
             request,
@@ -930,6 +932,27 @@ def task_overview(request):
             if blocked:
                 msg += f" Skipped {blocked} running task(s)."
             messages.success(request, msg)
+        elif action == "reset_stale_running":
+            stale_minutes_raw = (request.POST.get("stale_minutes") or "30").strip()
+            try:
+                stale_minutes = max(int(stale_minutes_raw), 1)
+            except ValueError:
+                stale_minutes = 30
+
+            cutoff = timezone.now() - timedelta(minutes=stale_minutes)
+            updated = selected_qs.filter(status="RUNNING", started_at__lt=cutoff).update(
+                status="READY",
+                started_at=None,
+                finished_at=None,
+                worker_ids="",
+                exception_class_path="",
+                traceback="",
+                return_value=None,
+            )
+            messages.success(
+                request,
+                f"Reset {updated} stale RUNNING task(s) to READY (older than {stale_minutes} minute(s)).",
+            )
         elif action == "delete_completed":
             deleted, _ = selected_qs.filter(status="SUCCESSFUL").delete()
             messages.success(request, f"Deleted {deleted} completed task record(s).")