diff --git a/atlas/admin.py b/atlas/admin.py index ab1dabb1..36572ebc 100755 --- a/atlas/admin.py +++ b/atlas/admin.py @@ -32,6 +32,8 @@ from reversion.admin import VersionAdmin from django.forms.widgets import RadioSelect from django.utils.html import format_html from django.urls import reverse +from django.utils import timezone +from datetime import timedelta from tinymce.widgets import TinyMCE @@ -226,6 +228,24 @@ if DBTaskResult is not None: msg += f" Skipped {blocked} running task(s)." self.message_user(request, msg, level=messages.SUCCESS) + @admin.action(description="Reset stale RUNNING tasks to READY (>30 min)") + def reset_stale_running_to_ready(self, request, queryset): + cutoff = timezone.now() - timedelta(minutes=30) + updated = queryset.filter(status="RUNNING", started_at__lt=cutoff).update( + status="READY", + started_at=None, + finished_at=None, + worker_ids="", + exception_class_path="", + traceback="", + return_value=None, + ) + self.message_user( + request, + f"Reset {updated} stale RUNNING task(s) to READY (older than 30 minutes).", + level=messages.SUCCESS, + ) + @admin.action(description="Delete selected COMPLETED tasks") def delete_selected_completed(self, request, queryset): completed_qs = queryset.filter(status="SUCCESSFUL") @@ -278,6 +298,7 @@ if DBTaskResult is not None: admin_cls.actions = ( "reset_failed_to_ready", "reset_selected_to_ready", + "reset_stale_running_to_ready", "delete_selected_completed", "delete_selected_failed", ) @@ -339,6 +360,7 @@ if DBTaskResult is not None: admin_cls.traceback_pretty = traceback_pretty admin_cls.reset_failed_to_ready = reset_failed_to_ready admin_cls.reset_selected_to_ready = reset_selected_to_ready + admin_cls.reset_stale_running_to_ready = reset_stale_running_to_ready admin_cls.delete_selected_completed = delete_selected_completed admin_cls.delete_selected_failed = delete_selected_failed admin_cls.has_add_permission = has_add_permission diff --git a/atlas/templates/atlas/task_overview.html b/atlas/templates/atlas/task_overview.html index 460407c9..e4284a08 100644 --- a/atlas/templates/atlas/task_overview.html +++ b/atlas/templates/atlas/task_overview.html @@ -82,10 +82,22 @@ + +
diff --git a/atlas/views.py b/atlas/views.py index e7d426ad..72d1ca85 100755 --- a/atlas/views.py +++ b/atlas/views.py @@ -4,6 +4,7 @@ import csv import random import copy import io +from datetime import timedelta from typing import Any from dal import autocomplete from django.db.models.base import Model as Model @@ -25,6 +26,7 @@ from django.template.defaultfilters import filesizeformat # from django.contrib.auth.models import User from django.contrib.auth.decorators import login_required, user_passes_test from django.contrib.auth.models import User +from django.contrib import messages from django.core.exceptions import ( FieldError, ObjectDoesNotExist, @@ -882,7 +884,7 @@ def series_reconstruct_status_htmx(request, series_id, task_id): @login_required @user_is_atlas_editor def task_overview(request): - """Atlas admin-style task overview page with active/completed/failed task summaries.""" + """Atlas editor task control page for inspecting and recovering queued/background tasks.""" if DBTaskResult is None: return render( request, @@ -930,6 +932,27 @@ def task_overview(request): if blocked: msg += f" Skipped {blocked} running task(s)." messages.success(request, msg) + elif action == "reset_stale_running": + stale_minutes_raw = (request.POST.get("stale_minutes") or "30").strip() + try: + stale_minutes = max(int(stale_minutes_raw), 1) + except ValueError: + stale_minutes = 30 + + cutoff = timezone.now() - timedelta(minutes=stale_minutes) + updated = selected_qs.filter(status="RUNNING", started_at__lt=cutoff).update( + status="READY", + started_at=None, + finished_at=None, + worker_ids="", + exception_class_path="", + traceback="", + return_value=None, + ) + messages.success( + request, + f"Reset {updated} stale RUNNING task(s) to READY (older than {stale_minutes} minute(s)).", + ) elif action == "delete_completed": deleted, _ = selected_qs.filter(status="SUCCESSFUL").delete() messages.success(request, f"Deleted {deleted} completed task record(s).")