feat: Add media cleanup functionality with UI and backend support for unused media management
This commit is contained in:
Ross
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth.models import Group
|
||||
from django.urls import reverse
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cid_manager_user(db, django_user_model):
|
||||
user = django_user_model.objects.create_user(
|
||||
username="cid_manager",
|
||||
email="cid_manager@example.com",
|
||||
password="password123",
|
||||
)
|
||||
group, _ = Group.objects.get_or_create(name="cid_user_manager")
|
||||
user.groups.add(group)
|
||||
return user
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_media_cleanup_requires_login(client):
|
||||
response = client.get(reverse("media_cleanup"))
|
||||
assert response.status_code == 302
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_media_cleanup_requires_manager_group(client, django_user_model):
|
||||
user = django_user_model.objects.create_user(
|
||||
username="regular_user",
|
||||
email="regular@example.com",
|
||||
password="password123",
|
||||
)
|
||||
client.force_login(user)
|
||||
|
||||
response = client.get(reverse("media_cleanup"))
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_media_cleanup_page_for_manager(client, cid_manager_user):
|
||||
client.force_login(cid_manager_user)
|
||||
|
||||
response = client.get(reverse("media_cleanup"))
|
||||
assert response.status_code == 200
|
||||
assert b"Unused Media Cleanup" in response.content
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_media_cleanup_preview_uses_dry_run(client, cid_manager_user, monkeypatch):
|
||||
client.force_login(cid_manager_user)
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_call_command(name, **kwargs):
|
||||
captured["name"] = name
|
||||
captured["kwargs"] = kwargs
|
||||
kwargs["stdout"].write("preview ok")
|
||||
|
||||
monkeypatch.setattr("rad.views.call_command", fake_call_command)
|
||||
|
||||
response = client.post(
|
||||
reverse("media_cleanup"),
|
||||
{
|
||||
"action": "preview",
|
||||
"minimum_file_age": "30",
|
||||
"remove_empty_dirs": "on",
|
||||
"exclude": "thumbnails/*\ncache/*",
|
||||
},
|
||||
HTTP_HX_REQUEST="true",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert captured["name"] == "cleanup_unused_media"
|
||||
assert captured["kwargs"]["dry_run"] is True
|
||||
assert captured["kwargs"]["no_input"] is True
|
||||
assert captured["kwargs"]["interactive"] is False
|
||||
assert captured["kwargs"]["minimum_file_age"] == 30
|
||||
assert captured["kwargs"]["remove_empty_dirs"] is True
|
||||
assert captured["kwargs"]["exclude"] == ["thumbnails/*", "cache/*"]
|
||||
assert b"preview ok" in response.content
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_media_cleanup_live_run_disables_dry_run(client, cid_manager_user, monkeypatch):
|
||||
client.force_login(cid_manager_user)
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_call_command(name, **kwargs):
|
||||
captured["name"] = name
|
||||
captured["kwargs"] = kwargs
|
||||
kwargs["stdout"].write("cleanup ok")
|
||||
|
||||
monkeypatch.setattr("rad.views.call_command", fake_call_command)
|
||||
|
||||
response = client.post(
|
||||
reverse("media_cleanup"),
|
||||
{
|
||||
"action": "cleanup",
|
||||
"minimum_file_age": "0",
|
||||
},
|
||||
HTTP_HX_REQUEST="true",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert captured["name"] == "cleanup_unused_media"
|
||||
assert captured["kwargs"]["dry_run"] is False
|
||||
assert captured["kwargs"]["interactive"] is False
|
||||
assert b"cleanup ok" in response.content
|
||||
assert b"Recovered space" in response.content
|
||||
@@ -94,6 +94,7 @@ urlpatterns = [
|
||||
),
|
||||
path("server", views.server, name="server"),
|
||||
path("logs/", views.logs_view, name="logs_view"),
|
||||
path("management/media-cleanup/", views.media_cleanup, name="media_cleanup"),
|
||||
path("people", views.people, name="people"),
|
||||
path("accounts/", views.UserListTableView.as_view(), name="accounts_list"),
|
||||
#path("accounts/", views.UserListView.as_view(), name="accounts_list"),
|
||||
|
||||
+102
@@ -2,6 +2,7 @@ import secrets
|
||||
from typing import Any, Optional
|
||||
import os
|
||||
import re
|
||||
from io import StringIO
|
||||
from django.conf import settings
|
||||
|
||||
from django_tables2 import SingleTableMixin
|
||||
@@ -48,6 +49,7 @@ from django.http import Http404, JsonResponse
|
||||
|
||||
from django.http import HttpResponseRedirect, HttpResponse
|
||||
from django.views.decorators.http import require_POST
|
||||
from django.core.management import call_command, CommandError
|
||||
|
||||
from physics.models import UserAnswer as PhysicsUserAnswer
|
||||
from physics.models import Exam as PhysicsExam
|
||||
@@ -107,6 +109,23 @@ import psutil
|
||||
|
||||
|
||||
|
||||
def _directory_size_bytes(path: str) -> int:
|
||||
"""Return recursive directory size in bytes, skipping unreadable files."""
|
||||
if not path or not os.path.exists(path):
|
||||
return 0
|
||||
|
||||
total = 0
|
||||
for root, _, files in os.walk(path):
|
||||
for filename in files:
|
||||
file_path = os.path.join(root, filename)
|
||||
try:
|
||||
total += os.path.getsize(file_path)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
return total
|
||||
|
||||
|
||||
def feedback_checker(user):
|
||||
return user.groups.filter(name="feedback_checker").exists()
|
||||
|
||||
@@ -409,6 +428,89 @@ def logs_view(request):
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
@user_is_cid_user_manager
|
||||
def media_cleanup(request):
|
||||
"""Run django-unused-media cleanup from a manager-facing maintenance page.
|
||||
|
||||
Scope: cid_user_manager group members and superusers.
|
||||
Functionality: supports safe preview (dry-run) and confirmed deletion runs,
|
||||
returning HTMX fragments for inline results.
|
||||
"""
|
||||
context: dict[str, Any] = {
|
||||
"ran": False,
|
||||
"action": "preview",
|
||||
"minimum_file_age": 60,
|
||||
"remove_empty_dirs": True,
|
||||
"exclude": "",
|
||||
}
|
||||
|
||||
if request.method == "POST":
|
||||
action = request.POST.get("action", "preview")
|
||||
dry_run = action != "cleanup"
|
||||
exclude_raw = (request.POST.get("exclude") or "").strip()
|
||||
exclude = [line.strip() for line in exclude_raw.splitlines() if line.strip()]
|
||||
remove_empty_dirs = request.POST.get("remove_empty_dirs") == "on"
|
||||
|
||||
min_age_raw = (request.POST.get("minimum_file_age") or "60").strip()
|
||||
try:
|
||||
minimum_file_age = max(0, int(min_age_raw))
|
||||
except (TypeError, ValueError):
|
||||
minimum_file_age = 0
|
||||
|
||||
out = StringIO()
|
||||
success = True
|
||||
|
||||
command_kwargs: dict[str, Any] = {
|
||||
"dry_run": dry_run,
|
||||
"no_input": True,
|
||||
"interactive": False,
|
||||
"minimum_file_age": minimum_file_age,
|
||||
"remove_empty_dirs": remove_empty_dirs,
|
||||
"verbosity": 2,
|
||||
"stdout": out,
|
||||
}
|
||||
if exclude:
|
||||
command_kwargs["exclude"] = exclude
|
||||
|
||||
error_message = ""
|
||||
media_size_before = _directory_size_bytes(settings.MEDIA_ROOT)
|
||||
media_size_after = media_size_before
|
||||
try:
|
||||
call_command("cleanup_unused_media", **command_kwargs)
|
||||
media_size_after = _directory_size_bytes(settings.MEDIA_ROOT)
|
||||
except CommandError as exc:
|
||||
success = False
|
||||
error_message = str(exc)
|
||||
except Exception as exc: # pragma: no cover
|
||||
success = False
|
||||
error_message = str(exc)
|
||||
|
||||
output = out.getvalue().strip()
|
||||
recovered_size = max(0, media_size_before - media_size_after)
|
||||
context.update(
|
||||
{
|
||||
"ran": True,
|
||||
"success": success,
|
||||
"action": action,
|
||||
"dry_run": dry_run,
|
||||
"minimum_file_age": minimum_file_age,
|
||||
"remove_empty_dirs": remove_empty_dirs,
|
||||
"exclude": exclude_raw,
|
||||
"output": output,
|
||||
"error_message": error_message,
|
||||
"media_size_before": media_size_before,
|
||||
"media_size_after": media_size_after,
|
||||
"recovered_size": recovered_size,
|
||||
}
|
||||
)
|
||||
|
||||
if request.htmx:
|
||||
return render(request, "rad/partials/_media_cleanup_result.html", context)
|
||||
|
||||
return render(request, "rad/media_cleanup.html", context)
|
||||
|
||||
|
||||
def cid_selector(request):
|
||||
return render(
|
||||
request,
|
||||
|
||||
+22
-3
@@ -257,9 +257,28 @@
|
||||
</li>
|
||||
{% endif %}
|
||||
|
||||
{% if request.user.is_staff %}
|
||||
<li class="nav-item">
|
||||
<a class="nav-link active" href="{% url 'admin:index' %}">Admin</a>
|
||||
{% if request.user.is_staff or request.user|has_group:"cid_user_manager" %}
|
||||
<li class="nav-item dropdown">
|
||||
<a class="nav-link dropdown-toggle active" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">
|
||||
Admin
|
||||
</a>
|
||||
<ul class="dropdown-menu dropdown-menu-end">
|
||||
{% if request.user.is_staff %}
|
||||
<li>
|
||||
<a class="dropdown-item" href="{% url 'admin:index' %}">Django admin</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
{% if request.user.is_superuser %}
|
||||
<li>
|
||||
<a class="dropdown-item" href="{% url 'logs_view' %}">Logs</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
{% if request.user.is_superuser or request.user|has_group:"cid_user_manager" %}
|
||||
<li>
|
||||
<a class="dropdown-item" href="{% url 'media_cleanup' %}">Unused media cleanup</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
</ul>
|
||||
</li>
|
||||
{% endif %}
|
||||
{% if request.user.is_authenticated %}
|
||||
|
||||
@@ -151,6 +151,7 @@
|
||||
<li>Manage <a href="{% url 'trainees' %}">trainees</a> · <a
|
||||
href="{% url 'accounts_list' %}">users</a> · <a
|
||||
href="{% url 'generic:manage_cids' %}">candidates</a></li>
|
||||
<li><a href="{% url 'media_cleanup' %}" class="link-primary">Unused media cleanup</a></li>
|
||||
{% endif %}
|
||||
{% if request.user.is_superuser %}
|
||||
<li><a href="{% url 'django_psutil_dash:dashboard' %}" class="link-primary">psutil</a></li>
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
{% extends 'base.html' %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container-fluid py-4">
|
||||
<div class="d-flex flex-wrap justify-content-between align-items-start gap-3 mb-4">
|
||||
<div>
|
||||
<h2 class="mb-1">Unused Media Cleanup</h2>
|
||||
<p class="text-muted mb-0">
|
||||
Run django-unused-media from the web UI to preview or remove orphaned files.
|
||||
</p>
|
||||
</div>
|
||||
<a href="{% url 'logs_view' %}" class="btn btn-outline-secondary btn-sm">
|
||||
<i class="bi bi-journal-text me-1"></i> View logs
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="card border-0 shadow-sm mb-4">
|
||||
<div class="card-body">
|
||||
<form
|
||||
method="post"
|
||||
hx-post="{% url 'media_cleanup' %}"
|
||||
hx-target="#media-cleanup-result"
|
||||
hx-swap="innerHTML"
|
||||
class="row g-3"
|
||||
>
|
||||
{% csrf_token %}
|
||||
|
||||
<div class="col-12 col-lg-4">
|
||||
<label for="id_minimum_file_age" class="form-label">Minimum file age (seconds)</label>
|
||||
<input
|
||||
id="id_minimum_file_age"
|
||||
type="number"
|
||||
min="0"
|
||||
step="1"
|
||||
name="minimum_file_age"
|
||||
class="form-control"
|
||||
value="{{ minimum_file_age|default:60 }}"
|
||||
>
|
||||
<div class="form-text">Skip very recent files to avoid race conditions with active uploads.</div>
|
||||
</div>
|
||||
|
||||
<div class="col-12 col-lg-8">
|
||||
<label for="id_exclude" class="form-label">Exclude masks (one per line)</label>
|
||||
<textarea
|
||||
id="id_exclude"
|
||||
name="exclude"
|
||||
rows="4"
|
||||
class="form-control"
|
||||
placeholder="example: thumbnails/*"
|
||||
>{{ exclude }}</textarea>
|
||||
<div class="form-text">Uses django-unused-media wildcard matching.</div>
|
||||
</div>
|
||||
|
||||
<div class="col-12">
|
||||
<div class="form-check">
|
||||
<input
|
||||
id="id_remove_empty_dirs"
|
||||
type="checkbox"
|
||||
class="form-check-input"
|
||||
name="remove_empty_dirs"
|
||||
{% if remove_empty_dirs %}checked{% endif %}
|
||||
>
|
||||
<label for="id_remove_empty_dirs" class="form-check-label">
|
||||
Remove empty directories after cleanup
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-12 d-flex flex-wrap gap-2">
|
||||
<button
|
||||
type="submit"
|
||||
name="action"
|
||||
value="preview"
|
||||
class="btn btn-primary"
|
||||
>
|
||||
<i class="bi bi-search me-1"></i> Preview unused files
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
name="action"
|
||||
value="cleanup"
|
||||
class="btn btn-danger"
|
||||
hx-confirm="Delete unused media files now? This cannot be undone."
|
||||
>
|
||||
<i class="bi bi-trash me-1"></i> Delete unused files
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="media-cleanup-result">
|
||||
{% include 'rad/partials/_media_cleanup_result.html' %}
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,60 @@
|
||||
{% if ran %}
|
||||
<div class="card border-0 shadow-sm">
|
||||
<div class="card-body">
|
||||
<div class="d-flex flex-wrap align-items-center gap-2 mb-2">
|
||||
{% if success %}
|
||||
<span class="badge bg-success">Completed</span>
|
||||
{% else %}
|
||||
<span class="badge bg-danger">Failed</span>
|
||||
{% endif %}
|
||||
|
||||
{% if dry_run %}
|
||||
<span class="badge bg-info text-dark">Dry run</span>
|
||||
{% else %}
|
||||
<span class="badge bg-warning text-dark">Live cleanup</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="small text-muted mb-3">
|
||||
minimum_file_age={{ minimum_file_age }}
|
||||
{% if remove_empty_dirs %} | remove_empty_dirs=on{% else %} | remove_empty_dirs=off{% endif %}
|
||||
{% if exclude %} | exclude masks provided{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="row g-2 mb-3">
|
||||
<div class="col-12 col-md-4">
|
||||
<div class="border rounded p-2 small">
|
||||
<div class="text-muted">Media size before</div>
|
||||
<div><strong>{{ media_size_before|filesizeformat }}</strong></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12 col-md-4">
|
||||
<div class="border rounded p-2 small">
|
||||
<div class="text-muted">Media size after</div>
|
||||
<div><strong>{{ media_size_after|filesizeformat }}</strong></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12 col-md-4">
|
||||
<div class="border rounded p-2 small">
|
||||
<div class="text-muted">Recovered space</div>
|
||||
<div><strong>{{ recovered_size|filesizeformat }}</strong></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if error_message %}
|
||||
<div class="alert alert-danger py-2">{{ error_message }}</div>
|
||||
{% endif %}
|
||||
|
||||
{% if output %}
|
||||
<pre class="mb-0 p-3 border rounded small" style="max-height: 28rem; overflow-y: auto;">{{ output }}</pre>
|
||||
{% else %}
|
||||
<div class="alert alert-secondary mb-0 py-2">No output was returned by the command.</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="alert alert-secondary mb-0">
|
||||
Use <strong>Preview unused files</strong> first, then run deletion when you are happy with the output.
|
||||
</div>
|
||||
{% endif %}
|
||||
Reference in New Issue
Block a user