From 359682a0cdfe3bd0a2cff4787e1525ef7f656be8 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 10:36:58 +0000 Subject: [PATCH 01/13] start prod docker migration --- .env.prod.example | 20 ++ deploy/nginx/prod.conf | 128 ++++++++++ deploy/settings_local.py.example | 60 +++++ docker/docker-compose.prod.yml | 72 ++++++ docker/docker-compose.test.yml | 26 ++ entrypoint.sh | 25 ++ rad/Dockerfile.prod | 44 ++++ temp1/default | 402 +++++++++++++++++++++++++++++++ temp1/settings_local.py | 54 +++++ 9 files changed, 831 insertions(+) create mode 100644 .env.prod.example create mode 100644 deploy/nginx/prod.conf create mode 100644 deploy/settings_local.py.example create mode 100644 docker/docker-compose.prod.yml create mode 100644 docker/docker-compose.test.yml create mode 100644 entrypoint.sh create mode 100644 rad/Dockerfile.prod create mode 100644 temp1/default create mode 100644 temp1/settings_local.py diff --git a/.env.prod.example b/.env.prod.example new file mode 100644 index 00000000..ee1da750 --- /dev/null +++ b/.env.prod.example @@ -0,0 +1,20 @@ +# Copy this to .env.prod on the server and fill in production values (do NOT commit secrets). + +# Django +SECRET_KEY=replace-me-with-a-secure-random-string +DEBUG=0 +ALLOWED_HOSTS=example.com + +# External DB (you mentioned production uses an external DB) +DATABASE_HOST=your.db.host +DATABASE_PORT=5432 +DATABASE_NAME=rad +DATABASE_USER=django +DATABASE_PASSWORD=strong-db-password + +# Redis +REDIS_URL=redis://redis:6379/0 + +# Gunicorn tuning +GUNICORN_WORKERS=3 +GUNICORN_LOGLEVEL=info diff --git a/deploy/nginx/prod.conf b/deploy/nginx/prod.conf new file mode 100644 index 00000000..a4f0de41 --- /dev/null +++ b/deploy/nginx/prod.conf @@ -0,0 +1,128 @@ +upstream app_server { + # In production we proxy to the 'web' service (gunicorn) listening on 8000 + server web:8000; +} + +server { + listen 80; + listen [::]:80; + server_name _; + + client_max_body_size 100M; + + # Serve static files from the static volume + location /static/ { + alias /usr/src/app/static/; + expires 30d; + add_header Cache-Control "public, max-age=2592000"; + } + + # Serve media files from the media volume. CORS handling preserved. + set $cors ""; + if ($http_origin ~* (https?://(localhost:5173|viewer\.penracourses\.org\.uk))) { + set $cors $http_origin; + } + + location /media { + if ($request_method = OPTIONS ) { + add_header 'Access-Control-Allow-Origin' $cors always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Headers' "Origin, X-Requested-With, Content-Type, Accept" always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 200; + } + add_header 'Access-Control-Allow-Origin' $cors always; + alias /usr/src/app/media; + } + + # Viewer / OHIF / other static frontends - mount these into the nginx container at deploy time + location /viewer { + alias /srv/www/viewer; + index index.html; + try_files $uri $uri/ /viewer/index.html; + add_header Cache-Control "no-store, no-cache, must-revalidate"; + add_header 'Cross-Origin-Resource-Policy' 'cross-origin'; + } + + location /ohif { + add_header 'Cross-Origin-Opener-Policy' 'same-origin' always; + add_header 'Cross-Origin-Embedder-Policy' 'require-corp' always; + add_header 'Cross-Origin-Resource-Policy' 'same-site' always; + alias /srv/www/ohif; + try_files $uri /ohif/index.html; + } + + # rts and other host-mounted folders + location /rts { + alias /srv/www/rts; + } + + location /rota { + alias /srv/proc/rota; + autoindex on; + } + + location /uploader { + alias /srv/uploader; + autoindex on; + } + + # Reporter proxy - if you run a reporter service, set it up on the 'reporter' network name + location ~ ^/reporter/(.*)$ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Authorization $http_authorization; + proxy_pass_header Authorization; + + proxy_pass http://reporter:5129/$1?$args; + proxy_set_header X-Forwarded-Prefix /reporter; + proxy_redirect http://reporter:5129/ /reporter/; + } + + # Default proxy to the Django/gunicorn upstream + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + proxy_redirect off; + proxy_buffering off; + add_header 'Cross-Origin-Embedder-Policy' 'require-corp' always; + add_header 'Cross-Origin-Resource-Policy' 'same-site' always; + + proxy_pass http://app_server; + } + + # Default dev / health server block; keep 443 managed outside of docker unless you mount certs + listen 80; +} + +# A minimal HTTPS server block that expects certs to be mounted at /etc/letsencrypt +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name www.penracourses.org.uk penracourses.org.uk viewer.penracourses.org.uk; + + ssl_certificate /etc/letsencrypt/live/www.penracourses.org.uk/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/www.penracourses.org.uk/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + client_max_body_size 4G; + + # Same static/media handling as above + location /static/ { alias /usr/src/app/static/; } + location /media { alias /usr/src/app/media; } + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://app_server; + } +} diff --git a/deploy/settings_local.py.example b/deploy/settings_local.py.example new file mode 100644 index 00000000..2d4204a0 --- /dev/null +++ b/deploy/settings_local.py.example @@ -0,0 +1,60 @@ +""" +Example `settings_local.py` derived from the production server file. +This file is a template only — DO NOT commit real secrets. Copy it to +`rad/deploy/settings_local.py` on your server (or provide your own) and +ensure the real file is excluded from git. +""" + +DEBUG = False + +INTERNAL_IPS = ["82.69.88.125", "217.155.198.96"] + +# Paths inside the nginx container / host-mounted deploy directories. +# On the server, these paths are mounted into nginx as described in the +# deployment README. Adjust as needed. +STATIC_ROOT = '/usr/src/app/static/' + +EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" + +if not DEBUG: + LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", + 'datefmt': "%d/%b/%Y %H:%M:%S", + }, + 'simple': { + 'format': '%(levelname)s %(message)s', + }, + }, + 'handlers': { + 'file': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': 'log.txt', + 'formatter': 'verbose', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['file'], + 'propagate': True, + 'level': 'DEBUG', + }, + 'atlas': { + 'handlers': ['file'], + 'level': 'DEBUG', + }, + } + } + +# External service credentials (placeholders) +CIMAR_USERNAME = "your.username@example.com" +CIMAR_PASSWORD = "" + +# Celery settings — when running in docker the redis service is +# reachable at the `redis` hostname provided by docker compose. +CELERY_BROKER_URL = "redis://redis:6379" +CELERY_RESULT_BACKEND = "redis://redis:6379" diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 00000000..d715c98d --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,72 @@ +version: "3.8" + +services: + redis: + image: redis:7.4.2-alpine + restart: always + volumes: + - redis_data:/data + healthcheck: + test: ["CMD","redis-cli","ping"] + interval: 10s + timeout: 5s + retries: 5 + + web: + build: + # build context is the repository root (one level up from this file) + context: .. + dockerfile: rad/Dockerfile.prod + restart: always + env_file: + - ../.env.prod + environment: + - DJANGO_SETTINGS_MODULE=rad.settings.production + depends_on: + redis: + condition: service_healthy + volumes: + - static_volume:/usr/src/app/static + - media_volume:/usr/src/app/media + # Mount the server-specific local settings if present (DO NOT commit secrets). + # On the server create: rad/deploy/settings_local.py and it will be mounted into the container. + - ../deploy/settings_local.py:/usr/src/app/rad/settings_local.py:ro + expose: + - "8000" + command: ["/usr/src/app/entrypoint.sh"] + + nginx: + image: nginx:stable-alpine + restart: always + ports: + - "80:80" + - "443:443" + depends_on: + - web + volumes: + - ../deploy/nginx/prod.conf:/etc/nginx/conf.d/default.conf:ro + - static_volume:/usr/src/app/static:ro + - media_volume:/usr/src/app/media:ro + - ../deploy/nginx/certs:/etc/letsencrypt:ro + # Host-provided front-end bundles and static folders. Populate these under rad/deploy on the server + - ../deploy/www/viewer:/srv/www/viewer:ro + - ../deploy/www/ohif:/srv/www/ohif:ro + - ../deploy/www/rts:/srv/www/rts:ro + - ../deploy/nicereporter:/srv/nicereporter:rw + - ../deploy/proc-rota:/srv/proc/rota:ro + - ../deploy/uploader:/srv/uploader:ro + healthcheck: + test: ["CMD-SHELL","nginx -t" ] + interval: 30s + timeout: 10s + retries: 3 + +volumes: + postgres_data: + redis_data: + static_volume: + media_volume: + +networks: + default: + driver: bridge diff --git a/docker/docker-compose.test.yml b/docker/docker-compose.test.yml new file mode 100644 index 00000000..11ba476e --- /dev/null +++ b/docker/docker-compose.test.yml @@ -0,0 +1,26 @@ +version: "3.8" + +# Local test compose overlay — adds a Postgres service so you can run the +# full stack locally without an external DB. Use it together with +# docker-compose.prod.yml: +# +# docker compose -f rad/docker/docker-compose.prod.yml -f rad/docker/docker-compose.test.yml up -d --build + +services: + db: + image: postgres:14-alpine + restart: always + environment: + POSTGRES_USER: ${POSTGRES_USER:-django} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-rad} + volumes: + - postgres_test_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL","pg_isready -U ${POSTGRES_USER:-django}" ] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + postgres_test_data: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 00000000..07db988d --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -e + +# Lightweight wait-for-postgres using nc (expects DATABASE_HOST and DATABASE_PORT env vars) +if [ -n "${DATABASE_HOST}" ]; then + host="$DATABASE_HOST" + port="${DATABASE_PORT:-5432}" + echo "Waiting for database at ${host}:${port}..." + while ! nc -z "$host" "$port"; do + sleep 1 + done +fi + +echo "Applying database migrations..." +python manage.py migrate --noinput + +echo "Collecting static files..." +python manage.py collectstatic --noinput + +echo "Starting gunicorn..." +exec gunicorn rad.wsgi:application \ + --name rad_gunicorn \ + --bind 0.0.0.0:8000 \ + --workers ${GUNICORN_WORKERS:-3} \ + --log-level ${GUNICORN_LOGLEVEL:-info} diff --git a/rad/Dockerfile.prod b/rad/Dockerfile.prod new file mode 100644 index 00000000..1630c1ba --- /dev/null +++ b/rad/Dockerfile.prod @@ -0,0 +1,44 @@ +FROM python:3.14-slim + +# Copy uv helper binary from the uv image so we can manage venvs and pip +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# Install build deps required for common Python packages and netcat for health wait +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + gcc \ + git \ + libpq-dev \ + pkg-config \ + libcairo2-dev \ + libgdk-pixbuf2.0-0 \ + libgdk-pixbuf2.0-dev \ + libjpeg-dev \ + zlib1g-dev \ + wget \ + ca-certificates \ + netcat \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src/app + +# Install python deps into a uv-managed venv for reproducible builds +COPY requirements.txt /usr/src/app/ +RUN uv venv /opt/venv \ + && export VIRTUAL_ENV=/opt/venv \ + && export PATH="/opt/venv/bin:$PATH" \ + && uv pip install --upgrade pip setuptools wheel \ + && uv pip install --no-cache-dir -r requirements.txt + +# Copy project files into image +COPY . /usr/src/app + +# Create non-root user +RUN useradd -m appuser && chown -R appuser /usr/src/app /opt/venv +USER appuser + +ENTRYPOINT ["/usr/src/app/entrypoint.sh"] diff --git a/temp1/default b/temp1/default new file mode 100644 index 00000000..908c9e88 --- /dev/null +++ b/temp1/default @@ -0,0 +1,402 @@ +upstream app_server { + server unix:/run/gunicorn.sock fail_timeout=0; +} + +server { + #listen 80; + + root /home/ross/web/viewer; + #index index.html index.htm; + + + server_name viewer.penracourses.org.uk; + + location / { + + + #if ($request_method = 'OPTIONS') { + # add_header 'Access-Control-Allow-Origin' * always; + # add_header 'Access-Control-Allow-Credentials' 'true' always; + # add_header 'Access-Control-Allow-Headers' "Origin, X-Requested-With, Content-Type, Accept" always; + # add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + # add_header 'Access-Control-Max-Age' 1728000; + # add_header 'Content-Type' 'text/plain; charset=utf-8'; + # add_header 'Content-Length' 0; + # return 204; + #} + + #alias /home/ross/web/viewer; + + #add_header Cross-Origin-Opener-Policy same-origin; + #add_header Cross-Origin-Embedder-Policy require-corp; + + #add_header 'Cross-Origin-Opener-Policy' 'cross-origin' always; + #add_header 'Cross-Origin-Embedder-Policy' 'credentialless' always; + + index index.html; + + try_files $uri $uri/ /index.html; + add_header Cache-Control "no-store, no-cache, must-revalidate"; + add_header 'Cross-Origin-Resource-Policy' 'cross-origin'; + } + + + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/viewer.penracourses.org.uk/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/viewer.penracourses.org.uk/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + + +server { + listen 80; + server_name penracourses.org.uk; + + return 301 https://www.penracourses.org.uk$request_uri; +} + +server { + listen [::]:443 ssl; # managed by Certbot + server_name penracourses.org.uk; + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/penracourses.org.uk/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/penracourses.org.uk/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + + return 301 https://www.penracourses.org.uk$request_uri; +} + + +server { + #listen 80; + + root /usr/share/nginx/html; + index index.html index.htm; + + client_max_body_size 4G; + #server_name _; + server_name www.penracourses.org.uk; + resolver 127.0.0.11; + + keepalive_timeout 20; + + set $cors ""; +if ($http_origin ~* (https?://(localhost:5173|viewer\.penracourses\.org\.uk))) { + set $cors $http_origin; +} + + + # Your Django project's media files - amend as required + location /media { + if ($request_method = OPTIONS ) { + add_header 'Access-Control-Allow-Origin' $cors always; + #add_header 'Access-Control-Allow-Origin' 'https://viewer.penracourses.org.uk' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Headers' "Origin, X-Requested-With, Content-Type, Accept" always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 200; + } + #add_header 'Access-Control-Allow-Origin' 'https://viewer.penracourses.org.uk' always; + add_header 'Access-Control-Allow-Origin' $cors always; + + + alias /home/ross/web/rad/media; + #add_header Access-Control-Allow-Origin *; + #add_header Access-Control-Allow-Origin http://localhost:8000; + #add_header Vary Origin; + } + + # your Django project's static files - amend as required + location /static { + alias /home/ross/web/static; + } + + # rts + location /rts { + alias /home/ross/web/rts; + } + # OHIF + location /viewer { + #add_header 'Cross-Origin-Opener-Policy' 'cross-origin' always; + #add_header 'Cross-Origin-Embedder-Policy' 'credentialless' always; + #add_header 'Cross-Origin-Resource-Policy' 'cross-origin' always; + alias /home/ross/web/viewer; + } + location /ohif { + add_header 'Cross-Origin-Opener-Policy' 'same-origin' always; + add_header 'Cross-Origin-Embedder-Policy' 'require-corp' always; + add_header 'Cross-Origin-Resource-Policy' 'same-site' always; + alias /home/ross/web/ohif; + try_files $uri /ohif/index.html; + #autoindex on; + } + + + # # Proxy the static assests for the Django Admin panel + # location /static/admin { + # alias /usr/lib/python3/dist-packages/django/contrib/admin/static/admin/; + # } + location /rota { + alias /home/ross/proc/proc-rota; + autoindex on; + } + + location /uploader { + alias /home/ross/uploader; + autoindex on; + } + + location ~ ^/reporter/(.*)$ { + alias /home/ross/web/nicereporter; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Authorization $http_authorization; + proxy_pass_header Authorization; + + proxy_pass http://127.0.0.1:5129/$1?$args; + proxy_set_header X-Forwarded-Prefix /reporter; + proxy_redirect http://127.0.0.1:5129/ /reporter/; + } + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + proxy_redirect off; + proxy_buffering off; + #add_header 'Cross-Origin-Opener-Policy' 'single-origin' always; + #add_header 'Cross-Origin-Embedder-Policy' 'require-corp' always; + #add_header 'Cross-Origin-Resource-Policy' 'same-site' always; + #add_header 'Cross-Origin-Opener-Policy' 'same-origin' always; + add_header 'Cross-Origin-Embedder-Policy' 'require-corp' always; + add_header 'Cross-Origin-Resource-Policy' 'same-site' always; + #add_header 'Access-Control-Allow-Origin' $cors always; + + proxy_pass http://app_server; + #add_header Access-Control-Allow-Origin *; + #add_header Vary Origin; + } + + #location ~* \.(jpg|jpeg|png|dicom)$ { + # add_header Access-Control-Allow-Origin *; + #} + # rota + + + listen [::]:443 ssl; # managed by Certbot + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/penracourses.org.uk/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/penracourses.org.uk/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + +#server { +#server_name 161.35.163.87; +#add_header X-Frame-Options "SAMEORIGIN"; +# +#return 301 $scheme://penracourses.org.uk$request_uri; +#} + + +#server { +# #listen 80; +# #server_name penracoureses.org.uk www.penracourses.org.uk; +# +# root /usr/share/nginx/html; +# index index.html index.htm; +# +# client_max_body_size 4G; +# #server_name _; +# server_name penracourses.org.uk; # managed by Certbot +# +# keepalive_timeout 5; +# +# # Your Django project's media files - amend as required +# location /media { +# alias /home/ross/web/rad/media; +# #add_header Access-Control-Allow-Origin *; +# #add_header Access-Control-Allow-Origin http://localhost:8000; +# #add_header Vary Origin; +# } +# +# # your Django project's static files - amend as required +# location /static { +# alias /home/ross/web/rad/static; +# } +# +# # rts +# location /rts { +# alias /home/ross/web/rts; +# } +# +# # # Proxy the static assests for the Django Admin panel +# # location /static/admin { +# # alias /usr/lib/python3/dist-packages/django/contrib/admin/static/admin/; +# # } +# +# location / { +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header Host $host; +# proxy_redirect off; +# proxy_buffering off; +# +# proxy_pass http://app_server; +# } +# +# #location ~* \.(jpg|jpeg|png|dicom)$ { +# # add_header Access-Control-Allow-Origin *; +# #} +# +# # rota +# location /rota { +# alias /home/ross/proc-rota; +# } +# +# location /rota2 { +# alias /home/ross/neos; +# } +# +# +# +# +# +# listen [::]:443 ssl ipv6only=on; # managed by Certbot +# listen 443 ssl; # managed by Certbot +# ssl_certificate /etc/letsencrypt/live/penracourses.org.uk/fullchain.pem; # managed by Certbot +# ssl_certificate_key /etc/letsencrypt/live/penracourses.org.uk/privkey.pem; # managed by Certbot +# include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot +# ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot +# +#} + + + +server { + root /home/ross/web/rts; + index index.html index.htm; + + client_max_body_size 4G; + #server_name _; + server_name dev.penracourses.org.uk www.dev.penracourses.org.uk; + + keepalive_timeout 5; + + # Your Django project's media files - amend as required + location /media { + # Simple requests + if ($request_method ~* "(GET|POST)") { + add_header "Access-Control-Allow-Origin" *; + } + + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + return 200; + } + alias /home/ross/web/rad/media; + #add_header Access-Control-Allow-Origin *; + #add_header Access-Control-Allow-Origin http://localhost:8000; + #add_header Vary Origin; + } + + # your Django project's static files - amend as required + location /static { + alias /home/ross/web/rad/static; + } + + # rts + location /rts { + alias /home/ross/web/rts; + } + + # # Proxy the static assests for the Django Admin panel + # location /static/admin { + # alias /usr/lib/python3/dist-packages/django/contrib/admin/static/admin/; + # } + + location / { + add_header 'Access-Control-Allow-Origin' '*' always; + try_files $uri $uri/ =404; + #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + #proxy_set_header X-Forwarded-Proto $scheme; + #proxy_set_header Host $host; + #proxy_redirect off; + #proxy_buffering off; + + #proxy_pass http://app_server; + #add_header Access-Control-Allow-Origin *; + #add_header Vary Origin; + } + + #location ~* \.(jpg|jpeg|png|dicom)$ { + # add_header Access-Control-Allow-Origin *; + #} + + + + + #listen [::]:443 ssl; # managed by Certbot + #listen 443 ssl; # managed by Certbot + #ssl_certificate /etc/letsencrypt/live/penracourses.org.uk/fullchain.pem; # managed by Certbot + #ssl_certificate_key /etc/letsencrypt/live/penracourses.org.uk/privkey.pem; # managed by Certbot + #include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + #ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + + +server { + if ($host = penracourses.org.uk) { + return 301 https://www.$host$request_uri; + } # managed by Certbot + + + listen 80 ; + listen [::]:80 ; + server_name penracourses.org.uk; + return 404; # managed by Certbot + + +} + +server { + if ($host = www.penracourses.org.uk) { + return 301 https://$host$request_uri; + } # managed by Certbot + + + listen 80 default_server; + listen [::]:80 default_server ipv6only=on; + server_name www.penracourses.org.uk; + return 404; # managed by Certbot + + +} + + + +server { + if ($host = viewer.penracourses.org.uk) { + return 301 https://$host$request_uri; + } # managed by Certbot + + + + server_name viewer.penracourses.org.uk; + listen 80; + return 404; # managed by Certbot + + +} diff --git a/temp1/settings_local.py b/temp1/settings_local.py new file mode 100644 index 00000000..ef40fa06 --- /dev/null +++ b/temp1/settings_local.py @@ -0,0 +1,54 @@ +DEBUG = False +INTERNAL_IPS = ["82.69.88.125", "217.155.198.96"] +#SECURE_SSL_REDIRECT = False +#SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'http') +#REMOTE_URL = "http://46.101.13.46:8123" +# +#MEDIA_ROOT = '/home/django/rad/media/' +STATIC_ROOT = '/home/ross/web/static/' + +#EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" + +if not DEBUG: + LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format' : "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", + 'datefmt' : "%d/%b/%Y %H:%M:%S" + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, + 'handlers': { + 'file': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': 'log.txt', + 'formatter': 'verbose' + }, + }, + 'loggers': { + 'django': { + 'handlers':['file'], + 'propagate': True, + 'level':'DEBUG', + }, + 'atlas': { + 'handlers': ['file'], + 'level': 'DEBUG', + }, + #"django.core.mail": {"handlers": ["console"], "level": "DEBUG", "propagate": False}, + #"smtplib": {"handlers": ["console"], "level": "DEBUG", "propagate": False}, + } + } + +CIMAR_USERNAME = "ross.kruger@nhs.net" +CIMAR_PASSWORD = "[prdr32@cimar]" + +# Celery settings +CELERY_BROKER_URL = "redis://localhost:6379" +CELERY_RESULT_BACKEND = "redis://localhost:6379" From 2157b6f781a4f207ffb2148c8d164b07c61583f0 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 11:12:01 +0000 Subject: [PATCH 02/13] some fixes? --- .env.dev | 18 ++++++++++++++++++ docker/.env.dev | 23 ----------------------- docker/docker-compose.dev.yml | 12 ++++++++++++ docker/docker-compose.prod.yml | 6 +++--- rad.code-workspace | 15 +++++++++++++++ rad/Dockerfile.prod | 31 ++++++++++++++++--------------- scripts/local-down.sh | 7 +++++++ scripts/local-up.sh | 22 ++++++++++++++++++++++ 8 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 .env.dev delete mode 100644 docker/.env.dev create mode 100644 docker/docker-compose.dev.yml create mode 100644 rad.code-workspace create mode 100755 scripts/local-down.sh create mode 100755 scripts/local-up.sh diff --git a/.env.dev b/.env.dev new file mode 100644 index 00000000..edb695aa --- /dev/null +++ b/.env.dev @@ -0,0 +1,18 @@ +# Django +DEBUG=1 +SECRET_KEY=w(s0&(_eb058wvmg@44_repv8)r9@5p8fx*g_@c)1dm&d*ew^u +DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1] + +# External DB (you mentioned production uses an external DB) +DATABASE_HOST=db-postgresql-lon1-05515-jan-22-backup-do-user-8165014-0.c.db.ondigitalocean.com +DATABASE_PORT=25060 +DATABASE_NAME=testing +DATABASE_USER=testing +DATABASE_PASSWORD=AVNS_ykXO4RxAq0zeJTI-1Nl + +# Redis +REDIS_URL=redis://redis:6379/0 + +# Gunicorn tuning +GUNICORN_WORKERS=3 +GUNICORN_LOGLEVEL=info diff --git a/docker/.env.dev b/docker/.env.dev deleted file mode 100644 index 73f8fd7f..00000000 --- a/docker/.env.dev +++ /dev/null @@ -1,23 +0,0 @@ -DEBUG=1 -SECRET_KEY=w(s0&(_eb058wvmg@44_repv8)r9@5p8fx*g_@c)1dm&d*ew^u -DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1] - -DB_NAME=rad -DB_HOST=db -DB_USER=django -DB_PASSWORD=postgres -DB_PORT=5432 - -MEMCACHE_HOST=cache - -# DEV-specific vars used by rad/settings_local.py -DEV_USE_POSTGRES=1 -DEV_DB_NAME=${DB_NAME} -DEV_DB_USER=${DB_USER} -DEV_DB_PASSWORD=${DB_PASSWORD} -DEV_DB_HOST=${DB_HOST} -DEV_DB_PORT=${DB_PORT} - -# pgAdmin creds -PGADMIN_DEFAULT_EMAIL=admin@example.com -PGADMIN_DEFAULT_PASSWORD=admin \ No newline at end of file diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 00000000..90d8cf54 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,12 @@ +# Development override: point the web service to rad/.env.dev and map nginx to +# non-privileged host ports so you can run without sudo. + +services: + web: + env_file: + - ../.env.dev + + nginx: + ports: + - "8080:80" + - "8443:443" diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index d715c98d..1dadf42c 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: redis: image: redis:7.4.2-alpine @@ -18,8 +16,10 @@ services: context: .. dockerfile: rad/Dockerfile.prod restart: always + # Use COMPOSE_ENV to select which env file to load (defaults to 'prod'). + # Example: COMPOSE_ENV=dev docker compose -f rad/docker/docker-compose.prod.yml -f rad/docker/docker-compose.dev.yml up -d env_file: - - ../.env.prod + - ../.env.${COMPOSE_ENV:-prod} environment: - DJANGO_SETTINGS_MODULE=rad.settings.production depends_on: diff --git a/rad.code-workspace b/rad.code-workspace new file mode 100644 index 00000000..24a0bd0b --- /dev/null +++ b/rad.code-workspace @@ -0,0 +1,15 @@ +{ + "folders": [ + { + "name": "rad", + "path": "." + } + ], + "settings": { + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true + } +} \ No newline at end of file diff --git a/rad/Dockerfile.prod b/rad/Dockerfile.prod index 1630c1ba..bd08e295 100644 --- a/rad/Dockerfile.prod +++ b/rad/Dockerfile.prod @@ -8,21 +8,22 @@ ENV PYTHONUNBUFFERED=1 # Install build deps required for common Python packages and netcat for health wait RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - gcc \ - git \ - libpq-dev \ - pkg-config \ - libcairo2-dev \ - libgdk-pixbuf2.0-0 \ - libgdk-pixbuf2.0-dev \ - libjpeg-dev \ - zlib1g-dev \ - wget \ - ca-certificates \ - netcat \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends \ + build-essential \ + gcc \ + git \ + libpq-dev \ + pkg-config \ + libcairo2-dev \ + # some distributions provide libgdk-pixbuf under the xlib name + libgdk-pixbuf-xlib-2.0-0 \ + libgdk-pixbuf-xlib-2.0-dev \ + libjpeg-dev \ + zlib1g-dev \ + wget \ + ca-certificates \ + netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* WORKDIR /usr/src/app diff --git a/scripts/local-down.sh b/scripts/local-down.sh new file mode 100755 index 00000000..c29fb5f8 --- /dev/null +++ b/scripts/local-down.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +docker compose -f docker/docker-compose.prod.yml -f docker/docker-compose.test.yml -f docker/docker-compose.dev.yml down --volumes --remove-orphans diff --git a/scripts/local-up.sh b/scripts/local-up.sh new file mode 100755 index 00000000..d471fde0 --- /dev/null +++ b/scripts/local-up.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -e + +# Simple local bring-up helper. Usage: +# ./rad/scripts/local-up.sh +# It requires rad/.env.dev to exist (do NOT auto-create from examples) and then +# runs the prod+dev compose stack using COMPOSE_ENV to select the env file. + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +if [ ! -f .env.dev ]; then + echo ".env.dev not found. Create .env.dev with your development values (pointing to the external dev DB) and re-run." + echo "You can copy .env.prod.example to start from a template, but DO NOT commit secrets." + exit 1 +fi + +# Default to 'dev' but allow overriding with COMPOSE_ENV environment variable +COMPOSE_ENV=${COMPOSE_ENV:-dev} + +echo "Starting compose with COMPOSE_ENV=$COMPOSE_ENV (using .env.$COMPOSE_ENV)" +COMPOSE_ENV=$COMPOSE_ENV docker compose -f docker/docker-compose.prod.yml -f docker/docker-compose.dev.yml up -d --build From f074e5b086472ed7707676db00b0d41895c4c73f Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 13:30:43 +0000 Subject: [PATCH 03/13] fix some ninja issuse --- anatomy/api.py | 31 +++++++++++++++---------------- atlas/api.py | 8 ++++---- entrypoint.sh | 0 3 files changed, 19 insertions(+), 20 deletions(-) mode change 100644 => 100755 entrypoint.sh diff --git a/anatomy/api.py b/anatomy/api.py index 251d802f..289c9a70 100644 --- a/anatomy/api.py +++ b/anatomy/api.py @@ -19,51 +19,50 @@ from django.contrib.auth import get_user_model router = Router() class CidUserSchema(ModelSchema): - class Config: + class Meta: model = CidUser - model_fields = ["cid", "name", "email"] + fields = ["cid", "name", "email"] class UserUserSchema(ModelSchema): - class Config: + class Meta: model = get_user_model() - model_fields = ["id", "email"] + fields = ["id", "email"] class CidUserExamSchema(ModelSchema): cid_user : CidUserSchema | None user_user : UserUserSchema | None - class Config: + class Meta: model = CidUserExam - model_fields = ["id"] + fields = ["id"] class ExamStatusSchema(ModelSchema): - class Config: + class Meta: model = ExamUserStatus - model_fields = ["id", "datetime", "status", "extra"] + fields = ["id", "datetime", "status", "extra"] class ExamUserStatusSchema(ModelSchema): #cid_user : CidUserSchema | None #user_user : UserUserSchema | None cid_user_exam : CidUserExamSchema | None - class Config: + class Meta: model = ExamUserStatus - model_fields = ["id", "datetime", "status", "extra"] + fields = ["id", "datetime", "status", "extra"] class QuestionSchema(ModelSchema): - class Config: + class Meta: model = Question - #model_fields = ["question", "history", "feedback", "normal", "laterality"] - model_fields = "__all__" + fields = "__all__" - #model_exclude = ["answers"] + #exclude = ["answers"] class ExamSchema(ModelSchema): - class Config: + class Meta: model = Exam - model_fields = ["id", "name", "active", "publish_results"] + fields = ["id", "name", "active", "publish_results"] @router.get('/') def list_questions(request): diff --git a/atlas/api.py b/atlas/api.py index 5dec2945..f2c8b5d3 100644 --- a/atlas/api.py +++ b/atlas/api.py @@ -32,9 +32,9 @@ router = Router() class SeriesSchema(ModelSchema): case_id: List[int] = [] - class Config: + class Meta: model = Series - model_fields = [ + fields = [ "id", "modality", "examination", @@ -48,9 +48,9 @@ class SeriesSchema(ModelSchema): class CaseSchema(ModelSchema): - class Config: + class Meta: model = Case - model_fields = ["id", "title"] + fields = ["id", "title"] @router.post("/upload_dicom", auth=django_auth) diff --git a/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 From fd6e25d55e245e901c5ba80bcc25153720f1f0d6 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 13:31:02 +0000 Subject: [PATCH 04/13] more ninja fixes --- longs/api.py | 13 +++++++------ physics/api.py | 12 ++++++------ rapids/api.py | 18 +++++++++--------- sbas/api.py | 11 +++++------ shorts/api.py | 12 ++++++------ 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/longs/api.py b/longs/api.py index 6c1bc513..75a5604b 100644 --- a/longs/api.py +++ b/longs/api.py @@ -14,19 +14,20 @@ router = Router() class QuestionSchema(ModelSchema): - class Config: + class Meta: model = Question - #model_fields = ["question", "history", "feedback", "normal", "laterality"] - model_fields = "__all__" + # Use `fields` per newer ninja ModelSchema API + #fields = ["question", "history", "feedback", "normal", "laterality"] + fields = "__all__" - #model_exclude = ["answers"] + #exclude = ["answers"] class ExamSchema(ModelSchema): - class Config: + class Meta: model = Exam - model_fields = ["id", "name", "active", "publish_results"] + fields = ["id", "name", "active", "publish_results"] @router.get('/') def list_questions(request): diff --git a/physics/api.py b/physics/api.py index f46306a1..abba04ea 100644 --- a/physics/api.py +++ b/physics/api.py @@ -15,19 +15,19 @@ router = Router() class QuestionSchema(ModelSchema): - class Config: + class Meta: model = Question - #model_fields = ["question", "history", "feedback", "normal", "laterality"] - model_fields = "__all__" + # Use `fields` per newer ninja ModelSchema API + fields = "__all__" - #model_exclude = ["answers"] + #exclude = ["answers"] class ExamSchema(ModelSchema): - class Config: + class Meta: model = Exam - model_fields = ["id", "name", "active", "publish_results"] + fields = ["id", "name", "active", "publish_results"] @router.get('/') def list_questions(request): diff --git a/rapids/api.py b/rapids/api.py index b0f4875c..2d78ddcd 100644 --- a/rapids/api.py +++ b/rapids/api.py @@ -15,19 +15,19 @@ router = Router() class RapidSchema(ModelSchema): - class Config: + # `ninja` newer versions require an inner `Meta` class instead of `Config` + # for ModelSchema. Replace `Config` with `Meta` to be compatible. + class Meta: model = Rapid - - #model_fields = ["question", "history", "feedback", "normal", "laterality"] - model_fields = "__all__" - - #model_exclude = ["answers"] + #model_fields (old API) -> use `fields` per newer ninja ModelSchema API + #fields can be a list of field names or the string "__all__". + #fields = ["question", "history", "feedback", "normal", "laterality"] + fields = "__all__" class ExamSchema(ModelSchema): - class Config: + class Meta: model = Exam - - model_fields = ["id", "name", "active", "publish_results"] + fields = ["id", "name", "active", "publish_results"] @router.get('/') def list_rapids(request): diff --git a/sbas/api.py b/sbas/api.py index 2da50baf..c2b3c568 100644 --- a/sbas/api.py +++ b/sbas/api.py @@ -15,19 +15,18 @@ router = Router() class QuestionSchema(ModelSchema): - class Config: + class Meta: model = Question - #model_fields = ["question", "history", "feedback", "normal", "laterality"] - model_fields = "__all__" + fields = "__all__" - #model_exclude = ["answers"] + #exclude = ["answers"] class ExamSchema(ModelSchema): - class Config: + class Meta: model = Exam - model_fields = ["id", "name", "active", "publish_results"] + fields = ["id", "name", "active", "publish_results"] @router.get('/') def list_questions(request): diff --git a/shorts/api.py b/shorts/api.py index 49736127..79b0fd56 100644 --- a/shorts/api.py +++ b/shorts/api.py @@ -15,19 +15,19 @@ router = Router() class QuestionSchema(ModelSchema): - class Config: + class Meta: model = Question - #model_fields = ["question", "history", "feedback", "normal", "laterality"] - model_fields = "__all__" + # Use `fields` per newer ninja ModelSchema API + fields = "__all__" - #model_exclude = ["answers"] + #exclude = ["answers"] class ExamSchema(ModelSchema): - class Config: + class Meta: model = Exam - model_fields = ["id", "name", "active", "publish_results"] + fields = ["id", "name", "active", "publish_results"] @router.get('/') def list_questions(request): From 9ed789ba90a0c11353c0f95690effb8bd86a820c Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 13:31:08 +0000 Subject: [PATCH 05/13] more docker updates --- .env.dev | 14 +++--- deploy/nginx/dev.conf | 30 +++++++++++++ deploy/nginx/prod.conf | 1 - deploy/settings_local.py | 78 ++++++++++++++++++++++++++++++++++ docker/.env.dev.local | 23 ++++++++++ docker/docker-compose.dev.yml | 10 +++-- docker/docker-compose.prod.yml | 6 +-- docker/docker-compose.yml | 2 +- entrypoint.sh | 0 rad/Dockerfile.prod | 38 ++++++++++++++--- scripts/local-up.sh | 24 ++++++++++- 11 files changed, 206 insertions(+), 20 deletions(-) create mode 100644 deploy/nginx/dev.conf create mode 100644 deploy/settings_local.py create mode 100644 docker/.env.dev.local mode change 100755 => 100644 entrypoint.sh diff --git a/.env.dev b/.env.dev index edb695aa..20e927f3 100644 --- a/.env.dev +++ b/.env.dev @@ -4,11 +4,11 @@ SECRET_KEY=w(s0&(_eb058wvmg@44_repv8)r9@5p8fx*g_@c)1dm&d*ew^u DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1] # External DB (you mentioned production uses an external DB) -DATABASE_HOST=db-postgresql-lon1-05515-jan-22-backup-do-user-8165014-0.c.db.ondigitalocean.com -DATABASE_PORT=25060 -DATABASE_NAME=testing -DATABASE_USER=testing -DATABASE_PASSWORD=AVNS_ykXO4RxAq0zeJTI-1Nl +DB_HOST=db-postgresql-lon1-05515-jan-22-backup-do-user-8165014-0.c.db.ondigitalocean.com +DB_PORT=25060 +DB_NAME=testing +DB_USER=django +DB_PASSWORD=AVNS_NG_s4i7SMMobWLO # Redis REDIS_URL=redis://redis:6379/0 @@ -16,3 +16,7 @@ REDIS_URL=redis://redis:6379/0 # Gunicorn tuning GUNICORN_WORKERS=3 GUNICORN_LOGLEVEL=info + +# Nginx host ports for local development (override prod defaults) +NGINX_HTTP_PORT=8000 +NGINX_HTTPS_PORT=8443 diff --git a/deploy/nginx/dev.conf b/deploy/nginx/dev.conf new file mode 100644 index 00000000..495c7e57 --- /dev/null +++ b/deploy/nginx/dev.conf @@ -0,0 +1,30 @@ +upstream app_server { + server web:8000; +} + +server { + listen 80; + listen [::]:80; + server_name _; + + client_max_body_size 100M; + + location /static/ { + alias /usr/src/app/static/; + expires 30d; + add_header Cache-Control "public, max-age=2592000"; + } + + location /media { + alias /usr/src/app/media; + } + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://app_server; + } +} diff --git a/deploy/nginx/prod.conf b/deploy/nginx/prod.conf index a4f0de41..d08ca5cc 100644 --- a/deploy/nginx/prod.conf +++ b/deploy/nginx/prod.conf @@ -97,7 +97,6 @@ server { } # Default dev / health server block; keep 443 managed outside of docker unless you mount certs - listen 80; } # A minimal HTTPS server block that expects certs to be mounted at /etc/letsencrypt diff --git a/deploy/settings_local.py b/deploy/settings_local.py new file mode 100644 index 00000000..e1dd6ec5 --- /dev/null +++ b/deploy/settings_local.py @@ -0,0 +1,78 @@ +""" +Example `settings_local.py` derived from the production server file. +This file is a template only — DO NOT commit real secrets. Copy it to +`rad/deploy/settings_local.py` on your server (or provide your own) and +ensure the real file is excluded from git. +""" +import os + +DEBUG = True + +INTERNAL_IPS = ["82.69.88.125", "217.155.198.96", "localhost"] + +# Paths inside the nginx container / host-mounted deploy directories. +# On the server, these paths are mounted into nginx as described in the +# deployment README. Adjust as needed. +STATIC_ROOT = '/usr/src/app/static/' + +EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" + +if not DEBUG: + LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", + 'datefmt': "%d/%b/%Y %H:%M:%S", + }, + 'simple': { + 'format': '%(levelname)s %(message)s', + }, + }, + 'handlers': { + 'file': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': 'log.txt', + 'formatter': 'verbose', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['file'], + 'propagate': True, + 'level': 'DEBUG', + }, + 'atlas': { + 'handlers': ['file'], + 'level': 'DEBUG', + }, + } + } + +# External service credentials (placeholders) +CIMAR_USERNAME = "your.username@example.com" +CIMAR_PASSWORD = "" + +# Celery settings — when running in docker the redis service is +# reachable at the `redis` hostname provided by docker compose. +CELERY_BROKER_URL = "redis://redis:6379" +CELERY_RESULT_BACKEND = "redis://redis:6379" + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": os.environ.get("DB_NAME", "django"), + "USER": os.environ.get("DB_USER", "django"), + "PASSWORD": os.environ.get("DB_PASSWORD", "AVNS_NG_s4i7SMMobWLO"), + #"HOST": os.environ.get("DB_HOST", "db-postgresql-lon1-05515-do-user-8165014-0.b.db.ondigitalocean.com"), + "HOST": os.environ.get("DB_HOST", "db-postgresql-lon1-05515-jan-22-backup-do-user-8165014-0.c.db.ondigitalocean.com"), + "PORT": os.environ.get("DB_PORT", "25060"), + #"NAME": os.environ.get("DB_NAME", "django"), + #"USER": os.environ.get("DB_USER", "django"), + #"PASSWORD": os.environ.get("DB_PASSWORD", "f7bf31dc9bda1256ea827953480d1917"), + #"HOST": os.environ.get("DB_HOST", "161.35.163.87"), + #"PORT": os.environ.get("DB_PORT", "5432"), + } +} \ No newline at end of file diff --git a/docker/.env.dev.local b/docker/.env.dev.local new file mode 100644 index 00000000..73f8fd7f --- /dev/null +++ b/docker/.env.dev.local @@ -0,0 +1,23 @@ +DEBUG=1 +SECRET_KEY=w(s0&(_eb058wvmg@44_repv8)r9@5p8fx*g_@c)1dm&d*ew^u +DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1] + +DB_NAME=rad +DB_HOST=db +DB_USER=django +DB_PASSWORD=postgres +DB_PORT=5432 + +MEMCACHE_HOST=cache + +# DEV-specific vars used by rad/settings_local.py +DEV_USE_POSTGRES=1 +DEV_DB_NAME=${DB_NAME} +DEV_DB_USER=${DB_USER} +DEV_DB_PASSWORD=${DB_PASSWORD} +DEV_DB_HOST=${DB_HOST} +DEV_DB_PORT=${DB_PORT} + +# pgAdmin creds +PGADMIN_DEFAULT_EMAIL=admin@example.com +PGADMIN_DEFAULT_PASSWORD=admin \ No newline at end of file diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 90d8cf54..9ae4f4c8 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -5,8 +5,10 @@ services: web: env_file: - ../.env.dev - nginx: - ports: - - "8080:80" - - "8443:443" + # Development nginx override: mount a simplified config that does not + # reference LetsEncrypt certs so the container can start without real + # certificates present on the host. + volumes: + - ../deploy/nginx/dev.conf:/etc/nginx/conf.d/default.conf:ro + diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 1dadf42c..da5b859d 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -21,7 +21,7 @@ services: env_file: - ../.env.${COMPOSE_ENV:-prod} environment: - - DJANGO_SETTINGS_MODULE=rad.settings.production + - DJANGO_SETTINGS_MODULE=rad.settings depends_on: redis: condition: service_healthy @@ -39,8 +39,8 @@ services: image: nginx:stable-alpine restart: always ports: - - "80:80" - - "443:443" + - "${NGINX_HTTP_PORT:-80}:80" + - "${NGINX_HTTPS_PORT:-443}:443" depends_on: - web volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ca9f6bb4..f73ef645 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -29,7 +29,7 @@ services: - 8000:8000 - 3459:3459 env_file: - - ./.env.dev + - ./.env.dev.local db: image: postgres:14.2-alpine volumes: diff --git a/entrypoint.sh b/entrypoint.sh old mode 100755 new mode 100644 diff --git a/rad/Dockerfile.prod b/rad/Dockerfile.prod index bd08e295..c25ef6b9 100644 --- a/rad/Dockerfile.prod +++ b/rad/Dockerfile.prod @@ -29,17 +29,45 @@ WORKDIR /usr/src/app # Install python deps into a uv-managed venv for reproducible builds COPY requirements.txt /usr/src/app/ +# Prebuild wheels into /wheels so compiled artifacts are cached in a layer. +# This speeds rebuilds when `requirements.txt` hasn't changed. The first build +# still needs to compile any packages without prebuilt wheels. +RUN mkdir -p /wheels \ + && python -m venv /tmp/venvbuild \ + && /tmp/venvbuild/bin/pip install --upgrade pip setuptools wheel \ + && /tmp/venvbuild/bin/pip wheel --wheel-dir=/wheels -r requirements.txt \ + && rm -rf /tmp/venvbuild + +# Create the final uv-managed venv and install from the cached wheels where +# possible. Using `--no-index --find-links=/wheels` makes pip prefer the +# prebuilt wheels, avoiding recompilation. RUN uv venv /opt/venv \ && export VIRTUAL_ENV=/opt/venv \ && export PATH="/opt/venv/bin:$PATH" \ && uv pip install --upgrade pip setuptools wheel \ - && uv pip install --no-cache-dir -r requirements.txt + && uv pip install --no-index --find-links=/wheels -r requirements.txt -# Copy project files into image -COPY . /usr/src/app +# Ensure the runtime environment uses the uv-managed venv by default. +# This persists the venv into image ENV so `python` and `gunicorn` point +# to the venv-installed binaries at container start. +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="/opt/venv/bin:$PATH" -# Create non-root user -RUN useradd -m appuser && chown -R appuser /usr/src/app /opt/venv +# Create non-root user before copying files so we can use Docker's +# `--chown` flag during COPY and avoid an expensive recursive `chown -R`. +RUN useradd -m appuser + +# Copy project files into image and set ownership during copy (faster than +# a separate `chown -R` step). This keeps layer reuse efficient. +COPY --chown=appuser:appuser . /usr/src/app + +# Ensure the venv is usable by the non-root user; chown only the venv path +# (much smaller than the whole project tree) to avoid a long recursive chown. +RUN chown -R appuser:appuser /opt/venv + +# Make the entrypoint executable (some files in the repo may not have the +# executable bit set). Do this as root before switching to the app user. +RUN [ -f /usr/src/app/entrypoint.sh ] && chmod +x /usr/src/app/entrypoint.sh || true USER appuser ENTRYPOINT ["/usr/src/app/entrypoint.sh"] diff --git a/scripts/local-up.sh b/scripts/local-up.sh index d471fde0..c8275443 100755 --- a/scripts/local-up.sh +++ b/scripts/local-up.sh @@ -19,4 +19,26 @@ fi COMPOSE_ENV=${COMPOSE_ENV:-dev} echo "Starting compose with COMPOSE_ENV=$COMPOSE_ENV (using .env.$COMPOSE_ENV)" -COMPOSE_ENV=$COMPOSE_ENV docker compose -f docker/docker-compose.prod.yml -f docker/docker-compose.dev.yml up -d --build +# Load variables from .env. into the environment so Compose variable +# substitution (e.g. ${NGINX_HTTP_PORT}) works. We export all variables from +# the file for the duration of the command. +ENV_FILE="$REPO_ROOT/.env.$COMPOSE_ENV" +if [ -f "$ENV_FILE" ]; then + # Export variables from the .env file safely without sourcing it. Some + # values include characters (parentheses, ampersands) that break POSIX + # shell parsing if the file is sourced directly. Read line-by-line, + # ignore comments/empty lines and export KEY=VALUE pairs. + while IFS= read -r _line || [ -n "$_line" ]; do + line="$_line" + case "$line" in + ''|\#*) continue ;; + esac + # Split on first '=' into key and value + key=${line%%=*} + val=${line#*=} + # Export directly (preserves special characters in the value) + export "$key=$val" + done < "$ENV_FILE" +fi + +COMPOSE_ENV=$COMPOSE_ENV docker compose -f docker/docker-compose.prod.yml -f docker/docker-compose.dev.yml up --build From e9a04e823c574b3a5f52b4c4612721df9fbbdcb7 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 14:01:46 +0000 Subject: [PATCH 06/13] more docker improvements --- .env.dev | 2 +- deploy/settings_local.py | 46 +++++++++++++++++++++++++++++++++-- docker/docker-compose.dev.yml | 10 ++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.env.dev b/.env.dev index 20e927f3..fe69fd87 100644 --- a/.env.dev +++ b/.env.dev @@ -19,4 +19,4 @@ GUNICORN_LOGLEVEL=info # Nginx host ports for local development (override prod defaults) NGINX_HTTP_PORT=8000 -NGINX_HTTPS_PORT=8443 +NGINX_HTTPS_PORT=8443 \ No newline at end of file diff --git a/deploy/settings_local.py b/deploy/settings_local.py index e1dd6ec5..03376c38 100644 --- a/deploy/settings_local.py +++ b/deploy/settings_local.py @@ -8,7 +8,35 @@ import os DEBUG = True -INTERNAL_IPS = ["82.69.88.125", "217.155.198.96", "localhost"] +INTERNAL_IPS = ["82.69.88.125", "217.155.198.96", "localhost", "127.0.0.1"] + +# When running in Docker (dev), requests will often come from an internal +# container/network address. Add any discovered host/container IPs (and a +# likely gateway address) to INTERNAL_IPS so the debug toolbar can render. +if DEBUG: + try: + import socket + + hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) + for ip in ips: + if ip not in INTERNAL_IPS: + INTERNAL_IPS.append(ip) + # common docker gateway pattern: replace last octet with '1' + parts = ip.split('.') + if len(parts) == 4: + parts[-1] = '1' + gw = '.'.join(parts) + if gw not in INTERNAL_IPS: + INTERNAL_IPS.append(gw) + except Exception: + # best-effort only; don't fail startup if this lookup doesn't work + pass + +# In local/dev enable the toolbar unconditionally to avoid IP/proxy +# headaches while developing inside Docker/behind nginx. +DEBUG_TOOLBAR_CONFIG = { + "SHOW_TOOLBAR_CALLBACK": lambda request: True, +} # Paths inside the nginx container / host-mounted deploy directories. # On the server, these paths are mounted into nginx as described in the @@ -75,4 +103,18 @@ DATABASES = { #"HOST": os.environ.get("DB_HOST", "161.35.163.87"), #"PORT": os.environ.get("DB_PORT", "5432"), } -} \ No newline at end of file +} + +# Development overrides to avoid forcing HTTPS and secure cookies in local dev +# (production should keep these True). +SECURE_SSL_REDIRECT = False +SESSION_COOKIE_SECURE = False +CSRF_COOKIE_SECURE = False +SECURE_HSTS_SECONDS = 0 + +# Allow requests from the local dev nginx/dev server and direct runserver +# (include scheme+host+port for Django's origin checking). +CSRF_TRUSTED_ORIGINS = [ + "http://127.0.0.1:8000", + "http://localhost:8000", +] \ No newline at end of file diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 9ae4f4c8..cc688c95 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -5,6 +5,16 @@ services: web: env_file: - ../.env.dev + # Development: run Django's autoreloading development server instead of + # the production entrypoint/gunicorn. Mount the repository into the + # container so code edits on the host trigger Django's autoreload. + entrypoint: ["python", "manage.py", "runserver", "0.0.0.0:8000"] + # Clear any command set by the prod compose file (which would be passed + # as an extra argument to manage.py). Setting an empty command prevents + # the image `command` from being appended to the entrypoint. + command: [] + volumes: + - ../:/usr/src/app:cached nginx: # Development nginx override: mount a simplified config that does not # reference LetsEncrypt certs so the container can start without real From 590e0b76ed3b5aac0c03a77424910c3df29108b3 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 14:50:11 +0000 Subject: [PATCH 07/13] Add series images endpoint and template for HTMX integration --- .../atlas/partials/series_images.html | 31 + atlas/templates/atlas/series_viewer.html | 640 +++++++++--------- atlas/urls.py | 1 + atlas/views.py | 11 + deploy/settings_local.py | 4 +- 5 files changed, 378 insertions(+), 309 deletions(-) create mode 100644 atlas/templates/atlas/partials/series_images.html diff --git a/atlas/templates/atlas/partials/series_images.html b/atlas/templates/atlas/partials/series_images.html new file mode 100644 index 00000000..20eb59b1 --- /dev/null +++ b/atlas/templates/atlas/partials/series_images.html @@ -0,0 +1,31 @@ +{% load static %} + +
+ {% for image in images %} +
+ {% if image.image %} + + image-{{ image.pk }} + + {% else %} +
+ {% endif %} + +
+
{{ image.image.name|default:"(no file)" }}
+ {% if image.image %} +
{{ image.image.size|filesizeformat }}
+ {% endif %} + {% if image.image_blake3_hash %} +
hash: {{ image.image_blake3_hash|truncatechars:12 }}
+ {% endif %} +
+ +
+ View DICOM +
+
+ {% empty %} +

No images found for this series.

+ {% endfor %} +
diff --git a/atlas/templates/atlas/series_viewer.html b/atlas/templates/atlas/series_viewer.html index e8e4f133..7a0d2c3e 100755 --- a/atlas/templates/atlas/series_viewer.html +++ b/atlas/templates/atlas/series_viewer.html @@ -1,337 +1,361 @@ {% load crispy_forms_tags %} {% with image_url_array_and_count=series.get_image_url_array_and_count %} -
-
-
-
-
-
-

Series: {{ series.modality }} - {{ series.examination }}

-
{{ series.plane }}, {{ series.contrast }}
-
Description: {{ series.description }}
-
Author: {{ series.get_author_display }}
+
+
+
+
+
+
+

Series: {{ series.modality }} - {{ series.examination }}

+
{{ series.plane }}, {{ series.contrast }}
+
Description: {{ series.description }}
+
Author: {{ series.get_author_display }}
+
+
+ View series + {% if can_edit %} + + {% else %} + + {% endif %} +
-
- View series +
+
+
+ +
+
+
+
+ {% with image_url_array_and_count=series.get_image_url_array_and_count %} +
+
+ {% endwith %} +
+
+ {% if can_edit and editing_finding < 1 %} + + + {% endif %} +
{{ series.get_image_count }} images
+
+
+
+ +
+
Finding form
+
+
+
+ {% if editing_finding > 0 %} +
Editing Finding
+

Editing finding with ID: {{editing_finding}}

+ {% else %} +
Add Finding
+

Click the button below to add a new finding.

+ {% endif %} +
+ {% csrf_token %} + {{series_finding_form|crispy}} +
+ + +
+
+
+
+
+
+
+ Findings +
+ {% for finding in series.findings.all %} +
+
+
+ + Finding(s): + {% for f in finding.findings.all %}{{f.get_link}}{% endfor %}
+ Structure(s): {% for s in finding.structures.all %}{{s.get_link}}{% endfor %}
+ Condition(s): {% for s in finding.conditions.all %}{{s.get_link}}{% endfor %}
+
{{finding.description}}
+
+
+ {% if request.user.is_superuser %} + +
+ 
{{finding.annotation_json}}
+ 
{{finding.viewport_json}}
+
+ {% endif %} + {% if can_edit %} +
+ Edit + Delete +
+ {% endif %} +
+
+
+ {% empty %} +

No findings available.

+ {% endfor %} +
+
+
+ + +
+
+ Images +
+ + Download + +
+
+
+
+

Click 'Load images' to fetch.

+
+
+
+ +
+ +
+
+
Cases
+
+
+ {% include 'atlas/partials/series_cases_list.html' %} +
{% if can_edit %} - - {% else %} - +
+ Add Series to Case +
+ {% csrf_token %} +
+ + +
+
+ + +
+
+
{% endif %}
+ +
+
Actions
+
+ Anonymise dicoms + Order by slice location + Order by instance number + Order by SeriesInstanceUID + Order by uploaded filename + Split by ImageType tag +
+
+ + + + {% if can_edit %} +
+
Truncate series
+
+

Limit the series to the selected bounds (the rest of the images will be deleted). This action is irreversible on site.

+
Warning: If you have reordered the series this may remove the wrong images.
+
+ +
+ + +
+
+
+ +
+ + +
+
+
+ + +
+
+
+
+ {% endif %}
-
-
-
-
- {% with image_url_array_and_count=series.get_image_url_array_and_count %} -
-
- {% endwith %} -
-
- {% if can_edit and editing_finding < 1 %} - - - {% endif %} -
{{ series.get_image_count }} images
-
-
-
- -
-
Finding form
-
-
-
- {% if editing_finding > 0 %} -
Editing Finding
-

Editing finding with ID: {{editing_finding}}

- {% else %} -
Add Finding
-

Click the button below to add a new finding.

- {% endif %} -
- {% csrf_token %} - {{series_finding_form|crispy}} -
- - -
-
-
-
-
-
-
- Findings -
- {% for finding in series.findings.all %} -
-
-
- - Finding(s): - {% for f in finding.findings.all %}{{f.get_link}}{% endfor %}
- Structure(s): {% for s in finding.structures.all %}{{s.get_link}}{% endfor %}
- Condition(s): {% for s in finding.conditions.all %}{{s.get_link}}{% endfor %}
-
{{finding.description}}
-
-
- {% if request.user.is_superuser %} - -
- 
{{finding.annotation_json}}
- 
{{finding.viewport_json}}
-
- {% endif %} - {% if can_edit %} -
- Edit - Delete -
- {% endif %} -
-
-
- {% empty %} -

No findings available.

- {% endfor %} -
-
-
-
- -
-
-
Cases
-
-
- {% include 'atlas/partials/series_cases_list.html' %} -
- {% if can_edit %} -
- Add Series to Case -
- {% csrf_token %} -
- - -
-
- - -
-
-
- {% endif %} -
-
- -
-
Actions
-
- Anonymise dicoms - Order by slice location - Order by instance number - Order by SeriesInstanceUID - Order by uploaded filename - Split by ImageType tag -
-
- - - - {% if can_edit %} -
-
Truncate series
-
-

Limit the series to the selected bounds (the rest of the images will be deleted). This action is irreversible on site.

-
Warning: If you have reordered the series this may remove the wrong images.
-
- -
- - -
-
-
- -
- - -
-
-
- - -
-
-
-
- {% endif %} + -
- + + {% endwith %} @@ -402,14 +426,14 @@ transition: background 0.2s, font-size 0.2s; } -details.styled-detail:not([open]) { - padding-bottom: 0; - min-height: 0; -} + details.styled-detail:not([open]) { + padding-bottom: 0; + min-height: 0; + } -details.styled-detail:not([open]) > *:not(summary) { - display: none; -} + details.styled-detail:not([open]) > *:not(summary) { + display: none; + } details.styled-detail:not([open]) > summary { padding: 0.7em 1.2em 0.3em 1.2em; diff --git a/atlas/urls.py b/atlas/urls.py index 868399fc..2097dc8d 100755 --- a/atlas/urls.py +++ b/atlas/urls.py @@ -457,6 +457,7 @@ urlpatterns = [ # TODO: case context series viewing (so that we can view series in the context of a case) path("series/", views.series_detail, name="series_detail"), path("series//viewer", views.series_viewer, name="series_viewer"), + path("series//images/", views.series_images_partial, name="series_images"), path("series//authors", views.SeriesAuthorUpdate.as_view(), name="series_authors"), path("series//finding/related", views.series_finding_related, name="series_finding_related"), path("series/", views.SeriesView.as_view(), name="series_view"), diff --git a/atlas/views.py b/atlas/views.py index 7d03c3d6..25709c5a 100755 --- a/atlas/views.py +++ b/atlas/views.py @@ -796,6 +796,17 @@ def series_detail(request, pk, finding_pk=None): ) +@login_required +@user_is_author_or_atlas_series_checker_or_atlas_marker_or_open_access +def series_images_partial(request, pk): + """Return a small fragment that lists images for a series. Intended for HTMX.""" + series = get_object_or_404(Series, pk=pk) + logger.debug(series) + images = series.images.filter(removed=False).order_by("pk") + logger.debug(images) + return render(request, "atlas/partials/series_images.html", {"series": series, "images": images}) + + @login_required # @user_is_atlas_editor def question_schema_detail(request, pk: int): diff --git a/deploy/settings_local.py b/deploy/settings_local.py index 03376c38..66c13638 100644 --- a/deploy/settings_local.py +++ b/deploy/settings_local.py @@ -117,4 +117,6 @@ SECURE_HSTS_SECONDS = 0 CSRF_TRUSTED_ORIGINS = [ "http://127.0.0.1:8000", "http://localhost:8000", -] \ No newline at end of file +] + +REMOTE_URL = "http://127.0.0.1:8000" \ No newline at end of file From 517c30ba0e793342efdc4b9a790ebe5a66582e2d Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 1 Dec 2025 16:43:40 +0000 Subject: [PATCH 08/13] Enhance Docker setup with logging and monitoring features - Added logging configuration to WSGI for persistent log collection. - Updated Docker Compose files for local and development environments, including Redis, PostgreSQL, and Grafana services. - Introduced Promtail configuration for log scraping and integration with Loki. - Created application log directory in Dockerfile for log persistence. --- docker/docker-compose.dev.yml | 45 +++++++++++++++++++ ...r-compose.yml => docker-compose.local.yml} | 0 docker/promtail-config.yml | 18 ++++++++ rad/Dockerfile.prod | 6 +++ rad/wsgi.py | 30 +++++++++++++ 5 files changed, 99 insertions(+) rename docker/{docker-compose.yml => docker-compose.local.yml} (100%) create mode 100644 docker/promtail-config.yml diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index cc688c95..92b8ec23 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -15,10 +15,55 @@ services: command: [] volumes: - ../:/usr/src/app:cached + # Mount the app log directory so logs are visible on the host at ./logs + - ../logs:/var/log/rad nginx: # Development nginx override: mount a simplified config that does not # reference LetsEncrypt certs so the container can start without real # certificates present on the host. + image: nginx:1.25-alpine volumes: - ../deploy/nginx/dev.conf:/etc/nginx/conf.d/default.conf:ro + loki: + image: grafana/loki:2.8.2 + command: -config.file=/etc/loki/local-config.yaml + ports: + - "3100:3100" + volumes: + - ./loki-config/local-config.yaml:/etc/loki/local-config.yaml:ro + - ./loki-data:/loki + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:3100/ready || exit 1"] + interval: 5s + timeout: 3s + retries: 6 + + promtail: + image: grafana/promtail:2.8.2 + volumes: + - ../logs:/var/log/rad:ro + - ./promtail-config.yml:/etc/promtail/promtail-config.yml:ro + command: -config.file=/etc/promtail/promtail-config.yml + depends_on: + - loki + restart: unless-stopped + + grafana: + image: grafana/grafana:10.2.0 + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - grafana-data:/var/lib/grafana + depends_on: + loki: + condition: service_healthy + restart: unless-stopped + +volumes: + grafana-data: + loki-data: + diff --git a/docker/docker-compose.yml b/docker/docker-compose.local.yml similarity index 100% rename from docker/docker-compose.yml rename to docker/docker-compose.local.yml diff --git a/docker/promtail-config.yml b/docker/promtail-config.yml new file mode 100644 index 00000000..43135394 --- /dev/null +++ b/docker/promtail-config.yml @@ -0,0 +1,18 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: rad_app_logs + static_configs: + - targets: + - localhost + labels: + job: rad_app + __path__: /var/log/rad/*.log diff --git a/rad/Dockerfile.prod b/rad/Dockerfile.prod index c25ef6b9..f65b66f8 100644 --- a/rad/Dockerfile.prod +++ b/rad/Dockerfile.prod @@ -57,6 +57,12 @@ ENV PATH="/opt/venv/bin:$PATH" # `--chown` flag during COPY and avoid an expensive recursive `chown -R`. RUN useradd -m appuser +# Create directory for application logs and ensure ownership is correct. +# Do this as root before switching to `appuser` so the directory exists +# and can be mounted by the dev compose override. +RUN mkdir -p /var/log/rad \ + && chown -R appuser:appuser /var/log/rad + # Copy project files into image and set ownership during copy (faster than # a separate `chown -R` step). This keeps layer reuse efficient. COPY --chown=appuser:appuser . /usr/src/app diff --git a/rad/wsgi.py b/rad/wsgi.py index 23078214..78186fbb 100644 --- a/rad/wsgi.py +++ b/rad/wsgi.py @@ -10,6 +10,36 @@ https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ import os from django.core.wsgi import get_wsgi_application +from loguru import logger +import sys +from pathlib import Path + +# Configure Loguru sinks early so app imports see consistent logging. +# Log to stdout (so `docker logs` shows messages) and to a file at +# `/var/log/rad/app.log` for persistent collection / host tailing. +try: + # ensure directory exists + log_dir = Path("/var/log/rad") + log_dir.mkdir(parents=True, exist_ok=True) + # Add stdout sink (if not already added by other code) + logger.remove() # remove default handlers to avoid duplicate lines + logger.add(sys.stdout, level="INFO", enqueue=True, backtrace=False, diagnose=False) + # Add rotating file sink for persistence in container. Use JSON lines + # (serialize=True) to make logs easy to query in Loki/Grafana. + logger.add( + str(log_dir / "app.log"), + rotation="10 MB", + retention="10 days", + level="DEBUG", + enqueue=True, + serialize=True, + ) +except Exception: + # Best-effort only — don't crash WSGI startup if logging can't be configured + try: + logger.add(sys.stderr) + except Exception: + pass os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rad.settings") From 63c6a648d78edaceacae1d924c41873936708e47 Mon Sep 17 00:00:00 2001 From: Ross Date: Thu, 4 Dec 2025 20:06:24 +0000 Subject: [PATCH 09/13] Add Loki, Promtail, and Grafana services for observability stack --- docker/docker-compose.prod.yml | 47 +++++++++++++++++++ docker/loki-config/local-config.yaml | 43 +++++++++++++++++ .../loki-data/chunks/loki_cluster_seed.json | 1 + 3 files changed, 91 insertions(+) create mode 100644 docker/loki-config/local-config.yaml create mode 100644 docker/loki-data/chunks/loki_cluster_seed.json diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index da5b859d..1808f0c5 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -61,11 +61,58 @@ services: timeout: 10s retries: 3 + # Optional local observability stack (can be enabled on servers that + # should run Loki/Grafana). These services use configs found under + # the repository `docker/` directory so the same config works in dev. + loki: + image: grafana/loki:2.8.2 + command: -config.file=/etc/loki/local-config.yaml + ports: + - "3100:3100" + volumes: + - ./loki-config/local-config.yaml:/etc/loki/local-config.yaml:ro + - ../docker/loki-data:/loki + # WAL directory: map a host folder so Loki (running as UID 10001) + # can create the write-ahead log. Host folder created under + # `docker/loki-wal` and owned by UID 10001. + - ../docker/loki-wal:/wal + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:3100/ready || exit 1"] + interval: 10s + timeout: 5s + retries: 6 + + promtail: + image: grafana/promtail:2.8.2 + volumes: + # In prod you may want to point this at the host system log directory. + - ../logs:/var/log/rad:ro + - ./promtail-config.yml:/etc/promtail/promtail-config.yml:ro + command: -config.file=/etc/promtail/promtail-config.yml + depends_on: + - loki + restart: unless-stopped + + grafana: + image: grafana/grafana:10.2.0 + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - grafana-data:/var/lib/grafana + depends_on: + loki: + condition: service_healthy + restart: unless-stopped + volumes: postgres_data: redis_data: static_volume: media_volume: + grafana-data: networks: default: diff --git a/docker/loki-config/local-config.yaml b/docker/loki-config/local-config.yaml new file mode 100644 index 00000000..8b3b7b2f --- /dev/null +++ b/docker/loki-config/local-config.yaml @@ -0,0 +1,43 @@ +server: + http_listen_port: 3100 + grpc_listen_port: 9095 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + chunk_idle_period: 5m + chunk_retain_period: 30s + max_transfer_retries: 0 + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/cache + shared_store: filesystem + filesystem: + directory: /loki/chunks + +# Compactor settings: ensure compactor working directory is inside the mounted +# `/loki` path so the process (running as UID 10001) can create files. +compactor: + working_directory: /loki/compactor + shared_store: filesystem + +limits_config: + enforce_metric_name: false + +auth_enabled: false diff --git a/docker/loki-data/chunks/loki_cluster_seed.json b/docker/loki-data/chunks/loki_cluster_seed.json new file mode 100644 index 00000000..037f8b98 --- /dev/null +++ b/docker/loki-data/chunks/loki_cluster_seed.json @@ -0,0 +1 @@ +{"UID":"f8cade59-3cf9-4ae3-965d-8b0a57007d58","created_at":"2025-12-01T17:14:05.101735276Z","version":{"version":"2.8.2","revision":"9f809eda7","branch":"HEAD","buildUser":"root@e401cfcb874f","buildDate":"2023-05-03T11:07:54Z","goVersion":"go1.20.4"}} \ No newline at end of file From 02a9118fafacaedb2efa41071315580bc13c2458 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 8 Dec 2025 09:29:30 +0000 Subject: [PATCH 10/13] . --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8d92a891..117005a9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,8 @@ venv *.log -temp/ \ No newline at end of file +temp/ + +# Loki runtime data +/docker/loki-data/ +/docker/loki-wal/ \ No newline at end of file From 0917947a9816030181405077129e9684f7ad8432 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 8 Dec 2025 09:52:18 +0000 Subject: [PATCH 11/13] . --- .env.dev | 5 +++-- .gitignore | 4 +++- deploy/nginx/dev.conf | 4 ++++ deploy/nginx/prod.conf | 8 ++++++++ docker/docker-compose.dev.yml | 1 + docker/docker-compose.prod.yml | 1 + generic/mixins.py | 3 +++ generic/views.py | 2 +- 8 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.env.dev b/.env.dev index fe69fd87..e7717e1f 100644 --- a/.env.dev +++ b/.env.dev @@ -18,5 +18,6 @@ GUNICORN_WORKERS=3 GUNICORN_LOGLEVEL=info # Nginx host ports for local development (override prod defaults) -NGINX_HTTP_PORT=8000 -NGINX_HTTPS_PORT=8443 \ No newline at end of file +NGINX_HTTP_PORT=8080 +# Pick a non-privileged HTTPS port to avoid conflicts with host installs +NGINX_HTTPS_PORT=8444 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 117005a9..5d244eab 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ temp/ # Loki runtime data /docker/loki-data/ -/docker/loki-wal/ \ No newline at end of file +/docker/loki-wal/ +# Local runtime logs collected by Promtail +/logs/ \ No newline at end of file diff --git a/deploy/nginx/dev.conf b/deploy/nginx/dev.conf index 495c7e57..6eb39ef9 100644 --- a/deploy/nginx/dev.conf +++ b/deploy/nginx/dev.conf @@ -7,6 +7,10 @@ server { listen [::]:80; server_name _; + # Write logs to the host-mounted folder so Promtail can scrape them + access_log /var/log/rad/nginx.access.log combined; + error_log /var/log/rad/nginx.error.log warn; + client_max_body_size 100M; location /static/ { diff --git a/deploy/nginx/prod.conf b/deploy/nginx/prod.conf index d08ca5cc..a4c7fea8 100644 --- a/deploy/nginx/prod.conf +++ b/deploy/nginx/prod.conf @@ -8,6 +8,10 @@ server { listen [::]:80; server_name _; + # Write logs to the host-mounted folder so Promtail can scrape them + access_log /var/log/rad/nginx.access.log combined; + error_log /var/log/rad/nginx.error.log warn; + client_max_body_size 100M; # Serve static files from the static volume @@ -112,6 +116,10 @@ server { client_max_body_size 4G; + # Write logs to the host-mounted folder so Promtail can scrape them + access_log /var/log/rad/nginx.access.log combined; + error_log /var/log/rad/nginx.error.log warn; + # Same static/media handling as above location /static/ { alias /usr/src/app/static/; } location /media { alias /usr/src/app/media; } diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 92b8ec23..598c2634 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -24,6 +24,7 @@ services: image: nginx:1.25-alpine volumes: - ../deploy/nginx/dev.conf:/etc/nginx/conf.d/default.conf:ro + - ../logs:/var/log/rad:rw loki: image: grafana/loki:2.8.2 diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 1808f0c5..68f0e361 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -45,6 +45,7 @@ services: - web volumes: - ../deploy/nginx/prod.conf:/etc/nginx/conf.d/default.conf:ro + - ../logs:/var/log/rad:rw - static_volume:/usr/src/app/static:ro - media_volume:/usr/src/app/media:ro - ../deploy/nginx/certs:/etc/letsencrypt:ro diff --git a/generic/mixins.py b/generic/mixins.py index 99765d4f..75898e4e 100644 --- a/generic/mixins.py +++ b/generic/mixins.py @@ -65,6 +65,9 @@ class AuthorMixin(): def is_author(self, user: User) -> bool: """Returns True if the user is an author of the object""" + if user.is_superuser: + return True + return self.author.filter(id=user.id).exists() diff --git a/generic/views.py b/generic/views.py index a7b14ffe..223dae71 100644 --- a/generic/views.py +++ b/generic/views.py @@ -1774,7 +1774,7 @@ class ExamViews(View, LoginRequiredMixin): def exam_list_collection(self, request, collection_id): collection = get_object_or_404(ExamCollection, pk=collection_id) - if not request.user in collection.author.all(): + if not collection.is_author(request.user): raise PermissionDenied return self.exam_list(request, all=True, collection=collection) From d29c23a7997ee330f3cdf67f1bb0c08a2030d284 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 8 Dec 2025 09:54:15 +0000 Subject: [PATCH 12/13] . --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..f10b7633 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +README +====== + +Quick notes for running this repository (development & observability) + +Running locally (development) +- Use the included compose files. To run the dev stack (overrides) set `COMPOSE_ENV=dev` so the correct env file is loaded: + + ```sh + # run nginx on non-privileged ports (see .env.dev) + COMPOSE_ENV=dev docker compose -f docker/docker-compose.prod.yml -f docker/docker-compose.dev.yml up -d --build + ``` + +- By default the development nginx ports are set in `.env.dev` to avoid colliding with a host nginx. The defaults now are: + - `NGINX_HTTP_PORT=8080` + - `NGINX_HTTPS_PORT=8444` + +Logging & observability (Loki + Promtail + Grafana) +- Promtail is configured to scrape `/var/log/rad/*.log` and push to Loki. Promtail's config is at `docker/promtail-config.yml` and the Loki config is at `docker/loki-config/local-config.yaml`. +- The nginx configs (both `deploy/nginx/prod.conf` and `deploy/nginx/dev.conf`) are configured to write access and error logs to `/var/log/rad/nginx.access.log` and `/var/log/rad/nginx.error.log` respectively. +- The compose files mount a host `logs/` folder into `/var/log/rad` so both nginx (write) and promtail (read) can access the same files: + - `../logs:/var/log/rad:rw` (nginx) + - `../logs:/var/log/rad:ro` (promtail) + +Host setup for logs +- Create the host logs folder at the repo root (this repo's `logs/`). In development you can use permissive permissions so containers can write to it quickly: + + ```sh + mkdir -p logs + # development: make writable by all (change this for production) + sudo chown $USER:$USER logs + chmod 0777 logs + ``` + +- In production prefer setting the folder owner to the nginx worker UID (or run nginx under a specific user) and use `0755` or `0750` as appropriate. Example: + + ```sh + # run on the host (determine nginx UID as needed) + sudo chown -R 101:101 /srv/rad/logs + sudo chmod 0755 /srv/rad/logs + ``` + +Viewing logs in Grafana +- Grafana is available on the port exposed by compose (default `3000`). +- The Loki datasource is configured to use `http://loki:3100` (see compose). In Grafana Explore choose the Loki datasource and run queries such as: + + - `{job="rad_app"}` (all collected logs) + - `{job="rad_app", filename="/var/log/rad/nginx.access.log"}` + +Notes & recommendations +- In development the `logs/` folder is ignored by git (`/.gitignore` updated). Do not commit runtime logs. +- Consider a log rotation/retention policy for production (e.g., `logrotate` or use Loki retention policies). +- If you need nginx to re-resolve the backend service IP without restarting, consider using `resolver` and variable-based upstreams in the nginx config. For many development workflows restarting nginx after web restarts is the simplest approach. + +If you want, I can also add a short `docs/OBSERVABILITY.md` with more details and recommended production settings (TLS/auth for Loki, retention, log rotation). From af616a9f4a9cc49db53de3fcfe3aa1aeee58b6d8 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 8 Dec 2025 10:05:44 +0000 Subject: [PATCH 13/13] Add tooltips for open access settings and improve UI elements in exam overview --- generic/templates/generic/exam_overview_headers.html | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/generic/templates/generic/exam_overview_headers.html b/generic/templates/generic/exam_overview_headers.html index 0c67cf8b..67b1850f 100644 --- a/generic/templates/generic/exam_overview_headers.html +++ b/generic/templates/generic/exam_overview_headers.html @@ -93,7 +93,7 @@
-
+
Open access: {{ exam.open_access }}
@@ -121,10 +121,10 @@
-
+
{# Use d-block on small screens so the summary text wraps nicely; md uses flex for spacing #} -
@@ -133,7 +133,6 @@
-
Select questions on the list above (checkboxes) then click one of the actions below.