From fd2766d0a904780479685868c599ebf06b0419c1 Mon Sep 17 00:00:00 2001 From: Ross Date: Thu, 16 Jul 2026 23:16:46 +0100 Subject: [PATCH] feat(auth): implement basic authentication for cache server and update URL --- scrapers/cache_server.py | 1015 ++++++++++++++++++++++- scrapers/statdx_passive_capture.user.js | 26 +- 2 files changed, 1032 insertions(+), 9 deletions(-) diff --git a/scrapers/cache_server.py b/scrapers/cache_server.py index 125cd22..bc703b0 100644 --- a/scrapers/cache_server.py +++ b/scrapers/cache_server.py @@ -3,10 +3,14 @@ import os import json import base64 import hashlib +import secrets from pathlib import Path -from typing import Dict, Optional -from fastapi import FastAPI, HTTPException +from typing import Dict, Optional, List +from collections import deque +from fastapi import FastAPI, HTTPException, Depends, status from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import HTMLResponse, FileResponse +from fastapi.security import HTTPBasic, HTTPBasicCredentials from pydantic import BaseModel # Import exact helpers from capture_passive_playwright_async @@ -34,8 +38,27 @@ OUTPUT_DIR = Path("xhr_captured_async") OUTPUT_DIR.mkdir(parents=True, exist_ok=True) INDEX_PATH = OUTPUT_DIR / "capture_index.jsonl" +# Basic Authentication Configuration +CACHE_AUTH_USER = os.getenv("CACHE_AUTH_USER", "admin") +CACHE_AUTH_PASS = os.getenv("CACHE_AUTH_PASS", "statdx-secure-cache-2026") + +security = HTTPBasic() + +def check_auth(credentials: HTTPBasicCredentials = Depends(security)): + correct_username = secrets.compare_digest(credentials.username, CACHE_AUTH_USER) + correct_password = secrets.compare_digest(credentials.password, CACHE_AUTH_PASS) + if not (correct_username and correct_password): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Basic"}, + ) + return credentials.username + # Persistent in-memory index in_memory_index: Dict[str, list] = {} +# Deque for the last 100 captures to display in the dashboard +recent_captures = deque(maxlen=100) def load_existing_index(): if INDEX_PATH.exists(): @@ -51,9 +74,11 @@ def load_existing_index(): ch = entry.get("content_hash") if ch: in_memory_index.setdefault(ch, []).append(entry) + recent_captures.append(entry) except Exception: pass print(f"Loaded {len(in_memory_index)} unique hashes into memory.") + print(f"Loaded last {len(recent_captures)} captures into recent history.") except Exception as e: print(f"Error loading index: {e}") @@ -72,13 +97,992 @@ class CapturePayload(BaseModel): body_base64: str content_type: str +# ----------------- Dashboard HTML ----------------- +DASHBOARD_HTML = """ + + + + + STATdx Cache Payload Monitor + + + + + + + + + +
+
+ +
+

STATdx Cache Monitor

+

Secure verification dashboard

+
+
+ +
+
+ Loading... +
+
+ + Active +
+
+
+ +
+
+
+
+ + +
+
+
All
+
XHR
+
Fetch
+
Images
+
Documents
+
+
+ +
+
+
+

Loading captured payloads...

+
+
+
+ +
+
+ +

Select a payload from the list to view captured contents

+
+
+
+ + + + +""" + +@app.get("/", response_class=HTMLResponse) +async def dashboard(username: str = Depends(check_auth)): + return HTMLResponse(content=DASHBOARD_HTML) + +@app.get("/api/recent", response_model=List[dict]) +async def get_recent(username: str = Depends(check_auth)): + return list(reversed(recent_captures)) + +@app.get("/api/payload/{content_hash}") +async def get_payload(content_hash: str, username: str = Depends(check_auth)): + entries = in_memory_index.get(content_hash) + if not entries: + raise HTTPException(status_code=404, detail="Payload not found in memory index") + + entry = entries[0] + meta_file = entry.get("meta_file") + if meta_file and os.path.exists(meta_file): + try: + with open(meta_file, "r", encoding="utf-8") as f: + meta = json.load(f) + return meta + except Exception: + pass + return entry + +@app.get("/api/file/{file_path:path}") +async def get_file(file_path: str, username: str = Depends(check_auth)): + base_dir = OUTPUT_DIR.resolve() + + # Safely resolve path relative to current directory + full_path = Path(os.getcwd()).joinpath(file_path).resolve() + + if not str(full_path).startswith(str(base_dir)): + raise HTTPException(status_code=403, detail="Access denied") + + if not full_path.exists() or not full_path.is_file(): + raise HTTPException(status_code=404, detail="File not found") + + return FileResponse(full_path) + @app.post("/api/check-hash") -async def check_hash(payload: HashCheckRequest): +async def check_hash(payload: HashCheckRequest, username: str = Depends(check_auth)): exists = payload.content_hash in in_memory_index return {"exists": exists} @app.post("/api/capture") -async def capture(payload: CapturePayload): +async def capture(payload: CapturePayload, username: str = Depends(check_auth)): try: # Decode body body_bytes = base64.b64decode(payload.body_base64) @@ -86,8 +1090,6 @@ async def capture(payload: CapturePayload): # Check deduplication if content_hash in in_memory_index: - # We already have this exact content saved - # Just return and do not write duplicate files return {"status": "skipped", "reason": "content_hash already cached"} url = payload.url @@ -176,6 +1178,7 @@ async def capture(payload: CapturePayload): # Update in-memory index in_memory_index.setdefault(content_hash, []).append(entry) + recent_captures.append(entry) print(f"SAVED: {url} -> {body_path}") return {"status": "saved", "path": body_path} diff --git a/scrapers/statdx_passive_capture.user.js b/scrapers/statdx_passive_capture.user.js index 6cbdb1f..04c3ef8 100644 --- a/scrapers/statdx_passive_capture.user.js +++ b/scrapers/statdx_passive_capture.user.js @@ -16,7 +16,21 @@ // CHANGE THIS URL to the IP address or hostname of your central cache server if running on remote machines // e.g., "http://192.168.1.50:8000" - const CACHE_SERVER_URL = "http://localhost:8000"; + const CACHE_SERVER_URL = "https://stat.xkjq.uk"; + + // Credentials for central cache server basic authentication + const CACHE_AUTH_USER = "admin"; + const CACHE_AUTH_PASS = "statdx-secure-cache-2026"; + + // Helper: Generate Basic Auth header + function getAuthHeader() { + try { + const credentials = btoa(`${CACHE_AUTH_USER}:${CACHE_AUTH_PASS}`); + return `Basic ${credentials}`; + } catch (e) { + return ""; + } + } const capturedUrls = new Set(); @@ -156,7 +170,10 @@ GM_xmlhttpRequest({ method: "POST", url: `${CACHE_SERVER_URL}/api/check-hash`, - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + "Authorization": getAuthHeader() + }, data: JSON.stringify({ content_hash: contentHash }), onload: function(res) { if (res.status === 200) { @@ -187,7 +204,10 @@ GM_xmlhttpRequest({ method: "POST", url: `${CACHE_SERVER_URL}/api/capture`, - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + "Authorization": getAuthHeader() + }, data: JSON.stringify(payload), onload: function(res) { if (res.status === 200) {