ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement passive XHR/fetch capture using Playwright

Browse Source 
- Added `capture_passive_playwright.py` for synchronous passive capture of XHR/fetch responses.
- Added `capture_passive_playwright_async.py` for asynchronous passive capture using Playwright's async API.
- Introduced `save_page_snapshots.py` to save full page HTML snapshots with deduplication.
- Removed obsolete CSRF token capture JSON file.
This commit is contained in:
Ross
2025-10-14 19:48:01 +01:00
parent 683dd06643
commit adc21f5795
7 changed files with 1181 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/statdx_live_capture.py
+28 -25
View File
@@ -22,7 +22,6 @@ Notes / limitations:
import os
import argparse
import logging
import time
import threading
import sys
@@ -32,9 +31,12 @@ from urllib.parse import urlparse
from dotenv import load_dotenv
import mimetypes
from loguru import logger
load_dotenv()
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
logger.remove()
logger.add(sys.stderr, level="INFO")
def sanitize_filename(url: str) -> str:
@@ -79,7 +81,7 @@ def auto_scroll(page, step_ms: int = 50, step_px: int = 300):
step_ms,
)
except Exception:
logging.exception('auto_scroll failed')
logger.exception('auto_scroll failed')
def save_response(resp, out_dir: str):
@@ -112,7 +114,7 @@ def save_response(resp, out_dir: str):
filename_base = sanitize_filename(url)
out_path = os.path.join(out_dir, filename_base + ext)
logging.info('Saving %s -> %s (content-type: %s)', url, out_path, ctype.split(';')[0])
logger.info(f'Saving {url} -> {out_path} (content-type: {ctype.split(";")[0]})')
if save_as_text:
try:
@@ -136,7 +138,7 @@ def save_response(resp, out_dir: str):
return True
except Exception as e:
logging.exception('Failed to save response: %s', e)
logger.exception(f'Failed to save response: {e}')
return False
@@ -147,21 +149,21 @@ def save_page_snapshot(page, out_dir: str, reason: str = ''):
filename_base = sanitize_filename(url)
safe_reason = ''.join(c if c.isalnum() or c in ('-', '_') else '_' for c in reason)[:40]
out_path = os.path.join(out_dir, f"snapshot_{filename_base}_{ts}_{safe_reason}.html")
logging.info('Saving page snapshot: %s -> %s', url, out_path)
logger.info(f'Saving page snapshot: {url} -> {out_path}')
content = page.content()
with open(out_path, 'w', encoding='utf-8') as fh:
fh.write(content)
# Screenshots are disabled (not saving visual snapshots per user request)
return True
except Exception:
logging.exception('Failed to save page snapshot')
logger.exception('Failed to save page snapshot')
return False
def main():
parser = argparse.ArgumentParser(description='Live-capture browsing responses using Playwright')
parser.add_argument('--output', '-o', default='captured', help='Directory to save captured responses')
parser.add_argument('--profile', '-p', default='.playwright_profile', help='Persistent profile directory')
parser.add_argument('--profile', '-p', default='playwright_profile', help='Persistent profile directory')
parser.add_argument('--domain', '-d', default=os.getenv('STATDX_DOMAIN', 'app.statdx.com'), help='Only capture responses whose URL contains this domain')
parser.add_argument('--headless', action='store_true', help='Run headless (no visible browser)')
parser.add_argument('--snapshot-delay', type=float, default=0.5, help='Seconds to wait after an XHR before saving a snapshot')
@@ -182,7 +184,7 @@ def main():
with sync_playwright() as p:
# Use persistent context so cookies/localStorage persist between runs
browser_type = p.chromium
logging.info('Launching Chromium (headless=%s) with profile: %s, channel=%s', args.headless, args.profile, args.channel)
logger.info(f'Launching Chromium (headless={args.headless}) with profile: {args.profile}, channel={args.channel}')
if args.channel:
context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel)
else:
@@ -198,6 +200,7 @@ def main():
pass
def handle_response(resp):
logger.debug(f'RESPONSE: {resp.url} ({resp.request.resource_type})')
try:
if args.domain not in resp.url:
return
@@ -214,12 +217,12 @@ def main():
try:
auto_scroll(page)
except Exception:
logging.exception('auto-scroll failed')
logger.exception('auto-scroll failed')
save_page_snapshot(page, args.output, reason=rtype)
except Exception:
logging.exception('snapshot after response failed')
logger.exception('snapshot after response failed')
except Exception:
logging.exception('Error in response handler')
logger.exception('Error in response handler')
page.on('response', handle_response)
@@ -227,19 +230,19 @@ def main():
# Log console messages and page errors to help diagnose missing content
def on_console(msg):
try:
logging.info('PAGE LOG [%s] %s', msg.type, msg.text)
logger.info(f'PAGE LOG [{msg.type}] {msg.text}')
except Exception:
logging.info('PAGE LOG: %s', msg)
logger.info(f'PAGE LOG: {msg}')
def on_page_error(exc):
logging.error('PAGE ERROR: %s', exc)
logger.error(f'PAGE ERROR: {exc}')
page.on('console', on_console)
page.on('pageerror', on_page_error)
logging.info('Open the browser window and interact with the site. Captured responses will be saved to: %s', args.output)
logging.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
logging.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
logger.info(f'Open the browser window and interact with the site. Captured responses will be saved to: {args.output}')
logger.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
logger.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
# Navigate to the domain root so you can log in manually if needed
start_url = f'https://{args.domain}/'
@@ -247,27 +250,27 @@ def main():
# Optionally wait for a selector that indicates the page finished rendering
if args.wait_for_selector:
try:
logging.info('Waiting for selector: %s', args.wait_for_selector)
logger.info(f'Waiting for selector: {args.wait_for_selector}')
page.wait_for_selector(args.wait_for_selector, timeout=20000)
except Exception:
logging.exception('Waiting for selector timed out')
logger.exception('Waiting for selector timed out')
def stdin_watcher():
# Run in a background thread so the main thread can keep the browser running
logging.debug('stdin watcher started')
logger.debug('stdin watcher started')
for line in sys.stdin:
cmd = line.strip().lower()
if cmd == '' or cmd == 's' or cmd == 'snapshot' or cmd == 'save':
logging.info('Manual snapshot requested via stdin')
logger.info('Manual snapshot requested via stdin')
try:
if args.auto_scroll:
try:
auto_scroll(page)
except Exception:
logging.exception('manual auto-scroll failed')
logger.exception('manual auto-scroll failed')
save_page_snapshot(page, args.output, reason='manual')
except Exception:
logging.exception('manual snapshot failed')
logger.exception('manual snapshot failed')
watcher = threading.Thread(target=stdin_watcher, daemon=True)
watcher.start()
@@ -277,7 +280,7 @@ def main():
while True:
time.sleep(1)
except KeyboardInterrupt:
logging.info('Shutting down...')
logger.info('Shutting down...')
finally:
context.close()