feat: Implement passive XHR/fetch capture using Playwright

- Added `capture_passive_playwright.py` for synchronous passive capture of XHR/fetch responses.
- Added `capture_passive_playwright_async.py` for asynchronous passive capture using Playwright's async API.
- Introduced `save_page_snapshots.py` to save full page HTML snapshots with deduplication.
- Removed obsolete CSRF token capture JSON file.

.gitignore

@@ -70,3 +70,4 @@ playwright_profile
 
 captured/
 xhr_captured/
+xhr_captured_async/

scrapers/capture_passive_playwright.py

@@ -0,0 +1,459 @@
+#!/usr/bin/env python3
+"""
+Passive XHR/fetch capture using Playwright.
+
+This script launches a persistent browser profile (so you can log in),
+optionally attempts auto-login, then attaches a response handler and saves
+every XHR/fetch response body and metadata into `--output-dir` while you
+browse. It is intentionally passive: it doesn't reload the page unless you
+ask it to; it simply listens and saves responses as they happen.
+
+Usage:
+  python scrapers/capture_passive_playwright.py --output-dir xhr_captured --continuous
+
+Options:
+  --username/--password or STATDX_USERNAME/STATDX_PASSWORD via .env for auto-login
+  --no-prompt to start listening immediately
+  --capture-types to adjust which resource types to record (default: xhr,fetch)
+"""
+from pathlib import Path
+import argparse
+import json
+import sys
+import time
+import mimetypes
+import hashlib
+from datetime import datetime, timezone
+import os
+
+from loguru import logger
+
+from playwright.sync_api import sync_playwright
+import asyncio
+try:
+    from playwright._impl._errors import TargetClosedError as _PlaywrightTargetClosedError
+    TARGET_CLOSED_EXCEPTIONS = (_PlaywrightTargetClosedError,)
+except Exception:
+    # If we cannot import Playwright internal error types, fall back to Exception
+    TARGET_CLOSED_EXCEPTIONS = (Exception,)
+from dotenv import load_dotenv
+
+logger.remove()
+logger.add("capture_passive_playwright.log", rotation="10 MB")
+logger.add(sys.stderr, level="TRACE")
+# Keep detailed logs in the file only to avoid noisy console output while you browse.
+# We will print a short summary to stdout when the browser/context is closed.
+
+load_dotenv()
+
+
+def sanitize(s: str) -> str:
+    out = []
+    for ch in s:
+        if ch.isalnum() or ch in ('-', '_', '.'):
+            out.append(ch)
+        else:
+            out.append('_')
+    name = ''.join(out)
+    if len(name) > 180:
+        name = name[:120] + '_' + hashlib.sha1(s.encode('utf-8')).hexdigest()[:8]
+    return name
+
+
+def guess_ext(ctype: str) -> str:
+    if not ctype:
+        return 'bin'
+    mt = ctype.split(';')[0].strip()
+    ext = mimetypes.guess_extension(mt)
+    if ext:
+        return ext.lstrip('.')
+    if mt.startswith('image/'):
+        return mt.split('/')[1]
+    if mt == 'application/json':
+        return 'json'
+    if mt == 'text/html':
+        return 'html'
+    if mt.startswith('text/'):
+        return 'txt'
+    return 'bin'
+
+
+def now_ts():
+    return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+
+
+def save_binary(path: Path, data: bytes):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'wb') as f:
+        f.write(data)
+
+
+def save_text(path: Path, text: str):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write(text)
+
+
+def try_pretty_json(s: str):
+    try:
+        parsed = json.loads(s)
+        return json.dumps(parsed, ensure_ascii=False, indent=2)
+    except Exception:
+        return None
+
+
+def attempt_autologin(page, username, password, post_login_selector, wait_after=3.0):
+    if not username or not password:
+        return False
+    logger.info('Attempting auto-login')
+    user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]']
+    pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
+
+    user_sel = None
+    for sel in user_selectors:
+        try:
+            if page.query_selector(sel):
+                user_sel = sel
+                break
+        except Exception:
+            continue
+
+    pass_sel = None
+    for sel in pass_selectors:
+        try:
+            if page.query_selector(sel):
+                pass_sel = sel
+                break
+        except Exception:
+            continue
+
+    if not user_sel or not pass_sel:
+        logger.warning('Could not find login fields for auto-login')
+        return False
+
+    try:
+        page.fill(user_sel, username)
+        time.sleep(0.1)
+        page.fill(pass_sel, password)
+        time.sleep(0.1)
+        # try to submit
+        submit_try = ['button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")']
+        clicked = False
+        for s in submit_try:
+            try:
+                btn = page.query_selector(s)
+                if btn:
+                    btn.click()
+                    clicked = True
+                    break
+            except Exception:
+                continue
+        if not clicked:
+            page.press(pass_sel, 'Enter')
+
+        if post_login_selector:
+            try:
+                page.wait_for_selector(post_login_selector, timeout=10000)
+            except Exception:
+                logger.info('Post-login selector not found in timeout')
+
+        time.sleep(wait_after)
+        return True
+    except Exception:
+        logger.exception('Auto-login failed')
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--url', '-u', default='https://app.statdx.com/')
+    parser.add_argument('--profile', '-p', default='playwright_profile')
+    parser.add_argument('--output-dir', '-o', default='xhr_captured')
+    parser.add_argument('--headless', action='store_true')
+    parser.add_argument('--continuous', action='store_true', help='Keep listening and saving as you browse')
+    parser.add_argument('--no-prompt', action='store_true', help='Start listening immediately without waiting for Enter')
+    parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
+    parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
+    parser.add_argument('--post-login-selector', default='#ds-app')
+    parser.add_argument('--capture-types', default='xhr,fetch,document,other', help='Comma-separated resource types to capture (empty = all)')
+    args = parser.parse_args()
+
+    output = Path(args.output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    index_path = output / 'capture_index.jsonl'
+
+    username = args.username or os.getenv('STATDX_USERNAME')
+    password = args.password or os.getenv('STATDX_PASSWORD')
+    capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()}
+
+    with sync_playwright() as p:
+        # launch persistent context
+        try:
+            context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel='chrome')
+        except Exception:
+            context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
+
+        page = context.new_page()
+        # NOTE: don't navigate until we've attached listeners so we don't miss
+        # early requests from the initial load.
+
+        # response handler (define before we attach listeners)
+        def on_response(resp):
+            """Safe response handler: reads response bodies but tolerates browser/context close.
+
+            Playwright can raise TargetClosedError or asyncio.CancelledError when the
+            browser or context closes while we're trying to read a body. Those are
+            normal during shutdown and should not produce noisy tracebacks. This
+            handler therefore catches those and returns quietly.
+            """
+            logger.debug(f'Response: {getattr(resp, "status", "?")} {getattr(resp, "url", "?")}')
+            try:
+                req = resp.request
+                rtype = (req.resource_type or '').lower()
+                # Log every response minimally so user sees activity
+                logger.info('Response event: %s %s %s', getattr(resp, 'status', '?'), rtype, req.url)
+
+                # write debug line to disk so we can verify events even if console is quiet
+                try:
+                    with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                        dbg.write(f"{now_ts()}\t{getattr(resp, 'status', '?')}\t{rtype}\t{req.url}\n")
+                except Exception:
+                    pass
+
+                # If capture_types is empty, capture everything. Otherwise check membership.
+                allow = False
+                if not capture_types:
+                    allow = True
+                elif rtype in capture_types:
+                    allow = True
+                # Heuristic: always capture document content endpoints we care about
+                if '/document/content/' in req.url or '/document/summary/' in req.url:
+                    allow = True
+                if not allow:
+                    return
+
+                url = req.url
+                ts = now_ts()
+                status = getattr(resp, 'status', None)
+                req_headers = dict(req.headers)
+                resp_headers = dict(resp.headers)
+
+                meta = {
+                    'url': url,
+                    'method': req.method,
+                    'resource_type': rtype,
+                    'status': status,
+                    'timestamp': ts,
+                    'request_headers': req_headers,
+                    'response_headers': resp_headers,
+                }
+
+                # request post data if any
+                try:
+                    post = req.post_data
+                    meta['request_post_data'] = post
+                except Exception:
+                    meta['request_post_data'] = None
+
+                ctype = (resp_headers.get('content-type') or '').lower()
+                safe = sanitize(url.replace('https://', '').replace('http://', ''))
+                h = hashlib.sha1((url + ts).encode('utf-8')).hexdigest()[:8]
+
+                body_path = None
+                excerpt = None
+                # try textual
+                try:
+                    if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype:
+                        try:
+                            txt = resp.text()
+                        except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,):
+                            logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '<unknown>'))
+                            return
+                        # pretty-print JSON if possible
+                        pretty = try_pretty_json(txt)
+                        if pretty is not None:
+                            fname = f"{safe}_{h}_{ts}.json"
+                            body_path = str(output / fname)
+                            save_text(output / fname, pretty)
+                            excerpt = pretty[:800]
+                        else:
+                            fname = f"{safe}_{h}_{ts}.txt"
+                            body_path = str(output / fname)
+                            save_text(output / fname, txt)
+                            excerpt = txt[:800]
+                    else:
+                        try:
+                            data = resp.body()
+                        except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,):
+                            logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '<unknown>'))
+                            return
+                        ext = guess_ext(ctype)
+                        fname = f"{safe}_{h}_{ts}.{ext}"
+                        body_path = str(output / fname)
+                        save_binary(output / fname, data)
+                except Exception as e:
+                    try:
+                        # fallback to binary
+                        try:
+                            data = resp.body()
+                        except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,):
+                            logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '<unknown>'))
+                            return
+                        ext = guess_ext(ctype)
+                        fname = f"{safe}_{h}_{ts}.{ext}"
+                        body_path = str(output / fname)
+                        save_binary(output / fname, data)
+                    except Exception:
+                        logger.debug('Failed to save body for %s: %s', url, e)
+
+                meta['response_body_file'] = body_path
+                meta['response_excerpt'] = excerpt
+
+                meta_name = f"{safe}_{h}_{ts}.json"
+                meta_path = output / meta_name
+                save_text(meta_path, json.dumps(meta, ensure_ascii=False, indent=2))
+
+                # append index
+                try:
+                    with open(index_path, 'a', encoding='utf-8') as idx:
+                        idx.write(json.dumps({'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(meta_path), 'excerpt': excerpt}, ensure_ascii=False) + '\n')
+                except Exception:
+                    logger.exception('Failed to write index entry')
+
+                logger.info('Captured %s %s -> %s', rtype, url, meta_path.name)
+            except Exception as e:
+                # suppress noisy Playwright shutdown traces
+                if isinstance(e, TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,)) or \
+                   (hasattr(e, 'args') and any('Target page, context or browser has been closed' in str(a) for a in e.args)):
+                    logger.debug('Response handler aborted: target closed')
+                    return
+                logger.exception('Error in response handler')
+
+        # helper to attach listeners to a page
+        def attach_listeners_to_page(pg):
+            try:
+                pg.on('response', on_response)
+                # log requests and finished requests for debug
+                def on_request(req):
+                    try:
+                        with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                            dbg.write(f"{now_ts()}\tREQUEST\t{req.method}\t{req.url}\n")
+                    except Exception:
+                        pass
+
+                def on_request_finished(req):
+                    try:
+                        with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                            dbg.write(f"{now_ts()}\tREQUEST_FINISHED\t{req.method}\t{req.url}\n")
+                    except Exception:
+                        pass
+
+                pg.on('request', on_request)
+                pg.on('requestfinished', on_request_finished)
+                # also log request failures for visibility
+                def on_req_failed(req):
+                    try:
+                        logger.warning('Request failed: %s %s', req.failure, req.url)
+                    except Exception:
+                        pass
+                pg.on('requestfailed', on_req_failed)
+                logger.debug('Attached listeners to page: %s', pg.url)
+            except Exception:
+                logger.exception('Failed to attach listeners to page')
+
+            try:
+                # navigation and lifecycle events
+                def on_navigate():
+                    try:
+                        with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                            dbg.write(f"{now_ts()}\tNAVIGATE\t{pg.url}\n")
+                    except Exception:
+                        pass
+                pg.on('load', lambda: on_navigate())
+                pg.on('domcontentloaded', lambda: on_navigate())
+                pg.on('framenavigated', lambda frame: on_navigate())
+            except Exception:
+                pass
+
+        # attach to any existing pages in the context
+        try:
+            for pg in context.pages:
+                attach_listeners_to_page(pg)
+        except Exception:
+            logger.debug('No existing pages to attach')
+
+        # attach to future pages
+        try:
+            context.on('page', lambda new_page: attach_listeners_to_page(new_page))
+        except Exception:
+            logger.debug('Could not attach context.page listener')
+
+        # also attach at context level to catch responses not tied to a single page
+        try:
+            context.on('response', lambda resp: on_response(resp))
+        except Exception:
+            logger.debug('Context-level response listener not available')
+
+        # add context-level request listeners too (some fetches/service-workers
+        # may appear at the context level rather than page)
+        try:
+            def _ctx_request(req):
+                try:
+                    with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                        dbg.write(f"{now_ts()}\tCTX_REQUEST\t{req.method}\t{req.url}\n")
+                except Exception:
+                    pass
+            def _ctx_req_finished(req):
+                try:
+                    with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                        dbg.write(f"{now_ts()}\tCTX_REQUEST_FINISHED\t{req.method}\t{req.url}\n")
+                except Exception:
+                    pass
+            context.on('request', lambda r: _ctx_request(r))
+            context.on('requestfinished', lambda r: _ctx_req_finished(r))
+        except Exception:
+            logger.debug('Context-level request listeners not available')
+
+        # try auto-login
+        if username and password:
+            attempt_autologin(page, username, password, args.post_login_selector)
+
+        # navigate after listeners attached so we capture initial load
+        try:
+            logger.info('Opening %s', args.url)
+            page.goto(args.url)
+        except Exception:
+            logger.debug('Initial navigation failed or was skipped')
+
+        # If interactive and not no-prompt, let user finish manual login/navigation
+        if not args.no_prompt:
+            try:
+                input('When you have logged in and are ready, press Enter to start passive capture...\n')
+            except Exception:
+                pass
+
+        logger.info(f'Starting passive response capture (types: {", ".join(sorted(capture_types))})')
+
+        # attach the response handler to the initial page
+        page.on('response', on_response)
+
+        logger.info('Passive listener attached. Continue browsing in the opened browser window.')
+
+        if args.continuous:
+            try:
+                while True:
+                    time.sleep(1)
+            except KeyboardInterrupt:
+                logger.info('Interrupted by user; finishing and closing browser')
+        else:
+            # one-shot: wait a short while to gather responses then exit
+            time.sleep(5)
+
+        try:
+            context.close()
+        except Exception:
+            pass
+
+
+if __name__ == '__main__':
+    main()

scrapers/capture_passive_playwright_async.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+"""Async passive XHR/fetch capture using Playwright's async API.
+
+This script mirrors the sync version but uses async/await to avoid blocking
+the event loop. It attaches listeners before navigating and prints a small
+heartbeat that shows how many responses we've captured so you can see
+activity live.
+
+Usage:
+  python scrapers/capture_passive_playwright_async.py --output-dir xhr_captured_async --continuous
+"""
+import argparse
+import asyncio
+import json
+import mimetypes
+import hashlib
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+from dotenv import load_dotenv
+load_dotenv()
+
+try:
+    from playwright.async_api import async_playwright
+except Exception:
+    raise
+
+
+def sanitize(s: str) -> str:
+    out = []
+    for ch in s:
+        if ch.isalnum() or ch in ('-', '_', '.'):
+            out.append(ch)
+        else:
+            out.append('_')
+    name = ''.join(out)
+    if len(name) > 180:
+        name = name[:120] + '_' + hashlib.sha1(s.encode('utf-8')).hexdigest()[:8]
+    return name
+
+
+def guess_ext(ctype: str) -> str:
+    if not ctype:
+        return 'bin'
+    mt = ctype.split(';')[0].strip()
+    ext = mimetypes.guess_extension(mt)
+    if ext:
+        return ext.lstrip('.')
+    if mt.startswith('image/'):
+        return mt.split('/')[1]
+    if mt == 'application/json':
+        return 'json'
+    if mt == 'text/html':
+        return 'html'
+    if mt.startswith('text/'):
+        return 'txt'
+    return 'bin'
+
+
+def now_ts():
+    return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+
+
+def save_text(path: Path, text: str):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write(text)
+
+
+def save_binary(path: Path, data: bytes):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'wb') as f:
+        f.write(data)
+
+
+async def run(args):
+    output = Path(args.output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    index_path = output / 'capture_index.jsonl'
+
+    capture_count = 0
+    capture_count_lock = asyncio.Lock()
+
+    async def heartbeat():
+        while True:
+            await asyncio.sleep(5)
+            async with capture_count_lock:
+                print(f"[heartbeat] captured={capture_count}")
+
+    async with async_playwright() as p:
+        browser_type = p.chromium
+        # Launch persistent context
+        if args.channel:
+            try:
+                context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel)
+            except Exception:
+                print(f'Launch with channel {args.channel} failed; falling back to bundled Chromium')
+                context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
+        else:
+            context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
+
+        page = await context.new_page()
+
+        async def on_response(resp):
+            nonlocal capture_count
+            try:
+                req = resp.request
+                rtype = (req.resource_type or '').lower()
+
+                # write quick debug line
+                try:
+                    with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                        dbg.write(f"{now_ts()}\tRESP\t{resp.status}\t{rtype}\t{req.url}\n")
+                except Exception:
+                    pass
+
+                allow = False
+                if not args.capture_types:
+                    allow = True
+                elif rtype in args.capture_types:
+                    allow = True
+                if '/document/content/' in req.url or '/document/summary/' in req.url:
+                    allow = True
+                if not allow:
+                    return
+
+                url = req.url
+                ts = now_ts()
+                resp_headers = dict(resp.headers)
+                ctype = (resp_headers.get('content-type') or '').lower()
+
+                safe = sanitize(url.replace('https://', '').replace('http://', ''))
+                h = hashlib.sha1((url + ts).encode('utf-8')).hexdigest()[:8]
+
+                body_path = None
+                excerpt = None
+
+                try:
+                    if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype:
+                        try:
+                            txt = await resp.text()
+                        except asyncio.CancelledError:
+                            return
+                        except Exception as e:
+                            if 'Target page, context or browser has been closed' in str(e):
+                                return
+                            raise
+                        # try pretty JSON
+                        try:
+                            parsed = json.loads(txt)
+                            pretty = json.dumps(parsed, ensure_ascii=False, indent=2)
+                            fname = f"{safe}_{h}_{ts}.json"
+                            body_path = str(output / fname)
+                            save_text(output / fname, pretty)
+                            excerpt = pretty[:800]
+                        except Exception:
+                            fname = f"{safe}_{h}_{ts}.txt"
+                            body_path = str(output / fname)
+                            save_text(output / fname, txt)
+                            excerpt = txt[:800]
+                    else:
+                        try:
+                            data = await resp.body()
+                        except asyncio.CancelledError:
+                            return
+                        except Exception as e:
+                            if 'Target page, context or browser has been closed' in str(e):
+                                return
+                            raise
+                        ext = guess_ext(ctype)
+                        # Put images into an images/ subdirectory, other binaries into assets/
+                        if ctype.startswith('image/'):
+                            subdir = output / 'images'
+                            subdir.mkdir(parents=True, exist_ok=True)
+                            fname = f"{safe}_{h}_{ts}.{ext}"
+                            body_path = str(subdir / fname)
+                            save_binary(subdir / fname, data)
+                        else:
+                            subdir = output / 'assets'
+                            subdir.mkdir(parents=True, exist_ok=True)
+                            fname = f"{safe}_{h}_{ts}.{ext}"
+                            body_path = str(subdir / fname)
+                            save_binary(subdir / fname, data)
+                except Exception as e:
+                    # final fallback: try binary
+                    try:
+                        data = await resp.body()
+                        ext = guess_ext(ctype)
+                        if ctype.startswith('image/'):
+                            subdir = output / 'images'
+                            subdir.mkdir(parents=True, exist_ok=True)
+                            fname = f"{safe}_{h}_{ts}.{ext}"
+                            body_path = str(subdir / fname)
+                            save_binary(subdir / fname, data)
+                        else:
+                            subdir = output / 'assets'
+                            subdir.mkdir(parents=True, exist_ok=True)
+                            fname = f"{safe}_{h}_{ts}.{ext}"
+                            body_path = str(subdir / fname)
+                            save_binary(subdir / fname, data)
+                    except Exception:
+                        # skip if still failing
+                        return
+
+                meta = {
+                    'url': url,
+                    'resource_type': rtype,
+                    'status': resp.status,
+                    'timestamp': ts,
+                    'response_headers': resp_headers,
+                    'response_body_file': body_path,
+                    'response_excerpt': excerpt,
+                }
+                meta_name = f"{safe}_{h}_{ts}.meta.json"
+                save_text(output / meta_name, json.dumps(meta, ensure_ascii=False, indent=2))
+                try:
+                    with open(index_path, 'a', encoding='utf-8') as idx:
+                        idx.write(json.dumps({'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(output / meta_name), 'excerpt': excerpt}, ensure_ascii=False) + '\n')
+                except Exception:
+                    pass
+
+                async with capture_count_lock:
+                    capture_count += 1
+
+            except Exception:
+                # swallow unexpected exceptions from handler so event loop stays healthy
+                try:
+                    with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
+                        dbg.write(f"{now_ts()}\tRESP_HANDLER_ERROR\n")
+                except Exception:
+                    pass
+
+        # helpers to attach to page
+        def attach_page_listeners(pg):
+            pg.on('response', lambda r: asyncio.create_task(on_response(r)))
+            pg.on('request', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tREQUEST\t{req.method}\t{req.url}\n"))
+            pg.on('requestfinished', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tREQUEST_FINISHED\t{req.method}\t{req.url}\n"))
+
+        # attach listeners to existing pages
+        for pg in context.pages:
+            attach_page_listeners(pg)
+
+        # attach to future pages
+        context.on('page', lambda new_page: attach_page_listeners(new_page))
+
+        # context-level response and request listeners
+        context.on('response', lambda r: asyncio.create_task(on_response(r)))
+        context.on('request', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tCTX_REQUEST\t{req.method}\t{req.url}\n"))
+
+        # Start heartbeat
+        hb_task = asyncio.create_task(heartbeat())
+
+        # Navigate after listeners attached
+        try:
+            await page.goto(args.url)
+        except Exception:
+            pass
+
+        # If interactive, let user log in; otherwise start capture immediately
+        if not args.no_prompt:
+            print('When you have logged in in the opened browser, press Enter here to continue and capture...')
+            try:
+                await asyncio.get_event_loop().run_in_executor(None, input)
+            except Exception:
+                pass
+
+        print(f'Starting async passive capture (types: {args.capture_types})')
+
+        try:
+            if args.continuous:
+                # keep running until KeyboardInterrupt
+                while True:
+                    await asyncio.sleep(1)
+            else:
+                await asyncio.sleep(5)
+        except KeyboardInterrupt:
+            print('Interrupted; closing...')
+        finally:
+            hb_task.cancel()
+            try:
+                await context.close()
+            except Exception:
+                pass
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--url', default='https://app.statdx.com/')
+    parser.add_argument('--profile', default='playwright_profile')
+    parser.add_argument('--output-dir', default='xhr_captured_async')
+    parser.add_argument('--headless', action='store_true')
+    parser.add_argument('--continuous', action='store_true')
+    parser.add_argument('--no-prompt', action='store_true')
+    parser.add_argument('--channel', default=os.getenv('PLAYWRIGHT_CHROME_CHANNEL', 'chrome'))
+    parser.add_argument('--capture-types', default='xhr,fetch,document,other')
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    # normalize capture types to a set
+    args.capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()}
+    try:
+        asyncio.run(run(args))
+    except KeyboardInterrupt:
+        pass

scrapers/capture_with_cdp.py

@@ -88,6 +88,8 @@ def main():
     parser.add_argument("--headless", action="store_true", help="Run headless")
     parser.add_argument("--capture-wait", type=int, default=8, help="Seconds to wait after reload to capture XHRs")
     parser.add_argument("--keep-open", action="store_true", help="Keep browser open after capture")
+    parser.add_argument("--continuous", action="store_true", help="Continue capturing requests as you browse until interrupted")
+    parser.add_argument("--no-prompt", action="store_true", help="Do not prompt before starting capture (useful for automated runs)")
     parser.add_argument("--username", help="STATdx username (or use STATDX_USERNAME env var)")
     parser.add_argument("--password", help="STATdx password (or use STATDX_PASSWORD env var)")
     parser.add_argument("--post-login-selector", default="#ds-app", help="Selector to wait for after login")
@@ -101,23 +103,31 @@ def main():
     with sync_playwright() as p:
         # try to use installed Chrome first
         try:
-            browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome")
+            context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome")
-            page = browser.pages[0] if browser.pages else browser.new_page()
+            page = context.pages[0] if context.pages else context.new_page()
         except Exception as e:
             logger.warning("Failed to launch Chrome channel: {}. Falling back to chromium bundle.", e)
-            browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
+            context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
-            page = browser.pages[0] if browser.pages else browser.new_page()
+            page = context.pages[0] if context.pages else context.new_page()
 
         # attach CDP session to the page
         try:
-            session = browser.new_cdp_session(page)
+            session = context.new_cdp_session(page)
         except Exception as e:
             logger.error("Failed to create CDP session: {}", e)
-            browser.close()
+            try:
+                context.close()
+            except Exception:
+                pass
             return
 
         logger.info("Enabling Network domain on CDP")
         session.send("Network.enable")
+        # Disable browser cache to force network requests (helps capture cached items)
+        try:
+            session.send("Network.setCacheDisabled", {"cacheDisabled": True})
+        except Exception:
+            logger.debug("Could not disable cache via CDP; continuing without cache disable")
 
         seen = set()
 
@@ -137,7 +147,9 @@ def main():
                     return
                 seen.add(key)
 
-                # attempt to get body
+                # attempt to get body; sometimes responseReceived doesn't allow
+                # getResponseBody immediately, so we'll try but also rely on
+                # a loadingFinished handler below.
                 body = None
                 base64_encoded = False
                 try:
@@ -146,7 +158,7 @@ def main():
                         body = result.get("body")
                         base64_encoded = bool(result.get("base64Encoded"))
                 except Exception as e:
-                    logger.debug("Network.getResponseBody failed for {}: {}", url, e)
+                    logger.debug("Network.getResponseBody failed (responseReceived) for {}: {}", url, e)
 
                 # fallback: if no body, skip writing but still store metadata
                 # build filenames
@@ -196,7 +208,54 @@ def main():
         # subscribe to CDP event
         session.on("Network.responseReceived", on_response_received)
 
-        logger.info("Opening page: {}. Please log in if needed, then press ENTER to start capture.", args.url)
+        # loadingFinished is a good fallback: some responses only have bodies
+        # available after loadingFinished. We'll attempt to fetch body there
+        # if not previously saved for the same requestId.
+        def on_loading_finished(params):
+            try:
+                requestId = params.get("requestId")
+                # call the same body retrieval logic via a small wrapper
+                # create a fake params dict to reuse code
+                fake = {"requestId": requestId, "response": {}}
+                # attempt to get the body now
+                try:
+                    result = session.send("Network.getResponseBody", {"requestId": requestId})
+                except Exception:
+                    result = None
+                if result:
+                    # build a minimal params for saving using response info from CDP
+                    resp = result
+                    # We don't have the URL here; try to fetch it via CDP - getRequestPostData isn't available.
+                    # Instead, reuse on_response_received's behavior by calling getResponseBody and writing
+                    # files directly here when possible.
+                    try:
+                        body = result.get("body")
+                        base64_encoded = bool(result.get("base64Encoded"))
+                        # fetch additional response info via Network.getResponseBody? we already have body.
+                        # write a minimal metadata file
+                        ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+                        safe = sanitize_fn(requestId)
+                        h = hashlib.sha1((requestId + str(time.time())).encode("utf-8")).hexdigest()[:8]
+                        ext = "bin"
+                        try:
+                            data = base64.b64decode(body) if base64_encoded else body.encode("utf-8")
+                            body_filename = f"{safe}_{h}_{ts}.{ext}"
+                            write_file(out_dir / body_filename, data)
+                            meta = {"requestId": requestId, "timestamp": ts, "response_body_file": str(out_dir / body_filename)}
+                            meta_filename = f"{safe}_{h}_{ts}.json"
+                            with open(out_dir / meta_filename, "w", encoding="utf-8") as mf:
+                                json.dump(meta, mf, ensure_ascii=False, indent=2)
+                            logger.info("Saved body via loadingFinished: {}", body_filename)
+                        except Exception:
+                            pass
+                    except Exception:
+                        pass
+            except Exception:
+                logger.exception("Error in loadingFinished handler")
+
+        session.on("Network.loadingFinished", on_loading_finished)
+
+        logger.info("Opening page: {}.", args.url)
         page.goto(args.url)
 
         # Automatic login if credentials provided
@@ -256,28 +315,46 @@ def main():
                 except Exception:
                     logger.exception('Automatic login attempt failed')
 
+        # Start capture: either prompt the user or begin immediately depending on flags
+        try:
+            if not args.no_prompt and not args.continuous:
+                try:
                     input("After logging in and navigating to the page you want to capture, press Enter to begin capture...\n")
+                except Exception:
+                    # non-interactive; proceed
+                    pass
 
             logger.info("Reloading page to trigger XHRs and network activity")
+            try:
                 page.reload()
+            except Exception:
+                logger.exception('Failed to reload page; continuing capture')
 
             # wait a bit to let XHRs fire and be processed by CDP
             wait = args.capture_wait
-        logger.info("Waiting {} seconds to collect responses...", wait)
+            logger.info("Waiting {} seconds to collect initial responses...", wait)
             time.sleep(wait)
 
-        logger.info("Capture pass complete. Metadata and bodies written to {}", out_dir)
+            logger.info("Initial capture pass complete. Metadata and bodies written to {}", out_dir)
 
-        if args.keep_open:
+            if args.continuous:
-            logger.info("Keeping browser open. Close manually when done.")
+                logger.info("Continuous capture enabled: saving responses as you browse. Press Ctrl+C to stop.")
                 try:
                     while True:
                         time.sleep(1)
                 except KeyboardInterrupt:
-                logger.info("Received interrupt, closing.")
+                    logger.info("Interrupted by user; closing browser and exiting")
-                browser.close()
             else:
-            browser.close()
+                logger.info("One-shot capture complete.")
+
+        finally:
+            try:
+                context.close()
+            except Exception:
+                try:
+                    context.close()
+                except Exception:
+                    pass
 
 
 if __name__ == "__main__":

scrapers/save_page_snapshots.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+Save full page HTML snapshots after page loads and periodically while you browse.
+
+This script launches a persistent Playwright profile (so you can log in),
+optionally attempts automatic login, then either performs a one-shot snapshot
+or continues saving snapshots as you browse. Snapshots are de-duplicated by
+content hash to avoid writing identical files repeatedly.
+
+Usage examples:
+  # manual login, one-shot snapshot after reload
+  python scrapers/save_page_snapshots.py --url https://app.statdx.com/ --output-dir xhr_captured
+
+  # auto-login and continuous snapshots while you browse
+  python scrapers/save_page_snapshots.py --continuous --username you --password secret --output-dir xhr_captured
+
+"""
+from pathlib import Path
+import argparse
+import time
+import hashlib
+import json
+import mimetypes
+import os
+from datetime import datetime, timezone
+
+try:
+    from loguru import logger
+except Exception:
+    import logging
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+    logger = logging.getLogger("save_page_snapshots")
+
+from playwright.sync_api import sync_playwright
+from dotenv import load_dotenv
+
+# load .env
+load_dotenv()
+
+
+def sanitize_filename(s: str) -> str:
+    safe = []
+    for ch in s:
+        if ch.isalnum() or ch in ('-', '_', '.'):
+            safe.append(ch)
+        else:
+            safe.append('-')
+    name = ''.join(safe)
+    if len(name) > 180:
+        h = hashlib.sha1(s.encode('utf-8')).hexdigest()[:8]
+        name = name[:120] + '-' + h
+    return name
+
+
+def write_text(path: Path, text: str):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write(text)
+
+
+def write_json(path: Path, obj):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'w', encoding='utf-8') as f:
+        json.dump(obj, f, ensure_ascii=False, indent=2)
+
+
+def now_ts():
+    return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+
+
+def snapshot_page(page, out_dir: Path, index_path: Path, last_hashes: dict, min_interval: float = 3.0):
+    """Capture page content and write HTML + meta if content changed or enough time passed."""
+    try:
+        url = page.url
+    except Exception:
+        url = 'unknown'
+
+    try:
+        content = page.content()
+    except Exception as e:
+        logger.exception('Failed to get page.content(): {}', e)
+        return None
+
+    h = hashlib.sha1(content.encode('utf-8')).hexdigest()
+    prev = last_hashes.get(url)
+    ts = now_ts()
+
+    # If unchanged and we saved recently, skip
+    if prev and prev.get('hash') == h and (time.time() - prev.get('ts', 0)) < min_interval:
+        logger.debug('Snapshot skipped (no change) for {}', url)
+        return None
+
+    # prepare filenames
+    name = sanitize_filename(url.replace('https://', '').replace('http://', ''))
+    short = h[:8]
+    html_name = f"{name}_{short}_{ts}.html"
+    meta_name = f"{name}_{short}_{ts}.json"
+
+    html_path = out_dir / html_name
+    meta_path = out_dir / meta_name
+
+    try:
+        write_text(html_path, content)
+    except Exception as e:
+        logger.exception('Failed to write HTML snapshot: {}', e)
+        return None
+
+    meta = {
+        'url': url,
+        'timestamp': ts,
+        'file': str(html_path),
+        'sha1': h,
+        'title': (page.title() if page else None),
+    }
+    try:
+        write_json(meta_path, meta)
+    except Exception:
+        logger.exception('Failed to write meta file')
+
+    # append to index
+    try:
+        entry = {'url': url, 'timestamp': ts, 'html_file': str(html_path), 'meta_file': str(meta_path), 'sha1': h}
+        with open(index_path, 'a', encoding='utf-8') as idx:
+            idx.write(json.dumps(entry, ensure_ascii=False) + '\n')
+    except Exception:
+        logger.exception('Failed to append to index')
+
+    last_hashes[url] = {'hash': h, 'ts': time.time()}
+    logger.info('Saved snapshot: {} ({} bytes)', html_path.name, len(content))
+    return html_path
+
+
+def attempt_autologin(page, username: str, password: str, post_login_selector: str, wait_after: float = 3.0):
+    if not username or not password:
+        return False
+    logger.info('Attempting automatic login')
+    # Common selectors for STATdx and similar pages
+    user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
+    pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
+
+    user_sel = None
+    for sel in user_selectors:
+        try:
+            if page.query_selector(sel):
+                user_sel = sel
+                break
+        except Exception:
+            continue
+
+    pass_sel = None
+    for sel in pass_selectors:
+        try:
+            if page.query_selector(sel):
+                pass_sel = sel
+                break
+        except Exception:
+            continue
+
+    if not user_sel or not pass_sel:
+        logger.warning('Could not find username/password fields for auto-login')
+        return False
+
+    try:
+        page.fill(user_sel, username)
+        time.sleep(0.1)
+        page.fill(pass_sel, password)
+        time.sleep(0.1)
+        # try submit
+        submit_selectors = ['button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")']
+        clicked = False
+        for s in submit_selectors:
+            try:
+                btn = page.query_selector(s)
+                if btn:
+                    btn.click()
+                    clicked = True
+                    logger.info('Clicked submit button using selector: {}', s)
+                    break
+            except Exception:
+                continue
+        if not clicked:
+            page.press(pass_sel, 'Enter')
+
+        # wait for post-login selector
+        try:
+            if post_login_selector:
+                page.wait_for_selector(post_login_selector, timeout=10000)
+        except Exception:
+            logger.info('Post-login selector not found within timeout')
+        time.sleep(wait_after)
+        return True
+    except Exception:
+        logger.exception('Auto-login failed')
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--url', '-u', default='https://app.statdx.com/')
+    parser.add_argument('--profile', '-p', default='playwright_profile')
+    parser.add_argument('--output-dir', '-o', default='xhr_captured')
+    parser.add_argument('--headless', action='store_true')
+    parser.add_argument('--continuous', action='store_true', help='Keep capturing while you browse until interrupted')
+    parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait before initial snapshot')
+    parser.add_argument('--poll-interval', type=float, default=5.0, help='Periodic snapshot interval when continuous')
+    parser.add_argument('--min-interval', type=float, default=3.0, help='Minimum seconds between snapshots for same URL')
+    parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
+    parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
+    parser.add_argument('--post-login-selector', default='#ds-app')
+    parser.add_argument('--no-prompt', action='store_true')
+    args = parser.parse_args()
+
+    out_dir = Path(args.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    index_path = out_dir / 'snapshot_index.jsonl'
+
+    username = args.username or os.getenv('STATDX_USERNAME')
+    password = args.password or os.getenv('STATDX_PASSWORD')
+
+    last_hashes = {}
+
+    with sync_playwright() as p:
+        try:
+            context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel='chrome')
+        except Exception:
+            context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
+
+        page = context.new_page()
+        logger.info('Opening %s', args.url)
+        page.goto(args.url)
+
+        # attempt auto-login
+        if username and password:
+            attempt_autologin(page, username, password, args.post_login_selector)
+
+        # wait before initial snapshot to let in-flight XHRs populate DOM
+        logger.info('Waiting %.1f sec before initial snapshot', args.capture_wait)
+        time.sleep(args.capture_wait)
+
+        # initial snapshot
+        snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
+
+        if args.continuous:
+            logger.info('Continuous snapshot mode: saving on navigation and every %.1f sec', args.poll_interval)
+
+            # on navigation events, take a snapshot (debounced by min_interval)
+            def on_load():
+                try:
+                    snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
+                except Exception:
+                    logger.exception('Error during on_load snapshot')
+
+            page.on('load', lambda: on_load())
+            page.on('domcontentloaded', lambda: on_load())
+            page.on('framenavigated', lambda frame: on_load())
+
+            try:
+                # periodic polling snapshot
+                while True:
+                    time.sleep(args.poll_interval)
+                    snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
+            except KeyboardInterrupt:
+                logger.info('Interrupted by user, closing')
+        else:
+            # one-shot: save a final snapshot after a short delay to capture dynamic updates
+            try:
+                time.sleep(1.0)
+                snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
+            except Exception:
+                logger.exception('Error taking final snapshot')
+
+        try:
+            context.close()
+        except Exception:
+            pass
+
+
+if __name__ == '__main__':
+    main()

scrapers/statdx_live_capture.py

@@ -22,7 +22,6 @@ Notes / limitations:
 
 import os
 import argparse
-import logging
 import time
 import threading
 import sys
@@ -32,9 +31,12 @@ from urllib.parse import urlparse
 from dotenv import load_dotenv
 import mimetypes
 
+from loguru import logger
+
 load_dotenv()
 
-logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logger.remove()
+logger.add(sys.stderr, level="INFO")
 
 
 def sanitize_filename(url: str) -> str:
@@ -79,7 +81,7 @@ def auto_scroll(page, step_ms: int = 50, step_px: int = 300):
             step_ms,
         )
     except Exception:
-        logging.exception('auto_scroll failed')
+        logger.exception('auto_scroll failed')
 
 
 def save_response(resp, out_dir: str):
@@ -112,7 +114,7 @@ def save_response(resp, out_dir: str):
         filename_base = sanitize_filename(url)
         out_path = os.path.join(out_dir, filename_base + ext)
 
-        logging.info('Saving %s -> %s (content-type: %s)', url, out_path, ctype.split(';')[0])
+        logger.info(f'Saving {url} -> {out_path} (content-type: {ctype.split(";")[0]})')
 
         if save_as_text:
             try:
@@ -136,7 +138,7 @@ def save_response(resp, out_dir: str):
 
         return True
     except Exception as e:
-        logging.exception('Failed to save response: %s', e)
+        logger.exception(f'Failed to save response: {e}')
         return False
 
 
@@ -147,21 +149,21 @@ def save_page_snapshot(page, out_dir: str, reason: str = ''):
         filename_base = sanitize_filename(url)
         safe_reason = ''.join(c if c.isalnum() or c in ('-', '_') else '_' for c in reason)[:40]
         out_path = os.path.join(out_dir, f"snapshot_{filename_base}_{ts}_{safe_reason}.html")
-        logging.info('Saving page snapshot: %s -> %s', url, out_path)
+        logger.info(f'Saving page snapshot: {url} -> {out_path}')
         content = page.content()
         with open(out_path, 'w', encoding='utf-8') as fh:
             fh.write(content)
         # Screenshots are disabled (not saving visual snapshots per user request)
         return True
     except Exception:
-        logging.exception('Failed to save page snapshot')
+        logger.exception('Failed to save page snapshot')
         return False
 
 
 def main():
     parser = argparse.ArgumentParser(description='Live-capture browsing responses using Playwright')
     parser.add_argument('--output', '-o', default='captured', help='Directory to save captured responses')
-    parser.add_argument('--profile', '-p', default='.playwright_profile', help='Persistent profile directory')
+    parser.add_argument('--profile', '-p', default='playwright_profile', help='Persistent profile directory')
     parser.add_argument('--domain', '-d', default=os.getenv('STATDX_DOMAIN', 'app.statdx.com'), help='Only capture responses whose URL contains this domain')
     parser.add_argument('--headless', action='store_true', help='Run headless (no visible browser)')
     parser.add_argument('--snapshot-delay', type=float, default=0.5, help='Seconds to wait after an XHR before saving a snapshot')
@@ -182,7 +184,7 @@ def main():
     with sync_playwright() as p:
         # Use persistent context so cookies/localStorage persist between runs
         browser_type = p.chromium
-        logging.info('Launching Chromium (headless=%s) with profile: %s, channel=%s', args.headless, args.profile, args.channel)
+        logger.info(f'Launching Chromium (headless={args.headless}) with profile: {args.profile}, channel={args.channel}')
         if args.channel:
             context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel)
         else:
@@ -198,6 +200,7 @@ def main():
             pass
 
         def handle_response(resp):
+            logger.debug(f'RESPONSE: {resp.url} ({resp.request.resource_type})')
             try:
                 if args.domain not in resp.url:
                     return
@@ -214,12 +217,12 @@ def main():
                             try:
                                 auto_scroll(page)
                             except Exception:
-                                logging.exception('auto-scroll failed')
+                                logger.exception('auto-scroll failed')
                         save_page_snapshot(page, args.output, reason=rtype)
                     except Exception:
-                        logging.exception('snapshot after response failed')
+                        logger.exception('snapshot after response failed')
             except Exception:
-                logging.exception('Error in response handler')
+                logger.exception('Error in response handler')
 
 
         page.on('response', handle_response)
@@ -227,19 +230,19 @@ def main():
         # Log console messages and page errors to help diagnose missing content
         def on_console(msg):
             try:
-                logging.info('PAGE LOG [%s] %s', msg.type, msg.text)
+                logger.info(f'PAGE LOG [{msg.type}] {msg.text}')
             except Exception:
-                logging.info('PAGE LOG: %s', msg)
+                logger.info(f'PAGE LOG: {msg}')
 
         def on_page_error(exc):
-            logging.error('PAGE ERROR: %s', exc)
+            logger.error(f'PAGE ERROR: {exc}')
 
         page.on('console', on_console)
         page.on('pageerror', on_page_error)
 
-        logging.info('Open the browser window and interact with the site. Captured responses will be saved to: %s', args.output)
+        logger.info(f'Open the browser window and interact with the site. Captured responses will be saved to: {args.output}')
-        logging.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
+        logger.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
-        logging.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
+        logger.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
 
         # Navigate to the domain root so you can log in manually if needed
         start_url = f'https://{args.domain}/'
@@ -247,27 +250,27 @@ def main():
         # Optionally wait for a selector that indicates the page finished rendering
         if args.wait_for_selector:
             try:
-                logging.info('Waiting for selector: %s', args.wait_for_selector)
+                logger.info(f'Waiting for selector: {args.wait_for_selector}')
                 page.wait_for_selector(args.wait_for_selector, timeout=20000)
             except Exception:
-                logging.exception('Waiting for selector timed out')
+                logger.exception('Waiting for selector timed out')
 
         def stdin_watcher():
             # Run in a background thread so the main thread can keep the browser running
-            logging.debug('stdin watcher started')
+            logger.debug('stdin watcher started')
             for line in sys.stdin:
                 cmd = line.strip().lower()
                 if cmd == '' or cmd == 's' or cmd == 'snapshot' or cmd == 'save':
-                    logging.info('Manual snapshot requested via stdin')
+                    logger.info('Manual snapshot requested via stdin')
                     try:
                         if args.auto_scroll:
                             try:
                                 auto_scroll(page)
                             except Exception:
-                                logging.exception('manual auto-scroll failed')
+                                logger.exception('manual auto-scroll failed')
                         save_page_snapshot(page, args.output, reason='manual')
                     except Exception:
-                        logging.exception('manual snapshot failed')
+                        logger.exception('manual snapshot failed')
 
         watcher = threading.Thread(target=stdin_watcher, daemon=True)
         watcher.start()
@@ -277,7 +280,7 @@ def main():
             while True:
                 time.sleep(1)
         except KeyboardInterrupt:
-            logging.info('Shutting down...')
+            logger.info('Shutting down...')
         finally:
             context.close()
 

xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json

@@ -1,45 +0,0 @@
-{
-  "url": "https://app.statdx.com/csrf/token",
-  "method": "GET",
-  "resource_type": "xhr",
-  "status": 200,
-  "request_headers": {
-    "sec-ch-ua-platform": "\"Linux\"",
-    "referer": "https://app.statdx.com/",
-    "x-requested-with": "XMLHttpRequest",
-    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    "accept": "*/*",
-    "sec-ch-ua": "\"Not=A?Brand\";v=\"24\", \"Chromium\";v=\"140\"",
-    "sec-ch-ua-mobile": "?0"
-  },
-  "response_headers": {
-    "version-pss-dsjs-config": "2025.9.11-134022-sdx-ASTRO-e2575728",
-    "version-pss-dsjs": "2025.9.11-134022-sdx-ASTRO-e2575728",
-    "content-encoding": "gzip",
-    "cf-cache-status": "DYNAMIC",
-    "etag": "W/\"10c-vLpriK5k7wHbwVGauL6RBVcuBHQ\"",
-    "x-ua-compatible": "IE=edge",
-    "date": "Tue, 14 Oct 2025 12:41:21 GMT",
-    "content-type": "application/json; charset=utf-8",
-    "vary": "Accept-Encoding",
-    "version-pss-sparks-event-client": "2.1.1",
-    "strict-transport-security": "max-age=31536000; includeSubDomains",
-    "content-security-policy": "script-src 'self' 'unsafe-eval' 'unsafe-inline' js-agent.newrelic.com static.cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; style-src 'self' 'unsafe-inline' fonts.googleapis.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; img-src 'self' data: app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com *.cloudfront.net; media-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com *.cloudfront.net; connect-src 'self' ws: cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; frame-src 'self' app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com; worker-src 'self' blob: scopus.com service.elsevier.com pendo.reaxys.com; object-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com",
-    "cache-control": "no-cache, no-store",
-    "tdm-reservation": "1",
-    "x-ratelimit-reset": "1760445742",
-    "hostname": "sdxapp3",
-    "cf-ray": "98e726878fefbea1-LHR",
-    "x-ratelimit-remaining": "499",
-    "version-pss-cmejs": "9.3.3",
-    "tdm-policy": "https://www.elsevier.com/tdm/tdmrep-policy.json",
-    "x-ratelimit-limit": "500",
-    "version-pss-authenticationjs": "7.1.0",
-    "x-powered-by": "Express",
-    "server": "cloudflare"
-  },
-  "timestamp": "20251014T124122Z",
-  "request_post_data": null,
-  "response_text": "{\"token\":\"c67a4a30b300096e11483fc08f9dcb69e679ef1152816abe6bcad85c694959d5ba2e29cab29da9e13a743b29daac792a44a0af3bcf7f0b6476c1691d52b01918472529e208a3431dc22a435435c7947b1212f546ae3b618ad1701e1f37f616c6f6a76cfa179fb82706566692759ca11b617ba0d9501d9ee88a7e1e6ac2c8be39\"}",
-  "response_body_file": null
-}